<jupyter_start><jupyter_text>MNIST<jupyter_code>from fastai.vision.all import *
from fastbook import *
path = untar_data(URLs.MNIST_SAMPLE)

threes = (path/'train'/'3').ls().sorted()
sevens = (path/'train'/'7').ls().sorted()
three_tensors = [tensor(Image.open(img)) for img in threes]
sevens_tensors = [tensor(Image.open(img)) for img in sevens]
stack_threes = torch.stack(three_tensors).float()/255
stack_sevens = torch.stack(sevens_tensors).float()/255
valid_3_tns = torch.stack([tensor(Image.open(o)) for o in (path/'valid'/'3').ls()])
valid_3_tns = valid_3_tns.float()/255

valid_7_tns = torch.stack([tensor(Image.open(o)) for o in (path/'valid'/'7').ls()])
valid_7_tns = valid_7_tns.float()/255
train_x = torch.cat([stack_threes, stack_sevens]).view(-1, 28*28)

train_y = tensor([1]*len(threes) + [0]*len(sevens)).unsqueeze(1)
train_x.shape, train_y.shape
dataset = list(zip(train_x, train_y))
x, y = dataset[0]
x.shape, y
valid_x = torch.cat([valid_3_tns, valid_7_tns]).view(-1, 28*28)

valid_y = tensor([1]*len(valid_3_tns) + [0]*len(valid_7_tns)).unsqueeze(1)

valid_dataset = list(zip(valid_x, valid_y))<jupyter_output><empty_output><jupyter_text>Now we need an (initially random) weight for every pixel<jupyter_code>def init_params(size, std=1.0):
    return (torch.randn(size)*std).requires_grad_()
weights = init_params(28*28, 1)
weights.shape, weights[1:10]
bias = init_params(1)<jupyter_output><empty_output><jupyter_text>y=ax+b -> y = weight * x + bias We can now calculate a prediction for one image:<jupyter_code>(train_x[0] * weights.T).sum() + bias<jupyter_output><empty_output><jupyter_text>In Python, matrix multiplication is represented with the @ operator<jupyter_code>def linear1(x_batch):
    return x_batch @ weights + bias

preds = linear1(train_x)
preds
corrects = (preds > 0.5).float() == train_y
corrects
corrects.float().mean().item()<jupyter_output><empty_output><jupyter_text>Check change in accuracy when small change in weights<jupyter_code>weights[0].data *= 1.0001
preds = linear1(train_x)

((preds > 0.0).float() == train_y).float().mean().item()<jupyter_output><empty_output><jupyter_text>LOSS FUNCTION<jupyter_code>targets = tensor([1, 0, 1])
predictions = tensor([0.9, 0.4, 0.2])
torch.where(targets==1, 1-predictions, predictions)<jupyter_output><empty_output><jupyter_text>torch.where(a_logic, b, c) == if a: b; else c but works on CUDA/GPU<jupyter_code>def mnist_loss(predictions, targest):
    return torch.where(targest==1, 1 - predictions, predictions).mean()
mnist_loss(predictions, targets)<jupyter_output><empty_output><jupyter_text>SIGMOID<jupyter_code>def sigmoid(x):
    return 1/(1+torch.exp(-x))
plot_function(torch.sigmoid, title='Sigmoid', min=-4, max=4)
def mnist_loss(predictions, targets):
    predictions = predictions.sigmoid()
    return torch.where(targets==1, 1-predictions, predictions).mean()<jupyter_output><empty_output><jupyter_text>SGD and MINI-BATCHES  A DataLoader can take any Python collection and turn it into an iterator over many batches<jupyter_code>coll = range(15)

dl = DataLoader(coll, batch_size=5, shuffle=True)

list(dl)<jupyter_output><empty_output><jupyter_text>A collection that contains tuples of independent and dependent variables is known in PyTorch as a Dataset<jupyter_code>ds = L(enumerate(string.ascii_lowercase))
ds<jupyter_output><empty_output><jupyter_text>When we pass a Dataset to a DataLoader we will get back many batches which are themselves tuples of tensors representing batches of independent and dependent variables<jupyter_code>dl = DataLoader(ds, batch_size=6, shuffle=True)
list(dl)<jupyter_output><empty_output><jupyter_text>Put it all together<jupyter_code>weights = init_params((28*28, 1))
bias = init_params(1)
dl = DataLoader(dataset, batch_size=256)

xb, yb = first(dl)
xb.shape, yb.shape
valid_dl = DataLoader(valid_dataset, batch_size=256)
batch = train_x[:4]
batch.shape
preds = linear1(batch)
preds
loss = mnist_loss(preds, train_y[:4])
loss
loss.mean().backward()

weights.grad.shape, weights.grad.mean(), bias.grad
def calc_grad(xb, yb, model):
    preds = model(xb)
    loss = mnist_loss(preds, yb)
    loss.backward()
calc_grad(batch, train_y[:4], linear1)
weights.grad.mean(), bias.grad<jupyter_output><empty_output><jupyter_text>Methods in PyTorch whose names end in an underscore modify their objects in place. For instance, bias.zero_() sets all elements of the tensor bias to 0.<jupyter_code>weights.grad.zero_(),
bias.grad.zero_();
def train_epoch(model, lr, params):
    for xb, yb in dl:
        calc_grad(xb, yb, model)
        for p in params:
            p.data -= p.grad*lr
            p.grad.zero_()
(preds > 0.0).float() == train_y[:4]
def batch_accuracy(xb, yb):
    preds = xb.sigmoid()
    correct = (preds > 0.5) == yb
    return correct.float().mean()
batch_accuracy(linear1(batch), train_y[:4])
def validate_epoch(model):
    accs = [batch_accuracy(model(xb), yb) for xb, yb in valid_dl]
    return round(torch.stack(accs).mean().item(), 4)
validate_epoch(linear1)<jupyter_output><empty_output><jupyter_text>Train for an epoch<jupyter_code>lr = 1.
params = weights, bias
train_epoch(linear1, lr, params)
validate_epoch(linear1)
for i in range(20):
    train_epoch(linear1, lr, params)
    print(validate_epoch(linear1), end=' ')<jupyter_output>0.8124 0.9067 0.9355 0.9467 0.9515 0.9569 0.9569 0.9584 0.9599 0.9608 0.9613 0.9638 0.9638 0.9653 0.9662 0.9677 0.9682 0.9692 0.9696 0.9711<jupyter_text>Create an optimizer<jupyter_code>linear_model = nn.Linear(28*28, 1)
w, b = linear_model.parameters()
w.shape, b.shape
class BasicOptim:
    def __init__(self, params, lr):
        self.params = list(params)
        self.lr = lr
    
    def step(self, *args, **kwargs):
        for p in self.params:
            p.data -= p.grad.data * self.lr
    
    def zero_grad(self, *args, **kwargs):
        for p in self.params:
            p.grad = None
opt = BasicOptim(linear_model.parameters(), lr)
def train_epoch(model):
    for xb, yb in dl:
        calc_grad(xb, yb, model)
        opt.step()
        opt.zero_grad()
validate_epoch(linear_model)
def train_model(model, epochs):
    for i in range(epochs):
        train_epoch(model)
        print(validate_epoch(model), end=' ')
train_model(linear_model, 20)
linear_model = nn.Linear(28*28, 1)
opt = SGD(linear_model.parameters(), lr)
train_model(linear_model, 20)
dls = DataLoaders(dl, valid_dl)
learn = Learner(dls, nn.Linear(28*28, 1), opt_func=SGD, loss_func=mnist_loss, metrics=batch_accuracy)
learn.fit(10, lr=lr)<jupyter_output><empty_output><jupyter_text>Adding non-linearity<jupyter_code>def simple_net(xb):
    res = xb @ w1 + b1
    res = res.max(tensor(0.0))
    res = res @ w2 + b2
    return res
w1 = init_params((28*28,30))
b1 = init_params(30)
w2 = init_params((30,1))
b2 = init_params(1)
plot_function(F.relu)
simple_net = nn.Sequential(
    nn.Linear(28*28, 30),
    nn.ReLU(),
    nn.Linear(30, 1)
)
learn = Learner(dls, simple_net, opt_func=SGD,
                loss_func=mnist_loss, metrics=batch_accuracy)
learn.fit(40, 0.1)
plt.plot(L(learn.recorder.values).itemgot(2));
learn.recorder.values[-1][2]
dls = ImageDataLoaders.from_folder(path)
learn = cnn_learner(dls, resnet18, pretrained=False,
                    loss_func=F.cross_entropy, metrics=accuracy)
learn.fit_one_cycle(1, 0.1)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup