{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Tutorial 2\n", "\n", "In this tutorial, we'll look at gradient descent, its variants and optimization algorithms." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Torch version: 2.8.0+cu128\n" ] } ], "source": [ "import numpy as np\n", "import math\n", "from matplotlib import pyplot as plt\n", "\n", "import torch\n", "from torch import nn\n", "from torch.utils.data import DataLoader\n", "from torchvision import datasets\n", "from torchvision import transforms\n", "\n", "print(\"Torch version:\", torch.__version__)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's first load the MNIST dataset and split it into a train and test set:" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "batch_size = 64" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "data_train = datasets.MNIST(\n", " root = \"data\",\n", " train = True,\n", " download = True,\n", " transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(0, 1)])\n", ")\n", "data_test = datasets.MNIST(\n", " root = \"data\",\n", " train = False,\n", " download = True,\n", " transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(0, 1)])\n", ")\n", "\n", "dataloader_train = DataLoader(data_train, batch_size = batch_size, shuffle = True)\n", "dataloader_test = DataLoader(data_test, batch_size = batch_size, shuffle = True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's defined helper functions totrain the model:" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def calculate_accuracy(model, dataloader):\n", " num_correct = 0\n", " \n", " with torch.no_grad():\n", " for (X, y) in dataloader:\n", " pred = model(X)\n", " num_correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n", "\n", " accuracy = num_correct / len(dataloader.dataset)\n", " return accuracy" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def calculate_loss_accuracy(model, dataloader, loss_fn):\n", " loss = 0\n", " num_correct = 0\n", " \n", " with torch.no_grad():\n", " for (X, y) in dataloader:\n", " pred = model(X)\n", " loss += loss_fn(pred, y).item()\n", " num_correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n", "\n", " loss /= len(dataloader)\n", " accuracy = num_correct / len(dataloader.dataset)\n", " return loss, accuracy" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def train_model(model, loss_fn, optimizer, epochs, dataloader_train, dataloader_test, early_stopper = None, log_period = 10000):\n", " for epoch in range(epochs):\n", " processed_since_log = 0\n", " for batch, (X, y) in enumerate(dataloader_train):\n", " model.train()\n", " pred = model(X)\n", " loss = loss_fn(pred, y)\n", "\n", " loss.backward()\n", " optimizer.step()\n", " optimizer.zero_grad()\n", "\n", " processed_since_log += dataloader_train.batch_size\n", "\n", " if processed_since_log >= log_period:\n", " current = min((batch + 1) * dataloader_train.batch_size, len(data_train))\n", " loss = loss.item()\n", " model.eval()\n", " train_acc = calculate_accuracy(model, dataloader_train)\n", " test_loss, test_acc = calculate_loss_accuracy(model, dataloader_test, loss_fn)\n", " print(f\"train loss: {loss:>7f} test loss: {test_loss:>7f} train accuracy: {train_acc:>3f} test accuracy: {test_acc:>3f} [sample {current:>5d}/{len(data_train):>5d}] [epoch {epoch+1:>2d}/{epochs:>2d}]\")\n", " processed_since_log -= log_period" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Since we will be creating multiple identical models, let's also create a function for that:" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "def create_model():\n", " model = nn.Sequential()\n", " model.append(nn.Flatten())\n", " model.append(nn.Linear(data_train.data.shape[1] * data_train.data.shape[2], 100))\n", " model.append(nn.ReLU())\n", " model.append(nn.Linear(100, 10))\n", " return model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Gradient descent" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "class Simple_SGD(torch.optim.Optimizer):\n", " def __init__(self, params, lr = 1e-3):\n", " defaults = dict(lr = lr)\n", " super().__init__(params, defaults)\n", "\n", " @torch.no_grad\n", " def step(self):\n", " for group in self.param_groups:\n", " for p in group[\"params\"]:\n", " p.add_(p.grad, alpha = -group[\"lr\"])" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train loss: 2.263719 test loss: 2.276265 train accuracy: 0.158967 test accuracy: 0.166300 [sample 10048/60000] [epoch 1/10]\n", "train loss: 2.236584 test loss: 2.245551 train accuracy: 0.233133 test accuracy: 0.240700 [sample 20032/60000] [epoch 1/10]\n", "train loss: 2.205888 test loss: 2.213709 train accuracy: 0.315933 test accuracy: 0.322500 [sample 30016/60000] [epoch 1/10]\n", "train loss: 2.185920 test loss: 2.180754 train accuracy: 0.411917 test accuracy: 0.418600 [sample 40000/60000] [epoch 1/10]\n", "train loss: 2.167347 test loss: 2.145731 train accuracy: 0.506400 test accuracy: 0.518100 [sample 50048/60000] [epoch 1/10]\n", "train loss: 2.135681 test loss: 2.108275 train accuracy: 0.575500 test accuracy: 0.582700 [sample 60000/60000] [epoch 1/10]\n", "train loss: 2.111817 test loss: 2.067934 train accuracy: 0.613100 test accuracy: 0.622300 [sample 10048/60000] [epoch 2/10]\n", "train loss: 1.998481 test loss: 2.026338 train accuracy: 0.649700 test accuracy: 0.656600 [sample 20032/60000] [epoch 2/10]\n", "train loss: 2.017172 test loss: 1.983081 train accuracy: 0.665033 test accuracy: 0.670500 [sample 30016/60000] [epoch 2/10]\n", "train loss: 1.899091 test loss: 1.937358 train accuracy: 0.688817 test accuracy: 0.693900 [sample 40000/60000] [epoch 2/10]\n", "train loss: 1.890191 test loss: 1.889460 train accuracy: 0.702650 test accuracy: 0.707400 [sample 50048/60000] [epoch 2/10]\n", "train loss: 1.885232 test loss: 1.840180 train accuracy: 0.713217 test accuracy: 0.720000 [sample 60000/60000] [epoch 2/10]\n", "train loss: 1.797119 test loss: 1.788791 train accuracy: 0.720317 test accuracy: 0.726900 [sample 10048/60000] [epoch 3/10]\n", "train loss: 1.793513 test loss: 1.737064 train accuracy: 0.726067 test accuracy: 0.731900 [sample 20032/60000] [epoch 3/10]\n", "train loss: 1.685632 test loss: 1.685528 train accuracy: 0.734650 test accuracy: 0.740700 [sample 30016/60000] [epoch 3/10]\n", "train loss: 1.573969 test loss: 1.632686 train accuracy: 0.741767 test accuracy: 0.747100 [sample 40000/60000] [epoch 3/10]\n", "train loss: 1.497673 test loss: 1.578514 train accuracy: 0.746517 test accuracy: 0.751600 [sample 50048/60000] [epoch 3/10]\n", "train loss: 1.495830 test loss: 1.526487 train accuracy: 0.750433 test accuracy: 0.756100 [sample 60000/60000] [epoch 3/10]\n", "train loss: 1.490834 test loss: 1.475138 train accuracy: 0.754183 test accuracy: 0.758400 [sample 10048/60000] [epoch 4/10]\n", "train loss: 1.387297 test loss: 1.424023 train accuracy: 0.757217 test accuracy: 0.761000 [sample 20032/60000] [epoch 4/10]\n", "train loss: 1.325215 test loss: 1.375769 train accuracy: 0.764867 test accuracy: 0.770800 [sample 30016/60000] [epoch 4/10]\n", "train loss: 1.330296 test loss: 1.328129 train accuracy: 0.768967 test accuracy: 0.775400 [sample 40000/60000] [epoch 4/10]\n", "train loss: 1.318292 test loss: 1.281981 train accuracy: 0.774917 test accuracy: 0.780300 [sample 50048/60000] [epoch 4/10]\n", "train loss: 1.223882 test loss: 1.239548 train accuracy: 0.781450 test accuracy: 0.788700 [sample 60000/60000] [epoch 4/10]\n", "train loss: 1.178159 test loss: 1.199669 train accuracy: 0.785567 test accuracy: 0.791900 [sample 10048/60000] [epoch 5/10]\n", "train loss: 1.183795 test loss: 1.158786 train accuracy: 0.788150 test accuracy: 0.794300 [sample 20032/60000] [epoch 5/10]\n", "train loss: 1.286700 test loss: 1.122575 train accuracy: 0.793633 test accuracy: 0.799500 [sample 30016/60000] [epoch 5/10]\n", "train loss: 1.196659 test loss: 1.087124 train accuracy: 0.801950 test accuracy: 0.806800 [sample 40000/60000] [epoch 5/10]\n", "train loss: 1.142709 test loss: 1.053563 train accuracy: 0.805267 test accuracy: 0.811600 [sample 50048/60000] [epoch 5/10]\n", "train loss: 0.987626 test loss: 1.023040 train accuracy: 0.810233 test accuracy: 0.816700 [sample 60000/60000] [epoch 5/10]\n", "train loss: 1.026534 test loss: 0.994809 train accuracy: 0.811917 test accuracy: 0.816900 [sample 10048/60000] [epoch 6/10]\n", "train loss: 0.992003 test loss: 0.967405 train accuracy: 0.817283 test accuracy: 0.822800 [sample 20032/60000] [epoch 6/10]\n", "train loss: 0.937600 test loss: 0.941365 train accuracy: 0.820550 test accuracy: 0.826100 [sample 30016/60000] [epoch 6/10]\n", "train loss: 0.932441 test loss: 0.916459 train accuracy: 0.820483 test accuracy: 0.826700 [sample 40000/60000] [epoch 6/10]\n", "train loss: 0.865825 test loss: 0.894424 train accuracy: 0.823517 test accuracy: 0.828700 [sample 50048/60000] [epoch 6/10]\n", "train loss: 0.864450 test loss: 0.871670 train accuracy: 0.827667 test accuracy: 0.834400 [sample 60000/60000] [epoch 6/10]\n", "train loss: 0.946063 test loss: 0.851903 train accuracy: 0.829300 test accuracy: 0.836500 [sample 10048/60000] [epoch 7/10]\n", "train loss: 0.939093 test loss: 0.833244 train accuracy: 0.831550 test accuracy: 0.836900 [sample 20032/60000] [epoch 7/10]\n", "train loss: 0.874471 test loss: 0.813415 train accuracy: 0.834433 test accuracy: 0.840300 [sample 30016/60000] [epoch 7/10]\n", "train loss: 0.793531 test loss: 0.797040 train accuracy: 0.835450 test accuracy: 0.841700 [sample 40000/60000] [epoch 7/10]\n", "train loss: 0.867384 test loss: 0.781236 train accuracy: 0.835950 test accuracy: 0.843100 [sample 50048/60000] [epoch 7/10]\n", "train loss: 0.855470 test loss: 0.765017 train accuracy: 0.838767 test accuracy: 0.845600 [sample 60000/60000] [epoch 7/10]\n", "train loss: 0.670806 test loss: 0.750746 train accuracy: 0.840433 test accuracy: 0.846700 [sample 10048/60000] [epoch 8/10]\n", "train loss: 0.675908 test loss: 0.737537 train accuracy: 0.842783 test accuracy: 0.848100 [sample 20032/60000] [epoch 8/10]\n", "train loss: 0.715992 test loss: 0.724245 train accuracy: 0.844150 test accuracy: 0.848400 [sample 30016/60000] [epoch 8/10]\n", "train loss: 0.776157 test loss: 0.712779 train accuracy: 0.846383 test accuracy: 0.851400 [sample 40000/60000] [epoch 8/10]\n", "train loss: 0.741091 test loss: 0.699592 train accuracy: 0.847333 test accuracy: 0.852300 [sample 50048/60000] [epoch 8/10]\n", "train loss: 0.767111 test loss: 0.688667 train accuracy: 0.848000 test accuracy: 0.853200 [sample 60000/60000] [epoch 8/10]\n", "train loss: 0.701922 test loss: 0.678205 train accuracy: 0.849417 test accuracy: 0.854700 [sample 10048/60000] [epoch 9/10]\n", "train loss: 0.627290 test loss: 0.666366 train accuracy: 0.850667 test accuracy: 0.856300 [sample 20032/60000] [epoch 9/10]\n", "train loss: 0.665451 test loss: 0.658279 train accuracy: 0.851950 test accuracy: 0.857800 [sample 30016/60000] [epoch 9/10]\n", "train loss: 0.595099 test loss: 0.649256 train accuracy: 0.853017 test accuracy: 0.859100 [sample 40000/60000] [epoch 9/10]\n", "train loss: 0.746273 test loss: 0.639954 train accuracy: 0.854250 test accuracy: 0.860500 [sample 50048/60000] [epoch 9/10]\n", "train loss: 0.626599 test loss: 0.631625 train accuracy: 0.854500 test accuracy: 0.860300 [sample 60000/60000] [epoch 9/10]\n", "train loss: 0.682037 test loss: 0.623792 train accuracy: 0.855550 test accuracy: 0.861700 [sample 10048/60000] [epoch 10/10]\n", "train loss: 0.650744 test loss: 0.614389 train accuracy: 0.857167 test accuracy: 0.863400 [sample 20032/60000] [epoch 10/10]\n", "train loss: 0.652605 test loss: 0.607539 train accuracy: 0.857383 test accuracy: 0.863700 [sample 30016/60000] [epoch 10/10]\n", "train loss: 0.657660 test loss: 0.599809 train accuracy: 0.859317 test accuracy: 0.865400 [sample 40000/60000] [epoch 10/10]\n", "train loss: 0.601811 test loss: 0.592257 train accuracy: 0.859550 test accuracy: 0.865500 [sample 50048/60000] [epoch 10/10]\n", "train loss: 0.512712 test loss: 0.585600 train accuracy: 0.860033 test accuracy: 0.866300 [sample 60000/60000] [epoch 10/10]\n" ] } ], "source": [ "model = create_model()\n", "optimizer = Simple_SGD(model.parameters())\n", "train_model(model, nn.CrossEntropyLoss(), optimizer, 10, dataloader_train, dataloader_test)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Momentum" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "class Simple_Momentum(torch.optim.Optimizer):\n", " def __init__(self, params, lr = 1e-3, momentum_gamma = 0.9):\n", " defaults = dict(lr = lr, momentum_gamma = momentum_gamma)\n", " super().__init__(params, defaults)\n", "\n", " @torch.no_grad\n", " def step(self):\n", " for group in self.param_groups:\n", " for p in group[\"params\"]:\n", " momentum_v = self.state[p].get(\"momentum_v\")\n", " if momentum_v is None:\n", " momentum_v = torch.clone(p.grad).detach()\n", " else:\n", " momentum_v.mul_(group[\"momentum_gamma\"]).add_(p.grad)\n", " \n", " self.state[p][\"momentum_v\"] = momentum_v\n", " p.add_(momentum_v, alpha = -group[\"lr\"])" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train loss: 2.024158 test loss: 2.006585 train accuracy: 0.639700 test accuracy: 0.653100 [sample 10048/60000] [epoch 1/10]\n", "train loss: 1.495003 test loss: 1.511178 train accuracy: 0.723150 test accuracy: 0.734500 [sample 20032/60000] [epoch 1/10]\n", "train loss: 1.102537 test loss: 1.073900 train accuracy: 0.801183 test accuracy: 0.808400 [sample 30016/60000] [epoch 1/10]\n", "train loss: 1.017867 test loss: 0.820161 train accuracy: 0.828517 test accuracy: 0.836400 [sample 40000/60000] [epoch 1/10]\n", "train loss: 0.618913 test loss: 0.679602 train accuracy: 0.842833 test accuracy: 0.848500 [sample 50048/60000] [epoch 1/10]\n", "train loss: 0.492513 test loss: 0.593564 train accuracy: 0.858800 test accuracy: 0.865300 [sample 60000/60000] [epoch 1/10]\n", "train loss: 0.685109 test loss: 0.536720 train accuracy: 0.864817 test accuracy: 0.872400 [sample 10048/60000] [epoch 2/10]\n", "train loss: 0.375390 test loss: 0.496623 train accuracy: 0.870533 test accuracy: 0.878900 [sample 20032/60000] [epoch 2/10]\n", "train loss: 0.567842 test loss: 0.466121 train accuracy: 0.876067 test accuracy: 0.884500 [sample 30016/60000] [epoch 2/10]\n", "train loss: 0.267405 test loss: 0.440520 train accuracy: 0.881333 test accuracy: 0.887200 [sample 40000/60000] [epoch 2/10]\n", "train loss: 0.432212 test loss: 0.423353 train accuracy: 0.884267 test accuracy: 0.890500 [sample 50048/60000] [epoch 2/10]\n", "train loss: 0.490672 test loss: 0.405512 train accuracy: 0.886850 test accuracy: 0.893000 [sample 60000/60000] [epoch 2/10]\n", "train loss: 0.426054 test loss: 0.397057 train accuracy: 0.888567 test accuracy: 0.893900 [sample 10048/60000] [epoch 3/10]\n", "train loss: 0.468828 test loss: 0.382410 train accuracy: 0.891917 test accuracy: 0.897200 [sample 20032/60000] [epoch 3/10]\n", "train loss: 0.398453 test loss: 0.373230 train accuracy: 0.894017 test accuracy: 0.898400 [sample 30016/60000] [epoch 3/10]\n", "train loss: 0.231376 test loss: 0.365052 train accuracy: 0.893417 test accuracy: 0.898800 [sample 40000/60000] [epoch 3/10]\n", "train loss: 0.270318 test loss: 0.358131 train accuracy: 0.897067 test accuracy: 0.901500 [sample 50048/60000] [epoch 3/10]\n", "train loss: 0.258386 test loss: 0.354128 train accuracy: 0.897983 test accuracy: 0.904000 [sample 60000/60000] [epoch 3/10]\n", "train loss: 0.448841 test loss: 0.347341 train accuracy: 0.899283 test accuracy: 0.904300 [sample 10048/60000] [epoch 4/10]\n", "train loss: 0.461823 test loss: 0.342868 train accuracy: 0.900783 test accuracy: 0.904700 [sample 20032/60000] [epoch 4/10]\n", "train loss: 0.280363 test loss: 0.334410 train accuracy: 0.901917 test accuracy: 0.906700 [sample 30016/60000] [epoch 4/10]\n", "train loss: 0.323257 test loss: 0.331176 train accuracy: 0.902133 test accuracy: 0.906100 [sample 40000/60000] [epoch 4/10]\n", "train loss: 0.308119 test loss: 0.326123 train accuracy: 0.904050 test accuracy: 0.907600 [sample 50048/60000] [epoch 4/10]\n", "train loss: 0.379277 test loss: 0.323765 train accuracy: 0.904633 test accuracy: 0.908800 [sample 60000/60000] [epoch 4/10]\n", "train loss: 0.375031 test loss: 0.318680 train accuracy: 0.906200 test accuracy: 0.910200 [sample 10048/60000] [epoch 5/10]\n", "train loss: 0.199382 test loss: 0.316104 train accuracy: 0.907133 test accuracy: 0.911900 [sample 20032/60000] [epoch 5/10]\n", "train loss: 0.449335 test loss: 0.314529 train accuracy: 0.907667 test accuracy: 0.911700 [sample 30016/60000] [epoch 5/10]\n", "train loss: 0.426005 test loss: 0.311017 train accuracy: 0.907467 test accuracy: 0.912400 [sample 40000/60000] [epoch 5/10]\n", "train loss: 0.327632 test loss: 0.306797 train accuracy: 0.909600 test accuracy: 0.914100 [sample 50048/60000] [epoch 5/10]\n", "train loss: 0.358053 test loss: 0.304473 train accuracy: 0.910300 test accuracy: 0.914700 [sample 60000/60000] [epoch 5/10]\n", "train loss: 0.223087 test loss: 0.303693 train accuracy: 0.910400 test accuracy: 0.913600 [sample 10048/60000] [epoch 6/10]\n", "train loss: 0.528402 test loss: 0.299673 train accuracy: 0.911250 test accuracy: 0.915400 [sample 20032/60000] [epoch 6/10]\n", "train loss: 0.433482 test loss: 0.297998 train accuracy: 0.912133 test accuracy: 0.917200 [sample 30016/60000] [epoch 6/10]\n", "train loss: 0.303720 test loss: 0.293804 train accuracy: 0.912933 test accuracy: 0.917400 [sample 40000/60000] [epoch 6/10]\n", "train loss: 0.278269 test loss: 0.292509 train accuracy: 0.913833 test accuracy: 0.917600 [sample 50048/60000] [epoch 6/10]\n", "train loss: 0.495534 test loss: 0.290309 train accuracy: 0.915133 test accuracy: 0.919600 [sample 60000/60000] [epoch 6/10]\n", "train loss: 0.317416 test loss: 0.288402 train accuracy: 0.915000 test accuracy: 0.919700 [sample 10048/60000] [epoch 7/10]\n", "train loss: 0.403600 test loss: 0.286199 train accuracy: 0.916567 test accuracy: 0.921100 [sample 20032/60000] [epoch 7/10]\n", "train loss: 0.163503 test loss: 0.286406 train accuracy: 0.916517 test accuracy: 0.921200 [sample 30016/60000] [epoch 7/10]\n", "train loss: 0.288699 test loss: 0.286191 train accuracy: 0.916933 test accuracy: 0.920600 [sample 40000/60000] [epoch 7/10]\n", "train loss: 0.275852 test loss: 0.279419 train accuracy: 0.917917 test accuracy: 0.920800 [sample 50048/60000] [epoch 7/10]\n", "train loss: 0.342238 test loss: 0.281323 train accuracy: 0.918533 test accuracy: 0.922400 [sample 60000/60000] [epoch 7/10]\n", "train loss: 0.406612 test loss: 0.277541 train accuracy: 0.918250 test accuracy: 0.922500 [sample 10048/60000] [epoch 8/10]\n", "train loss: 0.241967 test loss: 0.276891 train accuracy: 0.919350 test accuracy: 0.922500 [sample 20032/60000] [epoch 8/10]\n", "train loss: 0.300157 test loss: 0.273663 train accuracy: 0.920517 test accuracy: 0.923600 [sample 30016/60000] [epoch 8/10]\n", "train loss: 0.316659 test loss: 0.272043 train accuracy: 0.920417 test accuracy: 0.923400 [sample 40000/60000] [epoch 8/10]\n", "train loss: 0.282002 test loss: 0.270813 train accuracy: 0.921233 test accuracy: 0.924200 [sample 50048/60000] [epoch 8/10]\n", "train loss: 0.227715 test loss: 0.268753 train accuracy: 0.921533 test accuracy: 0.925800 [sample 60000/60000] [epoch 8/10]\n", "train loss: 0.182154 test loss: 0.267117 train accuracy: 0.921633 test accuracy: 0.925400 [sample 10048/60000] [epoch 9/10]\n", "train loss: 0.192655 test loss: 0.264470 train accuracy: 0.922733 test accuracy: 0.925000 [sample 20032/60000] [epoch 9/10]\n", "train loss: 0.259953 test loss: 0.265051 train accuracy: 0.923367 test accuracy: 0.925300 [sample 30016/60000] [epoch 9/10]\n", "train loss: 0.381528 test loss: 0.263207 train accuracy: 0.923667 test accuracy: 0.925600 [sample 40000/60000] [epoch 9/10]\n", "train loss: 0.307604 test loss: 0.260316 train accuracy: 0.923967 test accuracy: 0.926100 [sample 50048/60000] [epoch 9/10]\n", "train loss: 0.253930 test loss: 0.260361 train accuracy: 0.924633 test accuracy: 0.927900 [sample 60000/60000] [epoch 9/10]\n", "train loss: 0.156860 test loss: 0.257279 train accuracy: 0.925383 test accuracy: 0.927600 [sample 10048/60000] [epoch 10/10]\n", "train loss: 0.211944 test loss: 0.257173 train accuracy: 0.924833 test accuracy: 0.926800 [sample 20032/60000] [epoch 10/10]\n", "train loss: 0.259947 test loss: 0.254046 train accuracy: 0.926167 test accuracy: 0.927700 [sample 30016/60000] [epoch 10/10]\n", "train loss: 0.293786 test loss: 0.252905 train accuracy: 0.926333 test accuracy: 0.928100 [sample 40000/60000] [epoch 10/10]\n", "train loss: 0.307108 test loss: 0.251655 train accuracy: 0.927033 test accuracy: 0.929600 [sample 50048/60000] [epoch 10/10]\n", "train loss: 0.380677 test loss: 0.251115 train accuracy: 0.927650 test accuracy: 0.929800 [sample 60000/60000] [epoch 10/10]\n" ] } ], "source": [ "model = create_model()\n", "optimizer = Simple_Momentum(model.parameters())\n", "train_model(model, nn.CrossEntropyLoss(), optimizer, 10, dataloader_train, dataloader_test)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Task 1\n", "\n", "Create an optimizer for the Nesterov accelerated gradients (NAG) method." ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "class Simple_Nesterov(torch.optim.Optimizer):\n", " def __init__(self, params, lr = 1e-3, momentum_gamma = 0.9):\n", " defaults = dict(lr = lr, momentum_gamma = momentum_gamma)\n", " super().__init__(params, defaults)\n", "\n", " @torch.no_grad\n", " def step(self):\n", " for group in self.param_groups:\n", " for p in group[\"params\"]:\n", " momentum_v = self.state[p].get(\"momentum_v\")\n", " if momentum_v is None:\n", " momentum_v = torch.clone(p.grad).detach()\n", " else:\n", " momentum_v.mul_(group[\"momentum_gamma\"]).add_(p.grad)\n", "\n", " grad_like = p.grad.add(momentum_v, alpha = group[\"momentum_gamma\"])\n", " \n", " self.state[p][\"momentum_v\"] = momentum_v\n", " p.add_(grad_like, alpha = -group[\"lr\"])" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train loss: 2.028380 test loss: 1.973173 train accuracy: 0.672633 test accuracy: 0.675800 [sample 10048/60000] [epoch 1/10]\n", "train loss: 1.602706 test loss: 1.466081 train accuracy: 0.766783 test accuracy: 0.771900 [sample 20032/60000] [epoch 1/10]\n", "train loss: 0.922532 test loss: 1.040253 train accuracy: 0.796950 test accuracy: 0.802000 [sample 30016/60000] [epoch 1/10]\n", "train loss: 0.791584 test loss: 0.797734 train accuracy: 0.828100 test accuracy: 0.837300 [sample 40000/60000] [epoch 1/10]\n", "train loss: 0.634853 test loss: 0.665005 train accuracy: 0.845133 test accuracy: 0.852500 [sample 50048/60000] [epoch 1/10]\n", "train loss: 0.494954 test loss: 0.581608 train accuracy: 0.855983 test accuracy: 0.864500 [sample 60000/60000] [epoch 1/10]\n", "train loss: 0.603256 test loss: 0.527253 train accuracy: 0.863983 test accuracy: 0.871600 [sample 10048/60000] [epoch 2/10]\n", "train loss: 0.518996 test loss: 0.487960 train accuracy: 0.870617 test accuracy: 0.878700 [sample 20032/60000] [epoch 2/10]\n", "train loss: 0.415344 test loss: 0.457299 train accuracy: 0.876250 test accuracy: 0.883600 [sample 30016/60000] [epoch 2/10]\n", "train loss: 0.451841 test loss: 0.435456 train accuracy: 0.880367 test accuracy: 0.887000 [sample 40000/60000] [epoch 2/10]\n", "train loss: 0.330520 test loss: 0.417798 train accuracy: 0.883983 test accuracy: 0.890700 [sample 50048/60000] [epoch 2/10]\n", "train loss: 0.872627 test loss: 0.402097 train accuracy: 0.887217 test accuracy: 0.893600 [sample 60000/60000] [epoch 2/10]\n", "train loss: 0.351241 test loss: 0.390792 train accuracy: 0.889700 test accuracy: 0.894400 [sample 10048/60000] [epoch 3/10]\n", "train loss: 0.603006 test loss: 0.381084 train accuracy: 0.891383 test accuracy: 0.898000 [sample 20032/60000] [epoch 3/10]\n", "train loss: 0.289691 test loss: 0.370486 train accuracy: 0.893617 test accuracy: 0.897400 [sample 30016/60000] [epoch 3/10]\n", "train loss: 0.375915 test loss: 0.363273 train accuracy: 0.895433 test accuracy: 0.900000 [sample 40000/60000] [epoch 3/10]\n", "train loss: 0.325770 test loss: 0.356163 train accuracy: 0.896500 test accuracy: 0.901600 [sample 50048/60000] [epoch 3/10]\n", "train loss: 0.461542 test loss: 0.350524 train accuracy: 0.897817 test accuracy: 0.902300 [sample 60000/60000] [epoch 3/10]\n", "train loss: 0.436961 test loss: 0.346903 train accuracy: 0.898167 test accuracy: 0.903300 [sample 10048/60000] [epoch 4/10]\n", "train loss: 0.297691 test loss: 0.337939 train accuracy: 0.900600 test accuracy: 0.905700 [sample 20032/60000] [epoch 4/10]\n", "train loss: 0.238765 test loss: 0.333790 train accuracy: 0.901917 test accuracy: 0.906800 [sample 30016/60000] [epoch 4/10]\n", "train loss: 0.275689 test loss: 0.331250 train accuracy: 0.902950 test accuracy: 0.908000 [sample 40000/60000] [epoch 4/10]\n", "train loss: 0.346364 test loss: 0.327644 train accuracy: 0.903583 test accuracy: 0.908800 [sample 50048/60000] [epoch 4/10]\n", "train loss: 0.171065 test loss: 0.322663 train accuracy: 0.904917 test accuracy: 0.909100 [sample 60000/60000] [epoch 4/10]\n", "train loss: 0.415263 test loss: 0.320392 train accuracy: 0.905550 test accuracy: 0.910200 [sample 10048/60000] [epoch 5/10]\n", "train loss: 0.286689 test loss: 0.317389 train accuracy: 0.906633 test accuracy: 0.910600 [sample 20032/60000] [epoch 5/10]\n", "train loss: 0.350410 test loss: 0.314403 train accuracy: 0.906900 test accuracy: 0.911900 [sample 30016/60000] [epoch 5/10]\n", "train loss: 0.240909 test loss: 0.310776 train accuracy: 0.907517 test accuracy: 0.913900 [sample 40000/60000] [epoch 5/10]\n", "train loss: 0.360590 test loss: 0.309994 train accuracy: 0.909167 test accuracy: 0.913200 [sample 50048/60000] [epoch 5/10]\n", "train loss: 0.325543 test loss: 0.305014 train accuracy: 0.909817 test accuracy: 0.914400 [sample 60000/60000] [epoch 5/10]\n", "train loss: 0.640698 test loss: 0.303324 train accuracy: 0.911283 test accuracy: 0.914700 [sample 10048/60000] [epoch 6/10]\n", "train loss: 0.192288 test loss: 0.301410 train accuracy: 0.910367 test accuracy: 0.916600 [sample 20032/60000] [epoch 6/10]\n", "train loss: 0.361128 test loss: 0.297901 train accuracy: 0.912067 test accuracy: 0.916600 [sample 30016/60000] [epoch 6/10]\n", "train loss: 0.200255 test loss: 0.295348 train accuracy: 0.912900 test accuracy: 0.917100 [sample 40000/60000] [epoch 6/10]\n", "train loss: 0.235122 test loss: 0.296988 train accuracy: 0.913583 test accuracy: 0.917200 [sample 50048/60000] [epoch 6/10]\n", "train loss: 0.325066 test loss: 0.290221 train accuracy: 0.914717 test accuracy: 0.918900 [sample 60000/60000] [epoch 6/10]\n", "train loss: 0.211465 test loss: 0.290932 train accuracy: 0.914283 test accuracy: 0.917900 [sample 10048/60000] [epoch 7/10]\n", "train loss: 0.379385 test loss: 0.287412 train accuracy: 0.915483 test accuracy: 0.918400 [sample 20032/60000] [epoch 7/10]\n", "train loss: 0.238461 test loss: 0.287063 train accuracy: 0.915917 test accuracy: 0.918500 [sample 30016/60000] [epoch 7/10]\n", "train loss: 0.324883 test loss: 0.286194 train accuracy: 0.916950 test accuracy: 0.920900 [sample 40000/60000] [epoch 7/10]\n", "train loss: 0.335119 test loss: 0.284447 train accuracy: 0.917767 test accuracy: 0.920800 [sample 50048/60000] [epoch 7/10]\n", "train loss: 0.550468 test loss: 0.280950 train accuracy: 0.917633 test accuracy: 0.921600 [sample 60000/60000] [epoch 7/10]\n", "train loss: 0.304858 test loss: 0.278279 train accuracy: 0.918717 test accuracy: 0.921400 [sample 10048/60000] [epoch 8/10]\n", "train loss: 0.173078 test loss: 0.278741 train accuracy: 0.919367 test accuracy: 0.922100 [sample 20032/60000] [epoch 8/10]\n", "train loss: 0.247315 test loss: 0.276542 train accuracy: 0.919817 test accuracy: 0.921100 [sample 30016/60000] [epoch 8/10]\n", "train loss: 0.205080 test loss: 0.274186 train accuracy: 0.919817 test accuracy: 0.921800 [sample 40000/60000] [epoch 8/10]\n", "train loss: 0.294312 test loss: 0.271895 train accuracy: 0.921017 test accuracy: 0.923100 [sample 50048/60000] [epoch 8/10]\n", "train loss: 0.443174 test loss: 0.268991 train accuracy: 0.921117 test accuracy: 0.924500 [sample 60000/60000] [epoch 8/10]\n", "train loss: 0.302319 test loss: 0.268023 train accuracy: 0.921400 test accuracy: 0.924300 [sample 10048/60000] [epoch 9/10]\n", "train loss: 0.254351 test loss: 0.266356 train accuracy: 0.922517 test accuracy: 0.924400 [sample 20032/60000] [epoch 9/10]\n", "train loss: 0.215918 test loss: 0.265178 train accuracy: 0.923550 test accuracy: 0.925500 [sample 30016/60000] [epoch 9/10]\n", "train loss: 0.142430 test loss: 0.263484 train accuracy: 0.924100 test accuracy: 0.927200 [sample 40000/60000] [epoch 9/10]\n", "train loss: 0.231130 test loss: 0.262762 train accuracy: 0.924733 test accuracy: 0.926500 [sample 50048/60000] [epoch 9/10]\n", "train loss: 0.273160 test loss: 0.261117 train accuracy: 0.923750 test accuracy: 0.927400 [sample 60000/60000] [epoch 9/10]\n", "train loss: 0.240012 test loss: 0.260134 train accuracy: 0.926183 test accuracy: 0.926900 [sample 10048/60000] [epoch 10/10]\n", "train loss: 0.473665 test loss: 0.257737 train accuracy: 0.925533 test accuracy: 0.927800 [sample 20032/60000] [epoch 10/10]\n", "train loss: 0.212104 test loss: 0.256659 train accuracy: 0.926683 test accuracy: 0.928200 [sample 30016/60000] [epoch 10/10]\n", "train loss: 0.203648 test loss: 0.257183 train accuracy: 0.926417 test accuracy: 0.927300 [sample 40000/60000] [epoch 10/10]\n", "train loss: 0.216775 test loss: 0.253761 train accuracy: 0.926767 test accuracy: 0.926800 [sample 50048/60000] [epoch 10/10]\n", "train loss: 0.247392 test loss: 0.252304 train accuracy: 0.927800 test accuracy: 0.929300 [sample 60000/60000] [epoch 10/10]\n" ] } ], "source": [ "model = create_model()\n", "optimizer = Simple_Nesterov(model.parameters())\n", "train_model(model, nn.CrossEntropyLoss(), optimizer, 10, dataloader_train, dataloader_test)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Task 2\n", "\n", "Create an optimizer for the ADAM method." ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "class Simple_ADAM(torch.optim.Optimizer):\n", " def __init__(self, params, lr = 1e-3, beta_1 = 0.9, beta_2 = 0.999, epsilon = 1e-8):\n", " defaults = dict(lr = lr, beta_1 = beta_1, beta_2 = beta_2, epsilon = epsilon)\n", " super().__init__(params, defaults)\n", "\n", " @torch.no_grad\n", " def step(self):\n", " for group in self.param_groups:\n", " for p in group[\"params\"]:\n", " if len(self.state[p]) == 0:\n", " self.state[p][\"step\"] = torch.tensor(1.0)\n", " self.state[p][\"grad_average\"] = torch.zeros_like(p)\n", " self.state[p][\"grad_square_average\"] = torch.zeros_like(p)\n", " \n", " grad_average = self.state[p].get(\"grad_average\")\n", " grad_square_average = self.state[p].get(\"grad_square_average\")\n", "\n", " # Linear interpolation between current value and p.grad with weight as second argument\n", " grad_average.lerp_(p.grad, 1 - group[\"beta_1\"])\n", "\n", " # addcmul_ = element-wise multiply arg1 and arg2, multiply by scalar value and add the result to current tensor\n", " grad_square_average.mul_(group[\"beta_2\"]).addcmul_(p.grad, p.grad.conj(), value = 1 - group[\"beta_2\"])\n", "\n", " bias_correction1 = 1 - group[\"beta_1\"]**self.state[p][\"step\"]\n", " bias_correction2 = 1 - group[\"beta_2\"]**self.state[p][\"step\"]\n", "\n", " grad_average_corrected = grad_average.div(bias_correction1)\n", " grad_square_average_corrected = grad_square_average.div(bias_correction2)\n", " denominator = grad_square_average_corrected.sqrt().add(group[\"epsilon\"])\n", "\n", " self.state[p][\"step\"] += 1\n", " self.state[p][\"grad_average\"] = grad_average\n", " self.state[p][\"grad_square_average\"] = grad_square_average\n", "\n", " # addcdiv_ = element-wise divide arg1 by arg2, multiply by scalar value and add the result to current tensor\n", " p.addcdiv_(grad_average_corrected, denominator, value = -group[\"lr\"])" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train loss: 0.358754 test loss: 0.393576 train accuracy: 0.886700 test accuracy: 0.892000 [sample 10048/60000] [epoch 1/10]\n", "train loss: 0.263642 test loss: 0.296537 train accuracy: 0.912183 test accuracy: 0.916000 [sample 20032/60000] [epoch 1/10]\n", "train loss: 0.157109 test loss: 0.262567 train accuracy: 0.926483 test accuracy: 0.927700 [sample 30016/60000] [epoch 1/10]\n", "train loss: 0.195705 test loss: 0.235457 train accuracy: 0.932867 test accuracy: 0.935700 [sample 40000/60000] [epoch 1/10]\n", "train loss: 0.305004 test loss: 0.207930 train accuracy: 0.939900 test accuracy: 0.938300 [sample 50048/60000] [epoch 1/10]\n", "train loss: 0.359770 test loss: 0.194959 train accuracy: 0.944433 test accuracy: 0.944300 [sample 60000/60000] [epoch 1/10]\n", "train loss: 0.154418 test loss: 0.177484 train accuracy: 0.950217 test accuracy: 0.949600 [sample 10048/60000] [epoch 2/10]\n", "train loss: 0.171317 test loss: 0.161207 train accuracy: 0.954617 test accuracy: 0.952400 [sample 20032/60000] [epoch 2/10]\n", "train loss: 0.181187 test loss: 0.151491 train accuracy: 0.958650 test accuracy: 0.956600 [sample 30016/60000] [epoch 2/10]\n", "train loss: 0.422330 test loss: 0.150724 train accuracy: 0.959283 test accuracy: 0.957100 [sample 40000/60000] [epoch 2/10]\n", "train loss: 0.070270 test loss: 0.146953 train accuracy: 0.957867 test accuracy: 0.957900 [sample 50048/60000] [epoch 2/10]\n", "train loss: 0.318036 test loss: 0.134630 train accuracy: 0.965633 test accuracy: 0.961400 [sample 60000/60000] [epoch 2/10]\n", "train loss: 0.084781 test loss: 0.126912 train accuracy: 0.968167 test accuracy: 0.962300 [sample 10048/60000] [epoch 3/10]\n", "train loss: 0.158540 test loss: 0.125650 train accuracy: 0.966800 test accuracy: 0.963200 [sample 20032/60000] [epoch 3/10]\n", "train loss: 0.036393 test loss: 0.128684 train accuracy: 0.967017 test accuracy: 0.962700 [sample 30016/60000] [epoch 3/10]\n", "train loss: 0.156080 test loss: 0.110734 train accuracy: 0.972633 test accuracy: 0.965900 [sample 40000/60000] [epoch 3/10]\n", "train loss: 0.051416 test loss: 0.112899 train accuracy: 0.972517 test accuracy: 0.966200 [sample 50048/60000] [epoch 3/10]\n", "train loss: 0.026248 test loss: 0.103500 train accuracy: 0.974333 test accuracy: 0.969000 [sample 60000/60000] [epoch 3/10]\n", "train loss: 0.049699 test loss: 0.103272 train accuracy: 0.975700 test accuracy: 0.969600 [sample 10048/60000] [epoch 4/10]\n", "train loss: 0.072082 test loss: 0.099289 train accuracy: 0.976600 test accuracy: 0.971300 [sample 20032/60000] [epoch 4/10]\n", "train loss: 0.065200 test loss: 0.103546 train accuracy: 0.976483 test accuracy: 0.967600 [sample 30016/60000] [epoch 4/10]\n", "train loss: 0.019557 test loss: 0.102335 train accuracy: 0.976367 test accuracy: 0.969700 [sample 40000/60000] [epoch 4/10]\n", "train loss: 0.069237 test loss: 0.093584 train accuracy: 0.979817 test accuracy: 0.972000 [sample 50048/60000] [epoch 4/10]\n", "train loss: 0.023832 test loss: 0.092792 train accuracy: 0.979617 test accuracy: 0.971800 [sample 60000/60000] [epoch 4/10]\n", "train loss: 0.067009 test loss: 0.092154 train accuracy: 0.980483 test accuracy: 0.972900 [sample 10048/60000] [epoch 5/10]\n", "train loss: 0.130489 test loss: 0.090503 train accuracy: 0.981533 test accuracy: 0.971600 [sample 20032/60000] [epoch 5/10]\n", "train loss: 0.051078 test loss: 0.094044 train accuracy: 0.980600 test accuracy: 0.970700 [sample 30016/60000] [epoch 5/10]\n", "train loss: 0.114564 test loss: 0.089696 train accuracy: 0.983067 test accuracy: 0.972800 [sample 40000/60000] [epoch 5/10]\n", "train loss: 0.026710 test loss: 0.085745 train accuracy: 0.984850 test accuracy: 0.974000 [sample 50048/60000] [epoch 5/10]\n", "train loss: 0.128158 test loss: 0.080846 train accuracy: 0.984917 test accuracy: 0.974900 [sample 60000/60000] [epoch 5/10]\n", "train loss: 0.014971 test loss: 0.085267 train accuracy: 0.983333 test accuracy: 0.972300 [sample 10048/60000] [epoch 6/10]\n", "train loss: 0.049073 test loss: 0.084855 train accuracy: 0.985067 test accuracy: 0.974900 [sample 20032/60000] [epoch 6/10]\n", "train loss: 0.125507 test loss: 0.084620 train accuracy: 0.985333 test accuracy: 0.975100 [sample 30016/60000] [epoch 6/10]\n", "train loss: 0.150686 test loss: 0.095704 train accuracy: 0.982517 test accuracy: 0.971400 [sample 40000/60000] [epoch 6/10]\n", "train loss: 0.034527 test loss: 0.083917 train accuracy: 0.986317 test accuracy: 0.974200 [sample 50048/60000] [epoch 6/10]\n", "train loss: 0.031870 test loss: 0.077941 train accuracy: 0.987100 test accuracy: 0.975400 [sample 60000/60000] [epoch 6/10]\n", "train loss: 0.029252 test loss: 0.080159 train accuracy: 0.987550 test accuracy: 0.976600 [sample 10048/60000] [epoch 7/10]\n", "train loss: 0.053650 test loss: 0.079263 train accuracy: 0.987917 test accuracy: 0.976800 [sample 20032/60000] [epoch 7/10]\n", "train loss: 0.101778 test loss: 0.083090 train accuracy: 0.986483 test accuracy: 0.975200 [sample 30016/60000] [epoch 7/10]\n", "train loss: 0.069888 test loss: 0.076862 train accuracy: 0.988183 test accuracy: 0.976600 [sample 40000/60000] [epoch 7/10]\n", "train loss: 0.036101 test loss: 0.080738 train accuracy: 0.988400 test accuracy: 0.974900 [sample 50048/60000] [epoch 7/10]\n", "train loss: 0.028425 test loss: 0.085019 train accuracy: 0.987683 test accuracy: 0.974200 [sample 60000/60000] [epoch 7/10]\n", "train loss: 0.034247 test loss: 0.076769 train accuracy: 0.990150 test accuracy: 0.976700 [sample 10048/60000] [epoch 8/10]\n", "train loss: 0.107205 test loss: 0.078597 train accuracy: 0.989733 test accuracy: 0.976100 [sample 20032/60000] [epoch 8/10]\n", "train loss: 0.067831 test loss: 0.083379 train accuracy: 0.989583 test accuracy: 0.975300 [sample 30016/60000] [epoch 8/10]\n", "train loss: 0.015575 test loss: 0.079701 train accuracy: 0.990367 test accuracy: 0.975600 [sample 40000/60000] [epoch 8/10]\n", "train loss: 0.185844 test loss: 0.080109 train accuracy: 0.991300 test accuracy: 0.976600 [sample 50048/60000] [epoch 8/10]\n", "train loss: 0.064245 test loss: 0.077849 train accuracy: 0.991633 test accuracy: 0.976700 [sample 60000/60000] [epoch 8/10]\n", "train loss: 0.013686 test loss: 0.075694 train accuracy: 0.991583 test accuracy: 0.977600 [sample 10048/60000] [epoch 9/10]\n", "train loss: 0.145657 test loss: 0.076824 train accuracy: 0.991800 test accuracy: 0.976800 [sample 20032/60000] [epoch 9/10]\n", "train loss: 0.014221 test loss: 0.083793 train accuracy: 0.991233 test accuracy: 0.975100 [sample 30016/60000] [epoch 9/10]\n", "train loss: 0.045129 test loss: 0.083192 train accuracy: 0.991017 test accuracy: 0.974400 [sample 40000/60000] [epoch 9/10]\n", "train loss: 0.006949 test loss: 0.079214 train accuracy: 0.991750 test accuracy: 0.976600 [sample 50048/60000] [epoch 9/10]\n", "train loss: 0.079518 test loss: 0.091282 train accuracy: 0.988083 test accuracy: 0.971700 [sample 60000/60000] [epoch 9/10]\n", "train loss: 0.011970 test loss: 0.077894 train accuracy: 0.992233 test accuracy: 0.975500 [sample 10048/60000] [epoch 10/10]\n", "train loss: 0.030387 test loss: 0.079066 train accuracy: 0.993283 test accuracy: 0.975800 [sample 20032/60000] [epoch 10/10]\n", "train loss: 0.004845 test loss: 0.078944 train accuracy: 0.993350 test accuracy: 0.976600 [sample 30016/60000] [epoch 10/10]\n", "train loss: 0.055352 test loss: 0.076980 train accuracy: 0.994217 test accuracy: 0.976700 [sample 40000/60000] [epoch 10/10]\n", "train loss: 0.005562 test loss: 0.081251 train accuracy: 0.993083 test accuracy: 0.975900 [sample 50048/60000] [epoch 10/10]\n", "train loss: 0.330105 test loss: 0.083544 train accuracy: 0.992500 test accuracy: 0.974300 [sample 60000/60000] [epoch 10/10]\n" ] } ], "source": [ "model = create_model()\n", "optimizer = Simple_ADAM(model.parameters())\n", "train_model(model, nn.CrossEntropyLoss(), optimizer, 10, dataloader_train, dataloader_test)" ] } ], "metadata": { "kernelspec": { "display_name": "tzn", "language": "python", "name": "tzn" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.12" } }, "nbformat": 4, "nbformat_minor": 4 }