SimCLR/feature_eval/mini_batch_logistic_regression_evaluator.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "pytorch",
      "language": "python",
      "name": "pytorch"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6"
    },
    "colab": {
      "name": "Copy of mini-batch-logistic-regression-evaluator.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1b97f76ec8314fe3985e9183af3fdd9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1d516174fefa4c26a1d9232a9fc7e34b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f72a8a93cdd14fa4bfdc34fbf1061f1e",
              "IPY_MODEL_8a684a8419754a86b7b70b9d26b252a4"
            ]
          }
        },
        "1d516174fefa4c26a1d9232a9fc7e34b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f72a8a93cdd14fa4bfdc34fbf1061f1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1a4df18ac4034be1acc4b8ef56527fd1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89b38536b9da4cfdb914fd291aca0dfe"
          }
        },
        "8a684a8419754a86b7b70b9d26b252a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_77da6ecf9d63460ab420d41f28bb7f1d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:20<00:00, 54507700.03it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_45b89ec6a3504560b9643422cee95213"
          }
        },
        "1a4df18ac4034be1acc4b8ef56527fd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89b38536b9da4cfdb914fd291aca0dfe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "77da6ecf9d63460ab420d41f28bb7f1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "45b89ec6a3504560b9643422cee95213": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sthalles/SimCLR/blob/simclr-refactor/feature_eval/mini_batch_logistic_regression_evaluator.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YUemQib7ZE4D"
      },
      "source": [
        "import torch\n",
        "import sys\n",
        "import numpy as np\n",
        "import os\n",
        "import yaml\n",
        "import matplotlib.pyplot as plt\n",
        "import torchvision"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WSgRE1CcLqdS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a6477424-66e6-4a59-bef2-42e5cbada7cf"
      },
      "source": [
        "!pip install gdown"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: gdown in /usr/local/lib/python3.6/dist-packages (3.6.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from gdown) (1.15.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from gdown) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from gdown) (4.41.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (2020.12.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NOIJEui1ZziV"
      },
      "source": [
        "def get_file_id_by_model(folder_name):\n",
        "  file_id = {'resnet18_100-epochs_stl10': '14_nH2FkyKbt61cieQDiSbBVNP8-gtwgF',\n",
        "             'resnet18_100-epochs_cifar10': '1lc2aoVtrAetGn0PnTkOyFzPCIucOJq7C'}\n",
        "  return file_id.get(folder_name, \"Model not found.\")"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G7YMxsvEZMrX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "da3bc13b-f989-4a19-dc02-5172e5e370c0"
      },
      "source": [
        "folder_name = 'resnet18_100-epochs_cifar10'\n",
        "file_id = get_file_id_by_model(folder_name)\n",
        "print(folder_name, file_id)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "resnet18_100-epochs_cifar10 1lc2aoVtrAetGn0PnTkOyFzPCIucOJq7C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PWZ8fet_YoJm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "63d1d89d-ad11-48ba-8bb3-4da15b930073"
      },
      "source": [
        "# download and extract model files\n",
        "os.system('gdown https://drive.google.com/uc?id={}'.format(file_id))\n",
        "os.system('unzip {}'.format(folder_name))\n",
        "!ls"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "checkpoint_0100.pth.tar\n",
            "config.yml\n",
            "events.out.tfevents.1610901418.4cb2c837708d.2683796.0\n",
            "resnet18_100-epochs_cifar10.zip\n",
            "resnet18_100-epochs-cifar10.zip\n",
            "run.log\n",
            "sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3_nypQVEv-hn"
      },
      "source": [
        "from torch.utils.data import DataLoader\n",
        "import torchvision.transforms as transforms\n",
        "from torchvision import datasets"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lDfbL3w_Z0Od",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "028ac120-c51d-4eb2-cf00-da69aed6e310"
      },
      "source": [
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "print(\"Using device:\", device)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using device: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BfIPl0G6_RrT"
      },
      "source": [
        "def get_stl10_data_loaders(download, shuffle=False, batch_size=256):\n",
        "  train_dataset = datasets.STL10('./data', split='train', download=download,\n",
        "                                  transform=transforms.ToTensor())\n",
        "\n",
        "  train_loader = DataLoader(train_dataset, batch_size=batch_size,\n",
        "                            num_workers=0, drop_last=False, shuffle=shuffle)\n",
        "  \n",
        "  test_dataset = datasets.STL10('./data', split='test', download=download,\n",
        "                                  transform=transforms.ToTensor())\n",
        "\n",
        "  test_loader = DataLoader(test_dataset, batch_size=2*batch_size,\n",
        "                            num_workers=10, drop_last=False, shuffle=shuffle)\n",
        "  return train_loader, test_loader\n",
        "\n",
        "def get_cifar10_data_loaders(download, shuffle=False, batch_size=256):\n",
        "  train_dataset = datasets.CIFAR10('./data', train=True, download=download,\n",
        "                                  transform=transforms.ToTensor())\n",
        "\n",
        "  train_loader = DataLoader(train_dataset, batch_size=batch_size,\n",
        "                            num_workers=0, drop_last=False, shuffle=shuffle)\n",
        "  \n",
        "  test_dataset = datasets.CIFAR10('./data', train=False, download=download,\n",
        "                                  transform=transforms.ToTensor())\n",
        "\n",
        "  test_loader = DataLoader(test_dataset, batch_size=2*batch_size,\n",
        "                            num_workers=10, drop_last=False, shuffle=shuffle)\n",
        "  return train_loader, test_loader"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6N8lYkbmDTaK"
      },
      "source": [
        "with open(os.path.join('./config.yml')) as file:\n",
        "  config = yaml.load(file)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a18lPD-tIle6"
      },
      "source": [
        "if config.arch == 'resnet18':\n",
        "  model = torchvision.models.resnet18(pretrained=False, num_classes=10).to(device)\n",
        "elif config.arch == 'resnet50':\n",
        "  model = torchvision.models.resnet50(pretrained=False, num_classes=10).to(device)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4AIfgq41GuTT"
      },
      "source": [
        "checkpoint = torch.load('checkpoint_0100.pth.tar', map_location=device)\n",
        "state_dict = checkpoint['state_dict']\n",
        "\n",
        "for k in list(state_dict.keys()):\n",
        "\n",
        "  if k.startswith('backbone.'):\n",
        "    if k.startswith('backbone') and not k.startswith('backbone.fc'):\n",
        "      # remove prefix\n",
        "      state_dict[k[len(\"backbone.\"):]] = state_dict[k]\n",
        "  del state_dict[k]"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VVjA83PPJYWl"
      },
      "source": [
        "log = model.load_state_dict(state_dict, strict=False)\n",
        "assert log.missing_keys == ['fc.weight', 'fc.bias']"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_GC0a14uWRr6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 117,
          "referenced_widgets": [
            "1b97f76ec8314fe3985e9183af3fdd9b",
            "1d516174fefa4c26a1d9232a9fc7e34b",
            "f72a8a93cdd14fa4bfdc34fbf1061f1e",
            "8a684a8419754a86b7b70b9d26b252a4",
            "1a4df18ac4034be1acc4b8ef56527fd1",
            "89b38536b9da4cfdb914fd291aca0dfe",
            "77da6ecf9d63460ab420d41f28bb7f1d",
            "45b89ec6a3504560b9643422cee95213"
          ]
        },
        "outputId": "4382995f-e0fa-48fc-d341-71400a06b6d9"
      },
      "source": [
        "if config.dataset_name == 'cifar10':\n",
        "  train_loader, test_loader = get_cifar10_data_loaders(download=True)\n",
        "elif config.dataset_name == 'stl10':\n",
        "  train_loader, test_loader = get_stl10_data_loaders(download=True)\n",
        "print(\"Dataset:\", config.dataset_name)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1b97f76ec8314fe3985e9183af3fdd9b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n",
            "Dataset: cifar10\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pYT_KsM0Mnnr"
      },
      "source": [
        "# freeze all layers but the last fc\n",
        "for name, param in model.named_parameters():\n",
        "    if name not in ['fc.weight', 'fc.bias']:\n",
        "        param.requires_grad = False\n",
        "\n",
        "parameters = list(filter(lambda p: p.requires_grad, model.parameters()))\n",
        "assert len(parameters) == 2  # fc.weight, fc.bias"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPVh1S_eMRDU"
      },
      "source": [
        "optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, weight_decay=0.0008)\n",
        "criterion = torch.nn.CrossEntropyLoss().to(device)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edr6RhP2PdVq"
      },
      "source": [
        "def accuracy(output, target, topk=(1,)):\n",
        "    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n",
        "    with torch.no_grad():\n",
        "        maxk = max(topk)\n",
        "        batch_size = target.size(0)\n",
        "\n",
        "        _, pred = output.topk(maxk, 1, True, True)\n",
        "        pred = pred.t()\n",
        "        correct = pred.eq(target.view(1, -1).expand_as(pred))\n",
        "\n",
        "        res = []\n",
        "        for k in topk:\n",
        "            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)\n",
        "            res.append(correct_k.mul_(100.0 / batch_size))\n",
        "        return res"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qOder0dAMI7X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "48816318-655c-4c2d-b4fa-4549316a8477"
      },
      "source": [
        "epochs = 100\n",
        "for epoch in range(epochs):\n",
        "  top1_train_accuracy = 0\n",
        "  for counter, (x_batch, y_batch) in enumerate(train_loader):\n",
        "    x_batch = x_batch.to(device)\n",
        "    y_batch = y_batch.to(device)\n",
        "\n",
        "    logits = model(x_batch)\n",
        "    loss = criterion(logits, y_batch)\n",
        "    top1 = accuracy(logits, y_batch, topk=(1,))\n",
        "    top1_train_accuracy += top1[0]\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "  top1_train_accuracy /= (counter + 1)\n",
        "  top1_accuracy = 0\n",
        "  top5_accuracy = 0\n",
        "  for counter, (x_batch, y_batch) in enumerate(test_loader):\n",
        "    x_batch = x_batch.to(device)\n",
        "    y_batch = y_batch.to(device)\n",
        "\n",
        "    logits = model(x_batch)\n",
        "  \n",
        "    top1, top5 = accuracy(logits, y_batch, topk=(1,5))\n",
        "    top1_accuracy += top1[0]\n",
        "    top5_accuracy += top5[0]\n",
        "  \n",
        "  top1_accuracy /= (counter + 1)\n",
        "  top5_accuracy /= (counter + 1)\n",
        "  print(f\"Epoch {epoch}\\tTop1 Train accuracy {top1_train_accuracy.item()}\\tTop1 Test accuracy: {top1_accuracy.item()}\\tTop5 test acc: {top5_accuracy.item()}\")"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0\tTop1 Train accuracy 49.823020935058594\tTop1 Test accuracy: 57.63786697387695\tTop5 test acc: 94.96036529541016\n",
            "Epoch 1\tTop1 Train accuracy 59.0130729675293\tTop1 Test accuracy: 59.57088851928711\tTop5 test acc: 95.76114654541016\n",
            "Epoch 2\tTop1 Train accuracy 60.604671478271484\tTop1 Test accuracy: 60.32686233520508\tTop5 test acc: 96.07250213623047\n",
            "Epoch 3\tTop1 Train accuracy 61.547752380371094\tTop1 Test accuracy: 61.19715118408203\tTop5 test acc: 96.14946746826172\n",
            "Epoch 4\tTop1 Train accuracy 62.19586944580078\tTop1 Test accuracy: 61.48035430908203\tTop5 test acc: 96.37407684326172\n",
            "Epoch 5\tTop1 Train accuracy 62.677772521972656\tTop1 Test accuracy: 61.784236907958984\tTop5 test acc: 96.40337371826172\n",
            "Epoch 6\tTop1 Train accuracy 63.06640625\tTop1 Test accuracy: 62.2346076965332\tTop5 test acc: 96.50102996826172\n",
            "Epoch 7\tTop1 Train accuracy 63.40122604370117\tTop1 Test accuracy: 62.52527618408203\tTop5 test acc: 96.46196746826172\n",
            "Epoch 8\tTop1 Train accuracy 63.698577880859375\tTop1 Test accuracy: 62.83777618408203\tTop5 test acc: 96.54009246826172\n",
            "Epoch 9\tTop1 Train accuracy 63.90983581542969\tTop1 Test accuracy: 63.118682861328125\tTop5 test acc: 96.58892059326172\n",
            "Epoch 10\tTop1 Train accuracy 64.14102172851562\tTop1 Test accuracy: 63.20772171020508\tTop5 test acc: 96.68657684326172\n",
            "Epoch 11\tTop1 Train accuracy 64.33633422851562\tTop1 Test accuracy: 63.469093322753906\tTop5 test acc: 96.75609588623047\n",
            "Epoch 12\tTop1 Train accuracy 64.5057373046875\tTop1 Test accuracy: 63.556983947753906\tTop5 test acc: 96.71703338623047\n",
            "Epoch 13\tTop1 Train accuracy 64.6436538696289\tTop1 Test accuracy: 63.66325759887695\tTop5 test acc: 96.69750213623047\n",
            "Epoch 14\tTop1 Train accuracy 64.75326538085938\tTop1 Test accuracy: 63.62419509887695\tTop5 test acc: 96.68773651123047\n",
            "Epoch 15\tTop1 Train accuracy 64.87284851074219\tTop1 Test accuracy: 63.84650802612305\tTop5 test acc: 96.66820526123047\n",
            "Epoch 16\tTop1 Train accuracy 64.97688293457031\tTop1 Test accuracy: 64.00276184082031\tTop5 test acc: 96.72563934326172\n",
            "Epoch 17\tTop1 Train accuracy 65.05500793457031\tTop1 Test accuracy: 63.95392990112305\tTop5 test acc: 96.71587371826172\n",
            "Epoch 18\tTop1 Train accuracy 65.11439514160156\tTop1 Test accuracy: 64.01252746582031\tTop5 test acc: 96.72563934326172\n",
            "Epoch 19\tTop1 Train accuracy 65.21205139160156\tTop1 Test accuracy: 64.07112121582031\tTop5 test acc: 96.71587371826172\n",
            "Epoch 20\tTop1 Train accuracy 65.31169891357422\tTop1 Test accuracy: 64.06135559082031\tTop5 test acc: 96.73540496826172\n",
            "Epoch 21\tTop1 Train accuracy 65.40338134765625\tTop1 Test accuracy: 64.18830871582031\tTop5 test acc: 96.74517059326172\n",
            "Epoch 22\tTop1 Train accuracy 65.45320129394531\tTop1 Test accuracy: 64.1969223022461\tTop5 test acc: 96.74517059326172\n",
            "Epoch 23\tTop1 Train accuracy 65.53292083740234\tTop1 Test accuracy: 64.23828125\tTop5 test acc: 96.71587371826172\n",
            "Epoch 24\tTop1 Train accuracy 65.61064910888672\tTop1 Test accuracy: 64.30549621582031\tTop5 test acc: 96.71587371826172\n",
            "Epoch 25\tTop1 Train accuracy 65.68638610839844\tTop1 Test accuracy: 64.31526184082031\tTop5 test acc: 96.69634246826172\n",
            "Epoch 26\tTop1 Train accuracy 65.75055694580078\tTop1 Test accuracy: 64.39338684082031\tTop5 test acc: 96.66704559326172\n",
            "Epoch 27\tTop1 Train accuracy 65.80635833740234\tTop1 Test accuracy: 64.40315246582031\tTop5 test acc: 96.67681121826172\n",
            "Epoch 28\tTop1 Train accuracy 65.8581771850586\tTop1 Test accuracy: 64.39338684082031\tTop5 test acc: 96.67681121826172\n",
            "Epoch 29\tTop1 Train accuracy 65.91397857666016\tTop1 Test accuracy: 64.42268371582031\tTop5 test acc: 96.65727996826172\n",
            "Epoch 30\tTop1 Train accuracy 65.96340942382812\tTop1 Test accuracy: 64.42268371582031\tTop5 test acc: 96.63774871826172\n",
            "Epoch 31\tTop1 Train accuracy 66.00127410888672\tTop1 Test accuracy: 64.39338684082031\tTop5 test acc: 96.62798309326172\n",
            "Epoch 32\tTop1 Train accuracy 66.05707550048828\tTop1 Test accuracy: 64.39338684082031\tTop5 test acc: 96.65727996826172\n",
            "Epoch 33\tTop1 Train accuracy 66.10092163085938\tTop1 Test accuracy: 64.43244934082031\tTop5 test acc: 96.66704559326172\n",
            "Epoch 34\tTop1 Train accuracy 66.13480377197266\tTop1 Test accuracy: 64.44221496582031\tTop5 test acc: 96.64751434326172\n",
            "Epoch 35\tTop1 Train accuracy 66.16669464111328\tTop1 Test accuracy: 64.4801254272461\tTop5 test acc: 96.63774871826172\n",
            "Epoch 36\tTop1 Train accuracy 66.21452331542969\tTop1 Test accuracy: 64.4801254272461\tTop5 test acc: 96.63774871826172\n",
            "Epoch 37\tTop1 Train accuracy 66.2547836303711\tTop1 Test accuracy: 64.5191879272461\tTop5 test acc: 96.61821746826172\n",
            "Epoch 38\tTop1 Train accuracy 66.28069305419922\tTop1 Test accuracy: 64.5582504272461\tTop5 test acc: 96.62798309326172\n",
            "Epoch 39\tTop1 Train accuracy 66.32653045654297\tTop1 Test accuracy: 64.57662963867188\tTop5 test acc: 96.63774871826172\n",
            "Epoch 40\tTop1 Train accuracy 66.35881805419922\tTop1 Test accuracy: 64.62431335449219\tTop5 test acc: 96.61821746826172\n",
            "Epoch 41\tTop1 Train accuracy 66.37077331542969\tTop1 Test accuracy: 64.68290710449219\tTop5 test acc: 96.61821746826172\n",
            "Epoch 42\tTop1 Train accuracy 66.39269256591797\tTop1 Test accuracy: 64.66337585449219\tTop5 test acc: 96.61821746826172\n",
            "Epoch 43\tTop1 Train accuracy 66.41262817382812\tTop1 Test accuracy: 64.66337585449219\tTop5 test acc: 96.63774871826172\n",
            "Epoch 44\tTop1 Train accuracy 66.45248413085938\tTop1 Test accuracy: 64.62431335449219\tTop5 test acc: 96.65727996826172\n",
            "Epoch 45\tTop1 Train accuracy 66.48238372802734\tTop1 Test accuracy: 64.65361022949219\tTop5 test acc: 96.66704559326172\n",
            "Epoch 46\tTop1 Train accuracy 66.51825714111328\tTop1 Test accuracy: 64.65361022949219\tTop5 test acc: 96.67681121826172\n",
            "Epoch 47\tTop1 Train accuracy 66.56608581542969\tTop1 Test accuracy: 64.64384460449219\tTop5 test acc: 96.65727996826172\n",
            "Epoch 48\tTop1 Train accuracy 66.59996795654297\tTop1 Test accuracy: 64.61454772949219\tTop5 test acc: 96.67681121826172\n",
            "Epoch 49\tTop1 Train accuracy 66.64381408691406\tTop1 Test accuracy: 64.67314147949219\tTop5 test acc: 96.67681121826172\n",
            "Epoch 50\tTop1 Train accuracy 66.65178680419922\tTop1 Test accuracy: 64.70243835449219\tTop5 test acc: 96.69519805908203\n",
            "Epoch 51\tTop1 Train accuracy 66.65178680419922\tTop1 Test accuracy: 64.72196960449219\tTop5 test acc: 96.69519805908203\n",
            "Epoch 52\tTop1 Train accuracy 66.69363403320312\tTop1 Test accuracy: 64.70358276367188\tTop5 test acc: 96.72449493408203\n",
            "Epoch 53\tTop1 Train accuracy 66.70957946777344\tTop1 Test accuracy: 64.75241088867188\tTop5 test acc: 96.71472930908203\n",
            "Epoch 54\tTop1 Train accuracy 66.72552490234375\tTop1 Test accuracy: 64.81100463867188\tTop5 test acc: 96.71472930908203\n",
            "Epoch 55\tTop1 Train accuracy 66.73548889160156\tTop1 Test accuracy: 64.84892272949219\tTop5 test acc: 96.69519805908203\n",
            "Epoch 56\tTop1 Train accuracy 66.77734375\tTop1 Test accuracy: 64.82077026367188\tTop5 test acc: 96.71472930908203\n",
            "Epoch 57\tTop1 Train accuracy 66.78730773925781\tTop1 Test accuracy: 64.81100463867188\tTop5 test acc: 96.73426055908203\n",
            "Epoch 58\tTop1 Train accuracy 66.8092269897461\tTop1 Test accuracy: 64.82077026367188\tTop5 test acc: 96.73426055908203\n",
            "Epoch 59\tTop1 Train accuracy 66.82716369628906\tTop1 Test accuracy: 64.81962585449219\tTop5 test acc: 96.74402618408203\n",
            "Epoch 60\tTop1 Train accuracy 66.84510040283203\tTop1 Test accuracy: 64.83800506591797\tTop5 test acc: 96.74402618408203\n",
            "Epoch 61\tTop1 Train accuracy 66.875\tTop1 Test accuracy: 64.80009460449219\tTop5 test acc: 96.75379180908203\n",
            "Epoch 62\tTop1 Train accuracy 66.88894653320312\tTop1 Test accuracy: 64.79032897949219\tTop5 test acc: 96.76355743408203\n",
            "Epoch 63\tTop1 Train accuracy 66.91127014160156\tTop1 Test accuracy: 64.78056335449219\tTop5 test acc: 96.76355743408203\n",
            "Epoch 64\tTop1 Train accuracy 66.93319702148438\tTop1 Test accuracy: 64.76103210449219\tTop5 test acc: 96.77332305908203\n",
            "Epoch 65\tTop1 Train accuracy 66.96907043457031\tTop1 Test accuracy: 64.78056335449219\tTop5 test acc: 96.77332305908203\n",
            "Epoch 66\tTop1 Train accuracy 66.97704315185547\tTop1 Test accuracy: 64.79032897949219\tTop5 test acc: 96.77332305908203\n",
            "Epoch 67\tTop1 Train accuracy 67.00494384765625\tTop1 Test accuracy: 64.76103210449219\tTop5 test acc: 96.77332305908203\n",
            "Epoch 68\tTop1 Train accuracy 67.02487182617188\tTop1 Test accuracy: 64.74150085449219\tTop5 test acc: 96.77332305908203\n",
            "Epoch 69\tTop1 Train accuracy 67.04280853271484\tTop1 Test accuracy: 64.73173522949219\tTop5 test acc: 96.78308868408203\n",
            "Epoch 70\tTop1 Train accuracy 67.04280853271484\tTop1 Test accuracy: 64.77079772949219\tTop5 test acc: 96.77332305908203\n",
            "Epoch 71\tTop1 Train accuracy 67.0447998046875\tTop1 Test accuracy: 64.79032897949219\tTop5 test acc: 96.77332305908203\n",
            "Epoch 72\tTop1 Train accuracy 67.05078125\tTop1 Test accuracy: 64.75241088867188\tTop5 test acc: 96.77332305908203\n",
            "Epoch 73\tTop1 Train accuracy 67.06074523925781\tTop1 Test accuracy: 64.76217651367188\tTop5 test acc: 96.77332305908203\n",
            "Epoch 74\tTop1 Train accuracy 67.07270050048828\tTop1 Test accuracy: 64.74264526367188\tTop5 test acc: 96.77332305908203\n",
            "Epoch 75\tTop1 Train accuracy 67.0826644897461\tTop1 Test accuracy: 64.7340316772461\tTop5 test acc: 96.77332305908203\n",
            "Epoch 76\tTop1 Train accuracy 67.09263610839844\tTop1 Test accuracy: 64.7242660522461\tTop5 test acc: 96.78308868408203\n",
            "Epoch 77\tTop1 Train accuracy 67.1045913696289\tTop1 Test accuracy: 64.6949691772461\tTop5 test acc: 96.76470184326172\n",
            "Epoch 78\tTop1 Train accuracy 67.1105728149414\tTop1 Test accuracy: 64.6949691772461\tTop5 test acc: 96.75493621826172\n",
            "Epoch 79\tTop1 Train accuracy 67.13288879394531\tTop1 Test accuracy: 64.6949691772461\tTop5 test acc: 96.75493621826172\n",
            "Epoch 80\tTop1 Train accuracy 67.13887023925781\tTop1 Test accuracy: 64.7242660522461\tTop5 test acc: 96.76470184326172\n",
            "Epoch 81\tTop1 Train accuracy 67.14684295654297\tTop1 Test accuracy: 64.7145004272461\tTop5 test acc: 96.76470184326172\n",
            "Epoch 82\tTop1 Train accuracy 67.17076110839844\tTop1 Test accuracy: 64.7242660522461\tTop5 test acc: 96.75493621826172\n",
            "Epoch 83\tTop1 Train accuracy 67.20065307617188\tTop1 Test accuracy: 64.71565246582031\tTop5 test acc: 96.75493621826172\n",
            "Epoch 84\tTop1 Train accuracy 67.21659851074219\tTop1 Test accuracy: 64.72541809082031\tTop5 test acc: 96.74517059326172\n",
            "Epoch 85\tTop1 Train accuracy 67.21061706542969\tTop1 Test accuracy: 64.7437973022461\tTop5 test acc: 96.74517059326172\n",
            "Epoch 86\tTop1 Train accuracy 67.23851776123047\tTop1 Test accuracy: 64.7535629272461\tTop5 test acc: 96.74517059326172\n",
            "Epoch 87\tTop1 Train accuracy 67.25247192382812\tTop1 Test accuracy: 64.72541809082031\tTop5 test acc: 96.74517059326172\n",
            "Epoch 88\tTop1 Train accuracy 67.2584457397461\tTop1 Test accuracy: 64.71565246582031\tTop5 test acc: 96.73540496826172\n",
            "Epoch 89\tTop1 Train accuracy 67.26641845703125\tTop1 Test accuracy: 64.79377746582031\tTop5 test acc: 96.73540496826172\n",
            "Epoch 90\tTop1 Train accuracy 67.2704086303711\tTop1 Test accuracy: 64.79377746582031\tTop5 test acc: 96.72563934326172\n",
            "Epoch 91\tTop1 Train accuracy 67.2803726196289\tTop1 Test accuracy: 64.77424621582031\tTop5 test acc: 96.73540496826172\n",
            "Epoch 92\tTop1 Train accuracy 67.29033660888672\tTop1 Test accuracy: 64.78401184082031\tTop5 test acc: 96.74517059326172\n",
            "Epoch 93\tTop1 Train accuracy 67.29830932617188\tTop1 Test accuracy: 64.78401184082031\tTop5 test acc: 96.74517059326172\n",
            "Epoch 94\tTop1 Train accuracy 67.30429077148438\tTop1 Test accuracy: 64.78401184082031\tTop5 test acc: 96.74517059326172\n",
            "Epoch 95\tTop1 Train accuracy 67.30030059814453\tTop1 Test accuracy: 64.79377746582031\tTop5 test acc: 96.74517059326172\n",
            "Epoch 96\tTop1 Train accuracy 67.30827331542969\tTop1 Test accuracy: 64.77424621582031\tTop5 test acc: 96.72563934326172\n",
            "Epoch 97\tTop1 Train accuracy 67.31624603271484\tTop1 Test accuracy: 64.7926254272461\tTop5 test acc: 96.71587371826172\n",
            "Epoch 98\tTop1 Train accuracy 67.32222747802734\tTop1 Test accuracy: 64.8219223022461\tTop5 test acc: 96.71587371826172\n",
            "Epoch 99\tTop1 Train accuracy 67.32820129394531\tTop1 Test accuracy: 64.8121566772461\tTop5 test acc: 96.71587371826172\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtYqHZirMNZk"
      },
      "source": [
        ""
      ],
      "execution_count": 26,
      "outputs": []
    }
  ]
}