mirrors/SimCLR
1
0
Fork 0
mirror of https://github.com/sthalles/SimCLR.git synced 2025-06-03 15:03:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Major refactor, small fixes

Browse Source
This commit is contained in:
Thalles Silva 2021-01-17 20:07:59 -03:00
parent 2c9536f731
commit d0112ed55b
4 changed files with 473 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
data_aug/contrastive_learning_dataset.py
View File

@ -2,6 +2,7 @@ from torchvision.transforms import transforms
from data_aug.gaussian_blur import GaussianBlur from data_aug.gaussian_blur import GaussianBlur
from torchvision import transforms, datasets from torchvision import transforms, datasets
from data_aug.view_generator import ContrastiveLearningViewGenerator from data_aug.view_generator import ContrastiveLearningViewGenerator
from exceptions.exceptions import InvalidDatasetSelection
class ContrastiveLearningDataset: class ContrastiveLearningDataset:
@ -33,5 +34,9 @@ class ContrastiveLearningDataset:
n_views), n_views),
download=True)} download=True)}
dataset = valid_datasets.get(name, 'Invalid dataset option.')() try:
return dataset dataset_fn = valid_datasets[name]
except KeyError:
raise InvalidDatasetSelection()
else:
return dataset_fn()

4
exceptions/exceptions.py
View File

@ -4,3 +4,7 @@ class BaseSimCLRException(Exception):
class InvalidBackboneError(BaseSimCLRException): class InvalidBackboneError(BaseSimCLRException):
"""Raised when the choice of backbone Convnet is invalid.""" """Raised when the choice of backbone Convnet is invalid."""
class InvalidDatasetSelection(BaseSimCLRException):
"""Raised when the choice of dataset is invalid."""

880
feature_eval/mini_batch_logistic_regression_evaluator.ipynb
View File

@ -1,444 +1,442 @@
{ {
"nbformat": 4, "cells": [
"nbformat_minor": 0, {
"metadata": { "cell_type": "markdown",
"kernelspec": { "metadata": {
"display_name": "pytorch", "colab_type": "text",
"language": "python", "id": "view-in-github"
"name": "pytorch" },
}, "source": [
"language_info": { "<a href=\"https://colab.research.google.com/github/sthalles/SimCLR/blob/master/feature_eval/mini_batch_logistic_regression_evaluator.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
"codemirror_mode": { ]
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
},
"colab": {
"name": "mini-batch-logistic-regression-evaluator.ipynb",
"provenance": [],
"include_colab_link": true
},
"accelerator": "GPU"
}, },
"cells": [ {
{ "cell_type": "code",
"cell_type": "markdown", "execution_count": 0,
"metadata": { "metadata": {
"id": "view-in-github", "colab": {},
"colab_type": "text" "colab_type": "code",
}, "id": "YUemQib7ZE4D"
"source": [ },
"<a href=\"https://colab.research.google.com/github/sthalles/SimCLR/blob/master/feature_eval/mini_batch_logistic_regression_evaluator.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" "outputs": [],
] "source": [
}, "import torch\n",
{ "import sys\n",
"cell_type": "code", "import numpy as np\n",
"metadata": { "import os\n",
"id": "YUemQib7ZE4D", "from sklearn.neighbors import KNeighborsClassifier\n",
"colab_type": "code", "import yaml\n",
"colab": {} "import matplotlib.pyplot as plt\n",
}, "from sklearn.decomposition import PCA\n",
"source": [ "from sklearn.linear_model import LogisticRegression\n",
"import torch\n", "from sklearn import preprocessing\n",
"import sys\n", "import importlib.util"
"import numpy as np\n", ]
"import os\n", },
"from sklearn.neighbors import KNeighborsClassifier\n", {
"import yaml\n", "cell_type": "code",
"import matplotlib.pyplot as plt\n", "execution_count": 0,
"from sklearn.decomposition import PCA\n", "metadata": {
"from sklearn.linear_model import LogisticRegression\n", "colab": {},
"from sklearn import preprocessing\n", "colab_type": "code",
"import importlib.util" "id": "WSgRE1CcLqdS"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "!pip install gdown"
{ ]
"cell_type": "code", },
"metadata": { {
"id": "WSgRE1CcLqdS", "cell_type": "code",
"colab_type": "code", "execution_count": 0,
"colab": {} "metadata": {
}, "colab": {},
"source": [ "colab_type": "code",
"!pip install gdown" "id": "NOIJEui1ZziV"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "def get_file_id_by_model(folder_name):\n",
{ " file_id = {'resnet-18_40-epochs': '1c4eVon0sUd-ChVhH6XMpF6nCngNJsAPk',\n",
"cell_type": "code", " 'resnet-18_80-epochs': '1L0yoeY9i2mzDcj69P4slTWb-cfr3PyoT',\n",
"metadata": { " 'resnet-50_40-epochs': '1TZqBNTFCsO-mxAiR-zJeyupY-J2gA27Q',\n",
"id": "NOIJEui1ZziV", " 'resnet-50_80-epochs': '1is1wkBRccHdhSKQnPUTQoaFkVNSaCb35',\n",
"colab_type": "code", " 'resnet-18_100-epochs':'1aZ12TITXnajZ6QWmS_SDm8Sp8gXNbeCQ'}\n",
"colab": {} " return file_id.get(folder_name, \"Model not found.\")"
}, ]
"source": [ },
"def get_file_id_by_model(folder_name):\n", {
" file_id = {'resnet-18_40-epochs': '1c4eVon0sUd-ChVhH6XMpF6nCngNJsAPk',\n", "cell_type": "code",
" 'resnet-18_80-epochs': '1L0yoeY9i2mzDcj69P4slTWb-cfr3PyoT',\n", "execution_count": 0,
" 'resnet-50_40-epochs': '1TZqBNTFCsO-mxAiR-zJeyupY-J2gA27Q',\n", "metadata": {
" 'resnet-50_80-epochs': '1is1wkBRccHdhSKQnPUTQoaFkVNSaCb35',\n", "colab": {},
" 'resnet-18_100-epochs':'1aZ12TITXnajZ6QWmS_SDm8Sp8gXNbeCQ'}\n", "colab_type": "code",
" return file_id.get(folder_name, \"Model not found.\")" "id": "G7YMxsvEZMrX"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "folder_name = 'resnet-50_40-epochs'\n",
{ "file_id = get_file_id_by_model(folder_name)\n",
"cell_type": "code", "print(folder_name, file_id)"
"metadata": { ]
"id": "G7YMxsvEZMrX", },
"colab_type": "code", {
"colab": {} "cell_type": "code",
}, "execution_count": 0,
"source": [ "metadata": {
"folder_name = 'resnet-50_40-epochs'\n", "colab": {},
"file_id = get_file_id_by_model(folder_name)\n", "colab_type": "code",
"print(folder_name, file_id)" "id": "PWZ8fet_YoJm"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "# download and extract model files\n",
{ "os.system('gdown https://drive.google.com/uc?id={}'.format(file_id))\n",
"cell_type": "code", "os.system('unzip {}'.format(folder_name))\n",
"metadata": { "!ls"
"id": "PWZ8fet_YoJm", ]
"colab_type": "code", },
"colab": {} {
}, "cell_type": "code",
"source": [ "execution_count": 0,
"# download and extract model files\n", "metadata": {
"os.system('gdown https://drive.google.com/uc?id={}'.format(file_id))\n", "colab": {},
"os.system('unzip {}'.format(folder_name))\n", "colab_type": "code",
"!ls" "id": "3_nypQVEv-hn"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "from torch.utils.data import DataLoader\n",
{ "import torchvision.transforms as transforms\n",
"cell_type": "code", "from torchvision import datasets"
"metadata": { ]
"id": "3_nypQVEv-hn", },
"colab_type": "code", {
"colab": {} "cell_type": "code",
}, "execution_count": 0,
"source": [ "metadata": {
"from torch.utils.data import DataLoader\n", "colab": {},
"import torchvision.transforms as transforms\n", "colab_type": "code",
"from torchvision import datasets" "id": "lDfbL3w_Z0Od"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
{ "print(\"Using device:\", device)"
"cell_type": "code", ]
"metadata": { },
"id": "lDfbL3w_Z0Od", {
"colab_type": "code", "cell_type": "code",
"colab": {} "execution_count": 0,
}, "metadata": {
"source": [ "colab": {},
"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", "colab_type": "code",
"print(\"Using device:\", device)" "id": "IQMIryc6LjQd"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "checkpoints_folder = os.path.join(folder_name, 'checkpoints')\n",
{ "config = yaml.load(open(os.path.join(checkpoints_folder, \"config.yaml\"), \"r\"))\n",
"cell_type": "code", "config"
"metadata": { ]
"id": "IQMIryc6LjQd", },
"colab_type": "code", {
"colab": {} "cell_type": "code",
}, "execution_count": 0,
"source": [ "metadata": {
"checkpoints_folder = os.path.join(folder_name, 'checkpoints')\n", "colab": {},
"config = yaml.load(open(os.path.join(checkpoints_folder, \"config.yaml\"), \"r\"))\n", "colab_type": "code",
"config" "id": "BfIPl0G6_RrT"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "def get_stl10_data_loaders(download, shuffle=False, batch_size=128):\n",
{ " train_dataset = datasets.STL10('./data', split='train', download=download,\n",
"cell_type": "code", " transform=transforms.ToTensor())\n",
"metadata": { "\n",
"id": "BfIPl0G6_RrT", " train_loader = DataLoader(train_dataset, batch_size=batch_size,\n",
"colab_type": "code", " num_workers=0, drop_last=False, shuffle=shuffle)\n",
"colab": {} " \n",
}, " test_dataset = datasets.STL10('./data', split='test', download=download,\n",
"source": [ " transform=transforms.ToTensor())\n",
"def get_stl10_data_loaders(download, shuffle=False, batch_size=128):\n", "\n",
" train_dataset = datasets.STL10('./data', split='train', download=download,\n", " test_loader = DataLoader(test_dataset, batch_size=batch_size,\n",
" transform=transforms.ToTensor())\n", " num_workers=0, drop_last=False, shuffle=shuffle)\n",
"\n", " return train_loader, test_loader"
" train_loader = DataLoader(train_dataset, batch_size=batch_size,\n", ]
" num_workers=0, drop_last=False, shuffle=shuffle)\n", },
" \n", {
" test_dataset = datasets.STL10('./data', split='test', download=download,\n", "cell_type": "code",
" transform=transforms.ToTensor())\n", "execution_count": 0,
"\n", "metadata": {
" test_loader = DataLoader(test_dataset, batch_size=batch_size,\n", "colab": {},
" num_workers=0, drop_last=False, shuffle=shuffle)\n", "colab_type": "code",
" return train_loader, test_loader" "id": "a18lPD-tIle6"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "def _load_resnet_model(checkpoints_folder):\n",
{ " # Load the neural net module\n",
"cell_type": "code", " spec = importlib.util.spec_from_file_location(\"model\", os.path.join(checkpoints_folder, 'resnet_simclr.py'))\n",
"metadata": { " resnet_module = importlib.util.module_from_spec(spec)\n",
"id": "a18lPD-tIle6", " spec.loader.exec_module(resnet_module)\n",
"colab_type": "code", "\n",
"colab": {} " model = resnet_module.ResNetSimCLR(**config['model'])\n",
}, " model.eval()\n",
"source": [ "\n",
"def _load_resnet_model(checkpoints_folder):\n", " state_dict = torch.load(os.path.join(checkpoints_folder, 'model.pth'), map_location=torch.device('cpu'))\n",
" # Load the neural net module\n", " model.load_state_dict(state_dict)\n",
" spec = importlib.util.spec_from_file_location(\"model\", os.path.join(checkpoints_folder, 'resnet_simclr.py'))\n", " model = model.to(device)\n",
" resnet_module = importlib.util.module_from_spec(spec)\n", " return model"
" spec.loader.exec_module(resnet_module)\n", ]
"\n", },
" model = resnet_module.ResNetSimCLR(**config['model'])\n", {
" model.eval()\n", "cell_type": "markdown",
"\n", "metadata": {
" state_dict = torch.load(os.path.join(checkpoints_folder, 'model.pth'), map_location=torch.device('cpu'))\n", "colab_type": "text",
" model.load_state_dict(state_dict)\n", "id": "5nf4rDtWLjRE"
" model = model.to(device)\n", },
" return model" "source": [
], "## Protocol #2 Logisitc Regression"
"execution_count": 0, ]
"outputs": [] },
}, {
{ "cell_type": "code",
"cell_type": "markdown", "execution_count": 0,
"metadata": { "metadata": {
"id": "5nf4rDtWLjRE", "colab": {},
"colab_type": "text" "colab_type": "code",
}, "id": "7jjSxmDnHNQz"
"source": [ },
"## Protocol #2 Logisitc Regression" "outputs": [],
] "source": [
}, "class ResNetFeatureExtractor(object):\n",
{ " def __init__(self, checkpoints_folder):\n",
"cell_type": "code", " self.checkpoints_folder = checkpoints_folder\n",
"metadata": { " self.model = _load_resnet_model(checkpoints_folder)\n",
"id": "7jjSxmDnHNQz", "\n",
"colab_type": "code", " def _inference(self, loader):\n",
"colab": {} " feature_vector = []\n",
}, " labels_vector = []\n",
"source": [ " for batch_x, batch_y in loader:\n",
"class ResNetFeatureExtractor(object):\n", "\n",
" def __init__(self, checkpoints_folder):\n", " batch_x = batch_x.to(device)\n",
" self.checkpoints_folder = checkpoints_folder\n", " labels_vector.extend(batch_y)\n",
" self.model = _load_resnet_model(checkpoints_folder)\n", "\n",
"\n", " features, _ = self.model(batch_x)\n",
" def _inference(self, loader):\n", " feature_vector.extend(features.cpu().detach().numpy())\n",
" feature_vector = []\n", "\n",
" labels_vector = []\n", " feature_vector = np.array(feature_vector)\n",
" for batch_x, batch_y in loader:\n", " labels_vector = np.array(labels_vector)\n",
"\n", "\n",
" batch_x = batch_x.to(device)\n", " print(\"Features shape {}\".format(feature_vector.shape))\n",
" labels_vector.extend(batch_y)\n", " return feature_vector, labels_vector\n",
"\n", "\n",
" features, _ = self.model(batch_x)\n", " def get_resnet_features(self):\n",
" feature_vector.extend(features.cpu().detach().numpy())\n", " train_loader, test_loader = get_stl10_data_loaders(download=True)\n",
"\n", " X_train_feature, y_train = self._inference(train_loader)\n",
" feature_vector = np.array(feature_vector)\n", " X_test_feature, y_test = self._inference(test_loader)\n",
" labels_vector = np.array(labels_vector)\n", "\n",
"\n", " return X_train_feature, y_train, X_test_feature, y_test"
" print(\"Features shape {}\".format(feature_vector.shape))\n", ]
" return feature_vector, labels_vector\n", },
"\n", {
" def get_resnet_features(self):\n", "cell_type": "code",
" train_loader, test_loader = get_stl10_data_loaders(download=True)\n", "execution_count": 0,
" X_train_feature, y_train = self._inference(train_loader)\n", "metadata": {
" X_test_feature, y_test = self._inference(test_loader)\n", "colab": {},
"\n", "colab_type": "code",
" return X_train_feature, y_train, X_test_feature, y_test" "id": "kghx1govJq5_"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "resnet_feature_extractor = ResNetFeatureExtractor(checkpoints_folder)"
{ ]
"cell_type": "code", },
"metadata": { {
"id": "kghx1govJq5_", "cell_type": "code",
"colab_type": "code", "execution_count": 0,
"colab": {} "metadata": {
}, "colab": {},
"source": [ "colab_type": "code",
"resnet_feature_extractor = ResNetFeatureExtractor(checkpoints_folder)" "id": "S_JcznxVJ1Xj"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "X_train_feature, y_train, X_test_feature, y_test = resnet_feature_extractor.get_resnet_features()"
{ ]
"cell_type": "code", },
"metadata": { {
"id": "S_JcznxVJ1Xj", "cell_type": "code",
"colab_type": "code", "execution_count": 0,
"colab": {} "metadata": {
}, "colab": {},
"source": [ "colab_type": "code",
"X_train_feature, y_train, X_test_feature, y_test = resnet_feature_extractor.get_resnet_features()" "id": "oftbHXcdLjRM"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "import torch.nn as nn\n",
{ "\n",
"cell_type": "code", "class LogisticRegression(nn.Module):\n",
"metadata": { " \n",
"id": "oftbHXcdLjRM", " def __init__(self, n_features, n_classes):\n",
"colab_type": "code", " super(LogisticRegression, self).__init__()\n",
"colab": {} " self.model = nn.Linear(n_features, n_classes)\n",
}, "\n",
"source": [ " def forward(self, x):\n",
"import torch.nn as nn\n", " return self.model(x)"
"\n", ]
"class LogisticRegression(nn.Module):\n", },
" \n", {
" def __init__(self, n_features, n_classes):\n", "cell_type": "code",
" super(LogisticRegression, self).__init__()\n", "execution_count": 0,
" self.model = nn.Linear(n_features, n_classes)\n", "metadata": {
"\n", "colab": {},
" def forward(self, x):\n", "colab_type": "code",
" return self.model(x)" "id": "Ks73ePLtNWeV"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "class LogiticRegressionEvaluator(object):\n",
{ " def __init__(self, n_features, n_classes):\n",
"cell_type": "code", " self.log_regression = LogisticRegression(n_features, n_classes).to(device)\n",
"metadata": { " self.scaler = preprocessing.StandardScaler()\n",
"id": "Ks73ePLtNWeV", "\n",
"colab_type": "code", " def _normalize_dataset(self, X_train, X_test):\n",
"colab": {} " print(\"Standard Scaling Normalizer\")\n",
}, " self.scaler.fit(X_train)\n",
"source": [ " X_train = self.scaler.transform(X_train)\n",
"class LogiticRegressionEvaluator(object):\n", " X_test = self.scaler.transform(X_test)\n",
" def __init__(self, n_features, n_classes):\n", " return X_train, X_test\n",
" self.log_regression = LogisticRegression(n_features, n_classes).to(device)\n", "\n",
" self.scaler = preprocessing.StandardScaler()\n", " @staticmethod\n",
"\n", " def _sample_weight_decay():\n",
" def _normalize_dataset(self, X_train, X_test):\n", " # We selected the l2 regularization parameter from a range of 45 logarithmically spaced values between 10−6 and 105\n",
" print(\"Standard Scaling Normalizer\")\n", " weight_decay = np.logspace(-6, 5, num=45, base=10.0)\n",
" self.scaler.fit(X_train)\n", " weight_decay = np.random.choice(weight_decay)\n",
" X_train = self.scaler.transform(X_train)\n", " print(\"Sampled weight decay:\", weight_decay)\n",
" X_test = self.scaler.transform(X_test)\n", " return weight_decay\n",
" return X_train, X_test\n", "\n",
"\n", " def eval(self, test_loader):\n",
" @staticmethod\n", " correct = 0\n",
" def _sample_weight_decay():\n", " total = 0\n",
" # We selected the l2 regularization parameter from a range of 45 logarithmically spaced values between 10−6 and 105\n", "\n",
" weight_decay = np.logspace(-6, 5, num=45, base=10.0)\n", " with torch.no_grad():\n",
" weight_decay = np.random.choice(weight_decay)\n", " self.log_regression.eval()\n",
" print(\"Sampled weight decay:\", weight_decay)\n", " for batch_x, batch_y in test_loader:\n",
" return weight_decay\n", " batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n",
"\n", " logits = self.log_regression(batch_x)\n",
" def eval(self, test_loader):\n", "\n",
" correct = 0\n", " predicted = torch.argmax(logits, dim=1)\n",
" total = 0\n", " total += batch_y.size(0)\n",
"\n", " correct += (predicted == batch_y).sum().item()\n",
" with torch.no_grad():\n", "\n",
" self.log_regression.eval()\n", " final_acc = 100 * correct / total\n",
" for batch_x, batch_y in test_loader:\n", " self.log_regression.train()\n",
" batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n", " return final_acc\n",
" logits = self.log_regression(batch_x)\n", "\n",
"\n", "\n",
" predicted = torch.argmax(logits, dim=1)\n", " def create_data_loaders_from_arrays(self, X_train, y_train, X_test, y_test):\n",
" total += batch_y.size(0)\n", " X_train, X_test = self._normalize_dataset(X_train, X_test)\n",
" correct += (predicted == batch_y).sum().item()\n", "\n",
"\n", " train = torch.utils.data.TensorDataset(torch.from_numpy(X_train), torch.from_numpy(y_train).type(torch.long))\n",
" final_acc = 100 * correct / total\n", " train_loader = torch.utils.data.DataLoader(train, batch_size=396, shuffle=False)\n",
" self.log_regression.train()\n", "\n",
" return final_acc\n", " test = torch.utils.data.TensorDataset(torch.from_numpy(X_test), torch.from_numpy(y_test).type(torch.long))\n",
"\n", " test_loader = torch.utils.data.DataLoader(test, batch_size=512, shuffle=False)\n",
"\n", " return train_loader, test_loader\n",
" def create_data_loaders_from_arrays(self, X_train, y_train, X_test, y_test):\n", "\n",
" X_train, X_test = self._normalize_dataset(X_train, X_test)\n", " def train(self, X_train, y_train, X_test, y_test):\n",
"\n", " \n",
" train = torch.utils.data.TensorDataset(torch.from_numpy(X_train), torch.from_numpy(y_train).type(torch.long))\n", " train_loader, test_loader = self.create_data_loaders_from_arrays(X_train, y_train, X_test, y_test)\n",
" train_loader = torch.utils.data.DataLoader(train, batch_size=396, shuffle=False)\n", "\n",
"\n", " weight_decay = self._sample_weight_decay()\n",
" test = torch.utils.data.TensorDataset(torch.from_numpy(X_test), torch.from_numpy(y_test).type(torch.long))\n", "\n",
" test_loader = torch.utils.data.DataLoader(test, batch_size=512, shuffle=False)\n", " optimizer = torch.optim.Adam(self.log_regression.parameters(), 3e-4, weight_decay=weight_decay)\n",
" return train_loader, test_loader\n", " criterion = torch.nn.CrossEntropyLoss()\n",
"\n", "\n",
" def train(self, X_train, y_train, X_test, y_test):\n", " best_accuracy = 0\n",
" \n", "\n",
" train_loader, test_loader = self.create_data_loaders_from_arrays(X_train, y_train, X_test, y_test)\n", " for e in range(200):\n",
"\n", " \n",
" weight_decay = self._sample_weight_decay()\n", " for batch_x, batch_y in train_loader:\n",
"\n", "\n",
" optimizer = torch.optim.Adam(self.log_regression.parameters(), 3e-4, weight_decay=weight_decay)\n", " batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n",
" criterion = torch.nn.CrossEntropyLoss()\n", "\n",
"\n", " optimizer.zero_grad()\n",
" best_accuracy = 0\n", "\n",
"\n", " logits = self.log_regression(batch_x)\n",
" for e in range(200):\n", "\n",
" \n", " loss = criterion(logits, batch_y)\n",
" for batch_x, batch_y in train_loader:\n", "\n",
"\n", " loss.backward()\n",
" batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n", " optimizer.step()\n",
"\n", "\n",
" optimizer.zero_grad()\n", " epoch_acc = self.eval(test_loader)\n",
"\n", " \n",
" logits = self.log_regression(batch_x)\n", " if epoch_acc > best_accuracy:\n",
"\n", " #print(\"Saving new model with accuracy {}\".format(epoch_acc))\n",
" loss = criterion(logits, batch_y)\n", " best_accuracy = epoch_acc\n",
"\n", " torch.save(self.log_regression.state_dict(), 'log_regression.pth')\n",
" loss.backward()\n", "\n",
" optimizer.step()\n", " print(\"--------------\")\n",
"\n", " print(\"Done training\")\n",
" epoch_acc = self.eval(test_loader)\n", " print(\"Best accuracy:\", best_accuracy)"
" \n", ]
" if epoch_acc > best_accuracy:\n", },
" #print(\"Saving new model with accuracy {}\".format(epoch_acc))\n", {
" best_accuracy = epoch_acc\n", "cell_type": "code",
" torch.save(self.log_regression.state_dict(), 'log_regression.pth')\n", "execution_count": 0,
"\n", "metadata": {
" print(\"--------------\")\n", "colab": {},
" print(\"Done training\")\n", "colab_type": "code",
" print(\"Best accuracy:\", best_accuracy)" "id": "NE716m7SOkaK"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": [
}, "log_regressor_evaluator = LogiticRegressionEvaluator(n_features=X_train_feature.shape[1], n_classes=10)\n",
{ "\n",
"cell_type": "code", "log_regressor_evaluator.train(X_train_feature, y_train, X_test_feature, y_test)"
"metadata": { ]
"id": "NE716m7SOkaK", },
"colab_type": "code", {
"colab": {} "cell_type": "code",
}, "execution_count": 0,
"source": [ "metadata": {
"log_regressor_evaluator = LogiticRegressionEvaluator(n_features=X_train_feature.shape[1], n_classes=10)\n", "colab": {},
"\n", "colab_type": "code",
"log_regressor_evaluator.train(X_train_feature, y_train, X_test_feature, y_test)" "id": "_GC0a14uWRr6"
], },
"execution_count": 0, "outputs": [],
"outputs": [] "source": []
}, }
{ ],
"cell_type": "code", "metadata": {
"metadata": { "accelerator": "GPU",
"id": "_GC0a14uWRr6", "colab": {
"colab_type": "code", "include_colab_link": true,
"colab": {} "name": "mini-batch-logistic-regression-evaluator.ipynb",
}, "provenance": []
"source": [ },
"" "kernelspec": {
], "display_name": "Python [conda env:image]",
"execution_count": 0, "language": "python",
"outputs": [] "name": "conda-env-image-py"
} },
] "language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.12"
}
},
"nbformat": 4,
"nbformat_minor": 1
} }

25
simclr.py
View File

@ -34,6 +34,23 @@ def _save_config_file(model_checkpoints_folder, args):
yaml.dump(args, outfile, default_flow_style=False) yaml.dump(args, outfile, default_flow_style=False)
def accuracy(output, target, topk=(1,)):
"""Computes the accuracy over the k top predictions for the specified values of k"""
with torch.no_grad():
maxk = max(topk)
batch_size = target.size(0)
_, pred = output.topk(maxk, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
res.append(correct_k.mul_(100.0 / batch_size))
return res
class SimCLR(object): class SimCLR(object):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@ -97,10 +114,10 @@ class SimCLR(object):
self.optimizer.step() self.optimizer.step()
if n_iter % self.args.log_every_n_steps == 0: if n_iter % self.args.log_every_n_steps == 0:
predictions = torch.argmax(logits, dim=1) top1, top5 = accuracy(logits, labels, topk=(1,5))
acc = 100 * (predictions == labels).float().mean()
self.writer.add_scalar('loss', loss, global_step=n_iter) self.writer.add_scalar('loss', loss, global_step=n_iter)
self.writer.add_scalar('acc/top1', acc, global_step=n_iter) self.writer.add_scalar('acc/top1', top1[0], global_step=n_iter)
self.writer.add_scalar('acc/top5', top5[0], global_step=n_iter)
self.writer.add_scalar('learning_rate', self.scheduler.get_lr()[0], global_step=n_iter) self.writer.add_scalar('learning_rate', self.scheduler.get_lr()[0], global_step=n_iter)
n_iter += 1 n_iter += 1
@ -108,7 +125,7 @@ class SimCLR(object):
# warmup for the first 10 epochs # warmup for the first 10 epochs
if epoch_counter >= 10: if epoch_counter >= 10:
self.scheduler.step() self.scheduler.step()
logging.debug(f"Epoch: {epoch_counter}\tLoss: {loss}\tTop1 accuracy: {acc}") logging.debug(f"Epoch: {epoch_counter}\tLoss: {loss}\tTop1 accuracy: {top1[0]}")
logging.info("Training has finished.") logging.info("Training has finished.")
# save model checkpoints # save model checkpoints