We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Tutorial 3: small typo "strcture" -> "structure"
In [this section](https://docs.lightly.ai/self-supervised-learning/tutorials/package/tutorial_simclr_clothing.html#setup-data-augmentations-and-loaders), the first paragraph has the following typo:

"strcture" -> "structure"


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `docs/source/tutorials_source/package/tutorial_simclr_clothing.py`
Content:
```
1 """
2 .. _lightly-simclr-tutorial-3:
3 
4 Tutorial 3: Train SimCLR on Clothing
5 ==============================================
6 
7 In this tutorial, we will train a SimCLR model using lightly. The model,
8 augmentations and training procedure is from 
9 `A Simple Framework for Contrastive Learning of Visual Representations <https://arxiv.org/abs/2002.05709>`_.
10 
11 The paper explores a rather simple training procedure for contrastive learning.
12 Since we use the typical contrastive learning loss based on NCE the method
13 greatly benefits from having larger batch sizes. In this example, we use a batch
14 size of 256 and paired with the input resolution per image of 64x64 pixels and
15 a resnet-18 model this example requires 16GB of GPU memory.
16 
17 We use the 
18 `clothing dataset from Alex Grigorev <https://github.com/alexeygrigorev/clothing-dataset>`_ 
19 for this tutorial.
20 
21 In this tutorial you will learn:
22 
23 - How to create a SimCLR model
24 
25 - How to generate image representations
26 
27 - How different augmentations impact the learned representations
28 
29 """
30 
31 # %%
32 # Imports
33 # -------
34 #
35 # Import the Python frameworks we need for this tutorial.
36 import os
37 
38 import matplotlib.pyplot as plt
39 import numpy as np
40 import pytorch_lightning as pl
41 import torch
42 import torch.nn as nn
43 import torchvision
44 from PIL import Image
45 from sklearn.neighbors import NearestNeighbors
46 from sklearn.preprocessing import normalize
47 
48 from lightly.data import LightlyDataset
49 from lightly.transforms import SimCLRTransform, utils
50 
51 # %%
52 # Configuration
53 # -------------
54 #
55 # We set some configuration parameters for our experiment.
56 # Feel free to change them and analyze the effect.
57 #
58 # The default configuration with a batch size of 256 and input resolution of 128
59 # requires 6GB of GPU memory.
60 num_workers = 8
61 batch_size = 256
62 seed = 1
63 max_epochs = 20
64 input_size = 128
65 num_ftrs = 32
66 
67 # %%
68 # Let's set the seed for our experiments
69 pl.seed_everything(seed)
70 
71 # %%
72 # Make sure `path_to_data` points to the downloaded clothing dataset.
73 # You can download it using
74 # `git clone https://github.com/alexeygrigorev/clothing-dataset.git`
75 path_to_data = "/datasets/clothing-dataset/images"
76 
77 
78 # %%
79 # Setup data augmentations and loaders
80 # ------------------------------------
81 #
82 # The images from the dataset have been taken from above when the clothing was
83 # on a table, bed or floor. Therefore, we can make use of additional augmentations
84 # such as vertical flip or random rotation (90 degrees).
85 # By adding these augmentations we learn our model invariance regarding the
86 # orientation of the clothing piece. E.g. we don't care if a shirt is upside down
87 # but more about the strcture which make it a shirt.
88 #
89 # You can learn more about the different augmentations and learned invariances
90 # here: :ref:`lightly-advanced`.
91 transform = SimCLRTransform(input_size=input_size, vf_prob=0.5, rr_prob=0.5)
92 
93 # We create a torchvision transformation for embedding the dataset after
94 # training
95 test_transform = torchvision.transforms.Compose(
96     [
97         torchvision.transforms.Resize((input_size, input_size)),
98         torchvision.transforms.ToTensor(),
99         torchvision.transforms.Normalize(
100             mean=utils.IMAGENET_NORMALIZE["mean"],
101             std=utils.IMAGENET_NORMALIZE["std"],
102         ),
103     ]
104 )
105 
106 dataset_train_simclr = LightlyDataset(input_dir=path_to_data, transform=transform)
107 
108 dataset_test = LightlyDataset(input_dir=path_to_data, transform=test_transform)
109 
110 dataloader_train_simclr = torch.utils.data.DataLoader(
111     dataset_train_simclr,
112     batch_size=batch_size,
113     shuffle=True,
114     drop_last=True,
115     num_workers=num_workers,
116 )
117 
118 dataloader_test = torch.utils.data.DataLoader(
119     dataset_test,
120     batch_size=batch_size,
121     shuffle=False,
122     drop_last=False,
123     num_workers=num_workers,
124 )
125 
126 # %%
127 # Create the SimCLR Model
128 # -----------------------
129 # Now we create the SimCLR model. We implement it as a PyTorch Lightning Module
130 # and use a ResNet-18 backbone from Torchvision. Lightly provides implementations
131 # of the SimCLR projection head and loss function in the `SimCLRProjectionHead`
132 # and `NTXentLoss` classes. We can simply import them and combine the building
133 # blocks in the module.
134 
135 from lightly.loss import NTXentLoss
136 from lightly.models.modules.heads import SimCLRProjectionHead
137 
138 
139 class SimCLRModel(pl.LightningModule):
140     def __init__(self):
141         super().__init__()
142 
143         # create a ResNet backbone and remove the classification head
144         resnet = torchvision.models.resnet18()
145         self.backbone = nn.Sequential(*list(resnet.children())[:-1])
146 
147         hidden_dim = resnet.fc.in_features
148         self.projection_head = SimCLRProjectionHead(hidden_dim, hidden_dim, 128)
149 
150         self.criterion = NTXentLoss()
151 
152     def forward(self, x):
153         h = self.backbone(x).flatten(start_dim=1)
154         z = self.projection_head(h)
155         return z
156 
157     def training_step(self, batch, batch_idx):
158         (x0, x1), _, _ = batch
159         z0 = self.forward(x0)
160         z1 = self.forward(x1)
161         loss = self.criterion(z0, z1)
162         self.log("train_loss_ssl", loss)
163         return loss
164 
165     def configure_optimizers(self):
166         optim = torch.optim.SGD(
167             self.parameters(), lr=6e-2, momentum=0.9, weight_decay=5e-4
168         )
169         scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optim, max_epochs)
170         return [optim], [scheduler]
171 
172 
173 # %%
174 # Train the module using the PyTorch Lightning Trainer on a single GPU.
175 
176 model = SimCLRModel()
177 trainer = pl.Trainer(max_epochs=max_epochs, devices=1, accelerator="gpu")
178 trainer.fit(model, dataloader_train_simclr)
179 
180 # %%
181 # Next we create a helper function to generate embeddings
182 # from our test images using the model we just trained.
183 # Note that only the backbone is needed to generate embeddings,
184 # the projection head is only required for the training.
185 # Make sure to put the model into eval mode for this part!
186 
187 
188 def generate_embeddings(model, dataloader):
189     """Generates representations for all images in the dataloader with
190     the given model
191     """
192 
193     embeddings = []
194     filenames = []
195     with torch.no_grad():
196         for img, _, fnames in dataloader:
197             img = img.to(model.device)
198             emb = model.backbone(img).flatten(start_dim=1)
199             embeddings.append(emb)
200             filenames.extend(fnames)
201 
202     embeddings = torch.cat(embeddings, 0)
203     embeddings = normalize(embeddings)
204     return embeddings, filenames
205 
206 
207 model.eval()
208 embeddings, filenames = generate_embeddings(model, dataloader_test)
209 
210 # %%
211 # Visualize Nearest Neighbors
212 # ----------------------------
213 # Let's look at the trained embedding and visualize the nearest neighbors for
214 # a few random samples.
215 #
216 # We create some helper functions to simplify the work
217 
218 
219 def get_image_as_np_array(filename: str):
220     """Returns an image as an numpy array"""
221     img = Image.open(filename)
222     return np.asarray(img)
223 
224 
225 def plot_knn_examples(embeddings, filenames, n_neighbors=3, num_examples=6):
226     """Plots multiple rows of random images with their nearest neighbors"""
227     # lets look at the nearest neighbors for some samples
228     # we use the sklearn library
229     nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(embeddings)
230     distances, indices = nbrs.kneighbors(embeddings)
231 
232     # get 5 random samples
233     samples_idx = np.random.choice(len(indices), size=num_examples, replace=False)
234 
235     # loop through our randomly picked samples
236     for idx in samples_idx:
237         fig = plt.figure()
238         # loop through their nearest neighbors
239         for plot_x_offset, neighbor_idx in enumerate(indices[idx]):
240             # add the subplot
241             ax = fig.add_subplot(1, len(indices[idx]), plot_x_offset + 1)
242             # get the correponding filename for the current index
243             fname = os.path.join(path_to_data, filenames[neighbor_idx])
244             # plot the image
245             plt.imshow(get_image_as_np_array(fname))
246             # set the title to the distance of the neighbor
247             ax.set_title(f"d={distances[idx][plot_x_offset]:.3f}")
248             # let's disable the axis
249             plt.axis("off")
250 
251 
252 # %%
253 # Let's do the plot of the images. The leftmost image is the query image whereas
254 # the ones next to it on the same row are the nearest neighbors.
255 # In the title we see the distance of the neigbor.
256 plot_knn_examples(embeddings, filenames)
257 
258 # %%
259 # Color Invariance
260 # ---------------------
261 # Let's train again without color augmentation. This will force our model to
262 # respect the colors in the images.
263 
264 # Set color jitter and gray scale probability to 0
265 new_transform = SimCLRTransform(
266     input_size=input_size, vf_prob=0.5, rr_prob=0.5, cj_prob=0.0, random_gray_scale=0.0
267 )
268 
269 # let's update the transform on the training dataset
270 dataset_train_simclr.transform = new_transform
271 
272 # then train a new model
273 model = SimCLRModel()
274 trainer = pl.Trainer(max_epochs=max_epochs, devices=1, accelerator="gpu")
275 trainer.fit(model, dataloader_train_simclr)
276 
277 # and generate again embeddings from the test set
278 model.eval()
279 embeddings, filenames = generate_embeddings(model, dataloader_test)
280 
281 # %%
282 # other example
283 plot_knn_examples(embeddings, filenames)
284 
285 # %%
286 # What's next?
287 
288 # You could use the pre-trained model and train a classifier on top.
289 pretrained_resnet_backbone = model.backbone
290 
291 # you can also store the backbone and use it in another code
292 state_dict = {"resnet18_parameters": pretrained_resnet_backbone.state_dict()}
293 torch.save(state_dict, "model.pth")
294 
295 # %%
296 # THIS COULD BE IN A NEW FILE (e.g. inference.py)
297 #
298 # Make sure you place the `model.pth` file in the same folder as this code
299 
300 # load the model in a new file for inference
301 resnet18_new = torchvision.models.resnet18()
302 
303 # note that we need to create exactly the same backbone in order to load the weights
304 backbone_new = nn.Sequential(*list(resnet18_new.children())[:-1])
305 
306 ckpt = torch.load("model.pth")
307 backbone_new.load_state_dict(ckpt["resnet18_parameters"])
308 
309 # %%
310 # Next Steps
311 # ------------
312 #
313 # Interested in exploring other self-supervised models? Check out our other
314 # tutorials:
315 #
316 # - :ref:`lightly-moco-tutorial-2`
317 # - :ref:`lightly-simsiam-tutorial-4`
318 # - :ref:`lightly-custom-augmentation-5`
319 # - :ref:`lightly-detectron-tutorial-6`
320 #
321
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/docs/source/tutorials_source/package/tutorial_simclr_clothing.py b/docs/source/tutorials_source/package/tutorial_simclr_clothing.py
--- a/docs/source/tutorials_source/package/tutorial_simclr_clothing.py
+++ b/docs/source/tutorials_source/package/tutorial_simclr_clothing.py
@@ -84,7 +84,7 @@
 # such as vertical flip or random rotation (90 degrees).
 # By adding these augmentations we learn our model invariance regarding the
 # orientation of the clothing piece. E.g. we don't care if a shirt is upside down
-# but more about the strcture which make it a shirt.
+# but more about the structure which make it a shirt.
 #
 # You can learn more about the different augmentations and learned invariances
 # here: :ref:`lightly-advanced`.

{"golden_diff": "diff --git a/docs/source/tutorials_source/package/tutorial_simclr_clothing.py b/docs/source/tutorials_source/package/tutorial_simclr_clothing.py\n--- a/docs/source/tutorials_source/package/tutorial_simclr_clothing.py\n+++ b/docs/source/tutorials_source/package/tutorial_simclr_clothing.py\n@@ -84,7 +84,7 @@\n # such as vertical flip or random rotation (90 degrees).\n # By adding these augmentations we learn our model invariance regarding the\n # orientation of the clothing piece. E.g. we don't care if a shirt is upside down\n-# but more about the strcture which make it a shirt.\n+# but more about the structure which make it a shirt.\n #\n # You can learn more about the different augmentations and learned invariances\n # here: :ref:`lightly-advanced`.\n", "issue": "Tutorial 3: small typo \"strcture\" -> \"structure\"\nIn [this section](https://docs.lightly.ai/self-supervised-learning/tutorials/package/tutorial_simclr_clothing.html#setup-data-augmentations-and-loaders), the first paragraph has the following typo:\r\n\r\n\"strcture\" -> \"structure\"\r\n\n", "before_files": [{"content": "\"\"\"\n.. _lightly-simclr-tutorial-3:\n\nTutorial 3: Train SimCLR on Clothing\n==============================================\n\nIn this tutorial, we will train a SimCLR model using lightly. The model,\naugmentations and training procedure is from \n`A Simple Framework for Contrastive Learning of Visual Representations <https://arxiv.org/abs/2002.05709>`_.\n\nThe paper explores a rather simple training procedure for contrastive learning.\nSince we use the typical contrastive learning loss based on NCE the method\ngreatly benefits from having larger batch sizes. In this example, we use a batch\nsize of 256 and paired with the input resolution per image of 64x64 pixels and\na resnet-18 model this example requires 16GB of GPU memory.\n\nWe use the \n`clothing dataset from Alex Grigorev <https://github.com/alexeygrigorev/clothing-dataset>`_ \nfor this tutorial.\n\nIn this tutorial you will learn:\n\n- How to create a SimCLR model\n\n- How to generate image representations\n\n- How different augmentations impact the learned representations\n\n\"\"\"\n\n# %%\n# Imports\n# -------\n#\n# Import the Python frameworks we need for this tutorial.\nimport os\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pytorch_lightning as pl\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom PIL import Image\nfrom sklearn.neighbors import NearestNeighbors\nfrom sklearn.preprocessing import normalize\n\nfrom lightly.data import LightlyDataset\nfrom lightly.transforms import SimCLRTransform, utils\n\n# %%\n# Configuration\n# -------------\n#\n# We set some configuration parameters for our experiment.\n# Feel free to change them and analyze the effect.\n#\n# The default configuration with a batch size of 256 and input resolution of 128\n# requires 6GB of GPU memory.\nnum_workers = 8\nbatch_size = 256\nseed = 1\nmax_epochs = 20\ninput_size = 128\nnum_ftrs = 32\n\n# %%\n# Let's set the seed for our experiments\npl.seed_everything(seed)\n\n# %%\n# Make sure `path_to_data` points to the downloaded clothing dataset.\n# You can download it using\n# `git clone https://github.com/alexeygrigorev/clothing-dataset.git`\npath_to_data = \"/datasets/clothing-dataset/images\"\n\n\n# %%\n# Setup data augmentations and loaders\n# ------------------------------------\n#\n# The images from the dataset have been taken from above when the clothing was\n# on a table, bed or floor. Therefore, we can make use of additional augmentations\n# such as vertical flip or random rotation (90 degrees).\n# By adding these augmentations we learn our model invariance regarding the\n# orientation of the clothing piece. E.g. we don't care if a shirt is upside down\n# but more about the strcture which make it a shirt.\n#\n# You can learn more about the different augmentations and learned invariances\n# here: :ref:`lightly-advanced`.\ntransform = SimCLRTransform(input_size=input_size, vf_prob=0.5, rr_prob=0.5)\n\n# We create a torchvision transformation for embedding the dataset after\n# training\ntest_transform = torchvision.transforms.Compose(\n    [\n        torchvision.transforms.Resize((input_size, input_size)),\n        torchvision.transforms.ToTensor(),\n        torchvision.transforms.Normalize(\n            mean=utils.IMAGENET_NORMALIZE[\"mean\"],\n            std=utils.IMAGENET_NORMALIZE[\"std\"],\n        ),\n    ]\n)\n\ndataset_train_simclr = LightlyDataset(input_dir=path_to_data, transform=transform)\n\ndataset_test = LightlyDataset(input_dir=path_to_data, transform=test_transform)\n\ndataloader_train_simclr = torch.utils.data.DataLoader(\n    dataset_train_simclr,\n    batch_size=batch_size,\n    shuffle=True,\n    drop_last=True,\n    num_workers=num_workers,\n)\n\ndataloader_test = torch.utils.data.DataLoader(\n    dataset_test,\n    batch_size=batch_size,\n    shuffle=False,\n    drop_last=False,\n    num_workers=num_workers,\n)\n\n# %%\n# Create the SimCLR Model\n# -----------------------\n# Now we create the SimCLR model. We implement it as a PyTorch Lightning Module\n# and use a ResNet-18 backbone from Torchvision. Lightly provides implementations\n# of the SimCLR projection head and loss function in the `SimCLRProjectionHead`\n# and `NTXentLoss` classes. We can simply import them and combine the building\n# blocks in the module.\n\nfrom lightly.loss import NTXentLoss\nfrom lightly.models.modules.heads import SimCLRProjectionHead\n\n\nclass SimCLRModel(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n\n        # create a ResNet backbone and remove the classification head\n        resnet = torchvision.models.resnet18()\n        self.backbone = nn.Sequential(*list(resnet.children())[:-1])\n\n        hidden_dim = resnet.fc.in_features\n        self.projection_head = SimCLRProjectionHead(hidden_dim, hidden_dim, 128)\n\n        self.criterion = NTXentLoss()\n\n    def forward(self, x):\n        h = self.backbone(x).flatten(start_dim=1)\n        z = self.projection_head(h)\n        return z\n\n    def training_step(self, batch, batch_idx):\n        (x0, x1), _, _ = batch\n        z0 = self.forward(x0)\n        z1 = self.forward(x1)\n        loss = self.criterion(z0, z1)\n        self.log(\"train_loss_ssl\", loss)\n        return loss\n\n    def configure_optimizers(self):\n        optim = torch.optim.SGD(\n            self.parameters(), lr=6e-2, momentum=0.9, weight_decay=5e-4\n        )\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optim, max_epochs)\n        return [optim], [scheduler]\n\n\n# %%\n# Train the module using the PyTorch Lightning Trainer on a single GPU.\n\nmodel = SimCLRModel()\ntrainer = pl.Trainer(max_epochs=max_epochs, devices=1, accelerator=\"gpu\")\ntrainer.fit(model, dataloader_train_simclr)\n\n# %%\n# Next we create a helper function to generate embeddings\n# from our test images using the model we just trained.\n# Note that only the backbone is needed to generate embeddings,\n# the projection head is only required for the training.\n# Make sure to put the model into eval mode for this part!\n\n\ndef generate_embeddings(model, dataloader):\n    \"\"\"Generates representations for all images in the dataloader with\n    the given model\n    \"\"\"\n\n    embeddings = []\n    filenames = []\n    with torch.no_grad():\n        for img, _, fnames in dataloader:\n            img = img.to(model.device)\n            emb = model.backbone(img).flatten(start_dim=1)\n            embeddings.append(emb)\n            filenames.extend(fnames)\n\n    embeddings = torch.cat(embeddings, 0)\n    embeddings = normalize(embeddings)\n    return embeddings, filenames\n\n\nmodel.eval()\nembeddings, filenames = generate_embeddings(model, dataloader_test)\n\n# %%\n# Visualize Nearest Neighbors\n# ----------------------------\n# Let's look at the trained embedding and visualize the nearest neighbors for\n# a few random samples.\n#\n# We create some helper functions to simplify the work\n\n\ndef get_image_as_np_array(filename: str):\n    \"\"\"Returns an image as an numpy array\"\"\"\n    img = Image.open(filename)\n    return np.asarray(img)\n\n\ndef plot_knn_examples(embeddings, filenames, n_neighbors=3, num_examples=6):\n    \"\"\"Plots multiple rows of random images with their nearest neighbors\"\"\"\n    # lets look at the nearest neighbors for some samples\n    # we use the sklearn library\n    nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(embeddings)\n    distances, indices = nbrs.kneighbors(embeddings)\n\n    # get 5 random samples\n    samples_idx = np.random.choice(len(indices), size=num_examples, replace=False)\n\n    # loop through our randomly picked samples\n    for idx in samples_idx:\n        fig = plt.figure()\n        # loop through their nearest neighbors\n        for plot_x_offset, neighbor_idx in enumerate(indices[idx]):\n            # add the subplot\n            ax = fig.add_subplot(1, len(indices[idx]), plot_x_offset + 1)\n            # get the correponding filename for the current index\n            fname = os.path.join(path_to_data, filenames[neighbor_idx])\n            # plot the image\n            plt.imshow(get_image_as_np_array(fname))\n            # set the title to the distance of the neighbor\n            ax.set_title(f\"d={distances[idx][plot_x_offset]:.3f}\")\n            # let's disable the axis\n            plt.axis(\"off\")\n\n\n# %%\n# Let's do the plot of the images. The leftmost image is the query image whereas\n# the ones next to it on the same row are the nearest neighbors.\n# In the title we see the distance of the neigbor.\nplot_knn_examples(embeddings, filenames)\n\n# %%\n# Color Invariance\n# ---------------------\n# Let's train again without color augmentation. This will force our model to\n# respect the colors in the images.\n\n# Set color jitter and gray scale probability to 0\nnew_transform = SimCLRTransform(\n    input_size=input_size, vf_prob=0.5, rr_prob=0.5, cj_prob=0.0, random_gray_scale=0.0\n)\n\n# let's update the transform on the training dataset\ndataset_train_simclr.transform = new_transform\n\n# then train a new model\nmodel = SimCLRModel()\ntrainer = pl.Trainer(max_epochs=max_epochs, devices=1, accelerator=\"gpu\")\ntrainer.fit(model, dataloader_train_simclr)\n\n# and generate again embeddings from the test set\nmodel.eval()\nembeddings, filenames = generate_embeddings(model, dataloader_test)\n\n# %%\n# other example\nplot_knn_examples(embeddings, filenames)\n\n# %%\n# What's next?\n\n# You could use the pre-trained model and train a classifier on top.\npretrained_resnet_backbone = model.backbone\n\n# you can also store the backbone and use it in another code\nstate_dict = {\"resnet18_parameters\": pretrained_resnet_backbone.state_dict()}\ntorch.save(state_dict, \"model.pth\")\n\n# %%\n# THIS COULD BE IN A NEW FILE (e.g. inference.py)\n#\n# Make sure you place the `model.pth` file in the same folder as this code\n\n# load the model in a new file for inference\nresnet18_new = torchvision.models.resnet18()\n\n# note that we need to create exactly the same backbone in order to load the weights\nbackbone_new = nn.Sequential(*list(resnet18_new.children())[:-1])\n\nckpt = torch.load(\"model.pth\")\nbackbone_new.load_state_dict(ckpt[\"resnet18_parameters\"])\n\n# %%\n# Next Steps\n# ------------\n#\n# Interested in exploring other self-supervised models? Check out our other\n# tutorials:\n#\n# - :ref:`lightly-moco-tutorial-2`\n# - :ref:`lightly-simsiam-tutorial-4`\n# - :ref:`lightly-custom-augmentation-5`\n# - :ref:`lightly-detectron-tutorial-6`\n#\n", "path": "docs/source/tutorials_source/package/tutorial_simclr_clothing.py"}], "after_files": [{"content": "\"\"\"\n.. _lightly-simclr-tutorial-3:\n\nTutorial 3: Train SimCLR on Clothing\n==============================================\n\nIn this tutorial, we will train a SimCLR model using lightly. The model,\naugmentations and training procedure is from \n`A Simple Framework for Contrastive Learning of Visual Representations <https://arxiv.org/abs/2002.05709>`_.\n\nThe paper explores a rather simple training procedure for contrastive learning.\nSince we use the typical contrastive learning loss based on NCE the method\ngreatly benefits from having larger batch sizes. In this example, we use a batch\nsize of 256 and paired with the input resolution per image of 64x64 pixels and\na resnet-18 model this example requires 16GB of GPU memory.\n\nWe use the \n`clothing dataset from Alex Grigorev <https://github.com/alexeygrigorev/clothing-dataset>`_ \nfor this tutorial.\n\nIn this tutorial you will learn:\n\n- How to create a SimCLR model\n\n- How to generate image representations\n\n- How different augmentations impact the learned representations\n\n\"\"\"\n\n# %%\n# Imports\n# -------\n#\n# Import the Python frameworks we need for this tutorial.\nimport os\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pytorch_lightning as pl\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom PIL import Image\nfrom sklearn.neighbors import NearestNeighbors\nfrom sklearn.preprocessing import normalize\n\nfrom lightly.data import LightlyDataset\nfrom lightly.transforms import SimCLRTransform, utils\n\n# %%\n# Configuration\n# -------------\n#\n# We set some configuration parameters for our experiment.\n# Feel free to change them and analyze the effect.\n#\n# The default configuration with a batch size of 256 and input resolution of 128\n# requires 6GB of GPU memory.\nnum_workers = 8\nbatch_size = 256\nseed = 1\nmax_epochs = 20\ninput_size = 128\nnum_ftrs = 32\n\n# %%\n# Let's set the seed for our experiments\npl.seed_everything(seed)\n\n# %%\n# Make sure `path_to_data` points to the downloaded clothing dataset.\n# You can download it using\n# `git clone https://github.com/alexeygrigorev/clothing-dataset.git`\npath_to_data = \"/datasets/clothing-dataset/images\"\n\n\n# %%\n# Setup data augmentations and loaders\n# ------------------------------------\n#\n# The images from the dataset have been taken from above when the clothing was\n# on a table, bed or floor. Therefore, we can make use of additional augmentations\n# such as vertical flip or random rotation (90 degrees).\n# By adding these augmentations we learn our model invariance regarding the\n# orientation of the clothing piece. E.g. we don't care if a shirt is upside down\n# but more about the structure which make it a shirt.\n#\n# You can learn more about the different augmentations and learned invariances\n# here: :ref:`lightly-advanced`.\ntransform = SimCLRTransform(input_size=input_size, vf_prob=0.5, rr_prob=0.5)\n\n# We create a torchvision transformation for embedding the dataset after\n# training\ntest_transform = torchvision.transforms.Compose(\n    [\n        torchvision.transforms.Resize((input_size, input_size)),\n        torchvision.transforms.ToTensor(),\n        torchvision.transforms.Normalize(\n            mean=utils.IMAGENET_NORMALIZE[\"mean\"],\n            std=utils.IMAGENET_NORMALIZE[\"std\"],\n        ),\n    ]\n)\n\ndataset_train_simclr = LightlyDataset(input_dir=path_to_data, transform=transform)\n\ndataset_test = LightlyDataset(input_dir=path_to_data, transform=test_transform)\n\ndataloader_train_simclr = torch.utils.data.DataLoader(\n    dataset_train_simclr,\n    batch_size=batch_size,\n    shuffle=True,\n    drop_last=True,\n    num_workers=num_workers,\n)\n\ndataloader_test = torch.utils.data.DataLoader(\n    dataset_test,\n    batch_size=batch_size,\n    shuffle=False,\n    drop_last=False,\n    num_workers=num_workers,\n)\n\n# %%\n# Create the SimCLR Model\n# -----------------------\n# Now we create the SimCLR model. We implement it as a PyTorch Lightning Module\n# and use a ResNet-18 backbone from Torchvision. Lightly provides implementations\n# of the SimCLR projection head and loss function in the `SimCLRProjectionHead`\n# and `NTXentLoss` classes. We can simply import them and combine the building\n# blocks in the module.\n\nfrom lightly.loss import NTXentLoss\nfrom lightly.models.modules.heads import SimCLRProjectionHead\n\n\nclass SimCLRModel(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n\n        # create a ResNet backbone and remove the classification head\n        resnet = torchvision.models.resnet18()\n        self.backbone = nn.Sequential(*list(resnet.children())[:-1])\n\n        hidden_dim = resnet.fc.in_features\n        self.projection_head = SimCLRProjectionHead(hidden_dim, hidden_dim, 128)\n\n        self.criterion = NTXentLoss()\n\n    def forward(self, x):\n        h = self.backbone(x).flatten(start_dim=1)\n        z = self.projection_head(h)\n        return z\n\n    def training_step(self, batch, batch_idx):\n        (x0, x1), _, _ = batch\n        z0 = self.forward(x0)\n        z1 = self.forward(x1)\n        loss = self.criterion(z0, z1)\n        self.log(\"train_loss_ssl\", loss)\n        return loss\n\n    def configure_optimizers(self):\n        optim = torch.optim.SGD(\n            self.parameters(), lr=6e-2, momentum=0.9, weight_decay=5e-4\n        )\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optim, max_epochs)\n        return [optim], [scheduler]\n\n\n# %%\n# Train the module using the PyTorch Lightning Trainer on a single GPU.\n\nmodel = SimCLRModel()\ntrainer = pl.Trainer(max_epochs=max_epochs, devices=1, accelerator=\"gpu\")\ntrainer.fit(model, dataloader_train_simclr)\n\n# %%\n# Next we create a helper function to generate embeddings\n# from our test images using the model we just trained.\n# Note that only the backbone is needed to generate embeddings,\n# the projection head is only required for the training.\n# Make sure to put the model into eval mode for this part!\n\n\ndef generate_embeddings(model, dataloader):\n    \"\"\"Generates representations for all images in the dataloader with\n    the given model\n    \"\"\"\n\n    embeddings = []\n    filenames = []\n    with torch.no_grad():\n        for img, _, fnames in dataloader:\n            img = img.to(model.device)\n            emb = model.backbone(img).flatten(start_dim=1)\n            embeddings.append(emb)\n            filenames.extend(fnames)\n\n    embeddings = torch.cat(embeddings, 0)\n    embeddings = normalize(embeddings)\n    return embeddings, filenames\n\n\nmodel.eval()\nembeddings, filenames = generate_embeddings(model, dataloader_test)\n\n# %%\n# Visualize Nearest Neighbors\n# ----------------------------\n# Let's look at the trained embedding and visualize the nearest neighbors for\n# a few random samples.\n#\n# We create some helper functions to simplify the work\n\n\ndef get_image_as_np_array(filename: str):\n    \"\"\"Returns an image as an numpy array\"\"\"\n    img = Image.open(filename)\n    return np.asarray(img)\n\n\ndef plot_knn_examples(embeddings, filenames, n_neighbors=3, num_examples=6):\n    \"\"\"Plots multiple rows of random images with their nearest neighbors\"\"\"\n    # lets look at the nearest neighbors for some samples\n    # we use the sklearn library\n    nbrs = NearestNeighbors(n_neighbors=n_neighbors).fit(embeddings)\n    distances, indices = nbrs.kneighbors(embeddings)\n\n    # get 5 random samples\n    samples_idx = np.random.choice(len(indices), size=num_examples, replace=False)\n\n    # loop through our randomly picked samples\n    for idx in samples_idx:\n        fig = plt.figure()\n        # loop through their nearest neighbors\n        for plot_x_offset, neighbor_idx in enumerate(indices[idx]):\n            # add the subplot\n            ax = fig.add_subplot(1, len(indices[idx]), plot_x_offset + 1)\n            # get the correponding filename for the current index\n            fname = os.path.join(path_to_data, filenames[neighbor_idx])\n            # plot the image\n            plt.imshow(get_image_as_np_array(fname))\n            # set the title to the distance of the neighbor\n            ax.set_title(f\"d={distances[idx][plot_x_offset]:.3f}\")\n            # let's disable the axis\n            plt.axis(\"off\")\n\n\n# %%\n# Let's do the plot of the images. The leftmost image is the query image whereas\n# the ones next to it on the same row are the nearest neighbors.\n# In the title we see the distance of the neigbor.\nplot_knn_examples(embeddings, filenames)\n\n# %%\n# Color Invariance\n# ---------------------\n# Let's train again without color augmentation. This will force our model to\n# respect the colors in the images.\n\n# Set color jitter and gray scale probability to 0\nnew_transform = SimCLRTransform(\n    input_size=input_size, vf_prob=0.5, rr_prob=0.5, cj_prob=0.0, random_gray_scale=0.0\n)\n\n# let's update the transform on the training dataset\ndataset_train_simclr.transform = new_transform\n\n# then train a new model\nmodel = SimCLRModel()\ntrainer = pl.Trainer(max_epochs=max_epochs, devices=1, accelerator=\"gpu\")\ntrainer.fit(model, dataloader_train_simclr)\n\n# and generate again embeddings from the test set\nmodel.eval()\nembeddings, filenames = generate_embeddings(model, dataloader_test)\n\n# %%\n# other example\nplot_knn_examples(embeddings, filenames)\n\n# %%\n# What's next?\n\n# You could use the pre-trained model and train a classifier on top.\npretrained_resnet_backbone = model.backbone\n\n# you can also store the backbone and use it in another code\nstate_dict = {\"resnet18_parameters\": pretrained_resnet_backbone.state_dict()}\ntorch.save(state_dict, \"model.pth\")\n\n# %%\n# THIS COULD BE IN A NEW FILE (e.g. inference.py)\n#\n# Make sure you place the `model.pth` file in the same folder as this code\n\n# load the model in a new file for inference\nresnet18_new = torchvision.models.resnet18()\n\n# note that we need to create exactly the same backbone in order to load the weights\nbackbone_new = nn.Sequential(*list(resnet18_new.children())[:-1])\n\nckpt = torch.load(\"model.pth\")\nbackbone_new.load_state_dict(ckpt[\"resnet18_parameters\"])\n\n# %%\n# Next Steps\n# ------------\n#\n# Interested in exploring other self-supervised models? Check out our other\n# tutorials:\n#\n# - :ref:`lightly-moco-tutorial-2`\n# - :ref:`lightly-simsiam-tutorial-4`\n# - :ref:`lightly-custom-augmentation-5`\n# - :ref:`lightly-detectron-tutorial-6`\n#\n", "path": "docs/source/tutorials_source/package/tutorial_simclr_clothing.py"}]}