End of training

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230710_174519-01ve2s6v/run-01ve2s6v.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230710_174519-cqch7cwa/run-cqch7cwa.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230710_174519-qoix60bv/run-qoix60bv.wandb filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,21 @@
+
+---
+license: creativeml-openrail-m
+base_model: runwayml/stable-diffusion-v1-5
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- text-to-image
+- diffusers
+- lora
+inference: true
+---
+    
+# LoRA text2image fine-tuning - asrimanth/person-thumbs-up-lora-no-cap
+These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the Custom dataset dataset. You can find some example images in the following. 
+
+![img_0](./image_0.png)
+![img_1](./image_1.png)
+![img_2](./image_2.png)
+![img_3](./image_3.png)
+

checkpoint-1000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:746c62e5113737ef8b02c00326e3b5cdfdcb8f95180c91b330f3f3c94926def9
+size 6591685

checkpoint-1000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ac51b3392264e0485a7949271a2e16544b09a8a3473a6ef318301eb391aa62c
+size 3285965

checkpoint-1000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf626b00f0c0a21c1ee2f5631667c9175d425a61775d2fd75a66283361ef7cec
+size 16719

checkpoint-1000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68cff80b680ddf6e7abbef98b5f336b97f9b5963e2209307f639383870e8cc71
+size 557

checkpoint-1000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61808704787602dc93c00aa68d75b73415f1c6bfacfbeb97c14702c3e19cf9f2
+size 563

checkpoint-1500/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77a55cfe2d202d7658e497a13e57ebcdacaf27c1a51ee31ac50c39152218cbc0
+size 6591685

checkpoint-1500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c253af5591389039973a2ea17bd87d63f574889d6f050aca303f6ccd8a61b7f9
+size 3285965

checkpoint-1500/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:863bba4edd1ca97dc90e081725c55f3be037349b333f06678cdaad3d9501ed3e
+size 16719

checkpoint-1500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:203a72d6c29f42a0e2964fdddc8d7a98df1eccee78fea9de0fa416613390f5c6
+size 557

checkpoint-1500/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e89f6844bbfca1eeee319b6e2d6f3fd9d9df58879544d455482ad32bb3ad4df6
+size 563

checkpoint-2000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea56829d7f2a0745c8705572654104d7505898e7a2c20c053a47094f61c80742
+size 6591685

checkpoint-2000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec021ef8f4568ce10dddacb2b0c7f00374040c528a45e63243c4e86ca0ab60f1
+size 3285965

checkpoint-2000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fd21d1d8596fbae8788b9060a2386fe2636e634218aff7b075ff9bbd70231b3
+size 16719

checkpoint-2000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd2de9749828adacdf103bf6e9592702bb7067a2c1df27dd62ab38c1eb8c070f
+size 557

checkpoint-2000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00f6a413e75a46219e92e544cae3353a4577b60b728f62a720befd9033032acd
+size 563

checkpoint-2500/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c60850f36c09c2c47453cfdbc5a6b2f7a9628541c7ae7d666b4f84deb979b97f
+size 6591685

checkpoint-2500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:270f1731d5c47267878cc217744089f3c5db3353c18496879195fe47339aae49
+size 3285965

checkpoint-2500/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39f28d64d6912cc34eaaa0bb0cd0819897835a6aae2d96c1c19aa15468bb16c1
+size 16719

checkpoint-2500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fbcebc8f5487b0c117b5dd47f2ea304af3eebf408d297118d9307e1223927e1
+size 557

checkpoint-2500/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df00e6adcececef3aacd6addf17f6c85eae895b25ab91e673e9360b2a7f6ee86
+size 563

checkpoint-3000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6f85f4ee5aa71cbe6aec52bac688f8d18060f9fdc6ed4a24e18bcf229ac97dc
+size 6591685

checkpoint-3000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71d305a98f9f86636331f5c95d26bea6c72360da043b17d6f93ac1032e1e82c7
+size 3285965

checkpoint-3000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d12ebe8df23ebc27557c11a245e66d96279ac2a3ff7c8b15aac109084cee9e94
+size 16719

checkpoint-3000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1f9398b77268202e8e1465734a63d123b1ef11c27f20f2473677e9883a6869
+size 557

checkpoint-3000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:807d7d063623792ef9b8c66da2304691aed0888f14ad6e70a728375b9c90fe9d
+size 563

checkpoint-500/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3779ad72103a5ce89a74b972afd45f12daf23fa910cc7e57fbd14d70d3e6667
+size 6591685

checkpoint-500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:263b95b3ee399314308e8ddb3d24a1292e981f2798a39f8d235998de2aaaa27c
+size 3285965

checkpoint-500/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e670e6c021cd2c9f45d7068757a1a37ed11280498d332d84fc9b52b93dce0e7d
+size 16719

checkpoint-500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3f196a54202bb4ba1220e8c59f42f9cda0702d68ea83147d814c2fb2f36b8f2
+size 557

checkpoint-500/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bc0a22eef87625080f7c833304b0844a5e9a28131b856c035cff1e8f8d56121
+size 563

pytorch_lora_weights.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea51f54bb3ef6955de5e69c155b00b4ead4a80836046b7fb20b4c33f6a11b823
+size 3287771

wandb/debug.log

@@ -0,0 +1,27 @@
+2023-07-10 17:45:19,956 INFO    MainThread:538851 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-07-10 17:45:19,956 INFO    MainThread:538851 [wandb_setup.py:_flush():76] Configure stats pid to 538851
+2023-07-10 17:45:19,957 INFO    MainThread:538851 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings
+2023-07-10 17:45:19,957 INFO    MainThread:538851 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings
+2023-07-10 17:45:19,957 INFO    MainThread:538851 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2023-07-10 17:45:19,957 INFO    MainThread:538851 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-07-10 17:45:19,957 INFO    MainThread:538851 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'}
+2023-07-10 17:45:19,957 INFO    MainThread:538851 [wandb_init.py:_log_setup():507] Logging user logs to /l/vision/v5/sragas/easel_ai/models_no_cap/wandb/run-20230710_174519-qoix60bv/logs/debug.log
+2023-07-10 17:45:19,957 INFO    MainThread:538851 [wandb_init.py:_log_setup():508] Logging internal logs to /l/vision/v5/sragas/easel_ai/models_no_cap/wandb/run-20230710_174519-qoix60bv/logs/debug-internal.log
+2023-07-10 17:45:19,958 INFO    MainThread:538851 [wandb_init.py:init():547] calling init triggers
+2023-07-10 17:45:19,958 INFO    MainThread:538851 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2023-07-10 17:45:19,958 INFO    MainThread:538851 [wandb_init.py:init():596] starting backend
+2023-07-10 17:45:19,958 INFO    MainThread:538851 [wandb_init.py:init():600] setting up manager
+2023-07-10 17:45:19,961 INFO    MainThread:538851 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-07-10 17:45:19,965 INFO    MainThread:538851 [wandb_init.py:init():606] backend started and connected
+2023-07-10 17:45:19,968 INFO    MainThread:538851 [wandb_init.py:init():703] updated telemetry
+2023-07-10 17:45:19,969 INFO    MainThread:538851 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
+2023-07-10 17:45:20,272 INFO    MainThread:538851 [wandb_run.py:_on_init():2176] communicating current version
+2023-07-10 17:45:20,355 INFO    MainThread:538851 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2023-07-10 17:45:20,355 INFO    MainThread:538851 [wandb_init.py:init():787] starting run threads in backend
+2023-07-10 17:45:20,484 INFO    MainThread:538851 [wandb_run.py:_console_start():2155] atexit reg
+2023-07-10 17:45:20,484 INFO    MainThread:538851 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
+2023-07-10 17:45:20,485 INFO    MainThread:538851 [wandb_run.py:_redirect():2075] Wrapping output streams.
+2023-07-10 17:45:20,485 INFO    MainThread:538851 [wandb_run.py:_redirect():2100] Redirects installed.
+2023-07-10 17:45:20,486 INFO    MainThread:538851 [wandb_init.py:init():828] run started, returning control to user process

wandb/run-20230710_174519-01ve2s6v/files/config.yaml

@@ -0,0 +1,39 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.8.10
+    cli_version: 0.15.4
+    framework: huggingface
+    huggingface_version: 4.30.2
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    start_time: 1689025519.940496
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      3:
+      - 23
+      4: 3.8.10
+      5: 0.15.4
+      6: 4.30.2
+      8:
+      - 2
+      - 5

wandb/run-20230710_174519-01ve2s6v/files/output.log

@@ -0,0 +1,7 @@
+07/10/2023 17:45:20 - INFO - __main__ - Distributed environment: MULTI_GPU  Backend: nccl
+Num processes: 3
+Process index: 2
+Local process index: 2
+Device: cuda:2
+Mixed precision type: fp16
+Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 58947.81it/s]

wandb/run-20230710_174519-01ve2s6v/files/requirements.txt

@@ -0,0 +1,165 @@
+absl-py==1.4.0
+accelerate==0.20.3
+aiohttp==3.8.4
+aiosignal==1.3.1
+anyio==3.7.0
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==21.3.0
+asttokens==2.2.1
+async-lru==2.0.2
+async-timeout==4.0.2
+attrs==23.1.0
+babel==2.12.1
+backcall==0.2.0
+beautifulsoup4==4.12.2
+bleach==6.0.0
+cachetools==5.3.1
+certifi==2023.5.7
+cffi==1.15.1
+charset-normalizer==3.1.0
+click==8.1.3
+cmake==3.26.4
+comm==0.1.3
+datasets==2.13.1
+debugpy==1.6.7
+decorator==5.1.1
+defusedxml==0.7.1
+diffusers==0.18.0.dev0
+dill==0.3.6
+docker-pycreds==0.4.0
+exceptiongroup==1.1.2
+executing==1.2.0
+fastjsonschema==2.17.1
+filelock==3.12.2
+frozenlist==1.3.3
+fsspec==2023.6.0
+ftfy==6.1.1
+gitdb==4.0.10
+gitpython==3.1.31
+google-auth-oauthlib==1.0.0
+google-auth==2.21.0
+grpcio==1.56.0
+huggingface-hub==0.15.1
+idna==3.4
+importlib-metadata==6.7.0
+importlib-resources==5.12.0
+ipykernel==6.24.0
+ipython==8.12.2
+jedi==0.18.2
+jinja2==3.1.2
+json5==0.9.14
+jsonschema==4.17.3
+jupyter-client==8.3.0
+jupyter-core==5.3.1
+jupyter-events==0.6.3
+jupyter-lsp==2.2.0
+jupyter-server-terminals==0.4.4
+jupyter-server==2.7.0
+jupyterlab-pygments==0.2.2
+jupyterlab-server==2.23.0
+jupyterlab==4.0.2
+lit==16.0.6
+markdown==3.4.3
+markupsafe==2.1.3
+matplotlib-inline==0.1.6
+mistune==3.0.1
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.6.0
+nbformat==5.9.0
+nest-asyncio==1.5.6
+networkx==3.1
+notebook-shim==0.2.3
+numpy==1.24.4
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+oauthlib==3.2.2
+overrides==7.3.1
+packaging==23.1
+pandas==2.0.3
+pandocfilters==1.5.0
+parso==0.8.3
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==10.0.0
+pip==20.0.2
+pkg-resources==0.0.0
+pkgutil-resolve-name==1.3.10
+platformdirs==3.8.0
+prometheus-client==0.17.0
+prompt-toolkit==3.0.39
+protobuf==4.23.3
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==12.0.1
+pyasn1-modules==0.3.0
+pyasn1==0.5.0
+pycparser==2.21
+pygments==2.15.1
+pyre-extensions==0.0.29
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+pytz==2023.3
+pyyaml==6.0
+pyzmq==25.1.0
+regex==2023.6.3
+requests-oauthlib==1.3.1
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rsa==4.9
+safetensors==0.3.1
+send2trash==1.8.2
+sentry-sdk==1.27.0
+setproctitle==1.3.2
+setuptools==44.0.0
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+soupsieve==2.4.1
+stack-data==0.6.2
+sympy==1.12
+tensorboard-data-server==0.7.1
+tensorboard==2.13.0
+terminado==0.17.1
+tinycss2==1.2.1
+tokenizers==0.13.3
+tomli==2.0.1
+torch==2.0.1
+torchaudio==2.0.2
+torchvision==0.15.2
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.30.2
+triton==2.0.0
+typing-extensions==4.7.1
+typing-inspect==0.9.0
+tzdata==2023.3
+urllib3==2.0.3
+wandb==0.15.4
+wcwidth==0.2.6
+webencodings==0.5.1
+websocket-client==1.6.1
+werkzeug==2.3.6
+wheel==0.40.0
+xformers==0.0.20
+xxhash==3.2.0
+yarl==1.9.2
+zipp==3.15.0

wandb/run-20230710_174519-01ve2s6v/files/wandb-metadata.json

@@ -0,0 +1,127 @@
+{
+    "os": "Linux-5.15.0-73-generic-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2023-07-10T21:45:20.460414",
+    "startedAt": "2023-07-10T21:45:19.811914",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5",
+        "--train_data_dir=/l/vision/v5/sragas/easel_ai/thumbs_up_no_cap_dataset/",
+        "--resolution=512",
+        "--center_crop",
+        "--random_flip",
+        "--train_batch_size=2",
+        "--gradient_accumulation_steps=4",
+        "--num_train_epochs=300",
+        "--learning_rate=1e-6",
+        "--max_grad_norm=1",
+        "--lr_scheduler=cosine",
+        "--lr_warmup_steps=500",
+        "--output_dir=/l/vision/v5/sragas/easel_ai/models_no_cap/",
+        "--checkpointing_steps=500",
+        "--report_to=wandb",
+        "--validation_prompt=<tom_cruise> #thumbsup",
+        "--seed=15",
+        "--push_to_hub",
+        "--hub_model_id=person-thumbs-up-lora-no-cap"
+    ],
+    "state": "running",
+    "program": "train_text_to_image_lora.py",
+    "codePath": "train_text_to_image_lora.py",
+    "host": "magnemite",
+    "username": "sragas",
+    "executable": "/nfs/blitzle/home/data/vision5/sragas/easel_venv/bin/python3",
+    "cpu_count": 6,
+    "cpu_count_logical": 12,
+    "cpu_freq": {
+        "current": 1880.4280833333335,
+        "min": 1200.0,
+        "max": 3900.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2354.544,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1200.0,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1300.0,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1805.495,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1200.0,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2470.242,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1200.0,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1200.0,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2168.124,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2166.389,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1200.0,
+            "min": 1200.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1300.0,
+            "min": 1200.0,
+            "max": 3900.0
+        }
+    ],
+    "disk": {
+        "total": 116.31898498535156,
+        "used": 59.75596237182617
+    },
+    "gpu": "NVIDIA GeForce GTX TITAN X",
+    "gpu_count": 3,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce GTX TITAN X",
+            "memory_total": 12884901888
+        },
+        {
+            "name": "NVIDIA GeForce GTX TITAN X",
+            "memory_total": 12884901888
+        },
+        {
+            "name": "NVIDIA GeForce GTX TITAN X",
+            "memory_total": 12884901888
+        }
+    ],
+    "memory": {
+        "total": 62.69387435913086
+    }
+}

wandb/run-20230710_174519-01ve2s6v/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20230710_174519-01ve2s6v/logs/debug.log

@@ -0,0 +1,27 @@
+2023-07-10 17:45:19,929 INFO    MainThread:538852 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-07-10 17:45:19,929 INFO    MainThread:538852 [wandb_setup.py:_flush():76] Configure stats pid to 538852
+2023-07-10 17:45:19,929 INFO    MainThread:538852 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings
+2023-07-10 17:45:19,929 INFO    MainThread:538852 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings
+2023-07-10 17:45:19,929 INFO    MainThread:538852 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2023-07-10 17:45:19,929 INFO    MainThread:538852 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-07-10 17:45:19,930 INFO    MainThread:538852 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'}
+2023-07-10 17:45:19,930 INFO    MainThread:538852 [wandb_init.py:_log_setup():507] Logging user logs to /l/vision/v5/sragas/easel_ai/models_no_cap/wandb/run-20230710_174519-01ve2s6v/logs/debug.log
+2023-07-10 17:45:19,930 INFO    MainThread:538852 [wandb_init.py:_log_setup():508] Logging internal logs to /l/vision/v5/sragas/easel_ai/models_no_cap/wandb/run-20230710_174519-01ve2s6v/logs/debug-internal.log
+2023-07-10 17:45:19,930 INFO    MainThread:538852 [wandb_init.py:init():547] calling init triggers
+2023-07-10 17:45:19,930 INFO    MainThread:538852 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2023-07-10 17:45:19,930 INFO    MainThread:538852 [wandb_init.py:init():596] starting backend
+2023-07-10 17:45:19,930 INFO    MainThread:538852 [wandb_init.py:init():600] setting up manager
+2023-07-10 17:45:19,934 INFO    MainThread:538852 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-07-10 17:45:19,939 INFO    MainThread:538852 [wandb_init.py:init():606] backend started and connected
+2023-07-10 17:45:19,943 INFO    MainThread:538852 [wandb_init.py:init():703] updated telemetry
+2023-07-10 17:45:19,944 INFO    MainThread:538852 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
+2023-07-10 17:45:20,209 INFO    MainThread:538852 [wandb_run.py:_on_init():2176] communicating current version
+2023-07-10 17:45:20,400 INFO    MainThread:538852 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2023-07-10 17:45:20,400 INFO    MainThread:538852 [wandb_init.py:init():787] starting run threads in backend
+2023-07-10 17:45:20,593 INFO    MainThread:538852 [wandb_run.py:_console_start():2155] atexit reg
+2023-07-10 17:45:20,594 INFO    MainThread:538852 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
+2023-07-10 17:45:20,594 INFO    MainThread:538852 [wandb_run.py:_redirect():2075] Wrapping output streams.
+2023-07-10 17:45:20,594 INFO    MainThread:538852 [wandb_run.py:_redirect():2100] Redirects installed.
+2023-07-10 17:45:20,595 INFO    MainThread:538852 [wandb_init.py:init():828] run started, returning control to user process

wandb/run-20230710_174519-01ve2s6v/run-01ve2s6v.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c2f32aa537003af2bda504bc679cc9717175ecc6ebfd535e53b34b878a1f6fc
+size 2459037

wandb/run-20230710_174519-cqch7cwa/files/config.yaml

@@ -0,0 +1,187 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.8.10
+    cli_version: 0.15.4
+    framework: huggingface
+    huggingface_version: 4.30.2
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    start_time: 1689025519.94471
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      3:
+      - 23
+      - 24
+      4: 3.8.10
+      5: 0.15.4
+      6: 4.30.2
+      8:
+      - 2
+      - 5
+pretrained_model_name_or_path:
+  desc: null
+  value: runwayml/stable-diffusion-v1-5
+revision:
+  desc: null
+  value: null
+dataset_name:
+  desc: null
+  value: null
+dataset_config_name:
+  desc: null
+  value: null
+train_data_dir:
+  desc: null
+  value: /l/vision/v5/sragas/easel_ai/thumbs_up_no_cap_dataset/
+image_column:
+  desc: null
+  value: image
+caption_column:
+  desc: null
+  value: text
+validation_prompt:
+  desc: null
+  value: '<tom_cruise> #thumbsup'
+num_validation_images:
+  desc: null
+  value: 4
+validation_epochs:
+  desc: null
+  value: 1
+max_train_samples:
+  desc: null
+  value: null
+output_dir:
+  desc: null
+  value: /l/vision/v5/sragas/easel_ai/models_no_cap/
+cache_dir:
+  desc: null
+  value: /l/vision/v5/sragas/hf_models/
+seed:
+  desc: null
+  value: 15
+resolution:
+  desc: null
+  value: 512
+center_crop:
+  desc: null
+  value: true
+random_flip:
+  desc: null
+  value: true
+train_batch_size:
+  desc: null
+  value: 2
+num_train_epochs:
+  desc: null
+  value: 300
+max_train_steps:
+  desc: null
+  value: 3000
+gradient_accumulation_steps:
+  desc: null
+  value: 4
+gradient_checkpointing:
+  desc: null
+  value: false
+learning_rate:
+  desc: null
+  value: 1.0e-06
+scale_lr:
+  desc: null
+  value: false
+lr_scheduler:
+  desc: null
+  value: cosine
+lr_warmup_steps:
+  desc: null
+  value: 500
+snr_gamma:
+  desc: null
+  value: null
+use_8bit_adam:
+  desc: null
+  value: false
+allow_tf32:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_weight_decay:
+  desc: null
+  value: 0.01
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+push_to_hub:
+  desc: null
+  value: true
+hub_token:
+  desc: null
+  value: null
+prediction_type:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: person-thumbs-up-lora-no-cap
+logging_dir:
+  desc: null
+  value: logs
+mixed_precision:
+  desc: null
+  value: null
+report_to:
+  desc: null
+  value: wandb
+local_rank:
+  desc: null
+  value: 0
+checkpointing_steps:
+  desc: null
+  value: 500
+checkpoints_total_limit:
+  desc: null
+  value: null
+resume_from_checkpoint:
+  desc: null
+  value: null
+enable_xformers_memory_efficient_attention:
+  desc: null
+  value: false
+noise_offset:
+  desc: null
+  value: 0
+rank:
+  desc: null
+  value: 4