huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/bill-wurtz")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/1y01b0sy/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/27ysbe74/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on Bill Wurtz's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/1l9hr5w2) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/2f8oa51l) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/1l9hr5w2/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/2f8oa51l/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "huggingartists/bill-wurtz",
+  "_name_or_path": "bill-wurtz",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -18,7 +18,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -35,7 +37,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.9.2",
+  "transformers_version": "4.16.2",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 4.039820671081543, "eval_runtime": 0.5683, "eval_samples_per_second": 36.954, "eval_steps_per_second": 5.279, "epoch": 22.0}
+{"eval_loss": 2.747051954269409, "eval_runtime": 1.0203, "eval_samples_per_second": 19.602, "eval_steps_per_second": 2.94, "epoch": 6.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5eea9a74d99aea650690be8e5916b60d87d02d8e31dc45b59fa4bd1904a4b58
+oid sha256:060e7080e916a0cfca49c43ca58ab8d2e2597dde7fe50e0b1d8ddb0095bf8b92
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa1422428e38213576ff235f2001350e546a4c4643f2b2f472d00667df459b33
+oid sha256:982717a4b3eb11fd324c00becfc94519651828fb480697cf1ef1e21682a7a1bc
 size 995603825

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63382be8feee6fe2b303cf897efe67e158593589b9171b749dab61667b6c81f2
+oid sha256:0d1d3bf79540744461b44cf0dadab89cabcbce43cfed8f94046bf29deb00e002
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1d50a042836fb7244e3a57318b3664fd2cb3c84d1ecf895aeecf11cd286b9d1
+oid sha256:12e38e4c0903310b46906cc923622a2da9fcbd366af1ad3842402f38f5c0b222
 size 14567

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1aee7c44e1d1ea5ba4c4593abd0231fd0c216d063ddd82618d0f2d2c0e4a5dc
+oid sha256:ed9457e8f7ccd2a262fcd727a62c61efaa77aaf95187fa3bb6da72e39564d99e
 size 623

trainer_state.json

@@ -1,232 +1,54 @@
 {
-  "best_metric": 2.6812992095947266,
-  "best_model_checkpoint": "output/bill-wurtz/checkpoint-140",
-  "epoch": 5.0,
-  "global_step": 140,
+  "best_metric": 2.747051954269409,
+  "best_model_checkpoint": "output/bill-wurtz/checkpoint-28",
+  "epoch": 1.0,
+  "global_step": 28,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.19,
-      "learning_rate": 0.00012591446386292745,
-      "loss": 3.5789,
+      "epoch": 0.18,
+      "learning_rate": 0.00012668528006706036,
+      "loss": 2.3906,
       "step": 5
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 9.577107195028616e-05,
-      "loss": 3.3728,
+      "epoch": 0.36,
+      "learning_rate": 9.836442450346464e-05,
+      "loss": 2.3509,
       "step": 10
     },
     {
-      "epoch": 0.56,
-      "learning_rate": 5.668773501204858e-05,
-      "loss": 3.2429,
+      "epoch": 0.54,
+      "learning_rate": 6.091923693931308e-05,
+      "loss": 2.3424,
       "step": 15
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 2.152382364220488e-05,
-      "loss": 3.2491,
+      "epoch": 0.71,
+      "learning_rate": 2.582859959249092e-05,
+      "loss": 2.4842,
       "step": 20
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 1.849121878224087e-06,
-      "loss": 3.1415,
+      "epoch": 0.89,
+      "learning_rate": 3.849603540845946e-06,
+      "loss": 2.52,
       "step": 25
     },
     {
       "epoch": 1.0,
-      "eval_loss": 3.1433706283569336,
-      "eval_runtime": 0.6351,
-      "eval_samples_per_second": 42.515,
-      "eval_steps_per_second": 6.299,
-      "step": 27
-    },
-    {
-      "epoch": 1.11,
-      "learning_rate": 4.137086214086682e-06,
-      "loss": 2.9586,
-      "step": 30
-    },
-    {
-      "epoch": 1.3,
-      "learning_rate": 2.7634920609188867e-05,
-      "loss": 3.0624,
-      "step": 35
-    },
-    {
-      "epoch": 1.48,
-      "learning_rate": 6.461126473674133e-05,
-      "loss": 2.9663,
-      "step": 40
-    },
-    {
-      "epoch": 1.67,
-      "learning_rate": 0.00010290000000000001,
-      "loss": 2.8978,
-      "step": 45
-    },
-    {
-      "epoch": 1.85,
-      "learning_rate": 0.0001299031991261861,
-      "loss": 2.9441,
-      "step": 50
-    },
-    {
-      "epoch": 2.0,
-      "eval_loss": 3.056016445159912,
-      "eval_runtime": 0.6316,
-      "eval_samples_per_second": 42.746,
-      "eval_steps_per_second": 6.333,
-      "step": 54
-    },
-    {
-      "epoch": 1.96,
-      "learning_rate": 0.00013676865759867644,
-      "loss": 2.5878,
-      "step": 55
-    },
-    {
-      "epoch": 2.0,
-      "eval_loss": 2.762599468231201,
-      "eval_runtime": 0.5151,
-      "eval_samples_per_second": 40.769,
-      "eval_steps_per_second": 5.824,
-      "step": 56
-    },
-    {
-      "epoch": 2.14,
-      "learning_rate": 0.00013040646433810595,
-      "loss": 2.8115,
-      "step": 60
-    },
-    {
-      "epoch": 2.32,
-      "learning_rate": 0.00010509740044895205,
-      "loss": 2.5949,
-      "step": 65
-    },
-    {
-      "epoch": 2.5,
-      "learning_rate": 6.860000000000001e-05,
-      "loss": 2.6643,
-      "step": 70
-    },
-    {
-      "epoch": 2.68,
-      "learning_rate": 3.210259955104798e-05,
-      "loss": 2.5837,
-      "step": 75
-    },
-    {
-      "epoch": 2.86,
-      "learning_rate": 6.793535661894062e-06,
-      "loss": 2.7461,
-      "step": 80
-    },
-    {
-      "epoch": 3.0,
-      "eval_loss": 2.6894850730895996,
-      "eval_runtime": 0.5306,
-      "eval_samples_per_second": 39.579,
-      "eval_steps_per_second": 5.654,
-      "step": 84
-    },
-    {
-      "epoch": 3.04,
-      "learning_rate": 4.313424013235498e-07,
-      "loss": 2.4693,
-      "step": 85
-    },
-    {
-      "epoch": 3.21,
-      "learning_rate": 1.496636030269314e-05,
-      "loss": 2.4734,
-      "step": 90
-    },
-    {
-      "epoch": 3.39,
-      "learning_rate": 4.594285634987545e-05,
-      "loss": 2.5555,
-      "step": 95
-    },
-    {
-      "epoch": 3.57,
-      "learning_rate": 8.386493606940314e-05,
-      "loss": 2.498,
-      "step": 100
-    },
-    {
-      "epoch": 3.75,
-      "learning_rate": 0.0001171075251893971,
-      "loss": 2.3626,
-      "step": 105
-    },
-    {
-      "epoch": 3.93,
-      "learning_rate": 0.0001354800547756731,
-      "loss": 2.4603,
-      "step": 110
-    },
-    {
-      "epoch": 4.0,
-      "eval_loss": 2.6920154094696045,
-      "eval_runtime": 0.5456,
-      "eval_samples_per_second": 38.49,
-      "eval_steps_per_second": 5.499,
-      "step": 112
-    },
-    {
-      "epoch": 4.11,
-      "learning_rate": 0.00013335039645915404,
-      "loss": 2.3237,
-      "step": 115
-    },
-    {
-      "epoch": 4.29,
-      "learning_rate": 0.00011137140040750914,
-      "loss": 2.2052,
-      "step": 120
-    },
-    {
-      "epoch": 4.46,
-      "learning_rate": 7.62807630606869e-05,
-      "loss": 2.2121,
-      "step": 125
-    },
-    {
-      "epoch": 4.64,
-      "learning_rate": 3.883557549653544e-05,
-      "loss": 2.2551,
-      "step": 130
-    },
-    {
-      "epoch": 4.82,
-      "learning_rate": 1.0514719932939762e-05,
-      "loss": 2.2849,
-      "step": 135
-    },
-    {
-      "epoch": 5.0,
-      "learning_rate": 0.0,
-      "loss": 2.2654,
-      "step": 140
-    },
-    {
-      "epoch": 5.0,
-      "eval_loss": 2.6812992095947266,
-      "eval_runtime": 0.5264,
-      "eval_samples_per_second": 39.891,
-      "eval_steps_per_second": 5.699,
-      "step": 140
+      "eval_loss": 2.747051954269409,
+      "eval_runtime": 0.9153,
+      "eval_samples_per_second": 21.851,
+      "eval_steps_per_second": 3.278,
+      "step": 28
     }
   ],
-  "max_steps": 616,
-  "num_train_epochs": 22,
-  "total_flos": 145278369792000.0,
+  "max_steps": 168,
+  "num_train_epochs": 6,
+  "total_flos": 29264707584000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41eee99e83d6fdcd4ae4efff4b1f5e17120c0670785282724daf91004a53de18
-size 2671
+oid sha256:1417a02d3efa067f099a494a6c3d4dbe9db44852a2687418d8cd162ecbd5d91a
+size 3055