pod changes

norwegian_base.gin

@@ -10,7 +10,7 @@ import tasks
 
 MIXTURE_OR_TASK_NAME = "ncc_span_corruption_stream"
 TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
-TRAIN_STEPS = 1_100_000
+TRAIN_STEPS = 1_500_000
 DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
 INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
 PjitPartitioner.num_partitions = 4

tasks.py

@@ -128,3 +128,30 @@ TaskRegistry.add(
     output_features={"targets": DEFAULT_OUTPUT_FEATURES["targets"]},
     metric_fns=[]
 )
+
+# Final pretraining task used in Raffel et al., 2019 adaptated to NCC
+dataset_name = 'NbAiLab/scandinavian'
+dataset_params = {"path": dataset_name, "use_auth_token": True, "streaming": True}
+dataset_shapes = None
+TaskRegistry.add(
+    "scandinavian_span_corruption_stream",
+    source=seqio.FunctionDataSource(
+        dataset_fn=functools.partial(dataset_fn, dataset_params=dataset_params),
+        splits=("train", "validation"),
+        caching_permitted=False,
+        num_input_examples=dataset_shapes,
+        ),
+    preprocessors=[
+        functools.partial(
+            target_to_key, key_map={
+                "inputs": None,
+                "targets": None,
+            }, target_key="targets"),
+        seqio.preprocessors.tokenize,
+        # seqio.CacheDatasetPlaceholder(),
+        preprocessors.span_corruption,
+        seqio.preprocessors.append_eos_after_trim,
+    ],
+    output_features={"targets": DEFAULT_OUTPUT_FEATURES["targets"]},
+    metric_fns=[]
+)

train_base.sh

@@ -1,6 +1,6 @@
 PROJECT_DIR=${HOME}"/models/pk-nb-t5x"
 T5X_DIR="../../t5x"  # directory where the t5x is cloned.
-MODEL_DIR="gs://nb-t5x-us-central2/pk_nb_t5x_base_run2"
+MODEL_DIR="gs://nb-t5x-us-central2/norwegian_t5x_base"
 export PYTHONPATH=${PROJECT_DIR}
 
 python3 ${T5X_DIR}/t5x/train.py \