# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="rQph610BeO4I"
# # ALBERT pretraining on Japanese wikipedia corpus (v2)
#
# This notebook is assumed to be executed on Colaboratory notebook with TPU.
#
#

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 20835, "status": "ok", "timestamp": 1587707118430, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="Uk2yvKc9ndth" outputId="410cf962-ca58-491c-a04e-1a3ba19fa623"
# %tensorflow_version 1.x

# + colab={} colab_type="code" id="3Cv0EJiZ8vi6"
import tensorflow as tf

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 4303, "status": "ok", "timestamp": 1587707128616, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="R3AyT0Fq8yzD" outputId="99d7e009-a4f2-44fb-cff9-35614788458c"
tf.__version__

# + colab={"base_uri": "https://localhost:8080/", "height": 416} colab_type="code" executionInfo={"elapsed": 5644, "status": "ok", "timestamp": 1587707131907, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="EWkrwgQV80JK" outputId="8999fc10-3396-47e0-ac98-f1c32c22df48"
# !git clone --recursive https://github.com/alinear-corp/albert-japanese.git

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 475, "status": "ok", "timestamp": 1587707133242, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="OcKCR8GpITp5" outputId="f2ce5c8c-71aa-41f0-a4f6-77b77a5389e0"
# cd albert-japanese/ALBERT

# + colab={"base_uri": "https://localhost:8080/", "height": 191} colab_type="code" executionInfo={"elapsed": 2007, "status": "ok", "timestamp": 1587707135104, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="zD-xYUc4Bp0I" outputId="f6028a09-1e70-4063-e5b9-5417fe310be6"
# !git log --oneline | head

# + [markdown] colab_type="text" id="rYnDgi2xgDS7"
# Authentication to use TPU.

# + colab={"base_uri": "https://localhost:8080/", "height": 156} colab_type="code" executionInfo={"elapsed": 15928, "status": "ok", "timestamp": 1587707152389, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="lwRm42uDt8SN" outputId="a58711f2-dcf5-4673-e79f-d9da3e7f76bd"
from google.colab import auth
auth.authenticate_user()

# + [markdown] colab_type="text" id="yLWTT6NZ_ZYi"
# ## Check TPU devices

# + colab={"base_uri": "https://localhost:8080/", "height": 243} colab_type="code" executionInfo={"elapsed": 1539, "status": "ok", "timestamp": 1587707155210, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="pShRjorw_Lla" outputId="6d28e5ac-7916-4747-cee4-bb0e99e12204"
import datetime
import json
import os
import pprint
import random
import string
import sys
import tensorflow as tf

assert 'COLAB_TPU_ADDR' in os.environ, 'ERROR: Not connected to a TPU runtime; please see the first cell in this notebook for instructions!'
TPU_ADDRESS = 'grpc://' + os.environ['COLAB_TPU_ADDR']
print('TPU address is', TPU_ADDRESS)

with tf.Session(TPU_ADDRESS) as session:
  print('TPU devices:')
  pprint.pprint(session.list_devices())

  # Upload credentials to TPU.
  with open('/content/adc.json', 'r') as f:
    auth_info = json.load(f)
  tf.contrib.cloud.configure_gcs(session, credentials=auth_info)
  # Now credentials are set for all future sessions on this TPU.

# + [markdown] colab_type="text" id="iQM0ChEhg08g"
# ## Set input and output
#
# Need to put `all-maxseq(128|512).tfrecord` data for pre-traning on your GCS bucket.  
# Trained objects will be saved into a specified GCS bucket.

# + colab={} colab_type="code" id="54yVw-J4iVq6"
INPUT_DATA_GCS = 'gs://albert_japanese_v2/wiki_tfrecord'

# + colab={} colab_type="code" id="V3-OQkauSdXe"
TARGET_DIRS = [
  'AA',
  'AB',
  'AC',
  'AD',
  'AE',
  'AF',
  'AG',
  'AH',
  'AI',
  'AJ',
  'AK',
  'AL',
  'AM',
  'AN',
  'AO',
  'AP',
  'AQ',
  'AR',
  'AS',
  'AT',
  'AU',
  'AV',
  'AW',
  'AX',
  'AY',
  'AZ',
  'BA',
  'BB',
  'BC',
]

# + colab={} colab_type="code" id="BG46L7cKB9uT"
# MAX_SEQ_LEN = 128
MAX_SEQ_LEN = 512

# + colab={} colab_type="code" id="sa9C1sAdS0tZ"
INPUT_FILE = ','.join( [ '{}/{}/all-maxseq{}.tfrecord'.format(INPUT_DATA_GCS, elem, MAX_SEQ_LEN) for elem in TARGET_DIRS] )

# + colab={} colab_type="code" id="xK-4oH2ViNzW"
OUTPUT_GCS = 'gs://albert_japanese_v2/model'

# + [markdown] colab_type="text" id="JYYqrixMKrd0"
# ## Execute pre-training
#
#

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 506, "status": "ok", "timestamp": 1587707165821, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="H4crLHoEow92" outputId="2e5240c0-3f5a-4f40-fc41-88ce6655db30"
# cd /content/

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" executionInfo={"elapsed": 2979860, "status": "ok", "timestamp": 1587452551792, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="ejRY72r3BL4W" outputId="2daff9ce-1fb2-4537-8a0e-5c14ab177389"
# # !PYTHONPATH=/tensorflow-1.15.2/python3.6:/content/albert-japanese/ python albert-japanese/ALBERT/run_pretraining.py \
# #   --albert_config_file=albert-japanese/albert_config.json \
# #   --input_file={INPUT_FILE} \
# #   --output_dir={OUTPUT_GCS} \
# #   --use_tpu=True \
# #   --tpu_name={TPU_ADDRESS} \
# #   --num_tpu_cores=8 \
# #   --do_train=True \
# #   --do_eval=True \
# #   --train_batch_size=256 \
# #   --max_seq_length={MAX_SEQ_LEN} \
# #   --max_predictions_per_seq=20 \
# #   --num_train_steps=1000000 \
# #   --num_warmup_steps=10000 \
# #   --save_checkpoints_steps=10000 \
# #   --learning_rate=1e-4

# + colab={} colab_type="code" id="4TRrgdUiJxru"


# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" executionInfo={"elapsed": 12563850, "status": "ok", "timestamp": 1587719745476, "user": {"displayName": "\u4e57\u677e\u6f64\u77e2", "photoUrl": "", "userId": "14372592260560654082"}, "user_tz": -540} id="d3OEEo3XR93B" outputId="e59b8e52-329b-4710-dacd-7838a36fb366"
# !PYTHONPATH=/tensorflow-1.15.2/python3.6:/content/albert-japanese/ python albert-japanese/ALBERT/run_pretraining.py \
#   --albert_config_file=albert-japanese/albert_config.json \
#   --input_file={INPUT_FILE} \
#   --output_dir={OUTPUT_GCS} \
#   --use_tpu=True \
#   --tpu_name={TPU_ADDRESS} \
#   --num_tpu_cores=8 \
#   --do_train=True \
#   --do_eval=True \
#   --train_batch_size=64 \
#   --max_seq_length={MAX_SEQ_LEN} \
#   --max_predictions_per_seq=20 \
#   --num_train_steps=1400000 \
#   --num_warmup_steps=10000 \
#   --save_checkpoints_steps=10000 \
#   --learning_rate=1e-4

# + colab={} colab_type="code" id="GBNgHk-s4epb"


StarcoderdataJupyterScriptsDedupFiltered