# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.8.8 64-bit (''venv_tf2.4'': conda)'
#     language: python
#     name: python388jvsc74a57bd0e43aa85e0d007ca602e04b3033c86d6af4f225b69c2da0d32fa6602213775d26
# ---

import sys
sys.path.append("/home/sidhu/Projects/tf-transformers/src/")

# +
import tempfile
from transformers import AlbertTokenizer

from tf_transformers.text.sentencepiece_layer import extend_sentencepicemodel
from tf_transformers.text import SentencepieceTokenizer
# -





# +
# We will use T5 tokenizer , but we extend it with 2 more tokens

# 1. [CLS]
# 2. [MASK]
# We will use </s> as SEP token

temp_dir = tempfile.mkdtemp()
model_name = 'albert-base-v2'
tokenizer_hf = AlbertTokenizer.from_pretrained(model_name)
save_path = "{}/{}".format(temp_dir, model_name)
tokenizer_hf.save_pretrained(save_path)

in_file  = '{}/spiece.model'.format(save_path)
# out_file = '{}/new_spiece.model'.format(save_path)
# special_tokens = ['[CLS]', '[MASK]']

# extend_sentencepicemodel(in_file, out_file, special_tokens)
# -



# To use new sentencepiece model in T5 use like this
t5_kwargs = {'bos_token': '[CLS]',
 'eos_token': '</s>', 
 'unk_token': '<unk>', 
 'pad_token': '<pad>', 
 'mask_token': '[MASK]', 
 'name_or_path': '{}'.format(save_path), 
 'vocab_file': '{}/new_spiece.model'.format(save_path)}
tokenizer_hf = T5Tokenizer(**t5_kwargs)
tokenizer_hf.unique_no_split_tokens = tokenizer_hf.all_special_tokens

# +

# Use this one for fast processinf
tokenizer_tf = SentencepieceTokenizer(
        model_file_path=in_file,
        lower_case=True,
        special_tokens=['[CLS]', '[MASK]', '<unk>', '<pad>']
    )
print("total vocab is", tokenizer_tf.vocab_size)
# -



text = 'This is [MASK] token with [CLS] </s>'



tokenizer_hf.tokenize(text)



tokenizer_sp = tokenizer_tf._tokenizer

text = 'this is [MASK] token with [CLS] <unk>'
tokenizer_sp.string_to_id(text.split())



tokenizer_sp.string_to_id(text)

tokenizer_sp.split(text.lower())

tokenizer_sp.detokenize([   48,    25,   636, 23265,   500,    20,  2853,    29,   636,
        5316,    18,   500,    13,     1, 17081,     1])

tokenizer_sp.detokenize([ 3713,   159, 32001,     2,  4065, 32000]).numpy().decode()

tokenizer_sp.string_to_id(['▁This', '▁is', '[MASK]', '▁token', '▁with', '[CLS]', '</s>'])

tokenizer_hf.encode(text)



tokenizer_hf.decode([100, 19, 32001, 14145, 28, 32000, 1])

tokenizer_sp.detokenize([100, 19, 32001, 14145, 28, 32000, 1])

tokenizer_hf.decode([ 3713,   159, 32001,     2,  4065, 32000])

tokenizer_sp.tokenize("This is")



# +
from tf_transformers.text import sentencepiece_model_pb2


mp = sentencepiece_model_pb2.ModelProto()
mp.ParseFromString(open(in_file, "rb").read())

for token in special_tokens:
    new_token = sentencepiece_model_pb2.ModelProto().SentencePiece()
    new_token.piece = token
    new_token.score = 0
    new_token.type = 3
    mp.pieces.append(new_token)
    print(f'added {token}...')

with open(out_file, 'wb') as f:
    f.write(mp.SerializeToString())
# -



import sentencepiece as sp

sp_model = sp.SentencePieceProcessor()
import tensorflow as tf
model_proto = tf.io.gfile.GFile(in_file, "rb").read()
sp_model.LoadFromSerializedProto(model_proto)

sp_model.encode_as_pieces(text)

text = 'This is [MASK] token with [CLS] </s>'


tokenizer_sp.split(text)



tokenizer_sp.detokenize([  100,    19,     3, 32001, 14145,    28,     3, 32000,     3,
           2,    87,     7,  3155])



model_proto = tf.io.gfile.GFile(out_file, "rb").read()


import nltk

article = '''Wikipedia (/ˌwɪkɪˈpiːdiə/ (About this soundlisten) wik-ih-PEE-dee-ə or /ˌwɪki-/ (About this soundlisten) wik-ee-) is a free content, multilingual online encyclopedia written and maintained by a community of volunteer contributors through a model of open collaboration, using a wiki-based editing system. Wikipedia is the largest and most-read reference work in history,[3] and is consistently one of the 15 most popular websites as ranked by Alexa; as of 2021, it was ranked as the 13th most popular site.[3][4] The project carries no advertisements and is hosted by the Wikimedia Foundation, an American non-profit organization funded mainly through small donations.[5]

Wikipedia was launched on January 15, 2001, by <NAME>[6] and <NAME>; Sanger coined its name as a blending of "wiki" and "encyclopedia".[7][8] Initially available only in English, versions in other languages were quickly developed. Combined, Wikipedia's editions comprise more than 56 million articles, that attract an average of around 2 billion unique device visits and receive more than 17 million edits per month, or develop at a rate of 1.9 edits per second.[9][10]

Wikipedia has received praise for its enablement of the democratization of knowledge, extent of coverage, unique structure, culture, and reduced amount of commercial bias, but criticism for exhibiting systemic bias, particularly gender bias against women.[11] Its reliability was frequently criticized in the 2000s, but has improved over time and has been generally praised in the late 2010s and 2020s.[12][3][11] Its coverage of controversial topics such as American politics and major events such as the COVID-19 pandemic has received substantial media attention. At various points, Wikipedia has been censored by world governments, ranging from the blocking of specific pages to bans on the entire site. Wikipedia has become an element of popular culture, with references in books, films and academic studies. In 2018, Facebook and YouTube announced that they would help users detect fake news by suggesting fact-checking links to related Wikipedia articles.[13][14]'''

for i in nltk.tokenize.sent_tokenize(article):
    print(i)
    print("-----------------------------------")

nltk.download('punkt')



StarcoderdataJupyterScriptsDedupFiltered