# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:env_nlp_text_class]
#     language: python
#     name: conda-env-env_nlp_text_class-py
# ---

# + [markdown] Collapsed="false"
# # Environment

# + Collapsed="false"
import data.beam_debug as debug, src.preprocessing.test_dofn as dofn
import time, google.cloud.bigquery as bigquery

# + [markdown] Collapsed="false"
# # Data

# + Collapsed="false"
client = bigquery.Client()

# + Collapsed="false"
query_minute_1 = """
SELECT
  id,
  title,
  body
FROM
  stackoverflow.posts_p1
WHERE
  id IN {minute_1}""".format(
    minute_1=debug.minute_1
)

query_minute_5 = """
SELECT
  id,
  title,
  body
FROM
  stackoverflow.posts_p1
WHERE
  id IN {minute_5}""".format(
    minute_5=debug.minute_5
)

# + Collapsed="false"
df_minute_1 = client.query(query_minute_1).to_dataframe()

# + Collapsed="false"
df_minute_5 = client.query(query_minute_5).to_dataframe()

# + [markdown] Collapsed="false"
# # Instantianting DoFN Object

# + Collapsed="false"
preprocessing_object = dofn.NLP()

# + [markdown] Collapsed="false"
# # Testing Execution Speed (t >= 300s With DataFlow)

# + Collapsed="false"
df_minute_5.head()

# + Collapsed="false"
body = df_minute_5['body'].tolist()
title = df_minute_5['title'].tolist()

# + Collapsed="false"
times = []

# + Collapsed="false"
for body_element, title_element in zip(body, title):
    start_time = time.time()
    
    temp1 = preprocessing_object.nlp(body_element)
    temp2 = preprocessing_object.nlp(title_element)
    
    end_time = time.time()
    total_time = end_time-start_time
    
    times.append(total_time)

# + Collapsed="false"
times

# + Collapsed="false"
# Testing Memory Leakage

# + Collapsed="false"
import random
import spacy
import plac
import psutil
import sys
import objgraph
import gc

gc.set_debug(gc.DEBUG_SAVEALL)

def load_data():
    return ["This is a fake test document number %d."%i for i in random.sample(range(100_000), 10_000)]


# def print_memory_usage():
#     print(objgraph.show_growth(limit=5))
#     print("GC count="+str(gc.get_count()))
#     gc.collect()

class ReloadableNlp:
    def __init__(self, model, reload=1000):
        self.model = model
        self.reload = reload
        self.count = 0
        self.nlp = spacy.load(model)

    def get_nlp(self):
        self.count += 1
        if self.count % 1_000 == 0:
            del self.nlp
            gc.collect()
            self.nlp = spacy.load(self.model)
        return self.nlp



def parse_texts(reloadable, texts, iterations=1_000):
    for i in range(iterations):
        for doc in reloadable.get_nlp().pipe(texts, cleanup=True):
            yield doc

@plac.annotations(
    iterations=("Number of iterations", "option", "n", int),
    model=("spaCy model to load", "positional", None, str)
)
def main(model='en_core_web_sm', iterations=1_000):
    texts = load_data()
    reloadable = ReloadableNlp(model)
    for i, doc in enumerate(parse_texts(reloadable, texts, iterations=iterations)):
        if i % 10_000 == 0:
            print(i, psutil.virtual_memory().percent)
            #print_memory_usage()
            sys.stdout.flush()




# + Collapsed="false"
main()

# + Collapsed="false"
# ! python -m spacy validate

# + Collapsed="false"
# ! python -m spacy download en_core_web_sm

# + Collapsed="false"


StarcoderdataJupyterScriptsDedupFiltered