# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

from nltk.stem import WordNetLemmatizer 

string = 'initial clinical critical care covid19 patients seattle new york city chicago since first recognition cluster novel respiratory viral infections china late december 2019 intensivists united states growing concern infections severe acute respiratory syndrome coronavirus 2 sarscov2 named coronavirus disease 2019 spread hospitals united states covid19 extremely transmissible progress severe form respiratory failure potential overwhelm available critical care resources high critical care management covid19 patients spotlight covid19 arrived united states january anticipated dramatically increased usage critical care resources three hardesthit cities seattle new york city chicago combined total cases march 23 2020in special article describe initial clinical critical care covid19 areas attention clinical presentation laboratory values organ system effects treatment strategies resource management highlight clinical observations align differ already published reports represent early empiric experience authors intended serve recommendations guidelines practice rather starting point intensivists preparing address covid19 community'

lemmatizer = WordNetLemmatizer() 
print(string)

string_sep = string.split()

for i in range(len(string_sep)):
    current_word = string_sep[i]
    string_sep[i] = lemmatizer.lemmatize(current_word)

lemmatizer.lemmatize('microscopically')

string_sep

from nltk import pos_tag
nltk.download('averaged_perceptron_tagger')
pos_tag(string_sep)

string = 'initial clinical critical care covid19 patients seattle new york city chicago since first recognition cluster novel respiratory viral infections china late december 2019 intensivists united states growing concern infections severe acute respiratory syndrome coronavirus 2 sarscov2 named coronavirus disease 2019 spread hospitals united states covid19 extremely transmissible progress severe form respiratory failure potential overwhelm available critical care resources high critical care management covid19 patients spotlight covid19 arrived united states january anticipated dramatically increased usage critical care resources three hardesthit cities seattle new york city chicago combined total cases march 23 2020in special article describe initial clinical critical care covid19 areas attention clinical presentation laboratory values organ system effects treatment strategies resource management highlight clinical observations align differ already published reports represent early empiric experience authors intended serve recommendations guidelines practice rather starting point intensivists preparing address covid19 community'

''.join([i for i in string if not i.isdigit()])

with open('downstream/TextSGC_Bio/data/corpus/ohsumed.clean.txt', 'r') as f:
    lines = f.readlines()
    doc_content_list = [l.strip() for l in lines]
    for j in range(len(doc_content_list)):
        doc_content_list[j] = ''.join([i for i in doc_content_list[j] if not i.isdigit()])
        doc_content_list[j] = doc_content_list[j].strip()
print(doc_content_list[3].split())



StarcoderdataJupyterScriptsDedupFiltered