# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:pytorch]
#     language: python
#     name: conda-env-pytorch-py
# ---

import os
import re
from collections import Counter
import slimit
from slimit.parser import Parser
from slimit.visitors.nodevisitor import ASTVisitor
from slimit.lexer import Lexer
from functions import lex, remove_comments

input_len = 30

# +
# _actual is the result of removing all missing and undecodable files
file_dir = '/home/irteam/users/data/150kJavaScript/'
lexer = Lexer()

with open(file_dir+'programs_training_actual.txt') as f:
    file_list=f.readlines()
# -

idx=400
for i,file in enumerate(file_list[idx:idx+1]):
    file = file.strip()
    with open(file_dir+file) as f:
        text = f.read()
    tokens = lex(text,'value')

for line in ' '.join(tokens).split(';'):
    print(line)
    print('\n')

lexer.input('var a = ')

for token in lexer:
    print(token)

out_file.fll



StarcoderdataJupyterScriptsDedupFiltered