<jupyter_start><jupyter_text>Install tree-sitter parser & clang1) Install tree-sitter parser- ```cd evaluation/CodeBLEU/parser```- ```bash build.sh```2) Install clang- goto `data/github/preprocessing/src/code_tokenizer.py` and replace path with your `clang` path- **NOTE:** You may try to copy `clang` from `/tmp/clang`, but i'm not sure it will work  Preprocessing pipeline PLBART Tokenization/detokenization- **tokenization**: Code -> Tokens (a.k.a. python tokenizer) -> SPM.- so we need to do detokenization: SPM -> Tokens -> syntax highliting -> Printed Code<jupyter_code>from itertools import islice
import subprocess
from pygments import highlight
from pygments.lexers import Python3Lexer
from pygments.formatters import HtmlFormatter, TerminalFormatter, TerminalTrueColorFormatter, Terminal256Formatter
from IPython.core.display import HTML, display


import data.github.preprocessing.src.code_tokenizer as code_tokenizer

def peak(path: str, n=10) -> str:
    """reads first n lines from file"""
    with open(path, 'r') as f:
        head = list(islice(f, n)) 
    return "".join(head)

def detok(code, lang='python'):
    detokenize = getattr(code_tokenizer, f"detokenize_{lang}")
    return detokenize(code)

def tokenize(code, lang='python'):
    detokenize = getattr(code_tokenizer, f"tokenize_{lang}")
    return detokenize(code)


def colored(code: str, formatter=Terminal256Formatter) -> str:
    return highlight(code, Python3Lexer(), formatter())

def remove_bpe(s: str):
    return s.replace(" ", "").replace("▁", " ").lstrip(' ')<jupyter_output><empty_output><jupyter_text>How to detokenize- tokenizer/detokenizer is implemented in `PLBART/data/github/preprocessing`- you may also use `tokenize` function<jupyter_code># raw BPE tokens, brr...
print(peak(path_to_spm_file, 20))
# nice code!
print(colored(detok(
    remove_bpe(peak(path_to_spm_file, 20))
)))
tree_sitter_path=f"{HOME}/evaluation/CodeBLEU/parser/my-languages.so"
!ls $tree_sitter_path
from tree_sitter import Language, Parser, Tree
PY_LANGUAGE = Language(tree_sitter_path, 'python')
JAVA_LANGUAGE = Language(tree_sitter_path, 'java')



def traverse_tree(tree: Tree):
    cursor = tree.walk()

    reached_root = False
    while reached_root == False:
        yield cursor.node

        if cursor.goto_first_child():
            continue

        if cursor.goto_next_sibling():
            continue

        retracing = True
        while retracing:
            if not cursor.goto_parent():
                retracing = False
                reached_root = True

            if cursor.goto_next_sibling():
                retracing = False

class TreeSitterCode:
    """
    stores 1) the source code in bytes
           2) the tree
        allows to traverse nodes in the AST tree, replace values.
        updates both with `replace` method
        (tool to change identifiers)
    """
    def __init__(self, code: str, parser: Parser):
        self.code_bytes = bytes(
            code, "utf-8"
        )
        self.parser = parser
        self.tree = self.parser.parse(self.code_bytes)
        
    @property
    def code(self):
        return self.code_bytes.decode()
    
    def traverse(self):
        for item in traverse_tree(self.tree):
            yield item
            
    def identifiers(self):
        def is_identifier(node): return node.type=='identifier'
        for node in self.traverse():
            if is_identifier(node):
                yield node
                
    def get_value(self, node):
        return self.code_bytes[node.start_byte: node.end_byte]
    
    def inspect(self):
        print("="*80)
        for it in code_tree.traverse():
            print(it.type, '\t', code_tree.get_value(it), '\t', it.start_point, '\t', it.end_point, '\t', it.start_byte, '\t', it.end_byte)
            
    def replace(self, node: Tree, new_value: bytes):
        assert not b'\n' in new_value
        start_byte_old, end_byte_old = node.start_byte, node.end_byte
        length = len(new_value)
        self.code_bytes = self.code_bytes[:node.start_byte] + \
                new_value + \
                self.code_bytes[node.end_byte:]
        assert node.end_point[0] == node.start_point[0], "we should edit one line"
        # print(start_byte_old, end_byte_old, start_byte_old + length, node.start_point, node.end_point, (node.start_point[0], node.start_point[1] + length))
        self.tree.edit(
            start_byte=start_byte_old,
            old_end_byte=end_byte_old,
            new_end_byte=start_byte_old + length,
            start_point=node.start_point,
            old_end_point=node.end_point,
            new_end_point=(node.start_point[0], node.start_point[1] + length),
        )
        self.tree = self.parser.parse(self.code_bytes, self.tree)
        # self.tree = self.parser.parse(self.code_bytes)

        
code_example ="""
my_file = "input.txt"
with open(my_file, "r") as f:
    print(f.readlines())
"""
parser = Parser()
parser.set_language(PY_LANGUAGE)
code_tree = TreeSitterCode(code_example, parser)

print("<No bug>")
print(colored(code_tree.code))

for identifier in code_tree.identifiers():
    value = code_tree.get_value(identifier)
    if value == b'f':
        code_tree.replace(identifier, b'my_file') # insert bug
        break

print("<With bug>")
print(colored(code_tree.code))
        
# code_tree.inspect()
    
for identifier in code_tree.identifiers():
    value = code_tree.get_value(identifier)
    if value == b'f':
        code_tree.replace(identifier, b'my_file') # insert bug
        break
        
print("<With second bug>")
print(colored(code_tree.code))

# code_tree.inspect()<jupyter_output><No bug>
my_file [38;5;241m=[39m [38;5;124m"[39m[38;5;124minput.txt[39m[38;5;124m"[39m
[38;5;28;01mwith[39;00m [38;5;28mopen[39m(my_file, [38;5;124m"[39m[38;5;124mr[39m[38;5;124m"[39m) [38;5;28;01mas[39;00m f:
    [38;5;28mprint[39m(f[38;5;241m.[39mreadlines())

<With bug>
my_file [38;5;241m=[39m [38;5;124m"[39m[38;5;124minput.txt[39m[38;5;124m"[39m
[38;5;28;01mwith[39;00m [38;5;28mopen[39m(my_file, [38;5;124m"[39m[38;5;124mr[39m[38;5;124m"[39m) [38;5;28;01mas[39;00m my_file:
    [38;5;28mprint[39m(f[38;5;241m.[39mreadlines())

<With second bug>
my_file [38;5;241m=[39m [38;5;124m"[39m[38;5;124minput.txt[39m[38;5;124m"[39m
[38;5;28;01mwith[39;00m [38;5;28mopen[39m(my_file, [38;5;124m"[39m[38;5;124mr[39m[38;5;124m"[39m) [38;5;28;01mas[39;00m my_file:
    [38;5;28mprint[39m(my_file[38;5;241m.[39mreadlines())
StarcoderdataJupyterStructuredCleanDedup