U5chNdjU6p6rrXpl9U=" crossorigin="anonymous"></script><meta property="og:title" content="Tokenize_folder — tokenize_folder"><meta property="og:description" content="Tokenize text files in `path` in parallel using `n_workers`"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMM