<jupyter_start><jupyter_text>Read Files<jupyter_code>import pandas as pd
import numpy as np
from glob import glob
import os, psutil
from tqdm.notebook import tqdm
fns = glob('/media/robmulla/moardata/reddit_place2/*.parquet')
process = psutil.Process(os.getpid())
print(process.memory_info().rss)
fns.sort()
dfs = []
print(process.memory_info().rss) 
for f in tqdm(fns):
    df = pd.read_parquet(f)
    df['pixel_color'] = df['pixel_color'].astype('category')
    df['x'] = df['x'].astype('int32')
    df['y'] = df['y'].astype('int32')
    dfs.append(df)
print('About to concat')
print(process.memory_info().rss) 
# Does this double our memory use
dfs = pd.concat(dfs)
print('Done with concat')
print(process.memory_info().rss)
dfs.to_parquet('/media/robmulla/moardata/reddit_place2/combined_v1.parquet')
dfs.head()<jupyter_output><empty_output>