<jupyter_start><jupyter_text>Chapter Two - Data Loading, Saving and file Formats<jupyter_code>from optimus import Optimus
op = Optimus("pandas")<jupyter_output><empty_output><jupyter_text>Loading a file<jupyter_code>df = op.load.file("my_file.json")
display(df)<jupyter_output><empty_output><jupyter_text>Fixed number of rows<jupyter_code>df = op.load.csv("data/file.csv", n_rows=5) 
display(df)<jupyter_output><empty_output><jupyter_text>Without a header<jupyter_code>df = op.load.csv("data/file.csv", header=None) 
display(df)<jupyter_output><empty_output><jupyter_text>Assigning which value will be assumed as null<jupyter_code>df = op.load.csv("data/file.csv", null_value="Null")
display(df)<jupyter_output><empty_output><jupyter_text>Using a specific separator<jupyter_code>df = op.load.csv("data/file.csv", sep=";")
display(df)<jupyter_output><empty_output><jupyter_text>Wildcards<jupyter_code>df = op.load.csv("csv/*")
display(df)
df = op.load.csv("csv/file-*.csv") 
display(df)
df = op.load.csv("csv/file-?.csv")
display(df)
df = op.load.csv("csv/file-*[0-9].*.csv")
display(df)<jupyter_output><empty_output><jupyter_text>Loading large files<jupyter_code>op = Optimus("dask") 
df = op.load.csv("s3://my-storage/massive-file.csv")<jupyter_output><empty_output><jupyter_text>From a remote connection<jupyter_code>conn = op.connect.s3(endpoint_url="s3://my-storage/") 
df = op.load.csv("files/foo_file.csv", conn=conn) 
df2 = op.load.file("files/file_bar.xml", conn=conn)<jupyter_output><empty_output><jupyter_text>From a database<jupyter_code>db = op.connect.mysql(host="localhost", database="my_database")
df = op.load.database_table("foo_table", conn=db)<jupyter_output><empty_output><jupyter_text>Memory usage<jupyter_code>op = Optimus("dask") 
df = op.create.dataframe({ 
    "a": [1000,2000,3000,4000,5000]*10000, 
    "b": [1,2,3,4,5]*10000 
}) 
df.size() 
df = df.optimize()
df.size()<jupyter_output><empty_output><jupyter_text>Creating a dataframe from scratch<jupyter_code>df = op.create.dataframe({ 
    "name":["OPTIMUS", "BUMBLEBEE", "EJECT"], 
    "function":["Leader", "Espionage", " Electronic Surveillance"] 
})
df<jupyter_output><empty_output><jupyter_text>Saving a dataframe To a local file<jupyter_code>df.save.csv("foo_output.csv", sep=";")<jupyter_output><empty_output><jupyter_text>To a remote connection<jupyter_code>df.save.xml("files/foo_output.xml", conn=conn)<jupyter_output><empty_output><jupyter_text>To a database table<jupyter_code>df.save.database_table("foo_output_table", db=db)<jupyter_output><empty_output><jupyter_text>Repartition<jupyter_code>df = df.repartition(2)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup