<jupyter_start><jupyter_text><NAME> Analytic GoalMonitor the average return per hour of Bitcoin over the last month and look for statistically significant trends<jupyter_code>sc
%%help
%%configure -f
{
    "conf":{
        "spark.driver.memory": "7G",
        "spark.executor.memory": "7G",
        "spark.jars.packages": "org.mongodb.spark:mongo-spark-connector_2.11:2.3.1",
        "spark.mongodb.input.uri": "mongodb+srv://emre:<EMAIL>/msds697_project_2.crypto_exchange"
    }
}
df = spark.read.format('com.mongodb.spark.sql.DefaultSource').load()
df_rdd = df.rdd
from datetime import datetime


btc = (
    df_rdd.filter(lambda x: x[2] == "btcusd")
    .map(lambda x: (datetime.fromtimestamp(x[5] / 1000), x[1]))
    .filter(lambda x: (x[0].year == 2021 and x[0].minute == 0))
    .cache()
)

btc.collect()<jupyter_output><empty_output>