# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

import pandas as pd
import numpy as np
import os
#print os.getcwd()
dir = "/Users/walterdempsey/Box/MD2K Processed Data/smoking-lvm-cleaned-data/"
os.chdir(dir)
os.getcwd()

# read data
original_data = pd.read_csv(dir + 'random-ema.csv')
original_data = original_data.drop(['offset'], axis=1)
backup_data = pd.read_csv(dir + 'random-ema-backup.csv')

# check data types
print(original_data.dtypes)
print("")
print(backup_data.dtypes)

print(original_data.shape)
original_data['participant_id'].value_counts().sort_index()

print(backup_data.shape)
backup_data['participant_id'].value_counts().sort_index()

# +
# Look at the participant_id intersection of the backup and original datasets
unique_backup_ids = set(backup_data.participant_id)
unique_original_ids = set(original_data.participant_id)
ids_intersection = unique_backup_ids.intersection(unique_original_ids)

original_data_subset = original_data[original_data.participant_id.isin(ids_intersection)]
backup_data_subset = backup_data[backup_data.participant_id.isin(ids_intersection)]

print unique_backup_ids.difference(unique_original_ids)
print unique_original_ids.difference(unique_backup_ids)

# +
# In intersection, the backup is a superset of the observations. 
# Therefore, the backup shoud be used for analysis where possible.
print(original_data_subset.shape)
temp_original = pd.DataFrame(original_data_subset['participant_id'].value_counts().sort_index())
temp_backup = np.array(backup_data_subset['participant_id'].value_counts().sort_index())
temp_original['backup'] = temp_backup
temp_original.columns = ['original', 'backup']
print(temp_original)

sum(temp_original['backup'] - temp_original['original'])

# +
s = set()
d = {}

lst2 = []
for index, row in backup_data.iterrows():
    participant_id = row['participant_id']
    hour = row['hour']
    minute = row['minute']
    day_of_week = row['day_of_week']
    valid_key = (participant_id, hour, minute, day_of_week)
    s.add(valid_key)
    d[valid_key] = index

for index, row in original_data.iterrows():
    participant_id = row['participant_id']
    hour = row['hour']
    minute = row['minute']
    day_of_week = row['day_of_week']
    valid_key = (participant_id, hour, minute, day_of_week)
    if valid_key in s:
        s.remove(valid_key)
    else:
        lst2.append(index)

lst = []
for i in s:
    lst.append(d[i])

lst.sort()
difference = backup_data.ix[lst] 
print(difference.shape)
difference # rows that are only in the backup but not in the original
# -

difference = original_data.ix[lst2]
print(difference.shape)
difference # rows in the original but not the backup



StarcoderdataJupyterScriptsDedupFiltered