<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import random<jupyter_output><empty_output><jupyter_text>Import all data<jupyter_code># #data source : https://www.kaggle.com/daveianhickey/2000-16-traffic-flow-england-scotland-wales/version/8
# df2005_07 = pd.read_csv("accidents_2005_to_2007.csv",low_memory=False)
# df2009_11 = pd.read_csv("accidents_2009_to_2011.csv",low_memory=False)
# df2012_14 = pd.read_csv("accidents_2012_to_2014.csv",low_memory=False)
# df2014_lon = pd.read_csv("accidents_london_district1-32_2014.csv",low_memory=False)
# dfaadf = pd.read_csv("ukTrafficAADF.csv")
# clust = pd.read_csv("cluster_output.csv")
# df500clust = pd.read_csv("accidents_london_district2014w500clust.csv")


#Union the 3 files into 1 mega file and reset index
# df_all = pd.concat([df2005_07,df2009_11,df2012_14]).reset_index(drop=True)

df_all = pd.read_csv('data_processed.csv')
df_all = df_all[df_all['Year'] >= 2010]
df_all.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 756934 entries, 718592 to 1475525
Data columns (total 13 columns):
 #   Column           Non-Null Count   Dtype  
---  ------           --------------   -----  
 0   Cluster          756934 non-null  int64  
 1   Accident         756934 non-null  int64  
 2   Longitude        756934 non-null  float64
 3   Latitude         756934 non-null  float64
 4   Weather_Fine     756934 non-null  float64
 5   Weather_Raining  756934 non-null  float64
 6   Weather_Fog      756934 non-null  float64
 7   Weather_Other    756934 non-null  float64
 8   Day_of_Year      756934 non-null  int64  
 9   Month            756934 non-null  int64  
 10  Hour             756934 non-null  float64
 11  Day_of_Week      756934 non-null  int64  
 12  Year             756934 non-null  int64  
dtypes: float64(7), int64(6)
memory usage: 80.8 MB<jupyter_text>Generate Negative Samples (Non-accident data)<jupyter_code>T = df_all[df_all['Year'] == 2014]

df_nonacc = pd.DataFrame(columns=['Cluster','Day_of_year','Hour'])

i = 1

maxclust = df_all['Cluster'].max()

random.seed(888)

# while i < T.shape[0]*3:   #3 times more non-accident than accident records
while i < T.shape[0]:
# while i < 50:   #try 50 times first
    nday = random.randint(1, 365)
    nclust = random.randint(1, maxclust)  #maximum cluster id
    nhr = random.randint(0, 23)
    isacc = T.loc[(df_all['Cluster'] == nclust) & (T['Day_of_Year'] == nday) & (T['Hour'] == nhr)]
    if isacc.empty:
        i += 1
        dfnew = pd.DataFrame([[nclust,nday,nhr]], columns=['Cluster','Day_of_Year','Hour'])
        df_nonacc = df_nonacc.append(dfnew,ignore_index=True)
    print(i)
#     else:
#         print([nclust,nday,nhr])
#         print("Accident found!")

df_nonacc.to_csv("non_accident_2014.csv")<jupyter_output>8954
8955
8956
8957
8957
8958
8959
8960
8961
8961
8962
8963
8964
8964
8964
8965
8965
8966
8966
8967
8968
8969
8970
8970
8971
8972
8972
8973
8974
8974
8975
8975
8976
8977
8978
8979
8980
8981
8982
8982
8983
8984
8985
8986
8986
8987
8988
8989
8990
8991
8992
8992
8992
8992
8992
8993
8994
8994
8994
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005
9005
9006
9007
9008
9009
9010
9011
9012
9013
9014
9015
9016
9017
9017
9017
9017
9017
9018
9019
9020
9021
9022
9023
9023
9024
9024
9025
9025
9025
9026
9027
9028
9029
9030
9031
9032
9033
9034
9035
9036
9037
9038
9039
9040
9041
9042
9042
9042
9043
9043
9043
9044
9045
9046
9047
9048
9049
9050
9051
9052
9053
9053
9054
9055
9055
9056
9057
9057
9058
9058
9059
9060
9061
9062
9063
9064
9065
9065
9066
9066
9067
9068
9068
9069
9069
9069
9070
9071
9072
9073
9074
9074
9075
9076
9076
9076
9077
9077
9077
9077
9078
9079
9080
9081
9082
9083
9084
9085
9085
9085
9086
9087
9087
9088
9089
9090
9091
9092
9092
9092
9093
9094
9095
9095
9096
9097
9097
9097
9098
[...]
StarcoderdataJupyterStructuredCleanDedup