# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
  
####
#### Sept. 1
####

"""
Jupyter Notebook is called on iMac:
Acreages_ConfusionStyle_raw_n_regular_SOS
"""

import csv
import time
import scipy
import datetime
import itertools
import numpy as np
import os, os.path
import scipy.signal
import pandas as pd
from patsy import cr
from math import factorial
from IPython.display import Image
from sklearn.linear_model import LinearRegression
from statsmodels.sandbox.regression.predstd import wls_prediction_std

import matplotlib.pyplot as plt
import seaborn as sb

# import geopandas as gpd
# from shapely.geometry import Point, Polygon
# from pprint import pprint

import sys
start_time = time.time()

####################################################################################
###
###                      Aeolus Core path
###
####################################################################################

sys.path.append('/Users/hn/Documents/00_GitHub/Ag/remote_sensing/python/')
sys.path.append('/home/hnoorazar/remote_sensing_codes/')

import remote_sensing_core as rc
import remote_sensing_core as rcp


####################################################################################
###
###      Parameters                   
###
####################################################################################
SG_win_size = 5
SG_order = 3
delt = 0.4
do_plot  = 0
SF_year = 2017



given_county = "Grant"
SF_year = 2016
indekses = "EVI"

regularized = True
raw_or_regular = "regular"

####################################################################################
###
###                   Aeolus Directories
###
####################################################################################
param_dir = "/Users/hn/Documents/00_GitHub/Ag/remote_sensing/parameters/"
SF_data_dir = "/Users/hn/Documents/01_research_data/remote_sensing/01_Data_part_not_filtered/"

# the following data came from 70% cloud.
regular_in_dir = "/Users/hn/Documents/01_research_data/remote_sensing/test_aeolus_outputs/2Yrs_tables_regular/"
regular_output_dir = "/Users/hn/Documents/01_research_data/" + \
                     "remote_sensing/05_confusion_tables/00_separate_CountyYear/"

SF_years = [2016, 2017, 2018]
counties = ["Grant", "Whitman", "Asotin",
            "Garfield", "Ferry", "Franklin",
            "Columbia", "Adams","Benton",
            "Chelan", "Douglas", "Kittitas", "Klickitat",
            "Lincoln", "Okanogan", "Spokane", "Stevens",
            "Yakima",'Pend_Oreille', 'Walla_Walla']

for given_county in counties:
    for SF_year in SF_years:
        print ("given_county is " + given_county.replace("_", " "))

        data_dir = regular_in_dir    
        output_dir = regular_output_dir

        os.makedirs(output_dir, exist_ok=True)

        print ("_________________________________________________________")
        print ("data dir is: " + data_dir)
        print ("_________________________________________________________")
        print ("output_dir is: " +  output_dir)
        print ("_________________________________________________________")

        ####################################################################################
        ###
        ###                   Read parameters and data
        ###
        ####################################################################################
        double_crop_potens = pd.read_csv(param_dir + "double_crop_potential_plants.csv")
        
        WSDA_DataTable = pd.read_csv(SF_data_dir + "WSDA_DataTable_" + str(SF_year) + ".csv")
        WSDA_DataTable = WSDA_DataTable[WSDA_DataTable.county == given_county.replace("_", " ")]
        WSDA_DataTable["DataSrc"] = WSDA_DataTable["DataSrc"].str.lower()
        WSDA_DataTable["CropTyp"] = WSDA_DataTable["CropTyp"].str.lower()

        print (double_crop_potens.shape)
        print (WSDA_DataTable.shape)

        """
        Generate different datafamres based on the following variables
        2 * 2 * 2 * 2 different combinations of NASS, Double_by_Notes, Irrigated, LastSurveyYear
        For now, leave out the fucking last survey year!

        In the following we have abbreviated:

        AF: All Fields
        DP: Double Potential Fields (i.e. perennials out)
        Irr: Just Irrigated Fields
        BothIrr: both irrigated and non-irrigated

        """
        ####
        ####  parameters
        ####
        deltas = [0.1, 0.2, 0.3, 0.4]
        delta_windows_degrees = [[5, 1], [5, 3], [7, 3], [9, 3]]

        output_columns = ['parameters', 'actual_2_pred_2', 'actual_2_pred_Not2',
                          'actual_Not2_pred_2', 'actual_Not2_pred_Not2',]

        for indeks in [indekses]:
            for exactly_2_peaks in [False, True]:
                for double_by_Note in [False]:
                    for NASS_out in [False]:
                        # we have dropped out non-irrigated fields in the peak_finding step
                        for non_Irr_out in [True]: 
                            for perennials_out in [True]: # 
                                #### 
                                #### build output dataframe
                                #### 
                                output = pd.DataFrame(data=None, 
                                                      #delta_windows_degrees
                                                      index = np.arange(len(delta_windows_degrees)),
                                                      columns = output_columns)
                                output['parameters'] = delta_windows_degrees

                                #### 
                                #### Build shapeFile info accordingly
                                #### 
                                curr_SF = WSDA_DataTable.copy()

                                if double_by_Note == False:
                                    dbl_name = "_dblNotFiltered_"
                                else:
                                    curr_SF = rc.filter_double_by_Notes(curr_SF)
                                    dbl_name = "_onlyDblByNotes_"

                                if NASS_out == True:
                                    curr_SF = rc.filter_out_NASS(curr_SF)
                                    NASS_name = "NASSOut_"
                                else:
                                    NASS_name = "NASSin_"

                                if non_Irr_out == True:
                                    curr_SF = rc.filter_out_nonIrrigated(curr_SF)
                                    non_Irr_name = "JustIrr"
                                else:
                                    non_Irr_name = "BothIrr"

                                if perennials_out == True:
                                    curr_SF = curr_SF[curr_SF.CropTyp.isin(double_crop_potens['Crop_Type'])]
                                    Pere_name = "_PereOut_"
                                else:
                                    Pere_name = "_PereIn_"


                                print ("NASS_out: " + str(NASS_out) + ", non_Irr_out: " + str(non_Irr_out) + \
                                       ", perennials_out: " + str(perennials_out))

                                for location, params in enumerate(output['parameters']):
                    #                 if location % 15 ==0:
                    #                     print (location)
                                    window = params[0]
                                    degree = params[1]

                                    f_name = given_county  + "_" + str(SF_year) + "_regular_" + indeks + \
                                             "_SG_win" + str(window) + "_Order" + str(degree) + ".csv"
                                    print (f_name)

                                    doubl_pk_file = data_dir + f_name

                                    doubl_peak_table = pd.read_csv(doubl_pk_file, low_memory=False)

                                    doubl_peak_table["CropTyp"] = doubl_peak_table["CropTyp"].str.lower()

                                    if double_by_Note == True:
                                        doubl_peak_table = rc.filter_double_by_Notes(doubl_peak_table)

                                    if NASS_out == True:
                                        doubl_peak_table = rc.filter_out_NASS(doubl_peak_table)

                                     # print ("NASS_out: " + str(NASS_out))
                                     # print ("curr_SF.DataSrc.unique(): ")
                                     # print (curr_SF.DataSrc.unique())

                                     # print ("doubl_peak_table.DataSrc.unique(): ")
                                     # print (doubl_peak_table.DataSrc.unique())
                                     # print("____________________________________________________")

                                    if non_Irr_out == True:
                                        doubl_peak_table = rc.filter_out_nonIrrigated(doubl_peak_table)
                                    # print("_____________________________________________")
                                    # print("non_Irr_out " + str(non_Irr_out) + " doubl_peak_table.Irrigtn:")
                                    # print (doubl_peak_table.Irrigtn.unique())

                                    if perennials_out == True:
                                        doubl_peak_table = doubl_peak_table[\
                                                 doubl_peak_table.CropTyp.isin(double_crop_potens['Crop_Type'])]
                                     # print("perennials_out " + str(perennials_out))
                                     # print (doubl_peak_table.CropTyp.unique())

                                     # print ("No. of rows after dropping perennials \
                                     #          are [%(nrow)d]." % {"nrow":doubl_peak_table.shape[0]})

                                    # print ("No. of rows are [%(nrow)d]." % {"nrow":doubl_peak_table.shape[0]})
                                    doubl_peak_table.drop(['doy', 'EVI', 'system_start_time', 'Date', 
                                                           'human_system_start_time', 
                                                           'EVI_ratio','SOS', 'EOS'], axis=1, inplace=True)

                                    doubl_peak_table.drop_duplicates(inplace=True)
                                    # print ("No. of rows after dropping dupliates are [%(nrow)d]." % \
                                    # {"nrow":doubl_peak_table.shape[0]})

                                    #### 
                                    #### Populate output dataframe
                                    #### 
                                    actual_double_cropped = rc.filter_double_by_Notes(curr_SF)
                                    actual_Notdouble_cropped = rc.filter_Notdouble_by_Notes(curr_SF)

                                    if exactly_2_peaks == False:
                                        predicted_double_peaks = doubl_peak_table[\
                                                                 doubl_peak_table.season_count >= 2].copy()

                                        predicted_Notdouble_peaks = doubl_peak_table[\
                                                                    doubl_peak_table.season_count < 2].copy()
                                        exactly_2_peaks_name = "morethan2Peaks"
                                    else:
                                        predicted_double_peaks = doubl_peak_table[\
                                                        doubl_peak_table.season_count == 2].copy()


                                        predicted_Notdouble_peaks = doubl_peak_table[\
                                                                    doubl_peak_table.season_count != 2].copy()

                                        exactly_2_peaks_name = "exactly2Peaks"

                                    # print ("There are [%(nrow)d] IDs in curr_SF." % \
                                    #       {"nrow":len(curr_SF['ID'])})

                                    # print ("of which [%(nrow)d] are unique." % \
                                    #        {"nrow":len(curr_SF['ID'].unique())})

                                    actual_2_pred_2 = actual_double_cropped[\
                                                    actual_double_cropped['ID'].isin(\
                                                                            predicted_double_peaks['ID'])]

                                    actual_2_pred_2 = actual_2_pred_2['ExctAcr'].sum()

                                    actual_Not2_pred_2 = actual_Notdouble_cropped[\
                                                           actual_Notdouble_cropped['ID'].isin(\
                                                                          predicted_double_peaks['ID'])]
                                    actual_Not2_pred_2 = actual_Not2_pred_2['ExctAcr'].sum()

                                    # the follwoing two lines would not work, since we have not 
                                    # included the fields with no maximum in our output
                                    # actual_2_pred_Not2 = sum(actual_double_cropped['ID'].isin(\
                                    #                                  predicted_Notdouble_peaks['ID']))
                                    # actual_Not2_pred_Not2 = sum(actual_Notdouble_cropped['ID'].isin(\
                                    #                            predicted_Notdouble_peaks['ID']))

                                    actual_2_pred_Not2 = actual_double_cropped['ExctAcr'].sum() - actual_2_pred_2

                                    actual_Not2_pred_Not2 = actual_Notdouble_cropped['ExctAcr'].sum() - \
                                                              actual_Not2_pred_2

                                    fillin_col = ["actual_2_pred_2", "actual_2_pred_Not2", \
                                                  "actual_Not2_pred_2", "actual_Not2_pred_Not2"]

                                    fillin_vals = [actual_2_pred_2, actual_2_pred_Not2, \
                                                   actual_Not2_pred_2, actual_Not2_pred_Not2]

                                    output.loc[location, fillin_col] = fillin_vals


                                ###########
                                output['parameters'] = output['parameters'].astype("str")

                                filename = regular_output_dir + given_county + "_" + str(SF_year) + \
                                           "_" + indeks + \
                                           Pere_name + NASS_name + non_Irr_name + dbl_name + \
                                           "confusion_Acr_" + exactly_2_peaks_name + "_" + \
                                            raw_or_regular + ".csv"


                                output['actual_2_pred_2'] = output['actual_2_pred_2'].astype(float)
                                output['actual_2_pred_Not2'] = output['actual_2_pred_Not2'].astype(float)
                                output['actual_Not2_pred_2'] = output['actual_Not2_pred_2'].astype(float)
                                output['actual_Not2_pred_Not2'] = output['actual_Not2_pred_Not2'].astype(float)
                                output = output.round(decimals=2)
                                print (output)

                                output.to_csv(filename, index = False)






# -

import pandas as pd

data_dir = "/Users/hn/Documents/00_GitHub/Ag/remote_sensing/python/drivers/05_confusionTables/01_merge_tables/"
A = pd.read_csv(data_dir + "allCounties_2016_confusion.csv")
B = pd.read_csv(data_dir + "allCounties_separate_2016_confusion.csv")





StarcoderdataJupyterScriptsDedupFiltered