# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# foo.027	- Observed and Potential Crop Yields
#
# Description: http://www.earthstat.org/data-download/
# Download as Yield Gaps and Climate Bins for Major Crops - Geotiff format :http://www.earthstat.org/data-download/
#
# File type: Geotiff

# +
# Libraries for downloading data from remote server (may be ftp)
import requests
from urllib.request import urlopen
from contextlib import closing
import shutil

# Library for uploading/downloading data to/from S3
import boto3

# Libraries for handling data
import rasterio as rio
import numpy as np
# from netCDF4 import Dataset
# import pandas as pd
# import scipy

# Libraries for various helper functions
# from datetime import datetime
import os
import threading
import sys
from glob import glob
# -

# Define local file locations

# +
local_folder = "/Users/Max81007/Desktop/Python/Resource_Watch/Raster/foo.004/"

sub_folder1 = "Maize.2.crop.calendar.fill/"
sub_folder2 = "Maize.crop.calendar.fill/"
sub_folder3 = "Rice.2.crop.calendar.fill/"
sub_folder4 = "Rice.crop.calendar.fill/"
sub_folder5 = "Soybeans.crop.calendar.fill/"
sub_folder6 = "Wheat.crop.calendar.fill/"
sub_folder7 = "Wheat.Winter.crop.calendar.fill/"

file_name1 = "tot.days"
file_name2 = "plant.start"
file_name3 = "tot.days"
file_name4 = "plant.start"
file_name5 = "tot.days"
file_name6 = "plant.start"
file_name7 = "tot.days"
file_name8 = "plant.start"
file_name9 = "tot.days"
file_name10 = "plant.start"
file_name11 = "tot.days"
file_name12 = "plant.start"
file_name13 = "tot.days"
file_name14 = "plant.start"

local_orig1 = local_folder+ sub_folder1 + file_name1 + ".asc" 
local_orig2 = local_folder+ sub_folder1 + file_name2 + ".asc" 
local_orig3 = local_folder+ sub_folder2 + file_name3 + ".asc" 
local_orig4 = local_folder+ sub_folder2 + file_name4 + ".asc" 
local_orig5 = local_folder+ sub_folder3 + file_name5 + ".asc" 
local_orig6 = local_folder+ sub_folder3 + file_name6 + ".asc" 
local_orig7 = local_folder+ sub_folder4 + file_name7 + ".asc" 
local_orig8 = local_folder+ sub_folder4 + file_name8 + ".asc" 
local_orig9 = local_folder+ sub_folder5 + file_name9 + ".asc" 
local_orig10 = local_folder+ sub_folder5 + file_name10 + ".asc" 
local_orig11 = local_folder+ sub_folder6 + file_name11 + ".asc" 
local_orig12 = local_folder+ sub_folder6 + file_name12 + ".asc" 
local_orig13 = local_folder+ sub_folder7 + file_name13 + ".asc" 
local_orig14 = local_folder+ sub_folder7 + file_name14 + ".asc" 

orig_extension_length = 4 #4 for each char in .tif

local_edit1 = local_orig1[:-orig_extension_length] + "edit.tif" 
local_edit2 = local_orig2[:-orig_extension_length] + "edit.tif" 
local_edit3 = local_orig3[:-orig_extension_length] + "edit.tif" 
local_edit4 = local_orig4[:-orig_extension_length] + "edit.tif" 
local_edit5 = local_orig5[:-orig_extension_length] + "edit.tif" 
local_edit6 = local_orig6[:-orig_extension_length] + "edit.tif" 
local_edit7 = local_orig7[:-orig_extension_length] + "edit.tif" 
local_edit8 = local_orig8[:-orig_extension_length] + "edit.tif" 
local_edit9 = local_orig9[:-orig_extension_length] + "edit.tif" 
local_edit10 = local_orig10[:-orig_extension_length] + "edit.tif" 
local_edit11 = local_orig11[:-orig_extension_length] + "edit.tif" 
local_edit12 = local_orig12[:-orig_extension_length] + "edit.tif" 
local_edit13 = local_orig13[:-orig_extension_length] + "edit.tif" 
local_edit14 = local_orig14[:-orig_extension_length] + "edit.tif" 

# upload_name1 = "maize2_tot_days"
# upload_name2 = "maize2_plant_start"
# upload_name3 = "maize_tot_days"
# upload_name4 = "maize_plant_start"
# upload_name5 = "rice2_tot_days"
# upload_name6 = "rice2_plant_start"
# upload_name7 = "rice_tot_days"
# upload_name8 = "rice_plant_start"
# upload_name9 = "soybeans_tot_days"
# upload_name10 = "soybeans_plant_start"
# upload_name11 = "wheat_tot_days"
# upload_name12 = "wheat_plant_start"
# upload_name13 = "wheat_winter_tot_days"
# upload_name14 = "wheat_winter_plant_start"

upload_name1 = "maize2"
upload_name2 = "maize2"
upload_name3 = "maize"
upload_name4 = "maize"
upload_name5 = "rice2"
upload_name6 = "rice2"
upload_name7 = "rice"
upload_name8 = "rice"
upload_name9 = "soybeans"
upload_name10 = "soybean"
upload_name11 = "wheat"
upload_name12 = "wheat"
upload_name13 = "wheat_winter"
upload_name14 = "wheat"
# -

# s3

# +
s3_upload = boto3.client("s3")
s3_download = boto3.resource("s3")

s3_bucket = "wri-public-data"
s3_folder = "resourcewatch/raster/foo_027_yield_gaps_for_major_crops/"

s3_key_orig1 = s3_folder + upload_name1 + ".tif" 
s3_key_edit1 = s3_key_orig1[0:-4] + "_edit.tif"

s3_key_orig2 = s3_folder + upload_name2 + ".tif" 
s3_key_edit2 = s3_key_orig2[0:-4] + "_edit.tif"

s3_key_orig3 = s3_folder + upload_name3 + ".tif" 
s3_key_edit3 = s3_key_orig3[0:-4] + "_edit.tif"

s3_key_orig4 = s3_folder + upload_name4 + ".tif" 
s3_key_edit4 = s3_key_orig4[0:-4] + "_edit.tif"

s3_key_orig5 = s3_folder + upload_name5 + ".tif" 
s3_key_edit5 = s3_key_orig5[0:-4] + "_edit.tif"

s3_key_orig6= s3_folder + upload_name6 + ".tif" 
s3_key_edit6 = s3_key_orig6[0:-4] + "_edit.tif"

s3_key_orig7 = s3_folder + upload_name7 + ".tif" 
s3_key_edit7 = s3_key_orig7[0:-4] + "_edit.tif"

s3_key_orig8 = s3_folder + upload_name8 + ".tif" 
s3_key_edit8 = s3_key_orig8[0:-4] + "_edit.tif"

s3_key_orig9 = s3_folder + upload_name9 + ".tif" 
s3_key_edit9 = s3_key_orig9[0:-4] + "_edit.tif"

s3_key_orig10 = s3_folder + upload_name10 + ".tif" 
s3_key_edit10 = s3_key_orig10[0:-4] + "_edit.tif"

s3_key_orig11 = s3_folder + upload_name11 + ".tif" 
s3_key_edit11 = s3_key_orig11[0:-4] + "_edit.tif"

s3_key_orig12 = s3_folder + upload_name12 + ".tif" 
s3_key_edit12 = s3_key_orig12[0:-4] + "_edit.tif"

s3_key_orig13 = s3_folder + upload_name13 + ".tif" 
s3_key_edit13 = s3_key_orig13[0:-4] + "_edit.tif"

s3_key_orig14= s3_folder + upload_name14 + ".tif" 
s3_key_edit14 = s3_key_orig14[0:-4] + "_edit.tif"

class ProgressPercentage(object):
        def __init__(self, filename):
            self._filename = filename
            self._size = float(os.path.getsize(filename))
            self._seen_so_far = 0
            self._lock = threading.Lock()

        def __call__(self, bytes_amount):
            # To simplify we'll assume this is hooked up
            # to a single filename.
            with self._lock:
                self._seen_so_far += bytes_amount
                percentage = (self._seen_so_far / self._size) * 100
                sys.stdout.write("\r%s  %s / %s  (%.2f%%)"%(
                        self._filename, self._seen_so_far, self._size,
                        percentage))
                sys.stdout.flush()


# -

# band_ids = [upload_name1, upload_name2, upload_name3, upload_name4, upload_name5, upload_name6, upload_name7, upload_name8, upload_name9, upload_name10, upload_name11, upload_name12, upload_name13, upload_name14], 
band_ids = [upload_name1, upload_name3, upload_name5, upload_name7, upload_name9, upload_name11, upload_name13], 
merge_name = "foo_004_crop_calendar_total_day_merge.tif"
s3_key_merge = s3_folder + merge_name


# merge_files = [local_edit1, local_edit2, local_edit3, local_edit4, local_edit5, local_edit6, local_edit7, local_edit8, local_edit9, local_edit10, local_edit11, local_edit12, local_edit13, local_edit14]
merge_files = [local_edit1, local_edit3, local_edit5, local_edit7, local_edit9, local_edit11, local_edit13]
tmp_merge = local_folder + merge_name


# Use rasterio to reproject and compress

files = [local_orig1, local_orig2, local_orig3, local_orig4, local_orig5, local_orig6, local_orig7, local_orig8]
for file in files:
    with rio.open(file, 'r') as src:
        profile = src.profile
        print(profile)


# Note - this is the core of Vizz's netcdf2tif function
def convert_asc_to_tif(orig_name, edit_name):

    with rio.open(orig_name, 'r') as src:
        # This assumes data is readable by rasterio
        # May need to open instead with netcdf4.Dataset, for example

        data = src.read()[0]

        rows = data.shape[0]
        columns = data.shape[1]

        print(rows)
        print(columns)

        # Latitude bounds
        south_lat = -90
        north_lat = 90

        # Longitude bounds
        west_lon = -180
        east_lon = 180

        transform = rio.transform.from_bounds(west_lon, south_lat, east_lon, north_lat, columns, rows)

        # Profile

        no_data_val = 0
        target_projection = 'EPSG:4326'
        target_data_type = np.float64

        profile = {
            'driver':'GTiff', 
            'height':rows, 
            'width':columns, 
            'count':1, 
            'dtype':target_data_type, 
            'crs':target_projection, 
            'transform':transform, 
            'compress':'lzw', 
            'nodata': no_data_val
        }

        with rio.open(edit_name, "w", **profile) as dst:
            dst.write(data.astype(profile["dtype"]), 1)


convert_asc_to_tif(local_orig1, local_edit1)
# convert_asc_to_tif(local_orig2, local_edit2)
convert_asc_to_tif(local_orig3, local_edit3)
# convert_asc_to_tif(local_orig4, local_edit4)
convert_asc_to_tif(local_orig5, local_edit5)
# convert_asc_to_tif(local_orig6, local_edit6)
convert_asc_to_tif(local_orig7, local_edit7)
# convert_asc_to_tif(local_orig8, local_edit8)
convert_asc_to_tif(local_orig9, local_edit9)
# convert_asc_to_tif(local_orig10, local_edit10)
convert_asc_to_tif(local_orig11, local_edit11)
# convert_asc_to_tif(local_orig12, local_edit12)
convert_asc_to_tif(local_orig13, local_edit13)
# convert_asc_to_tif(local_orig14, local_edit14)


# +
with rio.open(merge_files[0]) as src:
    kwargs = src.profile
    
kwargs.update(
    count=len(merge_files)
)
    
with rio.open(tmp_merge, 'w', **kwargs) as dst:
    for idx, file in enumerate(merge_files):
        print(idx)
        with rio.open(file) as src:

            band = idx+1
            windows = src.block_windows()

            for win_id, window in windows:
                src_data = src.read(1, window=window)
                dst.write_band(band, src_data, window=window)

# -

# Upload orig and edit files to s3

# +
# Original
s3_upload.upload_file(local_orig1, s3_bucket, s3_key_orig1,
                         Callback=ProgressPercentage(local_orig1))
s3_upload.upload_file(local_orig3, s3_bucket, s3_key_orig3,
                         Callback=ProgressPercentage(local_orig3))
s3_upload.upload_file(local_orig5, s3_bucket, s3_key_orig5,
                         Callback=ProgressPercentage(local_orig5))
s3_upload.upload_file(local_orig7, s3_bucket, s3_key_orig7,
                         Callback=ProgressPercentage(local_orig7))
s3_upload.upload_file(local_orig9, s3_bucket, s3_key_orig9,
                         Callback=ProgressPercentage(local_orig9))
s3_upload.upload_file(local_orig11, s3_bucket, s3_key_orig11,
                         Callback=ProgressPercentage(local_orig11))
s3_upload.upload_file(local_orig13, s3_bucket, s3_key_orig13,
                         Callback=ProgressPercentage(local_orig13))


# Edit
s3_upload.upload_file(local_edit1, s3_bucket, s3_key_edit1,
                         Callback=ProgressPercentage(local_edit1))
s3_upload.upload_file(local_edit3, s3_bucket, s3_key_edit3,
                         Callback=ProgressPercentage(local_edit3))
s3_upload.upload_file(local_edit5, s3_bucket, s3_key_edit5,
                         Callback=ProgressPercentage(local_edit5))
s3_upload.upload_file(local_edit7, s3_bucket, s3_key_edit7,
                         Callback=ProgressPercentage(local_edit7))
s3_upload.upload_file(local_edit9, s3_bucket, s3_key_edit9,
                         Callback=ProgressPercentage(local_edit9))
s3_upload.upload_file(local_edit11, s3_bucket, s3_key_edit11,
                         Callback=ProgressPercentage(local_edit11))
s3_upload.upload_file(local_edit13, s3_bucket, s3_key_edit13,
                         Callback=ProgressPercentage(local_edit13))

# Merged 

s3_upload.upload_file(tmp_merge, s3_bucket, s3_key_merge,
           Callback=ProgressPercentage(tmp_merge))
# -

#

#

#

#
#

os.environ["Zs3_key"] = "s3://wri-public-data/" + s3_key_merge
os.environ["Zgs_key"] = "gs://resource-watch-public/" + s3_key_merge


# !gsutil cp %Zs3_key% %Zgs_key%


os.environ["asset_id"] = "users/resourcewatch/foo_004_crop_calendar_total_day"
# !earthengine upload image --asset_id=%asset_id% %Zgs_key%


os.environ["band_names"] = str(band_ids)
# !earthengine asset set -p band_names="%band_names%" %asset_id%






StarcoderdataJupyterScriptsDedupFiltered