# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python (delta)
#     language: python
#     name: delta
# ---

# +
# # !conda install python-snappy

# +
# # %matplotlib widget
# %matplotlib inline
# # %matplotlib qt 

import json

import requests
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import folium
import geopandas as gpd
from shapely.geometry import box, Polygon
from pyproj import CRS
from tqdm.notebook import tqdm
from tqdm import trange
# -

# remove folium warning
import warnings; warnings.filterwarnings('ignore', 'GeoSeries.notna', UserWarning)

# +
dates = requests.get('https://openaltimetry.org/data/icesat2/getTrackDate').json()
tracks = gpd.read_file(r"../data/icesat-2/icesat2_tracks.shp")

# NL
# extents = [5.35,53.4,6.2,53.5] 
extents = [5.8062744140625, 53.36202615413913, 6.1791229248046875, 53.5092846887053]
# zoom = 10

# DK
# extents = [8.0557, 54.8955, 8.7615, 55.5938]
# zoom = 8

# FR
extents = [-1.28608, 44.60415, -0.9908, 44.7969]
zoom = 10

polygon_geom = Polygon(zip([extents[0],extents[2],extents[2],extents[0]], [extents[1],extents[1],extents[3],extents[3]]))
polygon = gpd.GeoDataFrame(index=[0], crs=CRS('EPSG:4326'), geometry=[polygon_geom])       

# Clip data
tracks_clip = gpd.clip(tracks, polygon)
track_ids = tracks_clip.TrackId.values
# -

center = [polygon_geom.centroid.xy[1][0], polygon_geom.centroid.xy[0][0]]

m = folium.Map(center, zoom_start=zoom)
folium.GeoJson(polygon_geom).add_to(m)
folium.GeoJson(tracks_clip).add_to(m)
folium.LatLngPopup().add_to(m)
m


track_ids

# track_id, track_date
overpasses = []
for track_id in track_ids:
    for track_date in dates['track_{}'.format(track_id)].split(','):
        overpasses.append([track_id, track_date])

# +
rows = []

# download data for all track_id, track_date combinations
with tqdm(overpasses, ncols='100%') as t:
    log = lambda s: t.set_description(s); t.refresh()
    
    for track_id, track_date in t:
        log(f'Downloading data for track: {track_id}, date: {track_date}')

        # Paste the OpenAltimetry API URL for Photon here: 
        OA_API_URL = 'https://openaltimetry.org/data/api/icesat2/atl03?' \
            '&minx={}&miny={}&maxx={}&maxy={}&date={}&trackId={}' \
            '&beamName=gt3r&beamName=gt3l&beamName=gt2r&beamName=gt2l&beamName=gt1r&beamName=gt1l' \
            .format(extents[0], extents[1], extents[2], extents[3], track_date, track_id)

        # This function will request the 6 beams data using OpenAltimetry's API
        r = requests.get(OA_API_URL + '&client=jupyter')
        photon_data = r.json()

        # iterate over 6 beams
        for beam in photon_data:
            # every beam has data series with different confidences:
            #
            # {'beam_name': 'gt3r',
            #  'total_photon_count': 43383,
            #  'select_photon_count': 43383,
            #  'percentage': 100.0,
            #  'series': [{'name': 'Noise',
            #    'photon_count': 580,
            #    'data': [[55.48735236638138, 8.49802945875813, 37.546936],
            #     [55.48734604692815, 8.498028327844818, 42.50964],            
            beam_name = beam['beam_name']
            for s in beam['series']:
                # every series has name (confidence)
                series_name = s['name']
                
                for o in s['data']:
                    # add rows
                    row = {
                        'track_id': track_id,
                        'date': track_date,
                        'beam': beam_name,
                        'series': series_name,
                        'lon': round(o[1], 6),
                        'lat': round(o[0], 6),
                        'h': o[2],
                    }
            
                    rows.append(row)

# -

df = pd.DataFrame(rows)

df

df.memory_usage()

df.h.hist(bins=100)

# +
# df.to_parquet('../data/out/icesat-2-DK.parquet')
# df = pd.read_parquet('../data/out/icesat-2-DK.parquet')

# df.to_parquet('../data/out/icesat-2-NL.parquet')
# df = pd.read_parquet('../data/out/icesat-2-DK.parquet')

df.to_parquet('../data/out/icesat-2-FR.parquet')
# -

<<<<<<< LOCAL CELL DELETED >>>>>>>
df.to_parquet('../data/out/icesat-2-DK.parquet')
#df = pd.read_parquet('../data/out/icesat-2-DK.parquet')

df.columns

df.series.unique()

df.beam.unique()

track_ids = df.track_id.unique()
track_ids

[ [track_id, df[df.track_id == track_id].date.unique()] for track_id in track_ids]

# for track_id in track_ids:
#     track_df = df[df.track_id == track_id]
#     for date in track_df.date.unique():
#         track_df2 = track_df[track_df.date == date]
#         print(track_id, date, len(track_df2))
        

print(df.beam.unique())

# +
fig, ax = plt.subplots(1, 2, figsize=(20, 5))

# (df.series != 'Noise') & (df.series != 'Buffer') & 
data = df [(df.series != 'Noise') & (df.track_id == '229') & (df.date == '2020-01-10') & (df.beam == 'gt2l')]

ax[0].plot(data.h, '.', markersize=5, alpha=0.2)
ax[1].plot(df.x, df.y, 'k.', markersize=0.1, alpha=0.2)
ax[1].plot(data.x, data.y, 'r.', markersize=0.1, alpha=0.2)
plt.show()

# -



StarcoderdataJupyterScriptsDedupFiltered