<jupyter_start><jupyter_text>Import libraries<jupyter_code>from scraper import *
import pandas as pd
import isthmuslib as isli
from tqdm.auto import tqdm
import pathlib
import numpy as np<jupyter_output><empty_output><jupyter_text>Setup params<jupyter_code>cache_path: str = str(pathlib.Path.cwd() / '..' / 'data' / '2021_12_29_data.pickle')
style: isli.Style = isli.Style(watermark_text="             Isthmus - github.com/mitchellpkt/firo-forensics",
                               watermark_fontsize=12, grid=False)<jupyter_output><empty_output><jupyter_text>Initialize - load from cache or scrape<jupyter_code>if cache_path:
    df: pd.DataFrame = pd.read_pickle(cache_path).drop(columns='height')
else:
    for height in tqdm(range(437410, 438059)):
        df = df.append(get_metadata(height), ignore_index=True)
    df: pd.DataFrame = pd.DataFrame({'height': []})
    df.to_pickle(cache_path)
df.head()  # See the result
df.describe()<jupyter_output><empty_output><jupyter_text>Visualizations<jupyter_code>marker_line = lambda: isli.plt.axvline(437916, color='b', linestyle=':')
title: str = f'Firo on-chain data \n{isli.human_time(min(df.timestamp_unix), formatter="%Y-%m-%d %H:%M", include_timezone=False)} - {isli.human_time(max(df.timestamp_unix), formatter="%Y-%m-%d %H:%M")}\n heights: {min(df.block_height):.0f} - {max(df.block_height):.0f}'
isli.scatter(df, 'block_height', 'timestamp_unix', xlabel='block_height', ylabel='timestamp',
             title=f"{title}\n", style=style, legend_strings=[]);
marker_line();
isli.hist((diffs := [(x[0] - x[1]) / 60 for x in zip(df.timestamp_unix[1:], df.timestamp_unix)]), title=title, bins=40,
          xlabel='block discovery time (min)', style=style.override({'watermark_placement': (-0.05, 0.95)}));
isli.scatter(df.timestamp_unix[1:], diffs, ylabel='block discovery time (min)', xlabel='timestamp', title=title,
             style=style.override({'watermark_placement': (-0.05, 0.025)}), log_axes='y', );<jupyter_output><empty_output><jupyter_text>From node records Import and process the node records<jupyter_code>threshold = 500_000
df_node_raw: pd.DataFrame = pd.read_csv(
    pathlib.Path.cwd() / '..' / 'data_local_only' / 'debug-logs' / 'debug01_blocks.csv')
df_node: pd.DataFrame = df_node_raw.loc[df_node_raw.loc[:, 'height'] < threshold, :]
df_node.sort_values(by='height', ascending=True, inplace=True)
df_node.drop_duplicates(inplace=True)
df_node.reset_index(inplace=True, drop=True)
df_node['timestamp'] = [isli.machine_time(x) for x in df_node.timestamp_human]
df_node<jupyter_output>/home/mglrnd/Projects/GitHub/firo-forensics/env/lib/python3.8/site-packages/pandas/util/_decorators.py:311: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  return func(*args, **kwargs)
/tmp/ipykernel_75997/3413099976.py:8: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df_node['timestamp'] = [isli.machine_time(x) for x in df_node.timestamp_human]<jupyter_text>Peek at data qualityThese next few cells are boring, we just identify a duplicate record and remove it<jupyter_code>isli.basis_quality_plots(df_node.height)
marker_line()
df_node[df_node.height == 437844]
df_node.drop(index=[df_node[df_node.height == 437844].index[0]], inplace=True)
df_node.reset_index(inplace=True, drop=True)
df_node
isli.basis_quality_plots(df_node.height);<jupyter_output><empty_output><jupyter_text>Review the data from the node<jupyter_code>isli.scatter(df_node, 'height', 'timestamp', legend_strings='', title='Logs from node 1');
marker_line();<jupyter_output><empty_output><jupyter_text>Join the node data to the chain metadata<jupyter_code>df['height'] = df.block_height.astype(int)
dfj: pd.DataFrame = pd.DataFrame()
for h in df.height:
    if h in list(df_node.height):
        try:
            dfj = dfj.append({
                'height': h,
                'timestamp_node': int(df_node[df_node['height'] == h].timestamp) - 7 * 60 * 60,  # time zone adjustment
                'timestamp_block': int(df[df['block_height'] == h].timestamp_unix)}, ignore_index=True)
        except Exception as e:
            print(f"Exception: {e}")
dfj['diff'] = dfj['timestamp_node'] - dfj['timestamp_block']
dfj<jupyter_output><empty_output><jupyter_text>Compare the timestamps<jupyter_code>isli.scatter(dfj, 'timestamp_node', 'timestamp_block',
             title=(
                 t := f'Comparing block header timestamp to node records\n{isli.human_time(min(dfj.timestamp_node), formatter="%Y-%m-%d %H:%M", include_timezone=False)} - {isli.human_time(max(dfj.timestamp_node), formatter="%Y-%m-%d %H:%M")}\n'));
isli.plt.axvline(x=isli.machine_time('2021-12-30T00:10:00Z'), color='b', linestyle=':');
isli.hist(dfj['diff'], xlabel='(node receipt timestamp) - (block timestamp)', title=t, bins=100);<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup