adding postgres db as well as fixing API endpoints -- 'yfinance' broke

.gitignore

@@ -1,4 +1,5 @@
 secrets.json
 edgar-crawler/
 .venv/
-.env
+.env
+.streamlit/

packages.txt

@@ -0,0 +1 @@
+libpq-dev

pages/1_Fundamentals.py

@@ -6,8 +6,9 @@ import streamlit as st
 import yfinance as yf
 from datetime import timedelta
 from dotenv import load_dotenv
+import psycopg2
 
-load_dotenv()
+load_dotenv(override=True)
 
 def get_finnhub_data(example: str) -> json:
     """
@@ -96,8 +97,72 @@ def get_list_of_tickers():
           list_of_tickers.append(comp_info[i]['symbol'])
     return list_of_tickers
 
-# ---------------------------------------------------------------------------------------------- #
+def handle_json_data(data, ticker) -> tuple[list[pd.DataFrame], int]:
+
+  df_list = []
+
+  for key, val in data.items():
+    
+    if val and hasattr(val, '__getitem__') and isinstance(val[0], dict):
+      #print('IS time series data') 
+
+      temporal_df = pd.DataFrame(val)
+      temporal_df = temporal_df.rename({'v':key}, axis=1)
+      temporal_df['ticker'] = ticker
+      axis = 1
+
+      if not temporal_df.empty:
+        temporal_df = temporal_df.set_index(['period','ticker'])
+      df_list.append(temporal_df)
+      
+    elif len(val) > 0:
+        #print('not time series data') 
+        non_temporal_df = pd.DataFrame(data.items(), columns=['ticker', ticker])
+        non_temporal_df = non_temporal_df.set_index('ticker').T
+        df_list.append(non_temporal_df)
+        axis = 0
+        break
+    else:
+      print("dict key has no data.")
+
+  return df_list, axis
+
+def insert_dataframe(cursor, table_name, df):
+    """
+    Insert a DataFrame into the specified table.
+    """
+    # Prepare the INSERT statement dynamically
+    columns = df.columns
+    columns_str = ', '.join(columns)
+    #print('cols for insert', columns_str)
+    placeholders = ', '.join(['%s'] * len(columns))
+    # get primary keys for insert collision handling
+    pkey_query = f"""
+      SELECT c.column_name
+      FROM information_schema.table_constraints tc 
+      JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) 
+      JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema
+        AND tc.table_name = c.table_name AND ccu.column_name = c.column_name
+      WHERE constraint_type in ('UNIQUE', 'PRIMARY KEY') and tc.table_name = '{table_name}';
+      """
+    cursor.execute(pkey_query)
+    pkeys_tuple = cursor.fetchall()
+    pkeys_list = [col[0] for col in pkeys_tuple]
+    pkeys_str = ', '.join(pkeys_list)
+
+    insert_query = f"INSERT INTO {table_name} ({columns_str}) VALUES ({placeholders}) ON CONFLICT ({pkeys_str}) DO NOTHING"
+
+    # Ensure DataFrame has the required columns
+    df = df[columns]
+
+    # Convert DataFrame to list of tuples
+    data = list(df.itertuples(index=False, name=None))
+
+    # Execute batch insert
+    cursor.executemany(insert_query, data)
+    print('df inserted.')
 
+# ---------------------------------------------------------------------------------------------- #
 if 'tickers' not in st.session_state:
     tickers = [
         "AAPL", "MSFT", "GOOG", "NVDA", "TSLA", 
@@ -115,50 +180,85 @@ with st.form(key="selecting columns"):
     st.session_state['tickers'] = symbols
 
     if submit_button and symbols and strategy_selection == 'Value':
-        gains_data = {}
-        hash_map = {}
-        
-        for ticker in symbols:
-            # make all the API calls and capture return json
-            basic_info = get_industry(ticker) 
-            metric_data, annual_series_data, quarterly_series_data  = get_company_metrics(ticker)
-
-            # reformat all JSON returns to be flattened dictionaries 
-            roe_dict = {'roe': annual_series_data['roe'][0]['v']*100 if ('roe' in annual_series_data) and (len(annual_series_data['roe'])>1) and ('v' in annual_series_data['roe'][0]) else 0}
-            eps_dict = {'eps' :annual_series_data['eps'][0]['v'] if ('eps' in annual_series_data) and (len(annual_series_data['eps'])>1) and ('v' in annual_series_data['eps'][0]) else 0}
-            pe_dict = {'pe': annual_series_data['pe'][0]['v'] if ('pe' in annual_series_data) and (len(annual_series_data['pe'])>1) and ('v' in annual_series_data['pe'][0]) else 0}
-            ps_dict = {'ps': annual_series_data['ps'][0]['v'] if ('ps' in annual_series_data) and (len(annual_series_data['ps'])>1) and ('v' in annual_series_data['ps'][0]) else 0}
-            pb_dict = {'pb': annual_series_data['pb'][0]['v'] if ('pb' in annual_series_data) and (len(annual_series_data['pb'])>1) and ('v' in annual_series_data['pb'][0]) else 0}
-            pfcf_dict = {'pfcf': annual_series_data['pfcf'][0]['v'] if ('pfcf' in annual_series_data) and (len(annual_series_data['pfcf'])>1) and ('v' in annual_series_data['pfcf'][0]) else 0}
-
-            # merge all dictionary keys per ticker
-            combined_info = basic_info.copy()  # Make a copy of the basic info
-            combined_info = combined_info | metric_data | roe_dict | eps_dict | pe_dict | ps_dict | pb_dict | pfcf_dict
+
+      # Mapping variables to target tables. keys are dataframe names, values are (write mode, db table)
+      table_mapping = {
+          'quarterly_series_data': ('APPEND', 'quarterly_financials'),
+          'basic_info': ('OVERWRITE', 'companies')
+      }
+
+      dbparams = {
+          'dbname':"hf_fundamentals",
+          'user':"postgres",
+          'password':os.environ['postgres_pwd'],
+          'host':"localhost",
+          'port':"5432"
+      }
+
+      try:
+        with psycopg2.connect(**dbparams) as conn:
+          conn.autocommit=True
+
+          with conn.cursor() as cur:
             
-            hash_map[ticker] = combined_info
+            query_companies = "SELECT * FROM public.quarterly_financials where ticker = %s;" # works because of foreign key relation
+
+            all_data = {}
+
+            for ticker in symbols:
+              # first, check if the ticker has a record in the database
+              cur.execute(query_companies, (ticker,))
+              results = cur.fetchall()
+
+              if len(results) == 0: # if not in database, call api's and store in db
+                print("no data found in db")
+                # make all the API calls and capture return json
+                basic_info = get_industry(ticker) 
+                metric_data, annual_series_data, quarterly_series_data  = get_company_metrics(ticker)
+                
+                api_data_mapping = {'basic_info':basic_info,
+                                    #  'metric_data':metric_data, 
+                                    #'annual_series_data':annual_series_data,
+                                      'quarterly_series_data':quarterly_series_data}
+                
+                for df_name, df in api_data_mapping.items():
+                  df_main_list = []
+
+                  # creates a list of dataframes where each index corresponds to a ticker
+                  df_list, axis = handle_json_data(df, ticker)
+                  df_main = pd.concat(df_list, axis=axis) 
+                  df_main_list.append(df_main)
+              
+                  # craft a dict where key is data source and values are list of datafames for every ticker
+                  if df_name not in all_data:
+                    all_data[df_name] = df_main_list
+                  else:
+                    print('redundant.')
+
+              else:
+                print("retrieved from db.")
+                st.write(results[0])
             
-            # equity gains
-            _, div, close_price = get_equity_gains(ticker=ticker, period=1810)
-            gains_data[ticker] = [div, close_price]
-        
-
-        # Now, create a DataFrame from the hash_map
-        df_1 = pd.DataFrame.from_dict(hash_map, orient='index')[['finnhubIndustry','beta','pe','ps','pb','pfcf','eps','roe','roeTTM','dividendPerShareTTM','dividendGrowthRate5Y','epsGrowth5Y']]
-        df_2 = pd.DataFrame.from_dict(gains_data, orient='index', columns=['Recent Dividend','Price'])
-
-        df_final = df_1.join(df_2)
-        
-        # calculate additional columns 
-        df_final['1Y_SP500_growth'], _, _ = get_equity_gains(ticker= '^GSPC', period=720)
-        df_final['90_day_tbill'] = 4.06
-        df_final['dividendGrowthRate5Y'] = df_final['dividendGrowthRate5Y']/100
-        df_final['1Y_CAPM'] = df_final['90_day_tbill']/100 + df_final['beta'] * (df_final['1Y_SP500_growth'] - df_final['90_day_tbill']/100)
-        df_final['1Y_DDM'] = (df_final['Recent Dividend'] * (1+df_final['dividendGrowthRate5Y'])) / (df_final['1Y_CAPM'] - df_final['dividendGrowthRate5Y'])
-        df_final = df_final[['finnhubIndustry','Price','eps','roe','roeTTM','pe','ps','pb','pfcf','epsGrowth5Y','beta','Recent Dividend','90_day_tbill','1Y_SP500_growth','dividendPerShareTTM','dividendGrowthRate5Y','1Y_CAPM','1Y_DDM']]
-        df_final.rename({'finnhubIndustry':'Industry', 'eps':'EPS', 'roe':'ROE','pe':'P/E','ps':'P/S', 'pb':'P/B','pfcf':'P/FCF','beta':'Beta'}, inplace=True, axis=1)
-        st.write(df_final)
-
-        st.session_state['tickers'] = symbols
+            for dataframe_name, data in all_data.items():
+              
+              write_mode, target_table = table_mapping[dataframe_name]
+              print(write_mode, dataframe_name, target_table)
+
+              # combines all tickers into one dataframe for a given data source
+              df_all_tickers = pd.concat(data)
+
+              print(ticker)
+              unstacked_df = df_all_tickers.reset_index()
+
+              if 'index' in unstacked_df.columns: # hack fix for single index dataframes .. 
+                unstacked_df=unstacked_df.drop('ticker', axis=1)
+                unstacked_df.rename(columns={'index':'ticker'}, inplace = True)
+
+              insert_dataframe(cur, target_table, unstacked_df)
+
+            st.session_state['tickers'] = symbols
+      except:
+         print("You do not have an active Postgres instance running. Select 'Bypass' and continue to Portfolio Builder.")
 
     if submit_button and symbols and strategy_selection == 'Growth':
 
@@ -168,6 +268,7 @@ with st.form(key="selecting columns"):
           basic_info = get_industry(ticker) 
           metric_data, annual_series_data, quarterly_series_data  = get_company_metrics(ticker)
 
+
           # reformat all JSON returns to be flattened dictionaries 
           roe_dict = {'roe': annual_series_data['roe'][0]['v'] if ('roe' in annual_series_data) and (len(annual_series_data['roe'])>1) and ('v' in annual_series_data['roe'][0]) else 0}
           ev_dict = {'ev' :annual_series_data['ev'][0]['v'] if ('ev' in annual_series_data) and (len(annual_series_data['ev'])>1) and ('v' in annual_series_data['ev'][0]) else 0}
@@ -184,22 +285,22 @@ with st.form(key="selecting columns"):
           hash_map[ticker] = combined_info
           
           # equity gains
-          _, div, close_price = get_equity_gains(ticker=ticker, period=1810)
-          gains_data[ticker] = [div, close_price]
+          # _, div, close_price = get_equity_gains(ticker=ticker, period=1810)
+          # gains_data[ticker] = [div, close_price]
       
 
       # Now, create a DataFrame from the hash_map
       df_1 = pd.DataFrame.from_dict(hash_map, orient='index')[['finnhubIndustry','roe','marketCapitalization','ebitdPerShareAnnual','pe','ps','pb','salesPerShare','eps','epsGrowth5Y','ev','operatingMarginAnnual', 'ebitdPerShareTTM', 'ebitdaCagr5Y', 'ebitdaInterimCagr5Y']]
-      df_2 = pd.DataFrame.from_dict(gains_data, orient='index', columns=['Recent Dividend','Price'])
-      df_final = df_1.join(df_2)
+      # df_2 = pd.DataFrame.from_dict(gains_data, orient='index', columns=['Recent Dividend','Price'])
+      # df_final = df_1.join(df_2)
 
-      df_final['PE/G'] = df_final['pe'] / df_final['epsGrowth5Y']
-      df_final.rename({'finnhubIndustry':'Industry','marketCapitalization':'MarketCap','roe':'ROE', 'ev':'Enterp. Val', 'pe':'P/E', 'ps':'P/S', 'pb':'P/B', 'eps': 'EPS'}, inplace=True, axis=1)
-      st.write(df_final)
+      df_1['PE/G'] = df_1['pe'] / df_1['epsGrowth5Y']
+      df_1.rename({'finnhubIndustry':'Industry','marketCapitalization':'MarketCap','roe':'ROE', 'ev':'Enterp. Val', 'pe':'P/E', 'ps':'P/S', 'pb':'P/B', 'eps': 'EPS'}, inplace=True, axis=1)
+      st.write(df_1)
 
       st.session_state['tickers'] = symbols
     
     if submit_button and symbols and strategy_selection == 'Bypass':
-      st.write('You have selected to bypass the metrics calculation for the selected tickers.')
+      st.write('You have selected to bypass the metrics calculation for the selected tickers. Proceed to Portfolio Builder.')
       st.session_state['tickers'] = symbols
 

pages/2_Portfolio_Builder.py

@@ -115,7 +115,7 @@ with st.form(key="selecting columns"):
             obb
             .equity
             .price
-            .historical(tickers, start_date=start_date, end_date=end_date, provider="yfinance")
+            .historical(tickers, start_date=start_date, end_date=end_date, provider="fmp")
             .to_df()
             .pivot(columns="symbol", values="close")
         )

requirements.txt

@@ -1,3 +1,5 @@
+sqlalchemy
+psycopg2-binary
 openbb==4.1.4
 riskfolio-lib==5.0.1
 transformers==4.36.2
@@ -8,7 +10,7 @@ beautifulsoup4==4.12.3
 click==7.0
 cssutils==1.0.2
 numpy==1.24.4
-lxml==4.9.1
+#lxml==4.9.1
 requests>=2.31.0
 urllib3>=1.26.7
 pandas>=1.5.3
@@ -20,4 +22,4 @@ regex==2024.5.15
 yfinance==0.2.28
 python-dotenv==1.0.1
 asyncio==3.4.3
-nest_asyncio==1.6.0
+nest_asyncio==1.6.0

test.py

@@ -0,0 +1,12 @@
+import psycopg2
+
+connection = psycopg2.connect(database="hf_fundamentals", user="postgres", password="4x?S3nR", host="localhost", port=5432)
+
+cursor = connection.cursor()
+
+cursor.execute("SELECT * FROM pg_catalog.pg_tables;")
+
+# Fetch all rows from database
+record = cursor.fetchall()
+
+print("Data from Database:- ", record)