<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code

Enter your authorization code:
··········
Mounted at /content/drive<jupyter_text>imports<jupyter_code>#For the system
import os

#Manage of time
from datetime import datetime
from pytz import timezone
import time

#Manage of files
import pandas as pd
import csv

#scrap
from bs4 import BeautifulSoup
from openpyxl.workbook import Workbook
import requests
%cd 'drive/My Drive/Colab Notebooks/4SS/4SS_db'
!ls
def centered_len(max_len):
    def centered_string(string):
        if type(string) != 'string':
            string = str(string)
            
        rest = max_len - len(string)
        n_spaces = int(rest/2)

        string_spaces = ' ' * n_spaces
        
        centered_string = string_spaces + string + string_spaces

        if len(centered_string)<max_len:
            centered_string = ' ' + centered_string

        return centered_string
    return centered_string<jupyter_output><empty_output><jupyter_text>Identificador de series extras Se usa parquet por que es más rápido para cargar DF<jupyter_code># %cd '..'
%cd 'br/parquet'
!ls

databases = os.listdir()
n_databases = len(databases)
df = [None]*n_databases
n_headers = [None]*n_databases
headers = [None]*n_databases
largest_len_db = 0
n = 0


for db in databases:
    dataframe = pd.read_parquet(db)
    df[n] = db
    headers[n] = list(dataframe.columns)
    n_headers[n] = len(headers[n])
    if len(db) > largest_len_db:
        largest_len_db = len(db)
    n = n + 1
        
# print(largest_df)
centered_db = centered_len(largest_len_db)
centered_header = centered_db('masterdbs')
dashes = '-'*largest_len_db
print(f' {centered_header} | len | headers')
print(f' {dashes} | --- | ')


for d in range(len(df)):

    print(f'{centered_db(df[d])}  | {n_headers[d]}  | {headers[d]}')

    if d < (len(df)-1):
        if n_headers[d+1]>11:
            print(f' {dashes} | *** | ------')


    if n_headers[d]>11:
        edition = pd.read_parquet(df[d])
        edition['time'].fillna(edition['Date'], inplace=True)
        edition.drop(['Date'],axis=1, inplace=True)
        
        file_name = f'{df[d][:(len(df[d])-8)]}'

        csv_file = f'{file_name}.csv'
        excel_file = f'{file_name}.xlsx'
        parquet_file = f'{file_name}.parquet'

        edition.to_csv(csv_file,sep="|")
        edition.to_excel(excel_file, index=False)
        edition.to_parquet(parquet_file)

        new_headers = list(edition.columns)
        len_new_headers = len(new_headers)
        print('check:')
        print(f'{centered_db(df[d])}  | {len_new_headers}  | {new_headers}')

        print(f' {dashes} | *** | ------')
def extract_inter_rank(rank):
    inter_rank = int(rank[1:len(rank)])
    return inter_rank
ranks = edition['Rank'].str.extract(r'(\d+)')
ranks
edition = pd.read_parquet('br-master_db_music.parquet')
edition.dtypes
edition = pd.read_csv('master_db_digital-text.csv', sep="|")
right_serie = edition['Date']
right_serie
left_serie = edition['time']
right_serie = edition['Date']
# pd.merge(left_serie, right_serie, how="outer")
right_serie.fillna(left_serie)
edition['time'].fillna(edition['Date'], inplace=True)
edition.drop(['Date'],axis=1, inplace=True)
edition
edition.to_csv("master_db_digital-text_3.csv",sep="|")
edition.to_excel("master_db_digital-text_3.xlsx")<jupyter_output><empty_output><jupyter_text>Laboratorio<jupyter_code># #por pais en paises
# initial_time = datetime.now()
# print(initial_time)


# country_counter = 0
# print(' Added date/time | With date/time | csv_files | country |       Category     ')

# centered_missing = centered_len(len('Missing date/time'))
# centered_with = centered_len(len('With date/time'))
# centered_csv = centered_len(len('csv_files'))
# centered_country = centered_len(len('country'))
# #largest category name: musical-instruments
# centered_category = centered_len(len('musical-instruments'))

# total_csv_files = 0
# total_dirs = 0
# total_changed = 0
# error_count = 0
# errors = []

# for country in countries:
#     country_entries = os.listdir(country)
#     if country_counter > 0:
#         print(' ----------------- | -------------- | --------- | ------- | -------------------')
#     #por carpeta en paises
#     for entry in country_entries:
#         files_ok = False
#         try:
#             total_dirs = total_dirs + 1
#             #Archivos en carpetas
#             files_dir = f'{country}/{entry}'
#             folder_files = os.listdir(files_dir)
#             files_ok = True
#         except:
#             total_dirs = total_dirs + 1
#             pass
        
#         if files_ok == True:
#             if folder_files[:2] != country:
#                 csv_counter = 0
#                 with_dt = 0
#                 changed_dt = 0        

#                 for file_content in folder_files:
#                     total_dirs = total_dirs + 1    
#                     #Archivo csv
#                     if file_content[-3:] == 'csv':
#                         csv_counter = csv_counter + 1
#                         complete_file = f'{country}/{entry}/{file_content}'
                        
#                         with open(complete_file, mode="r", encoding='utf-8') as csv_file:
#                             csv_reader = csv.reader(csv_file, delimiter='|')
                            
#                             try:
#                                 header = next(csv_reader)
#                                 if header[0] == 'time' or header[0] == 'Date':
#                                     with_dt = with_dt + 1
#                                 else:
#                                     rename_folders_files(complete_file, ':', '_')
#                                     try:
#                                         add_date(complete_file)
#                                     except: 
#                                         pass
#                                     changed_dt = changed_dt + 1
#                                     total_changed = total_changed + 1

#                                 if header[0] == header[1]:
#                                     error_count = error_count + 1
#                                     errors.append(complete_file)
#                                 elif header[0] == header[1][:4]:
#                                     error_count = error_count + 1
#                                     errors.append(complete_file)

#                                 csv_file.close()
#                                 #File is empty
#                             except StopIteration:
#                                 csv_file.close()
                                

#                 print(f' {centered_missing(changed_dt)}', end=' | ')
#                 print(centered_with(with_dt), end=' | ')
#                 print(centered_csv(csv_counter), end=' | ')
#                 print(centered_country(country), end=' | ')
#                 print(centered_category(entry))
#                 total_csv_files = total_csv_files + csv_counter
                


#     country_counter = country_counter+1

# final_time = datetime.now()
# time_elapsed = final_time - initial_time
# print(f'\nPasaron {time_elapsed.total_seconds()} segundos')
# print(final_time)

# print(f'\nChanged to check: {total_changed}')
# print(f'Total of csv files = {total_csv_files}')
# print(f'Total dirs: {total_dirs}')
# print(f'\nErrors: {error_count}')
# print(errors)<jupyter_output><empty_output><jupyter_text>Experimento para unir bases con Pandas<jupyter_code>pd.options.display.max_rows = 300

country = 'mx'
folder = 'handmade'

entry = 'mx/handmade/mx-test_handmade_2020-07-31 16_00.csv'

master_db_csv = f'{country}/csv/master_db_{folder}.csv'
master_db_excel = f'{country}/excel/master_db_{folder}.xlsx'

add_1 = 'mx/handmade/mx-test_handmade_2020-07-31 16_28.csv'
add_2 = 'mx/handmade/mx-test_handmade_2020-07-31 16_49.csv'
add_3 = 'mx/handmade/mx-test_handmade_2020-07-31 17_10.csv'

to_add = [add_1, add_2, add_3]
main_df = pd.read_csv(entry, sep='|')

for csv_file in to_add:
    add_df = pd.read_csv(csv_file, sep='|')
    main_df = pd.concat([main_df, add_df])

main_df.reset_index(drop=True, inplace=True)
main_df.to_csv(master_db_csv, sep="|", index=False)
main_df.to_excel(master_db_excel, index=False)
main_df<jupyter_output><empty_output><jupyter_text>Centered Len<jupyter_code>def centered_len(max_len):
    def centered_string(string):
        if type(string) != 'string':
            string = str(string)
            
        rest = max_len - len(string)
        n_spaces = int(rest/2)

        string_spaces = ' ' * n_spaces
        
        centered_string = string_spaces + string + string_spaces

        if len(centered_string)<max_len:
            centered_string = ' ' + centered_string

        return centered_string
    return centered_string
from IPython.display import Javascript
display(Javascript("google.colab.output.resizeIframeToContent()"))
def add_box(add_csv, main_csv):
    with open(add_csv, mode="r", encoding="utf-8") as box_file:
        box_reader = csv.reader(box_file, delimiter='|')
        header = next(box_reader)
        box_data = [line for line in box_reader]
        # print(box_data)

    # # add row to CSV file
    with open(main_csv, "a", encoding="utf-8") as db_file:
        db_writer = csv.writer(db_file, delimiter='|')
        db_writer.writerow('\n')
        db_writer.writerows(box_data)<jupyter_output><empty_output><jupyter_text>DB Unifier<jupyter_code>%cd 'testing'
!ls
initial_time = datetime.now()
print(initial_time)
country_counter = 0
print(' country |       Category      | csv_files |  Created_Master_DB  | time')

centered_csv = centered_len(len('csv_files'))
centered_country = centered_len(len('country'))
#largest category name: musical-instruments
centered_category = centered_len(len('musical-instruments'))
centered_file = centered_len(len('Created_Master_DB'))

total_files = 0
total_dirs = 0
# total_folders = 0
total_bases = 0
error_count = 0
total_csv_files = 0
errors = []
error_message = []

countries = ['br']

for country in countries:
    # total_folders = total_folders + 1
    country_entries = os.listdir(country)
    if country_counter > 0:
        print(' ------- | ------------------- | --------- | ------------------- |')

    #por carpeta en paises
    for entry in country_entries:
        is_folder = False
        if entry == 'csv' or entry == 'excel' or entry == 'parquet':
            pass
        else:
            try:
                # total_folders = total_folders + 1
                #Archivos en carpetas
                files_dir = f'{country}/{entry}'
                folder_files = os.listdir(files_dir)
                is_folder = True
            except:
                pass
            
            if is_folder == True:
                # total_folders = total_folders + 1

                #Archivos en carpetas
                folder_files = os.listdir(country + '/' + entry)
                if folder_files[:2] != country:
                    print(f' {centered_country(country)}', end=' | ')
                    print(centered_category(entry), end=' | ')

                    csv_counter = 0   

                    for file_content in folder_files:
                        total_dirs = total_dirs + 1    
                        #Archivo csv
                        if file_content[-3:] == 'csv':
                            
                            start_master = datetime.now()
                            #here's why csv_counter never enters as 0 in main_df
                            csv_counter = csv_counter + 1
                            complete_file = country + '/' + entry + '/' + file_content
                            
                            #For a start
                            if csv_counter == 1:
                                # print('entered main')
                                main_df = pd.read_csv(complete_file, sep='|')
                            
                            #Add the rest
                            else:
                                try:
                                    add_df = pd.read_csv(complete_file, sep='|')
                                    main_df = pd.concat([main_df, add_df])
                                    # print(f'loaded {csv_counter}')
                                except Exception as e:
                                    errors.append(complete_file)
                                    error_message.append(e)
                                    error_count = error_count + 1
                                    pass

                    print(centered_csv(csv_counter), end=' | ')
                    total_csv_files = total_csv_files + csv_counter

                    master_db_csv = f'{country}/csv/{country}-master_db_{entry}.csv'
                    master_db_excel = f'{country}/excel/{country}-master_db_{entry}.xlsx'
                    master_db_parquet = f'{country}/parquet/{country}-master_db_{entry}.parquet'

                    finish_master = datetime.now()
                    time_file_making = finish_master - start_master

                    main_df.reset_index(drop=True, inplace=True)
                    main_df.to_csv(master_db_csv, sep="|", index=False)
                    main_df.to_excel(master_db_excel, index=False)
                    main_df.to_parquet(master_db_parquet)

                    print(centered_file('csv, excel, parquet'),"| ", time_file_making.total_seconds(), " sec")
                    total_bases = total_bases + 1
                

    country_counter = country_counter + 1

final_time = datetime.now()
time_elapsed = final_time - initial_time
print(f'\nPasaron {time_elapsed.total_seconds()} segundos')
print(final_time)

print(f'\nBases creadas: {total_bases} en csv y excel')
print(f'Total of csv files = {total_csv_files}')
print(f'Total dirs: {total_dirs}')
if error_count > 0:
    print(f'Errores en archivos: {error_count}. Estos son los archivos:')
    for e in range(len(errors)):
        print(f'Message: {error_message[e]}    | file: {errors[e]} ')<jupyter_output>2020-08-12 04:48:53.964066
 country |       Category      | csv_files |  Created_Master_DB  | time
    br   |       kitchen       |    114    | csv, excel, parquet |  0.012722  sec
    br   |          hi         |     97    | csv, excel, parquet |  0.302133  sec
    br   |     electronics     |    115    | csv, excel, parquet |  0.296863  sec
    br   |        sports       |    115    | csv, excel, parquet |  0.37554  sec
    br   |    baby-products    |     98    | csv, excel, parquet |  0.29418  sec
    br   |    amazon-devices   |    116    | csv, excel, parquet |  0.17662  sec
    br   |      furniture      |     92    | csv, excel, parquet |  0.314285  sec
    br   |   lawn-and-garden   |     92    | csv, excel, parquet |  0.280246  sec
    br   |      appliances     |     92    | csv, excel, parquet |  0.17013  sec
    br   |         home        |     92    | csv, excel, parquet |  0.177683  sec
    br   |        beauty       |     92    | csv, excel, parquet |  0.265231  sec
   [...]<jupyter_text>Corrigiendo todo<jupyter_code>MX__COL_tz = 'America/Mexico_City'
timezone_MXCOL = timezone(MX__COL_tz) 
prueba = datetime.now(timezone_MXCOL)
prueba

def extract_soup(url, preview=True):
    response = requests.get(url)
    status = response.status_code

    soup = BeautifulSoup(response.text, 'lxml')

    if preview==True:
        print(soup.prettify())

    return soup, status
def top_amazon_boxes(soup):
    boxes = soup.find_all('div', attrs={'class':"a-section a-spacing-none aok-relative"})

    return boxes
def scrap_boxes(boxes, domain, dict_to_rows=False):
    
    ranks = [None]*50
    product_names = [None]*50
    image_urls = [None]*50
    product_links = [None]*50
    star_ratings = [None]*50
    reviews = [None]*50
    authors_companies = [None]*50
    editions_consoles = [None]*50
    min_prices = [None]*50
    max_prices = [None]*50
    time_log = [None]*50

    amz_mx_url = 'https://www.amazon.com.mx'
    
    n_box = 0
    for box in boxes:
        MX__COL_tz = 'America/Mexico_City'
        timezone_MXCOL = timezone(MX__COL_tz) 
        time_log[n_box] = datetime.now(timezone_MXCOL)

        rank_box = box.find_all('span', attrs={'class':'zg-badge-text'})
        products_and_image_box = box.find_all('div', attrs={'class' : 'a-section a-spacing-small'})
        product_links_box = box.find_all('a', attrs={'class' : 'a-link-normal'})
        star_ratings_box = box.find_all('span', attrs={'class' : 'a-icon-alt'})
        reviews_box = box.find_all('a', attrs={'class' : 'a-size-small a-link-normal'})
        authors_company_box = box.find_all('span', attrs={'class' : 'a-size-small a-color-base'})
        editions_console_box = box.find_all('span', attrs={'class' : 'a-size-small a-color-secondary'})
        prices_box = box.find_all('span', attrs={'class' : "p13n-sc-price"})
        
        
        ranks[n_box] = int(rank_box[0].get_text()[1:])

        #In case the element was removed (yes, it happens)
        try:
            product_names[n_box] = products_and_image_box[0].img.get('alt')
            image_urls[n_box] = products_and_image_box[0].img.get('src')
            product_links[n_box] = amz_mx_url + product_links_box[0].get('href')
        except: pass

        try:
            star_ratings[n_box] = float(star_ratings_box[0].get_text()[:3])
            reviews[n_box] = int(reviews_box[0].get_text().replace(',',''))
        except: pass
        #Individual cases
        try:
            authors_companies[n_box] = authors_company_box[0].get_text()
        except:pass
        
        try:
            editions_consoles[n_box] = editions_console_box[0].get_text()
        except:pass

        #Courrencies
        if domain == 'mx':
            coin_symbol = 1
        elif domain == 'br':
            coin_symbol = 2

        try:
            min_prices[n_box] = float(prices_box[0].get_text()[coin_symbol:].replace(',',''))
        except: pass

        try:    
            max_prices[n_box] = float(prices_box[1].get_text()[coin_symbol:].replace(',',''))
        except: pass

        n_box = n_box + 1 

    # Dictionary
    boxes_dict = {
    'time' : time_log,
    "Rank" : ranks,
    "Product Names": product_names,
    "Image urls": image_urls,
    "Product links": product_links,
    "Stars": star_ratings,
    "Reviews": reviews,
    "Authors/Company": authors_companies,
    "Edition/Console": editions_consoles,
    "Price_std_or_min" : min_prices,
    "Max_prices" : max_prices
    }

    if dict_to_rows == True:
        dict_rows = [None]*50
        for n in range(len(dict_rows)):
            dict_rows[n] = [
                boxes_dict["time"][n],
                boxes_dict["Rank"][n],
                boxes_dict["Product Names"][n],
                boxes_dict["Image urls"][n],
                boxes_dict["Product links"][n],
                boxes_dict["Stars"][n],
                boxes_dict["Reviews"][n],
                boxes_dict["Authors/Company"][n],
                boxes_dict["Edition/Console"][n],
                boxes_dict["Price_std_or_min"][n],
                boxes_dict["Max_prices"][n]]

    return boxes_dict, dict_rows<jupyter_output><empty_output><jupyter_text>Experimento con de Append con reader y writer Desde un archivo csv<jupyter_code>box_file = 'box_scraped.csv'
with open(box_file, mode="r", encoding="utf-8") as box_file:
    box_reader = csv.reader(box_file, delimiter='|')
    header = next(box_reader)
    box_data = [line for line in box_reader]
    # print(box_data)

# # add row to CSV file
database_file = 'Testof master_db_music.csv'
with open(database_file, "a", encoding="utf-8") as db_file:
    db_writer = csv.writer(db_file, delimiter='|')
    db_writer.writerow('\n')
    db_writer.writerows(box_data)

# # entry = 'mx/handmade/mx-test_handmade_2020-07-31 16_00.csv'
main_df = pd.read_csv(database_file, sep='|')
main_df<jupyter_output><empty_output><jupyter_text>Desde el scraper<jupyter_code>!ls
amz_rows = [None]*50
for n in 49:
    amz_rows[n] = [
        amz_key_top_boxes["time"][n],
        amz_key_top_boxes["Rank"][n],
        amz_key_top_boxes["Product Names"][n],
        amz_key_top_boxes["Image urls"][n],
        amz_key_top_boxes["Product links"][n],
        amz_key_top_boxes["Stars"][n],
        amz_key_top_boxes["Reviews"][n],
        amz_key_top_boxes["Authors/Company"][n],
        amz_key_top_boxes["Edition/Console"][n],
        amz_key_top_boxes["Price_std_or_min"][n],
        amz_key_top_boxes["Max_prices"][n]]
    print(amz_rows[n])
url = f'https://www.amazon.com.mx/gp/bestsellers/music/ref=zg_bs_nav_0'
            
soup, status = extract_soup(url, preview=False)
print(f'status de la solicitud: {status}')

boxes = top_amazon_boxes(soup)

amz_key_top_boxes, amz_rows = scrap_boxes(boxes, 'mx', dict_to_rows=True)

database_file = 'Testof_master_db_music.csv'
main_df = pd.read_csv(database_file, sep='|')

with open(database_file, "a", encoding="utf-8") as db_file:
    db_writer = csv.writer(db_file, delimiter='|')
    for row in amz_rows:
        db_writer.writerow(row)

main_df
def update_csv(file, rows, sep=','):
    with open(file, "a", encoding="utf-8") as csv_file:
        csv_file = csv.writer(csv_file, delimiter=sep)
        for row in rows:
            csv_file.writerow(row)
url = f'https://www.amazon.com.mx/gp/bestsellers/music/ref=zg_bs_nav_0'
            
soup, status = extract_soup(url, preview=False)
print(f'status de la solicitud: {status}')

boxes = top_amazon_boxes(soup)

amz_key_top_boxes, amz_rows = scrap_boxes(boxes, 'mx', dict_to_rows=True)

database_file = 'Testof_master_db_music.csv'
main_df = pd.read_csv(database_file, sep='|')
update_csv(database_file, amz_rows, sep='|')
main_df.to_parquet(database_file)
domain_dict = {'Mexico' : 'mx',
               'Brazil' : 'br'}
!ls
domain_path = '4SS/4SS_db/testing/{}'
date = datetime.utcnow()
date<jupyter_output><empty_output><jupyter_text>main scraper<jupyter_code>def scrap_amazon():
    MX__COL_tz = 'America/Mexico_City'
    timezone_MXCOL = timezone(MX__COL_tz) 

    log_status = [None]*45
    log_date = [None]*45
    log_domain = [None]*45
    log_category = [None]*45

    n_log = 0

    country_count = 0
    print('          Log Date          | Country |        category       | status | Loaded |')

    for country_name in domain_dict:
        country_count = country_count + 1

        if country_count > 1:
            print(' -------------------------- | ------- | --------------------- | ------ | ------ |')

        country = domain_dict[country_name]

        if country == 'mx':
            categories_dict = mx_dict

        elif country == 'br':
            categories_dict = br_dict

        for key in categories_dict:
            #Date-time
            log_date[n_log] = datetime.now(timezone_MXCOL)
            print(f' {log_date[n_log]}', end=' | ')
            
            #Country where you are at
            log_domain[n_log] = country
            print(f'   {log_domain[n_log]}  ', end=' | ')

            #Category that is scraping
            category = categories_dict[key]
            log_category[n_log] = category
            centered_category = centered_string(log_category[n_log])
            print(centered_category, end=' | ')
            
            #The main scrap
            url = f'https://www.amazon.com.{country}/gp/bestsellers/{category}/ref=zg_bs_nav_0'
            soup, status = extract_soup(url, preview=False)
            
            if status == 503:
                while status == 503:
                    time.sleep(1)
                    soup, status = extract_soup(url, preview=False)
                    log_status[n_log] = status
                    log_date[n_log] = datetime.now()

            if status ==200:
                log_status[n_log] = status
                print(f'  {log_status[n_log]} ', end=' | ')
                
                boxes = top_amazon_boxes(soup)
                amz_key_top_boxes, amz_rows = scrap_boxes(boxes, country)
                add_df = pd.DataFrame.from_dict(amz_key_top_boxes)

                #Updating csv files
                csv_file = f'/{country}-master_db_{category}.csv'
                dir_csv_testing = f'/content/drive/My Drive/Colab Notebooks/4SS/4SS_db/testing/{country}/csv/{csv_file}'
                update_csv(dir_csv_testing, amz_rows, sep='|')
                
                #Updating Parquet files
                parquet_file = f'/{country}-master_db_{category}.parquet'
                dir_parquet_maindb = f'/content/drive/My Drive/Colab Notebooks/4SS/4SS_db/testing/{country}/parquet/{parquet_file}'
                main_df = pd.read_parquet(dir_parquet_maindb)
                main_df = pd.concat([main_df, add_df])
                main_df.to_parquet(dir_parquet_maindb)

                excel_file = f'/{country}-master_db_{category}.xlsx'
                dir_excel_maindb = f'/content/drive/My Drive/Colab Notebooks/4SS/4SS_db/testing/{country}/excel/{excel_file}'
                #For the testing
                main_df.to_excel(dir_excel_maindb, index=False)

                print('  Yes  |')

            else:
                log_status[n_log] == status
                print(f'  {log_status[n_log]} ', end=' | ')
                
                log_date[n_log] = datetime.now()
                print(' +.No.+ |')

            n_log = n_log + 1

    log_dict = {
        'log_date' : log_date,
        'category' : log_category,
        'country' : log_domain,
        'status' : log_status
    }

    log_rows = [None]*45
    
    for n in range(len(dict_rows)):
        dict_rows[n] = [
            log_date[n],
            log_category[n],
            log_domain[n],
            log_status[n]]

    log_file = f'/master_db_logs.csv'
    dir_log_testing = f'/content/drive/My Drive/Colab Notebooks/4SS/4SS_db/testing/{log_file}'
    update_csv(dir_log_testing, log_rows, sep='|')

    log_df = pd.DataFrame.from_dict(log_dict)


    log_print = """
                                ----------------------
                                |   Log file loaded   |
                                ----------------------

    """
    print(log_print)<jupyter_output><empty_output>