# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Importing necessary packages TEST test
from datetime import date, timedelta
import os
import requests
import shutil
import pandas as pd
import numpy as np
import spotipy
from spotipy.oauth2 import SpotifyClientCredentials
import tqdm

# +
# Set start date and end date, with 7 days interval
start = date(2020, 1, 3)
end = date.today()
week = timedelta(days=7)

# Creating a list with dates from start to end and 7-days interval
dates_string = []
mydate = start
while mydate < end:
    dates_string.append("{date.year}-{date.month:02}-{date.day:02}".format(date=mydate))
    mydate += week

# +
URL_job_temp = []

# Creating a list with URL's for each week top 200 charts
for i in dates_string[0:-1]:
    URL_job_temp.append(f'https://spotifycharts.com/regional/dk/weekly/{i}--')
    
URL_job= ["{}{}".format(i, j) for i, j in zip(URL_job_temp, dates_string[1:])]

# + tags=[]
# Finds current directory and makes a new folder 'Spotify_files' and changes directory to 'Spotify_files'
path = os.getcwd()
os.mkdir(f'{path}/Spotify_files')
os.chdir(f'{path}/Spotify_files')

# Downloads every weeks top 200 charts as csv file and place them in the folder 'Spotify_files'.
for i in URL_job:
    url = f'{i}/download'
    r = requests.get(url, verify=False,stream=True)
    if r.status_code!=200:
        print( "Failure")
        exit()
    else:
        r.raw.decode_content = True
        with open(f'{i[45:67]}.csv', 'wb',) as f:
            shutil.copyfileobj(r.raw, f)
        print("Success")

# Changes directory back to original path
os.chdir(path)

# +
dataframes = []
# Makes a list of the downloaded csv files
for i in URL_job:
    df = pd.read_csv(f'{path}/Spotify_files/{i[45:67]}.csv', header=1)
    df['date'] = i[45:67]
    dataframes.append(df)

# Concatenates all the csv files into a dataframe
combined_csv = pd.concat(dataframes).reset_index(drop=True)

# Slices dates such that they only consist of the first date of the the week
combined_csv['date'] = combined_csv['date'].str.slice(0,10)

# Changes format to datetime
combined_csv['date'] = pd.to_datetime(combined_csv['date'])

# Makes a new variable 'uri', slightly different from URL
combined_csv['uri'] = 'spotify:track:' + combined_csv['URL'].str.slice(31,53)
# -

combined_csv

# Passes client id and client secret to spotipy
cid = 'e3da1cac4264428cbe476f7d055db765'
secret = '<KEY>'
client_credentials_manager = SpotifyClientCredentials(client_id=cid, client_secret=secret)
sp = spotipy.Spotify(client_credentials_manager = client_credentials_manager)


# + tags=[]
def get_track_features(id):
    '''Takes an uri, URL or track id and returns a list of lists with some metadata
    and its musical features''' 
    meta = sp.track(id)
    features = sp.audio_features(id)
    
    # Metadata
    name = meta['name']
    release_date = meta['album']['release_date']
    length = meta['duration_ms']
    
    # Musical features
    acousticness = features[0]['acousticness']
    danceability = features[0]['danceability']
    energy = features[0]['energy']
    instrumentalness = features[0]['instrumentalness']
    liveness = features[0]['liveness']
    loudness = features[0]['loudness']
    speechiness = features[0]['speechiness']
    valence = features[0]['valence']
    tempo = features[0]['tempo']
    time_signature = features[0]['time_signature']
    URL = features[0]['uri']
    track = [name, release_date ,length, danceability, acousticness, energy, instrumentalness, liveness, loudness, speechiness, valence, tempo, time_signature, URL]
    return track

# Running get_track_features function on all tracks
tracks_features = []
for i in tqdm.tqdm(combined_csv['URL']):
    tracks_features.append(get_track_features(i))

# +
# Converts list of all track features to a dataframe
df = pd.DataFrame(tracks_features, columns = ['name','release_date', 'length', 'danceability', 'acousticness', 'energy', 'instrumentalness', 'liveness', 'loudness', 'speechiness', 'valence','tempo', 'time_signature','uri'])

# Merges dataframe with weekly top 200 charts with dataframe containing all track features. 
# Merge is made on index since the order of the dataframes are similar
combined_csv_joined = pd.merge(combined_csv, df, left_index=True, right_index=True)\
                        .drop('name', axis=1)\
                        .to_csv('songs_and_audio_features.csv')

# -

df2 = pd.read_csv('songs_and_audio_features.csv')
df2

musical_variables =  ["danceability", "acousticness", "energy", "instrumentalness", "liveness","speechiness", "valence"]

df_split2 = df2[["date", "danceability", "acousticness", "energy", "instrumentalness", "liveness","speechiness", "valence"]]
df_group = df_split2.groupby(by = 'date').mean()
df_group

# +
import matplotlib.pyplot as plt
import pandas as pd
from math import pi
plt.style.use("seaborn")

# number of variable
categories=musical_variables
N = len(categories)
 
# We are going to plot the first line of the data frame.
# But we need to repeat the first value to close the circular graph:
#values=df_group.loc['2020-01-03'].values.flatten().tolist()
#values += values[:1]
#values

# What will be the angle of each axis in the plot? (we divide the plot / number of variable)
angles = [n / float(N) * 2 * pi for n in range(N)]
angles += angles[:1]
 
# Initialise the spider plot
fig = plt.figure(figsize=(10,10))
ax = plt.subplot(111, polar=True)
 
# Draw one axe per variable + add labels labels yet
plt.xticks(angles[:-1], categories, color='black', size=20)

# Draw ylabels
ax.set_rlabel_position(0)
plt.yticks([0.25,0.5,0.75], ["0.25","0.5","0.75"], color="black", size=20)
plt.ylim(0,0.75)
 
# Plot data
ax.plot(angles, values, linewidth=1, linestyle='solid')
 
# Fill area
#ax.fill(angles, values, 'b', alpha=0.1)

values=df_group.loc['2020-03-06'].values.flatten().tolist()
values += values[:1]
ax.plot(angles, values, linewidth=1, linestyle='solid', label="2020-03-06")
#ax.fill(angles, values, 'b', alpha=0.1)
 
# Ind2
values=df_group.loc['2020-03-20'].values.flatten().tolist()
values += values[:1]
ax.plot(angles, values, linewidth=1, linestyle='solid', label="2020-03-20")
#ax.fill(angles, values, 'r', alpha=0.1)

# Ind2
values=df_group.loc['2020-04-10'].values.flatten().tolist()
values += values[:1]
ax.plot(angles, values, linewidth=1, linestyle='solid', label="2020-04-10")
#ax.fill(angles, values, 'r', alpha=0.1)

plt.legend(loc='upper right', bbox_to_anchor=(0.1,0.1))
# -

df2 = df2.drop(['URL','uri_y','Unnamed: 0'], axis=1).rename(columns={'uri_x':'URI'})

df200 = df2.copy
df100 = (df2.loc[df2['Position']<=100]).reset_index(drop=True)
df50 = (df2.loc[df2['Position']<=50]).reset_index(drop=True)
df10 = (df2.loc[df2['Position']<=10]).reset_index(drop=True)

df10_split = df10[["danceability", "acousticness", "energy", "instrumentalness", "liveness","speechiness", "valence"]]
df50_split = df50[["danceability", "acousticness", "energy", "instrumentalness", "liveness","speechiness", "valence"]]
df100_split = df100[["danceability", "acousticness", "energy", "instrumentalness", "liveness","speechiness", "valence"]]
df200_split = df2[["danceability", "acousticness", "energy", "instrumentalness", "liveness","speechiness", "valence"]]

# +
import matplotlib.pyplot as plt
import pandas as pd
from math import pi
plt.style.use("seaborn")
 
# number of variable
categories=musical_variables
N = len(categories)
 
# What will be the angle of each axis in the plot? (we divide the plot / number of variable)
angles = [n / float(N) * 2 * pi for n in range(N)]
angles += angles[:1]
 
# Initialise the spider plot
fig = plt.figure(figsize=(10,10))
ax = plt.subplot(111, polar=True)
 
# Draw one axe per variable + add labels labels yet
plt.xticks(angles[:-1], categories, color='black', size=20)

# Draw ylabels
ax.set_rlabel_position(324)
plt.yticks([0.25,0.5,0.75], ["0.25","0.5","0.75"], color="black", size=20)
plt.ylim(0,0.80)
 
# Plot data
ax.plot(angles, values, linewidth=1, linestyle='solid')
 
# Fill area
#ax.fill(angles, values, 'b', alpha=0.1)

values=df10_split.mean().values.flatten().tolist()
values += values[:1]
ax.plot(angles, values, linewidth=1, linestyle='solid', label="Top 10")
#ax.fill(angles, values, 'b', alpha=0.1)
 
values=df50_split.mean().values.flatten().tolist()
values += values[:1]
ax.plot(angles, values, linewidth=1, linestyle='solid', label="Top 50")
#ax.fill(angles, values, 'r', alpha=0.1)

values=df100_split.mean().values.flatten().tolist()
values += values[:1]
ax.plot(angles, values, linewidth=1, linestyle='solid', label="Top 100")
#ax.fill(angles, values, 'r', alpha=0.1)

values=df200_split.mean().values.flatten().tolist()
values += values[:1]
ax.plot(angles, values, linewidth=1, linestyle='solid', label="Top 200")
#ax.fill(angles, values, 'r', alpha=0.1)

plt.legend(loc='upper right', bbox_to_anchor=(0.1,0.1))
# -

df = pd.read_csv('songs_and_audio_features.csv')\
       .drop(['URL','uri_y','Unnamed: 0'], axis=1)\
       .rename(columns={'uri_x': 'URI'})
df_top10_temp = (df.loc[df['Position'] <= 10])\
              .reset_index()\
              .drop('index', axis=1)
df_top50_temp = (df.loc[df['Position'] <= 50])\
              .reset_index()\
              .drop('index', axis=1)
df_top100_temp = (df.loc[df['Position'] <= 100])\
              .reset_index()\
              .drop('index', axis=1)
df_top200_temp = df.copy()
list_of_dfs = [df_top10_temp, df_top50_temp, df_top100_temp, df_top200_temp]
list_of_temps = []
final_list =[]

# +
for i in list_of_dfs:
    i['weeks_in_top'] = pd.Series(np.random.randn(len(i))) #, index=i.index)
    temp = i.groupby('URI')['weeks_in_top'].count()
    list_of_temps.append(temp)

    
for i, j in zip (list_of_dfs,list_of_temps):
    final_list.append(pd.merge(i, j,on ='URI',how ='inner')\
              .drop('weeks_in_top_x', axis=1)\
              .rename(columns={'weeks_in_top_y': 'weeks_in_top'})\
              .drop_duplicates('URI')\
              .reset_index()\
              .drop('index', axis=1))

# -

df_top10 = final_list[0]
df_top50 = final_list[1]
df_top100 = final_list[2]
df_top200 = final_list[3]


df_top10





'''# Definining functions 
def get_artist(name):
    # Returns the artist object with respective data
    results = sp.search(q='artist:' + name, type='artist')
    items = results['artists']['items']
    if len(items) > 0:
        return items[0]
    else:
        return None
def get_artist_albums_ids(artist):
    # Given an artist object, returns a list of with each of the album's ids
    albums_ids = []
    for i in range (len(sp.artist_albums(artist['id'], album_type='album',limit=50)['items'])):
        albums_ids.append(sp.artist_albums(artist['id'], album_type='album',limit=50)['items'][i]['id'])
    return albums_ids
def filter_albums_ids(albums_ids):
    # Spotify has many versions of the same album. 
    # This function filters a list of albums ids and returns the most popular ones
    album_names = []    
    for album_id in albums_ids:
        album_names.append(sp.album(album_id)['name'])       
    album_pop = []    
    for album_id in albums_ids:
        album_pop.append(sp.album(album_id)['popularity'])    
    d = {'id': albums_ids,'name': album_names, 'popularity': album_pop}
    df = pd.DataFrame(data=d)    
    df = df.sort_values('popularity', ascending=False).drop_duplicates('name').sort_index()
    df = df.reset_index(drop = True)   
    return (df['id'])
def get_artist_albums_tracks_ids(albums_ids):
    # Given a list of albums ids, returns a list of ids of the albums' tracks
    tracks_ids = []
    for album_id in albums_ids:
        for i in range(len(sp.album_tracks(album_id)['items'])):
            tracks_ids.append(sp.album_tracks(album_id)['items'][i]['id'])
    return(tracks_ids)
def get_track_features(id):
    # Given a track id, returns a nested list with its musical features as described in the introduction
    meta = sp.track(id)
    features = sp.audio_features(id)
    # Meta
    name = meta['name']
    album = meta['album']['name']
    artist = meta['album']['artists'][0]['name']
    release_date = meta['album']['release_date']
    length = meta['duration_ms']
    popularity = meta['popularity']
    # Features
    acousticness = features[0]['acousticness']
    danceability = features[0]['danceability']
    energy = features[0]['energy']
    instrumentalness = features[0]['instrumentalness']
    liveness = features[0]['liveness']
    loudness = features[0]['loudness']
    speechiness = features[0]['speechiness']
    valence = features[0]['valence']
    tempo = features[0]['tempo']
    time_signature = features[0]['time_signature']
    track = [name, album, artist, release_date, length, popularity, danceability, acousticness, energy, instrumentalness, liveness, loudness, speechiness, valence, tempo, time_signature]
    return track
def get_tracks_features(tracks_ids):
    # given a list of ids of tracks, returns a list with their features
    tracks = []
    for i in range(0, len(tracks_ids)):
        track = get_track_features(tracks_ids[i])
        tracks.append(track)
    return(tracks)
def tracks_features_to_csv(tracks_features, csv_title):
    # transforms the list of track features into a csv file
    df = pd.DataFrame(tracks_features, columns = ['name', 'album', 'artist', 'release_date', 'length', 'popularity', 'danceability', 'acousticness', 'energy', 'instrumentalness', 'liveness', 'loudness', 'speechiness', 'valence','tempo', 'time_signature'])
    df.to_csv(csv_title + ".csv", sep = ',')
def get_discography(artist_name):
    # a function that returns the discography of an artist given its name (string)
    # do not use, it is quite unreliable since the discography is not filtered
    tracks_features = get_tracks_features(get_artist_albums_tracks_ids(filter_albums_ids(get_artist_albums_ids(get_artist(artist_name)))))
    return tracks_features_to_csv(tracks_features, artist_name)
def get_playlist_track_ids(user, playlist_id):
    # initially I wanted to use a playlist to get an artist's discography
    # however, finding a good playlist proved to be quite difficult
    ids = []
    playlist = sp.user_playlist(user, playlist_id)
    for item in playlist['tracks']['items']:
        track = item['track']
        ids.append(track['id'])
    return ids'''





StarcoderdataJupyterScriptsDedupFiltered