<jupyter_start><jupyter_text>load artist_qid<jupyter_code># open artist-metadata csv
filename = '../data/artist_qid_html_new.csv'
#csv_file = csv.reader(open(filename, "r"))
#df_new = pd.read_csv(filename)
df_new.to_csv(filename, index = False, header = True)
df_new.columns
filename = '../data/artist_qid_rijks.csv'
#csv_file = csv.reader(open(filename, "r"))
#df_urls = pd.read_csv(filename)
#df['URLs Rijk'] = df_urls['URLs Rijk']
#df['URLs Rijk'][288].strip('[').strip(']').strip('\'')
df_new['cloudinary artworks'][4]
# clean the formats in Rijks URLs list

ix_clean = []
for i in df_new['cloudinary artworks']:
    rix = []
    if i == '[]':
        rix = []
        ix_clean.append(rix)
    elif i!=i:
        rix = []
        ix_clean.append(rix)
    elif len(i.split(',')) == 1:
        rix = [(str(i).strip('[').strip(']').strip("'"))]
        ix_clean.append(rix)
        #print(int(str(i).strip('[').strip(']').strip("'")))
    elif len(i.split(',')) >1:
        for j in i.split(','):
            rix.append(j.strip('[').strip(']').strip("'").strip("\"").strip(" \'"))
        
        ix_clean.append(rix)
        
len(ix_clean)    
#rijks
df_new['cloudinary artworks new'] = ix_clean
df_new['cloudinary artworks new'][4]<jupyter_output><empty_output><jupyter_text>still formatting<jupyter_code># place two artworks into thumbnails
for i, j in enumerate(df_new['thumbnail']):
    if j!=j:
        #print(df_new['Name'][i],i)
        if df_new['URLs Rijk new'][i] != []:
            #df_new['thumbnail'][i] = df_new['URLs Rijk new'][i][0]
            print (df_new['thumbnail'][i])
         #   print(df['Name'][i])<jupyter_output><empty_output><jupyter_text>Download thumbnails<jupyter_code>index = 284
index
# open thumbnail, get width, height

# limit the iterator (8 per call)

for i in range(8):
    # load file
    if isnan(df_new['thumbnail'][(int(index))]): # sort nan values out
        index += 1
        if index < 10:
            filename = '../data/thumbnails/thumbn_c/thumbnailDUMMY_00' + str(index) + '.png'
        elif index < 100:
            filename = '../data/thumbnails/thumbn_c/thumbnailDUMMY_0' + str(index) + '.png'
        else:
            filename = '../data/thumbnails/thumbn_c/thumbnailDUMMY_' + str(index) + '.png'
        
    else:
        new_img = getsquared_image(df_new['thumbnail'][((index))])
    
    # filename
        if index < 10:
            filename = '../data/thumbnails/thumbn_c/thumbnail_00' + str(index) + '.png'
        elif index < 100:
            filename = '../data/thumbnails/thumbn_c/thumbnail_0' + str(index) + '.png'
        else:
            filename = '../data/thumbnails/thumbn_c/thumbnail_' + str(index) + '.png'
        index +=1
    new_img.save(filename, 'PNG')   
list_of_cloud_thumbnail_filenames = os.listdir('../data/thumbnails2_cloudinary/')
# build filename list for .startswith comparison
index2 = 0
filename_list = []
for i in range(292):
    
    if index2 < 10:
        filename = "thumbnail_00" + str(index2) 
    elif index2 < 100:
        filename = "thumbnail_0" + str(index2)
    else:
        filename = "thumbnail_" + str(index2)
    index2+=1
    filename_list.append(filename)
#filename_list
#https://res.cloudinary.com/dfyzsulq8/image/upload/v1622639726/thumbnails2/thumbnail_291_fehaff.png
# get the fitting cloudinary filename for the thumbnails in a column
new_list = []
for i in filename_list:
    check = 0
    for j in list_of_cloud_thumbnail_filenames:
        if j.startswith(i):
            new_list.append('https://res.cloudinary.com/dfyzsulq8/image/upload/v1622639726/thumbnails2/'+j)
            check = 1
    if check == 0:
        new_list.append('')

#new_list
df_new['cloudinary thumbnails new'] = new_list

type(df_new['cloudinary artworks new'][4])
df_new.to_csv('BACKUP_df_NEW.csv', index=False, header=True)
df_new.columns<jupyter_output><empty_output><jupyter_text>get all the Rijksmuseum artworks<jupyter_code>df_new['URLs Rijk new'][30] == []
# open all the URLs and save the squared images names [artist-index]_[image-index].png

count = 0

# iterate over artists
for i in range(len(df_new)):
    
    if df_new['URLs Rijk new'][i] == []:
        print(count, i, df_new['URLs Rijk new'][i])
        count += 1

    else:
        
        index = 0
        # iterate over list of urls
        for j in df_new['URLs Rijk new'][i]:
    
            if j == 'nan':
                index += 1
                continue
            elif j == '':
                index += 1
                continue
            else:
                img = getsquared_image(j + '-s1400') #.replace(' ', '%20')
                filename = '../data/Rijks/' + str(i) + '_' + str(index) + '.png'
                img.save(filename, 'PNG')
                index += 1
        
        print(count)
        count += 1
 
for h in (df_new['URLs Rijk new'][25]):
    print(h)
df_new['Unnamed: 0'][0]
list_of_artworks = os.listdir('../data/Rijks_cloudinary/')
#list_of_artworks
# get the fitting cloudinary filename for the Rijks artworks
# https://res.cloudinary.com/dfyzsulq8/image/upload/v1622644688/artworks_rijk/286_1_vdkwpx.png
artwork_list = []
for i in df_new['Unnamed: 0']:
    check = 0
    storage = []
    for j in list_of_artworks:
        if j.startswith(str(i) + '_'):
            storage.append('https://res.cloudinary.com/dfyzsulq8/image/upload/v1622644688/artworks_rijk/' + str(j))
            check = 1
    if check == 0:
        artwork_list.append('')
    else:
        artwork_list.append(storage)
df_new[' Rijks cloudinary artworks new'] = artwork_list
(artwork_list)
df_new[' Rijks cloudinary artworks new']<jupyter_output><empty_output><jupyter_text>finally make the artwork column!<jupyter_code>df_new.columns
final_storage = []

for i, j in enumerate(df_new['cloudinary thumbnails new']):
    store = list()
    store.append(j)
    for k in df_new['cloudinary artworks new'][i]:
        store.append(k)
    for l in df_new[' Rijks cloudinary artworks new'][i]:
        store.append(l)
    #.extend().extend(df_new[' Rijks cloudinary artworks new'][i])
    final_storage.append(store)
len(final_storage)
df_new['cloudinary thumbnails new']
thumbnail = ''
for i in df['']:
    death+=str(i)+','
    
death = death.strip(',')
#final_storage[233]
with open('')
df_new['final artworks'] = final_storage<jupyter_output><empty_output><jupyter_text>END of formatting make final cables dataframe<jupyter_code>#final_storage
position_in_timeline = [1271,1266,1650,1254,1282,1249,1276,1268,1304,1400,1316,1311,1326,1305,1314,1319,1308,1314.5,1285,1309,1324,1328,1341,1340,1412,1373,1395,1365,1287,1395.5,1321,1380,1384,1319.5,1398,1392,1388,1407,1453,1406,1410,1446,1396.0,1440,1480,1527,1436,1416,1405,1415,1420,1414,1411,1426,1434,1439,1429,1448,1461,1454,1424,1449,1487,1438,1429.5,1449.5,1462,1466,1433,1465,1468,1458,1655,1463,1472,1433.5,1431,1390,1425,1376,1435,1453.5,1461.5,1470,1441,1443,1457,1448.5,1431.5,1447,1456,1498,1504,1474,1430.0,1473,1463.5,1475,1514,1468.5,1467,1470.5,1477,1471,1466.5,1478,1488,1486,1464,1469,1445,1473.5,1465.5,1457.5,1490,1535,1464.5,1475.5,1482,1462.5,1499,1484,1485,1481,1479,1526,1510,1500,1496,1407.5,1383,1360,1404,1335,1355,1467.5,1495,1483,1495.5,1469.5,1491,1492,1481.5,1487.5,1496.5,1494,1507,1521,1521.5,1483.5,1518,1522,1501,1478.5,1532,1482.5,1489,1512,1488.5,1474.5,1489.5,1490.5,1493,1497.0,1476.0,1491.5,1502,1497.5,1477.5,1484.5,1476.5,1505,1492.5,1511,1493.5,1494.5,1509,1498.5,1530,1499.5,1500.5,1516,1508,1501.5,1503,1503.5,1505.5,1556,1567,1580,1517,1530.5,1520,1506,1519,1507.5,1512.5,1513,1515,1516.5,1504.5,1508.5,1509.5,1502.5,1506.5,1517.5,1510.5,1513.5,1471.5,1511.5,1514.5,1515.5,1524,1518.5,1519.5,1520.5,1522.5,1523.0,1523.5,1524.5,1525.0,1525.5,1526.5,1542,1527.5,1528.0,1528.5,1529.0,1529.5,1531.0,1538,1531.5,1532.5,1558,1565,1533.0,1533.5,1534.0,1534.5,1535.5,1536,1536.5,1485.5,1479.5,1537.0,1537.5,1551,1540,1538.5,1539.0,1539.5,1548,1540.5,1543,1574,1541.0,1543.5,1544,1541.5,1561,1554,1542.5,1544.5,1550,1562,1545.0,1545.5,1557,1553,1560,1546.0,1546.5,1557.5,1547.0,1547.5,1548.5,1549.0,1566,1594,1549.5,1550.5,1551.5,1552.0,1552.5,1660,1553.5,1554.5,1555.0,1575,1555.5,1556.5,1558.5]
len(position_in_timeline)
# make new dataframe for cables -> to be converted to json array

index = []
for i in range(292):
    index.append(i)
df_cables = pd.DataFrame()
df_cables['index'] = index
df_cables['Name'] = df_new['Name']
df_cables['birth'] = df_new['birth']
df_cables['death'] = df_new['death']
df_cables['timeline_position'] = df_new['timeline_position']
df_cables['timeline_position_extended'] = position_in_timeline
df_cables['Wikipedia'] = df_new['Wikipedia-URL']
df_cables['Summary'] = df_new['Summary']
df_cables['extract_html'] = df_new['extract_html']                     # make new!
df_cables['cloudinary thumbnails'] = df_new['cloudinary thumbnails new'] # make new!
df_cables['final artworks'] = final_storage   # make new!


df_cables['Summary'][2]
df_final = df_cables.sort_values(by = ['timeline_position_extended'], inplace= False)
df_final
df_final.to_csv('FINAL.csv', index = False, header = True)
len(df_final)
df_final.to_json('FINAL_json_indexf.json', orient = 'index')
os.listdir('../')
# make list of jsons from dataframe
list_of_json = []
for i in range(len(df_final2)):
    x = df_final2.iloc[i].to_json()
    list_of_json.append(x)
len(list_of_json)
#df_cables.iloc[4].to_json()
with open('json_listTEST.json', 'w+') as output:
    output.write(str(list_of_json))
    output.close()
another_list = []
for i in list_of_json:
    data = json.loads(i)
    another_list.append(data)
type(another_list[0])
with open('json_list_NEW_testtest.json', 'w') as outfile:
    json.dump(another_list, outfile)
(df_final2)['indexf'] = range(292)
df_final2 = df_final.set_index('indexf')
df_final2
thumbnail = ''
for i in df_final2['cloudinary thumbnails']:
    thumbnail+=str(i)+','
    
death = thumbnail.strip(',')
f = open('thumbnails_squared.txt', 'wb') # names
f.write(death.encode("utf-8"))
f.close()
df_final2.columns<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup