# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.8.12 ('img')
#     language: python
#     name: python3
# ---

# +
import numpy as np
from PIL import Image
from tqdm import tqdm
import json
import os

#data = json.load(open('/Volumes/Elements/236_data/origin/data.json'))
base = '/Volumes/Elements/236_data/origin/'


from transformers import pipeline
# -

summarizer = pipeline("summarization")


#find data where 'id' = 107629
for i in range(len(data)):
    if data[i]['id'] == 107629:
        print(data[i])
        break

# +
text = """For four long years members of the US Congress had to smile or scowl as a TV star played the role of president.

<NAME> became infamous for the art of lying. On Wednesday another TV performer turned national leader came before Congress. But this one captivated his viewers with truth telling.

The Ukrainian president, <NAME>, a former actor and comedian facing down the Russian war machine, has an instinctive understanding of the camera but is proving a more serious man for more serious times.

Despite being under siege in Kyiv, Zelenskiy has been on a virtual tour of western capitals over the past three weeks, tailoring his speeches to each nation. Speaking virtually to the British parliament, he cited <NAME> and <NAME>, while he asked members of its Canadian equivalent to imagine waking at 4am to bombs dropping on Ottawa’s airport or Toronto’s CN Tower.

The Axios website described it as a “signature blend of praising, chastising and pleading with his audience to understand the global stakes of Ukraine’s resistance” which has produced unexpected commitments such as oil and Swift banking sanctions.

So it was that in a packed auditorium in the basement of the US Capitol in Washington, Zelenskiy, whose words were translated from Ukrainian into English by a female interpreter, conjured the demons of two days when America was attacked from the skies to renew his plea for a no-fly zone above Ukraine.

“Remember <NAME>, the terrible morning of December 7, 1941, when your sky was black from the planes attacking you,” said Zelenskiy, looming large on a cinema screen, wearing perfectly trimmed hair and beard and a green T-shirt, against a white backdrop with a Ukrainian flag to one side.

“Remember September 11, a terrible day in 2001 when evil tried to turn your cities, independent territories, into battlefields. When innocent people were attacked from the air. Our country is experiencing the same every day, right now, at this moment. Every night for three weeks now … Russia has turned the Ukrainian sky into a source of death for thousands of people.”

Combined with references to Mount Rushmore and <NAME> King’s “I have a dream speech”, Zelenskiy, was pushing America’s most emotive buttons with words. But he also knows that this is the nation of network television, cable news, Hollywood, Netflix and social media. So words alone would not do.

Zelenskiy asked the members of the House of Representatives and Senate to watch a searing video compilation showing the hell that Russian troops have rained down on Ukraine and its citizens. It contrasted idyllic images of children playing in peaceful towns and cities with explosions, destruction, sobbing, refugees, hospitals and corpses, accompanied by the lament of a violin.

According to a pooled report by the Associated Press, “As Zelenskiy played the video of violence, the room was very quiet and members were mostly still. Some shook their heads or wiped eyes or took video. Small amount of applause afterward.”

Then came a simple message written in white letters on a black backdrop: “Close the sky over Ukraine.”

Tragic in the truest sense because this is the one thing that Congress, and <NAME>, will not do, fearing that a no-fly zone, in which US pilots shoot down Russians, could trigger a third world war. Perhaps aware of this reluctance, Zelenskiy did not dwell on the issue for long, pivoting to a request for surface-to-air missile systems and urging Washington to “do more”.

But the video had a wider purpose. It was shown to millions of American TV viewers just after 9am. It caught TV executives by surprise and they did not have time to censor it; some anchors apologised for its graphic content. It spread far and wide on social media. In the court of public opinion, the video humanised the victims and conveyed the message that our struggle is your struggle.

Zelenskiy had again shown himself to be a master of the medium, inviting comparisons with Vladimir Putin’s efforts to lie low, clamp down on media, crush all dissent and turn Russia into North Korea. Zelenskiy is running rings around Putin in the soft power arena with his speeches and intimate phone videos; Russia is not faring especially well with hard power either.

On Wednesday the Ukrainian president ended his speech by addressing the room in English. “Now, I’m almost 45 years old,” he said. “Today my age stopped when the heart of more than 100 children stopped beating. I see no sense in life if it cannot stop the deaths.”

There was also a direct appeal to Biden: “I wish for you to be the leader of the world. Being the leader of the world means to be the leader of peace.”

The auditorium erupted in a bipartisan standing ovation. <NAME>, a Democratic senator, tweeted: “There’s no member of Congress left that room without thinking what more the United States can do to stop this carnage. Just a gut wrenching speech. #SlavaUkraine.”

In an era of Trumpism, fake news and disinformation, Zelenskiy, who used to play a fictional president, had cut through with his sincerity. For him and Ukraine, it already feels like a third world war; that is their truth. And the temptation for America to flex its superpower muscles is stronger than ever.

Tragic in the truest sense because this is the one thing that Congress, and <NAME>, will not do, fearing that a no-fly zone, in which US pilots shoot down Russians, could trigger a third world war. Perhaps aware of this reluctance, Zelenskiy did not dwell on the issue for long, pivoting to a request for surface-to-air missile systems and urging Washington to “do more”.

But the video had a wider purpose. It was shown to millions of American TV viewers just after 9am. It caught TV executives by surprise and they did not have time to censor it; some anchors apologised for its graphic content. It spread far and wide on social media. In the court of public opinion, the video humanised the victims and conveyed the message that our struggle is your struggle.

Zelenskiy had again shown himself to be a master of the medium, inviting comparisons with Vladimir Putin’s efforts to lie low, clamp down on media, crush all dissent and turn Russia into North Korea. Zelenskiy is running rings around Putin in the soft power arena with his speeches and intimate phone videos; Russia is not faring especially well with hard power either.

On Wednesday the Ukrainian president ended his speech by addressing the room in English. “Now, I’m almost 45 years old,” he said. “Today my age stopped when the heart of more than 100 children stopped beating. I see no sense in life if it cannot stop the deaths.”

There was also a direct appeal to Biden"""


summarizer(text)


# -

def summarized_docs(ids):
    summaries = {}
    for id in tqdm(ids):
        #find data with id
        art_path = None
        caption = None
        for d in data:
            if d['id'] == id:
                art_path = base + d['article_path'][2:]
                caption = d['caption']
                print(art_path)
                break
        #print article  
        with open(art_path, 'r') as f:
            art = f.read()
            #split into words
            art = art.split()
            #take first 700 words
            art = art[:500]
            #join words
            art = ' '.join(art)
            print(len(art))

            text = summarizer(art, min_length=5, max_length=50)[0]['summary_text']
            #add to dict
            summaries[id] = {'caption': caption, 'summary': text}
            print(text)

    return summaries



# +
#pick 128 random images from the ./test folder
random_imgs = np.random.choice(os.listdir('./test'), 128, replace=False)
#remove the .jpg extension from the images
random_imgs = [img[:-4] for img in random_imgs]
#convert to list of ints
random_imgs = [int(img) for img in random_imgs]
summaries = summarized_docs(random_imgs)

#save to json
with open('./test_summaries.json', 'w') as f:
    json.dump(summaries, f)

# -

# !pwd

# +
fake_summ_ft_path = '/Users/isaiahwilliams/thumbnail/fake/summary/ft/'
fake_caption_ft_path = '/Users/isaiahwilliams/thumbnail/fake/caption/ft/'
fake_summ_base_path = '/Users/isaiahwilliams/thumbnail/fake/summary/base/'
fake_caption_base_path = '/Users/isaiahwilliams/thumbnail/fake/caption/base/'

for i in range (1, len(os.listdir(fake_summ_ft_path))):
    #get os list files only
    fake_summ_ft_files = os.listdir(fake_summ_ft_path)
    fake_summ_ft_files = [f for f in fake_summ_ft_files if os.path.isfile(fake_summ_ft_path + f)]
    folder_name = fake_summ_ft_files[0].split('_')[0][2:]
    #make folder of folder name
    os.mkdir(fake_summ_ft_path + folder_name)
    #move all files containing folder name to folder
    for f in fake_summ_ft_files:
        if folder_name in f:
            os.rename(fake_summ_ft_path + f, fake_summ_ft_path + folder_name + '/' + f)
        
    ########################################################################################################################
    fake_caption_ft_files = os.listdir(fake_caption_ft_path)
    fake_caption_ft_files = [f for f in fake_caption_ft_files if os.path.isfile(fake_caption_ft_path + f)]
    folder_name = fake_caption_ft_files[0].split('_')[0][2:]
    #make folder of folder name
    os.mkdir(fake_caption_ft_path + folder_name)
    #move all files containing folder name to folder
    for f in fake_caption_ft_files:
        if folder_name in f:
            os.rename(fake_caption_ft_path + f, fake_caption_ft_path + folder_name + '/' + f)


    ########################################################################################################################
    fake_summ_base_files = os.listdir(fake_summ_base_path)
    fake_summ_base_files = [f for f in fake_summ_base_files if os.path.isfile(fake_summ_base_path + f)]
    folder_name = fake_summ_base_files[0].split('_')[0][2:]
    #make folder of folder name
    os.mkdir(fake_summ_base_path + folder_name)
    #move all files containing folder name to folder
    for f in fake_summ_base_files:
        if folder_name in f:
            os.rename(fake_summ_base_path + f, fake_summ_base_path + folder_name + '/' + f)


    ########################################################################################################################
    fake_caption_base_files = os.listdir(fake_caption_base_path)
    fake_caption_base_files = [f for f in fake_caption_base_files if os.path.isfile(fake_caption_base_path + f)]
    folder_name = fake_caption_base_files[0].split('_')[0][2:]
    #make folder of folder name
    os.mkdir(fake_caption_base_path + folder_name)
    #move all files containing folder name to folder
    for f in fake_caption_base_files:
        if folder_name in f:
            os.rename(fake_caption_base_path + f, fake_caption_base_path + folder_name + '/' + f)


###THIS RETURNS AN ERROR BUT STILL WORKS WITH ZIP FILE



# +
#find data where id = '111268'

for 
# -

fake_summ_ft_path = '/Users/isaiahwilliams/thumbnail/fake/summary/ft/'
from tqdm import tqdm
for path in tqdm(os.listdir(fake_summ_ft_path)):
    #get os list files only
    if os.path.isdir(fake_summ_ft_path + path):
        #print(path)
        id = path
        real_img_path = '/Users/isaiahwilliams/thumbnail/test/' + str(id) + '.jpg'
        new_dest = '/Users/isaiahwilliams/thumbnail/real/' + str(id) + '.jpg'
        # !cp '{real_img_path}' '{new_dest}'

for i in range(len(data)):
    if data[i]['id'] == id:
        print(data[i])
        break

#loop through all files in test_f and put them in their own folder
for filename in os.listdir('./test_f'):
    #make folder with filename
    folder_name = filename[:-4]
    os.mkdir('./test_f/' + folder_name)
    #move file into folder
    os.rename('./test_f/' + filename, './test_f/' + folder_name + '/' + filename)


# +
#loop through all files in test_f and duplicate them 3 tiems
import shutil
for filename in os.listdir('./test_f'):
    #check if filename is a folder
    #chec kif filename is an int
    if os.path.isdir('./test_f/' + filename) and filename.isdigit():
        for filename2 in os.listdir('./test_f/' + filename):
            shutil.copy('./test_f/' + filename + '/' + filename2, './test_f/' + filename + '/' + filename2[:-4] + '_2.jpg')
            shutil.copy('./test_f/' + filename + '/' + filename2, './test_f/' + filename + '/' + filename2[:-4] + '_3.jpg')
            shutil.copy('./test_f/' + filename + '/' + filename2, './test_f/' + filename + '/' + filename2[:-4] + '_4.jpg')
            
            
# -

#loop through all files in test_f and duplicate them 3 tiems
import shutil
for filename in os.listdir('./test_f'):
    #check if filename is a folder
    #chec kif filename is an int
    if os.path.isdir('./test_f/' + filename) and filename.isdigit():
        #remove all files that are not jpg
        for filename2 in os.listdir('./test_f/' + filename):
            if filename2[-4:] != '.jpg':
                os.remove('./test_f/' + filename + '/' + filename2)

# ## RUN THIS TO GET FID SCORES ON DATA
#

# +
fake_summ_ft_path = '/Users/isaiahwilliams/thumbnail/fake/summary/ft/'
fake_caption_ft_path = '/Users/isaiahwilliams/thumbnail/fake/caption/ft/'
fake_summ_base_path = '/Users/isaiahwilliams/thumbnail/fake/summary/base/'
fake_caption_base_path = '/Users/isaiahwilliams/thumbnail/fake/caption/base/'
real_path = '/Users/isaiahwilliams/thumbnail/real/'

# !python -m pytorch_fid $fake_summ_ft_path $real_path
# !python -m pytorch_fid $fake_caption_ft_path $real_path
# !python -m pytorch_fid $fake_summ_base_path $real_path
# !python -m pytorch_fid $fake_caption_base_path $real_path




   


# +
# fake_summ_ft_path = '/Users/isaiahwilliams/thumbnail/fake/summary/ft/'
# fake_caption_ft_path = '/Users/isaiahwilliams/thumbnail/fake/caption/ft/'
# fake_summ_base_path = '/Users/isaiahwilliams/thumbnail/fake/summary/base/'
# fake_caption_base_path = '/Users/isaiahwilliams/thumbnail/fake/caption/base/'
# real_path = '/Users/isaiahwilliams/thumbnail/real/'

# files = []
# #loop through all files in fake_summ_ft_path
# for filename in os.listdir(fake_summ_ft_path):
#     filename_cf = fake_caption_ft_path + filename.replace('sf', 'cf')
#     filename_sb = fake_summ_base_path + filename.replace('sf', 'sb')
#     filename_cb = fake_caption_base_path + filename.replace('sf', 'cb')
#     filename_real = real_path + filename.split('_')[0][2:] + '.jpg'
#     filename_sf = fake_summ_ft_path + filename
#     files.extend([filename_cf, filename_sb, filename_cb, filename_real, filename_sf])

#     #check if file is a file

# #loop through all files in files

# +
# for path in os.listdir('/Users/isaiahwilliams/thumbnail/fake/'):
#     if os.path.isdir('/Users/isaiahwilliams/thumbnail/fake/' + path):
#         for filename in os.listdir('/Users/isaiahwilliams/thumbnail/fake/' + path):
#             if os.path.isdir('/Users/isaiahwilliams/thumbnail/fake/' + path + '/' + filename):
#                 for filename2 in os.listdir('/Users/isaiahwilliams/thumbnail/fake/' + path + '/' + filename):
#                     full_path = '/Users/isaiahwilliams/thumbnail/fake/' + path + '/' + filename + '/' + filename2
#                     #split on '_'
#                     number = int(filename2.split('_')[1][:-4])
#                     #if number is greater than 3 then delete
#                     if number > 3:
#                         os.remove(full_path)

#                     if full_path not in files:
#                         os.remove(full_path)
#                         print(full_path)

# for path in os.listdir('/Users/isaiahwilliams/thumbnail/real'):
#     full_path = '/Users/isaiahwilliams/thumbnail/real/' + path
#     if full_path not in files:
#         os.remove(full_path)
#         print(full_path)

# -



StarcoderdataJupyterScriptsDedupFiltered