# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="TpQCOZh1wLp8"
# # Inference
#
# This notebook is dedicated to testing and visualizing results for both the wiki and podcast datasets
#
# Note: 
# Apologies for the gratuitous warnings. Tensorflow is aware of these issues and has rectified them in later versions of TensorFlow. Unfortunately, they persist for version 1.13. 

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 28132, "status": "ok", "timestamp": 1607092424385, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="Vu3MjSBcwN4p" outputId="4d2377bf-bb21-4297-ff3c-7ab0a86588fb"
from google.colab import drive
drive.mount('/content/gdrive')
# %cd gdrive/My Drive/MSci/SliceCast
# ! git pull

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} executionInfo={"elapsed": 264391, "status": "ok", "timestamp": 1607092694561, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="l0pQ5s_Jla0H" outputId="5b20e0d9-240f-4f0d-aab0-85bccb7d6138"
#Run ALL these if first time opening 
# ! pip3 install -r requirements.txt
# ! pip install objectpath
# ! pip install git+https://github.com/boudinfl/pke.git
# ! pip install keras==2.3.0 #1.15 doesnt exist

# ! pip install tensorflow==1.15
# ! pip install tensorflow_hub>=0.6.0
# ! pip3 install tensorflow_text==1.15

# ! pip install --upgrade tables


# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 1855, "status": "ok", "timestamp": 1607093025673, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="xKv2jNVx39d_" outputId="ae6ce3ac-8b7b-403a-94ab-014989a5bd64"
#Run
import tensorflow as tf
import tensorflow_hub as hub
import keras

print('keras version:', keras.__version__)  # should be 2.3.0
print('tensorflow version', tf.__version__) # should be 1.15.0


# + colab={"base_uri": "https://localhost:8080/", "height": 1000} executionInfo={"elapsed": 23335, "status": "ok", "timestamp": 1607093054175, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="QNKK9FTLTueq" outputId="ac910707-54ad-4cb0-c288-bed10ff11c79"
#Come back to and run if issue with pke in next cell
# ! pip install git+https://github.com/boudinfl/pke.git

# ! python -m nltk.downloader stopwords
# ! python -m nltk.downloader universal_tagset
# ! python -m spacy download en # download the english model

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 20422, "status": "ok", "timestamp": 1607093112162, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="j_DYJyF8wLqD" outputId="951fd3e8-7915-4728-d4d1-8aef1914769d" pycharm={"is_executing": true}
#Run
import nltk
import numpy
nltk.download('stopwords')

import pke

from src.SliceNet import SliceNet
from src.netUtils import getSingleExample
import matplotlib.pyplot as plt
from pathlib import Path
import numpy as np
import pandas as pd
import seaborn as sns
import random
import math

import nltk
nltk.download('punkt')

import warnings
warnings.filterwarnings('ignore')

tf.logging.set_verbosity(tf.logging.ERROR)
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
if type(tf.contrib) != type(tf): tf.contrib._warning = None
# %load_ext autoreload
# %autoreload 2

# + pycharm={"name": "#%%\n"}
# ! pip install numpy 

# + pycharm={"name": "#%%\n"}
## Sample predictions on unseen wiki articles
Note that this section relies on wikipedia
# -

## Sample predictions on unseen wiki articles
Note that this section relies on wikipedia

# + [markdown] id="leyLNXISwLqO"
# # Choose whether to use the base network or the network with self-attention
# attention = True
#
# # Current best networks
# best_base_wiki = './models/04_20_2019_2300_final.h5'
# best_base_podcast = './models/04_26_2019_1000_podcast.h5'
# best_attn_wiki = './models/05_03_2019_0800_attn.h5'
# best_attn_podcast = './models/05_02_2019_2200_attn_podcast.h5'
#
# if attention:
#     weights_wiki = best_attn_wiki
#     weights_podcast = best_attn_podcast
# else:
#     weights_wiki = best_base_wiki
#     weights_podcast = best_base_podcast
#
# #Instantiate Network
# net = SliceNet(classification=True,
#                class_weights=[1.0, 7, 0.2],
#                attention=attention)
#

# + id="udbhQVMG4-l4"
dataPath = Path('./wiki-sample/')
files = [str(x) for x in dataPath.glob('**/*') if x.suffix=='.hdf5']
mask = random.sample(range(0,len(files)), 1)

# randomly select a file to test
test_file = [x for (i,x) in enumerate(files) if i in mask][0]
print('test_file', test_file)

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 857, "status": "ok", "timestamp": 1606232013180, "user": {"displayName": "Shona CW", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="AV3erf9mwLqO" outputId="7af06748-3a78-426c-abb4-a8acaa920337"
k = 4
num_samples = 16

preds, labels, pk = net.predict(test_file=test_file,
                                num_samples=num_samples,
                                weights_path=weights_wiki,
                                k=k)
print('Average PK score with k={} on {} examples is: {:0.3f}'.format(k, num_samples, pk))

# + colab={"base_uri": "https://localhost:8080/", "height": 232} executionInfo={"elapsed": 489, "status": "error", "timestamp": 1606306609830, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="PHKHUNrMwLqS" outputId="7b0563c5-bf1d-47c7-b6b3-99e6a3c5dc50"
np.set_printoptions(suppress=True)

preds = np.argmax(preds, axis=2)
labels = np.argmax(labels, axis=2)

# + id="qLAFje2QwLqV"
# Choose the index of the document you want to examine
idx = 2
# You can keep running this cell with different indices to visualize different
# documents within this batch of testing
# Note: The graph displays n sentences where n is the length of the longest
# document in the batch. As such, there may be padding sections at the beginning
# of the document with label and prediction of value 2

df = pd.DataFrame()
df['preds'] = preds[idx,:]
df['labels'] = labels[idx,:]
df['sent_number'] = df.index

fig, axes = plt.subplots(nrows=2, ncols=1)

df.plot(x='sent_number', y='preds', figsize=(10,5), grid=True, ax=axes[0])
df.plot(x='sent_number', y='labels', figsize=(10,5), grid=True, ax=axes[1], color='green')

# + colab={"base_uri": "https://localhost:8080/", "height": 352} executionInfo={"elapsed": 1284, "status": "ok", "timestamp": 1606232052020, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="sQPABogCwLqX" outputId="95e671e8-d0b6-4a36-d105-fdf65a8898a9"
## Sample predictions on unseen podcast data

# + [markdown] id="VoBAa2iLwLqa"
#
# test_file = './data/podcasts/hdf5/batch0_0.hdf5'
# k = 33
# num_samples = 2
# preds, labels, pk = net.predict(test_file=test_file,
#                     num_samples=num_samples,
#                     weights_path=weights_podcast,
#                     k=k)
#
# print('Average PK score with k={} on {} examples is: {:0.3f}'.format(k, num_samples, pk))

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 7546, "status": "ok", "timestamp": 1606384441536, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="OMshsXcUwLqb" outputId="66eb2c66-9feb-4d74-c274-5bb074013d0c"
np.set_printoptions(suppress=True)

preds = np.argmax(preds, axis=2)
labels = np.argmax(labels, axis=2)

# + id="tzGUmppswLqd"
# Choose the document you want to examine
idx = 1

df = pd.DataFrame()
df['preds'] = preds[idx,:]
df['labels'] = labels[idx,:]
df['sent_number'] = df.index

fig, axes = plt.subplots(nrows=2, ncols=1)

df.plot(x='sent_number', y='preds', figsize=(10,5), grid=True, ax=axes[0])
df.plot(x='sent_number', y='labels', figsize=(10,5), grid=True, ax=axes[1], color='green')
df_unseenpod = df

# + colab={"base_uri": "https://localhost:8080/", "height": 335} executionInfo={"elapsed": 2226, "status": "ok", "timestamp": 1606384446228, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="wxXes95TwLqf" outputId="c604aef1-01ab-497e-df80-fa555bc00cee"
df_unseenpod

# + colab={"base_uri": "https://localhost:8080/", "height": 419} executionInfo={"elapsed": 566, "status": "ok", "timestamp": 1606304855002, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="0beLMM36Ot7z" outputId="f50f0501-f47b-4a91-e758-3283397770b9"
## Predictions on a single text file

# + [markdown] id="q7aUri9qwLqj"
# # # ! python -m spacy validate

# + id="OIeLq-ZT3kLW"
# ! python -m spacy download en_core_web_sm

# + id="A3kVsgXv3nlo"
# The joe1254.txt doc has '========,9,title.' tags at manually labelled segments locations
text_file = './data/podcasts/with_timestamps/joe1254.txt'
is_labeled = True
weights_path = weights_podcast # transfer learning

sents, labels = getSingleExample(fname=text_file, is_labeled=is_labeled)
sents = np.expand_dims(sents, axis=0)
preds = net.singlePredict(sents, weights_path=weights_path)

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 13320, "status": "ok", "timestamp": 1606389751057, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="TwSxVo8awLqj" outputId="cb29a794-3439-4f95-c45a-b630333ed383"
# Place data into a pandas dataframe for analysis
df = pd.DataFrame()

preds = np.argmax(np.squeeze(preds), axis=-1)
df['raw_sentences'] = sents[0]
if is_labeled:
    df['labels'] = labels
df['preds'] = preds
df['sent_number'] = df.index

fig, axes = plt.subplots(nrows=2, ncols=1)

df.plot(x='sent_number', y='preds', figsize=(10,5), grid=True, ax=axes[0])
df.plot(x='sent_number', y='labels', figsize=(10,5), grid=True, ax=axes[1], color='green')

# + colab={"base_uri": "https://localhost:8080/", "height": 352} executionInfo={"elapsed": 842, "status": "ok", "timestamp": 1606306644992, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="UX-ied3IwLqm" outputId="581fd89b-8265-4791-a7b5-b207a766d335"
## Keyword Extraction
The following cells are experimental code to extract keywords for each segment in order to provide context for each segment.

# + [markdown] id="pGsfr0olwLqo"
# from src.postprocess import getSummaries, getTimeStamps
# import nltk
#
# nltk.download('stopwords')
#
# keywords = getSummaries(sents[0], preds)
# stamps = getTimeStamps(sents[0], './data/podcasts/with_timestamps/joe1254.json', preds)

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 9408, "status": "ok", "timestamp": 1606307114683, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="yTjIamqvwLqp" outputId="c630885b-48f0-493c-f620-c7944a014f04"
seconds = [x%60 for x in stamps]
minutes = [math.floor(x/60) for x in stamps]
for i, (x, y)in enumerate(zip(minutes, seconds)):
    print("{}:{}".format(x, y), end="")
    print([x[0] for x in keywords[i]])

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 6344, "status": "ok", "timestamp": 1606307114684, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="mgz0wgY6wLqr" outputId="3c8360cd-103e-4254-8257-86cfab7c4f0b"
# Shona Section


# + [markdown] id="R34zveslS680"
#

# + id="aUOG0DfsHCHE"
#Run to format the shortened version of the Joe Rogan podcast

from pathlib import Path
import re

def reformat_sgcw_remove_all(text):
    """
    Takes the Joe Rogan podcast transcript (which I already slightly pre-processed
    a while back) and turns it into a single long string of text without speakers 
    or times
    """

    print('Text format before preprocessing:\n', text[:300])

    # Strip all characters but letters and whitespace inside the speech
    content_1 = re.sub('\w+\s\w+;[0-9]{2}:[0-9]{2};', "", text)
    # Strip new-lines
    content_2 = re.sub('\n', " ", content_1)

    print('\nText format after preprocessing:\n', content_2)

    return content_2

# Paths
path_shorter = Path('./SGCW/shorter_formatted.txt')
path_shorter_and_formatted = Path('./SGCW/shorter_formatted_plain.txt')

# Reformat the (shortened) Joe Rogan transcript to be consistent w SliceCast podcasts
with open(path_shorter) as f_shorter:
    content = f_shorter.read()
better_format_text = reformat_sgcw_remove_all(content)

# Save shortened + formatted version of the Joe Rogan transcript
form_shorter_doc = open(path_shorter_and_formatted, 'w')
form_shorter_doc.write(better_format_text)
form_shorter_doc.close()

# + colab={"base_uri": "https://localhost:8080/", "height": 446} executionInfo={"elapsed": 7294, "status": "ok", "timestamp": 1606750771413, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="ZYd8MN1mkTxU" outputId="a9d86fc1-bfcc-4c9f-bfc0-c2e6d8a188b0"
# Segmentation using SliceCast on JoeRogan podcast
text_file = './SGCW/shorter_formatted_plain_labelled.txt'
is_labeled = True                             # I manually added segment tokens
weights_path = weights_podcast                # Transfer learning

sents, labels = getSingleExample(fname=text_file, is_labeled=is_labeled)
sents = np.expand_dims(sents, axis=0)

preds = net.singlePredict(sents, weights_path=weights_path)

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 10431, "status": "ok", "timestamp": 1606391638269, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="m1h_AkqrhZu9" outputId="fdc9fbc6-7451-4344-f3ae-0f1f01d72f08"
# Place data into a pandas dataframe for analysis...
df = pd.DataFrame()
preds = np.argmax(np.squeeze(preds), axis=-1)
df['raw_sentences'] = sents[0]
if is_labeled:
    df['labels'] = labels
df['preds'] = preds
df['sent_number'] = df.index

# Plot
print('Segmentation of Joe Rogan Podcast, predictions vs. Labels:')
fig, axes = plt.subplots(nrows=2, ncols=1)
df.plot(x='sent_number', y='preds', figsize=(10,5), grid=True, ax=axes[0])
df.plot(x='sent_number', y='labels', figsize=(10,5), grid=True, ax=axes[1], color='green')

# Also save dataframe of segments to hdf5 file
df.to_hdf('./SGCW/segd_shorter_formatted_plain.h5', key='dfs', mode='w')

# + colab={"base_uri": "https://localhost:8080/", "height": 352} executionInfo={"elapsed": 1106, "status": "ok", "timestamp": 1606391643272, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="uBhluvVd-qEG" outputId="49644988-ccac-43a7-a046-4231ab791697"
# Produce tables of data about the segmentation process, and extract Keywords

from src.postprocess import getTimeStamps
import nltk
nltk.download('stopwords')

def getSummaries(doc, labels, n=3):
    """
    Added variable n number for multiple keyword extraction
    """
    summaries = []
    segment = ''
    numSent = 0
    #k = 0 #not using?
    for i, sent in enumerate(doc):
      ##looping through sentences
      if labels[i]==1 and segment != '':
        ## label==1 if its the first sentence of a segment 
        ##(but doesn't enter if it's the first sentence of the first segment)
        extractor = pke.unsupervised.TopicRank()
        extractor.load_document(input=segment, language='en')
        extractor.candidate_selection()
        extractor.candidate_weighting()
        summaries.append(extractor.get_n_best(n))
        #k = k+1 #not using?
        segment = sent
        numSent = 1
      else:
        ##First sentence enters here
        ##+ then all sentences WITHIN segments after (i.e. not the first sents)
        segment = segment + " " + sent
        numSent = numSent + 1
    
    ##'segment' is a long string of sentences from the last segment
    ## the next few lines deal with the final segment (i think..)
    extractor = pke.unsupervised.TopicRank()
    extractor.load_document(input=segment, language='en')
    extractor.candidate_selection()
    extractor.candidate_weighting()

    summaries.append(extractor.get_n_best(n))

    return summaries

def get_keywords(text, n=100):
  """
  get top 100 keywords for the entire transcript
  """
  extractor = pke.unsupervised.TopicRank()
  extractor.load_document(input=text, language='en')
  extractor.candidate_selection()
  extractor.candidate_weighting()

  top_keywords = extractor.get_n_best(n)
  return top_keywords

def get_segments_info(df, timestamps=False):
  """

  """
  segments_dict = {'first_sent_numbers' : [], 'length_of_segment': [],
                   'keyword_list' : [], }

  df_mini = df[df['preds']==1]  
  old_idx = 0
  for idx, row in df_mini.iterrows():
    segments_dict['first_sent_numbers'].append(idx)   # POSITION of each section
    length = np.int(idx) - np.int(old_idx)            # LENGTH of each section
    segments_dict['length_of_segment'].append(length) 
    old_idx = idx

  keywords = getSummaries(df['raw_sentences'], df['preds'], n=5) # KEYWORDS
  if timestamps:
    segments_dict['first_sent_times'] = []
    stamps = getTimeStamps(df['raw_sentences'], 
                         './data/podcasts/with_timestamps/joe1254.json', 
                         df['preds'])                            # TIMESTAMPS 

    seconds = [x%60 for x in stamps]
    minutes = [math.floor(x/60) for x in stamps]
    for i, (x, y)in enumerate(zip(minutes, seconds)):
        segments_dict['first_sent_times'].append(f'{x}:{y}')
  
  for i in range(len(keywords)):
    segments_dict['keyword_list'].append([x[0] for x in keywords[i]])

  # Convert dictionary to dataframe
  df = pd.DataFrame({k: pd.Series(l) for k, l in segments_dict.items()})
  
  return df

def get_predicted_segs_info(df, JoeRogan=True):
  """

  """
  label_segments_dict = {'first_sent_numbers' : [], 'length_of_segment': [], 
                   'keyword_list' : [], }
  if JoeRogan:
    keywords_list_sgcw = [
                          ['Neural Nets', 'AI', 'Brain'], 
                          ['Moving House', 'Materialistic Belongings'], 
                          ['Materialistic Belongings', 'Money', 'Wealth'],
                          ['People','Need','Manufacturing', 'Making', 'Fewer Law Grads'],
                          ['Materialistic Belongings'],
                          ['Gene Wilder', "Gene Wilder's House"],
                          ['Time Management', 'Productivity'],
                          ['Neuralink', 'Neuralink Implant Procedure', 'Benefits'],
                          ['AI symbiosis'], 
                          ['Communication', 'Language','Brain', 'Complexity'],
                          ['Alien', 'Biological Being'],
                          ['Memory', 'Changing your past'],
                          ['Simulation', 'Brain', 'Senses'],
                          ['Conciousness'],
                          ['Self-Improvement', 'Being less-wrong']
                          ]
  label_segments_dict['keyword_list'] = keywords_list_sgcw

  df_mini = df[df['labels']==1]  
  old_idx = 0
  for idx, row in df_mini.iterrows():
    label_segments_dict['first_sent_numbers'].append(idx)   # POSITION of each section
    length = np.int(idx) - np.int(old_idx)            # LENGTH of each section
    label_segments_dict['length_of_segment'].append(length) 
    old_idx = idx

  # Convert dictionary to dataframe
  df = pd.DataFrame({k: pd.Series(l) for k, l in label_segments_dict.items()})
  
  return df

## TRIAL with a SliceCast example podcast...
# df = pd.read_hdf('./SGCW/df_single_text_file.h5')
# trial_segments_df = get_segments_info(df)
# trial_segments_df

path = Path('./SGCW/shorter_formatted_plain.txt')
with open(path) as f:
    content = f.read()
keywords = get_keywords(content)
top_keywords = []
for i in range(len(keywords)):
  top_keywords.append(keywords[i][0])
print('len(top_keywords)', len(top_keywords))
print('top_keywords', top_keywords)

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 4768, "status": "ok", "timestamp": 1607079842426, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="LuZECMblDU7W" outputId="69d658d7-49a8-4cb5-c73a-4312d42577df"
# RUN: Inference-Segmented Data 'Slice'
df = pd.read_hdf('./SGCW/segd_shorter_formatted_plain.h5')

seg_df = get_segments_info(df)
seg_df.to_hdf('./SGCW/segs_slice_info.h5', key='dfs', mode='w')
seg_df

# + colab={"base_uri": "https://localhost:8080/", "height": 266} executionInfo={"elapsed": 6375, "status": "ok", "timestamp": 1606756475246, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="Vk7CWjMsD8w0" outputId="8ff48d00-7163-4b7f-b36b-d41d867416c9"
# RUN: Manually-Segmented Data 'Manual'
df = pd.read_hdf('./SGCW/segd_shorter_formatted_plain.h5')

label_seg_df = get_predicted_segs_info(df)
label_seg_df.to_hdf('./SGCW/segs_manual_info.h5', key='dfs', mode='w')
label_seg_df

# + colab={"base_uri": "https://localhost:8080/", "height": 514} executionInfo={"elapsed": 1216, "status": "ok", "timestamp": 1606756477633, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="Xk2A4N2iVtBr" outputId="ffc8d7d2-a6d6-43bc-c634-8f6c23775d44"


# + id="hLZfuBhbW0l8"
# Visualisation of Topics


# + [markdown] id="_IJf4ObbjSOE"
#
# # # ! pip install --upgrade gensim

# + id="L___ylv7k0Fs"
from sklearn.manifold import TSNE
from gensim.models import Word2Vec
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import nltk.data
import string
from gensim.models import Phrases


def Use_Whole_Transcript(path):
  content = []
  # Reformat the (shortened) Joe Rogan transcript to be consistent w SliceCast podcasts
  with open(path) as f:
    content = f.read()

    # # reading each line  
    # for line in f:
    #     content.append(line)
    #     # reading each word         
    #     for word in line.split():            
    #         content.append(word)

  # Split into sentences
  sents = nltk.tokenize.sent_tokenize(content)
  #print(sents)

  # Remove punctuation
  sents_ohne_punc = [''.join(c for c in s if c not in string.punctuation) for s in sents]
  # Remove empty strings
  final_sents = [s for s in sents_ohne_punc if s]
  #print(final_sents)

  content = []
  for sentence in final_sents:
      words = sentence.split()
      words_lower = [word.lower() for word in words]
      content.append(words_lower)

  return content

def Run_Model(keywords):
  # train model
  model = Word2Vec(keywords, min_count=1)
  # summarize the loaded model
  # print(model)
  # summarize vocabulary
  words = list(model.wv.vocab)
  # print(words)
  # access vector for one word
  # print(model['sentence'])
  # save model
  model.save('model.bin')
  # load model
  new_model = Word2Vec.load('model.bin')

  # fit a 2d PCA model to the vectors
  X = model[model.wv.vocab]
  pca = PCA(n_components=2)
  results = pca.fit_transform(X)  

  return results, words


def Detect_Key_Phrases(words_out, keywords):
  """
  the words 
  """



just_keywords = False # Trains only on the keywords extracted from each segment
all_words = False     # Will train Word2Vec on whole transcript
gensim_phraser = True # Will extract phrases uses gensim.models.phrases

if just_keywords:
  ## LOAD DATA---------------------------
  segs_manual_info_df = pd.read_hdf('./SGCW/segs_manual_info.h5', key='dfs')
  keywords_manual = segs_manual_info_df['keyword_list'].values
  print('\nkeywords_manual (Content put into model):', keywords_manual)

  segs_slice_info_df = pd.read_hdf('./SGCW/segs_slice_info.h5', key='dfs')
  keywords_slice = segs_slice_info_df['keyword_list'].values
  print('\nkeywords_slice (Content put into model):', keywords_slice, '\n')

  ## MANUAL ------------------------------
  results_manual, words_manual = Run_Model(keywords_manual)
  xs_manual = results_manual[:, 0]
  ys_manual = results_manual[:, 1]
  print('words_manual (Content out of model): ', words_manual)

  ## SLICECAST ---------------------------
  results_slice, words_slice= Run_Model(keywords_slice)
  xs_slice = results_slice[:, 0]
  ys_slice = results_slice[:, 1]
  print('words_slice (Content out of model): ', words_slice)

  ## SCATTER PLOTS -------------
  f, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, sharex=True, sharey=True,
                              figsize=(18,9))
  f.suptitle('Word2Vec Word Embedding Plots')

  ax1.scatter(xs_manual, ys_manual)
  for i, word in enumerate(words_manual):
    ax1.annotate(word, xy=(results_manual[i, 0], results_manual[i, 1]))
  ax1.set_title('Manual')

  ax2.scatter(xs_slice, ys_slice)
  for i, word in enumerate(words_slice):
    ax2.annotate(word, xy=(results_slice[i, 0], results_slice[i, 1]))
  ax2.set_title('SliceCast + ')

  f.show()

if all_words:
  print('top_keywords', top_keywords)
  results, words = Run_Model(top_keywords)
  xs = results[:, 0]
  ys = results[:, 1]
  print('words out: ', words)


  segs_slice_info_df = pd.read_hdf('./SGCW/segs_slice_info.h5', key='dfs')
  keywords_slice = segs_slice_info_df['keyword_list'].values
  keywords_slice_flat = [item for sublist in keywords_slice for item in sublist]
  print('\nkeywords_slice:', keywords_slice_flat, '\n')

  # Only want to plot they keywords
  (xs_key, ys_key, ordered_keywords) = ([], [], [])
  for i, word in enumerate(words):
    if word in keywords_slice_flat:
      xs_key.append(xs[i])
      ys_key.append(ys[i])
      ordered_keywords.append(word)

  plt.figure()
  plt.scatter(xs_key, ys_key)
  for i, word in enumerate(ordered_keywords):
      plt.annotate(word, xy=(xs_key[i], ys_key[i]))
  plt.title('SliceCast keywords with Word2Vec trained on Entire transcript')

  plt.show()

if gensim_phraser:
  path = Path('./SGCW/shorter_formatted_plain.txt')
  content = Use_Whole_Transcript(path)
  print('content being put in: ', content)
  phrases = Phrases(content, min_count=4, threshold=3)
  #sent = [u'the', u'mayor', u'of', u'new', u'york', u'was', u'there']
  sent = [u'also', u'i’ve', u'spent', u'a', u'lot', u'of', u'time', u'on', u'ai', u'and', u'neural', u'nets']
  print(phrases[sent])

  sents = [[u'trees', u'graph', u'minors'], [u'graph', u'minors']]
  for phrase in phrases[content]:
      print(phrase)


# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 1563, "status": "ok", "timestamp": 1607093799828, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="tUzrYB26jZj0" outputId="238cfa58-7003-4c93-8398-c812771e451e"
# Phrase selection


# + [markdown] id="KJ6iPWIA5ECR"
# from gensim.models import Phrases
#
# documents = ["the mayor of new york was there", "machine learning can be useful sometimes","new york mayor was present"]
#
# sentence_stream = [doc.split(" ") for doc in documents]
#  = Phrases(sentence_stream, min_count=1, threshold=1)
# sent = [u'the', u'mayor', u'of', u'new', u'york', u'was', u'there']
# print(bigram[sent])

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 977, "status": "ok", "timestamp": 1607088229408, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="ZrsoJ0R67DWC" outputId="44c8c209-476d-43fe-d29a-5236be35f583"
#from gensim.test.utils import datapath
from gensim.models.word2vec import Text8Corpus
from gensim.models.phrases import Phrases
#from gensim.models import phrases


# !pwd
# Load corpus and train a model.
sentences = Text8Corpus(datapath('testcorpus.txt'))
phrases = Phrases(sentences, min_count=1, threshold=1) #connector_words=phrases.ENGLISH_CONNECTOR_WORDS

# Use the model to detect phrases in a new sentence.
sent = [u'trees', u'graph', u'minors']
print(phrases[sent])
#[u'trees_graph', u'minors']

# Or transform multiple sentences at once.
sents = [[u'trees', u'graph', u'minors'], [u'graph', u'minors']]
for phrase in phrases[sents]:
    print(phrase)
#[u'trees_graph', u'minors']
[u'graph_minors']
#>>>
# Export a FrozenPhrases object that is more efficient but doesn't allow any more training.
#frozen_phrases = phrases.freeze()
#print(frozen_phrases[sent])
#[u'trees_graph', u'minors']

# + colab={"base_uri": "https://localhost:8080/"} executionInfo={"elapsed": 660, "status": "ok", "timestamp": 1607090599709, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="FjiRjNBw-v9s" outputId="ebc056af-c64e-4bdc-d166-2c926b3b9ba0"
#represent sentences by a 2-d matrix of cleaned tokens
import re

def get_sentences(input_file_pointer):
    while True:
        line = input_file_pointer.readline()
        if not line:
            break

        yield line

def clean_sentence(sentence):
    sentence = sentence.lower().strip()
    sentence = re.sub(r’[^a-z0-9\s]’, '’, sentence)
    return re.sub(r’\s{2,}’, ' ', sentence)

from spacy.lang.en.stop_words import STOP_WORDS
def tokenize(sentence):
    return [token for token in sentence.split() if token not in STOP_WORDS]


# + id="fghrzJSv5YSM"
## N-GRAM EXTRACTION 
from gensim.models.phrases import Phrases, Phraser

def build_phrases(sentences):
    phrases = Phrases(sentences,
                      min_count=2,
                      threshold=3,
                      progress_per=1000)
    return Phraser(phrases)

phrases_model.save('phrases_model.txt')

      

# + id="m9yad7EL2JA7"

phrases_model= Phraser.load('phrases_model.txt')

def sentence_to_bi_grams(phrases_model, sentence):
    return ' '.join(phrases_model[sentence])

def sentences_to_bi_grams(n_grams, input_file_name, output_file_name):
    with open(input_file_name, 'r') as input_file_pointer:
        with open(output_file_name, 'w+') as out_file:
            for sentence in get_sentences(input_file_pointer):
                cleaned_sentence = clean_sentence(sentence)
                tokenized_sentence = tokenize(cleaned_sentence)
                parsed_sentence = sentence_to_bi_grams(n_grams, tokenized_sentence)
                out_file.write(parsed_sentence + '\n')


# + id="BzZIi3HN2qUd"
# Rest of stuff



# + [markdown] id="3LXIxSDY2Xp1"
# ## Create Dict out of Word Embeddings
# # Manual-----------------------------
# Keyword_Embeddings_dict_manual = {'Word':[], 'X': [], 'Y': []}
#
# for i, word in enumerate(words_manual):
#   Keyword_Embeddings_dict_manual['Word'].append(word)
#   Keyword_Embeddings_dict_manual['X'].append(result_manual[i, 0])
#   Keyword_Embeddings_dict_manual['Y'].append(result_manual[i, 1])
#
# Keyword_Embeddings_df_manual = pd.DataFrame({k: pd.Series(l) for k, l in Keyword_Embeddings_dict_manual.items()})
# Keyword_Embeddings_df_manual.to_hdf('./SGCW/Keyword_Embeddings_df_manual.h5', key='dfs', mode='w')
#
# Keyword_Embeddings_df_manual

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} executionInfo={"elapsed": 575, "status": "ok", "timestamp": 1606757743974, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="RoBAVeQGlEBL" outputId="92d3bf86-4561-4009-a994-e8e79d821be9"
## Create Dict out of Word Embeddings
# SliceCast-------------------------
Keyword_Embeddings_dict_slice = {'Word':[], 'X': [], 'Y': []}

for i, word in enumerate(words_slice):
  Keyword_Embeddings_dict_slice['Word'].append(word)
  Keyword_Embeddings_dict_slice['X'].append(result_slice[i, 0])
  Keyword_Embeddings_dict_slice['Y'].append(result_slice[i, 1])

Keyword_Embeddings_df_slice = pd.DataFrame({k: pd.Series(l) for k, l in Keyword_Embeddings_dict_slice.items()})
Keyword_Embeddings_df_slice.to_hdf('./SGCW/Keyword_Embeddings_df_slice.h5', key='dfs', mode='w')
Keyword_Embeddings_df_slice

# + colab={"base_uri": "https://localhost:8080/", "height": 979} executionInfo={"elapsed": 1228, "status": "ok", "timestamp": 1606757765203, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="44LaxbziOV1M" outputId="e322bfe1-391d-419a-a65b-71a806fa8453"
## CALCULATE AVERAGE TOPIC POSITION for each segment
# i.e. average of the word vectors in each list (each list corresponds to a segment)
Keywo_Embed_df_manual = pd.read_hdf('./SGCW/Keyword_Embeddings_df_manual.h5', key='dfs')
Keywo_Embed_df_slice = pd.read_hdf('./SGCW/Keyword_Embeddings_df_slice.h5', key='dfs')

segs_manual_info_df = pd.read_hdf('./SGCW/segs_manual_info.h5', key='dfs')
keywords_manual = segs_manual_info_df['keyword_list'].values

segs_slice_info_df = pd.read_hdf('./SGCW/segs_slice_info.h5', key='dfs')
keywords_slice = segs_slice_info_df['keyword_list'].values

(av_xs_manual, av_ys_manual, segment_numbs_manual) = ([], [], [])
(av_xs_slice, av_ys_slice, segment_numbs_slice) = ([], [], [])


for idx, list_manual in enumerate(keywords_manual):
  ## MANUAL
  # Access XY coord for each word in the sublist
  word_list_X = []
  word_list_Y = []
  for word in list_manual:
    #print(word)
    word_list_X.append(Keywo_Embed_df_manual[Keywo_Embed_df_manual['Word']==word].X.values[0])
    word_list_Y.append(Keywo_Embed_df_manual[Keywo_Embed_df_manual['Word']==word].Y.values[0])

  # Finding average positions
  av_xs_manual.append(np.mean(word_list_X))
  av_ys_manual.append(np.mean(word_list_Y))
  
  segment_numbs_manual.append(idx)

for idx, list_slice in enumerate(keywords_slice):
  ## SLICE--------------------------------------
  # Access XY coord for each word in the sublist
  word_list_X = []
  word_list_Y = []
  for word in list_slice:
    #print(word)
    word_list_X.append(Keywo_Embed_df_slice[Keywo_Embed_df_slice['Word']==word].X.values[0])
    word_list_Y.append(Keywo_Embed_df_slice[Keywo_Embed_df_slice['Word']==word].Y.values[0])

  # Finding average positions
  av_xs_slice.append(np.mean(word_list_X))
  av_ys_slice.append(np.mean(word_list_Y))


  segment_numbs_slice.append(idx)
  
  
Topic_avs_df_manual = pd.DataFrame()
Topic_avs_df_slice = pd.DataFrame()
# Add to new df with Topic Number, Words, Av word embedding Position
Topic_avs_df_manual['Topic_Num'] = segment_numbs_manual
Topic_avs_df_slice['Topic_Num'] = segment_numbs_slice

Topic_avs_df_manual['Av_X'] = av_xs_manual
Topic_avs_df_manual['Av_Y'] = av_ys_manual

Topic_avs_df_slice['Av_X'] = av_xs_slice
Topic_avs_df_slice['Av_Y'] = av_ys_slice

# Plot the scatter of Av word embedding Position
Topic_avs_df_manual.to_hdf('./SGCW/Topic_avs_df_manual.h5', key='dfs', mode='w')
Topic_avs_df_slice.to_hdf('./SGCW/Topic_avs_df_slice.h5', key='dfs', mode='w')

Topic_avs_df_manual
#should have 15 topics

# + colab={"base_uri": "https://localhost:8080/", "height": 514} executionInfo={"elapsed": 3034, "status": "ok", "timestamp": 1606759182424, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="yAcYEHJOmUih" outputId="0f736baf-acd7-40a6-d695-4afc170fe96b"
Topic_avs_df_slice
# should have fewer topics than manual above (6)

# + colab={"base_uri": "https://localhost:8080/", "height": 266} executionInfo={"elapsed": 663, "status": "ok", "timestamp": 1606759184980, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="2K9n3O7MS0Rp" outputId="ae4a7a56-f93d-4f8b-c191-260e97285470"
df_manual = pd.read_hdf('./SGCW/Topic_avs_df_manual.h5', key='dfs')
df_slice = pd.read_hdf('./SGCW/Topic_avs_df_slice.h5', key='dfs')

f, (ax1, ax2) = plt.subplots(nrows=1, ncols=2,
                             figsize=(18,9)) #sharex=True
f.suptitle('Averaged word embeddings for Keywords in each Podcast Segment')


## MANUAL 
labels = df_manual['Topic_Num'].values
xs = df_manual['Av_X'].values
ys = df_manual['Av_Y'].values

ax1.quiver(xs[:-1], ys[:-1], xs[1:]-xs[:-1], ys[1:]-ys[:-1], scale_units='xy', 
           angles='xy', scale=1, color='b', width=0.005)

# zip joins x and y coordinates in pairs
for x, y, label in zip(xs, ys, labels):

    ax1.annotate(label+1, # this is the text
                 (x,y), # this is the point to label
                 textcoords="offset points", # how to position the text
                 xytext=(0,10), # distance from text to points (x,y)
                 ha='center') # horizontal alignment can be left, right or center

#plot special colours for the first and last point
ax1.plot([xs[0]], [ys[0]], 'o', color='green', markersize=10, label='Beginning of Conversation')
ax1.plot([xs[-1]], [ys[-1]], 'o', color='red', markersize=10, label='End of Conversation')
ax1.set_title('Manual')

## SLICECAST
labels = df_slice['Topic_Num'].values
xs = df_slice['Av_X'].values
ys = df_slice['Av_Y'].values

ax2.quiver(xs[:-1], ys[:-1], xs[1:]-xs[:-1], ys[1:]-ys[:-1], scale_units='xy', 
           angles='xy', scale=1, color='b', width=0.005)

# zip joins x and y coordinates in pairs
for x, y, label in zip(xs, ys, labels):

    ax2.annotate(label+1, # this is the text
                 (x,y), # this is the point to label
                 textcoords="offset points", # how to position the text
                 xytext=(0,10), # distance from text to points (x,y)
                 ha='center') # horizontal alignment can be left, right or center

#plot special colours for the first and last point
ax2.plot([xs[0]], [ys[0]], 'o', color='green', markersize=10)
ax2.plot([xs[-1]], [ys[-1]], 'o', color='red', markersize=10)

ax2.set_title('SliceCast')
f.legend()
f.show()

# + colab={"base_uri": "https://localhost:8080/", "height": 610} executionInfo={"elapsed": 2249, "status": "ok", "timestamp": 1606760363024, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="AF5NExF0oa_h" outputId="9cd2d92e-9bf1-4543-8d46-25d6ac56a749"
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d.axes3d import get_test_data
from mpl_toolkits import mplot3d

df_manual = pd.read_hdf('./SGCW/Topic_avs_df_manual.h5', key='dfs')
labels_manual = df_manual['Topic_Num'].values
df_slice = pd.read_hdf('./SGCW/Topic_avs_df_slice.h5', key='dfs')
labels_slice = df_slice['Topic_Num'].values

# set up a figure twice as wide as it is tall
fig = plt.figure(figsize=(22,11)) #plt.figaspect(0.5))
fig.suptitle('Movement through topic space over time')

# set up the axes for the first plot
ax1 = fig.add_subplot(1, 2, 1, projection='3d')

## MANUAL------------------------------------------
# Data for a three-dimensional line
word_emb_xs = df_manual['Av_X'].values
word_emb_ys = df_manual['Av_Y'].values
segment_numbers = range(len(word_emb_xs))

#ax.plot3D(xs, segment_numbers, ys, 'bo-')
ax1.set_xlabel('$time (Segment Number)$', fontsize=13)
ax1.set_zlabel('$Y$', fontsize=20)
ax1.set_ylabel('$X$', fontsize=20, rotation = 0)
ax1.zaxis.set_rotate_label(False) 
ax1.set_title('Manual')

for x, y, z, label in zip(segment_numbers, word_emb_xs, word_emb_ys, labels_manual):
  ax1.plot([x], [y], [z],'o-')
  ax1.text(x, y, z, label+1, size=10)

## AXIS STUFF
ax1.dist = 13
ax1 = plt.gca()
#ax1.xaxis.set_ticklabels([])
ax1.yaxis.set_ticklabels([])
ax1.zaxis.set_ticklabels([])

# for line in ax1.xaxis.get_ticklines():
#     line.set_visible(False)
for line in ax1.yaxis.get_ticklines():
    line.set_visible(False)
for line in ax1.zaxis.get_ticklines():
    line.set_visible(False)

## SLICE---------------------------------------------
# Data for a three-dimensional line
ax2 = fig.add_subplot(1, 2, 2, projection='3d')
word_emb_xs = df_slice['Av_X'].values
word_emb_ys = df_slice['Av_Y'].values
segment_numbers = range(len(word_emb_xs))

#ax.plot3D(xs, segment_numbers, ys, 'bo-')
ax2.set_xlabel('$time (Segment Number)$', fontsize=13)
ax2.set_zlabel('$Y$', fontsize=20)
ax2.set_ylabel('$X$', fontsize=20, rotation = 0)
ax2.zaxis.set_rotate_label(False) 
ax2.set_title('SliceCast')

for x, y, z, label in zip(segment_numbers, word_emb_xs, word_emb_ys, labels_slice):
  ax2.plot([x], [y], [z],'o-')
  ax2.text(x, y, z, label+1, size=10)

## AXIS STUFF
ax2.dist = 13
ax2 = plt.gca()
#ax2.xaxis.set_ticklabels([])
ax2.yaxis.set_ticklabels([])
ax2.zaxis.set_ticklabels([])

# for line in ax2.xaxis.get_ticklines():
#     line.set_visible(False)
for line in ax2.yaxis.get_ticklines():
    line.set_visible(False)
for line in ax2.zaxis.get_ticklines():
    line.set_visible(False)

fig.show()

#NOT PLOTTING ALL THE POINTS! MANUAL SHOULD HAVE LIKE 15!

# + colab={"base_uri": "https://localhost:8080/", "height": 685} executionInfo={"elapsed": 2496, "status": "ok", "timestamp": 1606762440412, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="sgc_1GrWrU_w" outputId="2feb35a7-dd22-41c3-ff41-9c5c8a789bf8"
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d.axes3d import get_test_data
from mpl_toolkits import mplot3d
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.patches import FancyArrowPatch
from mpl_toolkits.mplot3d import proj3d

class Arrow3D(FancyArrowPatch):
    def __init__(self, xs, ys, zs, *args, **kwargs):
        FancyArrowPatch.__init__(self, (0,0), (0,0), *args, **kwargs)
        self._verts3d = xs, ys, zs

    def draw(self, renderer):
        xs3d, ys3d, zs3d = self._verts3d
        xs, ys, zs = proj3d.proj_transform(xs3d, ys3d, zs3d, renderer.M)
        self.set_positions((xs[0],ys[0]),(xs[1],ys[1]))
        FancyArrowPatch.draw(self, renderer)
    
df_manual = pd.read_hdf('./SGCW/Topic_avs_df_manual.h5', key='dfs')
labels_manual = df_manual['Topic_Num'].values
df_slice = pd.read_hdf('./SGCW/Topic_avs_df_slice.h5', key='dfs')
labels_slice = df_slice['Topic_Num'].values

# set up a figure twice as wide as it is tall
fig = plt.figure(figsize=(22,11)) #plt.figaspect(0.5))
fig.suptitle('Movement through topic space over time')

# set up the axes for the first plot
ax1 = fig.add_subplot(1, 2, 1, projection='3d')

##MANUAL ----------------------------------------------------------
# Data for a three-dimensional line
word_emb_xs = df_manual['Av_X'].values
word_emb_ys = df_manual['Av_Y'].values
label_seg_df = pd.read_hdf('./SGCW/segs_manual_info.h5', key='dfs')
segment_numbers = label_seg_df['first_sent_numbers'].values
print('segment_numbers', segment_numbers)

#ax.plot3D(xs, segment_numbers, ys, 'bo-')
ax1.set_xlabel('$time (Segment Number)$', fontsize=13)
ax1.set_ylabel('$X$', fontsize=20, rotation = 0)
ax1.set_zlabel('$Y$', fontsize=20)
ax1.zaxis.set_rotate_label(False) 
ax1.set_title('Manual')

cnt = 0
# (old_x, old_y, old_z) = (0, 0, 0)
for x, y, z, label in zip(segment_numbers, word_emb_xs, word_emb_ys, labels_manual):
  cnt +=1
  ax1.plot([x], [y], [z],'o') #markerfacecolor='k', markeredgecolor='k', marker='o', markersize=5, alpha=0.6)
  ax1.text(x, y, z, label+1, size=10)
  if cnt ==1:
    (old_x, old_y, old_z) = (x, y, z)
    continue

  a = Arrow3D([old_x, x], [old_y,y], [old_z, z], mutation_scale=20, lw=3, arrowstyle="-|>", color="r")
  ax1.add_artist(a)

  (old_x, old_y, old_z) = (x, y, z)


## AXIS STUFF
ax1.dist = 13
ax1 = plt.gca()
# ax.xaxis.set_ticklabels([])
ax1.yaxis.set_ticklabels([])
ax1.zaxis.set_ticklabels([])

# for line in ax.xaxis.get_ticklines():
#     line.set_visible(False)
for line in ax1.yaxis.get_ticklines():
    line.set_visible(False)
for line in ax1.zaxis.get_ticklines():
    line.set_visible(False)

##SLICE ----------------------------------------------------------

# set up the axes for the second plot
ax2 = fig.add_subplot(1, 2, 2, projection='3d')

# Data for a three-dimensional line
word_emb_xs = df_slice['Av_X'].values
word_emb_ys = df_slice['Av_Y'].values
label_seg_df = pd.read_hdf('./SGCW/segs_slice_info.h5', key='dfs')
segment_numbers = label_seg_df['first_sent_numbers'].values

#ax.plot3D(xs, segment_numbers, ys, 'bo-')
ax2.set_xlabel('$time (Segment Number)$', fontsize=13)
ax2.set_ylabel('$X$', fontsize=20, rotation = 0)
ax2.set_zlabel('$Y$', fontsize=20)
ax2.zaxis.set_rotate_label(False) 
ax2.set_title('Slice')

cnt = 0
# (old_x, old_y, old_z) = (0, 0, 0)
for x, y, z, label in zip(segment_numbers, word_emb_xs, word_emb_ys, labels_slice):
  cnt +=1
  ax2.plot([x], [y], [z],'o') #markerfacecolor='k', markeredgecolor='k', marker='o', markersize=5, alpha=0.6)
  ax2.text(x, y, z, label+1, size=10)
  if cnt ==1:
    (old_x, old_y, old_z) = (x, y, z)
    continue

  a = Arrow3D([old_x, x], [old_y,y], [old_z, z], mutation_scale=20, lw=3, arrowstyle="-|>", color="r")
  ax2.add_artist(a)

  (old_x, old_y, old_z) = (x, y, z)

## AXIS STUFF
ax2.dist = 13
ax2 = plt.gca()
# ax2.xaxis.set_ticklabels([])
ax2.yaxis.set_ticklabels([])
ax2.zaxis.set_ticklabels([])

# for line in ax2.xaxis.get_ticklines():
#     line.set_visible(False)
for line in ax2.yaxis.get_ticklines():
    line.set_visible(False)
for line in ax2.zaxis.get_ticklines():
    line.set_visible(False)

fig.show()

# + colab={"base_uri": "https://localhost:8080/", "height": 702} executionInfo={"elapsed": 2542, "status": "ok", "timestamp": 1606906913052, "user": {"displayName": "Shona CW", "photoUrl": "", "userId": "17927708844475205952"}, "user_tz": 0} id="oduv2AIhyNQ9" outputId="087e92ec-9713-4c6d-9ec2-f1246fdcb762"

# mkdir GloVe
curl -Lo GloVe/glove.840B.300d.zip http://nlp.stanford.edu/data/glove.840B.300d.zip
unzip GloVe/glove.840B.300d.zip -d GloVe/
# mkdir fastText
curl -Lo fastText/crawl-300d-2M.vec.zip https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M.vec.zip
unzip fastText/crawl-300d-2M.vec.zip -d fastText/

# + id="Wt6oMTJpzSjI"


StarcoderdataJupyterScriptsDedupFiltered