# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="q73_fLRvTQYI"
# Sound Event Detection using Approach 1

# + [markdown] colab_type="text" id="X3PASjCITQC0"
# This colab demonstrates how to extract the AudioSet embeddings, using a VGGish deep neural network (DNN).

# + [markdown] colab_type="text" id="IKSLc0bIB1QS"
# Based on the directions at: https://github.com/tensorflow/models/tree/master/research/audioset

# + colab={"base_uri": "https://localhost:8080/", "height": 749} colab_type="code" executionInfo={"elapsed": 9304, "status": "ok", "timestamp": 1584893793375, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="228fji9C6c-q" outputId="9febc986-a23d-437c-bcd5-b12bef28e230"
# !lscpu
# !nvidia-smi

# + colab={"base_uri": "https://localhost:8080/", "height": 124} colab_type="code" executionInfo={"elapsed": 33533, "status": "ok", "timestamp": 1584893817627, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="xNUdYfA11vGR" outputId="df02d4a3-232e-438c-f597-1b0e9a079c8b"
#Google drive access
import os
from google.colab import drive
drive.mount('/content/gdrive',force_remount = True)

#Directory
root_path = 'gdrive/My Drive/SoundEventDetection/modelTraining/'
os.chdir(root_path)

# + colab={} colab_type="code" id="O1YVQb-MBiUx"
# %%capture
#Install necessary software
# !pip install six
# !pip install h5py
# !pip install pydub
# !pip install numpy
# !pip install scipy
# !pip install keras
# !pip install future
# !pip install resampy
# !pip install ipython
# !pip install soundfile
# !pip install pysoundfile
# !pip install scikit-learn
# !apt-get install libsndfile1

# !pip install python==3.6
# !pip install matplotlib
# !pip install cudnn==7.1.2
# !pip install cudatoolkit==9
# !pip install tensorflow-gpu==1.12.0

#Install: cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb
# !dpkg -i cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb
# !apt-key add /var/cuda-repo-9-0-local/7fa2af80.pub
# !apt-get update
# !apt-get install cuda=9.0.176-1
#OR
# #!wget https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb

#VGGish model checkpoint, in TensorFlow checkpoint format.
import os 
os.chdir("trained_models")
# !wget https://storage.googleapis.com/audioset/vggish_model.ckpt
os.chdir("..")

# + colab={} colab_type="code" id="jmhLEKk8bxW2"
# %%capture
#Important DO NOT DELETE
#import six
import sys
#import h5py
#import math
#import glob
import json
#import time
import numpy as np
import pandas as pd
import soundfile as sf
import tensorflow as tf
import matplotlib.pyplot as plt
#from pydub.playback import play
#from pydub import AudioSegment
#from scipy.io import wavfile
#from scipy.io.wavfile import write

sys.path.insert(1, os.path.join(sys.path[0], '../'))

#ML imports
#from keras.layers import Input, Dense, BatchNormalization, Dropout, Activation, Concatenate
from keras.layers import Lambda
from keras.optimizers import Adam
from keras.models import load_model
from keras.models import Model
import keras.backend as K

#External .py scripts
from lib import mel_features
from lib import vggish_input
from lib import vggish_params
from lib import vggish_postprocess
from lib import vggish_slim
from lib import utilities
from lib import data_generator
from lib.train_functions import evaluateCore, trainCore, average_pooling, max_pooling, attention_pooling, pooling_shape, train, writeToFile

# + colab={} colab_type="code" id="ANV8Fhf2SeDj"
# %%capture
trainingType = ['decision_level_max_pooling', 
                'decision_level_average_pooling', 
                'decision_level_single_attention', 
                'decision_level_multi_attention', 
                'feature_level_attention']

balanced = ['no_balance', 'balance_in_batch']


def loadModel(num1,num2):
  args = {
          "data_dir" : "packed_features/",
          "workspace" : "workspace/",
          "mini_data" : False,
          "balance_type" : balanced[num2], #'no_balance', 'balance_in_batch'
          "model_type" : trainingType[num1], 
          "learning_rate" : 1e-3,
          "filename":utilities.get_filename("work/")
  }

  #Output directories
  sub_dir = os.path.join(args["filename"],
                        'balance_type={}'.format(args["balance_type"]),
                        'model_type={}'.format(args["model_type"]))
  
  modelFiles = [['200k_DLMP_B','200k_DLMP_U'],['200k_DLAP_B','200k_DLAP_U'],
                ['200k_DLSA_B','200k_DLSA_U'],['200k_DLMA_B','200k_DLMA_U'],
                ['200k_FLA_B','200k_FLA_U']]

  #CNN Architecture
  model = train(args, 1)
  optimizer = Adam(lr=args["learning_rate"])
  model.compile(loss='binary_crossentropy', optimizer=optimizer)

  #Model dir
  #models_dir ="workspace/models/work/balance_type=balance_in_batch/model_type=feature_level_attention"
  models_dir = "trained_models/"+modelFiles[num1][num2]+".h5"

  #latest_file ="workspace/models/work/balance_type=no_balance/model_type=decision_level_multi_attention/md_4000_iters.h5"
  print(models_dir)
  model.load_weights(models_dir)
  print(type(model))
  return model
  
model = loadModel(2,1)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 360237, "status": "ok", "timestamp": 1584894144366, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="TVHHYowNsZEK" outputId="36c91ae1-4f0b-4158-8d18-29e6b0bc5af5"
'''
from keras.models import load_model
model = load_model( "trained_models/"+"200k_DLSA_U"+".h5")
'''


# + colab={} colab_type="code" id="e_AoyOcYSd5S"
#start_time = time.time()
def getTime(num, start_time):
  print(str(num)+ ":" + str(time.time() - start_time))
  return time.time()


# + colab={} colab_type="code" id="Huv3co4e1JL2"
#Get labels
class_labels = pd.read_csv("lib/class.csv").drop(columns=['index','mid'])['display_name'].to_numpy()

def getLabeling(class_labels,num):
   return class_labels[num]


# + colab={} colab_type="code" id="rx-mdaBcSeA1"
#Predict on input
def predictOnVggishInput(df, model):
  results = []
  attention_results = []
  #Handling edge case of 10 seconds
  length = df.shape[0]
  if(length==10):
    length = 11

  for endNumber in range(10,length):
    startNumber = endNumber - 10
    df_final = [df[startNumber:endNumber]]
    df_final = np.asarray(df_final)
    df_final = (np.float32(df_final) - 128.) / 128.
    output = model.predict(df_final)
    #attention_layer_output=model.get_layer(layer_name).output
    #intermediate_model=tf.keras.models.Model(inputs=model.input,outputs=layer_output)
    #attention_layer_prediction=intermediate_model.predict(df_final)
    #attention_results.append(attention_layer_prediction)
    results.append(output.astype(np.float32))
  return results


# + colab={} colab_type="code" id="bqkvenjFSp89"
def plotResults(df):
  #Create plot
  plt.rcParams["figure.figsize"] = (20,10)
  groups = df.groupby('Class')
  # Plot
  fig, ax = plt.subplots()
  ax.margins(0.05) # Optional, just adds 5% padding to the autoscaling
  for name, group in groups:
      ax.plot(group.StartTime, group.Probability, marker='o', linestyle='-', ms=6, label=getLabeling(class_labels,name))

  f = "output/dogprobsld.csv"
  df = pd.read_csv(f)
  #plt.plot(df[df['Class']==0]['Second'],df[df['Class']==0]['Class']+1.1, marker='o', linestyle='', ms=6,color='C0', label="Speech SLD") 
  #plt.plot(df[df['Class']==74]['Second'],df[df['Class']==74]['Class']-72.8, marker='o', linestyle='', ms=6,color='C3', label="Dog SLD")

  ax.legend()
  #plt.xlim((0, 180))
  plt.show()


# + colab={} colab_type="code" id="BwtZ__sWIUX0"
#VGGish transformation
def vggish_fullprocess(wav_file,data,sr,switch):
  # wav_file = "output/dog.wav" #tempSave
  checkpoint = "trained_models/vggish_model.ckpt"
  pca_params = "trained_models/vggish_pca_params.npz"

  if switch==1:
    examples_batch = vggish_input.wavfile_to_examples(wav_file)
  else:
    examples_batch = vggish_input.waveform_to_examples(data,sr)

  # Prepare a postprocessor to munge the model embeddings.
  pproc = vggish_postprocess.Postprocessor(pca_params)
  with tf.Graph().as_default(), tf.Session() as sess:
    # Define the model in inference mode, load the checkpoint, and
    # locate input and output tensors.
    vggish_slim.define_vggish_slim(training=False)
    vggish_slim.load_vggish_slim_checkpoint(sess, checkpoint)
    features_tensor = sess.graph.get_tensor_by_name(vggish_params.INPUT_TENSOR_NAME)
    embedding_tensor = sess.graph.get_tensor_by_name(vggish_params.OUTPUT_TENSOR_NAME)

    # Run inference and postprocessing.
    [embedding_batch] = sess.run([embedding_tensor],feed_dict={features_tensor: examples_batch})

  postprocessed_batch = pproc.postprocess(embedding_batch)

  return postprocessed_batch



# + colab={} colab_type="code" id="M5gCYFOcS2Ej"
# #Load vggish output
# filename = 'output/data.h5'

def predictOnAudioFile(wav_file):
  wav_data, sr = sf.read(wav_file, dtype='int16')
  assert wav_data.dtype == np.int16, 'Bad sample type: %r' % wav_data.dtype
  samples = wav_data / 32768.0  # Convert to [-1.0, +1.0]
  if len(samples.shape) > 1:
    data = np.mean(samples, axis=1)
  else:
    data = samples
  data = np.asarray(data)
  result_list = []

  step = int(data.shape[0]/sr)
  #step = 30
  for segment in np.arange(0,int(data.shape[0]/sr),step):
    if((data.shape[0]/sr)-segment<step):
      break

    if segment>0:
      segmentStart = segment - 9
    else:
      segmentStart = segment

    data_segment = data[sr*segmentStart:sr*(segment+step)]
    data_embedding = vggish_fullprocess(wav_file,data_segment,sr,2)
    result = predictOnVggishInput(data_embedding, model)
    result_list.append(result)
  return result_list

result_list = predictOnAudioFile('output/dog.wav')

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 377437, "status": "ok", "timestamp": 1584894161616, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="B7ob0l45TPG6" outputId="c2c88c22-270a-468d-f1b6-74ae020cab52"
len(result_list[0][0])

# + colab={} colab_type="code" id="Hhtx0-HHZnWx"
results = []
for i in range(0, len(result_list[0])):
  [t]  = result_list[0][i]
  for j in range(0,len(t)):
    if t[j] < 0.25:
      t[j] = 0
  t = t.tolist()
  results.append(t)

results = np.asarray(results)

results = np.transpose(results)

# + colab={"base_uri": "https://localhost:8080/", "height": 243} colab_type="code" executionInfo={"elapsed": 377424, "status": "ok", "timestamp": 1584894161623, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="Q4DKT8epcyHk" outputId="1f83c8bb-c817-4695-86ed-e1fc6d86909d"
results

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 377416, "status": "ok", "timestamp": 1584894161625, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="gPMdrJBXcCkZ" outputId="a104cce9-4c60-4a0a-a7c1-fb8bdbbf2796"
len(result_list)

# + colab={"base_uri": "https://localhost:8080/", "height": 54} colab_type="code" executionInfo={"elapsed": 377706, "status": "ok", "timestamp": 1584894161926, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="q9zgSWS_6ntI" outputId="ed5cff26-f93e-464b-e58a-b7f5f84ce63f"
masterList = []
counter = 1
for i in result_list[0]:
  a = np.argpartition(i[0], -10)[-10:]
  secondsMap={}
  valuesMap={}
  for aa in a:
    valuesMap[getLabeling(class_labels,aa)] = 1.*i[0][aa]
  secondsMap['second'] = counter
  secondsMap['values'] = valuesMap
  masterList.append(secondsMap)
  counter = counter + 1

json.dumps(masterList)

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" executionInfo={"elapsed": 1813, "status": "ok", "timestamp": 1584894255997, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="d18Lrj5QS2QU" outputId="e032ab7b-73f1-4de0-c62c-740dd9ce1469"
# Manipulate results
result_table = pd.DataFrame(columns=['StartTime','Class','Probability'])
i = 0
threshold = 0.2
for result_segment in result_list:
  for result in result_segment:
    result = result[0]
    classes = np.arange(len(result))[result>threshold]
    probs = result[result>threshold]
    for j in np.arange(len(probs)):
      result_table = result_table.append({'StartTime' : 0+i,'Class' : classes[j] , 'Probability' : probs[j]}, ignore_index=True)
    i=i+1

  result_table["Class"] = result_table["Class"].astype(int)

# Change column type
# result_table['Class'] = pd.Categorical(result_table.Class)
plotResults(result_table)
result_table.head(20)

# + colab={} colab_type="code" id="8YnB8SLU056F"
rdf = result_table
rdf['EndTime'] = rdf['StartTime'] + 10


# + colab={} colab_type="code" id="FWO2BzUvZbBU"
def calc_weighted_prob(data):
  data['middle_second'] = data[['StartTime','EndTime']].mean(axis=1)
  data['weight'] = (5- abs(data['second'] - data['middle_second']))+1
  weight_sum = data['weight'].sum()
  data['weight'] = data['weight']/weight_sum
  data['weighted_prob'] = data['Probability'] * data['weight']
  d = {'second': data['second'].iloc[0],'Class': data['Class'].iloc[0] , 'smoothed_prob': [data['weighted_prob'].sum()]}
  smoothed_prob = pd.DataFrame(data=d)
  return smoothed_prob


# + colab={} colab_type="code" id="oG3vn_dYOHtK"
def plotSmoothedResults(df):
  #Create plot
  plt.rcParams["figure.figsize"] = (20,10)
  groups = df.groupby('Class')
  # Plot
  fig, ax = plt.subplots()
  ax.margins(0.05) # Optional, just adds 5% padding to the autoscaling
  for name, group in groups:
      ax.plot(group.second, group.smoothed_prob, marker='o', linestyle='-', ms=6, label=getLabeling(class_labels,name))

  f = "output/dogprobsld.csv"
  df = pd.read_csv(f)
  #plt.plot(df[df['Class']==0]['Second'],df[df['Class']==0]['Class']+1.1, marker='o', linestyle='', ms=6,color='C0', label="Speech SLD") 
  #plt.plot(df[df['Class']==74]['Second'],df[df['Class']==74]['Class']-72.8, marker='o', linestyle='', ms=6,color='C3', label="Dog SLD")

  ax.legend()
  #plt.xlim((0, 180))
  plt.show()


# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" executionInfo={"elapsed": 384795, "status": "ok", "timestamp": 1584894169052, "user": {"displayName": "<NAME>", "photoUrl": "https://lh4.googleusercontent.com/-TpQl_yrDGdU/AAAAAAAAAAI/AAAAAAAAAlQ/HoAfg_rDDrA/s64/photo.jpg", "userId": "01397164172225109514"}, "user_tz": -60} id="R-xeoFQIze06" outputId="23c648a9-ccd5-40c1-e3a6-801b85492be5"
first_second = np.min(rdf['StartTime'])
last_second = np.max(rdf['EndTime'])

smoothed_results = pd.DataFrame(columns=['second','Class','smoothed_prob'])

for second in np.arange(first_second,last_second+1):
  data_second = rdf[(rdf['StartTime'] <=second) & (rdf['EndTime'] >= second)]
  data_second['second'] = second
  for Class in np.unique(np.asarray(data_second['Class'])):
    df = calc_weighted_prob(data_second[data_second['Class']==Class])
    smoothed_results = smoothed_results.append(df)

plotSmoothedResults(smoothed_results)

# + colab={} colab_type="code" id="1hFIXsLG5so3"


StarcoderdataJupyterScriptsDedupFiltered