<jupyter_start><jupyter_text>Monophonic SED on UrbanSound8K dataset  1 Import major libraries<jupyter_code>import numpy as np
import plaidml.keras
import os
plaidml.keras.install_backend()
os.environ["KERAS_BACKEND"] = "plaidml.keras.backend"

from keras import backend as K
print(K)

import keras
from keras.preprocessing.image import ImageDataGenerator

print("Keras version %s" %keras.__version__)<jupyter_output><module 'plaidml.keras.backend' from 'E:\\anaconda3\\envs\\plaidML\\lib\\site-packages\\plaidml\\keras\\backend.py'>
Keras version 2.2.4<jupyter_text>2 Load meta file<jupyter_code>import pandas as pd

# Set the path to the full UrbanSound dataset 
fulldatasetpath = '/Xception/UrbanSound8K/audio/'

meta = pd.read_csv('E:\\Xception\\UrbanSound8K\\metadata\\UrbanSound8K.csv')<jupyter_output><empty_output><jupyter_text>3 Data analysis  Labels distribution in dataset<jupyter_code># Class distribution
meta['class'].value_counts()<jupyter_output><empty_output><jupyter_text>Audio properties<jupyter_code>import struct
import re
import os

fulldatasetpath = '/Xception/UrbanSound8K/audio/'

# support to read file header and return audio properties
def read_header(filename):
    wave = open(filename,"rb")
    riff = wave.read(12)
    fmat = wave.read(36)
    num_channels_string = fmat[10:12]
    num_channels = struct.unpack('<H', num_channels_string)[0]
    sample_rate_string = fmat[12:16]
    sample_rate = struct.unpack("<I",sample_rate_string)[0]
    bit_depth_string = fmat[22:24]
    bit_depth = struct.unpack("<H",bit_depth_string)[0]
    return (num_channels, sample_rate, bit_depth)

# Read every file header to collect audio properties
audiodata = []
for index, row in meta.iterrows():
    cat = str(row["class"])
    fold = 'fold'+str(row["fold"])
    name = str(row["slice_file_name"])
    file_name = os.path.join(fulldatasetpath, fold, name)
    audio_props = read_header(file_name)
    duration = row['end'] - row['start']
    audiodata.append((name, fold, cat, duration) + audio_props)

# Convert into a Pandas dataframe
audiodatadf = pd.DataFrame(audiodata, columns=['file', 'fold', 'class', 'duration', 'channels','sample_rate','bit_depth'])<jupyter_output><empty_output><jupyter_text>Audio duration visualization<jupyter_code>import matplotlib.pyplot as plt

plt.hist(audiodatadf['duration'], rwidth=0.9, color='#86bf91')

plt.xlabel('Duration')
plt.ylabel('Population')
plt.title('Histogram of audio lengths')
plt.grid(False)
plt.show()<jupyter_output><empty_output><jupyter_text>Audio channels<jupyter_code>print("Audio channels:\n")
print(audiodatadf.channels.value_counts(normalize=True))<jupyter_output>Audio channels:

2    0.915369
1    0.084631
Name: channels, dtype: float64<jupyter_text>Bit depth<jupyter_code>print("Bit depths:\n")
print(audiodatadf.bit_depth.value_counts(normalize=True))<jupyter_output>Bit depths:

16    0.659414
24    0.315277
32    0.019354
8     0.004924
4     0.001031
Name: bit_depth, dtype: float64<jupyter_text>Sample rate<jupyter_code>print("Sample rates:\n")
print(audiodatadf.sample_rate.value_counts(normalize=True))<jupyter_output>Sample rates:

44100     0.614979
48000     0.286532
96000     0.069858
24000     0.009391
16000     0.005153
22050     0.005039
11025     0.004466
192000    0.001947
8000      0.001374
11024     0.000802
32000     0.000458
Name: sample_rate, dtype: float64<jupyter_text>4 Data Preprocessing<jupyter_code># Load various imports 
import pandas as pd
import os
import librosa

max_pad_len = 174

def extract_features(file_name):
   
    try:
        audio, sample_rate = librosa.load(file_name, res_type='kaiser_fast') 
        mfccs = librosa.feature.mfcc(y=audio, sr=sample_rate, n_mfcc=40)
        pad_width = max_pad_len - mfccs.shape[1]
        mfccs = np.pad(mfccs, pad_width=((0, 0), (0, pad_width)), mode='constant')
        
    except Exception as e:
        print("Error encountered while parsing file: ", file_name)
        return None 
     
    return mfccs

features = []

# Iterate through each sound file and extract the features 
for index, row in meta.iterrows():
    
    if index%500==0:
        print("processing element ",index)
    
    file_name = os.path.join(os.path.abspath(fulldatasetpath),'fold'+str(row["fold"]),str(row["slice_file_name"]))
    
    class_label = row["class"]
    data = extract_features(file_name)
    
    features.append([data, class_label])

# Convert into a Panda dataframe 
featuresdf = pd.DataFrame(features, columns=['feature','class_label'])

print('Finished feature extraction from ', len(featuresdf), ' files')<jupyter_output>processing element  0
processing element  250
processing element  500
processing element  750
processing element  1000
processing element  1250
processing element  1500
processing element  1750
processing element  2000
processing element  2250
processing element  2500
processing element  2750
processing element  3000
processing element  3250
processing element  3500<jupyter_text>5 Preprocessed data manipulation  Save processed data to pickle file<jupyter_code>featuresdf.to_pickle('E:\\Xception\\UrbanSound8K\\audio\\processed_data_frame.pkl')<jupyter_output><empty_output><jupyter_text>Load processed data from pickle file<jupyter_code>import pandas as pd
featuresdf = pd.read_pickle('E:\\Xception\\UrbanSound8K\\audio\\processed_data_frame.pkl')
print("imported", featuresdf.shape[0],"elements")<jupyter_output>imported 8732 elements<jupyter_text>Process data to input the network<jupyter_code>from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical

# Convert features and corresponding classification labels into numpy arrays
X = np.array(featuresdf.feature.tolist())
y = np.array(featuresdf.class_label.tolist())

# Encode the classification labels
le = LabelEncoder()
yy = to_categorical(le.fit_transform(y)) 

# split the dataset 
from sklearn.model_selection import train_test_split 

x_train, x_test, y_train, y_test = train_test_split(X, yy, test_size=0.2, random_state = 42)

# compute number of output classes
num_labels = featuresdf['class_label'].nunique()<jupyter_output><empty_output><jupyter_text>6 Model definitions  Baseline<jupyter_code>from keras.models import Model
from keras.layers import SeparableConv2D, ZeroPadding2D, Activation, Dropout, Dense, \
                            Conv2D, MaxPooling2D, Reshape, GRU
from keras.layers.normalization import BatchNormalization
from keras import Input, optimizers

num_rows = X.shape[1]
num_columns = X.shape[2]
num_channels = 1

x_train = x_train.reshape(x_train.shape[0], num_columns, num_rows, num_channels)
x_test = x_test.reshape(x_test.shape[0], num_columns, num_rows, num_channels)

def Net():
    
    # input layer
    inputs = Input(shape=(num_columns, num_rows, num_channels))
    
    # DWS-CNN layer 1
    x = Conv2D(256,kernel_size=(5,5), strides=(1,1), padding='same')(inputs)
    x = Activation('relu')(x)
    # Batch Normalisation before passing it to the next layer
    x = BatchNormalization()(x)
    # Pooling
    x = MaxPooling2D(pool_size=(1,5), padding='same')(x)
    # Dropout
    x = Dropout(0.25)(x)
    
    # DWS-CNN layer 2
    x = Conv2D(256,kernel_size=(5,5), strides=(1,1), padding='same')(x)
    x = Activation('relu')(x)
    # Batch Normalisation before passing it to the next layer
    x = BatchNormalization()(x)
    # Pooling
    x = MaxPooling2D(pool_size=(1,4), padding='same')(x)
    # Dropout
    x = Dropout(0.25)(x)
    
    # DWS-CNN layer 3
    x = Conv2D(256,kernel_size=(5,5), strides=(1,1), padding='same')(x)
    x = Activation('relu')(x)
    # Batch Normalisation before passing it to the next layer
    x = BatchNormalization()(x)
    # Pooling
    x = MaxPooling2D(pool_size=(1,2), padding='same')(x)
    # Dropout
    x = Dropout(0.25)(x)
    
    x = Reshape((174, 256))(x)
    
    # GRU
    x = GRU(256)(x)
        
    # classifier layer
    outputs = Dense(num_labels,activation='softmax')(x)
    
    
    # model compilation for training
    adam = optimizers.Adam(lr=0.0001)
    model = Model(inputs, outputs)                            
    model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=["accuracy"])
    
    return model


# create the model
model = Net()
model.summary()<jupyter_output>INFO:plaidml:Opening device "opencl_amd_gfx1010.0"<jupyter_text>Proposed model, low accuracy<jupyter_code>from keras.models import Model
from keras.layers import SeparableConv2D, ZeroPadding2D, Activation, Dropout, Dense, \
                            Conv2D, MaxPooling2D, GlobalAveragePooling2D, Reshape, Permute
from keras.layers.normalization import BatchNormalization
from keras import Input, optimizers

num_rows = X.shape[1]
num_columns = X.shape[2]
num_channels = 1

x_train = x_train.reshape(x_train.shape[0], num_columns, num_rows, num_channels)
x_test = x_test.reshape(x_test.shape[0], num_columns, num_rows, num_channels)

def Net(dilated_kernel, dilation, dilated_padding):
    
    # input layer
    inputs = Input(shape=(num_columns, num_rows, num_channels))
    
    # DWS-CNN layer 1
    x = ZeroPadding2D(padding=(2))(inputs)
    # use valid padding since padding is introduced before since it has a special form
    x = SeparableConv2D(256,kernel_size=(5,5), strides=(1,1), padding='valid')(x)
    x = Activation('relu')(x)
    # Batch Normalisation before passing it to the next layer
    x = BatchNormalization()(x)
    # Pooling
    x = MaxPooling2D(pool_size=(1,5), strides=(1,5), padding='valid')(x)
    # Dropout
    x = Dropout(0.25)(x)
    
    # DWS-CNN layer 2
    x = ZeroPadding2D(padding=(2))(x)
    # use valid padding since padding is introduced before since it has a special form
    x = SeparableConv2D(256,kernel_size=(5,5), strides=(1,1), padding='valid')(x)
    x = Activation('relu')(x)
    # Batch Normalisation before passing it to the next layer
    x = BatchNormalization()(x)
    # Pooling
    x = MaxPooling2D(pool_size=(1,4), strides=(1,4), padding='valid')(x)
    # Dropout
    x = Dropout(0.25)(x)
    
    # DWS-CNN layer 3
    x = ZeroPadding2D(padding=(2))(x)
    # use valid padding since padding is introduced before since it has a special form
    x = SeparableConv2D(256,kernel_size=(5,5), strides=(1,1), padding='valid')(x)
    x = Activation('relu')(x)
    # Batch Normalisation before passing it to the next layer
    x = BatchNormalization()(x)
    # Pooling
    x = MaxPooling2D(pool_size=(1,2), strides=(1,2), padding='valid')(x)
    # Dropout
    x = Dropout(0.25)(x)
    
    x = Permute((1,3,2))(x)
    
    # DIL-CNN 
    x = ZeroPadding2D(padding=(dilated_padding*dilation, 0))(x)
    x = Conv2D(256, kernel_size=dilated_kernel, dilation_rate=(dilation, 1))(x)#, strides=(1,3))(x)
    x = Activation('relu')(x)
    x = BatchNormalization()(x)
    
    # original idea with 1x1 convolution not working good
    #initializer = keras.initializers.Ones()
    #x = Conv2D(256, (1,1), strides=(1,3), kernel_initializer=initializer, trainable = False)(x)
    #x = MaxPooling2D(pool_size=(1,3), strides=(1,3), padding='valid')(x)
    
    # newer idea with global pooling, not working good either
    x = GlobalAveragePooling2D(name='avg_pool')(x)
    
    # classifier layer
    outputs = Dense(num_labels,activation='sigmoid')(x)
    
    # model compilation for training
    adam = optimizers.Adam(lr=0.0001)
    model = Model(inputs, outputs)                            
    model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
    
    return model


# create the model
dilated_kernel = (3,3)
dilation = (10)
dilated_padding = 1
model = Net(dilated_kernel,dilation,dilated_padding)
model.summary()<jupyter_output>INFO:plaidml:Opening device "opencl_amd_gfx1010.0"<jupyter_text>7 Model Training<jupyter_code>from keras.callbacks import ModelCheckpoint, EarlyStopping
from datetime import datetime 

num_epochs = 250
# low batch size due to memory maximum dimension, modify if using smaller dataset or larger VRAM
num_batch_size = 6

callbacks = [ModelCheckpoint(filepath='E:\\Xception\\UrbanSound8K\\model-{val_loss:.2f}.h5', 
                               verbose=1, save_best_only=True, monitor="val_loss"),
                EarlyStopping(monitor='val_loss', patience=30)]

start = datetime.now()

history = model.fit(x_train, y_train, batch_size=num_batch_size, epochs=num_epochs, validation_split=0.16, verbose=1, callbacks=callbacks)

duration = datetime.now() - start
print("Training completed in time: ", duration)<jupyter_output>Train on 5867 samples, validate on 1118 samples
Epoch 1/250
5867/5867 [==============================] - 352s 60ms/step - loss: 2.2666 - acc: 0.1824 - val_loss: 2.2912 - val_acc: 0.1127

Epoch 00001: val_loss improved from inf to 2.29120, saving model to E:\Xception\UrbanSound8K\model-2.29.h5
Epoch 2/250
5867/5867 [==============================] - 235s 40ms/step - loss: 2.2678 - acc: 0.1587 - val_loss: 2.2732 - val_acc: 0.1002

Epoch 00002: val_loss improved from 2.29120 to 2.27316, saving model to E:\Xception\UrbanSound8K\model-2.27.h5
Epoch 3/250
5867/5867 [==============================] - 235s 40ms/step - loss: 2.2613 - acc: 0.1198 - val_loss: 2.2694 - val_acc: 0.1020

Epoch 00003: val_loss improved from 2.27316 to 2.26942, saving model to E:\Xception\UrbanSound8K\model-2.27.h5
Epoch 4/250
5867/5867 [==============================] - 235s 40ms/step - loss: 2.2596 - acc: 0.1198 - val_loss: 2.2639 - val_acc: 0.1002

Epoch 00004: val_loss improved from 2.26942 to 2.26390, saving mode[...]<jupyter_text>Save model of last epoch<jupyter_code>model.save('E:\\Xception\\UrbanSound8K\\model-new-0.24_lastEpoch.h5')<jupyter_output><empty_output><jupyter_text>8 Model Evaluation  Evaluate model<jupyter_code># Evaluate the model on the test data using `evaluate`
print("Evaluate on test data")
results = model.evaluate(x_test, y_test, batch_size=num_batch_size)
print("test loss, test acc:", results)<jupyter_output>Evaluate on test data
1747/1747 [==============================] - 17s 10ms/step
test loss, test acc: [2.046015382358942, 0.26273612499782956]<jupyter_text>Plot history graphs<jupyter_code>import matplotlib.pyplot as plt

# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()<jupyter_output><empty_output><jupyter_text>Compute metrics of model<jupyter_code>import sklearn.metrics 
from sklearn.metrics import classification_report, confusion_matrix

classnames = ["air conditioner", "car horn", "children playing", "dog bark", "drilling", "engine idling", "gun shot", "jackhammer", "siren", "street music"]

preds = model.predict(x_test,verbose=1)

Ypred = np.argmax(preds, axis=1)
Ytest = np.argmax(y_test, axis=1)

print(classification_report(Ytest, Ypred, labels=None, target_names=classnames, digits=3))<jupyter_output>1747/1747 [==============================] - 21s 12ms/step
                  precision    recall  f1-score   support

 air conditioner      0.071     0.005     0.009       203
        car horn      0.286     0.023     0.043        86
children playing      0.171     0.158     0.164       183
        dog bark      0.218     0.398     0.282       201
        drilling      0.218     0.286     0.247       206
   engine idling      0.000     0.000     0.000       193
        gun shot      0.354     0.792     0.489        72
      jackhammer      0.328     0.423     0.370       208
           siren      0.336     0.218     0.265       165
    street music      0.281     0.465     0.350       230

        accuracy                          0.263      1747
       macro avg      0.226     0.277     0.222      1747
    weighted avg      0.213     0.263     0.217      1747
StarcoderdataJupyterStructuredCleanDedup