# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow.contrib.layers.python.layers import initializers
from tensorflow.examples.tutorials.mnist import input_data
from math import sqrt
import numpy as np
from sklearn.manifold import TSNE
get_ipython().magic('matplotlib inline')
get_ipython().magic('pylab inline')

mnist = input_data.read_data_sets('MNIST_data', one_hot=False)

print(mnist.test.labels)

from keras.utils import np_utils# fix random seed for reproducibility
y_test = np_utils.to_categorical(mnist.test.labels)
print(y_test)

learning_rate = 0.00001
training_epochs = 15
batch_size = 100
display_step = 1
logs_path = './tensorflow_logs/mnist_metrics'
# Network Parameters
n_hidden_1 = 256 # 1st layer number of features
n_hidden_2 = 256 # 2nd layer number of features
n_input = 28*28 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
margin = 1.0

    x_left = tf.placeholder(tf.float32, shape=[None, n_input], name='InputDataLeft')
    x_right = tf.placeholder(tf.float32, shape=[None, n_input], name='InputDataRight')
    label = tf.placeholder(tf.float32, shape=[None, 1], name='LabelData') # 0 if the same, 1 is different

    x_image_left = x_left
    x_image_right = x_right

  # def NN(inputs):


def tfNN(x, weights, biases):
    x = tf.scalar_mul(1.0/256.0, x)
    layer_1 = tf.add(tf.matmul(x, weights['w1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)
    layer_2 = tf.add(tf.matmul(layer_1, weights['w2']), biases['b2'])
    layer_2 = tf.nn.relu(layer_2)
    layer_3 = tf.add(tf.matmul(layer_2, weights['w3']), biases['b3'])
    out_layer = tf.add(tf.matmul(layer_3, weights['w4']), biases['b4'])
    return out_layer


    # Store layers weight & bias
    #tf.random_uniform(
    #shape,
    #minval=0,
    #maxval=None,
    #dtype=tf.float32,
    #seed=None,
    #name=None
    #)
    weights = {
    'w1': tf.Variable(tf.random_uniform([n_input, n_hidden_1], minval=-4*np.sqrt(6.0/(n_input + n_hidden_1)), maxval=4*np.sqrt(6.0/(n_input + n_hidden_1))), name='W1'),
    'w2': tf.Variable(tf.random_uniform([n_hidden_1, n_hidden_2], minval=-4*np.sqrt(6.0/(n_hidden_1 + n_hidden_2)), maxval=4*np.sqrt(6.0/(n_hidden_1 + n_hidden_2))), name='W2'),
    'w3': tf.Variable(tf.random_uniform([n_hidden_2, n_classes], minval=-4*np.sqrt(6.0/(n_hidden_2 + n_classes)), maxval=4*np.sqrt(6.0/(n_hidden_2 + n_classes))), name='W3'),
    'w4': tf.Variable(tf.random_uniform([n_classes, n_classes], minval=-4*np.sqrt(6.0/(n_classes + 2)), maxval=4*np.sqrt(6.0/(n_classes + 2))), name='W4')
    }
    biases = {
    'b1': tf.Variable(tf.truncated_normal([n_hidden_1]) / sqrt(n_hidden_1), name='b1'),
    'b2': tf.Variable(tf.truncated_normal([n_hidden_2]) / sqrt(n_hidden_2), name='b2'),
    'b3': tf.Variable(tf.truncated_normal([n_classes]) / sqrt(n_classes), name='b3'),
    'b4': tf.Variable(tf.truncated_normal([n_classes]) / sqrt(n_classes), name='b4')
    }

with tf.name_scope('Model'):
    # Model
    pred_left = tfNN(x_image_left, weights, biases)
    pred_right = tfNN(x_image_right, weights, biases)
    with tf.name_scope('Loss'):
        d = tf.reduce_sum(tf.square(tf.subtract(pred_left, pred_right)), 1, keep_dims=True)
    d_sqrt = tf.sqrt(d)
    loss = label * tf.square(tf.maximum(0.0, margin - d_sqrt)) + (1 - label) * d
    loss = 0.5 * tf.reduce_mean(loss)

    with tf.name_scope('AdamOptimizer'):
        optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

    # Initializing the variables
    init = tf.global_variables_initializer()
    # Create a summary to monitor cost tensor
    tf.summary.scalar("loss", loss)

    # Merge all summaries into a single op
    merged_summary_op = tf.summary.merge_all()


    # Launch the graph
sess = tf.Session()
sess.run(init)
    # op to write logs to Tensorboard
#summary_writer = tf.train.SummaryWriter(logs_path, graph=tf.get_default_graph())

    # Training cycle
for epoch in range(training_epochs):
    avg_loss = 0.0
    total_batch = int(mnist.train.num_examples / batch_size)
    # Loop over all batches
    for i in range(total_batch):
        left_batch_xs, left_batch_ys = mnist.train.next_batch(batch_size)
        right_batch_xs, right_batch_ys = mnist.train.next_batch(batch_size)
        labels = np.zeros((batch_size, 1))
        for l in range(batch_size):
            if left_batch_ys[l] == right_batch_ys[l]:
                labels[l, 0] = 0.0
            else:
                labels[l, 0] = 1.0
        _, l, summary = sess.run([optimizer, loss, merged_summary_op],
                                 feed_dict = {
                                              x_left: left_batch_xs, 
                                              x_right: right_batch_xs,
                                              label: labels,
                                             })
        # Write logs at every iteration
        #summary_writer.add_summary(summary, epoch * total_batch + i)
        # Compute average loss
        avg_loss += l / total_batch
    # Display logs per epoch step
    if (epoch+1) % display_step == 0:
        print ("Epoch:", '%04d' % (epoch+1), "loss =", "{:.9f}".format(avg_loss))

   # print ("Optimization Finished!")

    #print ("Run the command line:\n"       "--> tensorboard --logdir=./tensorflow_logs "       "\nThen open http://0.0.0.0:6006/ into your web browser")


print(left_batch_xs)

    # Test model
    # Calculate accuracy
    test_xs, test_ys = mnist.train.next_batch(5000)
    ans = sess.run([pred_left], feed_dict = { x_left: test_xs})

ans = ans[0]
print(ans)
print(len(ans))
print(test_ys)

print(mnist.train.num_examples)

    # Test model
    # Calculate accuracy
    test_xs, test_ys = mnist.train.next_batch(5000)
    ans = sess.run([pred_left], feed_dict = { x_left: test_xs})
    print(test_xs)
    print(test_ys[1])

print(len(test_ys))

print(ans[0])

# +
import matplotlib.pyplot as pyplot

# %matplotlib inline

def plot_data(features,labels,axis,alpha=1.0):
    # separate features according to their class
    X0,X1,X2,X3,X4,X5,X6,X7,X8,X9 = features[labels==0], features[labels==1], features[labels==2], features[labels==3], features[labels==4], features[labels==5], features[labels==6], features[labels==7], features[labels==8], features[labels==9]
    
    # class 0 data
    axis.plot(X0[:,0], X0[:,1], 'o', color='green', markersize=1, alpha=alpha)
    # class 1 data
    axis.plot(X1[:,0], X1[:,1], 'o', color='red', markersize=1, alpha=alpha)
    # class 2 data
    axis.plot(X2[:,0], X2[:,1], 'o', color='blue', markersize=1, alpha=alpha)
    
    axis.plot(X3[:,0], X3[:,1], 'o', color='yellow', markersize=1, alpha=alpha)
    
    axis.plot(X4[:,0], X4[:,1], 'o', color='brown', markersize=1, alpha=alpha)
    
    axis.plot(X5[:,0], X5[:,1], 'o', color='black', markersize=1, alpha=alpha)
    
    axis.plot(X6[:,0], X6[:,1], 'o', color='pink', markersize=1, alpha=alpha)
    
    axis.plot(X7[:,0], X7[:,1], 'o', color='orange', markersize=1, alpha=alpha)
    
    axis.plot(X8[:,0], X8[:,1], 'o', color='cyan', markersize=1, alpha=alpha)
    
    axis.plot(X9[:,0], X9[:,1], 'o', color='gray', markersize=1, alpha=alpha)
    
    # set axes limits
    axis.set_xlim(3.5,8)
    axis.set_ylim(0,6)
    axis.set_aspect('equal')
    
    axis.set_xlabel('x')
    axis.set_ylabel('y')

figure,axis = pyplot.subplots(1,1)
plot_data(ans[0],test_ys,axis)
pyplot.show()
# -

print(weights['w1'][0][0])

# +
# Python optimisation variables
import tensorflow as tf
learning_rate = 0.01
epochs = 30
batch_size = 100

# declare the training data placeholders
# input x - for 28 x 28 pixels = 784
x = tf.placeholder(tf.float32, [None, 10])
# now declare the output data placeholder - 10 digits
y = tf.placeholder(tf.float32, [None, 10])
# -

# now declare the weights connecting the input to the hidden layer
W1 = tf.Variable(tf.random_normal([10, 300], stddev=0.03), name='W1')
b1 = tf.Variable(tf.random_normal([300]), name='b1')
# and the weights connecting the hidden layer to the output layer
W2 = tf.Variable(tf.random_normal([300, 10], stddev=0.03), name='W2')
b2 = tf.Variable(tf.random_normal([10]), name='b2')

# calculate the output of the hidden layer
hidden_out = tf.add(tf.matmul(x, W1), b1)
hidden_out = tf.nn.relu(hidden_out)

y_ = tf.nn.softmax(tf.add(tf.matmul(hidden_out, W2), b2))

y_clipped = tf.clip_by_value(y_, 1e-10, 0.9999999)
cross_entropy = -tf.reduce_mean(tf.reduce_sum(y * tf.log(y_clipped)
                         + (1 - y) * tf.log(1 - y_clipped), axis=1))

# add an optimiser
optimiser = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cross_entropy)


# +
# finally setup the initialisation operator
init_op = tf.global_variables_initializer()

# define an accuracy assessment operation
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# -


    # Launch the graph
#sess = tf.Session()

    # op to write logs to Tensorboard
#summary_writer = tf.train.SummaryWriter(logs_path, graph=tf.get_default_graph())

    # Training cycle
with tf.Session() as sess:
    sess.run(init_op)
    for epoch in range(30):
        avg_cost=0
        total_batch = int(mnist.train.num_examples / batch_size)
    # Loop over all batches
        for i in range(total_batch):
            left_batch_xs, left_batch_ys = mnist.train.next_batch(batch_size)
            ans = sess.run([pred_left], feed_dict = { x_left: left_batch_xs})
            ans=ans[0]
            left_batch_ys=np_utils.to_categorical(left_batch_ys)
        #right_batch_xs, right_batch_ys = mnist.train.next_batch(batch_size)
        #labels = np.zeros((batch_size, 1))
            _, c = sess.run([optimiser, cross_entropy], feed_dict={x: ans, y: left_batch_ys})
            avg_cost += c / total_batch
        print("Epoch:", (epoch + 1), "cost =", "{:.3f}".format(avg_cost)) 
    test_xs, test_ys = mnist.train.next_batch(5000)
    #test_ys=np_utils.to_categorical(test_ys)
    #print(sess.run(accuracy, feed_dict={x_left: test_xs, y: test_ys}))
    #test_xs, test_ys = mnist.train.next_batch(5000)
    #ans = sess.run([pred_left], feed_dict = { x_left: test_xs})
  

print(test_ys)

print(weights['w1'])
print(W1)

StarcoderdataJupyterScriptsDedupFiltered