# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import numpy as np
import tensorflow as tf

mu = 0.5
sigma = 1

t = 0.45

def est_deriv_mu(samples):
    approx = np.square(samples * sigma + mu - t) * samples / sigma
    return np.mean(approx)

def est_deriv_sigma(samples):
    approx = np.square(samples * sigma + mu - t) * (np.square(samples) - 1) / sigma
    return np.mean(approx)

def get_noise(num):
    return np.random.normal(size=(num, 1))


for i in range(7):
    num_samples = 10 ** i
    samples = get_noise(num_samples)
    est_mu, est_sigma = est_deriv_mu(samples), est_deriv_sigma(samples)
    print("sample: {} est_mu: {} est_sigma: {}".format(num_samples, est_mu, est_sigma))

# +
weights ={
    'wg1': tf.Variable(tf.random_normal([1, 3], stddev=0.01)),
    'wg2': tf.Variable(tf.random_normal([3, 1], stddev=0.01)),
}

biases ={
    'bg1': tf.Variable(tf.zeros([3])),
    'bg2': tf.Variable(tf.zeros([1])),
}

Z = tf.placeholder(tf.float32, [None, 1], name = 'gan_X')
reward = tf.placeholder(tf.float32, [None, 1], name = 'reward')
X = sigma * Z + mu
noise = tf.nn.relu(tf.matmul(X, weights['wg1']) + biases['bg1'])
surr_reward = tf.matmul(noise, weights['wg2']) + biases['bg2']
rev_reward = (reward - surr_reward) 

var_e = sigma + 1e-6
grads_mean = tf.reduce_mean(rev_reward * (Z / var_e) + tf.gradients(surr_reward, X))
grads_var = tf.reduce_mean(rev_reward * ((tf.square(Z) / var_e) - 1 / var_e) + (tf.gradients(surr_reward, X) * Z)) 


S_var_list = [weights['wg1'], biases['bg1'], weights['wg2'], biases['bg2']]

# Gradient of S
opt_S = tf.train.AdamOptimizer(1e-5)
grads_S_m = tf.gradients(grads_mean * grads_mean, S_var_list)
grads_S_v = tf.gradients(grads_var * grads_var, S_var_list)
grads_S = [grads_S_m[i] + grads_S_v[i] for i in range(len(S_var_list))]
grads_and_vars_S = zip(grads_S, S_var_list)


# Ask the optimizer to apply the capped gradients.
train_S = opt_S.apply_gradients(grads_and_vars_S)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

for i in range(1000):
    num_samples = 1000
    samples = get_noise(num_samples)
    rewards = np.square(samples * sigma + mu - t)
    _, surr_rew, est_mean, est_var = sess.run([train_S, surr_reward, grads_mean, grads_var], feed_dict={Z: samples, reward: rewards})
    print("iter: {}/1000".format(i), "surrogate reward: {}, mean: {}, var: {}".format(np.mean(surr_rew), est_mean, est_var))
# -





StarcoderdataJupyterScriptsDedupFiltered