# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: RL
#     language: python
#     name: rl
# ---

# +
import os
import random
import time
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Sequential

from libs.utils import *
from libs.generate_boxes import *
# -

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0'
tf.get_logger().setLevel('INFO')
tf.keras.backend.floatx()

plt.style.use('fivethirtyeight')
plt.rcParams['figure.figsize'] = (20,10)


def build_actor(state_size, action_size):
    actor = Sequential(name='actor')
    actor.add(Dense(24, input_dim = state_size, activation='relu'))
    
    if action_size == 1:
        actor.add(Dense(action_size, activation='linear'))
    else:
        actor.add(Dense(action_size, activation='softmax'))
    actor.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.001))
    return actor


def build_critic(state_size):
    critic = Sequential(name='critic')
    critic.add(Dense(24, input_dim = state_size, activation='relu'))
    critic.add(Dense(1, activation='linear'))
    critic.compile(loss='mse', optimizer=Adam(lr=0.005))
    return critic


class Bpp3DEnv():
    def __init__(self,length=20, breadth=20, height=20):
        super(Bpp3DEnv, self).__init__()
        self.length=length
        self.breadth=breadth
        self.height=height
        self.container=np.zeros((self.length,self.breadth))
    
    def step(self, upleft,bxl,bxb,bxh):
        self.container[upleft[0]:upleft[0]+bxl,upleft[1]:upleft[1]+bxb] += bxh
        return self.container, self.terminal_reward()
    
    def reset(self):
        self.container=np.zeros((self.length,self.breadth))
    
    def terminal_reward(self):
        return np.sum(self.container)/(self.length*self.breadth*self.height)


env = Bpp3DEnv()
N_MDD = 50
env.reset()
boxes, gt_pos = generation_3dbox(case_size=[[20,20,20]],min_s=1,N_mdd=N_MDD,is_prediv='2h')
state_size = 20*20
action_size = 4
actor = build_actor(state_size, action_size)
critic = build_critic(state_size)

box_list, pos_list = [], []

history = env.container.copy()
boxes = boxes[0]
gt_pos = gt_pos[0]
boxes

step = 0
box = boxes[step]
pos = gt_pos[step]
box, pos

vis_box([box], [pos])

w_upleft = whole_upleft(*box_cornel([0,0],20,20),box[0],box[1])
f_upleft = feasible_location(history, w_upleft, box[0],box[1],box[2],
                            state_H = env.height, is2d=False)
f_upleft

history.shape

history = history.flatten()
history.shape

history = np.reshape(history, [1, state_size])
history.shape

policy = actor.predict(history, batch_size=1)
policy

policy = policy.flatten()
policy

action = np.random.choice(action_size, 1, p=policy)
#action = np.random.choice(2, 1, p=policy[:2])
action

action = action[0]
action

box_list.append(box)
pos_list.append(f_upleft[action])

next_state, reward = env.step(f_upleft[action], box[0], box[1], box[2])
next_state, reward

next_state = np.reshape(next_state, [1, state_size])
next_state.shape

target = np.zeros((1,1))
advantages = np.zeros((1,action_size))
target, advantages

value = critic.predict(history)
value

next_value = critic.predict(next_state)
next_value

value = value[0]
next_value = next_value[0]
value, next_value

advantages[0][action] = reward + 0.99 * (next_value)-value
target[0][0] = reward + 0.99 * next_value
target, advantages

advantages[0]

actor_history = actor.fit(history, advantages, epochs=1)
actor_history

critic_history = critic.fit(history, target, epochs = 1)
critic_history

box_list[0], np.array([pos_list[0][0], pos_list[0][1], 0])

vis_box([box_list[0]], [np.array([pos_list[0][0], pos_list[0][1], 0])])

step += 1
box = boxes[step]

box

history = env.container.copy()
history

w_upleft = whole_upleft(*box_cornel([0,0],20,20),box[0],box[1])
f_upleft = feasible_location(history, w_upleft, box[0],box[1],box[2],
                            state_H = env.height, is2d=False)
f_upleft

history = np.reshape(history, [1,state_size])
history.shape

policy = actor.predict(history, batch_size=1)
policy

policy = policy.flatten()
policy

this_action_size = 1
policy = policy[:this_action_size]
policy

p_list = []
for p in policy:
    p_list.append(round(p,2))
p_list = np.array(p_list)
p_list

m = 1 / sum(p_list)
m = round(m, 2)
m

for i in range(len(p_list)):
    p_list[i] = round(p_list[i] * m, 2)
p_list

sum(p_list)

action = np.random.choice(this_action_size, 1, p=p_list)
action

action = action[0]
action

box_list.append(box)
pos_list.append(f_upleft[action])

next_state, reward = env.step(f_upleft[action], box[0], box[1], box[2])
next_state, reward

next_state = np.reshape(next_state, [1, state_size])

target = np.zeros((1, 1))
advantages = np.zeros((1, action_size))
target, advantages

value = critic.predict(history)[0]
next_value = critic.predict(next_state)[0]
value, next_value

advantages[0][action] = reward + 0.99 * (next_value) - value
target[0][0] = reward + 0.99 * next_value
advantages, target

history.shape, advantages.shape

actor.fit(history, advantages, epochs=1)

critic.fit(history, target, epochs=1)

box_list, pos_list

p_list = []
for pos in pos_list:
    p_list.append(np.array([pos[0], pos[1], 0]))
p_list

vis_box(box_list, p_list)

step += 1
box = boxes[step]

history = env.container.copy()
box

history

w_upleft = whole_upleft(*box_cornel([0,0],20,20),box[0],box[1])
f_upleft = feasible_location(history, w_upleft, box[0],box[1],box[2],
                            state_H = env.height, is2d=False)
f_upleft

history = np.reshape(history, [1, state_size])
history.shape

policy = actor.predict(history, batch_size = 1)
policy

policy = policy.flatten()
policy

action = np.random.choice(action_size, 1, p=policy)[0]
action

box_list.append(box)
pos_list.append(f_upleft[action])

box_list

next_state, reward = env.step(f_upleft[action], box[0], box[1], box[2])
next_state, reward

h = 3

next_state = np.reshape(next_state, [1,state_size])

target = np.zeros((1,1))
advantages = np.zeros((1,action_size))
target, advantages

value = critic.predict(history)[0]
next_value = critic.predict(next_state)[0]
value, next_value

advantages[0][action] = reward + 0.99 * (next_value) - value
target[0][0] = reward + 0.99 * next_value
advantages, target

actor.fit(history, advantages, epochs=1)

critic.fit(history, advantages, epochs = 1)

p_list = []
h_list = [0,0,3]
for pos, h in zip(pos_list, h_list):
    p_list.append(np.array([pos[0], pos[1], h]))
p_list


# +
def cuboid_data2(o, size=(1,1,1)):
    X = [[[0, 1, 0], [0, 0, 0], [1, 0, 0], [1, 1, 0]],
         [[0, 0, 0], [0, 0, 1], [1, 0, 1], [1, 0, 0]],
         [[1, 0, 1], [1, 0, 0], [1, 1, 0], [1, 1, 1]],
         [[0, 0, 1], [0, 0, 0], [0, 1, 0], [0, 1, 1]],
         [[0, 1, 0], [0, 1, 1], [1, 1, 1], [1, 1, 0]],
         [[0, 1, 1], [0, 0, 1], [1, 0, 1], [1, 1, 1]]]
    X = np.array(X).astype(float)
    for i in range(3):
        X[:,:,i] *= size[i]
    X += np.array(o)
    return X

def plotCubeAt2(positions,sizes=None,colors=None, **kwargs):
    if not isinstance(colors,(list,np.ndarray)): colors=["C0"]*len(positions)
    if not isinstance(sizes,(list,np.ndarray)): sizes=[(1,1,1)]*len(positions)
    g = []
    for p,s,c in zip(positions,sizes,colors):
        g.append( cuboid_data2(p, size=s) )
    return Poly3DCollection(np.concatenate(g),facecolors=np.repeat(colors,6), alpha=0.5, **kwargs)

def get_colors(n_box):
    color_names=["crimson","limegreen","grey","brown","orange","olive","blue","purple","yellow","pink","skyblue","red","aqua","gold"]
    colors = color_names*(n_box//len(color_names))+color_names[:n_box%len(color_names)]
    return colors

def vis_box(sizes,positions,fs=(3,3)):
    colors = get_colors(len(positions))
    fig = plt.figure(figsize=fs)
    ax = fig.gca(projection='3d')
    ax.set_aspect('auto')
    pc = plotCubeAt2(positions,sizes,colors=colors, edgecolor="w")
    ax.add_collection3d(pc)    
    ax.set_xlim([-5,25])
    ax.set_ylim([-5,25])
    ax.set_zlim([-5,25])
    return ax


# -

vis_box(box_list, p_list)

ax = vis_box(box_list, p_list)
ax.view_init(90,-90)

ax = vis_box(box_list, p_list)
ax.view_init(0,0)

ax = vis_box(box_list, p_list)
ax.view_init(0,-90)

import gym
env = gym.make('CartPole-v0')
env.action_space.n, env.observation_space.shape[0]

# +
state_size = env.observation_space.shape[0]
action_size = env.action_space.n

actor = build_actor(state_size, action_size)
critic = build_critic(state_size)

state = env.reset()
state, state.shape
# -

state = np.reshape(state, [1,state_size])
state, state.shape

policy = actor.predict(state, batch_size=1)
policy

policy = policy.flatten()
policy

action = np.random.choice(action_size, 1, p=policy)
action

action = action[0]
action

next_state, reawrd, done, _ = env.step(action)
next_state, reward

next_state.shape

next_state = np.reshape(next_state, [1,state_size])
next_state, next_state.shape

target = np.zeros((1,1))
advantages = np.zeros((1, action_size))
target, advantages

value = critic.predict(state)
value

next_value = critic.predict(next_state)
next_value

value = value[0]
next_value = next_value[0]
value, next_value

advantages[0][action] = reward + 0.99 * (next_value) - value
advantages

target[0][0] = reward + 0.99 * next_value
target

state.shape, advantages.shape

history = actor.fit(state, advantages, epochs=1)

history.history['loss']

state.shape, target.shape

history_c = critic.fit(state, target, epochs=1)

history_c.history['loss']

state = next_state
state, state.shape

policy = actor.predict(state, batch_size=1)
policy

# +
import sys
import gym
import pylab
import numpy as np
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam

EPISODES = 100


# A2C(Advantage Actor-Critic) agent for the Cartpole
class A2CAgent:
    def __init__(self, state_size, action_size):
        # if you want to see Cartpole learning, then change to True
        self.render = False
        self.load_model = False
        # get size of state and action
        self.state_size = state_size
        self.action_size = action_size
        self.value_size = 1

        # These are hyper parameters for the Policy Gradient
        self.discount_factor = 0.99
        self.actor_lr = 0.001
        self.critic_lr = 0.005

        # create model for policy network
        self.actor = self.build_actor()
        self.critic = self.build_critic()

        if self.load_model:
            self.actor.load_weights("./save_model/cartpole_actor.h5")
            self.critic.load_weights("./save_model/cartpole_critic.h5")

    # approximate policy and value using Neural Network
    # actor: state is input and probability of each action is output of model
    def build_actor(self):
        actor = Sequential()
        actor.add(Dense(24, input_dim=self.state_size, activation='relu',
                        kernel_initializer='he_uniform'))
        actor.add(Dense(self.action_size, activation='softmax',
                        kernel_initializer='he_uniform'))
        actor.summary()
        # See note regarding crossentropy in cartpole_reinforce.py
        actor.compile(loss='categorical_crossentropy',
                      optimizer=Adam(lr=self.actor_lr))
        return actor

    # critic: state is input and value of state is output of model
    def build_critic(self):
        critic = Sequential()
        critic.add(Dense(24, input_dim=self.state_size, activation='relu',
                         kernel_initializer='he_uniform'))
        critic.add(Dense(self.value_size, activation='linear',
                         kernel_initializer='he_uniform'))
        critic.summary()
        critic.compile(loss="mse", optimizer=Adam(lr=self.critic_lr))
        return critic

    # using the output of policy network, pick action stochastically
    def get_action(self, state):
        policy = self.actor.predict(state, batch_size=1).flatten()
        return np.random.choice(self.action_size, 1, p=policy)[0]

    # update policy network every episode
    def train_model(self, state, action, reward, next_state, done):
        target = np.zeros((1, self.value_size))
        advantages = np.zeros((1, self.action_size))

        value = self.critic.predict(state)[0]
        next_value = self.critic.predict(next_state)[0]

        if done:
            advantages[0][action] = reward - value
            target[0][0] = reward
        else:
            advantages[0][action] = reward + self.discount_factor * (next_value) - value
            target[0][0] = reward + self.discount_factor * next_value

        self.actor.fit(state, advantages, epochs=1, verbose=0)
        self.critic.fit(state, target, epochs=1, verbose=0)


if __name__ == "__main__":
    # In case of CartPole-v1, maximum length of episode is 500
    env = gym.make('CartPole-v1')
    # get size of state and action from environment
    state_size = env.observation_space.shape[0]
    action_size = env.action_space.n

    # make A2C agent
    agent = A2CAgent(state_size, action_size)

    scores, episodes = [], []

    for e in range(EPISODES):
        done = False
        score = 0
        state = env.reset()
        state = np.reshape(state, [1, state_size])
        step = 0

        while not done:
            if agent.render:
                env.render()

            action = agent.get_action(state)
            next_state, reward, done, info = env.step(action)
            next_state = np.reshape(next_state, [1, state_size])
            # if an action make the episode end, then gives penalty of -100
            reward = reward if not done or score == 499 else -100
            
            agent.train_model(state, action, reward, next_state, done)

            score += reward
            state = next_state
            step += 1

            if done:
                # every episode, plot the play time
                score = score if score == 500.0 else score + 100
                scores.append(score)
                episodes.append(e)
                pylab.plot(episodes, scores, 'b')
                pylab.savefig("cartpole_a2c.png")
                print("episode:", e, "  score:", score)

                # if the mean of scores of last 10 episode is bigger than 490
                # stop training
                if np.mean(scores[-min(10, len(scores)):]) > 490:
                    sys.exit()
# -



StarcoderdataJupyterScriptsDedupFiltered