# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="fK95xzJZGSSH" colab_type="code" outputId="6cb2d9e7-7da9-4158-d4fb-867fd76cd9ac" colab={"base_uri": "https://localhost:8080/", "height": 227}
# !wget https://gitlab.com/federicozzo/electiveai/raw/master/Desktop/uni/elective_AI/electiveai/bdd100K_img.zip?inline=false

# + id="9aKnXvlJ5fsF" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 81} outputId="12834acd-44b6-450d-94b6-e29f11f796bd"
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
import numpy as np
import IPython.display as display
import cv2
import json
import os
from tqdm import tqdm
import matplotlib.pyplot as plt

# + id="hDnZHlEE30_I" colab_type="code" outputId="c8c2d81d-acff-42de-ea73-72c94100fd41" colab={"base_uri": "https://localhost:8080/", "height": 35}
from google.colab import drive
drive.mount('/content/drive', force_remount=True)

# + id="OU2YivkqK_7N" colab_type="code" colab={}
drive.flush_and_unmount()

# + id="DiFp9aU6Hz4k" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 203} outputId="0fdf1128-1115-4863-87b5-1fc12b379a8f"
with open("/content/bdd100k/labels/bdd100k_labels_images_train.json", "r") as f:
    train_images_label = json.load(f)

# + id="FLjjh08Y2zAZ" colab_type="code" colab={}
with open("/content/bdd100k/labels/bdd100k_labels_images_val.json", "r") as f:
    test_images_label = json.load(f)


# + id="oVIR7o9ULTVX" colab_type="code" colab={}
def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


# + id="M8_Z8gDvMevj" colab_type="code" colab={}
def load_image(addr):
    # read an image and resize to (128, 128)
    # cv2 load images as BGR, convert it to RGB
    img = cv2.imread(addr)
    img = cv2.resize(img, (128, 128), interpolation=cv2.INTER_CUBIC)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = img.astype(np.float32)
    return img


# + id="b2OR1JXpM_4n" colab_type="code" colab={}
def find_img_in_labels(name):
    name = name.split("/")[-1]
    for image in train_images_label:
        if image['name'] == name:
            return image
    return None


# + id="I6cVWq272vVj" colab_type="code" colab={}
def find_img_in_test_labels(name):
    name = name.split("/")[-1]
    for image in test_images_label:
        if image['name'] == name:
            return image
    return None


# + id="SLWgNVk7PDIM" colab_type="code" colab={}
for image in train_images_label:
    if image['attributes']['timeofday'] not in ['daytime', 'night', 'dawn/dusk', 'undefined']:
        print(image['name'], image['attributes']['timeofday'])
        

# + id="L0hIun-oLX2z" colab_type="code" colab={}
def writeTfRecord(input_path, output_path, setname):
    # open the TFRecords file
    writer = tf.python_io.TFRecordWriter(output_path)
    images_filenames = [input_path+fn for fn in os.listdir(path=input_path)]
    for image_fn in tqdm(images_filenames):
        # Load the image
        img = load_image(image_fn)
        
        if setname == 'train':
            attributes = find_img_in_labels(image_fn)
        else:
            attributes = find_img_in_test_labels(image_fn)

        if attributes is None:
            print(image_fn)
            continue
        label = attributes['attributes']['timeofday']
        
        if label not in ['daytime', 'night']:
            continue
            
        # Create a feature
        feature = {'label': _bytes_feature(tf.compat.as_bytes(label)),
                'image': _bytes_feature(tf.compat.as_bytes(img.tostring()))}
        # Create an example protocol buffer
        example = tf.train.Example(features=tf.train.Features(feature=feature))
        
        # Serialize to string and write on the file
        writer.write(example.SerializeToString())    
    writer.close()


# + id="0RxXcwfDP8E-" colab_type="code" colab={}
train_path = '/content/bdd100k/images/100k/train/'
test_path = '/content/bdd100k/images/100k/test/'
val_path = '/content/bdd100k/images/100k/val/'
train_out = 'train.tfrecords'
test_out = 'test.tfrecords'

# + id="F7d9z0EHQKgC" colab_type="code" colab={}
writeTfRecord(train_path, train_out)

# + id="_G84W2m0S6IM" colab_type="code" outputId="ad528709-a56a-4cd7-f31c-ef65bfba0dec" colab={"base_uri": "https://localhost:8080/", "height": 35}
writeTfRecord(val_path, test_out, 'val')

# + [markdown] colab_type="text" id="lbjCWhvL4IoW"
# # ***Read from TfRecords File***
#
#

# + id="i8lEMFzJK1hM" colab_type="code" colab={}
sess = tf.InteractiveSession()

# + id="oLDXd-GaW3qK" colab_type="code" colab={}
### IMPORTANT : First you have to unzip the tfRecord to import! ###

# + id="wPKE8J2CRcFM" colab_type="code" colab={}
tfrecord_path = 'test.tfrecords'
dataset = tf.data.TFRecordDataset(tfrecord_path)


# + id="3DDXqKMKRhI3" colab_type="code" colab={}
def decode(serialized_example):
    """
    Parses an image and label from the given `serialized_example`.
    It is used as a map function for `dataset.map`
    """
    IMAGE_SHAPE = (128,128,3)

    # 1. define a parser
    features = tf.parse_single_example(
        serialized_example,
        # Defaults are not specified since both keys are required.
        features={
            'image': tf.FixedLenFeature([], tf.string),
            'label': tf.FixedLenFeature([], tf.string),
        })

    # 2. Convert the data
    image = tf.decode_raw(features['image'], tf.float32)
    label = features['label']

    # 3. reshape
    image = tf.convert_to_tensor(tf.reshape(image, IMAGE_SHAPE))
    
    return image, label



# + id="TkWhDN81WyxN" colab_type="code" colab={}
def normalize(image, label):
  """Convert `image` from [0, 255] -> [-0.5, 0.5] floats."""
  image = tf.cast(image, tf.float32) * (1. / 255)
  return image, label


# + id="cYToDVmKR1wQ" colab_type="code" colab={}
# Parse the record into tensors with map. 
# map takes a Python function and applies it to every sample.
dataset = dataset.map(decode)

# + id="j0wyyQH1R7Ds" colab_type="code" colab={}
dataset = dataset.map(normalize)
batch_size = 1000
dataset = dataset.batch(batch_size)

# + id="JyQVgEF8SPNe" colab_type="code" colab={}
# Creating an iterator
iterator = dataset.make_one_shot_iterator()
image_batch, label_batch = iterator.get_next()

# + id="Gy0zTC_0SVFO" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 52} outputId="b910ae27-0000-4dca-dded-6889ffcd1412"
image_batch, label_batch = sess.run([image_batch, label_batch])
print(image_batch.shape)
print(label_batch.shape)

# + id="42jykAaGSbNG" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 285} outputId="0c60ebf2-6852-4169-eac6-60cfc366df60"
plt.imshow(image_batch[0])

StarcoderdataJupyterScriptsDedupFiltered