# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.7.8 64-bit
#     name: python3
# ---

# +
import numpy as np
import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
import tensorflow_datasets as tfds
from tensorflow.keras import layers
import matplotlib.cm as cm
import random
import glob
import os
from skimage.segmentation import chan_vese
import numpy as np
import PIL 
from PIL import Image 
from tensorboard.plugins.hparams import api as hp


# Display
# from IPython.display import Image, display
import matplotlib.pyplot as plt
import matplotlib.cm as cm
# -

import my_functions as mf

from tf_explain.callbacks.grad_cam import GradCAMCallback

batch_size = 32
img_height = 180
img_width = 180
image_size = (img_height, img_width)

# +
train_ds = tf.keras.utils.image_dataset_from_directory(
   'datasets/ct_scan_3/train',
   labels='inferred',
   label_mode='int',
   class_names=None,
   color_mode='rgb',
   batch_size=batch_size,
   image_size=image_size,
   shuffle=True, 
   seed=100, 
   interpolation='bilinear', 
   follow_links=False,
   crop_to_aspect_ratio=False)

val_ds = tf.keras.utils.image_dataset_from_directory(
   'datasets/ct_scan_3/val',
   labels='inferred',
   label_mode='int',
   class_names=None,
   color_mode='rgb',
   batch_size=batch_size,
   image_size=image_size,
   shuffle=True, 
   seed=100, 
   interpolation='bilinear', 
   follow_links=False,
   crop_to_aspect_ratio=False)

# -

mf.check_dataset(train_ds)

import pandas as pd

covid_metadata = pd.read_csv ('datasets\ct_scan_3\meta_data_covid.csv')
non_covid_metadata = pd.read_csv ('datasets\ct_scan_3\meta_data_normal.csv')

covid_metadata.head()

non_covid_metadata.head()

# # Cada um dos pascientes diagnosticado com covid ou vai para o dataset de treino ou para o dataset de validação

## Testando selecionar todas as imagens de um dado pasciente com covid
covid_metadata.head()

covid_metadata['Patient ID'].unique()

df.loc[df['Patient ID'] == 'patient103']

df.loc[df['Patient ID'] == 'patient101']['File name'][0]

from shutil import copy
base_src_folder = 'datasets\\ct_scan_3\\full-COVID-positive\\'
src = base_src_folder + df.loc[df['Patient ID'] == 'patient101']['File name'][0]
dst = 'datasets\\ct_scan_3\\train'
copy(src, dst)

len(covid_metadata['Patient ID'].unique())


def copy_all_images_from_one_patient(patient_id, src_folder, dst_folder, metadata):
   try:
      patient_dataset = metadata.loc[metadata['Patient ID'] == patient_id]
      files = patient_dataset['File name']
      for file in files:
         src = src_folder + '\\' + file
         copy(src, dst_folder)
   except Exception as e:
      print('Copy error: {}'.format(e))


patient_id = 'P040'
src_folder = 'datasets\\ct_scan_3\\full-COVID-positive'
dst_folder = 'datasets\\ct_scan_3\\train'
metadata = covid_metadata
copy_all_images_from_one_patient(patient_id, src_folder, dst_folder, metadata)

# Testando um split aleatório dos pacientes na esperança que a aleatoriedade salve a diferença de split final
covid_metadata['Patient ID'].unique()


def random_split_patients(unique_patients_ids, validation_split):
   number_of_patients = len(unique_patients_ids)
   number_of_train_patients = int(number_of_patients * (1 - validation_split))
   
   np.random.shuffle(unique_patients_ids)
   train_patients_ids = unique_patients_ids[:number_of_train_patients]
   val_patients_ids = unique_patients_ids[number_of_train_patients:]
   
   return train_patients_ids, val_patients_ids


train_patients_ids, val_patients_ids = random_split_patients(covid_metadata['Patient ID'].unique(), 0.2)

len(train_patients_ids) + len(val_patients_ids) == len(covid_metadata['Patient ID'].unique())

# ---------------

train_covid_patients_ids, val_covid_patients_ids = random_split_patients(covid_metadata['Patient ID'].unique(), 0.2)

for patient in train_covid_patients_ids:
   copy_all_images_from_one_patient(
      patient_id=patient, 
      src_folder='datasets\\ct_scan_3\\full-COVID-positive', 
      dst_folder = 'datasets\\ct_scan_3\\train\\COVID-positive', 
      metadata = covid_metadata)

for patient in val_covid_patients_ids:
   copy_all_images_from_one_patient(
      patient_id=patient, 
      src_folder='datasets\\ct_scan_3\\full-COVID-positive', 
      dst_folder = 'datasets\\ct_scan_3\\val\\COVID-positive', 
      metadata = covid_metadata)

train_non_covid_patients_ids, val_non_covid_patients_ids = random_split_patients(non_covid_metadata['Patient ID'].unique(), 0.2)

for patient in train_non_covid_patients_ids:
   copy_all_images_from_one_patient(
      patient_id=patient, 
      src_folder='datasets\\ct_scan_3\\full-COVID-negative', 
      dst_folder = 'datasets\\ct_scan_3\\train\\COVID-negative', 
      metadata = non_covid_metadata)

for patient in val_non_covid_patients_ids:
   copy_all_images_from_one_patient(
      patient_id=patient, 
      src_folder='datasets\\ct_scan_3\\full-COVID-negative', 
      dst_folder = 'datasets\\ct_scan_3\\val\\COVID-negative', 
      metadata = non_covid_metadata)

5462 + 6307 + 1431 + 1283 == 6893 + 7590

train_ds = tf.keras.utils.image_dataset_from_directory(
   'datasets/ct_scan_3/train',
   labels='inferred',
   label_mode='int',
   class_names=None,
   color_mode='rgb',
   batch_size=batch_size,
   image_size=image_size,
   shuffle=True, 
   seed=100, 
   interpolation='lanczos3', 
   follow_links=False,
   crop_to_aspect_ratio=False)

val_ds = tf.keras.utils.image_dataset_from_directory(
   'datasets/ct_scan_3/val',
   labels='inferred',
   label_mode='int',
   class_names=None,
   color_mode='rgb',
   batch_size=batch_size,
   image_size=image_size,
   shuffle=True, 
   seed=100, 
   interpolation='lanczos3', 
   follow_links=False,
   crop_to_aspect_ratio=False)

2714 / 11769 

mf.check_dataset(train_ds)

mf.check_dataset(val_ds)



StarcoderdataJupyterScriptsDedupFiltered