# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# [View in Colaboratory](https://colab.research.google.com/github/findingfoot/ML_practice-codes/blob/master/Data_Collection.ipynb)

# + id="OLiZ91247O6P" colab_type="code" colab={}
import warnings
warnings.filterwarnings('ignore')

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow.python.framework import ops
ops.reset_default_graph()

sess = tf.Session()

# + id="bM7Evstg7ZKN" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 85} outputId="937d5d22-944b-4468-e075-ac3826989af7"

#Getting data from sklearn
from sklearn.datasets import load_iris


iris = load_iris()
print(len(iris.data))
print(len(iris.target))
print(iris.data[14])
print(set(iris.target))

# + id="Sl0GOjuY7idv" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 51} outputId="e1de94e1-d228-4529-f168-019c50d957c4"
# downloading data from github

import requests

birthdata_url = 'https://github.com/nfmcclure/tensorflow_cookbook/raw/master/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat'
birth_file = requests.get(birthdata_url)
birth_data = birth_file.text.split('\r\n')
birth_header = birth_data[0].split('\t')
birth_data = [[float(x) for x in y.split('\t') if len(x)>=1] for y in birth_data[1:] if len(y)>=1]
print(len(birth_data))
print(len(birth_data[0]))


# + id="PMcTJkTS8FHr" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 2886} outputId="fbaad018-a2d8-462d-c811-9eae095ac225"
import requests

housing_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data'
housing_header = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV']
housing_file =  requests.get(housing_url)
housing_data = [[float(x) for x in y.split(' ') if len(x)>=1] for y in housing_file.text.split('\n') if len(y)>=1]
print(len(housing_data))
print(len(housing_data[0]))
print(housing_data)

# + id="_G3AOzzp_Kj7" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 598} outputId="54ebc784-32d8-4d87-e7b5-2ca794273540"
from tensorflow.examples.tutorials.mnist import input_data 

mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)

print(len(mnist.train.images))
print(len(mnist.test.images))
print(len(mnist.validation.images))
print(mnist.train.labels[2,:])

# + id="qraglttLu9kk" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 368} outputId="4af43012-3d0b-48d1-d44f-05fec1a7ddd2"
from PIL import Image


(X_train, y_train) ,(X_test,y_test) = tf.contrib.keras.datasets.cifar10.load_data()

print(X_train.shape)

print(X_test.shape)

print(y_train.shape)

# %matplotlib inline
img = Image.fromarray(X_train[0,:,:,:])
plt.imshow(img)

# + id="LlO65NSEyZUm" colab_type="code" colab={}
import requests
import io
from zipfile import ZipFile

# Get/read zip file
zip_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip'
r = requests.get(zip_url)
z = ZipFile(io.BytesIO(r.content))
file = z.read('SMSSpamCollection')
# Format Data
text_data = file.decode()
text_data = text_data.encode('ascii',errors='ignore')
text_data = text_data.decode().split('\n')
text_data = [x.split('\t') for x in text_data if len(x)>=1]
[text_data_target, text_data_train] = [list(x) for x in zip(*text_data)]
print(len(text_data_train))
print(set(text_data_target))
print(text_data_train[1])

# + id="Q9YXam301Hez" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 85} outputId="7befc381-8128-43e6-cd62-5d7a69c4b0a8"
import requests
import io
import tarfile

movie_data_url = 'http://www.cs.cornell.edu/people/pabo/movie-review-data/rt-polaritydata.tar.gz'
r = requests.get(movie_data_url)

streaming_data = io.BytesIO(r.content)
temp = io.BytesIO()

while True:
    s = streaming_data.read(16384)
    if not s:  
        break
    temp.write(s)
streaming_data.close()
temp.seek(0)

# Extract tar file
tar_file = tarfile.open(fileobj=temp, mode="r:gz")
pos = tar_file.extractfile('rt-polaritydata/rt-polarity.pos')
neg = tar_file.extractfile('rt-polaritydata/rt-polarity.neg')
# Save pos/neg reviews
pos_data = []
for line in pos:
    pos_data.append(line.decode('ISO-8859-1').encode('ascii',errors='ignore').decode())
neg_data = []
for line in neg:
    neg_data.append(line.decode('ISO-8859-1').encode('ascii',errors='ignore').decode())
tar_file.close()

print(len(pos_data))
print(len(neg_data))
print(pos_data[0])

# + id="mrFNQLDc3qRY" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="6c310c69-06bb-4417-9cb6-6ea17a14024f"

shakespeare_url = 'http://www.gutenberg.org/cache/epub/100/pg100.txt'
sp_data = requests.get(shakespeare_url)

sp_file = sp_data.content

sp_text = sp_file.decode('utf-8')
sp_text = sp_text[2308:]
print(len(sp_text))

# + id="LTTt2ELz3y3x" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 68} outputId="9403344b-e905-4ee0-f176-c0a3bbfe64af"
import requests
import io
from zipfile import ZipFile
sentence_url = 'http://www.manythings.org/anki/deu-eng.zip'
r = requests.get(sentence_url)
z = ZipFile(io.BytesIO(r.content))
file = z.read('deu.txt')
# Format Data
eng_ger_data = file.decode()
eng_ger_data = eng_ger_data.encode('ascii',errors='ignore')
eng_ger_data = eng_ger_data.decode().split('\n')
eng_ger_data = [x.split('\t') for x in eng_ger_data if len(x)>=1]
[english_sentence, german_sentence] = [list(x) for x in zip(*eng_ger_data)]
print(len(english_sentence))
print(len(german_sentence))
print(eng_ger_data[14])

# + id="2PxCIqDs4hUX" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered