# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <img src="https://drive.google.com/uc?id=1v7YY_rNBU2OMaPnbUGmzaBj3PUeddxrw" alt="ITI MCIT EPITA" style="width: 750px;"/>
#
# ___

# # Data Preparation & Exploration
#
# **By**: <NAME>, <EMAIL>
# ____
#
# ## Session 03

# #### Topics: 
#
# * Presentation of rest services 
# * Dealing with remote json content fetching
# * Connecting to websites and fetching raw content 
# * Connect to websites using selenium headless (python) or HTML Unit (java) 
# * Integration is performed through Orange studio.

# ### Web Service

# Simply put, a web service is a resource that’s made available over the internet. Therefore, web services, by definition, require a network. The term “web service” is defined by W3C (the World Wide Web Consortium) and so it technically follows a whole host of standards.
#
#     A Web service is a software system designed to support interoperable machine-to-machine interaction over a network. It has an interface described in a machine-processable format (specifically WSDL). Other systems interact with the Web service in a manner prescribed by its description using SOAP-messages, typically conveyed using HTTP with an XML serialization in conjunction with other Web-related standards.
#     
#     
#         SOAP (Simple Object Access Protocol)
#         UDDI (Universal Description, Discovery and Integration)
#         WSDL (Web Services Description Language)
#         
#
# web services are not API, and it supports only XML

# ### Application Programming Interface (API)

# APIs enables communication and data exchange between two separate software systems
#
# A SW system implementing an API, contains functions which can be executed by another SW system.
# Structured request, and structured response
#
# Ex. Web services, one category of APIs, bring the Web beyond links from one page to another, and beyond pages for humans to read. Instead, they allow us to exchange structured data across the Web and provide a coherent way for our applications to request data in rich, programmatic ways

#     When you log into a website using your Facebook profile
#     When you switch on Netflix and see dozens of new movies flood the screen
#     When you look for flights on Google
#     
#     
# supports XML and JSON 

#     APIs enables communication and data exchange between two separate software systems
#     A SW system implementing an API, contains functions which can be executed by another SW system.
#     Structured request, and structured response
# Ex. Web services, one category of APIs, bring the Web beyond links from one page to another, and beyond pages for humans to read. Instead, they allow us to exchange structured data across the Web and provide a coherent way for our applications to request data in rich, programmatic ways

# ### Data Collection through code

#     Web services and APIs are an important source of data.
#     Clients need to programmatically access data
#     Requests over local web applications and Databases
#
#     Send and Receive data
#     Send Authorization/Authentication Information
#     Cookies and authorization token for multiple requests to same server

# ### Data Scraping

#     HTML scraping
#     HTML forms
#
# SW can be programmed to access data the same way human user would do.
#
# We need to navigate the large amount of formatting information that is used to display the data (and other content) in a Web browser
#

# ### REST Service

# REpresentational State Transfer, a SW architecture standard for communication between web services
#
# REST leverages URLs as identifiers of resources which we can also think of as functions operating on data
#
# REST API can be used by any site or application no matter what language it is written in because the requests are based on the universal HTTP protocol, and the information is usually returned in the JSON format that almost all of the programming languages can read.

# Two other approaches to Web services that are still in use, but are being dominated by REST, are XML-RPC and SOAP.
#
# Web Services, Anonymous access for public data, registration token might be needed to monitor who is accessing, and access private data (OAuth)

# Web service APIs that adhere to the REST architectural constraints are called RESTful APIs.
# Besides a main component for an API, the URL, another component is the standard HTML methods:

# +
import requests

url = "https://community-open-weather-map.p.rapidapi.com/weather"

querystring = {"q":"Cairo,EG","lat":"0","lon":"0","callback":"test","id":"2172797","lang":"null","units":"\"metric\" or \"imperial\"","mode":"xml, html"}

headers = {
    'x-rapidapi-key': "<KEY>",
    'x-rapidapi-host': "community-open-weather-map.p.rapidapi.com"
    }

response = requests.request("GET", url, headers=headers, params=querystring)

print(response.text)

# + active=""
# {
#    "coord": {
#       "lon": 31.2497,
#       "lat": 30.0626
#    },
#    "weather": [
#       {
#          "id": 800,
#          "main": "Clear",
#          "description": "clear sky",
#          "icon": "01d"
#       }
#    ],
#    "base": "stations",
#    "main": {
#       "temp": 308.15,
#       "feels_like": 305.51,
#       "temp_min": 308.15,
#       "temp_max": 308.15,
#       "pressure": 1012,
#       "humidity": 11
#    },
#    "visibility": 10000,
#    "wind": {
#       "speed": 5.66,
#       "deg": 30
#    },
#    "clouds": {
#       "all": 0
#    },
#    "dt": 1618663968,
#    "sys": {
#       "type": 1,
#       "id": 2514,
#       "country": "EG",
#       "sunrise": 1618629944,
#       "sunset": 1618676577
#    },
#    "timezone": 7200,
#    "id": 360630,
#    "name": "Cairo",
#    "cod": 200
# }
# -

# ### selenium 

# + active=""
# conda install selenium
# pip install selenium
# -

# Selenium requires a driver to interface with the chosen browser. Firefox, for example, requires geckodriver
#
#     Chrome: 	https://sites.google.com/a/chromium.org/chromedriver/downloads
#     Edge: 	https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/
#     Firefox: 	https://github.com/mozilla/geckodriver/releases
#     Safari: 	https://webkit.org/blog/6900/webdriver-support-in-safari-10/

# https://github.com/mozilla/geckodriver/releases

# + active=""
# export PATH=$PATH:/opt/geckodriver >> ~/.profile
# export PATH=$PATH:/opt/chromedriver >> ~/.profile

# +
from selenium import webdriver
driver = webdriver.Chrome(r'/opt/chromedriver')
driver.maximize_window()
driver.get("http://www.seleniumeasy.com/test/basic-first-form-demo.html")
assert "Selenium Easy Demo - Simple Form to Automate using Selenium" in driver.title

eleUserMessage = driver.find_element_by_id("user-message")
eleUserMessage.clear()
eleUserMessage.send_keys("<PASSWORD>")

eleShowMsgBtn=driver.find_element_by_css_selector('#get-input > .btn')
eleShowMsgBtn.click()

eleYourMsg=driver.find_element_by_id("display")
assert "Test Python" in eleYourMsg.text
driver.close()

# +
from selenium import webdriver
from selenium.webdriver.common.keys import Keys

driver = webdriver.Chrome("/opt/chromedriver")
#url = "https://openweathermap.org/city/360630"
url = "https://www.youm7.com/"
driver.get(url)
#Temp_xpath = "//*[@id=\"weather-widget\"]/div[2]/div[1]/div[1]/div[2]/div[1]/span"
Temp_xpath = "/html/body/main/div[3]/div/div[3]/div/div/div[1]/div/h3/a"
fds = driver.find_elements_by_xpath(Temp_xpath)

print(fds[0].text)

# +
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome("/opt/chromedriver")

user_name = "YOUR EMAILID"
password = "<PASSWORD>"
driver.get("https://www.facebook.com")
element = driver.find_element_by_id("email")
element.send_keys(user_name)
element = driver.find_element_by_id("pass")
element.send_keys(password)
element.send_keys(Keys.RETURN)
element.close()
# -

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
# Step 1) Open Firefox 
browser = webdriver.Firefox()
# Step 2) Navigate to Facebook
browser.get("http://www.facebook.com")
# Step 3) Search & Enter the Email or Phone field & Enter Password
username = browser.find_element_by_id("email")
password = browser.find_element_by_id("pass")
submit   = browser.find_element_by_id("loginbutton")
username.send_keys("YOUR <PASSWORD>")
password.send_keys("<PASSWORD>")
# Step 4) Click Login
submit.click()
wait = WebDriverWait( browser, 5 )
page_title = browser.title
assert page_title == "Facebook"

# ## Practice

# +
import requests
import json

url = "https://community-open-weather-map.p.rapidapi.com/forecast"

querystring = {"q":"Cairo,EG"}

headers = {
    'x-rapidapi-key': "<KEY>",
    'x-rapidapi-host': "community-open-weather-map.p.rapidapi.com"
    }

response = requests.request("GET", url, headers=headers, params=querystring)



p = json.loads(response.text)
print(p["list"])
# -





StarcoderdataJupyterScriptsDedupFiltered