# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# # Common Crawl dataset

# ### Summary of the Common Crawl Data
# * Three formats in this dataset: WARC(Raw data), WAT(Meta data for WARC), WET(Extracted text data)
# * We should focus on WET because WARC and WAT are with too much useless information
# * This dataset is humongous, split in month. We cannot work on all. 
# * For July 2015, it is said to have 1.81 billion pages and 145 TB data
# * By checking the July 2015 data, the break-down is like this
#  * 99 directories holding 33957 .gz pair of (WARC, WAT and WET) files
#  * each directory holds 343 files
#  * each file is around 110 MB in .gz and 250 MB in uncompressed format
#  * each file has around 51K pages
# * We choose 1 of the 99 directory to start process
#  * s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981921.1/
# * We focus only on WET file for the assignment 2
#  * WET provides the parsed plaintext rather than the HTML contents
#  * But it could not reveal the raw content in the browser as the HTML tags are gone
#
# ### What is in WET
# * extracted plaintext on the page
# * URL
# * content length
# * no metadata for WET so there is no offset information for each page. 
#
# ### Comparison to NZ dataset
# * The WET data is more clean than NZ
# * NZ population speaks Roman languages and doesn't have that many Unicode pages than CCrawl data
# * The CCrawl WET data has no meta data
# * NZ dataset has metadata, can have offset for each page in the bigfile
#
#

# ### Check July 2015 Common Crawl dataset 
# http://blog.commoncrawl.org/2015/08/july-2015-crawl-archive-available/
#
#
# ```
# $ aws s3 ls s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2015-32/
#                            PRE segments/
# 2015-08-13 20:45:44        632 segment.paths.gz
# 2015-08-13 20:45:44     104599 warc.paths.gz
# 2015-08-13 20:45:45     104321 wat.paths.gz
# 2015-08-13 20:45:45     104322 wet.paths.gz
#
# $ wc -l *.paths
#       99 segment.paths
#    33957 warc.paths
#    33957 wat.paths
#    33957 wet.paths
#   101970 total
#
# $ head wet.paths
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00000-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00001-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00002-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00003-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00005-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00006-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00007-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00008-ip-10-236-191-2.ec2.internal.warc.wet.gz
# common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00009-ip-10-236-191-2.ec2.internal.warc.wet.gz
#
# $ aws s3 ls s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/ --summarize --human-readable | tail -n3
#
# Total Objects: 343
#    Total Size: 33.9 GiB
#
# $ aws s3 ls s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/warc/ --summarize --human-readable | tail -n3
#
# Total Objects: 343
#    Total Size: 300.2 GiB
#
# $ aws s3 ls s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wat/ --summarize --human-readable | tail -n3
#
# Total Objects: 343
#    Total Size: 95.1 GiB
#
# $ grep 'CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal' *.paths | grep 1460
# warc.paths:common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/warc/CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.gz
# wat.paths:common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wat/CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wat.gz
# wet.paths:common-crawl/crawl-data/CC-MAIN-2015-32/segments/1438042981460.12/wet/CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wet.gz
#
# ```
#

# ### Checking WET
#
# ```
# $ ls -l *.gz
# -rw-r--r--  1 robert  wheel  298358000 Aug  7 22:09 CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wat.gz
# -rw-r--r--  1 robert  wheel  106093786 Aug  7 22:09 CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wet.gz
#
# $ ls -l CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wet
# -rw-r--r--  1 robert  wheel  250969654 Aug  7 22:09 CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wet
#
# ```
# ```
#       1 WARC/1.0^M
#       2 WARC-Type: warcinfo^M
#       3 WARC-Date: 2015-08-08T01:50:27Z^M
#       4 WARC-Filename: CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wet.gz^M
#       5 WARC-Record-ID: <urn:uuid:8981c989-bba8-4c65-99dc-e80edb9b9bd4>^M
#       6 Content-Type: application/warc-fields^M
#       7 Content-Length: 257^M
#       8 ^M
#       9 Software-Info: ia-web-commons.1.0-SNAPSHOT-20150727093232^M
#      10 Extracted-Date: Sat, 08 Aug 2015 01:50:27 GMT^M
#      11 robots: classic^M
#      12 isPartOf: CC-MAIN-2015-32^M
#      13 operator: CommonCrawl Admin^M
#      14 description: Wide crawl of the web for August 2015^M
#      15 publisher: CommonCrawl^M
#      16 ^M
#      17 ^M
#      18 ^M
#      19 WARC/1.0^M
#      20 WARC-Type: conversion^M
#      21 WARC-Target-URI: http://0205921396.reader.chegg.com/homework-help/questions-and-answers/physics-archive-2013-may-07^M
#      22 WARC-Date: 2015-07-28T01:25:02Z^M
#      23 WARC-Record-ID: <urn:uuid:ba5ab6f4-c6f4-4384-abb0-7eb39b077482>^M
#      24 WARC-Refers-To: <urn:uuid:6c1f9943-8ef1-4a61-8437-089c3545087f>^M
#      25 WARC-Block-Digest: sha1:NAN6E2CTH4GJ3CF5ASQZRTZLZ65OSSGS^M
#      26 Content-Type: text/plain^M
#      27 Content-Length: 47^M
#      28 ^M
#      29 Chegg eReader
#      30 THE PROTOTYPE TEST
#      31 : EQUIVALENCE ^M
#      32 ^M
#  ```

# ### Checking WAT
#
# #### This dataset is not useful for our assignment: Confirmed after checking
#  
# ```
# $ ls -l *.wat.gz
# -rw-r--r--  1 robert  wheel  298358000 Aug  7 22:09 CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wat.gz
#
# $ gzip -d *.wat.gz
# $ ls -l *.wat
# -rw-r--r--  1 robert  wheel  1164909718 Aug  7 22:09 CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.wat
#     
# ```
#
# ```
#      1 WARC/1.0
#      2 WARC-Type: warcinfo
#      3 WARC-Date: 2015-08-08T01:50:27Z
#      4 WARC-Filename: CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.gz
#      5 WARC-Record-ID: <urn:uuid:6569b698-57b7-464c-9d52-43f7e71c5918>
#      6 Content-Type: application/warc-fields
#      7 Content-Length: 108
#      8
#      9 Software-Info: ia-web-commons.1.0-SNAPSHOT-20150727093232
#     10 Extracted-Date: Sat, 08 Aug 2015 01:50:27 GMT
#     11
#     12
#     13
#     14 WARC/1.0
#     15 WARC-Type: metadata
#     16 WARC-Target-URI: CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.gz
#     17 WARC-Date: 2015-08-06T11:12:07Z
#     18 WARC-Record-ID: <urn:uuid:7dd56397-6d91-410a-827a-d26b744596e5>
#     19 WARC-Refers-To: <urn:uuid:4da255da-ba75-40f9-8f3c-83b91c64a633>
#     20 Content-Type: application/json
#     21 Content-Length: 1150
#     22
#     23 {"Envelope":{"Format":"WARC","WARC-Header-Length":"273","Block-Digest":"sha1:SLBF7AGTH2IQQQ22RDT6SHVDPHQSP2Z6","Actual-Content-Length":"342","WARC-Header-Metadata":{"WAR       C-Type":"warcinfo","WARC-Filename":"CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.gz","WARC-Date":"2015-08-06T11:12:07Z","Content-Length":"342","WARC-Re       cord-ID":"<urn:uuid:4da255da-ba75-40f9-8f3c-83b91c64a633>","Content-Type":"application/warc-fields"},"Payload-Metadata":{"Trailing-Slop-Length":"0","Actual-Content-Type"       :"application/warc-fields","Actual-Content-Length":"342","Headers-Corrupt":true,"WARC-Info-Metadata":{"robots":"classic","software":"Nutch 1.6 (CC)/CC WarcExport 1.0","d       escription":"Wide crawl of the web for August 2015","hostname":"ip-10-236-191-2.ec2.internal","format":"WARC File Format 1.0","isPartOf":"CC-MAIN-2015-32","operator":"Co       mmonCrawl Admin","publisher":"CommonCrawl"}}},"Container":{"Compressed":true,"Gzip-Metadata":{"Footer-Length":"8","Deflate-Length":"433","Header-Length":"10","Inflated-C       RC":"-1161559046","Inflated-Length":"619"},"Offset":"0","Filename":"CC-MAIN-20150728002301-00004-ip-10-236-191-2.ec2.internal.warc.gz"}}
#     24
# ```

StarcoderdataJupyterScriptsDedupFiltered