# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# #DATASCI W261: Machine Learning at Scale

# ## Problem: Find out the pages which have more than 400 visits

# Complete code in MrJob class file 

# +
# %%writefile top_pages.py
"""Find Vroots with more than 400 visits.

This program will take a CSV data file and output tab-seperated lines of

    Vroot -> number of visits

To run:

    python top_pages.py anonymous-msweb.data

To store output:

    python top_pages.py anonymous-msweb.data > top_pages.out
"""
from mrjob.job import MRJob
import csv

def csv_readline(line):
    """Given a sting CSV line, return a list of strings."""
    for row in csv.reader([line]):
        return row

class TopPages(MRJob):

    def mapper(self, line_no, line):
        """Extracts the Vroot that was visited"""
        cell = csv_readline(line)
        if cell[0] == 'V':
            yield cell[1], 1
                  # What  Key, Value  do we want to output?

    def reducer(self, vroot, visit_counts):
        """Sumarizes the visit counts by adding them together.  If total visits
        is more than 400, yield the results"""
        total = sum(i for i in visit_counts)
                # How do we calculate the total visits from the visit_counts?
        
        if total > 400:
            yield vroot, total
                  # What  Key, Value  do we want to output?
        
if __name__ == '__main__':
    TopPages.run()
# -

# Driver code is ready

# +
# %reload_ext autoreload
# %autoreload 2
from top_pages import TopPages
import csv

count = 0
mr_job = TopPages(args=['anonymous-msweb.data'])
with mr_job.make_runner() as runner:
    runner.run()
    for line in runner.stream_output():
        print mr_job.parse_output_line(line)
        count +=1 
print count
# -



StarcoderdataJupyterScriptsDedupFiltered