<jupyter_start><jupyter_text>0. Importing PySpark<jupyter_code>from pyspark import SparkContext
from pyspark.sql import SparkSession

sc = SparkContext()
spark = SparkSession(sc)
# run this cell only once<jupyter_output><empty_output><jupyter_text>1. Loading the dataset<jupyter_code>import csv 
rdd1 = sc.textFile("digikala_comments.csv")\
        .mapPartitions(lambda line: csv.reader(line, delimiter=',', quotechar='"'))\
        .filter(lambda line: len(line) >= 2 and line[0] != 'product_id')
rdd1.take(1)<jupyter_output><empty_output><jupyter_text>2. Most popular item<jupyter_code>rdd1.map(lambda l: (l[0], int(l[4]) - int(l[5])))\
    .reduceByKey(lambda v1, v2: v1 + v2)\
    .max(key = lambda x: x[1])<jupyter_output><empty_output><jupyter_text>3. Percentage of unverified comments<jupyter_code>total_count = rdd1.count()
not_verified_count = rdd1.filter(lambda l: l[6] != 'verified').count()
print((not_verified_count / total_count ) * 100)<jupyter_output>1.7600964436407474<jupyter_text>4. The largest word in the comment section<jupyter_code>rdd1.map(lambda l: l[9].split())\
    .flatMap(lambda x: x)\
    .map(lambda x: (x, len(x)))\
    .reduce(lambda w1, w2: w1 if w1[1] > w2[1] else w2)<jupyter_output><empty_output><jupyter_text>5. Top 10 words in advantages and disadvantages<jupyter_code>import ast
persian_chars=["آ", "ا", "ب", "پ", "ت", "ث", "ج", "چ", "ح", "خ", "د", "ذ", "ر", "ز", "ژ", "س", "ش",
                  "ص", "ض", "ط", "ظ", "ع", "غ", "ف", "ق", "ک" ,"گ", "ل", "م", "ن", "و" ,"ه", "ی"]
def preprocess_words(x):
    for c in x:
        if c not in persian_chars:
            x = x.replace(c, " ")
    return x
def get_top_10_words(column_num):
    return rdd1.filter(lambda l: l[column_num] != '')\
                .map(lambda l: l[column_num])\
                .map(preprocess_words)\
                .map(lambda x: x.split())\
                .flatMap(lambda x: x)\
                .map(lambda x: (x, 1))\
                .reduceByKey(lambda x, y: x + y)\
                .takeOrdered(10, key=lambda x: -x[1])<jupyter_output><empty_output><jupyter_text>Advantages column:<jupyter_code>get_top_10_words(10)<jupyter_output><empty_output><jupyter_text>Disadvantages column:<jupyter_code>get_top_10_words(11)<jupyter_output><empty_output><jupyter_text>6. Most popular character in product's title<jupyter_code>rdd1.map(lambda l: l[1])\
    .flatMap(lambda x: x)\
    .filter(lambda x: x in persian_chars)\
    .map(lambda x: (x, 1))\
    .reduceByKey(lambda x, y: x + y)\
    .takeOrdered(1, key=lambda x: -x[1])<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup