<jupyter_start><jupyter_text>Importer de librairies Spark nécessaires<jupyter_code>from pyspark.sql.functions import *
from pyspark.sql.types import *
from pyspark.sql.window import Window
%cd data
#%ls
from pyspark.context import SparkContext
from pyspark.sql.session import SparkSession
sc = SparkContext.getOrCreate()
spark = SparkSession(sc)<jupyter_output><empty_output><jupyter_text>importer nos données en indiquant le path et utiliser read_csv   Interpretation de la base de données<jupyter_code>data = spark.read.csv("/Users/garbamoussa/Desktop/Python_ML/SparkML-master/data/2010-12-01.csv",header = 'True',inferSchema='True')
spark.version
data.show()
data.printSchema()
data.select("InvoiceNo").show()
data.select(data['InvoiceNo'], data['UnitPrice'] + 1).show()<jupyter_output>+---------+-----------------+
|InvoiceNo|  (UnitPrice + 1)|
+---------+-----------------+
|   536365|             3.55|
|   536365|4.390000000000001|
|   536365|             3.75|
|   536365|4.390000000000001|
|   536365|4.390000000000001|
|   536365|             8.65|
|   536365|             5.25|
|   536366|             2.85|
|   536366|             2.85|
|   536367|             2.69|
|   536367|              3.1|
|   536367|              3.1|
|   536367|             4.75|
|   536367|             2.65|
|   536367|             5.25|
|   536367|             5.95|
|   536367|            10.95|
|   536367|             6.95|
|   536367|             6.95|
|   536367|             8.95|
+---------+-----------------+
only showing top 20 rows<jupyter_text>Total number of transactions in bureau dat<jupyter_code>print(data.count())
data.filter(data['UnitPrice'] > 5).show()
data.groupBy("Country").count().show()
data.groupBy("Quantity").count().show()

data.groupBy("device_type").count().show()
data.createOrReplaceTempView("Dataset")
sqlDF = spark.sql("SELECT * FROM Dataset")
sqlDF.show()
data.show(n=2, truncate=False, vertical=True)
#print(data.toPandas())
click = data.filter(col('click')==1).count()
no_click = data.filter(col('click')==0).count()

print(click/(click+ no_click)*100)
#data.filter(col('click') == 5748394).count().show()

data.filter(data['click'] == 5748394).show()
from pyspark.sql import SparkSession
from pyspark.ml import Pipeline
from pyspark.sql.functions import mean,col,split, col, regexp_extract, when, lit
from pyspark.ml.feature import StringIndexer
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.feature import QuantileDiscretizer

campaign_count = data.count()
print(campaign_count)
data.describe().show()
type(data)
data.select("click","buyer_bid","geo_city").show()
data.groupBy("click").count().show()
data.groupBy("buyer_bid").count().show()
gropuBy_output = data.groupBy("click").count()
display(gropuBy_output)
#data.na.drop(subset=["buyer_id"])
#data.groupBy("click","buyer_id").count().show()
## Créer une fonction permettant d'afficher les colonnes ayant de valeurs manquantes et le nombre de valeurs manquantes

def null_value_count(df):
    null_columns_counts = []
    numRows = df.count()
    for k in df.columns:
        nullRows = df.where(col(k).isNull()).count()
        if(nullRows > 0):
            temp = k,nullRows
            null_columns_counts.append(temp)
    return(null_columns_counts)
null_columns_count_list = null_value_count(data)
spark.createDataFrame(null_columns_count_list, ['Column_With_Null_Value', 'Null_Values_Count']).show()
#spark.sql("SELECT * FROM global_temp.dataset").show()
data.limit(2).toPandas()<jupyter_output><empty_output><jupyter_text>Feature 1 : Numer of past loans per custermer<jupyter_code>data_100 = data.limit(100) # limiter le nombre à afficher à 100
buyerbid_par_click = data_100.select('click', 'buyer_bid').groupBy('click').count().withColumnRenamed("count", "data_buyer_bid_COUNT")
data_100 = data_100.join(buyerbid_par_click, ['click'],how ='left')
print((data_100.count(), len(data_100.columns)))
mean_buyer_bid = data.select(mean('buyer_bid')).collect()[0][0]
print(mean_buyer_bid)

data.select("creative_freq").show()
data.select("buyer_bid").distinct().show()
data.select("click").distinct().show()
#data.groupby('click').avg('buyer_id').collect()
import pandas_profiling
import numpy as np
from pathlib import Path
data1 = data.limit(100).toPandas()
profile_data = pandas_profiling.ProfileReport(data1) 
profile_data
feature = VectorAssembler(inputCols=data.columns[1:],outputCol="features")
feature_vector= feature.transform(data)
(trainingData, testData) = feature_vector.randomSplit([0.8, 0.2],seed = 11)
from pyspark.ml.classification import LogisticRegression
lr = LogisticRegression(labelCol="click", featuresCol="features")
#Training algo
lrModel = lr.fit(trainingData)
lr_prediction = lrModel.transform(testData)
lr_prediction.select("prediction", "click", "features").show()
evaluator = MulticlassClassificationEvaluator(labelCol="click", predictionCol="prediction", metricName="accuracy")

data.createOrReplaceTempView("data1")
from pyspark.sql.functions import max 

#data.select(max("count")).take(1)
#maxSql = spark.sql("""SELECT DEST_buyer_id, sum(count) as buyer_total
#FROM data1
#GROUP BY DEST_buyer_id
#ORDER BY sum(count) DESC
#LIMIT 5""")
from pyspark.sql.functions import max 
data1\
    .groupBy("DEST_buyer_id")\
    .sum(count)\
    .withColumnRenamed("sum(count)", "buyer_total")\
    .sort(desc("buyer_total"))\
    .limit(5)\
    .show()
from pyspark.sql.functions import sum
df = spark.createDataFrame(data1,["count","buyer_id"])
df.show()
from pyspark.sql.functions import sum
df = spark.createDataFrame(data1,["count","click"])
df.show()
total = data.groupBy().sum()
total.show()<jupyter_output>+------------------+-----------------+---------------+---------------------+------------+-------------------------+-----------------------+----------------+----------------+------------------+-----------------+------------+-------------+----------------+--------------------+----------+
|sum(fold_position)|   sum(buyer_bid)|sum(geo_region)|sum(operating_system)|sum(browser)|sum(advertiser_frequency)|sum(advertiser_recency)|sum(campaign_id)|sum(creative_id)|sum(creative_freq)|sum(creative_rec)|sum(geo_dma)|sum(geo_city)|sum(device_type)|sum(geo_postal_code)|sum(click)|
+------------------+-----------------+---------------+---------------------+------------+-------------------------+-----------------------+----------------+----------------+------------------+-----------------+------------+-------------+----------------+--------------------+----------+
|            486228|8267747.191345905|        3420869|             29748876|     4271812|                   777599|             1299080736|[...]
StarcoderdataJupyterStructuredCleanDedup