<jupyter_start><jupyter_text>TP 3   Utilisation des données textuelles<jupyter_code>import org.apache.spark.ml.feature.{IDF, Tokenizer, RegexTokenizer, StopWordsRemover, CountVectorizer, StringIndexer, OneHotEncoder, VectorAssembler}
import org.apache.spark.ml.{Pipeline, PipelineModel}
import org.apache.spark.ml.classification.LogisticRegression
 val df : DataFrame = spark.read
      .parquet("data/prepared_trainingset")
df.show()
df.select($"text".isNull).groupBy($"(text IS NULL)").count().show()
val tokenizer = new RegexTokenizer()
  .setPattern("\\W+")
  .setGaps(true)
  .setInputCol("text")
  .setOutputCol("tokens")
val stopWordsRemover = new StopWordsRemover()
    .setInputCol("tokens")
    .setOutputCol("tokensWOstopwords")
val cvModel = new CountVectorizer()
  .setInputCol("tokensWOstopwords")
  .setOutputCol("countedWord")
  .setMinDF(2) //a word has to appear 2 times to be in the vocabulary 
  .fit(stopWordsRemover.transform(tokenizer.transform(df)))

val idf = new IDF()
    .setInputCol("countedWord")
    .setOutputCol("tfidf")<jupyter_output><empty_output><jupyter_text>Conversion des variables catégorielles en variables numériques<jupyter_code>val indexerCountry = new StringIndexer()
  .setInputCol("country2")
  .setOutputCol("country_indexed")
val indexerCurrency = new StringIndexer()
  .setInputCol("currency2")
  .setOutputCol("currency_indexed")
val onehotencoderCountry = new OneHotEncoder()
    .setInputCol("country_indexed")
    .setOutputCol("country_onehot")
val onehotencoderCurrency = new OneHotEncoder()
    .setInputCol("currency_indexed")
    .setOutputCol("currency_onehot")
val assembler = new VectorAssembler()
  .setInputCols(Array("tfidf", "days_campaign", "hours_prepa", "goal", "country_onehot", "currency_onehot"))
  .setOutputCol("features")
val lr = new LogisticRegression()
  .setElasticNetParam(0.0)
  .setFitIntercept(true)
  .setFeaturesCol("features")
  .setLabelCol("final_status")
  .setStandardization(true)
  .setPredictionCol("predictions")
  .setRawPredictionCol("raw_predictions")
  .setThresholds(Array(0.7, 0.3))
  .setTol(1.0e-6)
  .setMaxIter(20)
val splits = df.randomSplit(Array(0.9, 0.1))
val training = splits(0).cache()
val test = splits(1)
val pipeline = new Pipeline()
  .setStages(Array(tokenizer, stopWordsRemover,cvModel,idf, indexerCountry,indexerCurrency,
                   onehotencoderCountry, onehotencoderCurrency, assembler, lr))
val model = pipeline.fit(training)
df.show()
val predic = model.transform(test)<jupyter_output><empty_output><jupyter_text>predic.select("features","raw_predictions","probability","predictions","final_status").show()<jupyter_code>import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
val evaluator = new MulticlassClassificationEvaluator()
  .setLabelCol("final_status")
  .setPredictionCol("predictions")
  .setMetricName("f1")
val result = evaluator.evaluate(predic)
val sameModel = PipelineModel.load("spark-logistic-regression-model")
val paramGrid = new ParamGridBuilder()
  .addGrid(cvModel.minDF, Array(55.0,75.0,95.0))
  .addGrid(lr.regParam, Array(10e-8, 10e-6, 10e-4, 10e-2))
  .build()

val lrtv = TrainValidationSplit()
    .setEstimator(pipeline)
    .setEstimatorParamMaps(paramGrid)
    .setEvaluator(evaluator)

val modelGrid = lrtv.fit(training)
import org.apache.spark.ml.feature.{IDF, Tokenizer, RegexTokenizer, StopWordsRemover, CountVectorizer, StringIndexer, OneHotEncoder, VectorAssembler,CountVectorizerModel}
import org.apache.spark.ml.{Pipeline, PipelineModel}
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
import org.apache.spark.ml.tuning.{ParamGridBuilder, TrainValidationSplit,TrainValidationSplitModel, CrossValidator, CrossValidatorModel}

val paramGrid = new ParamGridBuilder()
      .addGrid(cvModel.minDF, Array(55.0,75.0,95.0))
      .addGrid(lr.regParam, Array(10e-8, 10e-6, 10e-4, 10e-2))
      .build()
val lrcv = new CrossValidator()
      .setEstimator(pipeline)
      .setEstimatorParamMaps(paramGrid)
      .setEvaluator(evaluator)
      .setNumFolds(5)

    val modelGridCV = lrcv.fit(training)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup