# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Import Libraries for NLP

from pyspark.sql import SparkSession
spark = SparkSession.builder.appName(`nlp`).getOrCreate()

# Import libraries for Tokenizer 

from pyspark.ml.feature import Tokenizer, RegexTokenizer
from pyspark.sql.functions import col, udf
from pyspark.sql.types import IntegerType

senDF = spark.createDataFrame([
    (0, "Hi I heard about Spark"), 
    (1, "I wish Java could use case classes"), 
    (2, "Logistic,Regression,models,are,neat")
], ["id", "Sentence"])



StarcoderdataJupyterScriptsDedupFiltered