# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Sparkmagic (PySpark)
#     language: ''
#     name: pysparkkernel
# ---

# # Pubmed abstracts analysis
#

# +
import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job

glueContext = GlueContext(SparkContext.getOrCreate())
# -

df=spark.read.json("s3n://aegovan-data/pubmed-json/")
df.printSchema()

df.count()

df.groupBy('pub_date.year').count().orderBy("year").show(n=500)

StarcoderdataJupyterScriptsDedupFiltered