<jupyter_start><jupyter_text><jupyter_code>!pip install datasets
from datasets import load_dataset, load_metric, list_datasets, list_metrics
datasets = list_datasets()

from pprint import pprint
pprint(datasets, compact=True)


glue_dataset = list_datasets(with_details=True)[datasets.index('glue')]

pprint(glue_dataset.__dict__)  # It's a simple python dataclass

actual_task = 'sst2'
dataset = load_dataset("glue", actual_task)
metric = load_metric('glue', actual_task)

dataset
dataset["train"][100]
import datasets
import random
import pandas as pd
from IPython.display import display, HTML

def show_random_elements(dataset, num_examples=10):
    assert num_examples <= len(dataset), "Can't pick more elements than there are in the dataset."
    picks = []
    for _ in range(num_examples):
        pick = random.randint(0, len(dataset)-1)
        while pick in picks:
            pick = random.randint(0, len(dataset)-1)
        picks.append(pick)
    
    df = pd.DataFrame(dataset[picks])
    for column, typ in dataset.features.items():
        if isinstance(typ, datasets.ClassLabel):
            df[column] = df[column].transform(lambda i: typ.names[i])
    display(HTML(df.to_html()))
show_random_elements(dataset["train"])
print(dataset.column_names)
print(len(dataset["train"]))
print(len(dataset["validation"]))
print(len(dataset["test"]))
dataset["train"].features<jupyter_output><empty_output><jupyter_text><jupyter_code>df = dataset["train"].to_pandas()
df.hist()
sns.countplot(x=dataset["train"]['labels'])
plt.xlabel('labels');<jupyter_output><empty_output><jupyter_text>Tokenize dataset<jupyter_code><jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup