# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Verify language

# +
import math

import pandas as pd

from comprehend import get_dominant_language
# -

df = pd.read_pickle('text_20200409.pkl')

# ## Add `characters_num` column

df['characters_num'] = df.text.str.len()


# ## Add `units_num` column

def count_units(characters):
    units = characters / 100.0
    units = math.ceil(units)
    if units < 3:
        units = 3
    return units


df['units_num'] = df.characters_num.apply(count_units)

# ## Count price of Amazon Comprehend language detection

df.units_num.sum() * 0.0001


# ## Add `comprehend` column

def get_comprehend(text):
    result = get_dominant_language(text)
    if result == 1:
        return None, None
    return result


df['comprehend'] = df.text.apply(get_comprehend)

# ## Save to pickle file

df.to_pickle('comprehend_20201104.pkl')

# ### Test pickle file

pd.read_pickle('comprehend_20201104.pkl')


# ## Add `language_by_aws` column

def get_language_by_aws(data):
    return data[0]


df['language_by_aws'] = df.comprehend.apply(get_language_by_aws)


# ## Add `aws_score` column

def get_aws_score(data):
    return data[1]


df['aws_score'] = df.comprehend.apply(get_aws_score)

# ## Select only verified English

dff = df[df.language_by_aws == 'en'][df.aws_score > 0.9]

# ## Save to pickle file

dff.to_pickle('english_verified_20201104.pkl')

# ### Test pickle file

pd.read_pickle('english_verified_20201104.pkl')

StarcoderdataJupyterScriptsDedupFiltered