# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.8.11 64-bit (''otopy38'': conda)'
#     name: python3
# ---

# # Schema from example

oto_data = {
        "_id" : "5e3868eb61e48d0017ab68b0",
        "kind" : "OutlierModel",
        "bt" : 1580755178368101,
        "tt" : 0,
        "modelId" : "5c1a7b648a6df285f82bdd4f",
        "distance" : 204311,
        "closest_centroid" : 82,
        "outlier_score" : 90.9653,
        "centroid_rareness" : 4.26592,
        "is_outlier" : 1,
        "source" : "40FWF4_R-",
        "account" : "analogdevices_dev"
    }


# # Using schema

# ## Validation of a schema

# +
from schema import Schema, And, Use, Optional, SchemaError

schema = Schema([{'name': And(str, len),
                   'age':  And(Use(int), lambda n: 18 <= n <= 99),
                   Optional('gender'): And(str, Use(str.lower),
                                           lambda s: s in ('squid', 'kid'))}])

data = [{'name': 'Sue', 'age': '28', 'gender': 'Squid'},
         {'name': 'Sam', 'age': '42'},
        {'name': 'Sacha', 'age': '20', 'gender': 'KID'}]



# +
validated = schema.validate(data)

assert validated == [{'name': 'Sue', 'age': 28, 'gender': 'squid'},
                      {'name': 'Sam', 'age': 42},
                      {'name': 'Sacha', 'age' : 20, 'gender': 'kid'}]
# -

"""{'_id': str,
 'kind': str,
 'bt': int,
 'tt': int,
 'modelId': str,
 'distance': int,
 'closest_centroid': int,
 'outlier_score': float,
 'centroid_rareness': float,
 'is_outlier': int,
 'source': str,
 'account': str}
"""
schema_otosense= Schema([{'_id': str,
                          'kind':Use(str),
                          'bt': Use(int),
                          'tt':Use(int),
                          'modelId':Use(str),
                          'distance':Use(int),
                          'closest_centroid':Use(int),
                          'outlier_score':Use(int),
                          'centroid_rareness':Use(float),
                          'is_outlier':Use(bool),
                          'source':Use(str),
                          'account':And(str,
                                           lambda s: s in ('analogdevices_dev', 'analogdevices_prod'))
}])
                  

schema_otosense.validate([oto_data])

# # Validation using Great expectations

import great_expectations as ge
import pandas as pd

data_list = [{'_id': '5e3868eb61e48d0017ab68b0',
  'kind': 'OutlierModel',
  'bt': 1580755178368101,
  'tt': 0,
  'modelId': '5c1a7b648a6df285f82bdd4f',
  'distance': 204311,
  'closest_centroid': 82,
  'outlier_score': 90,
  'centroid_rareness': 4.26592,
  'is_outlier': True,
  'source': '40FWF4_R-',
  'account': 'analogdevices_dev'},
  {'_id': '4e3868eb61e48d0017ab6898',
  'kind': 'OutlierModel',
  'bt': 1580755178368156,
  'tt': 0,
  'modelId': '5c1a7b648a6df285f82bdd4f',
  'distance': 204316,
  'closest_centroid': 80,
  'outlier_score': 10,
  'centroid_rareness': 4.26592,
  'is_outlier': True,
  'source': '40FWF4_R-',
  'account': 'analogdevices_dev'},
{'_id': '3e3868eb61e48d0017ab6800',
  'kind': 'OutlierModel',
  'bt': 1580755178368101,
  'tt': 0,
  'modelId': '5c1a7b648a6df285f82bdd4f',
  'distance': 20400,
  'closest_centroid': 17,
  'outlier_score': 9,
  'centroid_rareness': 3.1,
  'is_outlier': True,
  'source': '40FWF4_R-',
  'account': 'analogdevices_prod'}

  ]

df_csv = pd.DataFrame(data_list).to_csv('data_great_expect.csv')
df=ge.read_csv('data_great_expect.csv')

df

# +
feature_columns = ['kind', 'bt', 'tt','modelId', 'distance','closest_centroid']
for col in feature_columns:
    df.expect_column_to_exist(col)

df.expect_column_values_to_be_of_type('kind', 'str')
# -

df.get_expectation_suite()

# # Generating data from a model

from sdv import SDV #synthetic data vault

sdv = SDV()
df_data = pd.DataFrame(data_list)
sdv.fit(tables =[df_data])

#

df_data



StarcoderdataJupyterScriptsDedupFiltered