# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + papermill={"duration": 0.009778, "end_time": "2021-07-29T04:02:31.896578", "exception": false, "start_time": "2021-07-29T04:02:31.886800", "status": "completed"} tags=[]


# + [markdown] papermill={"duration": 0.009778, "end_time": "2021-07-29T04:02:31.896578", "exception": false, "start_time": "2021-07-29T04:02:31.886800", "status": "completed"} tags=[]
#
#

# + _cell_guid="93620de9-b3b8-4129-8b7b-0b276cc58d68" _uuid="932f2c25-03ca-46e7-a87f-7daadf0247a6" jupyter={"outputs_hidden": false} papermill={"duration": 0.021164, "end_time": "2021-07-29T04:02:31.927913", "exception": false, "start_time": "2021-07-29T04:02:31.906749", "status": "completed"} tags=[]

import os
from pathlib import Path
in_folder_path = Path('../input/k/leolu1998/clrp-finetune-roberta-large')
scripts_dir = Path(in_folder_path / 'scripts')

# + papermill={"duration": 7.942391, "end_time": "2021-07-29T04:02:39.878942", "exception": false, "start_time": "2021-07-29T04:02:31.936551", "status": "completed"} tags=[]

os.chdir(scripts_dir)
exec(Path("imports.py").read_text())
exec(Path("config.py").read_text())
exec(Path("dataset.py").read_text())
exec(Path("model.py").read_text())
os.chdir('/kaggle/working')


# + _cell_guid="f182401e-add4-489a-97ad-253995c77ea4" _uuid="db10450f-b0f8-4687-aed6-d1a986657ce6" jupyter={"outputs_hidden": false} papermill={"duration": 124.510896, "end_time": "2021-07-29T04:04:44.398451", "exception": false, "start_time": "2021-07-29T04:02:39.887555", "status": "completed"} tags=[]

test_df = pd.read_csv("/kaggle/input/commonlitreadabilityprize/test.csv")
tokenizer = torch.load('../input/mytokenizers/roberta-tokenizer.pt')
models_folder_path = Path(in_folder_path / 'models')
models_preds = []
n_models = 10

for model_num in range(n_models):
    print(f'Inference#{model_num+1}/{n_models}')
    test_ds = CLRPDataset(data=test_df, tokenizer=tokenizer, max_len=Config.max_len, is_test=True)
    test_sampler = SequentialSampler(test_ds)
    test_dataloader = DataLoader(test_ds, sampler = test_sampler, batch_size=Config.batch_size)
    model = torch.load(models_folder_path / f'best_model_{model_num}.pt').to(Config.device)

    all_preds = []
    
    model.eval()

    for step,batch in enumerate(test_dataloader):
        sent_id, mask = batch['input_ids'].to(Config.device), batch['attention_mask'].to(Config.device)
        with torch.no_grad():
            preds = model(sent_id, mask)
            all_preds += preds.flatten().cpu().tolist()
            
    models_preds.append(all_preds)
del model, tokenizer, test_dataloader, test_sampler
gc.collect()
torch.cuda.empty_cache()



# + [markdown] papermill={"duration": 0.011979, "end_time": "2021-07-29T04:04:44.422070", "exception": false, "start_time": "2021-07-29T04:04:44.410091", "status": "completed"} tags=[]
# # ENS 2

# + papermill={"duration": 55.753162, "end_time": "2021-07-29T04:05:40.186466", "exception": false, "start_time": "2021-07-29T04:04:44.433304", "status": "completed"} tags=[]

import os
from pathlib import Path
in_folder_path2 = Path('../input/clrprobertalarge463-lb')
scripts_dir2 = Path(in_folder_path2 / 'scripts')

os.chdir(scripts_dir2)
exec(Path("imports.py").read_text())
exec(Path("config.py").read_text())
exec(Path("dataset.py").read_text())
exec(Path("model.py").read_text())
os.chdir('/kaggle/working')

test_df = pd.read_csv("/kaggle/input/commonlitreadabilityprize/test.csv")
tokenizer = torch.load('../input/mytokenizers/roberta-tokenizer.pt')
models_folder_path2 = Path(in_folder_path2 / 'models')
models_preds2 = []
n_models = 5

for model_num in range(n_models):
    print(f'Inference#{model_num+1}/{n_models}')
    test_ds = CLRPDataset(data=test_df, tokenizer=tokenizer, max_len=Config.max_len, is_test=True)
    test_sampler = SequentialSampler(test_ds)
    test_dataloader = DataLoader(test_ds, sampler = test_sampler, batch_size=Config.batch_size)
    model2 = torch.load(models_folder_path2 / f'best_model_{model_num}.pt').to(Config.device)

    all_preds2 = []
    
    model2.eval()

    for step,batch in enumerate(test_dataloader):
        sent_id, mask = batch['input_ids'].to(Config.device), batch['attention_mask'].to(Config.device)
        with torch.no_grad():
            preds2 = model2(sent_id, mask)
            all_preds2 += preds2.flatten().cpu().tolist()
            
    models_preds2.append(all_preds2)

####
del model2, tokenizer, test_dataloader, test_sampler
gc.collect()
torch.cuda.empty_cache()

# + [markdown] papermill={"duration": 0.013301, "end_time": "2021-07-29T04:05:40.214563", "exception": false, "start_time": "2021-07-29T04:05:40.201262", "status": "completed"} tags=[]
# # ENS 3

# + papermill={"duration": 21.6759, "end_time": "2021-07-29T04:06:01.903271", "exception": false, "start_time": "2021-07-29T04:05:40.227371", "status": "completed"} tags=[]

import os
from pathlib import Path
in_folder_path3 = Path('../input/mymodelrobertabase')
scripts_dir3 = Path(in_folder_path3 / 'scripts')

os.chdir(scripts_dir3)
exec(Path("imports.py").read_text())
exec(Path("config.py").read_text())
exec(Path("dataset.py").read_text())
exec(Path("model.py").read_text())
os.chdir('/kaggle/working')

test_df = pd.read_csv("/kaggle/input/commonlitreadabilityprize/test.csv")
tokenizer = torch.load('../input/mytokenizers/roberta-tokenizer.pt')
models_folder_path3 = Path(in_folder_path3 / 'models')
models_preds3 = []
n_models = 5

for model_num in range(n_models):
    print(f'Inference#{model_num+1}/{n_models}')
    test_ds = CLRPDataset(data=test_df, tokenizer=tokenizer, max_len=Config.max_len, is_test=True)
    test_sampler = SequentialSampler(test_ds)
    test_dataloader = DataLoader(test_ds, sampler = test_sampler, batch_size=Config.batch_size)
    model3 = torch.load(models_folder_path3 / f'best_model_{model_num}.pt').to(Config.device)

    all_preds3 = []
    
    model3.eval()

    for step,batch in enumerate(test_dataloader):
        sent_id, mask = batch['input_ids'].to(Config.device), batch['attention_mask'].to(Config.device)
        with torch.no_grad():
            preds3 = model3(sent_id, mask)
            all_preds3 += preds3.flatten().cpu().tolist()
            
    models_preds3.append(all_preds3)



# + papermill={"duration": 0.024919, "end_time": "2021-07-29T04:06:01.942474", "exception": false, "start_time": "2021-07-29T04:06:01.917555", "status": "completed"} tags=[]
all_preds

# + papermill={"duration": 0.023637, "end_time": "2021-07-29T04:06:01.982583", "exception": false, "start_time": "2021-07-29T04:06:01.958946", "status": "completed"} tags=[]
all_preds2

# + papermill={"duration": 0.022981, "end_time": "2021-07-29T04:06:02.019573", "exception": false, "start_time": "2021-07-29T04:06:01.996592", "status": "completed"} tags=[]
all_preds3

# + _cell_guid="ed2be4e8-71fe-4eb2-a4cf-7cc6b30c5519" _uuid="6116227b-04bd-4c60-988b-e244833da61d" jupyter={"outputs_hidden": false} papermill={"duration": 0.306695, "end_time": "2021-07-29T04:06:02.340672", "exception": false, "start_time": "2021-07-29T04:06:02.033977", "status": "completed"} tags=[]
models_preds_ens = 0.51*np.array(models_preds).mean(axis=0)+(np.array(models_preds2)*0.3+np.array(models_preds3)*0.2).mean(axis=0)
print(models_preds_ens.shape)
print(models_preds_ens)
all_preds_ens = models_preds_ens

result_df = pd.DataFrame(
    {
        'id': test_df.id,
        'target': all_preds_ens
    })


result_df.to_csv('submission.csv', index=False)
result_df.head(10)

StarcoderdataJupyterScriptsDedupFiltered