# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Book Linkage Tasks
#
# **Date:** 15/1/2019
#
# **Author:** <NAME>
#
# The aim of this notebook is to close out one of the major record linkage tasks for *Mapping Print, Charting Enlightenment*. There are three sets of book data that need to be more thoroughly joined to the rest of the database:
#
# * **The 'List of banned books' (BNF Ms Fr 21928-9):** This lists banned titles and was used by book inspectors in eighteenth-century France to police the trade. Each item on the list must be assigned a `super_book_code`.
# * **The 'Bastille Register' (BNF Arsenal MS 10305):** This lists books found in the Bastille after it was stormed during the revolution. In principle it records information about particular editions, so each book in the list should be assigned a `book_code`.
# * **The MMF-2 database:** This is a bibliographic database of French fiction from the eighteenth century. It records every known edition of every French novel from the period 1700-1800. Each record in the database should therefore assigned a `book_code` in the wider *MPCE* architecture.
#
# The data has all be drawn from the databases and preprocessed in R, transforming a record linkage task into a deduplication task. For each of the three tasks above, a csv has been prepared. A model will be trained to detect which rows refer to the same book, based on as much of the available information as possible.

# Import main libraries
import dedupe as dd
import pandas as pd
import os as os
import time
import numpy as np
import random
from dedupe_helper_functions import dedupe_initialise, run_deduper, save_clusters
import json

# ## Section 1: The List of Banned Books

# +
# Cell 1.1 Set main paths

# Root folder for banned books data
bbr = "banned_books_list/"

# Set paths
bb_input = bbr + "banned_books_list_dddata.csv"
bb_output = bbr + "banned_books_deduped.csv"
bb_settings = bbr + "banned_books_learned_settings"
bb_training = bbr + "banned_books_training.json"

# +
# Cell 1.2 Import data

bb_df = pd.read_csv(bb_input, encoding = "utf-8")

print(f'bb_df has {bb_df.shape[0]} rows and {bb_df.shape[1]} columns:\n{", ".join(list(bb_df))}\n')
print(f'{len(bb_df[pd.isnull(bb_df.super_book_code)])} books require super_book_codes.')

# +
# Cell 1.3 Initialise deduper

# Define the fields that the model will examine
bb_fields = [
    {'field':'super_book_title', 'type': 'String'},
    {'field':'author_name', 'type': 'String', 'has missing':True}
]

# Create Dedupe object
bb_model = dedupe_initialise(bb_df, bb_fields, training_file = bb_training)
# -

# Cell 1.4 Label training pairs
dd.consoleLabel(bb_model)

# +
# Cell 1.5 Run the Deduper

bb_model, matches = run_deduper(bb_model, bb_df, bb_settings, bb_training, recall_weight = 1.5)
# -

# Save the results
_ = save_clusters(matches, bb_df, bb_output)

bb_df[pd.notnull(bb_df.cluster) & pd.notnull(bb_df.ID)]

# ## Section 2: The Bastille Register

# +
# Cell 2.1 Paths and data

# Root folder for banned books data
basr = "bastille_register/"

# Paths
bas_input = basr + "bastille_register_dddata.csv"
bas_output = basr + "bas_reg_deduped.csv"
bas_settings = basr + "bas_reg_learned_settings"
bas_training = basr + "bas_reg_training.json"

# Data
bas_df = pd.read_csv(bas_input, encoding = "utf-8")
print(f'bas_df has {bas_df.shape[0]} rows and {bas_df.shape[1]} columns:\n{", ".join(list(bas_df))}\n')
print(f'{len(bas_df[pd.isnull(bas_df.book_code)])} books require book_codes.')

# +
# Cell 2.2 Initialise Deduper

# Fields that the model will examine
bas_fields = [
    {'field':'full_book_title', 'type': 'String'},
    {'field':'author_name', 'type': 'String', 'has missing':True},
    {'field':'stated_publication_years', 'type':'DateTime', 'has missing':True},
    {'field':'stated_publication_places', 'type': 'String', 'has missing':True},
]

# Create Dedupe object
bas_model = dedupe_initialise(bas_df, bas_fields, training_file = bas_training)
# -

# Cell 2.3 Label tranining examples
dd.consoleLabel(bas_model)

# The labelling in this instance was rough and difficult. There are so many editions, and so little information about them, that it is impossible to quickly establish a 'ground truth'. Indeed, it is impossible in many cases to establish a ground truth at all.

_, matches = run_deduper(bas_model, bas_df, bas_settings, bas_training, recall_weight = 2)

_ = save_clusters(matches, bas_df, bas_output)

# ## Section 3: The Big One (MMF-2)
#
# This section is only provisional, because at the time of writing I do not have <NAME>'s latest version of the MMF-2 database. But since he is largely working on books to 1720, it is unlikely that too many new editions from our other FBTEE datasets will be missing.

# +
# Cell 3.1 Paths and data

# Root folder for banned books data
mmfr = "mmf_2/"

# Paths
mmf_input = mmfr + "mmf_2_dddata.csv"
mmf_output = mmfr + "mmf_deduped.csv"
mmf_settings = mmfr + "mmf_settings"
mmf_training = mmfr + "mmf_training.json"

# Data
mmf_df = pd.read_csv(mmf_input, encoding = "utf-8")
print(f'bas_df has {mmf_df.shape[0]} rows and {mmf_df.shape[1]} columns:\n{", ".join(list(mmf_df))}\n')
print(f'{len(mmf_df[pd.isnull(mmf_df.book_code)])} books require book_codes.')

# +
# Cell 2.2 Initialise Deduper

# Need to do some light preprocessing on the date column
mmf_df['date'] = mmf_df.date.astype(str).str.replace("\.0", "")

# Fields that the model will examine
mmf_fields = [
    {'field':'long_title', 'type':'String'},
    {'field':'publisher', 'type':'String', 'has missing':True},
    {'field':'place', 'type':'String', 'has missing':True},
    {'field':'date', 'type':'DateTime', 'has missing':True},
    {'field':'author_name', 'type':'String', 'has missing':True}
]

# Create Dedupe object
mmf_model = dedupe_initialise(mmf_df, mmf_fields)
# -

# Cell 2.3 Label tranining examples
dd.consoleLabel(mmf_model)

_, mmf_matches = run_deduper(mmf_model, mmf_df, mmf_settings, mmf_training, recall_weight = 1.5)

_ = save_clusters(mmf_matches, mmf_df, mmf_output)

StarcoderdataJupyterScriptsDedupFiltered