# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # fuzzy-search
# Fuzzy search modules for searching lists of words in low quality OCR and HTR text.
#
# ## Usage

# +
from fuzzy_search.fuzzy_phrase_searcher import FuzzyPhraseSearcher
from fuzzy_search.fuzzy_phrase_model import PhraseModel

# highger matching thresholds for higher quality OCR/HTR (higher precision, recall should be good anyway)
# lower matching thresholds for lower quality OCR/HTR (higher recall, as that's the main problem)

config = {
    "char_match_threshold": 0.6,
    "ngram_threshold": 0.5,
    "levenshtein_threshold": 0.6,
    "ignorecase": False,
    "max_length_variance": 3,
    "ngram_size": 2,
    "skip_size": 2,
}

# initialize a new searcher instance with the config
fuzzy_searcher = FuzzyPhraseSearcher(config)

# create a list of domain phrases
domain_phrases = [
    # terms for the chair and attendants of a meeting
    "PRAESIDE",
    "PRAESENTIBUS",
    # some weekdays in Latin
    "Veneris", 
    "Mercurii",
    # some date phrase where any date in January 1725 should match
    "den .. Januarii 1725"
]

phrase_model = PhraseModel(phrases=domain_phrases)

# register the keywords with the searcher
fuzzy_searcher.index_phrase_model(phrase_model)

# take some example texts: meetings of the Dutch States General in January 1725
text1 = "ie Veucris den 5. Januaris 1725. PR&ASIDE, Den <NAME>. PRASENTIEBUS, De Heeren Jan Welderen , van Dam, Torck , met een extraordinaris Gedeputeerde uyt de Provincie van Gelderlandt. Van Maasdam , vanden Boeizelaar , Raadtpenfionaris van Hoornbeeck , met een extraordinaris Gedeputeerde uyt de Provincie van Hollandt ende Welt-Vrieslandt. Velters, Ockere , Noey; van Hoorn , met een extraordinaris Gedeputeerde uyt de Provincie van Zeelandt. Van Renswoude , van Voor{t. Van Schwartzenbergh, v<NAME>, <NAME>{elmuden. Van Iddekinge ‚ van Tamminga."

text2 = "Mercuri: den 10. Jangarii, 1725. ia PRESIDE, Den Heere an Iddekinge. PRA&SENTIBUS, De Heeren /an Welderen , van Dam, <NAME>, Torck, met een extraordinaris Gedeputeerde uyt de Provincie van Gelderland. Van Maasdam , Raadtpenfionaris van Hoorn=beeck. Velters, Ockerfe, Noey. Taats van Amerongen, van Renswoude. <NAME> , Vegilin, ’ Bentinck, van I(elmaden. Van Tamminga."


# -

# The `find_matches` method returns match objects:

# look for matches in the first example text
for match in fuzzy_searcher.find_matches(text1):
    print(match)


#
# Printing the matches directly yields the following output:

# look for matches in the first example text
for match in fuzzy_searcher.find_matches(text1):
    print(match)


# Alternatively, each match object can generate a JSON representation of the match containing all information:
#

# look for matches in the first example text
for match in fuzzy_searcher.find_matches(text1):
    print(match.json())


# Running the searcher on the second text:

# look for matches in the second example text
for match in fuzzy_searcher.find_matches(text2):
    print(match.json())


# Match objects can also generate Web Annotation representations:

# +
# look for matches in the second example text
text2_with_id = {
    "text": text2,
    "id": "urn:republic:3783_0076:page=151:para=4"
}
matches = fuzzy_searcher.find_matches(text2_with_id)

import json

print(json.dumps(matches[0].as_web_anno(), indent=2))
# -

# %reload_ext autoreload
# %autoreload 2

# +
from fuzzy_search.fuzzy_phrase_searcher import FuzzyPhraseSearcher

# init searcher with default parameter settings
fuzzy_searcher = FuzzyPhraseSearcher()
# register phrase you want to search
fuzzy_searcher.index_phrases(['Makelaars', 'Tabak', 'Koffie'])

# A text with OCR mistakes
text = 'De Makelaets sullen verkopen twee balen Tobacco en Javaansche Koffy.'
# Find all fuzzy matches
fuzzy_searcher.find_matches(text)


# +
config = {
    # these thresholds work when there are few OCR errors
    "char_match_threshold": 0.8,
    "ngram_threshold": 0.6,
    "levenshtein_threshold": 0.8,
    # Is upper/lowercase a meaningful signal?
    "ignorecase": False,
    # should matches follow word boundaries?
    "use_word_boundaries": False,
    # for phrases that have variant phrasings
    "include_variants": False,
    # avoid matching with similar but different phrases
    "filter_distractors": False,
    # matching string can be lower/shorter than prhase
    "max_length_variance": 3,
    # higher ngram size allows fewer character differences
    "ngram_size": 3,
    # fewer skips is much faster but less exhaustive
    "skip_size": 1,
}

# init searcher, overriding some defaults
fuzzy_searcher = FuzzyPhraseSearcher(config)


# +
from fuzzy_search.fuzzy_phrase_searcher import FuzzyPhraseSearcher

# init searcher with default parameter settings
fuzzy_searcher = FuzzyPhraseSearcher({'include_variants': True})
# register phrases and optional variants
phrases = [
    {'phrase': 'Makelaars'},
    {'phrase': 'Tabak', 'variants': ['Tobacco']},
    {'phrase': 'Koffie'}
]

fuzzy_searcher.index_phrase_model(phrases)

# A text with OCR mistakes
text = 'De Makelaets sullen verkopen twee balen Tobacco en Javaansche Koffy.'
# Find all fuzzy matches
fuzzy_searcher.find_matches(text)


# +
from fuzzy_search.fuzzy_phrase_model import PhraseModel

phrase_model = PhraseModel(phrases)


# -

# A text with a similar but different phrase
text = 'De Metselaers sullen verkopen twee zaken cement.'
# Find all fuzzy matches
fuzzy_searcher.find_matches(text)


# +
# registering a phrase with a distractor
phrases = [{'phrase': 'Makelaars', 'distractors': ['Metselaars']},]

fuzzy_searcher.index_phrase_model(phrases)

# A text with OCR mistakes
text = 'De Metselaers sullen verkopen twee zaken cement.'
# Find all fuzzy matches
fuzzy_searcher.find_matches(text, filter_distractors=True)


# +
fuzzy_searcher = FuzzyPhraseSearcher({'include_variants': True, 'filter_distractors': True})

phrases = [
    {'phrase': 'Makelaars', 'label': ['person_role', 'auction_broker'], 'distractors': ['Metselaars']},
    {'phrase': 'Tabak',     'label': 'auction_good',                    'variants': ['Tobacco']},
    {'phrase': 'Koffie',    'label': 'auction_good'},
]

fuzzy_searcher.index_phrase_model(phrases)

# A text with OCR mistakes
text = 'De Makelaets sullen verkopen twee balen Tobacco en Javaansche Koffy. ' + \
        'De Metselaers sullen verkopen twee zaken cement.'
# Find all fuzzy matches
matches = fuzzy_searcher.find_matches(text)
for match in matches:
    print(f"{match.offset: >4}\t{match.string: <20}\t{match.phrase.phrase_string: <20}", 
          match.label_list)

# -

text = "Auction op Prime Tobaccos. The Executors of the late JOHN BENNETT," + \
       " Tobacco Merchant,will Sell by AUCTION, at HALL'S Sale Room," + \
       " Commercial Buildings, Cork, TUESDAY the 14th October."


# +
from fuzzysearch import find_near_matches

phrases = [
    {'phrase': 'Makelaars'},
    {'phrase': 'Tabak', 'variants': ['Tobacco']},
    {'phrase': 'Koffie'}
]

# A text with OCR mistakes
text = 'De Makelaets sullen verkopen twee balen Tobacco en Javaansche Koffy.'
for phrase in phrases:
    matches = find_near_matches(phrase['phrase'], text, max_l_dist=2)
    print(matches)

# +
# Text from Delpher newspaper archive
text = """n 't Volck inSpanje en Portugacl ten tijdn van de Slag van Almauza , 
tc geven! W i|l de Intikeoingcn in deExchequer van dc 600000 Ponden, toegeftaert 
door middel van Lijfrenten te veikoopcn, door de Alakei&ers by na gecompletecrt 
zijn (die fy Wc'er mei groot propje vetkoopen) werden al itilcke lotekeinngtn te 
niet geraaeckt door etnCl»uful,die bjr dc Lijfremeo-Bil, 
dpwclcke nu ftact te pafleren,gevoegt is }"""

print(find_near_matches('Makelaars', text, max_l_dist=5))
# -

fuzzy_searcher = FuzzyPhraseSearcher()
fuzzy_searcher.index_phrases(['Makelaars'])
matches = fuzzy_searcher.find_matches(text)
for match in matches:
    print(match.json())

# +
from fuzzy_search.fuzzy_phrase_model import PhraseModel
from fuzzy_search.fuzzy_template_searcher import FuzzyTemplateSearcher, FuzzyTemplate

phrases = [
    {'phrase': 'Makelaars', 'label': ['person_role', 'auction_broker'], 'distractors': ['Metselaars']},
    {'phrase': 'Tabak',     'label': 'auction_good',                    'variants': ['Tobacco']},
    {'phrase': 'Koffie',    'label': 'auction_good'},
]
phrase_model = PhraseModel(phrases)


# +
template = ['auction_broker', 'auction_good']

fuzzy_template = FuzzyTemplate(phrase_model, template)
template_searcher = FuzzyTemplateSearcher(fuzzy_template, {'include_variants': True, 'filter_distractors': True})

# A text with OCR mistakes
text = 'De Makelaets sullen verkopen twee balen Tobacco en Javaansche Koffy. ' + \
        'De Metselaers sullen verkopen twee zaken cement.'
# Find all fuzzy matches
phrase_matches = template_searcher.find_matches(text)
template_matches = template_searcher.find_template_matches(phrase_matches)
for template_match in template_matches:
    for element_match in template_match.element_matches:
        print('Template element:', element_match['label'])
        for phrase_match in element_match['phrase_matches']:
            print(f'\t{phrase_match.phrase.phrase_string: <15}{phrase_match.string: <15}{phrase_match.offset: >4}')
# -

template = {
    'label': 'auction',
    'ordered': True,
    'type': 'group',
    'elements': [
        {
            'label': 'auction_event',
            'ordered': True,
            'type': 'group',
            'elements': [
                {'label': 'auction_broker',   'required': True,  'cardinality': 'single'},
                {'label': 'auction_location', 'required': True,  'cardinality': 'single'},
                {'label': 'auction_date',     'required': False, 'cardinality': 'single'},
            ]
        },
        {
            'label': 'auction_event',
            'ordered': False,
            'type': 'group',
            'elements': [
                {'label': 'auction_unit',     'required': False,  'cardinality': 'multi'},
                {'label': 'auction_good',     'required': True,  'cardinality': 'multi'},
            ]
        }
    ]
}


StarcoderdataJupyterScriptsDedupFiltered