# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: blog
#     language: python
#     name: blog
# ---

# # Regular expressions in Python
#
# - hide: true
# - toc: true
# - badges: true
# - comments: true
# - categories: [python]

# ## Preliminaries

# ### Raw strings
#
# > Raw string notation keeps regular expressions sane. [`re` tutorial](https://docs.python.org/3/library/re.html#raw-string-notation)

# Just like the regex engine, Python uses `\` to [escape](https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals) characters in strings that otherwise have special meaning (e.g. `'` and `\`) and to create tokens with special meaning (e.g. `\n`).

'It's raining'

'It\'s raining'

print('Hello\nWorld')

# A string is processed by the interpreter before being passed on to the regex engine. This means that to search for a single literal backslash, `\`, we need to escape twice and search for the regex `\\\\`. The interpreter reads this as `\\` and passes it to the regex engine, which then reads it as `\` as desired.

s = 'a \ b'
m = re.search('\\\\', s)
print(m.group())

# A useful alternative is to use raw strings `r''`, which make the interpreter read special characters as literals, obviating the first set of escapes. Hence, it's a good idea to use raw strings in Python regex expressions.

m = re.search(r'\\', s)
print(m.group())

# ## `re` module

import re

# Overview of search methods

# +
pattern = 'a'
string = '<NAME>'

methods = [
    ('re.match (start of string)', re.match(pattern, string)),
    ('re.search (anywhere in string)', re.search(pattern, string)),
    ('re.findall (all matches)', re.findall(pattern, string)),
    ('re.finditer (all matches as iterator)', re.finditer(pattern, string))
]

for desc, result in methods:
    print('{:40} -> {}'.format(desc, result))
# -

# ### `re.findall()`
#
# Returns list of all matches if no capturing groups specified, and a list of capturing groups otherwise.

# +
data = """
 012
foo34 
     56
78bar
9
 a10b
"""

data
# -

# no capturing groups
proper_digits = '\s+\d+\s+'
re.findall(proper_digits, data, flags=re.MULTILINE)

# single capturing group
proper_digits = '(?m)\s+(\d+)\s+'
re.findall(proper_digits, data, flags=re.MULTILINE)

# mutiple capturing group
proper_digits = '\s+(\d)(\d+)?\s+'
re.findall(proper_digits, data, flags=re.MULTILINE)

# + [markdown] tags=[]
# ### `re.match()`
#
# Find pattern at the beginning of a string

# +
line = '"688293"|"777"|"2011-07-20"|"1969"|"20K to 30K"|"WA1 4"|"E01012553"|"E02002603"|"M"|"2012-01-25"|"262916"|"NatWest Bank"|"Current"|"364.22"|"9572 24jan12 , tcs bowdon , bowdon gb - pos"|"Debit"|"25.03"|"No Tag"|"No Tag"|"No Tag"|"No Merchant"|"Unknown Merchant"|"2011-07-20"|"2020-07-21 20:32:00"|"2014-07-18"|"2017-10-24"|"U"'

pattern = '"\d+"\|"(?P<user_id>\d+)"'
match = re.match(pattern, line)
print(line, end='\n\n')
print(match)
print(match.group('user_id'))
# -

match.

# +
from itertools import compress

addresses = [
    '5412 N CLARK',
    '5148 N CLARK',
    '5800 E 58TH',
    '2122 N CLARK'
    '5645 N RAVENSWOOD',
    '1060 W ADDISON',
    '4801 N BROADWAY',
    '1039 W GRANVILLE',
]

def large_house_number(address, threshold=2000):
    house_number = int(re.search('\d+', address)[0])
    return house_number > threshold

large_number = [large_house_number(x) for x in addresses]
list(compress(addresses, large_number))
# -

match.group()

# ### `re.sub()`

import string

# Stip a string of whitespace and punctuation.

s = 'String. With! Punctu@tion# and _whitespace'
re.sub(r'[\W_]', '', s)

# ### `re.escape()`

# I want to match "(other)". To match the parentheses literally, I'd have to escape them. If I don't, the regex engine interpres them as a capturing group.

m = re.search('(other)', 'some (other) word')
print(m)
m.group()

# I can escape manually.

re.search('\(other\)', 'some (other) word')

# But if I have many fields with metacharacters (e.g. variable values that contain parentheses) this is a massive pain. The solution is to just use `re.escape()`, which does all the work for me.

re.search(re.escape('(other)'), 'some (other) word')

# ## `regex` module
#
# Awesome [comparison](https://github.com/rexdwyer/Splitsville/blob/master/Splitsville.ipynb) between `re` and `regex`.

import regex as re

pattern = 'e\Z'
subject = 'apple\norange'
re.search(pattern, subject)

# ## Pandas

re.Match.group


# +
def colname_cleaner(df):
    """Convert column names to stripped lowercase with underscores."""
    df.columns = df.columns.str.lower().str.strip()
    return df

def str_cleaner(df):
    """Convert string values to stripped lowercase."""
    str_cols = df.select_dtypes('object')
    for col in str_cols:
        df[col] = df[col].str.lower().str.strip()
    return df
    
movies = (data.movies()
          .pipe(colname_cleaner)
          .pipe(str_cleaner))
movies.head(2)
# -

import re

# ### Finding a single pattern in text

# +
pattern = 'hello'
text = 'hello world it is a beautiful day.'

match = re.search(pattern, text)
match.start(), match.end(), match.group()
# -

# In Pandas

movies.title.str.extract('(love)')

# - `contains()`: Test if pattern or regex is contained within a string of a Series or Index.
# - `match()`: Determine if each string starts with a match of a regular expression.
# - `fullmatch()`: 
# - `extract()`: Extract capture groups in the regex pat as columns in a DataFrame.
# - `extractall()`: Returns all matches (not just the first match).
# - `find()`: 
# - `findall()`:
# - `replace()`:

movies.title.replace('girls', 'hello')



# Let's drop all movies by distributors with "Pictures" and "Universal" in their title.

# +
# inverted masking

names = ['Universal', 'Pictures']
pattern = '|'.join(names)
mask = movies.distributor.str.contains(pattern, na=True)
result = movies[~mask]
result.head(2)

# +
# negated regex

names = ['Universal', 'Pictures']
pattern = '\|'.join(names)
neg_pattern = f'[^{pattern}]'
neg_pattern
mask = movies.distributor.str.contains(neg_pattern, na=False)
result2 =movies[mask]
# -

neg_pattern

result == result2


def drop_card_repayments(df):
    """Drop card repayment transactions from current accounts."""
    tags = ['credit card repayment', 'credit card payment', 'credit card']
    pattern = '|'.join(tags)
    mask = df.auto_tag.str.contains(pattern) & df.account_type.eq('current')
    return df[~mask]


# + [markdown] tags=[]
# ## Sources
#
# - [Python string documentation](https://docs.python.org/3/library/string.html#string-formatting)
# - [Pyformat](https://pyformat.info)
# - [Fluent Python](https://www.oreilly.com/library/view/fluent-python/9781491946237/)
# - [Python Cookbook](https://www.oreilly.com/library/view/python-cookbook-3rd/9781449357337/)
# - [Learning Python](https://www.oreilly.com/library/view/learning-python-5th/9781449355722/)
# - [Python for Data Analysis](https://www.oreilly.com/library/view/python-for-data/9781491957653/)
# - [Python Data Science Handbook](https://www.oreilly.com/library/view/python-data-science/9781491912126/)
# -



StarcoderdataJupyterScriptsDedupFiltered