<jupyter_start><jupyter_text>Sample 3 Revision Sampling===Includes all but the first 2018 revision in namespace 0.We're sampling only in 2018 (so that we have all of 2019 to observe reverts)<jupyter_code>import mwapi
import mwxml
import mwxml.utilities
import mwcli
import mwreverts
import oresapi
import mwparserfromhell
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import os
from tqdm import tqdm
import bz2
import gzip
import json
import re
import hashlib
from datetime import datetime
import nltk
import scipy.stats
import para
from itertools import groupby
from collections import Counter
git_root_dir = !git rev-parse --show-toplevel
git_root_dir = git_root_dir[0]
git_root_dir
raw_data_dir = "/export/scratch2/wiki_data"
derived_data_dir = os.path.join(git_root_dir, "data", "derived")
raw_data_dir, derived_data_dir
working_dir = os.path.join(derived_data_dir, 'revision_sample')
os.makedirs(working_dir, exist_ok=True)
working_dir
# NOTE TIMESTAMP BUG
# These incorrectly use CT, rather than UTC, as the boundaries for the data collection period
# This is completely fine, but it's probably not what most analysts would think
start_date = datetime.fromisoformat('2018-01-01')
start_timestamp = int(start_date.timestamp())
end_date = datetime.fromisoformat('2020-01-01')
end_timestamp = int(end_date.timestamp())
start_timestamp, end_timestamp
sample_start_timestamp = start_timestamp
sample_end_date = datetime.fromisoformat('2019-01-01')
sample_end_timestamp = int(sample_end_date.timestamp())<jupyter_output><empty_output><jupyter_text>Load in all revisionsWhich excludes revisions outside of 2018-2020 and only for pages in ns0 and non-redirects.<jupyter_code>start = datetime.now()
stub_history_reverts_dir = os.path.join(derived_data_dir, 'stub-history-reverts')
rev_ids_filepath = os.path.join(stub_history_reverts_dir, 'rev_ids_sorted.csv')
df = pd.read_csv(rev_ids_filepath,
                 header=None, 
                 names=['page_id', 'rev_id', 'rev_timestamp', 'is_revert_target', 'is_reverted', 'is_reverting']
)
print(f"{datetime.now() - start}")
len(df)
df.head()
# number of unique pages in this sample
page_ids_set = set(df.page_id)
len(page_ids_set)
# Read the revert info
stub_history_reverts_dir = os.path.join(derived_data_dir, 'stub-history-reverts')
revert_df_filepath = os.path.join(stub_history_reverts_dir, 'revert_df.pkl')
revert_df = pd.read_pickle(revert_df_filepath)
len(revert_df)
revert_df.head(3)
start = datetime.now()
page_df = df.groupby('page_id').agg({
    'rev_id': len,
    'is_reverted': np.sum,
    'is_reverting': np.sum
}).rename(columns={
    'rev_id': 'revision_count',
    'is_reverted': 'reverted_count',
    'is_reverting': 'revert_count'
})
print(f"{datetime.now() - start}")
len(page_df)
page_df.sample(3)
eligible_for_sampling = []
curr_page_id = -1
first_page_omitted_in_2018_count = 0
for tup in tqdm(df.itertuples(), total=len(df)):
    if tup.page_id != curr_page_id:
        curr_page_id = tup.page_id
        # can never sample the FIRST post-2018 revision
        eligible_for_sampling.append(False)
        # keep track of the number of revisions that are omitted entirely because they are the first
        # (should be <= the number of unique pages)
        if tup.rev_timestamp <= sample_end_timestamp:
            first_page_omitted_in_2018_count += 1
    else:
        is_eligible = tup.rev_timestamp <= sample_end_timestamp
        eligible_for_sampling.append(is_eligible)
first_page_omitted_in_2018_count
df['is_sample_eligible'] = eligible_for_sampling
# after filtering, only 43.9% are actually eligible...
np.sum(df.is_sample_eligible), np.sum(df.is_sample_eligible) / len(df)
df = df.reset_index().rename(columns={'index': 'full_index'})<jupyter_output><empty_output><jupyter_text>Sample version 3Includes all revisions that meet the sampling criteria.<jupyter_code>sample = df[df.is_sample_eligible]
len(sample)
# 11.9% of the sample are reverted
np.sum(sample.is_reverted), np.sum(sample.is_reverted) / len(sample)
# 9.0% of the sample are reverts
np.sum(sample.is_reverting), np.sum(sample.is_reverting) / len(sample)
# 6.1% of pages with 1+ revision are included in the sample
len(set(sample.page_id)), len(set(sample.page_id)) / len(page_ids_set)
matched_sample = df.loc[sample.index - 1]
assert len(matched_sample) == len(sample)
assert np.all(sample.page_id.reset_index(drop=True) == matched_sample.page_id.reset_index(drop=True))
sample = sample.assign(prev_rev_id=matched_sample.rev_id.tolist())
sample.head(3)
matched_sample = df.loc[sample.index + 1]
assert len(matched_sample) == len(sample)
sample['next_rev_id'] = -1
idx = np.array(sample.page_id.tolist()) == np.array(matched_sample.page_id.tolist())
print(f"{np.sum(idx)} / {len(sample)} sampled revisions have 1+ subsequent revision in 2018 or 2019.")
sample.loc[idx, 'next_rev_id'] = matched_sample[idx].rev_id.tolist()
# get the timestamp of the previous and next revisions
rev_id_timestamp_dict = {tup.rev_id: tup.rev_timestamp for tup in tqdm(df.itertuples(), total=len(df))}
sample['prev_rev_timestamp'] = sample.prev_rev_id.map(lambda prev_rev_id: rev_id_timestamp_dict[prev_rev_id])
sample['next_rev_timestamp'] = sample.next_rev_id.map(lambda next_rev_id: rev_id_timestamp_dict[next_rev_id] if next_rev_id != -1 else -1)
no_next_rev_count = np.sum(sample.next_rev_timestamp == -1)
print(f"{no_next_rev_count} sample revisions ({no_next_rev_count / len(sample)*100:.2f}%) have no next revision in the collected data range.")
sample.head()
sample_reverting_rev_ids = set(sample[sample.is_reverting==1].rev_id)
reverting_rev_id_to_reverted_ids_dict = {
    row.reverting_rev_id: row.reverted_rev_ids
    for row in tqdm(revert_df.itertuples(), total=len(revert_df))
    if row.reverting_rev_id in sample_reverting_rev_ids
}
# for reverting revisions in the sample, merge in the list of reverted_rev_ids
# using the dictionary computed in the cell above
reverted_rev_ids_list = []
for tup in sample.itertuples(): 
    if tup.is_reverting == 1:
        reverted_rev_ids = reverting_rev_id_to_reverted_ids_dict[tup.rev_id]
        reverted_rev_ids_list.append(reverted_rev_ids)
    else:
        reverted_rev_ids_list.append([])
sample['reverted_rev_ids'] = reverted_rev_ids_list
reverted_to_reverting_rev_id_dict = {}
for tup in tqdm(revert_df.itertuples(), total=len(revert_df)):
    for rev_id in tup.reverted_rev_ids:
        reverted_to_reverting_rev_id_dict[rev_id] = tup.reverting_rev_id
# grab the reverting id for reverted revisions
sample['reverting_rev_id'] = -1
sample.loc[sample.is_reverted==1,'reverting_rev_id'] = [
    reverted_to_reverting_rev_id_dict[rev_id] for rev_id in sample.loc[sample.is_reverted==1].rev_id
]
# merge in the time of the reverting revision
sample['reverting_rev_timestamp'] = -1
reverting_rev_timestamp_dict = {tup.rev_id: tup.rev_timestamp for tup in df[df.is_reverting==1].itertuples()}
sample.loc[sample.is_reverted==1,'reverting_rev_timestamp'] = [
    reverting_rev_timestamp_dict[rev_id] for rev_id in sample.loc[sample.is_reverted==1].reverting_rev_id
]
sample[sample.is_reverted==1].head()
sample = sample.drop(columns='full_index')
sample.head(1)
# save the sample
sample_filepath = os.path.join(working_dir, 'sample3_all.pkl')
sample.to_pickle(sample_filepath)
print("Finished.")
# read in the sample dataframe
revision_sample_dir = os.path.join(derived_data_dir, 'revision_sample')
sample_filepath = os.path.join(revision_sample_dir, 'sample3_all.pkl')
rev_df = pd.read_pickle(sample_filepath)
len(rev_df)
rev_df.head()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup