# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] run_control={"frozen": false, "read_only": false}
# # Identify user's affiliation with IBM
# Author: <NAME>  
# Last modified: 2017-05-28

# + [markdown] run_control={"frozen": false, "read_only": false}
# # Roadmap
# 1. Tag native tweets for keyword 'ibm' in 'text' field (multiprocessing)
# 2. Aggreate number of tweets and number of IBM tweets for each user
# 3. Identify affiliation based on the proportion of IBM tweets

# + [markdown] run_control={"frozen": false, "read_only": false}
# # Steps

# + run_control={"frozen": false, "read_only": false}
"""
Initialization
"""

'''
Data analysis modules: pandas, matplotlib, numpy, and etc
'''
# %matplotlib inline
# %config InlineBackend.figure_format = 'retina' # render double resolution plot output for Retina screens 
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

'''
Standard modules, MongoDB modules
''' 
import os, sys, json, datetime, pickle, multiprocessing, logging
from pprint import pprint

import pymongo
from pymongo import IndexModel, ASCENDING, DESCENDING

'''
Custom tool modules
'''
import mongodb  # module for setting up connection with (local) MongoDB database
import multiprocessing_workers  # module for splitting workloads between processes
import utilities  # module for various custom utility functions
from config import * # import all global configuration variables

'''
Misc
'''
NB_NAME = '20170504-user_affiliation_2'

# + [markdown] heading_collapsed=true run_control={"frozen": false, "read_only": false}
# ## Tag native tweets for keyword 'ibm' in 'text' field (multiprocessing)

# + hidden=true run_control={"frozen": true, "read_only": true}
# %%time
# """
# Use multiprocessing to tag the 'text' field of native tweets for keyword 'ibm'
# Worker function 'worker_tag_kws_in_tw' is wrapped in multiprocessing_workers.py
# """
# if 0 == 1:
#     multiprocessing.log_to_stderr(logging.DEBUG)
#     
#     procedure_name = 'tag_{}_text_ibm'.format(TW_NT_COL)
#     kw_lst = ['ibm']
#     process_n = multiprocessing.cpu_count() - 1 # set processes number to CPU numbers minus 1
#     suffix = 'json'
#     inter_files = utilities.gen_inter_filenames_list(NB_NAME, procedure_name, process_n, suffix)
#     
#     jobs = []
#     for batch_i in range(process_n):
#         p = multiprocessing.Process(target=multiprocessing_workers.worker_tag_kws_in_tw,
#                                     args=(DB_NAME, TW_NT_COL, batch_i, process_n, inter_files[batch_i], kw_lst),
#                                     name='Process-{}/{}'.format(batch_i, process_n))
#         jobs.append(p)
#     
#     for job in jobs:
#         job.start()
#         
#     for job in jobs:
#         job.join()

# + hidden=true run_control={"frozen": true, "read_only": true}
# %%time
# """
# Build a new collection for keyword 'ibm' tag on 'text' field of native tweets
# Register in config:
#     TW_NT_TXT_IBM_TAG_COL
# """
# if 0 == 1:
#     procedure_name = 'tag_{}_text_ibm'.format(TW_NT_COL)
#     kw_lst = ['ibm']
#     process_n = multiprocessing.cpu_count() - 1 # set processes number to CPU numbers minus 1
#     suffix = 'json'
#     inter_files = utilities.gen_inter_filenames_list(NB_NAME, procedure_name, process_n, suffix)
#
#     tw_nt_txt_ibm_tag_col = mongodb.initialize(db_name=DB_NAME, collection_name=TW_NT_TXT_IBM_TAG_COL)
#     for inter_file in inter_files:
#         print('Reading {}...'.format(inter_file), end=' ')
#         lines = open(inter_file).readlines()
#         parsed_jsons = [json.loads(line) for line in lines]
#         print('Importing into {}.{}...'.format(DB_NAME, TW_NT_TXT_IBM_TAG_COL))
#         tw_nt_txt_ibm_tag_col.insert_many(parsed_jsons)
#     print('Done')

# + hidden=true run_control={"frozen": false, "read_only": false}
"""
Build compound index 'user_id'-'id' on new colleciton
"""
if 0 == 1:
    tw_nt_txt_ibm_tag_col = mongodb.initialize(db_name=DB_NAME, collection_name=TW_NT_TXT_IBM_TAG_COL)
    index_lst = [('user_id', pymongo.ASCENDING),
                 ('id', pymongo.ASCENDING)]
    print('Building compond index {}...'.format(index_lst))
    tw_nt_txt_ibm_tag_col.create_index(keys=index_lst)
    print('Done')

# + hidden=true run_control={"frozen": false, "read_only": false}
"""
Check how many native tweets are tagged as having 'ibm' keyword in 'text' field
"""
if 0 == 1:
    tw_nt_txt_ibm_tag_col = mongodb.initialize(db_name=DB_NAME, collection_name=TW_NT_TXT_IBM_TAG_COL)

    tw_nt_num = tw_nt_txt_ibm_tag_col.count()

    tw_nt_txt_ibm_num = tw_nt_txt_ibm_tag_col.count(filter={'X_0': {'$eq': True}})
    print('Native tweets tagged as having "ibm" keyword: {} ({:.2%} out of total) '.format(tw_nt_txt_ibm_num, tw_nt_txt_ibm_num / tw_nt_num))

# + [markdown] run_control={"frozen": false, "read_only": false}
# ## Aggreate number of tweets and number of IBM tweets for each user

# + [markdown] run_control={"frozen": false, "read_only": false}
# Aggregate information into list of dictionaries. Example:  
# ```{'user_id': id of the user,
#    'tweets_num': number of tweets (native) authored by the user,
#    'ibm_tweets_num': number of IBM tweets (native) authored by the user}```

# + run_control={"frozen": false, "read_only": false}
"""
Aggreate 'tweets_num' and 'ibm_tweets_num' for each user and write to tmp pickle
"""

user_tw_ibmtw_num_lst_pkl = os.path.join(TMP_DIR, '{}-{}'.format(NB_NAME, 'user_tw_ibmtw_num.lst.pkl'))

if 0 == 1:
    print("Building pickle from database...")
    data_lst = []
    
    group_dict = {'$group': {'_id': '$user_id',                            
                             'tweets_num': {'$sum': 1},
                             'ibm_tweets_num': {'$sum': {'$cond': ['$X_0', 1, 0]}}}} # if 'X_0' is true, add count for IBM tweets
    
    project_dict = {'$project': {'_id': 0,
                                'user_id': '$_id',
                                'tweets_num': 1,
                                'ibm_tweets_num': 1}}
    
    ppl_lst = [group_dict, project_dict]
    print('Aggreating on collection "{}"'.format(TW_NT_TXT_IBM_TAG_COL))
    tw_nt_ibm_tag_col = mongodb.initialize(db_name=DB_NAME, collection_name=TW_NT_TXT_IBM_TAG_COL)
    cursor = tw_nt_ibm_tag_col.aggregate(pipeline=ppl_lst,
                                         allowDiskUse=True) # Exceeded memory limit for $group, but didn't allow external sort. Pass allowDiskUse:true to opt in.
    
    for doc in cursor:
        data_lst.append(doc)
    
    with open(user_tw_ibmtw_num_lst_pkl, 'wb') as f:
        pickle.dump(data_lst, f)
    print('Done')

# + [markdown] run_control={"frozen": false, "read_only": false}
# ## Identify affiliation based on the proportion of IBM tweets

# + [markdown] run_control={"frozen": false, "read_only": false}
# ### Plot the proportion of IBM tweets against number of tweets

# + run_control={"frozen": false, "read_only": false}
"""
Load pickled data
"""
if 1 == 1:
    data_lst = []
    with open(user_tw_ibmtw_num_lst_pkl, 'rb') as f:
        data_lst = pickle.load(f)
        
    df_user_nt = pd.DataFrame(data=data_lst,
                              columns=['user_id', 'tweets_num', 'ibm_tweets_num'], # explicitly pass in names of columns
                              dtype=int)

    # compute the proportion of IBM tweets
    df_user_nt['ibm_tweets_prop'] = df_user_nt['ibm_tweets_num'] / df_user_nt['tweets_num']

# + run_control={"frozen": false, "read_only": false}
pcls = np.arange(start=0.1, stop=1, step=0.1)
df_user_nt.describe()

# + run_control={"frozen": false, "read_only": false}
df_user_nt[df_user_nt['ibm_tweets_num'] >= 1].describe()

# + run_control={"frozen": false, "read_only": false}
"""
Plot the proportion of IBM tweets against number of tweets
"""

ibm_tweets_prop_fig = os.path.join(FIG_DIR, 'ibm_tweets_prop.png')

if 1 == 1:
    '''
    Prepare data
    '''
    # set minimum tweets_num so ibm_tweets_prop would be meaningful
    df_tmp = df_user_nt[df_user_nt['tweets_num'] >= 3] 
    
    X = df_tmp['tweets_num']
    Y = df_tmp['ibm_tweets_prop']
    
    
    '''
    Plot
    '''
    fig, ax = plt.subplots(figsize=(12, 8))
    title_fontdict = {'weight': 'bold', 'size': 'x-large'}
    # ax.set_title('Proportion of IBM tweets', fontdict=title_fontdict)
    
    label_fontdict = {'size': 'large'}
    ax.set_xlabel('Number of tweets', fontdict=label_fontdict)
    ax.set_ylabel('Proportion of IBM tweets', fontdict=label_fontdict)

#     plt.axvline(x=6, color='g')
#     plt.axhline(y=0.0475, color='g')
#     plt.axhline(y=0.2, color='g')
    
    plt.xscale('log')
#     plt.yscale('log')
    plt.plot(X, Y ,'b.')

    '''
    Save figure
    '''
    plt.savefig(ibm_tweets_prop_fig, dpi=200)

# + [markdown] run_control={"frozen": false, "read_only": false}
# ### Identifying affiliation with different combinations of (ibm_tweets_num, ibm_tweets_prop)

# + [markdown] run_control={"frozen": false, "read_only": false}
# We are looking for users:
# 1. ~~with _reasonable_ 'tweets_num' value. So the 'ibm_tweets_prop' value would be meaningful.~~ ??
# 2. with _high_ 'ibm_tweets_prop' value, which means the user frequently mentions keyword 'ibm' in her tweets

# + run_control={"frozen": false, "read_only": false}
"""
Load in IBM/non-IBM users identified by 'descritption' field.
Check the differences of 'ibm_tweets_num' and 'ibm_tweets_prop' between them
"""
if 1 == 1:
    '''
    Load pickled data
    '''
    user_nt_ibm_desc_ids_lst = []
    with open(USER_NT_IBM_DESC_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_desc_ids_lst = pickle.load(f)
    print('List length: "{}"'.format(len(user_nt_ibm_desc_ids_lst)))
    
    user_nt_nonibm_desc_ids_lst = []
    with open(USER_NT_NONIBM_DESC_IDS_LST_PKL, 'rb') as f:
        user_nt_nonibm_desc_ids_lst = pickle.load(f)
    print('List length: "{}"'.format(len(user_nt_nonibm_desc_ids_lst)))
    
    '''
    Subset users
    '''
    user_nt_ibm_desc_cond = df_user_nt['user_id'].isin(user_nt_ibm_desc_ids_lst)
    user_nt_nonibm_desc_cond = df_user_nt['user_id'].isin(user_nt_nonibm_desc_ids_lst)
    
    df_user_nt_ibm_desc = df_user_nt[user_nt_ibm_desc_cond]
    df_user_nt_nonibm_desc = df_user_nt[user_nt_nonibm_desc_cond]

# + [markdown] run_control={"frozen": false, "read_only": false}
# Check distribution the of 'tweets_num', 'ibm_tweets_num', and 'ibm_tweets_prop'

# + run_control={"frozen": false, "read_only": false}
"""
IBM users identified based on 'description' field
"""
df_user_nt_ibm_desc[['tweets_num', 'ibm_tweets_num', 'ibm_tweets_prop']].describe()
# np.arange(start=0.1, stop=1, step=0.1)

# + run_control={"frozen": false, "read_only": false}
"""
non-IBM users identified based on 'description' field
"""
df_user_nt_nonibm_desc[['tweets_num', 'ibm_tweets_num', 'ibm_tweets_prop']].describe()
# np.arange(start=0.1, stop=1, step=0.1)

# + [markdown] run_control={"frozen": false, "read_only": false}
# We can see that, for IBM/non-IBM users identified based on their 'description' field
# 1. IBM users have slightly higher 'tweets_num'.
# 2. IBM users have significantly higher 'ibm_tweets_num' and 'ibm_tweets_prop'
#
# This means 'ibm_tweets_num'/'ibm_tweets_prop' can be used to identify IBM users as a supplement of the 'description' field based method

# + [markdown] run_control={"frozen": false, "read_only": false}
# ### Different conditions

# + [markdown] run_control={"frozen": false, "read_only": false}
# We set three identification conditions:  
# 1. Cond1: mimimal requirements ('ibm_tweets_num' >= 1)
# 2. Cond2: IBM tweets prop >= first quartile ('ibm_tweets_prop' >= 0.063002)
# 3. Cond3: IBM tweets prop >= median ('ibm_tweets_prop' >= 0.562500)
#
# **Note**: To make 'ibm_tweets_prop' value meaningful and not affected by large amount of users with few tweets, we set 'tweets_num' >= 3, which is the median of 'tweets_num' of IBM users from 'description' field based method.

# + run_control={"frozen": false, "read_only": false}
df_user_nt[df_user_nt['tweets_num'] >= 3].describe(np.arange(0.1, 1, 0.1))

# + run_control={"frozen": false, "read_only": false}
"""
Users with >=3 tweets and >=1 IBM tweets
"""
user_nt_ibm_tw_prop_cond1 = (df_user_nt['ibm_tweets_num'] >= 1) & (df_user_nt['tweets_num'] >= 3)
df_user_nt_ibm_tw_prop_1 = df_user_nt[user_nt_ibm_tw_prop_cond1]
df_user_nt_ibm_tw_prop_1.describe()

# + run_control={"frozen": true, "read_only": true}
# """
# Make pickle
# Register in config:
#     USER_NT_IBM_TW_PROP_1_IDS_LST_PKL
#     USER_NT_NONIBM_TW_PROP_1_IDS_LST_PKL
# """
# if 1 == 1:
#     ibm_user_ids_lst = list(df_user_nt_ibm_tw_prop_1['user_id'])
#     
#     print('Dumping to pickle: {}...'.format(USER_NT_IBM_TW_PROP_1_IDS_LST_PKL))
#     with open(USER_NT_IBM_TW_PROP_1_IDS_LST_PKL, 'wb') as f:
#         pickle.dump(ibm_user_ids_lst, f)
#         
#     df_rest_user = df_user_nt[~ user_nt_ibm_tw_prop_cond1]
#     rest_user_ids_lst = list(df_rest_user['user_id'])
#     print('Dumping to pickle: {}...'.format(USER_NT_NONIBM_TW_PROP_1_IDS_LST_PKL))
#     with open(USER_NT_NONIBM_TW_PROP_1_IDS_LST_PKL, 'wb') as f:
#         pickle.dump(rest_user_ids_lst, f)
#
#     print('Done')

# + run_control={"frozen": false, "read_only": false}
"""
Users with >=3 tweets and IBM tweets proportion >= first quartile
"""
user_nt_ibm_tw_prop_cond2 = (df_user_nt['ibm_tweets_prop'] >= 0.063002) & (df_user_nt['tweets_num'] >= 3)
# user_nt_ibm_tw_prop_cond2 = (df_user_nt['ibm_tweets_prop'] > 0.063002) & (df_user_nt['tweets_num'] > 1)
df_user_nt_ibm_tw_prop_2 = df_user_nt[user_nt_ibm_tw_prop_cond2]
df_user_nt_ibm_tw_prop_2.describe()

# + run_control={"frozen": true, "read_only": true}
# """
# Make pickle
# Register in config:
#     USER_NT_IBM_TW_PROP_2_IDS_LST_PKL
#     USER_NT_NONIBM_TW_PROP_2_IDS_LST_PKL
# """
# if 0 == 1:
#     ibm_user_ids_lst = list(df_user_nt_ibm_tw_prop_2['user_id'])
#     
#     print('Dumping to pickle: {}...'.format(USER_NT_IBM_TW_PROP_2_IDS_LST_PKL))
#     with open(USER_NT_IBM_TW_PROP_2_IDS_LST_PKL, 'wb') as f:
#         pickle.dump(ibm_user_ids_lst, f)
#         
#     df_rest_user = df_user_nt[~ user_nt_ibm_tw_prop_cond2]
#     rest_user_ids_lst = list(df_rest_user['user_id'])
#     print('Dumping to pickle: {}...'.format(USER_NT_NONIBM_TW_PROP_2_IDS_LST_PKL))
#     with open(USER_NT_NONIBM_TW_PROP_2_IDS_LST_PKL, 'wb') as f:
#         pickle.dump(rest_user_ids_lst, f)
#
#     print('Done')

# + run_control={"frozen": false, "read_only": false}
"""
Users with >=3 tweets and IBM tweets proportion >= median
"""
user_nt_ibm_tw_prop_cond3 = (df_user_nt['ibm_tweets_prop'] >= 0.562500) & (df_user_nt['tweets_num'] >= 3)
df_user_nt_ibm_tw_prop_3 = df_user_nt[user_nt_ibm_tw_prop_cond3]
df_user_nt_ibm_tw_prop_3.describe()

# + run_control={"frozen": false, "read_only": false}
"""
Make pickle
Register in config:
    USER_NT_IBM_TW_PROP_3_IDS_LST_PKL
    USER_NT_NONIBM_TW_PROP_3_IDS_LST_PKL
"""
if 0 == 1:
    ibm_user_ids_lst = list(df_user_nt_ibm_tw_prop_3['user_id'])
    
    print('Dumping to pickle: {}...'.format(USER_NT_IBM_TW_PROP_3_IDS_LST_PKL))
    with open(USER_NT_IBM_TW_PROP_3_IDS_LST_PKL, 'wb') as f:
        pickle.dump(ibm_user_ids_lst, f)
        
    df_rest_user = df_user_nt[~ user_nt_ibm_tw_prop_cond3]
    rest_user_ids_lst = list(df_rest_user['user_id'])
    print('Dumping to pickle: {}...'.format(USER_NT_NONIBM_TW_PROP_3_IDS_LST_PKL))
    with open(USER_NT_NONIBM_TW_PROP_3_IDS_LST_PKL, 'wb') as f:
        pickle.dump(rest_user_ids_lst, f)

    print('Done')

# + [markdown] run_control={"frozen": false, "read_only": false}
# ### Compare with 'description' field based method

# + run_control={"frozen": false, "read_only": false}
"""
Check the overlap with 'description' field based method
"""
if 1 == 1:
    user_nt_ibm_desc_ids_lst = []
    with open(USER_NT_IBM_DESC_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_desc_ids_lst = pickle.load(f)

    user_nt_ibm_tw_prop_1_ids_lst = []
    with open(USER_NT_IBM_TW_PROP_1_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_tw_prop_1_ids_lst = pickle.load(f)
    
    user_nt_ibm_tw_prop_2_ids_lst = []
    with open(USER_NT_IBM_TW_PROP_2_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_tw_prop_2_ids_lst = pickle.load(f) 

    user_nt_ibm_tw_prop_3_ids_lst = []
    with open(USER_NT_IBM_TW_PROP_3_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_tw_prop_3_ids_lst = pickle.load(f) 
    
    user_nt_ibm_desc_ids_set = set(user_nt_ibm_desc_ids_lst)
    user_nt_ibm_tw_prop_1_ids_set = set(user_nt_ibm_tw_prop_1_ids_lst)
    user_nt_ibm_tw_prop_2_ids_set = set(user_nt_ibm_tw_prop_2_ids_lst)
    user_nt_ibm_tw_prop_3_ids_set = set(user_nt_ibm_tw_prop_3_ids_lst)

    common_num_1 = len(user_nt_ibm_desc_ids_set.intersection(user_nt_ibm_tw_prop_1_ids_set))
    common_num_2 = len(user_nt_ibm_desc_ids_set.intersection(user_nt_ibm_tw_prop_2_ids_set))
    common_num_3 = len(user_nt_ibm_desc_ids_set.intersection(user_nt_ibm_tw_prop_3_ids_set))
    
    print('IBM users in M1: {}'.format(len(user_nt_ibm_desc_ids_lst)))
    print('IBM users in M2_1: {}; common IBM users with M1: {}'.format(len(user_nt_ibm_tw_prop_1_ids_lst), common_num_1))
    print('IBM users in M2_2: {}; common IBM users with M1: {}'.format(len(user_nt_ibm_tw_prop_2_ids_lst), common_num_2))
    print('IBM users in M2_3: {}; common IBM users with M1: {}'.format(len(user_nt_ibm_tw_prop_3_ids_lst), common_num_3))

# + run_control={"frozen": false, "read_only": false}
"""
Plot different choices of IBM tweets proportion affect the common users identified with 'description' field based method
"""
if 0 == 1:
    '''
    Prepare data
    '''
    ibm_tweets_prop_lst = np.arange(0.005, 0.995, 0.0025)
    
    cmm_ibm_user_pct_lst = []
    total_users_pct_lst = []
    
    data_lst = []
    with open(user_tw_ibmtw_num_lst_pkl, 'rb') as f:
        data_lst = pickle.load(f)
        
    df_user_nt = pd.DataFrame(data=data_lst,
                              columns=['user_id', 'tweets_num', 'ibm_tweets_num'], # explicitly pass in names of columns
                              dtype=int)

    # compute the proportion of IBM tweets
    df_user_nt['ibm_tweets_prop'] = df_user_nt['ibm_tweets_num'] / df_user_nt['tweets_num']
    
    user_nt_ibm_desc_ids_lst = []
    with open(USER_NT_IBM_DESC_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_desc_ids_lst = pickle.load(f)

    user_nt_ibm_tw_prop_1_ids_lst = []
    with open(USER_NT_IBM_TW_PROP_1_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_tw_prop_1_ids_lst = pickle.load(f)

    
    for ibm_tweets_prop in ibm_tweets_prop_lst:
        
        user_nt_ibm_tw_num = len(user_nt_ibm_tw_prop_1_ids_lst)
        
        select_cond = df_user_nt['ibm_tweets_prop'] > ibm_tweets_prop
        df_tmp = df_user_nt[select_cond]
        
        tmp_user_ids_set = set(df_tmp['user_id'])
        cmm_ibm_user_pct = len(set(user_nt_ibm_desc_ids_lst).intersection(tmp_user_ids_set)) / len(user_nt_ibm_desc_ids_lst)
        cmm_ibm_user_pct_lst.append(cmm_ibm_user_pct)
        
        row_n = df_tmp.shape[0] # number of users identified
        total_users_pct = row_n / user_nt_ibm_tw_num
        total_users_pct_lst.append(total_users_pct)
    
    '''
    Plot
    '''
    fig, ax1 = plt.subplots()
    fig.set_size_inches((12,8))
    ax1.set_title('IBM tweets proportions')
    
    X = ibm_tweets_prop_lst
    Y1 = cmm_ibm_user_pct_lst
    ax1.plot(X, Y1, 'b-')
    ax1.set_ylabel("% of IBM users in common with method1", color='blue')
    ax1.set_ylim(0, 1)
    ax1.set_xticks(np.arange(0, 1.1, 0.1))
    X_vals = ax1.get_xticks()
    ax1.set_xticklabels(['{:.1%}'.format(X_val) for X_val in X_vals])
    ax1.grid()
    
    Y1_vals = ax1.get_yticks()
    ax1.set_yticklabels(['{:.1%}'.format(Y1_val) for Y1_val in Y1_vals])
    

    ax2 = ax1.twinx()
    Y2 = total_users_pct_lst
    ax2.plot(X, Y2, 'g-')
    ax2.set_ylabel("% of IBM tweets users", color='g')
#     ax2.set_ylim(0, 0.1)
    Y2_vals = ax2.get_yticks()
    ax2.set_yticklabels(['{:.1%}'.format(Y2_val) for Y2_val in Y2_vals])
    
    '''
    Save figure
    '''

# + [markdown] heading_collapsed=true run_control={"frozen": false, "read_only": false}
# ## Check accounts @Natasha_D_G and @jameskobielus

# + [markdown] hidden=true run_control={"frozen": false, "read_only": false}
# While by using method_1, we can identify all users have keyword 'ibm' in their description field as affiliated with IBM, method_1 is limited that not all users would explicitly express their affiliation with IBM in their description field.  
# So, the users identified by method_1 should be a subset of actual users affiliated with IBM. In other words, there are more users affiliated with IBM.
# The idea we use here is that, if a person frequently mention keyword 'ibm' in tweets, he or she is highly likely affiliated with IBM. We use the proportion of tweets with keyword 'ibm' to identify these users. 

# + hidden=true run_control={"frozen": false, "read_only": false}
if 1 == 1:
    screen_name_1 = 'Natasha_D_G'
    screen_name_2 = 'jameskobielus'
    
    id_1 = 39413322
    id_2 = 14072398
            
    user_nt_ibm_tw_prop_1_ids_lst = []
    with open(USER_NT_IBM_TW_PROP_1_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_tw_prop_1_ids_lst = pickle.load(f)
    
    user_nt_ibm_tw_prop_2_ids_lst = []
    with open(USER_NT_IBM_TW_PROP_2_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_tw_prop_2_ids_lst = pickle.load(f) 

    user_nt_ibm_tw_prop_3_ids_lst = []
    with open(USER_NT_IBM_TW_PROP_3_IDS_LST_PKL, 'rb') as f:
        user_nt_ibm_tw_prop_3_ids_lst = pickle.load(f) 

    
    print('User {} exists in M2_1 IBM-user set? {}'.format(screen_name_1, id_1 in set(user_nt_ibm_tw_prop_1_ids_lst)))
    print('User {} exists in M2_2 IBM-user set? {}'.format(screen_name_1, id_1 in set(user_nt_ibm_tw_prop_2_ids_lst)))
    print('User {} exists in M2_3 IBM-user set? {}'.format(screen_name_1, id_1 in set(user_nt_ibm_tw_prop_3_ids_lst)))
    
    print('User {} exists in M2_1 IBM-user set? {}'.format(screen_name_2, id_2 in set(user_nt_ibm_tw_prop_1_ids_lst)))
    print('User {} exists in M2_2 IBM-user set? {}'.format(screen_name_2, id_2 in set(user_nt_ibm_tw_prop_2_ids_lst)))
    print('User {} exists in M2_3 IBM-user set? {}'.format(screen_name_2, id_2 in set(user_nt_ibm_tw_prop_3_ids_lst)))

# + [markdown] hidden=true run_control={"frozen": false, "read_only": false}
# Since @jameskobielus (<NAME>) is not identified as IBM-user by our method_2, we check the tweets we captured for him.

# + hidden=true run_control={"frozen": false, "read_only": false}
"""
How many tweets we have in our database authored by @jameskobielus?
"""
tw_nt_txt_ibm_tag_col = mongodb.initialize(db_name=DB_NAME, collection_name=TW_NT_TXT_IBM_TAG_COL)
jameskobielus_nt_num = tw_nt_txt_ibm_tag_col.count(filter={'user_id': id_2})
jameskobielus_ibm_nt_num = tw_nt_txt_ibm_tag_col.count(filter={'user_id': id_2, 'X_0': True})
print('{} native tweets we have for user {}'.format(jameskobielus_nt_num, screen_name_2))
print('{} native IBM tweets we have for user {}'.format(jameskobielus_ibm_nt_num, screen_name_2))

# + hidden=true run_control={"frozen": false, "read_only": false}
"""
Check the content of these tweets
"""
cursor = updated_col.find(filter={'user.id': id_2}, projection={'_id':0, 'id': 1, 'text': 1})
count = 0
for doc in cursor:
    count += 1
    tweet_id = doc['id']
    tweet_text = doc['text']
    print('{}. ({})'.format(count, tweet_id))
    print(tweet_text)

# + [markdown] hidden=true run_control={"frozen": false, "read_only": false}
# After manully checking all his tweets, 9 tweets were found including 'ibm' keyword.

# + [markdown] run_control={"frozen": false, "read_only": false}
# # Notes

StarcoderdataJupyterScriptsDedupFiltered