<jupyter_start><jupyter_text>User Mapping<jupyter_code>orig_members = pd.read_csv('kad_mrgcn_public/data/ironmarch/raw_files/orig_members.csv')
sorted_member_id = orig_members['member_id'].sort_values()
member_id_map ={}
start_id = 1
for id in sorted_member_id:
  member_id_map[id] = start_id
  start_id = start_id + 1<jupyter_output><empty_output><jupyter_text>Posts<jupyter_code>orig_posts = pd.read_csv('kad_mrgcn_public/data/ironmarch/raw_files/orig_posts.csv')
orig_posts.drop(orig_posts.columns[[0,1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]], axis =1, inplace=True)<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def preprocess_text(text):


    soup = BeautifulSoup(text, "lxml")
    text = soup.body.text

    # Get a list of sentences, removes "\n"
    lst = sent_tokenize(text)
    text = " ".join(lst)

    text = text.replace("\n", "")

    # Preprocess using NLTK
    text = text.lower()

    # Remove Punctuation
    text_p = "".join([char for char in text if char not in string.punctuation])

    # Get a List of words 
    words = word_tokenize(text_p)

    words = " ".join(words)

    # Remove stopwords
    stop_words = stopwords.words('english')
    filtered_words = [word for word in words if word not in stop_words]

    ret = " ".join(filtered_words)

    return ret
def getBERT(post):

# Get the embedding
  inputs = tokenizer(post, return_tensors="pt")
  outputs = model(**inputs)
  last_hidden_states = outputs.last_hidden_state

# Take the average
  n_tensor = last_hidden_states.shape[1]
  vector = (last_hidden_states.sum(axis=1)/n_tensor)

  return vector
def get_word_split(text1):
  l_total = []
  l_partial = []
  if len(text1.split())//150 >0:
    n = len(text1.split())//150 + 1
  else: 
    n = 1
  # print(n)
  for w in range(n):
    if w == 0:
      l_partial = text1.split()[:200]
      l_total.append(" ".join(l_partial))
    else:
      l_partial = text1.split()[w*150:w*150 + 200]
      l_total.append(" ".join(l_partial))
  return l_total<jupyter_output><empty_output><jupyter_text>Variables<jupyter_code>embedding = np.zeros((orig_members.shape[0], 768))
count = np.zeros(orig_members.shape[0])
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')<jupyter_output><empty_output><jupyter_text>Iterations<jupyter_code>for i, row in orig_posts.iterrows():
  row = orig_posts.iloc[i]
  author_id = row['author_id']
  if author_id not in member_id_map:
          continue
  id = member_id_map[author_id]
  doc = row['post']

  clean_doc = preprocess_text(doc)
  clean_chunks = get_word_split(clean_doc)

  
  embed_doc = np.zeros((1, 768))

  for chunkNo, chunk in enumerate(clean_chunks):
    embed_chunk = getBERT(chunk)
    embed_chunk = embed_chunk.detach().numpy()
    embed_doc = (embed_doc*chunkNo + embed_chunk)/(chunkNo+1)


  embedding[id-1] = (count[id-1]*embedding[id-1] + embed_doc)/(count[id-1]+1)
  count[id-1] = count[id-1]+1

  if i%5000==0:
    num = int(i/5000)
    np.save("/content/drive/MyDrive/text gcn/node_feature/ironmarch/processed/posts_embedding_chunk_v{}".format(num+1))
    np.save("/content/drive/MyDrive/text gcn/node_feature/ironmarch/processed/posts_embedding_chunk_count_v{}".format(i+1), count)


np.save("/content/drive/MyDrive/text gcn/node_feature/ironmarch/processed/posts_embedding_chunk_final", embedding)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup