<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Preprocessing2&nbsp;&nbsp;Manual fit the word frequency count2.1&nbsp;&nbsp;Translate the sparse matrix in actual word3&nbsp;&nbsp;Automate the extraction4&nbsp;&nbsp;Extract from all columns<jupyter_code>import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
df = pd.read_excel('interviews_20180710.xls')
df<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>df.columns = [
    'name',              # Name
    'smart_contract',    # How do you handle smart contract verif & security?
    'bounties',          # Other bounties
    'who_what',          # Who are you and what are you working on?
    'tooling',           # What are the tools/libraries/frameworks you use?
    'frustrations',      # What are your biggest frustrations?
    'testing',           # How do you handle testing?
    'missing_tools',     # What tools don’t exist at the moment?
    'domain_questions',  # Other domain specific questions?
    'hardest_part',      # What was the hardest part to develop with Ethereum?
    'excited_about',     # What are you most excited about in the short term?
    'easier_expected',   # Was anything easier than expected?
    'people_talk_to',    # Who do you think we should talk to?
    'best_resources',    # What are the best educational resources?
    'questions_to_ask'   # Are there any other questions we should be asking?
]

df.fillna('', inplace = True)
topics = {'solidity', 'gas', 'protocol', 'transaction', 'event', 'network', 'documentation', 'blockchain',
          'client', 'bounty', 'deployment', 'audit', 'contract', 'verification', 'token', 'governance',
         'payment', 'consensus', 'NFT', 'compiler', 'signature', 'hash', 'testing', 'wallet', 'community',
         'decentralized'}

projects = {'Ethereum', 'Geth', 'Truffle', 'Infura', 'MetaMask', 'Parity', 'Remix', 'Mythril',
            'Casper', 'Plasma', 'Status', 'Embark', 'Swarm', 'eWASM', 'Sharding', 'State channels',
           'Gitcoin', 'Augur', 'Vyper', 'Bamboo'}<jupyter_output><empty_output><jupyter_text>Manual fit the word frequency count<jupyter_code>counter = CountVectorizer(vocabulary=topics)
smart_contract = counter.fit_transform(df['smart_contract'])
print(smart_contract)<jupyter_output>(0, 1)	1
  (0, 10)	2
  (0, 24)	1
  (2, 1)	1
  (4, 8)	1
  (4, 13)	1
  (7, 24)	1
  (8, 8)	1
  (9, 1)	1
  (16, 1)	1
  (16, 3)	2
  (16, 5)	2
  (20, 1)	1
  (20, 8)	1
  (22, 20)	2
  (22, 21)	1
  (33, 2)	1
  (33, 8)	3
  (33, 9)	1
  (33, 14)	1
  (33, 24)	2
  (35, 21)	1
  (37, 5)	1
  (37, 20)	1
  (38, 16)	1
  :	:
  (53, 3)	2
  (53, 24)	1
  (55, 2)	2
  (61, 10)	1
  (61, 11)	1
  (63, 1)	1
  (63, 8)	1
  (66, 24)	1
  (73, 1)	1
  (73, 20)	1
  (73, 24)	2
  (75, 6)	1
  (75, 20)	1
  (77, 1)	1
  (77, 24)	1
  (80, 1)	1
  (80, 3)	1
  (85, 1)	1
  (85, 2)	1
  (85, 5)	1
  (85, 13)	1
  (86, 1)	1
  (103, 8)	2
  (103, 18)	2
  (103, 24)	1<jupyter_text>Translate the sparse matrix in actual word<jupyter_code>for row in smart_contract[0:3]:
    d = dict(zip(counter.get_feature_names(),
                 row.toarray().ravel()
            ))
    print(d)
df['smart_contract'][0]
# Notice that audits (plural) is not taken into account
# We need a proper lemmatizer
# Cleanup
del counter
del smart_contract<jupyter_output><empty_output><jupyter_text>Automate the extraction<jupyter_code>count_topics = CountVectorizer(vocabulary=topics)
count_projects = CountVectorizer(vocabulary=projects)
def dictFreq(df, field):
    counts_t = count_topics.transform(df[field])
    counts_p = count_projects.transform(df[field])
    
    result = pd.Series(
        {
            'topics': dict(zip(count_topics.get_feature_names(), counts_t[i].toarray().ravel())),
            'projects': dict(zip(count_projects.get_feature_names(), counts_p[i].toarray().ravel()))
        } for i in range(df.index.__len__())
    )
    return result
dictFreq(df, 'smart_contract')<jupyter_output><empty_output><jupyter_text>Extract from all columns<jupyter_code>cols = [
    'smart_contract',    # How do you handle smart contract verif & security?
    'bounties',          # Other bounties
    'who_what',          # Who are you and what are you working on?
    'tooling',           # What are the tools/libraries/frameworks you use?
    'frustrations',      # What are your biggest frustrations?
    'testing',           # How do you handle testing?
    'missing_tools',     # What tools don’t exist at the moment?
    'domain_questions',  # Other domain specific questions?
    'hardest_part',      # What was the hardest part to develop with Ethereum?
    'excited_about',     # What are you most excited about in the short term?
    'easier_expected',   # Was anything easier than expected?
    'people_talk_to',    # Who do you think we should talk to?
    'best_resources',    # What are the best educational resources?
    'questions_to_ask'   # Are there any other questions we should be asking?
]
for col in cols:
    df[col + '_wordcount'] = dictFreq(df, col)
df
df.to_json('interviews_wordcount.json', orient = 'index')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup