<jupyter_start><jupyter_text>View the qualitative feedback:<jupyter_code>for i, f in data_df['feedback'].iteritems():
    
    print(f)
    print<jupyter_output>Ranking was quite challenging, especially given that several statements were highly similar or conceptually related.
I think it would be helpful to have some very stupid arguments thrown in so that you can have greater variability in your measurement.
Glad to see you taking an interest in social psychology, little sister!



ranking is difficult, scoring and/or selecting is easier

Ticking the 5 most relevant points to support my argument is the easiest form of feedback. Placing the different points in order is the most difficult and requires the most time as you have to read through each point multiple times in order to make comparisons and form the list in the order you want. Ranking the points on a scale of 1-10 is also relatively easy, but may not give the most accurate results as I often just end up choosing a random number in the region of important (6-10) or unimportant (which in this case I just left as 1 as I had 5 points already).





I liked the first metric (separate score[...]<jupyter_text>function [regret] = set_regret(profile,S,T)% Regret of set S for set Tif isempty(T)    regret = 0;    returnend[I,J] = find(ismember(profile,S));[~,ind] = unique(I,'first');first_indices_S = J(ind);[I,J] = find(ismember(profile,T));[~,ind] = unique(I,'first');first_indices_T = J(ind);regret = sum((first_indices_T < first_indices_S).*(1./first_indices_T));end<jupyter_code>def set_regret(profile, S, T):
    '''
    Minimax Regret Set (naive method)
    '''

    if len(T) == 0:
        regret = 0;
        return regret

    
T = set([2,3,7,11])
S = set([1,5])
len(T) == 0
profile = [list(np.random.permutation([1,2,3,4,5,6,7,8,9,10])) for i in range(10)]

# first_indices_T = np.array([[i for i,elem in enumerate(x) if x[i] in T][0] + 1 for x in profile])
# first_indices_S = np.array([[i for i,elem in enumerate(x) if x[i] in S][0] + 1 for x in profile])

# regret = np.sum((first_indices_T < first_indices_S)*(1./first_indices_T));
# regret
import itertools
k = 3
permutations1 = itertools.permutations([1,2,3,4,5,6,7,8,9,10], k)
output_regret = np.inf
permutations = None
for S in permutations1:
    
    max_reg = 0
    permutation = None
    
    permutations2 = itertools.permutations([1,2,3,4,5,6,7,8,9,10], k)
    for T in permutations2:
        
        first_indices_T = np.array([[i for i,elem in enumerate(x) if x[i] in T][0] + 1 for x in profile])
        first_indices_S = np.array([[i for i,elem in enumerate(x) if x[i] in S][0] + 1 for x in profile])

        regret = np.sum((first_indices_T < first_indices_S)*(1./first_indices_T));
        
        if regret > max_reg:
            
            max_reg = regret
            
    if max_reg < output_regret:

        output_regret = max_reg
        permutations = S
output_regret
permutations
import pickle
q1_sort_profile = data_df['question1'][data_df['question1']['qustion_type'] == 1][[1,2,3,4,5,6,7,8,9,10]].values
with open('./q1_sort_profile.p','wb') as file_:
    pickle.dump(q1_sort_profile, file_)
q2_sort_profile = data_df['question2'][data_df['question2']['qustion_type'] == 1][[1,2,3,4,5,6,7,8,9,10]].values
with open('./q2_sort_profile.p','wb') as file_:
    pickle.dump(q2_sort_profile, file_)
q3_sort_profile = data_df['question3'][data_df['question3']['qustion_type'] == 1][[1,2,3,4,5,6,7,8,9,10]].values
with open('./q3_sort_profile.p','wb') as file_:
    pickle.dump(q3_sort_profile, file_)
def rate_to_rank(question_num):
    rate_profile = data_df['question{}'.format(question_num)][data_df['question{}'.format(question_num)]['qustion_type'] == 3][[1,2,3,4,5,6,7,8,9,10]].values
    rate_list = []
    for i in range(len(rate_profile)):
        unique, counts = np.unique(rate_profile[i], return_counts=True)
    #     print "before", unique, counts
        for k in range(len(counts)):
            if counts[k]>1:
                cntr = 1
                while np.sum(rate_profile[i]==unique[k])>1:
                    indices = np.where(rate_profile[i]==unique[k])
                    idx = np.random.choice(indices[0])
                    rate_profile[i][idx] += cntr*.0000001
                    cntr+=1
        ranking = zip(list(rate_profile[i]), range(10))
    #     print(sorted(ranking))
    #     print([x[1]+1 for x in reversed(sorted(ranking))])
        rate_list.append([x[1]+1 for x in reversed(sorted(ranking))])

    rate_list = np.array(rate_list)
#     print(rate_list)
    with open('./q{}_rate_profile.p'.format(question_num),'wb') as file_:
        pickle.dump(rate_list, file_)
rate_to_rank(1)
rate_to_rank(2)
rate_to_rank(3)
for k in range(1,4):
    for typ in ['rate','sort']:
        with open('./q{}_{}_results.p'.format(k,typ), 'rb') as file_:
            results = pickle.load(file_)
        print("Question {}, Type: {}".format(k,typ))
        print results
        for i in range(len(results)):
            print(question_mapping['question{}'.format(k)]['answers']['{}'.format(results[i])])
        print('\n')
# print(question_mapping['question1']['answers']['6'])
# print(question_mapping['question1']['answers']['9'])
# print(question_mapping['question1']['answers']['8'])
# print
# print(question_mapping['question2']['answers']['3'])
# print(question_mapping['question2']['answers']['5'])
# print(question_mapping['question2']['answers']['1'])
# print
# print(question_mapping['question3']['answers']['6'])
# print(question_mapping['question3']['answers']['6'])
# print(question_mapping['question3']['answers']['10'])
def calc_spearman(question_num):
    rate_profile = data_df['question{}'.format(question_num)][data_df['question{}'.format(question_num)]['qustion_type'] == 3][[1,2,3,4,5,6,7,8,9,10]].values
    rate_list = []
    for i in range(len(rate_profile)):
        unique, counts = np.unique(rate_profile[i], return_counts=True)
    #     print "before", unique, counts
        for k in range(len(counts)):
            if counts[k]>1:
                cntr = 1
                while np.sum(rate_profile[i]==unique[k])>1:
                    indices = np.where(rate_profile[i]==unique[k])
                    idx = np.random.choice(indices[0])
                    rate_profile[i][idx] += cntr*.0000001
                    cntr+=1
        ranking = zip(list(rate_profile[i]), range(10))
    #     print(sorted(ranking))
    #     print([x[1]+1 for x in reversed(sorted(ranking))])
        rate_list.append([x[1]+1 for x in reversed(sorted(ranking))])

    rate_list = np.array(rate_list)
    srs = []
    for i in range(len(rate_list)):
        for j in range(i+1,len(rate_list)):
#             print spearmanr(rate_list[i],rate_list[j])
            srs.append(spearmanr(rate_list[i],rate_list[j]))
    return srs<jupyter_output><empty_output><jupyter_text>Rating Spearman Corr Coeff<jupyter_code>sr_list_1 = calc_spearman(1)
print(np.mean([sr_list_1[i][0] for i in range(len(sr_list_1))]))
sr_list_2 = calc_spearman(2)
print(np.mean([sr_list_2[i][0] for i in range(len(sr_list_2))]))
sr_list_3 = calc_spearman(3)
print(np.mean([sr_list_3[i][0] for i in range(len(sr_list_3))]))<jupyter_output>0.0020202020202
-0.037037037037
-0.00826446280992<jupyter_text>Ranking Spearman Corr Coeff<jupyter_code>def avg_scc_q(question_num):
    sort_profile = data_df['question{}'.format(question_num)][data_df['question{}'.format(question_num)]['qustion_type'] == 1][[1,2,3,4,5,6,7,8,9,10]].values
    srs = []
    wc = []
    kt = []
    mw = []
    for i in range(len(sort_profile)):
        for j in range(i+1,len(sort_profile)):
    #             print spearmanr(rate_list[i],rate_list[j])
            srs.append(spearmanr(sort_profile[i],sort_profile[j]))
            wc.append(wilcoxon(sort_profile[i],sort_profile[j]))
            kt.append(kendalltau(sort_profile[i],sort_profile[j]))
            mw.append(mannwhitneyu(sort_profile[i],sort_profile[j]))
    print(np.mean([srs[i][0] for i in range(len(srs))]))
    print(np.mean([wc[i][0] for i in range(len(wc))]))
    print(np.mean([kt[i][0] for i in range(len(kt))]))
    print(np.mean([mw[i][0] for i in range(len(mw))]))
avg_scc_q(1)
avg_scc_q(2)
avg_scc_q(3)<jupyter_output>/Users/soph/anaconda/envs/py27/lib/python2.7/site-packages/scipy/stats/morestats.py:2397: UserWarning: Warning: sample size too small for normal approximation.
  warnings.warn("Warning: sample size too small for normal approximation.")