confidence weighted accuracy assuming the scores are probabilities and using .5 as treshold
1244
public int[] cwaArray() {
    int[] arr = new int[numSamples()];
    for (int recall = 1; recall <= numSamples(); recall++) {
      arr[recall - 1] = logPrecision(recall);
    }
    return arr;
  }
1085589
private double computeConfidence(int support, int samples) {
    final double z = NormalDistribution.standardNormalQuantile(alpha);
    final double eprob = support / (double) samples;
    return Math.max(0., eprob - z * FastMath.sqrt((eprob * (1 - eprob)) / samples));
  }
1183373
def error_rate(predictions, labels):
  """"""
  return 100.0 - (
      100.0 *
      np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) /
      predictions.shape[0])
210810
def accuracy_thresh(y_pred:Tensor, y_true:Tensor, thresh:float=0.5, sigmoid:bool=True)->Rank0Tensor:
    ""
    if sigmoid: y_pred = y_pred.sigmoid()
    return ((y_pred>thresh)==y_true.byte()).float().mean()
152047
private static boolean checkCriticalValue(double score, int n, int m, boolean is_twoTailed, double aLevel) {
        double probability=ContinuousDistributions.fCdf(score,n-1,m-1);

        boolean rejectH0=false;

        double a=aLevel;
        if(is_twoTailed) { //if to tailed test then split the statistical significance in half
            a=aLevel/2;
        }
        if(probability<=a || probability>=(1-a)) {
            rejectH0=true;
        }

        return rejectH0;
    }
72154
def confidence_interval_continuous(
        point_estimate, stddev, sample_size, confidence=.95, **kwargs
):
    """"""
    alpha = ppf((confidence + 1) / 2, sample_size - 1)

    margin = stddev / sqrt(sample_size)
    return (point_estimate - alpha * margin, point_estimate + alpha * margin)
1325374
def confidence_level(self):
        
        z = self.z_score
        if isinstance(z, string_types):
            return z

        z = abs(round(z, 3))

        if z == 0.0:
            return "No Change"
        elif z < 1.65:
            return "No Confidence"
        elif z < 2.33:
            return "95% Confidence"
        elif z < 3.08:
            return "99% Confidence"
        return "99.9% Confidence"
525547
def weighted_std(values, weights):
    
    average = np.average(values, weights=weights)
    variance = np.average((values-average)**2, weights=weights)
    return np.sqrt(variance)
1049188
def padded_accuracy_topk(predictions,
                         labels,
                         k,
                         weights_fn=common_layers.weights_nonzero):
  """"""
  with tf.variable_scope("padded_accuracy_topk", values=[predictions, labels]):
    padded_predictions, padded_labels = common_layers.pad_with_zeros(
        predictions, labels)
    weights = weights_fn(padded_labels)
    effective_k = tf.minimum(k,
                             common_layers.shape_list(padded_predictions)[-1])
    _, outputs = tf.nn.top_k(padded_predictions, k=effective_k)
    outputs = tf.to_int32(outputs)
    padded_labels = tf.to_int32(padded_labels)
    padded_labels = tf.expand_dims(padded_labels, axis=-1)
    padded_labels += tf.zeros_like(outputs)  # Pad to same shape.
    same = tf.to_float(tf.equal(outputs, padded_labels))
    same_topk = tf.reduce_sum(same, axis=-1)
    return same_topk, weights
884514
def f1_score(prediction, ground_truth):
    '''
    
    '''
    prediction_tokens = normalize_answer(prediction).split()
    ground_truth_tokens = normalize_answer(ground_truth).split()
    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
    num_same = sum(common.values())
    if num_same == 0:
        return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(ground_truth_tokens)
    f1_result = (2 * precision * recall) / (precision + recall)
    return f1_result
786992
def accuracy(input:Tensor, targs:Tensor)->Rank0Tensor:
    ""
    n = targs.shape[0]
    input = input.argmax(dim=-1).view(n,-1)
    targs = targs.view(n,-1)
    return (input==targs).float().mean()
657367
def padded_sequence_accuracy(predictions,
                             labels,
                             weights_fn=common_layers.weights_nonzero):
  """"""
  # If the last dimension is 1 then we're using L1/L2 loss.
  if common_layers.shape_list(predictions)[-1] == 1:
    return rounding_sequence_accuracy(
        predictions, labels, weights_fn=weights_fn)
  with tf.variable_scope(
      "padded_sequence_accuracy", values=[predictions, labels]):
    padded_predictions, padded_labels = common_layers.pad_with_zeros(
        predictions, labels)
    weights = weights_fn(padded_labels)

    # Flatten, keeping batch dim (and num_classes dim for predictions)
    # TPU argmax can only deal with a limited number of dimensions
    predictions_shape = common_layers.shape_list(padded_predictions)
    batch_size = predictions_shape[0]
    num_classes = predictions_shape[-1]
    flat_size = common_layers.list_product(
        common_layers.shape_list(padded_labels)[1:])
    padded_predictions = tf.reshape(
        padded_predictions,
        [batch_size, common_layers.list_product(predictions_shape[1:-1]),
         num_classes])
    padded_labels = tf.reshape(padded_labels, [batch_size, flat_size])
    weights = tf.reshape(weights, [batch_size, flat_size])

    outputs = tf.to_int32(tf.argmax(padded_predictions, axis=-1))
    padded_labels = tf.to_int32(padded_labels)
    not_correct = tf.to_float(tf.not_equal(outputs, padded_labels)) * weights
    axis = list(range(1, len(outputs.get_shape())))
    correct_seq = 1.0 - tf.minimum(1.0, tf.reduce_sum(not_correct, axis=axis))
    return correct_seq, tf.constant(1.0)
886823
private static boolean checkCriticalValue(double score, int df, boolean is_twoTailed, double aLevel) {
        double probability=ContinuousDistributions.studentsCdf(score,df);

        boolean rejectH0=false;

        double a=aLevel;
        if(is_twoTailed) { //if to tailed test then split the statistical significance in half
            a=aLevel/2;
        }
        if(probability<=a || probability>=(1-a)) {
            rejectH0=true;
        }

        return rejectH0;
    }
1330028
def local_accuracy(X, y, model_generator, method_name):
    

    def score_map(true, pred):
        """ Converts local accuracy from % of standard deviation to numerical scores for coloring.
        """

        v = min(1.0, np.std(pred - true) / (np.std(true) + 1e-8))
        if v < 1e-6:
            return 1.0
        elif v < 0.01:
            return 0.9
        elif v < 0.05:
            return 0.75
        elif v < 0.1:
            return 0.6
        elif v < 0.2:
            return 0.4
        elif v < 0.3:
            return 0.3
        elif v < 0.5:
            return 0.2
        elif v < 0.7:
            return 0.1
        else:
            return 0.0
    def score_function(X_train, X_test, y_train, y_test, attr_function, trained_model, random_state):
        return measures.local_accuracy(
            X_train, y_train, X_test, y_test, attr_function(X_test),
            model_generator, score_map, trained_model
        )
    return None, __score_method(X, y, None, model_generator, score_function, method_name)
1212739
def reject_sv(m, s, y):
    
    mp = m + 0.5 * s**2 * (-1. + y**2 * np.exp(-m))
    ntries = 0
    while True:
        ntries += 1
        x = stats.norm.rvs(loc=mp, scale=s)
        u = stats.uniform.rvs()
        if np.log(u) < - 0.5 * y**2 * (np.exp(-x) - np.exp(-m) * (1. + m - x)):
            break
        if ntries > 1000:
            print('1000 failed attempt, m,s,y=%f, %f, %f' % (m, s, y))
            break
    return x
601677
def _accuracy_in_session(self, sess, other_var, X_feat, X_seq, y):
        
        y_pred = self._predict_in_session(sess, other_var, X_feat, X_seq)
        return ce.mse(y_pred, y)
572414
def p(i, sample_size, weights):
        

        # Determine the initial pick values
        weight_i = weights[i]
        weights_sum = sum(weights)

        # Build a list of weights that don't contain the weight `i`. This list will
        # be used to build the possible picks before weight `i`.
        other_weights = list(weights)
        del other_weights[i]

        # Calculate the probability
        probability_of_i = 0
        for picks in range(0, sample_size):

            # Build the list of possible permutations for this pick in the sample
            permutations = list(itertools.permutations(other_weights, picks))

            # Calculate the probability for this permutation
            permutation_probabilities = []
            for permutation in permutations:

                # Calculate the probability for each pick in the permutation
                pick_probabilities = []
                pick_weight_sum = weights_sum

                for pick in permutation:
                    pick_probabilities.append(pick / pick_weight_sum)

                    # Each time we pick we update the sum of the weight the next
                    # pick is from.
                    pick_weight_sum -= pick

                # Add the probability of picking i as the last pick
                pick_probabilities += [weight_i / pick_weight_sum]

                # Multiply all the probabilities for the permutation together
                permutation_probability = reduce(
                    lambda x, y: x * y, pick_probabilities
                    )
                permutation_probabilities.append(permutation_probability)

            # Add together all the probabilities for all permutations together
            probability_of_i += sum(permutation_probabilities)

        return probability_of_i