avg dist from one lamost point to nearest 10 training points
516160
def calc_dist(lamost_point, training_points, coeffs):
    
    diff2 = (training_points - lamost_point)**2
    dist = np.sqrt(np.sum(diff2*coeffs, axis=1))
    return np.mean(dist[dist.argsort()][0:10])
683261
def smooth_location_by_average(opts = {})
      seconds_either_side = opts[:averaging_window] || 20

      # calculate the first and last points to which the smoothing should be applied
      earliest = (find_point_by_time_or_offset(opts[:start]) || @earliest_point).time
      latest = (find_point_by_time_or_offset(opts[:end]) || @latest_point).time

      tmp_points = []

      @points.each do |point|
        if point.time > latest || point.time < earliest
          tmp_points.push point # add the point unaltered
          next
        end
        lat_av = 0.to_f
        lon_av = 0.to_f
        alt_av = 0.to_f
        n = 0
        # k ranges from the time of the current point +/- 20s
        (-1 * seconds_either_side..seconds_either_side).each do |k|
          # find the point nearest to the time offset indicated by k
          contributing_point = closest_point(point.time + k)
          # sum up the contributions to the average
          lat_av += contributing_point.lat
          lon_av += contributing_point.lon
          alt_av += contributing_point.elevation
          n += 1
        end
        # calculate the averages
        tmp_point = point.clone
        tmp_point.lon = (lon_av / n).round(7)
        tmp_point.elevation = (alt_av / n).round(2)
        tmp_point.lat = (lat_av / n).round(7)
        tmp_points.push tmp_point
      end
      @points.clear
      reset_meta_data
      # now commit the averages back and recalculate the distances
      tmp_points.each do |point|
        append_point(point)
      end
    end
116246
def estimate_global_norm_range(N_t, func, num_of_sample=10, func_args={},
                               percen_lower=1, percen_upper=99, show_progress=True):
    

    ## Check input arguments
    for arg in [N_t, num_of_sample]:
        assert is_integer_valued_real(arg)
    assert num_of_sample <= N_t
    for arg in [percen_lower, percen_upper]:
        assert is_real_number(arg)
        assert (0 <= arg) and (arg <= 100)
    assert percen_lower <= percen_upper

    assert callable(func)
    assert type(func_args) is dict

    assert type(show_progress) is bool

    sample_indices = np.random.randint(0, N_t-1, num_of_sample)

    if show_progress:
        progress_bar = Progress_Bar(num_of_sample)
    uppers = []
    lowers = []

    for idx, sample_index in enumerate(sample_indices):
        frame_data = func(sample_index, **func_args)
        lowers.append(np.percentile(frame_data, percen_lower))
        uppers.append(np.percentile(frame_data, percen_upper))
        if show_progress:
            progress_bar.print(idx)

    vmin = np.percentile(lowers, percen_lower)
    vmax = np.percentile(uppers, percen_upper)

    return vmin, vmax
335535
def move_point_num(point, to_clust, from_clust, cl_attr_sum, cl_memb_sum):
    """"""
    # Update sum of attributes in cluster.
    for iattr, curattr in enumerate(point):
        cl_attr_sum[to_clust][iattr] += curattr
        cl_attr_sum[from_clust][iattr] -= curattr
    # Update sums of memberships in cluster
    cl_memb_sum[to_clust] += 1
    cl_memb_sum[from_clust] -= 1
    return cl_attr_sum, cl_memb_sum
1102639
def _guess_lines(ys, max_lines=50, confidence_minimum=0.0):
        """"""
        ys = ys.astype(numpy.float32)
        compactness_list, means_list, diffs, deviations = [], [], [], []
        start_n = 1
        for k in range(start_n, min(len(ys), max_lines)):
            compactness, classified_points, means = cv2.kmeans(data=ys, K=k, bestLabels=None, criteria=(
            cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_MAX_ITER, 1, 10), attempts=2, flags=cv2.KMEANS_PP_CENTERS)
            means = numpy.sort(means, axis=0)
            means_list.append(means)
            compactness_list.append(compactness)
            if k < 3:
                tmp1 = [1, 2, 500, 550]  # forge data for bad clusters
            else:
                # calculate the center of each cluster. Assuming lines are equally spaced...
                tmp1 = numpy.diff(means, axis=0)  # diff will be equal or very similar
            tmp2 = numpy.std(tmp1) / numpy.mean(means)  # so variance is minimal
            tmp3 = numpy.sum((tmp1 - numpy.mean(tmp1)) ** 2)  # root mean square deviation, more sensitive than std
            diffs.append(tmp1)
            deviations.append(tmp3)

        compactness_list = numpy.diff(
            numpy.log(numpy.array(compactness_list) + 0.01))  # sum small amount to avoid log(0)
        deviations = numpy.array(deviations[1:])
        deviations[0] = numpy.mean(deviations[1:])
        compactness_list = (compactness_list - numpy.mean(compactness_list)) / numpy.std(compactness_list)
        deviations = (deviations - numpy.mean(deviations)) / numpy.std(deviations)
        aglomerated_metric = 0.1 * compactness_list + 0.9 * deviations

        i = numpy.argmin(aglomerated_metric) + 1
        lines = means_list[i]

        # calculate confidence
        betterness = numpy.sort(aglomerated_metric, axis=0)
        confidence = (betterness[1] - betterness[0]) / (betterness[2] - betterness[1])
        if confidence < confidence_minimum:
            raise Exception("low confidence")
        return lines
1347687
def normalize_dense(M, norm="frag", order=1, iterations=3):
    

    s = np.array(M, np.float64)
    floatorder = np.float64(order)

    if norm == "SCN":
        for _ in range(0, iterations):

            sumrows = s.sum(axis=1)
            maskrows = (sumrows != 0)[:, None] * (sumrows != 0)[None, :]
            sums_row = sumrows[:, None] * np.ones(sumrows.shape)[None, :]
            s[maskrows] = 1. * s[maskrows] / sums_row[maskrows]

            sumcols = s.sum(axis=0)
            maskcols = (sumcols != 0)[:, None] * (sumcols != 0)[None, :]
            sums_col = sumcols[None, :] * np.ones(sumcols.shape)[:, None]
            s[maskcols] = 1. * s[maskcols] / sums_col[maskcols]

    elif norm == "mirnylib":
        try:
            from mirnylib import numutils as ntls
            s = ntls.iterativeCorrection(s, iterations)[0]
        except ImportError as e:
            print(str(e))
            print("I can't find mirnylib.")
            print("Please install it from "
                  "https://bitbucket.org/mirnylab/mirnylib")
            print("I will use default norm as fallback.")
            return normalize_dense(M, order=order, iterations=iterations)

    elif norm == "frag":
        for _ in range(1, iterations):
            s_norm_x = np.linalg.norm(s, ord=floatorder, axis=0)
            s_norm_y = np.linalg.norm(s, ord=floatorder, axis=1)
            s_norm = np.tensordot(s_norm_x, s_norm_y, axes=0)
            s[s_norm != 0] = 1. * s[s_norm != 0] / s_norm[s_norm != 0]

    elif norm == "global":
        s_norm = np.linalg.norm(s, ord=floatorder)
        s /= 1. * s_norm

    elif callable(norm):
        s = norm(M)

    else:
        print("Unknown norm. Returning input as fallback")

    return (s + s.T) / 2
672833
def do_training(num_epoch, optimizer, kvstore, learning_rate, model_prefix, decay):
    """"""
    summary_writer = SummaryWriter(args.tblog_dir)
    lr_scheduler = SimpleLRScheduler(learning_rate)
    optimizer_params = {'lr_scheduler': lr_scheduler}
    module.init_params()
    module.init_optimizer(kvstore=kvstore,
                          optimizer=optimizer,
                          optimizer_params=optimizer_params)
    n_epoch = 0
    while True:
        if n_epoch >= num_epoch:
            break
        train_iter.reset()
        val_iter.reset()
        loss_metric.reset()
        for n_batch, data_batch in enumerate(train_iter):
            module.forward_backward(data_batch)
            module.update()
            module.update_metric(loss_metric, data_batch.label)
            loss_metric.get_batch_log(n_batch)
        train_acc, train_loss, train_recon_err = loss_metric.get_name_value()
        loss_metric.reset()
        for n_batch, data_batch in enumerate(val_iter):
            module.forward(data_batch)
            module.update_metric(loss_metric, data_batch.label)
            loss_metric.get_batch_log(n_batch)
        val_acc, val_loss, val_recon_err = loss_metric.get_name_value()

        summary_writer.add_scalar('train_acc', train_acc, n_epoch)
        summary_writer.add_scalar('train_loss', train_loss, n_epoch)
        summary_writer.add_scalar('train_recon_err', train_recon_err, n_epoch)
        summary_writer.add_scalar('val_acc', val_acc, n_epoch)
        summary_writer.add_scalar('val_loss', val_loss, n_epoch)
        summary_writer.add_scalar('val_recon_err', val_recon_err, n_epoch)

        print('Epoch[%d] train acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, train_acc, train_loss,
                                                                        train_recon_err))
        print('Epoch[%d] val acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, val_acc, val_loss, val_recon_err))
        print('SAVE CHECKPOINT')

        module.save_checkpoint(prefix=model_prefix, epoch=n_epoch)
        n_epoch += 1
        lr_scheduler.learning_rate = learning_rate * (decay ** n_epoch)
735975
def _compute_nearest_weights_edge(idcs, ndist, variant):
    """"""
    # Get out-of-bounds indices from the norm_distances. Negative
    # means "too low", larger than or equal to 1 means "too high"
    lo = (ndist < 0)
    hi = (ndist > 1)

    # For "too low" nodes, the lower neighbor gets weight zero;
    # "too high" gets 1.
    if variant == 'left':
        w_lo = np.where(ndist <= 0.5, 1.0, 0.0)
    else:
        w_lo = np.where(ndist < 0.5, 1.0, 0.0)

    w_lo[lo] = 0
    w_lo[hi] = 1

    # For "too high" nodes, the upper neighbor gets weight zero;
    # "too low" gets 1.
    if variant == 'left':
        w_hi = np.where(ndist <= 0.5, 0.0, 1.0)
    else:
        w_hi = np.where(ndist < 0.5, 0.0, 1.0)

    w_hi[lo] = 1
    w_hi[hi] = 0

    # For upper/lower out-of-bounds nodes, we need to set the
    # lower/upper neighbors to the last/first grid point
    edge = [idcs, idcs + 1]
    edge[0][hi] = -1
    edge[1][lo] = 0

    return w_lo, w_hi, edge
155590
def main(
        gpu:Param("GPU to run on", str)=None,
        lr: Param("Learning rate", float)=1e-3,
        size: Param("Size (px: 128,192,224)", int)=128,
        debias_mom: Param("Debias statistics", bool)=False,
        debias_sqr: Param("Debias statistics", bool)=False,
        opt: Param("Optimizer: 'adam','genopt','rms','sgd'", str)='genopt',
        alpha: Param("Alpha", float)=0.99,
        mom: Param("Momentum", float)=0.9,
        eps: Param("epsilon", float)=1e-7,
        decay: Param("Decay AvgStatistic (momentum)", bool)=False,
        epochs: Param("Number of epochs", int)=5,
        bs: Param("Batch size", int)=128,
        ):
    """"""

    # Pick one of these
    gpu = setup_distrib(gpu)
    if gpu is None: bs *= torch.cuda.device_count()

    moms = (mom,mom)
    stats = [
            AvgStatistic('momentum', mom,   scope=StatScope.Weight, decay=decay, debias=debias_mom),
            AvgSquare   ('alpha',    alpha, scope=StatScope.Weight, debias=debias_sqr),
            ConstStatistic('eps', eps), ConstStatistic('clip', 0.001),
            ]
    if   opt=='adam'  : opt_func = partial(optim.Adam, betas=(mom,alpha), eps=eps)
    elif opt=='rms'   : opt_func = partial(optim.RMSprop, alpha=alpha)
    elif opt=='genopt': opt_func = partial(GeneralOptimizer, on_step=on_step, stats=stats)
    else: raise Exception(f'unknown opt: {opt}')

    #opt_func = optim.SGD
    #learn = (cnn_learner(data, models.xresnet50, pretrained=False, concat_pool=False, lin_ftrs=[], split_on=bn_and_final,
    print(f'lr: {lr}; size: {size}; debias_mom: {debias_mom}; debias_sqr: {debias_sqr}; opt: {opt}; alpha: {alpha}; mom: {mom}; eps: {eps}; decay: {decay}')
    print('imagenette')
    get_learner(lr, size, False, bs, opt_func, gpu, epochs)
    gc.collect()

    print('imagewoof')
    get_learner(lr, size, True, bs, opt_func, gpu, epochs)
940611
def nearest_neighbor(x, tSet):
    
    assert isinstance(x, tuple) and isinstance(tSet, dict)
    current_key = ()
    min_d = float('inf')
    for key in tSet:
        d = distance(x, key)
        if d < min_d:
            min_d = d
            current_key = key
    return tSet[current_key]
210838
def next_frame_base_range(rhp):
  """"""
  rhp.set_float("dropout", 0.2, 0.6)
  rhp.set_discrete("hidden_size", [64, 128, 256])
  rhp.set_int("num_compress_steps", 5, 8)
  rhp.set_discrete("batch_size", [4, 8, 16, 32])
  rhp.set_int("num_hidden_layers", 1, 3)
  rhp.set_int("filter_double_steps", 1, 6)
  rhp.set_float("learning_rate_constant", 1., 4.)
  rhp.set_int("learning_rate_warmup_steps", 500, 3000)
  rhp.set_float("initializer_gain", 0.8, 1.8)
128060
def vq_nearest_neighbor(x, means,
                        soft_em=False, num_samples=10, temperature=None):
  """"""
  bottleneck_size = common_layers.shape_list(means)[0]
  x_norm_sq = tf.reduce_sum(tf.square(x), axis=-1, keepdims=True)
  means_norm_sq = tf.reduce_sum(tf.square(means), axis=-1, keepdims=True)
  scalar_prod = tf.matmul(x, means, transpose_b=True)
  dist = x_norm_sq + tf.transpose(means_norm_sq) - 2 * scalar_prod
  if soft_em:
    x_means_idx = tf.multinomial(-dist, num_samples=num_samples)
    x_means_hot = tf.one_hot(
        x_means_idx, depth=common_layers.shape_list(means)[0])
    x_means_hot = tf.reduce_mean(x_means_hot, axis=1)
  else:
    if temperature is None:
      x_means_idx = tf.argmax(-dist, axis=-1)
    else:
      x_means_idx = tf.multinomial(- dist / temperature, 1)
      x_means_idx = tf.squeeze(x_means_idx, axis=-1)
    if (common_layers.should_generate_summaries() and
        not common_layers.is_xla_compiled()):
      tf.summary.histogram("means_idx", tf.reshape(x_means_idx, [-1]))
    x_means_hot = tf.one_hot(x_means_idx, bottleneck_size)
  x_means_hot_flat = tf.reshape(x_means_hot, [-1, bottleneck_size])
  x_means = tf.matmul(x_means_hot_flat, means)
  e_loss = tf.reduce_mean(tf.squared_difference(x, tf.stop_gradient(x_means)))
  return x_means_hot, e_loss, dist
482986
def closest_points(S):
    
    shuffle(S)
    assert len(S) >= 2
    p = S[0]
    q = S[1]
    d = dist(p, q)
    while d > 0:
        r = improve(S, d)
        if r:
            d, p, q = r
        else:
            break
    return p, q
265640
def _find_neighbors(self, inst, avg_dist):
        
        NN = []
        min_indicies = []

        for i in range(self._datalen):
            if inst != i:
                locator = [inst, i]
                if i > inst:
                    locator.reverse()
                d = self._distance_array[locator[0]][locator[1]]
                if d < avg_dist:  # Defining the neighborhood with an average distance radius.
                    min_indicies.append(i)
        for i in range(len(min_indicies)):
            NN.append(min_indicies[i])
        return np.array(NN, dtype=np.int32)
827931
def main(net):
  '''
  
  '''
  # calculate the distance between the data points within the same category and
  # compare to null distribution
  for inst_rc in ['row', 'col']:

    inst_nodes = deepcopy(net.dat['nodes'][inst_rc])

    inst_index = deepcopy(net.dat['node_info'][inst_rc]['clust'])

    # reorder based on clustered order
    inst_nodes = [ inst_nodes[i] for i in inst_index]

    # make distance matrix dataframe
    dm = dist_matrix_lattice(inst_nodes)

    node_infos = list(net.dat['node_info'][inst_rc].keys())

    all_cats = []
    for inst_info in node_infos:
      if 'dict_cat_' in inst_info:
        all_cats.append(inst_info)

    for cat_dict in all_cats:

      tmp_dict = net.dat['node_info'][inst_rc][cat_dict]

      pval_name = cat_dict.replace('dict_','pval_')
      net.dat['node_info'][inst_rc][pval_name] = {}

      for cat_name in tmp_dict:

        subset = tmp_dict[cat_name]

        inst_median = calc_median_dist_subset(dm, subset)

        hist = calc_hist_distances(dm, subset, inst_nodes)

        pval = 0

        for i in range(len(hist['prob'])):
          if i == 0:
            pval = hist['prob'][i]
          if i >= 1:
            if inst_median >= hist['bins'][i]:
              pval = pval + hist['prob'][i]

        net.dat['node_info'][inst_rc][pval_name][cat_name] = pval
137413
def distLinf(x1,y1,x2,y2):
    """"""
    return int(max(abs(x2-x1),abs(y2-y1)))
1302618
def index_of_nearest(p, hot_points, distance_f=distance):
    
    min_dist = None
    nearest_hp_i = None
    for i, hp in enumerate(hot_points):
        dist = distance_f(p, hp)
        if min_dist is None or dist < min_dist:
            min_dist = dist
            nearest_hp_i = i
    return nearest_hp_i