With compressed attention.
160541
def xmoe2_v1_l4k_compressed_c4():
  """"""
  hparams = xmoe2_v1_l4k()
  hparams.decoder_layers = [
      "compressed_att" if l == "att" else l for l in hparams.decoder_layers]
  hparams.compression_factor = 4
  return hparams
948691
def attention_lm_translation():
  """"""
  hparams = attention_lm_base()
  hparams.layer_preprocess_sequence = "n"
  hparams.layer_postprocess_sequence = "da"
  hparams.learning_rate = 0.4
  hparams.prepend_mode = "prepend_inputs_masked_attention"
  hparams.max_length = 512
  hparams.label_smoothing = 0.1
  hparams.shared_embedding_and_softmax_weights = True
  return hparams
605662
def attention_lm_ae_extended():
  """"""
  hparams = attention_lm_moe_base_long_seq()
  hparams.attention_layers = "eeee"
  hparams.attention_local = True
  # hparams.factored_logits=1  # Necessary when the number of expert grow bigger
  hparams.attention_moe_k = 2
  hparams.attention_exp_factor = 4
  # hparams.attention_exp_inputdim = 128

  hparams.layer_preprocess_sequence = "n"
  hparams.layer_postprocess_sequence = "da"
  return hparams
1280955
function onReadable() {
    if (self._assimilationState.waiting) {
      self._assimilationState.waiting = false;
      self._pushChunks();
    }
  }
1121483
def compressed
      @compressed ||= case @type
      when 'css'
        Sass::Engine.new(joined, :syntax => :scss, :cache => false, :style => :compressed).render rescue joined
      when 'js'
        Uglifier.compile(joined, :mangle => false, :comments => :none) rescue joined
      end
    end
58317
def focus_prev_matching(self, querystring):
        """"""
        self.focus_property(lambda x: x._message.matches(querystring),
                            self._tree.prev_position)
759522
def full_self_attention(x,
                        self_attention_bias,
                        hparams,
                        q_padding="LEFT",
                        kv_padding="LEFT"):
  """"""
  x, x_shape, is_4d = maybe_reshape_4d_to_3d(x)
  if self_attention_bias is not None:
    self_attention_bias = get_self_attention_bias(x)
  with tf.variable_scope("self_att"):
    y = common_attention.multihead_attention(
        x,
        None,
        self_attention_bias,
        hparams.attention_key_channels or hparams.hidden_size,
        hparams.attention_value_channels or hparams.hidden_size,
        hparams.hidden_size,
        hparams.num_heads,
        hparams.attention_dropout,
        q_filter_width=hparams.q_filter_width,
        kv_filter_width=hparams.kv_filter_width,
        q_padding=q_padding,
        kv_padding=kv_padding,
        name="self_att")
    if is_4d:
      y = tf.reshape(y, [x_shape[0], x_shape[1], x_shape[2], x_shape[3]])
      y.set_shape([None, None, None, hparams.hidden_size])
    return y
831187
function (bufferSize, numberOfIterations) {

    var bufSize = bufferSize || 16000,
        iterations = numberOfIterations || 100;

    for (var i = 0; i < iterations; i++) {
        var data = [];

        for (var k = 0; k < bufSize; k++) {
            data.push((parseFloat(Math.random() * 1.0) - 0.5));
        }

        // Dispatch an event
        var event = new CustomEvent('audioinput');
        event.data = data;
        window.dispatchEvent(event);
    }
}
587557
def mean_with_attention(x, name, num_heads=4):
  """"""
  with tf.variable_scope(name):
    shape = shape_list(x)
    m = tf.reduce_mean(x, [1, 2])
    a = layers().Dense(num_heads, name="mean_attn")(x)
    s = tf.reshape(a, [shape[0], -1, num_heads])
    s = tf.nn.softmax(s, axis=1)
    s = tf.reshape(s, shape[:-1] + [1, num_heads])
    am = tf.reduce_mean(tf.expand_dims(x, axis=-1) * s, [1, 2])
    l = tf.concat([am, tf.expand_dims(m, axis=-1)], axis=-1)
    return layers().Dense(2 * shape[-1], name="mean_attn_final")(
        tf.reshape(l, [shape[0], (num_heads+1) * shape[-1]]))
1291464
def attention(query: torch.Tensor,
              key: torch.Tensor,
              value: torch.Tensor,
              mask: torch.Tensor = None,
              dropout: Callable = None) -> Tuple[torch.Tensor, torch.Tensor]:
    """'"""
    d_k = query.size(-1)
    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)
    if mask is not None:
        scores = scores.masked_fill(mask == 0, -1e9)
    p_attn = F.softmax(scores, dim=-1)
    if dropout is not None:
        p_attn = dropout(p_attn)
    return torch.matmul(p_attn, value), p_attn
1000190
def local_attention1d_spatial_decoder(x, kv_dim, heads_dim,
                                      feedforward_dim, hparams):
  """"""
  batch_dim, length_dim, model_dim = x.shape.dims
  blocks_w_dim = mtf.Dimension("blocksw", hparams.block_length)
  num_w_blocks_dim = mtf.Dimension("num_wblocks",
                                   length_dim.size // blocks_w_dim.size)
  x = mtf.reshape(
      x, mtf.Shape([batch_dim, num_w_blocks_dim, blocks_w_dim, model_dim]))
  # [ self attention - ffn - residual + dropout] x n
  for layer in range(hparams.num_decoder_layers):
    layer_name = "decoder_layer_%d" % layer
    with tf.variable_scope(layer_name):
      # Self attention layer
      x += layer_prepostprocess_dropout(
          mtf.layers.local_self_attention_spatial_blocks(
              mtf.layers.layer_norm(x, model_dim, name="layer_norm_att"),
              kv_dim,
              heads_dim,
              memory_w_dim=blocks_w_dim,
              mask_right=True,
              name="self_att"), hparams)
      # ffn layer
      x += layer_prepostprocess_dropout(
          mtf.layers.dense_relu_dense(
              mtf.layers.layer_norm(x, model_dim, name="layer_norm_ffn"),
              feedforward_dim,
              hparams.dropout,
              dropout_broadcast_dims=[length_dim]), hparams)

  output = mtf.layers.layer_norm(x, model_dim, name="final_layer_norm")
  return output
1247157
def adaptive_universal_transformer_multilayer_hard():
  """"""
  hparams = adaptive_universal_transformer_multilayer_tpu()
  hparams.batch_size = 256
  hparams.hard_attention_k = 8
  hparams.add_step_timing_signal = True
  # hparams.add_sru = True  # This is very slow on GPUs, does it help?
  hparams.self_attention_type = "dot_product_relative_v2"
  hparams.max_relative_position = 256
  return hparams
865603
def attention_lm_moe_small():
  
  hparams = attention_lm_moe_base()
  hparams.num_hidden_layers = 4
  hparams.hidden_size = 512
  hparams.filter_size = 2048
  hparams.moe_num_experts = 128
  hparams.moe_layers = "2"
  return hparams
1306723
protected void sparsify()
    {
        final int N = vecs.size();
        int accSize = accelCache == null ? 0 : accelCache.size()/N;
        int svCount = 0;
        for(int i = 0; i < N; i++)
            if(alphas[i] != 0)//Its a support vector
            {
                ListUtils.swap(vecs, svCount, i);
                if(accelCache != null)
                    for(int j = i*accSize; j < (i+1)*accSize; j++)
                        ListUtils.swap(accelCache, svCount*accSize+j-i*accSize, j);
                alphas[svCount++] = alphas[i];
            }

        vecs = new ArrayList<Vec>(vecs.subList(0, svCount));
        alphas = Arrays.copyOfRange(alphas, 0, svCount);
    }
184211
def lost_focus(self):
        """"""
        if (self.is_running and not self.any_has_focus() and
            not self.setting_data and not self.hidden):
            self.hide_tips()
842582
public void compress ()
    {
        for (Iterator<int[]> itr = values().iterator(); itr.hasNext(); ) {
            if (itr.next()[0] == 0) {
                itr.remove();
            }
        }
    }
23417
def mtf_image_transformer_tiny():
  """"""
  hparams = mtf_image_transformer_base()
  hparams.hidden_size = 128
  hparams.d_ff = 256
  hparams.batch_size = 4
  hparams.num_encoder_layers = 1
  hparams.num_decoder_layers = 4
  hparams.num_heads = 4
  hparams.attention_key_size = 128
  hparams.attention_value_size = 128
  hparams.block_length = 32
  # data parallelism and model-parallelism
  hparams.mesh_shape = "batch:2"
  hparams.layout = "batch:batch"
  return hparams