Scan the data tree on the given data_channel to create a corresponding
        InputSetGenerator tree.
514308
def create_from_data_channel(cls, data_channel):
        
        gather_depth = cls._get_gather_depth(data_channel)

        generator = InputSetGeneratorNode()
        for (data_path, data_node) in data_channel.get_ready_data_nodes(
                [], gather_depth):
            flat_data_node = data_node.flattened_clone(save=False)
            input_item = InputItem(
                flat_data_node, data_channel.channel,
                data_channel.as_channel, mode=data_channel.mode)
            generator._add_input_item(data_path, input_item)
        return generator
719117
void scanNewNodes(Set<AstChange> newNodes) {
    BuildGlobalNamespace builder = new BuildGlobalNamespace();

    for (AstChange info : newNodes) {
      if (!info.node.isQualifiedName() && !NodeUtil.mayBeObjectLitKey(info.node)) {
        continue;
      }
      scanFromNode(builder, info.module, info.scope, info.node);
    }
  }
36536
@Process(actionType = InitJberet.class)
    public void init(final Channel channel) {
        ResourceAddress rootAddress = ROOT_ADDRESS.resolve(statementContext);
        Operation readRoot = new Operation.Builder(READ_RESOURCE_OPERATION, rootAddress).build();
        Operation inMemoryRepository = new Operation.Builder(READ_CHILDREN_RESOURCES_OPERATION, rootAddress)
                .param(CHILD_TYPE, "in-memory-job-repository")
                .build();
        Operation jdbcRepository = new Operation.Builder(READ_CHILDREN_RESOURCES_OPERATION, rootAddress)
                .param(CHILD_TYPE, "jdbc-job-repository")
                .build();
        Operation threadFactory = new Operation.Builder(READ_CHILDREN_RESOURCES_OPERATION, rootAddress)
                .param(CHILD_TYPE, "thread-factory")
                .build();
        Operation threadPool = new Operation.Builder(READ_CHILDREN_RESOURCES_OPERATION, rootAddress)
                .param(CHILD_TYPE, "thread-pool")
                .build();
        Composite composite = new Composite(readRoot, inMemoryRepository, jdbcRepository, threadFactory, threadPool);
        dispatcher.execute(new DMRAction(composite), new AsyncCallback<DMRResponse>() {
            @Override
            public void onFailure(final Throwable caught) {
                channel.nack(caught);
            }

            @Override
            public void onSuccess(final DMRResponse response) {
                ModelNode result = response.get();
                if (result.isFailure()) {
                    channel.nack(result.getFailureDescription());
                } else {
                    ModelNode payload = result.get(RESULT);
                    ModelNode step = payload.get("step-1");
                    if (step.get(RESULT).isDefined()) {
                        defaults = step.get(RESULT);
                    }
                    step = payload.get("step-2");
                    if (step.get(RESULT).isDefined()) {
                        inMemoryRepositories.clear();
                        inMemoryRepositories.addAll(step.get(RESULT).asPropertyList());
                    }
                    step = payload.get("step-3");
                    if (step.get(RESULT).isDefined()) {
                        jdbcRepositories.clear();
                        jdbcRepositories.addAll(step.get(RESULT).asPropertyList());
                    }
                    step = payload.get("step-4");
                    if (step.get(RESULT).isDefined()) {
                        threadFactories.clear();
                        threadFactories.addAll(step.get(RESULT).asPropertyList());
                    }
                    step = payload.get("step-5");
                    if (step.get(RESULT).isDefined()) {
                        threadPools.clear();
                        threadPools.addAll(step.get(RESULT).asPropertyList());
                    }
                    channel.ack();
                }
            }
        });
    }
1305904
private Iterator<MultiTableScanResult> scanMultiTableRows(
            final TableSet tables, final DeltaPlacement placement, final ByteBufferRange rowRange,
            final LimitCounter limit, final boolean includeDroppedTables, final boolean includeMirrorTables,
            final int largeRowThreshold, final ReadConsistency consistency, @Nullable final Instant cutoffTime) {

        // Avoiding pinning multiple decoded rows into memory at once.
        return limit.limit(new AbstractIterator<MultiTableScanResult>() {
            private PeekingIterator<Row<ByteBuffer, DeltaKey>> _iter = Iterators.peekingIterator(
                    rowScan(placement, rowRange, _maxColumnsRange, LimitCounter.max(), consistency));

            private long _lastTableUuid = -1;
            private AstyanaxTable _table = null;
            private boolean _droppedTable;
            private boolean _primaryTable;

            @Override
            protected MultiTableScanResult computeNext() {
                while (_iter.hasNext()) {
                    Row<ByteBuffer, DeltaKey> row = _iter.next();
                    ColumnList<DeltaKey> rowColumns = row.getColumns();

                    // Convert the results into a Record object, lazily fetching the rest of the columns as necessary.
                    ByteBuffer rowKey = row.getRawKey();

                    long tableUuid = AstyanaxStorage.getTableUuid(rowKey);
                    if (_lastTableUuid != tableUuid) {
                        _lastTableUuid = tableUuid;
                        try {
                            _table = (AstyanaxTable) tables.getByUuid(tableUuid);
                        } catch (UnknownTableException e) {
                            _table = AstyanaxTable.createUnknown(tableUuid, placement, e.getTable());
                        } catch (DroppedTableException e) {
                            _table = AstyanaxTable.createUnknown(tableUuid, placement, e.getPriorTable());
                        }
                        _droppedTable = _table.isUnknownTable();
                        _primaryTable = _table.getReadStorage().hasUUID(tableUuid);
                    }

                    // Skip dropped and mirror tables if configured
                    if ((!includeDroppedTables && _droppedTable) || (!includeMirrorTables && !_primaryTable)) {
                        _iter = skipToNextTable(tableUuid);
                        continue;
                    }

                    int shardId = AstyanaxStorage.getShardId(rowKey);
                    String key = AstyanaxStorage.getContentKey(rowKey);
                    Record record = newRecord(new Key(_table, key), rowKey, rowColumns, largeRowThreshold, consistency, cutoffTime);
                    return new MultiTableScanResult(rowKey, shardId, tableUuid, _droppedTable, record);
                }

                return endOfData();
            }

            private PeekingIterator<Row<ByteBuffer, DeltaKey>> skipToNextTable(long tableUuid) {
                // Iterate over the next 50 rows first to check for a table switch.  This avoids starting a new range
                // query if the number of rows in the undesired table is small.
                int skipLimit = 50;
                Row<ByteBuffer, DeltaKey> row = null;

                while (skipLimit != 0 && _iter.hasNext()) {
                    row = _iter.peek();
                    long nextTableUuid = AstyanaxStorage.getTableUuid(row.getRawKey());
                    if (nextTableUuid != tableUuid) {
                        // This is the first row of a new table
                        return _iter;
                    } else {
                        _iter.next();
                        skipLimit -= 1;
                    }
                }

                if (_iter.hasNext()) {
                    // Skip the table entirely by starting a new query on the next possible table
                    assert row != null;
                    int shardId = AstyanaxStorage.getShardId(row.getRawKey());
                    ByteBuffer nextPossibleTableStart = AstyanaxStorage.getRowKeyRaw(shardId, tableUuid + 1, "");
                    ByteBuffer end = rowRange.getEnd();

                    if (AstyanaxStorage.compareKeys(nextPossibleTableStart, end) < 0) {
                        // We haven't reached the last end boundary of the original range scan
                        ByteBufferRange updatedRange = new ByteBufferRangeImpl(nextPossibleTableStart, end, -1, false);
                        return Iterators.peekingIterator(
                                rowScan(placement, updatedRange, _maxColumnsRange, LimitCounter.max(), consistency));
                    }
                }

                return Iterators.peekingIterator(Iterators.<Row<ByteBuffer, DeltaKey>>emptyIterator());
            }
        });
    }
331822
@SuppressWarnings("unused")
    public synchronized Channel findOrCreateChannel(ChannelData channelData) throws ChannelException {
        String channelName = channelData.getName();
        Channel ret = this.existingChannels.get(channelName);
        if (ret == null) {
            // Create the new channel with the input configuration
            if (null != this.globalBufferSize) {
                // if the factory setting exists, see if the channel overrides it
                // otherwise apply it now
                Map<Object, Object> props = channelData.getPropertyBag();
                if (!props.containsKey(PROP_BUFFERSIZE)) {
                    props.put(PROP_BUFFERSIZE, this.globalBufferSize);
                }
            }
            ret = new HttpDispatcherChannel(channelData, this);
            this.existingChannels.put(channelName, ret);
        }
        return ret;
    }
142770
def generate(ast_tree: ast.Tree, model_name: str):
    
    component_ref = ast.ComponentRef.from_string(model_name)
    ast_tree_new = copy.deepcopy(ast_tree)
    ast_walker = TreeWalker()
    flat_tree = flatten(ast_tree_new, component_ref)
    sympy_gen = SympyGenerator()
    ast_walker.walk(sympy_gen, flat_tree)
    return sympy_gen.src[flat_tree]
1010523
def build_and_train(iterations, log_stride, test=False):
  """"""

  # Test mode settings.
  batch_size = 2 if test else FLAGS.batch_size
  num_mems = 2 if test else FLAGS.num_mems
  num_heads = 1 if test else FLAGS.num_mems
  num_blocks = 1 if test else FLAGS.num_mems
  head_size = 4 if test else FLAGS.head_size
  max_length = 3 if test else FLAGS.max_length
  max_nest = 2 if test else FLAGS.max_nest
  mlp_size = (20,) if test else (256, 256, 256, 256)

  with tf.Graph().as_default():
    t0 = time.time()

    # Initialize the dataset.
    lte_train = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest)
    lte_test = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest, mode=learn_to_execute.Mode.TEST)
    train_data_iter = lte_train.make_one_shot_iterator().get_next()
    test_data_iter = lte_test.make_one_shot_iterator().get_next()
    output_size = lte_train.state.vocab_size

    # Create the model.
    core = snt.RelationalMemory(
        mem_slots=num_mems,
        head_size=head_size,
        num_heads=num_heads,
        num_blocks=num_blocks,
        gate_style=FLAGS.gate_style)
    final_mlp = snt.nets.MLP(
        output_sizes=mlp_size,
        activate_final=True)
    model = SequenceModel(
        core=core,
        target_size=output_size,
        final_mlp=final_mlp)
    tf.logging.info("Instantiated models ({:3f})".format(time.time() - t0))

    # Define the loss & accuracy.
    def loss_fn(inputs, targets, input_sequence_length, output_sequence_length):
      """Creates the loss and the exports."""
      logits = model(
          inputs, targets, input_sequence_length, output_sequence_length)
      targets = tf.cast(targets, tf.int32)
      sq_sz_out_max = targets.shape[0].value

      # Create a mask to ignore accuracy on buffer characters.
      sequence_sizes = tf.cast(output_sequence_length, tf.float32)
      lengths_transposed = tf.expand_dims(sequence_sizes, 1)
      range_row = tf.expand_dims(
          tf.range(0, sq_sz_out_max, 1, dtype=tf.float32), 0)
      mask = tf.cast(tf.transpose(tf.less(range_row, lengths_transposed)),
                     tf.float32)

      # Compute token accuracy and solved.
      correct = tf.equal(tf.argmax(logits, 2), tf.argmax(targets, 2))
      solved = tf.reduce_all(tf.boolean_mask(correct, tf.squeeze(mask)), axis=0)
      token_acc = tf.reduce_sum(tf.cast(correct, tf.float32) * mask)
      token_acc /= tf.reduce_sum(sequence_sizes)

      # Compute Loss.
      mask = tf.cast(tf.tile(tf.expand_dims(mask, 2), (1, 1, logits.shape[2])),
                     tf.float32)
      masked_logits = logits * mask
      masked_target = tf.cast(targets, tf.float32) * mask
      logits_flat = tf.reshape(masked_logits,
                               [sq_sz_out_max * batch_size, -1])
      target_flat = tf.reshape(masked_target,
                               [sq_sz_out_max * batch_size, -1])
      xent = tf.nn.softmax_cross_entropy_with_logits(logits=logits_flat,
                                                     labels=target_flat)
      loss = tf.reduce_mean(xent)
      return loss, token_acc, solved

    # Get training step counter.
    global_step = tf.train.get_or_create_global_step()

    # Create the optimizer.
    learning_rate_op = tf.reduce_max([
        tf.train.exponential_decay(
            FLAGS.learning_rate,
            global_step,
            decay_steps=FLAGS.epochs // 100,
            decay_rate=0.9,
            staircase=False),
        FLAGS.min_learning_rate
    ])
    optimizer = tf.train.AdamOptimizer(learning_rate_op)

    # Compute loss, accuracy & the step op.
    inputs, targets, _, input_lengths, output_lengths = train_data_iter
    train_loss, train_acc, train_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    step_op = optimizer.minimize(train_loss, global_step=global_step)
    inputs, targets, _, input_lengths, output_lengths = test_data_iter
    _, test_acc, test_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    tf.logging.info("Created losses and optimizers ({:3f})".format(
        time.time() - t0))

    # Begin Training.
    t0 = time.time()
    tf.logging.info("Starting training ({:3f})".format(time.time() - t0))
    with tf.train.SingularMonitoredSession() as sess:
      for it in six.moves.range(iterations):
        sess.run([step_op, learning_rate_op])
        if it % log_stride == 0:
          loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v = sess.run([
              train_loss, train_acc, test_acc, train_sol, test_sol])
          elapsed = time.time() - t0
          tf.logging.info(
              "iter: {:2d}, train loss {:3f}; train acc {:3f}; test acc {:3f};"
              " train solved {:3f}; test solved {:3f}; ({:3f})".format(
                  it, loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v,
                  elapsed))
280816
def make_dataloader(data_train, data_val, data_test, args,
                    use_average_length=False, num_shards=0, num_workers=8):
    """"""
    data_train_lengths = get_data_lengths(data_train)
    data_val_lengths = get_data_lengths(data_val)
    data_test_lengths = get_data_lengths(data_test)
    train_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),
                                  btf.Stack(dtype='float32'), btf.Stack(dtype='float32'))
    test_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),
                                 btf.Stack(dtype='float32'), btf.Stack(dtype='float32'),
                                 btf.Stack())
    target_val_lengths = list(map(lambda x: x[-1], data_val_lengths))
    target_test_lengths = list(map(lambda x: x[-1], data_test_lengths))
    if args.bucket_scheme == 'constant':
        bucket_scheme = nlp.data.ConstWidthBucket()
    elif args.bucket_scheme == 'linear':
        bucket_scheme = nlp.data.LinearWidthBucket()
    elif args.bucket_scheme == 'exp':
        bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)
    else:
        raise NotImplementedError
    train_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_train_lengths,
                                                      batch_size=args.batch_size,
                                                      num_buckets=args.num_buckets,
                                                      ratio=args.bucket_ratio,
                                                      shuffle=True,
                                                      use_average_length=use_average_length,
                                                      num_shards=num_shards,
                                                      bucket_scheme=bucket_scheme)
    logging.info('Train Batch Sampler:\n%s', train_batch_sampler.stats())
    train_data_loader = nlp.data.ShardedDataLoader(data_train,
                                                   batch_sampler=train_batch_sampler,
                                                   batchify_fn=train_batchify_fn,
                                                   num_workers=num_workers)

    val_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_val_lengths,
                                                    batch_size=args.test_batch_size,
                                                    num_buckets=args.num_buckets,
                                                    ratio=args.bucket_ratio,
                                                    shuffle=False,
                                                    use_average_length=use_average_length,
                                                    bucket_scheme=bucket_scheme)
    logging.info('Valid Batch Sampler:\n%s', val_batch_sampler.stats())
    val_data_loader = gluon.data.DataLoader(data_val,
                                            batch_sampler=val_batch_sampler,
                                            batchify_fn=test_batchify_fn,
                                            num_workers=num_workers)
    test_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_test_lengths,
                                                     batch_size=args.test_batch_size,
                                                     num_buckets=args.num_buckets,
                                                     ratio=args.bucket_ratio,
                                                     shuffle=False,
                                                     use_average_length=use_average_length,
                                                     bucket_scheme=bucket_scheme)
    logging.info('Test Batch Sampler:\n%s', test_batch_sampler.stats())
    test_data_loader = gluon.data.DataLoader(data_test,
                                             batch_sampler=test_batch_sampler,
                                             batchify_fn=test_batchify_fn,
                                             num_workers=num_workers)
    return train_data_loader, val_data_loader, test_data_loader
704545
def construct_channel(self, **kwargs):
        
        channel = self.get_channel(**kwargs)
        json_tree_path = self.get_json_tree_path(**kwargs)
        json_tree = read_tree_from_json(json_tree_path)
        build_tree_from_json(channel, json_tree['children'])
        raise_for_invalid_channel(channel)
        return channel
32787
void readOTreeSet(int iSize, ObjectInputStream s, V defaultVal) throws java.io.IOException, ClassNotFoundException {
    buildFromSorted(iSize, null, s, defaultVal);
  }
1331889
public static void main(String[] args) {
    Timing.startTime();
    Treebank treebank = new DiskTreebank(new TreeReaderFactory() {
      public TreeReader newTreeReader(Reader in) {
        return new PennTreeReader(in);
      }
    });
    Treebank treebank2 = new MemoryTreebank(new TreeReaderFactory() {
      public TreeReader newTreeReader(Reader in) {
        return new PennTreeReader(in);
      }
    });
    treebank.loadPath(args[0]);
    treebank2.loadPath(args[0]);
    CompositeTreebank c = new CompositeTreebank(treebank, treebank2);
    Timing.endTime();
    TreeTransformer myTransformer = new MyTreeTransformer();
    TreeTransformer myTransformer2 = new MyTreeTransformer2();
    TreeTransformer myTransformer3 = new MyTreeTransformer3();
    Treebank tf1 = c.transform(myTransformer).transform(myTransformer2).transform(myTransformer3);
    Treebank tf2 = new TransformingTreebank(new TransformingTreebank(new TransformingTreebank(c, myTransformer), myTransformer2), myTransformer3);
    TreeTransformer[] tta = { myTransformer, myTransformer2, myTransformer3 };
    TreeTransformer tt3 = new CompositeTreeTransformer(Arrays.asList(tta));
    Treebank tf3 = c.transform(tt3);

    System.out.println("-------------------------");
    System.out.println("COMPOSITE (DISK THEN MEMORY REPEATED VERSION OF) INPUT TREEBANK");
    System.out.println(c);
    System.out.println("-------------------------");
    System.out.println("SLOWLY TRANSFORMED TREEBANK, USING TransformingTreebank() CONSTRUCTOR");
    Treebank tx1 = new TransformingTreebank(c, myTransformer);
    System.out.println(tx1);
    System.out.println("-----");
    Treebank tx2 = new TransformingTreebank(tx1, myTransformer2);
    System.out.println(tx2);
    System.out.println("-----");
    Treebank tx3 = new TransformingTreebank(tx2, myTransformer3);
    System.out.println(tx3);
    System.out.println("-------------------------");
    System.out.println("TRANSFORMED TREEBANK, USING Treebank.transform()");
    System.out.println(tf1);
    System.out.println("-------------------------");
    System.out.println("PRINTING AGAIN TRANSFORMED TREEBANK, USING Treebank.transform()");
    System.out.println(tf1);
    System.out.println("-------------------------");
    System.out.println("TRANSFORMED TREEBANK, USING TransformingTreebank() CONSTRUCTOR");
    System.out.println(tf2);
    System.out.println("-------------------------");
    System.out.println("TRANSFORMED TREEBANK, USING CompositeTreeTransformer");
    System.out.println(tf3);
    System.out.println("-------------------------");
    System.out.println("COMPOSITE (DISK THEN MEMORY REPEATED VERSION OF) INPUT TREEBANK");
    System.out.println(c);
    System.out.println("-------------------------");
  }
179962
def data_to_stream(self, data_element, stream):
        
        generator = \
            self._make_representation_generator(stream, self.resource_class,
                                                self._mapping)
        generator.run(data_element)
7835
def dataset_generator(filepath,
                      dataset,
                      chunk_size=1,
                      start_idx=None,
                      end_idx=None):
  """"""
  encoder = dna_encoder.DNAEncoder(chunk_size=chunk_size)
  with h5py.File(filepath, "r") as h5_file:
    # Get input keys from h5_file
    src_keys = [s % dataset for s in ["%s_in", "%s_na", "%s_out"]]
    src_values = [h5_file[k] for k in src_keys]
    inp_data, mask_data, out_data = src_values
    assert len(set([v.len() for v in src_values])) == 1

    if start_idx is None:
      start_idx = 0
    if end_idx is None:
      end_idx = inp_data.len()

    for i in range(start_idx, end_idx):
      if i % 100 == 0:
        print("Generating example %d for %s" % (i, dataset))
      inputs, mask, outputs = inp_data[i], mask_data[i], out_data[i]
      ex_dict = to_example_dict(encoder, inputs, mask, outputs)
      # Original data has one output for every 128 input bases. Ensure that the
      # ratio has been maintained given the chunk size and removing EOS.
      assert (len(ex_dict["inputs"]) - 1) == ((
          128 // chunk_size) * ex_dict["targets_shape"][0])
      yield ex_dict
652828
def populate(self, node_generator):
        
        for data in node_generator:
            hashed = hash_byte_array(data)
            self.tree.add_leaf(hashed)
1276568
void expectGeneratorSupertype(Node n, JSType type, String msg) {
    if (!getNativeType(GENERATOR_TYPE).isSubtypeOf(type)) {
      mismatch(n, msg, type, GENERATOR_TYPE);
    }
  }
1141885
def generate(
        cls: Type[T], data: Generic, name: str = None, *, recursive: bool = True
    ) -> T:
        """"""
        if name is None:
            name = cls.__name__
        kls = make_class(name, {k: ib(default=None) for k in data}, bases=(cls,))
        data = {
            k: (
                cls.generate(v, k.title())
                if recursive and isinstance(v, Mapping)
                else v
            )
            for k, v in data.items()
        }
        return kls(**data)
1089654
void readTreeSet(int size, java.io.ObjectInputStream s, V defaultVal)
        throws java.io.IOException, ClassNotFoundException {
        buildFromSorted(size, null, s, defaultVal);
    }