Builds lexical chains, as an adjacency matrix,
        using a disambiguated term-concept map.
774695
def _lexical_chains(self, doc, term_concept_map):
        
        concepts = list({c for c in term_concept_map.values()})

        # Build an adjacency matrix for the graph
        # Using the encoding:
        # 1 = identity/synonymy, 2 = hypernymy/hyponymy, 3 = meronymy, 0 = no edge
        n_cons = len(concepts)
        adj_mat = np.zeros((n_cons, n_cons))

        for i, c in enumerate(concepts):
            # TO DO can only do i >= j since the graph is undirected
            for j, c_ in enumerate(concepts):
                edge = 0
                if c == c_:
                    edge = 1
                # TO DO when should simulate root be True?
                elif c_ in c._shortest_hypernym_paths(simulate_root=False).keys():
                    edge = 2
                elif c in c_._shortest_hypernym_paths(simulate_root=False).keys():
                    edge = 2
                elif c_ in c.member_meronyms() + c.part_meronyms() + c.substance_meronyms():
                    edge = 3
                elif c in c_.member_meronyms() + c_.part_meronyms() + c_.substance_meronyms():
                    edge = 3

                adj_mat[i,j] = edge

        # Group connected concepts by labels
        concept_labels = connected_components(adj_mat, directed=False)[1]
        lexical_chains = [([], []) for i in range(max(concept_labels) + 1)]
        for i, concept in enumerate(concepts):
            label = concept_labels[i]
            lexical_chains[label][0].append(concept)
            lexical_chains[label][1].append(i)

        # Return the lexical chains as (concept list, adjacency sub-matrix) tuples
        return [(chain, adj_mat[indices][:,indices]) for chain, indices in lexical_chains]
784605
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'text' in _dict:
            args['text'] = _dict.get('text')
        if 'keywords' in _dict:
            args['keywords'] = [
                SemanticRolesKeyword._from_dict(x)
                for x in (_dict.get('keywords'))
            ]
        return cls(**args)
1098524
public void buildTaxonomy() {
        
        // Determine if a full or incremental calculation is required
        if(!hasBeenIncrementallyClassified) {
            buildTaxonomyConcurrent();
            //buildTaxonomySequential();
        } else {
            final IConceptMap<IConceptSet> newConceptSubs = getNewSubsumptions();
            final IConceptMap<IConceptSet> affectedConceptSubs = getAffectedSubsumptions();
            
            // 1. Keep only the subsumptions that involve real atomic concepts
            IConceptMap<IConceptSet> allNew = new SparseConceptMap<IConceptSet>(newConceptSubs.size());

            IConceptMap<IConceptSet> allAffected = new SparseConceptMap<IConceptSet>(newConceptSubs.size());

            for (IntIterator itr = newConceptSubs.keyIterator(); itr.hasNext();) {
                final int x = itr.next();
                if (!factory.isVirtualConcept(x)) {
                    IConceptSet set = new SparseConceptHashSet();
                    allNew.put(x, set);
                    for (IntIterator it = newConceptSubs.get(x).iterator(); it.hasNext();) {
                        int next = it.next();
                        if (!factory.isVirtualConcept(next)) {
                            set.add(next);
                        }
                    }
                }
            }

            for (IntIterator itr = affectedConceptSubs.keyIterator(); itr.hasNext();) {
                final int x = itr.next();
                if (!factory.isVirtualConcept(x)) {
                    IConceptSet set = new SparseConceptHashSet();
                    allAffected.put(x, set);
                    for (IntIterator it = affectedConceptSubs.get(x).iterator(); it.hasNext();) {
                        int next = it.next();
                        if (!factory.isVirtualConcept(next)) {
                            set.add(next);
                        }
                    }
                }
            }
            
            // 2. Create nodes for new concepts and connect to node hierarchy
            // a. First create the nodes and add to index
            for (IntIterator itr = allNew.keyIterator(); itr.hasNext();) {
                final String key = factory.lookupConceptId(itr.next()).toString();
                Node cn = new Node();
                cn.getEquivalentConcepts().add(key);
                conceptNodeIndex.put(key, cn);
            }

            // b. Now connect the nodes disregarding redundant connections
            Node bottomNode = conceptNodeIndex.get(au.csiro.ontology.model.NamedConcept.BOTTOM);
            for (IntIterator itr = allNew.keyIterator(); itr.hasNext();) {
                int id = itr.next();
                final String key = factory.lookupConceptId(id).toString();
                Node cn = conceptNodeIndex.get(key);
                IConceptSet parents = allNew.get(id);
                for (IntIterator itr2 = parents.iterator(); itr2.hasNext();) {
                    // Create a connection to each parent
                    int parentId = itr2.next();
                    if (parentId == id)
                        continue;
                    Node parent = conceptNodeIndex.get(factory.lookupConceptId(parentId));
                    cn.getParents().add(parent);
                    parent.getChildren().add(cn);
                    // All nodes that get new children and are connected to BOTTOM
                    // must be disconnected
                    if (parent.getChildren().contains(bottomNode)) {
                        parent.getChildren().remove(bottomNode);
                        bottomNode.getParents().remove(parent);
                    }
                }
            }
            
            Set<Integer> toRemoveFromAffected = new HashSet<Integer>();
            for (IntIterator itr = allAffected.keyIterator(); itr.hasNext();) {
                final int id = itr.next();
                final String key = factory.lookupConceptId(id).toString();
                Node cn = conceptNodeIndex.get(key);
                IConceptSet parents = allAffected.get(id);
                
                if(parents.contains(IFactory.BOTTOM_CONCEPT)) {
                    // Special case - bottom is parent
                    
                    // a. add equivalents to bottom node
                    bottomNode.getEquivalentConcepts().addAll(cn.getEquivalentConcepts());
                    
                    Set<Node> tempParents = cn.getParents();
                    Set<Node> tempChildren = cn.getChildren();
                    
                    // b. reconnect parents to children
                    for(Node parent : tempParents) {
                        parent.getChildren().remove(cn);
                        parent.getChildren().addAll(tempChildren);
                    }
                    
                    for(Node child : tempChildren) {
                        child.getParents().remove(cn);
                        child.getParents().addAll(tempParents);
                    }
                    
                    for(String k : cn.getEquivalentConcepts()) {
                        conceptNodeIndex.remove(k);
                        conceptNodeIndex.put(key, bottomNode);
                    }
                    toRemoveFromAffected.add(id);
                } else {
                    for (IntIterator itr2 = parents.iterator(); itr2.hasNext();) {
                        // Create a connection to each parent
                        int parentId = itr2.next();
                        if (parentId == id)
                            continue;
                        Node parent = conceptNodeIndex.get(factory.lookupConceptId(parentId));
                        cn.getParents().add(parent);
                        parent.getChildren().add(cn);
                        // All nodes that get new children and are connected to BOTTOM must be disconnected
                        if (parent.getChildren().contains(bottomNode)) {
                            parent.getChildren().remove(bottomNode);
                            bottomNode.getParents().remove(parent);
                        }
                    }
                }
            }
            
            for(Integer i : toRemoveFromAffected) {
                allAffected.remove(i.intValue());
                allNew.remove(i.intValue());
            }

            // 3. Connect new nodes without parents to TOP
            Node topNode = conceptNodeIndex.get(au.csiro.ontology.model.NamedConcept.TOP);

            for (IntIterator itr = allNew.keyIterator(); itr.hasNext();) {
                final String key = factory.lookupConceptId(itr.next()).toString();
                Node cn = conceptNodeIndex.get(key);
                if (cn.getParents().isEmpty()) {
                    cn.getParents().add(topNode);
                    topNode.getChildren().add(cn);
                }
            }

            // 4. Fix connections for new and affected concepts
            // a. Check for equivalents
            Set<Pair> pairsToMerge = new HashSet<Pair>();
            for (IntIterator itr = allNew.keyIterator(); itr.hasNext();) {
                final String key = factory.lookupConceptId(itr.next()).toString();
                Node cn = conceptNodeIndex.get(key);
                for (Node parent : cn.getParents()) {
                    if (parent.getParents().contains(cn)) {
                        pairsToMerge.add(new Pair(cn, parent));
                    }
                }
            }
            for (IntIterator itr = allAffected.keyIterator(); itr.hasNext();) {
                final String key = factory.lookupConceptId(itr.next()).toString();
                Node cn = conceptNodeIndex.get(key);
                for (Node parent : cn.getParents()) {
                    if (parent.getParents().contains(cn)) {
                        pairsToMerge.add(new Pair(cn, parent));
                    }
                }
            }

            Set<Node> affectedByMerge = new HashSet<Node>();

            // Merge equivalents
            for (Pair p : pairsToMerge) {
                Node cn1 = p.getA();
                Node cn2 = p.getB();

                affectedByMerge.addAll(cn1.getChildren());
                affectedByMerge.addAll(cn2.getChildren());

                // Merge into cn1 - remove cn2 from index and replace with cn1
                for (String n : cn2.getEquivalentConcepts()) {
                    conceptNodeIndex.put(n, cn1);
                }

                cn1.getEquivalentConcepts().addAll(cn2.getEquivalentConcepts());

                // Remove relationships between merged concepts
                cn1.getParents().remove(cn2);
                cn2.getChildren().remove(cn1);
                cn2.getParents().remove(cn1);
                cn1.getChildren().remove(cn2);

                // Taxonomy is bidirectional
                cn1.getParents().addAll(cn2.getParents());
                for (Node parent : cn2.getParents()) {
                    parent.getChildren().remove(cn2);
                    parent.getChildren().add(cn1);
                }
                cn1.getChildren().addAll(cn2.getChildren());
                for (Node child : cn2.getChildren()) {
                    child.getParents().remove(cn2);
                    child.getParents().add(cn1);
                }

                cn2 = null; // nothing should reference cn2 now
            }

            // b. Fix all new and affected nodes
            Set<Node> all = new HashSet<Node>();
            for (IntIterator it = allNew.keyIterator(); it.hasNext();) {
                all.add(conceptNodeIndex.get(factory.lookupConceptId(it.next())));
            }

            for (IntIterator it = allAffected.keyIterator(); it.hasNext();) {
                all.add(conceptNodeIndex.get(factory.lookupConceptId(it.next())));
            }

            for (Node cn : affectedByMerge) {
                all.add(cn);
            }

            // Add also the children of the affected nodes
            Set<Node> childrenToAdd = new HashSet<Node>();
            for (Node cn : all) {
                for (Node ccn : cn.getChildren()) {
                    if (ccn.equals(bottomNode))
                        continue;
                    childrenToAdd.add(ccn);
                }
            }
            all.addAll(childrenToAdd);

            // Find redundant relationships
            for (Node cn : all) {
                Set<Node> ps = cn.getParents();

                Object[] parents = ps.toArray(new Object[ps.size()]);
                Set<Node> toRemove = new HashSet<Node>();
                for (int i = 0; i < parents.length; i++) {
                    for (int j = i + 1; j < parents.length; j++) {
                        if (isChild((Node)parents[j], (Node)parents[i])) {
                            toRemove.add((Node)parents[i]);
                            continue;
                        }
                        if (isChild((Node)parents[i], (Node)parents[j])) {
                            toRemove.add((Node)parents[j]);
                            continue;
                        }
                    }
                }
                for (Node tr : toRemove) {
                    cn.getParents().remove(tr);
                    tr.getChildren().remove(cn);
                }
            }

            // 5. Connect bottom to new and affected concepts with no children
            for (IntIterator itr = allNew.keyIterator(); itr.hasNext();) {
                final int key = itr.next();
                Node cn = conceptNodeIndex.get(factory.lookupConceptId(key));
                if (cn.getChildren().isEmpty()) {
                    cn.getChildren().add(bottomNode);
                    bottomNode.getParents().add(cn);
                }
            }
            for (IntIterator itr = allAffected.keyIterator(); itr.hasNext();) {
                final int key = itr.next();
                Node cn = conceptNodeIndex.get(factory.lookupConceptId(key));
                if (cn.getChildren().isEmpty()) {
                    cn.getChildren().add(bottomNode);
                    bottomNode.getParents().add(cn);
                }
            }

            // 6. Connect the top node to new and affected concepts with no parents
            for (IntIterator itr = allNew.keyIterator(); itr.hasNext();) {
                final int key = itr.next();
                Node cn = conceptNodeIndex.get(factory.lookupConceptId(key));
                if (cn.getParents().isEmpty()) {
                    cn.getParents().add(topNode);
                    topNode.getChildren().add(cn);
                }
            }
            for (IntIterator itr = allAffected.keyIterator(); itr.hasNext();) {
                final int key = itr.next();
                Node cn = conceptNodeIndex.get(factory.lookupConceptId(key));
                if (cn.getParents().isEmpty()) {
                    cn.getParents().add(topNode);
                    topNode.getChildren().add(cn);
                }
            }
        }  
    }
681906
def _buildDict(self):
        '''
        
        '''
        lexDict = {}
        with io.open(self.islePath, "r", encoding='utf-8') as fd:
            wordList = [line.rstrip('\n') for line in fd]
        
        for row in wordList:
            word, pronunciation = row.split(" ", 1)
            word, extraInfo = word.split("(", 1)
            
            extraInfo = extraInfo.replace(")", "")
            extraInfoList = [segment for segment in extraInfo.split(",")
                             if ("_" not in segment and "+" not in segment and
                                 ':' not in segment and segment != '')]
            
            lexDict.setdefault(word, [])
            lexDict[word].append((pronunciation, extraInfoList))
        
        return lexDict
1005752
def _get_augmented_label_matrix(self, L, higher_order=False):
        
        # Create a helper data structure which maps cliques (as tuples of member
        # sources) --> {start_index, end_index, maximal_cliques}, where
        # the last value is a set of indices in this data structure
        self.c_data = {}
        for i in range(self.m):
            self.c_data[i] = {
                "start_index": i * self.k,
                "end_index": (i + 1) * self.k,
                "max_cliques": set(
                    [
                        j
                        for j in self.c_tree.nodes()
                        if i in self.c_tree.node[j]["members"]
                    ]
                ),
            }

        L_ind = self._create_L_ind(L)

        # Get the higher-order clique statistics based on the clique tree
        # First, iterate over the maximal cliques (nodes of c_tree) and
        # separator sets (edges of c_tree)
        if higher_order:
            L_aug = np.copy(L_ind)
            for item in chain(self.c_tree.nodes(), self.c_tree.edges()):
                if isinstance(item, int):
                    C = self.c_tree.node[item]
                    C_type = "node"
                elif isinstance(item, tuple):
                    C = self.c_tree[item[0]][item[1]]
                    C_type = "edge"
                else:
                    raise ValueError(item)
                members = list(C["members"])
                nc = len(members)

                # If a unary maximal clique, just store its existing index
                if nc == 1:
                    C["start_index"] = members[0] * self.k
                    C["end_index"] = (members[0] + 1) * self.k

                # Else add one column for each possible value
                else:
                    L_C = np.ones((self.n, self.k ** nc))
                    for i, vals in enumerate(product(range(self.k), repeat=nc)):
                        for j, v in enumerate(vals):
                            L_C[:, i] *= L_ind[:, members[j] * self.k + v]

                    # Add to L_aug and store the indices
                    if L_aug is not None:
                        C["start_index"] = L_aug.shape[1]
                        C["end_index"] = L_aug.shape[1] + L_C.shape[1]
                        L_aug = np.hstack([L_aug, L_C])
                    else:
                        C["start_index"] = 0
                        C["end_index"] = L_C.shape[1]
                        L_aug = L_C

                    # Add to self.c_data as well
                    id = tuple(members) if len(members) > 1 else members[0]
                    self.c_data[id] = {
                        "start_index": C["start_index"],
                        "end_index": C["end_index"],
                        "max_cliques": set([item]) if C_type == "node" else set(item),
                    }
            return L_aug
        else:
            return L_ind
955146
def conceptscheme_from_uri(conceptscheme_uri, **kwargs):
    '''
    
    '''

    # get the conceptscheme
    # ensure it only ends in one slash
    conceptscheme_uri = conceptscheme_uri.strip('/') + '/'
    s = kwargs.get('session', requests.Session())
    graph = uri_to_graph('%s.rdf' % (conceptscheme_uri), session=s)

    notes = []
    labels = []
    if graph is not False:
        for s, p, o in graph.triples((URIRef(conceptscheme_uri), RDFS.label, None)):
            label = Label(o.toPython(), "prefLabel", 'en')
            labels.append(label)

    conceptscheme = ConceptScheme(
        conceptscheme_uri,
        labels=labels,
        notes=notes
    )
    return conceptscheme
175463
def build(term_to_index_dict):
		'''
		
		'''
		idxstore = IndexStore()
		idxstore._val2i = term_to_index_dict
		idxstore._next_i = len(term_to_index_dict)
		idxstore._i2val = [None for _ in range(idxstore._next_i)]
		for term, idx in idxstore._val2i.items():
			idxstore._i2val[idx] = term
		return idxstore
60668
public void buildVocab() {


        val constructor = new VocabConstructor.Builder<T>().addSource(iterator, minWordFrequency)
                        .setTargetVocabCache(vocab).fetchLabels(trainSequenceVectors).setStopWords(stopWords)
                        .enableScavenger(enableScavenger).setEntriesLimit(vocabLimit)
                        .allowParallelTokenization(configuration.isAllowParallelTokenization())
                        .setUnk(useUnknown && unknownElement != null ? unknownElement : null).build();

        if (existingModel != null && lookupTable instanceof InMemoryLookupTable
                        && existingModel.lookupTable() instanceof InMemoryLookupTable) {
            log.info("Merging existing vocabulary into the current one...");
            /*
                if we have existing model defined, we're forced to fetch labels only.
                the rest of vocabulary & weights should be transferred from existing model
             */

            constructor.buildMergedVocabulary(existingModel, true);

            /*
                Now we have vocab transferred, and we should transfer syn0 values into lookup table
             */
            ((InMemoryLookupTable<VocabWord>) lookupTable)
                            .consume((InMemoryLookupTable<VocabWord>) existingModel.lookupTable());
        } else {
            log.info("Starting vocabulary building...");
            // if we don't have existing model defined, we just build vocabulary


            constructor.buildJointVocabulary(false, true);

            /*
            if (useUnknown && unknownElement != null && !vocab.containsWord(unknownElement.getLabel())) {
                log.info("Adding UNK element...");
                unknownElement.setSpecial(true);
                unknownElement.markAsLabel(false);
                unknownElement.setIndex(vocab.numWords());
                vocab.addToken(unknownElement);
            }
            */


            // check for malformed inputs. if numWords/numSentences ratio is huge, then user is passing something weird
            if (vocab.numWords() / constructor.getNumberOfSequences() > 1000) {
                log.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
                log.warn("!                                                                                       !");
                log.warn("! Your input looks malformed: number of sentences is too low, model accuracy may suffer !");
                log.warn("!                                                                                       !");
                log.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
            }
        }


    }
163520
protected void addContextTerms(SparseDoubleVector meaning,
                                   Queue<String> words,
                                   int distance) {
        // Iterate through each of the context words.
        for (String term : words) {
            if (!term.equals(IteratorFactory.EMPTY_TOKEN)) {
                // Ignore any features that have no valid dimension.
                int dimension = basis.getDimension(term);
                if (dimension == -1)
                    continue;

                // Add the feature to the context vector and increase the
                // distance from the focus word.
                meaning.set(dimension, weighting.weight(distance, windowSize));
                ++distance;
            }
        }
    }
384046
def tcmap(self):
        '''  '''
        tcmap = dd(list)
        for concept in self.__concept_map.values():
            for w in concept.tokens:
                tcmap[w].append(concept)
        return tcmap
1176880
def join(self, featuresets):
        """"""
        concepts = (f.concept for f in featuresets)
        join = self.lattice.join(concepts)
        return self._featuresets[join.index]
807814
private Graph<Module> buildGraph(Map<Module, Set<Module>> edges) {
        Graph.Builder<Module> builder = new Graph.Builder<>();
        Set<Module> visited = new HashSet<>();
        Deque<Module> deque = new LinkedList<>();
        edges.entrySet().stream().forEach(e -> {
            Module m = e.getKey();
            deque.add(m);
            e.getValue().stream().forEach(v -> {
                deque.add(v);
                builder.addEdge(m, v);
            });
        });

        // read requires transitive from ModuleDescriptor
        Module source;
        while ((source = deque.poll()) != null) {
            if (visited.contains(source))
                continue;

            visited.add(source);
            builder.addNode(source);
            Module from = source;
            requiresTransitive(from).forEach(m -> {
                deque.add(m);
                builder.addEdge(from, m);
            });
        }
        return builder.build();
    }
348891
def build(self, words):
        """"""
        words = [self._normalize(tokens) for tokens in words]
        self._dawg = dawg.CompletionDAWG(words)
        self._loaded_model = True
699998
private void clusterTerm(String senseName,
                             List<SparseDoubleVector> contextSet,
                             Properties props) {
        // Convert the data points to a sparse matrix.
        SparseMatrix contexts = Matrices.asSparseMatrix(contextSet);

        // Cluster the context set.
        LOG.info("Clustering term: " + senseName);
        Assignments assignments = (numClusters > 0) 
            ? clustering.cluster(contexts, numClusters, props)
            : clustering.cluster(contexts, props);
        LOG.info("Finished clustering term: " + senseName);

        SparseDoubleVector[] centroids = assignments.getSparseCentroids();

        // Add the centroids to the splitSenses map.
        for (int index = 0; index < centroids.length; ++index) {
            String sense = (index > 0)
                    ? senseName + "-" + index
                    : senseName;
            wordSpace.put(sense, centroids[index]);
        }

        LOG.info("Finished creating centroids for term: " + senseName);

        // Empty out the stored contexts to free up memory for later processes.
        contextSet.clear();

        // If the reporter is null, avoid making any report.
        if (reporter == null)
            return;

        // Generate the secondary context labels for each data point.
        String[] contextLabels = reporter.contextLabels(senseName);
        if (contextLabels.length == 0)
            return;

        LOG.info("Making assignment report: " + senseName);
        // Report the assignments for each clustered data point.  Note that some
        // data points might not have been clustered (Cluto based clustering
        // does this on occasion) so we must check for the number of assignments
        // first.
        for (int i = 0; i < assignments.size(); ++i)
            if (assignments.get(i).assignments().length > 0)
                reporter.updateAssignment(senseName, contextLabels[i],
                                          assignments.get(i).assignments()[0]);
        LOG.info("Finished making assignment report: " + senseName);
    }
1355750
def concept_cuts(direction, node_indices, node_labels=None):
    """"""
    for partition in mip_partitions(node_indices, node_indices):
        yield KCut(direction, partition, node_labels)
232303
def build_graph(lstm_depth=3, batch_size=32, num_embedding=32, num_hidden=128,
                truncation_length=64, sample_length=1000, max_grad_norm=5,
                initial_learning_rate=0.1, reduce_learning_rate_multiplier=0.1,
                optimizer_epsilon=0.01):
  """"""

  # Get datasets.
  dataset_train = dataset_shakespeare.TinyShakespeareDataset(
      num_steps=truncation_length,
      batch_size=batch_size,
      subset="train",
      random=True,
      name="shake_train")

  dataset_valid = dataset_shakespeare.TinyShakespeareDataset(
      num_steps=truncation_length,
      batch_size=batch_size,
      subset="valid",
      random=False,
      name="shake_valid")

  dataset_test = dataset_shakespeare.TinyShakespeareDataset(
      num_steps=truncation_length,
      batch_size=batch_size,
      subset="test",
      random=False,
      name="shake_test")

  # Define model.
  model = TextModel(
      num_embedding=num_embedding,
      num_hidden=num_hidden,
      lstm_depth=lstm_depth,
      output_size=dataset_valid.vocab_size,
      use_dynamic_rnn=True,
      use_skip_connections=True)

  # Get the training loss.
  train_input_sequence, train_target_sequence = dataset_train()
  train_output_sequence_logits, train_final_state = model(train_input_sequence)  # pylint: disable=not-callable
  train_loss = dataset_train.cost(train_output_sequence_logits,
                                  train_target_sequence)

  # Get the validation loss.
  valid_input_sequence, valid_target_sequence = dataset_valid()
  valid_output_sequence_logits, _ = model(valid_input_sequence)  # pylint: disable=not-callable
  valid_loss = dataset_valid.cost(valid_output_sequence_logits,
                                  valid_target_sequence)

  # Get the test loss.
  test_input_sequence, test_target_sequence = dataset_test()
  test_output_sequence_logits, _ = model(test_input_sequence)  # pylint: disable=not-callable
  test_loss = dataset_test.cost(test_output_sequence_logits,
                                test_target_sequence)

  # Build graph to sample some strings during training.
  initial_logits = train_output_sequence_logits[truncation_length - 1]
  train_generated_string = model.generate_string(
      initial_logits=initial_logits,
      initial_state=train_final_state,
      sequence_length=sample_length)

  # Set up global norm clipping of gradients.
  trainable_variables = tf.trainable_variables()
  grads, _ = tf.clip_by_global_norm(
      tf.gradients(train_loss, trainable_variables), max_grad_norm)

  # Get learning rate and define annealing.
  learning_rate = tf.get_variable(
      "learning_rate",
      shape=[],
      dtype=tf.float32,
      initializer=tf.constant_initializer(initial_learning_rate),
      trainable=False)
  reduce_learning_rate = learning_rate.assign(
      learning_rate * reduce_learning_rate_multiplier)

  # Get training step counter.
  global_step = tf.get_variable(
      name="global_step",
      shape=[],
      dtype=tf.int64,
      initializer=tf.zeros_initializer(),
      trainable=False,
      collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP])

  # Define optimizer and training step.
  optimizer = tf.train.AdamOptimizer(
      learning_rate, epsilon=optimizer_epsilon)
  train_step = optimizer.apply_gradients(
      zip(grads, trainable_variables),
      global_step=global_step)

  graph_tensors = {
      "train_loss": train_loss,
      "valid_loss": valid_loss,
      "test_loss": test_loss,
      "train_generated_string": train_generated_string,
      "reduce_learning_rate": reduce_learning_rate,
      "global_step": global_step,
      "train_step": train_step
  }

  # Return dataset_train for translation to human readable text.
  return graph_tensors, dataset_train
168703
def setup_data_split(X, y, tokenizer, proc_data_dir, **kwargs):
    
    X_train, X_val, X_test, y_train, y_val, y_test = split_data(X, y)

    # only build vocabulary on training data
    tokenizer.build_vocab(X_train)

    process_save(X_train, y_train, tokenizer, path.join(
        proc_data_dir, 'train.bin'), train=True, **kwargs)
    process_save(X_val, y_val, tokenizer, path.join(
        proc_data_dir, 'val.bin'), **kwargs)
    process_save(X_test, y_test, tokenizer, path.join(
        proc_data_dir, 'test.bin'), **kwargs)