Train a classifier from documents.

@param docs
An objectbank representation of documents. Changed this type from
ObjectBank to Collection for generality (mihai)
501083
@Override
  public void train(Collection<List<IN>> docs, DocumentReaderAndWriter<IN> readerAndWriter) {
    Timing timer = new Timing();
    timer.start();
    makeAnswerArraysAndTagIndex(docs);
    long elapsedMs = timer.stop();
    System.err.println("Time to convert docs to feature indices: " + Timing.toSecondsString(elapsedMs) + " seconds");
    if (flags.exportFeatures != null) {
      timer.start();
      CRFFeatureExporter<IN> featureExporter = new CRFFeatureExporter<IN>(this);
      featureExporter.printFeatures(flags.exportFeatures, docs);
      elapsedMs = timer.stop();
      System.err.println("Time to export features: " + Timing.toSecondsString(elapsedMs) + " seconds");
    }
    for (int i = 0; i <= flags.numTimesPruneFeatures; i++) {
      timer.start();
      Pair<int[][][][], int[][]> dataAndLabels = documentsToDataAndLabels(docs);
      elapsedMs = timer.stop();
      System.err.println("Time to convert docs to data/labels: " + Timing.toSecondsString(elapsedMs) + " seconds");

      Evaluator[] evaluators = null;
      if (flags.evaluateIters > 0) {
        List<Evaluator> evaluatorList = new ArrayList<Evaluator>();
        evaluatorList.add(new MemoryEvaluator());
        if (flags.evaluateTrain) {
          CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Train set", this);
          List<Pair<int[][][], int[]>> trainDataAndLabels = new ArrayList<Pair<int[][][], int[]>>();
          for (int j = 0; j < dataAndLabels.first().length; j++) {
            Pair<int[][][], int[]> p = new Pair<int[][][], int[]>(dataAndLabels.first()[j], dataAndLabels.second()[j]);
            trainDataAndLabels.add(p);
          }
          crfEvaluator.setTestData(docs, trainDataAndLabels);
          crfEvaluator.setEvalCmd(flags.evalCmd);
          evaluatorList.add(crfEvaluator);
        }
        if (flags.testFile != null) {
          CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set (" + flags.testFile + ")",
              this);
          ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter);
          List<Pair<int[][][], int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank);
          crfEvaluator.setTestData(testObjBank, testDataAndLabels);
          crfEvaluator.setEvalCmd(flags.evalCmd);
          evaluatorList.add(crfEvaluator);
        }
        if (flags.testFiles != null) {
          String[] testFiles = flags.testFiles.split(",");
          for (String testFile : testFiles) {
            CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set ("
                + testFile + ")", this);
            ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(testFile, readerAndWriter);
            List<Pair<int[][][], int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank);
            crfEvaluator.setTestData(testObjBank, testDataAndLabels);
            crfEvaluator.setEvalCmd(flags.evalCmd);
            evaluatorList.add(crfEvaluator);
          }
        }
        evaluators = new Evaluator[evaluatorList.size()];
        evaluatorList.toArray(evaluators);
      }

      if (flags.numTimesPruneFeatures == i) {
        docs = null; // hopefully saves memory
      }
      // save feature index to disk and read in later
      File featIndexFile = null;

      if (flags.saveFeatureIndexToDisk) {
        try {
          System.err.println("Writing feature index to temporary file.");
          featIndexFile = IOUtils.writeObjectToTempFile(featureIndex, "featIndex" + i + ".tmp");
          featureIndex = null;
        } catch (IOException e) {
          throw new RuntimeException("Could not open temporary feature index file for writing.");
        }
      }

      // first index is the number of the document
      // second index is position in the document also the index of the
      // clique/factor table
      // third index is the number of elements in the clique/window thase
      // features are for (starting with last element)
      // fourth index is position of the feature in the array that holds them
      // element in data[i][j][k][m] is the index of the mth feature occurring
      // in position k of the jth clique of the ith document
      int[][][][] data = dataAndLabels.first();
      // first index is the number of the document
      // second index is the position in the document
      // element in labels[i][j] is the index of the correct label (if it
      // exists) at position j in document i
      int[][] labels = dataAndLabels.second();

      if (flags.loadProcessedData != null) {
        List processedData = loadProcessedData(flags.loadProcessedData);
        if (processedData != null) {
          // enlarge the data and labels array
          int[][][][] allData = new int[data.length + processedData.size()][][][];
          int[][] allLabels = new int[labels.length + processedData.size()][];
          System.arraycopy(data, 0, allData, 0, data.length);
          System.arraycopy(labels, 0, allLabels, 0, labels.length);
          // add to the data and labels array
          addProcessedData(processedData, allData, allLabels, data.length);
          data = allData;
          labels = allLabels;
        }
      }

      if (flags.useFloat) {
        CRFLogConditionalObjectiveFloatFunction func = new CRFLogConditionalObjectiveFloatFunction(data, labels,
            featureIndex, windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
        func.crfType = flags.crfType;

        QNMinimizer minimizer;
        if (flags.interimOutputFreq != 0) {
          FloatFunction monitor = new ResultStoringFloatMonitor(flags.interimOutputFreq, flags.serializeTo);
          minimizer = new QNMinimizer(monitor);
        } else {
          minimizer = new QNMinimizer();
        }

        if (i == 0) {
          minimizer.setM(flags.QNsize);
        } else {
          minimizer.setM(flags.QNsize2);
        }

        float[] initialWeights;
        if (flags.initialWeights == null) {
          initialWeights = func.initial();
        } else {
          try {
            System.err.println("Reading initial weights from file " + flags.initialWeights);
            DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(
                flags.initialWeights))));
            initialWeights = Convert.readFloatArr(dis);
          } catch (IOException e) {
            throw new RuntimeException("Could not read from float initial weight file " + flags.initialWeights);
          }
        }
        System.err.println("numWeights: " + initialWeights.length);
        float[] weights = minimizer.minimize(func, (float) flags.tolerance, initialWeights);
        this.weights = ArrayMath.floatArrayToDoubleArray(func.to2D(weights));

      } else {

        /*double[] estimate = null;

        if(flags.estimateInitial){
          int[][][][] approxData = new int[data.length/100][][][];
          int[][] approxLabels = new int[data.length/100][];

          Random generator = new Random(1);
          for(int k=0;k<approxData.length; k++){
            int thisInd = generator.nextInt(data.length);
            approxData[k] = data[ thisInd];
            approxLabels[k] = labels[ thisInd];
          }

          CRFLogConditionalObjectiveFunction approxFunc = new CRFLogConditionalObjectiveFunction(approxData, approxLabels, featureIndex, windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
          approxFunc.crfType = flags.crfType;
          minimizer = new QNMinimizer(10);

          if (flags.initialWeights == null) {
            estimate = approxFunc.initial();
          } else {
            try {
              System.err.println("Reading initial weights from file " + flags.initialWeights);
              DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(flags.initialWeights))));
              estimate = Convert.readDoubleArr(dis);
            } catch (IOException e) {
              throw new RuntimeException("Could not read from double initial weight file " + flags.initialWeights);
            }
          }

          estimate = minimizer.minimize(approxFunc, 1e-2, estimate);

        }
        */
        CRFLogConditionalObjectiveFunction func = new CRFLogConditionalObjectiveFunction(data, labels, featureIndex,
            windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
        func.crfType = flags.crfType;
        if (evaluators != null) {
          for (Evaluator eval : evaluators) {
            if (eval instanceof CRFClassifierEvaluator) {
              ((CRFClassifierEvaluator) eval).setHelperFunction(func);
            }
          }
        }
        Minimizer minimizer = getMinimizer(i, evaluators);

        double[] initialWeights;
        if (flags.initialWeights == null) {
          initialWeights = func.initial();
        } else {
          try {
            System.err.println("Reading initial weights from file " + flags.initialWeights);
            DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(
                flags.initialWeights))));
            initialWeights = Convert.readDoubleArr(dis);
          } catch (IOException e) {
            throw new RuntimeException("Could not read from double initial weight file " + flags.initialWeights);
          }
        }
        System.err.println("numWeights: " + initialWeights.length);

        if (flags.testObjFunction) {
          StochasticDiffFunctionTester tester = new StochasticDiffFunctionTester(func);
          if (tester.testSumOfBatches(initialWeights, 1e-4)) {
            System.err.println("Testing complete... exiting");
            System.exit(1);
          } else {
            System.err.println("Testing failed....exiting");
            System.exit(1);
          }

        }
        double[] weights = minimizer.minimize(func, flags.tolerance, initialWeights);
        this.weights = func.to2D(weights);
      }

      // save feature index to disk and read in later
      if (flags.saveFeatureIndexToDisk) {
        try {
          System.err.println("Reading temporary feature index file.");
          featureIndex = (Index<String>) IOUtils.readObjectFromFile(featIndexFile);
        } catch (Exception e) {
          throw new RuntimeException("Could not open temporary feature index file for reading.");
        }
      }

      if (i != flags.numTimesPruneFeatures) {
        dropFeaturesBelowThreshold(flags.featureDiffThresh);
        System.err.println("Removing features with weight below " + flags.featureDiffThresh + " and retraining...");
      }

    }
  }
434307
public void setDocumentClassificationJobPropertiesList(java.util.Collection<DocumentClassificationJobProperties> documentClassificationJobPropertiesList) {
        if (documentClassificationJobPropertiesList == null) {
            this.documentClassificationJobPropertiesList = null;
            return;
        }

        this.documentClassificationJobPropertiesList = new java.util.ArrayList<DocumentClassificationJobProperties>(documentClassificationJobPropertiesList);
    }
123845
void choose_smartly(int ndocs, List<Document> docs)
    {
        int siz = size();
        double[] closest = new double[siz];
        if (siz < ndocs)
            ndocs = siz;
        int index, count = 0;

        index = random.nextInt(siz);  // initial center
        docs.add(documents_.get(index));
        ++count;
        double potential = 0.0;
        for (int i = 0; i < documents_.size(); i++)
        {
            double dist = 1.0 - SparseVector.inner_product(documents_.get(i).feature(), documents_.get(index).feature());
            potential += dist;
            closest[i] = dist;
        }

        // choose each center
        while (count < ndocs)
        {
            double randval = random.nextDouble() * potential;

            for (index = 0; index < documents_.size(); index++)
            {
                double dist = closest[index];
                if (randval <= dist)
                    break;
                randval -= dist;
            }
            if (index == documents_.size())
                index--;
            docs.add(documents_.get(index));
            ++count;

            double new_potential = 0.0;
            for (int i = 0; i < documents_.size(); i++)
            {
                double dist = 1.0 - SparseVector.inner_product(documents_.get(i).feature(), documents_.get(index).feature());
                double min = closest[i];
                if (dist < min)
                {
                    closest[i] = dist;
                    min = dist;
                }
                new_potential += min;
            }
            potential = new_potential;
        }
    }
511302
def docs_init_to_class(self):
        
        result = False
        if not self.parsed:
            self._parse()
        einit = []
        eclass = []
        for e in self.docs_list:
            if len(eclass) == len(einit) + 1 and e['docs'].element['name'] == '__init__':
                einit.append(e)
            elif not eclass and e['docs'].element['type'] == 'class':
                eclass.append(e)
        for c, i in zip(eclass, einit):
            start, _ = c['location']
            if start < 0:
                start, _ = i['location']
                if start > 0:
                    result = True
                    cspaces = c['docs'].get_spaces()
                    ispaces = i['docs'].get_spaces()
                    c['docs'].set_spaces(ispaces)
                    i['docs'].set_spaces(cspaces)
                    c['docs'].generate_docs()
                    i['docs'].generate_docs()
                    c['docs'], i['docs'] = i['docs'], c['docs']
        return result
1329987
@Override
  public List<IN> classify(List<IN> document) {
    if (flags.doGibbs) {
      try {
        return classifyGibbs(document);
      } catch (Exception e) {
        System.err.println("Error running testGibbs inference!");
        e.printStackTrace();
        return null;
      }
    } else if (flags.crfType.equalsIgnoreCase("maxent")) {
      return classifyMaxEnt(document);
    } else {
      throw new RuntimeException("Unsupported inference type: " + flags.crfType);
    }
  }
536156
def train(klass, text)
      th = TextHash.new(text)
      th.each { |word, count|
        @storage.incr_word_count klass, word, count
        yield word, count if block_given?
      }
      @storage.incr_total_word_count klass, th.word_count
      doccount = (text.kind_of? Array) ? text.length : 1
      @storage.incr_doc_count klass, doccount
      @classnames << klass unless @classnames.include? klass
      # cache is now dirty of these vars
      @doc_count_totals = nil
      @vocab_sizes = nil
      th
    end
780096
public double[][] classify(List corpus) throws Exception {
		Document[] documents = (Document[]) corpus.toArray(new Document[corpus
				.size()]);
		return classify(documents);
	}
1218119
def guess_labels(self, doc):
        
        doc = doc.clone()  # make sure it can be serialized safely
        return self.index.guess_labels(doc)
772350
public void processDocument(BufferedReader document) throws IOException {
        int docIndex = documentCounter.getAndIncrement();

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        // As we read in the document, generate a compressed version of it,
        // which we will use during the process space method to recompute all of
        // the word vectors' semantics
        ByteArrayOutputStream compressedDocument = 
            new ByteArrayOutputStream(4096);
        DataOutputStream dos = new DataOutputStream(compressedDocument);
        int tokens = 0; // count how many are in this document
        int unfilteredTokens = 0; // how many tokens remained after filtering

        IntegerVector docVector = createVector();
        docToVector.put(docIndex, docVector);

        while (documentTokens.hasNext()) {
            tokens++;
            String focusWord = documentTokens.next();

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);

	    // If the filter does not accept this word, skip the semantic
	    // processing, continue with the next word
            if (!calculateSemantics) {
                // Do not write out any removed tokens to save space
		continue;
	    }

            // Update the occurrences of this token
            unfilteredTokens++;
            add(docVector, getTermIndexVector(focusWord));

            // Update the compress version of the document with the token.
            //
            // NOTE: this call to termToIndex *must* come after the
            // getTermIndexVector() call, which is responsible for adding this
            // mapping if it doesn't already exist.
	    int focusIndex = termToIndex.get(focusWord);

            // write the term index into the compressed for the document for
            // later corpus reprocessing
            dos.writeInt(focusIndex);
        }

        document.close();
        
        dos.close();
        byte[] docAsBytes = compressedDocument.toByteArray();

        // Once the document is finished, write the compressed contents to the
        // corpus stream
        synchronized(compressedDocumentsWriter) {
            // Write how many terms were in this document after filtering
            compressedDocumentsWriter.writeInt(unfilteredTokens);
            compressedDocumentsWriter.write(docAsBytes, 0, docAsBytes.length);
        }
    }
1337791
@SuppressWarnings("unchecked")
    public static org.bson.Document decodeMongoDoc(org.bson.Document doc) {
        org.bson.Document newDoc = new org.bson.Document();
        for (String key : doc.keySet()) {
            Object value = doc.get(key);
            if (value instanceof org.bson.Document)
                value = decodeMongoDoc(doc.get(key, org.bson.Document.class));
            else if (value instanceof List<?>) {
                for (int i=0; i < ((List<?>)value).size(); i++) {
                    Object obj = ((List<?>)value).get(i);
                    if (obj instanceof org.bson.Document)
                        ((List<org.bson.Document>)value).set(i, decodeMongoDoc((org.bson.Document)obj));
                }
            }

            String newKey = key;
            if (key.startsWith("\\uff04"))
                newKey = "$" + key.substring(6);
            if (key.contains("\\uff0e")) {
                newKey = newKey.replace("\\uff0e", ".");
            }
            newDoc.put(newKey, value);
        }
        return newDoc;
    }
376304
def doc_children(self, doctype, limiters=[]):
        
        result = []
        for doc in self.docstring:
            if len(limiters) == 0 or doc.doctype in limiters:
                result.extend(doc.children(doctype))

        return result
402516
def groups_to_docsets(groups, docset)
      results = []
      results << {
        :tagname => :class,
        :type => docset[:type],
        :comment => groups[:class],
        :code => docset[:code],
        :linenr => docset[:linenr],
      }
      groups[:cfg].each do |cfg|
        results << {
          :tagname => :cfg,
          :type => docset[:type],
          :comment => cfg,
          :code => {},
          :linenr => docset[:linenr],
        }
      end
      if groups[:constructor].length > 0
        # Remember that a constructor is already found and ignore if a
        # constructor is detected from code.
        @constructor_found = true

        results << {
          :tagname => :method,
          :type => docset[:type],
          :comment => groups[:constructor],
          :code => {},
          :linenr => docset[:linenr],
        }
      end
      results
    end
296485
public void execute(@NonNull Document document) {
      for (CaduceusRule rule : rules) {
         TokenMatcher matcher = rule.getRegex().matcher(document);
         while (matcher.find()) {
            ArrayListMultimap<String, Annotation> groups = ArrayListMultimap.create();
            ArrayListMultimap<CaduceusAnnotationProvider, Annotation> providers = ArrayListMultimap.create();

            //Process all the annotation providers
            rule.getAnnotationProviders().forEach(ap -> {
               if (ap.getGroup().equals("*")) {
                  Annotation annotation = createOrGet(document, ap.getAnnotationType(), matcher.group(),
                                                      ap.getAttributes());
                  groups.put(ap.getGroup(), annotation);
                  providers.put(ap, annotation);
               } else {
                  matcher.group(ap.getGroup()).forEach(g -> {
                     Annotation annotation = createOrGet(document, ap.getAnnotationType(), g, ap.getAttributes());
                     groups.put(ap.getGroup(), annotation);
                     providers.put(ap, annotation);
                  });
               }
            });

            if (!groups.containsKey("*")) {
               groups.putAll("*", matcher.group().tokens());
            }


            HashMultimap<String, Tuple2<Annotation, Relation>> relations = HashMultimap.create();
            for (CaduceusRelationProvider rp : rule.getRelationProviders()) {
               List<Annotation> sourceAnnotations = rp.getSource().getAnnotations(groups, matcher);
               List<Annotation> targetAnnotations = rp.getTarget().getAnnotations(groups, matcher);
               for (Annotation source : sourceAnnotations) {
                  for (Annotation target : targetAnnotations) {
                     relations.put(rp.getName(), Tuple2.of(source,
                                                           new Relation(rp.getRelationType(), rp.getRelationValue(),
                                                                        target.getId())));
                     if (rp.isReciprocal()) {
                        relations.put(rp.getName(), Tuple2.of(target,
                                                              new Relation(rp.getRelationType(), rp.getRelationValue(),
                                                                           source.getId())));
                     }
                  }
               }
            }


            Set<String> finalRelations = new HashSet<>();
            rule.getRelationProviders().stream()
                .filter(rp -> StringUtils.isNullOrBlank(rp.getRequires()) || relations.containsKey(rp.getRequires()))
                .forEach(rp -> {
                   relations.get(rp.getName()).forEach(t -> {
                      t.getV1().add(t.getV2());
                      finalRelations.add(rp.getName());
                   });
                });

            providers.entries().stream()
                     .filter(entry -> !finalRelations.containsAll(entry.getKey().getRequires()))
                     .forEach(entry -> document.remove(entry.getValue()));

         }
      }


   }
446814
def inherit_docstrings(cls):
    
    @functools.wraps(cls)
    def _inherit_docstrings(cls):
        if not isinstance(cls, (type, colorise.compat.ClassType)):
            raise RuntimeError("Type is not a class")

        for name, value in colorise.compat.iteritems(vars(cls)):
            if isinstance(getattr(cls, name), types.MethodType):
                if not getattr(value, '__doc__', None):
                    for base in cls.__bases__:
                        basemethod = getattr(base, name, None)

                        if basemethod and getattr(base, '__doc__', None):
                            value.__doc__ = basemethod.__doc__

        return cls

    return _inherit_docstrings(cls)
344088
public void printFirstOrderProbsDocuments(ObjectBank<List<IN>> documents) {
    for (List<IN> doc : documents) {
      printFirstOrderProbsDocument(doc);
      System.out.println();
    }
  }
1118092
function( documents ) {
    
    documents = this.map( documents )
    
    for( var key in documents ) {
      this.documents[ key ] = documents[ key ]
    }
    
  }
205523
public void createFeatures(List<String> features, String[] tokens) {

    for (DocumentFeatureGenerator generator : generators) {
      generator.createFeatures(features, tokens);
    }
  }