Run the classifier on the documents in an ObjectBank, and print the
answers to a given PrintWriter (with timing to stderr). The value of
flags.documentReader is used to determine testFile format.

@param documents The ObjectBank to test on.
358725
public void classifyAndWriteAnswersKBest(ObjectBank<List<IN>> documents, int k, PrintWriter printWriter,
                                           DocumentReaderAndWriter<IN> readerAndWriter) throws IOException {
    Timing timer = new Timing();
    int numWords = 0;
    int numSentences = 0;

    for (List<IN> doc : documents) {
      Counter<List<IN>> kBest = classifyKBest(doc, AnswerAnnotation.class, k);
      numWords += doc.size();
      List<List<IN>> sorted = Counters.toSortedList(kBest);
      int n = 1;
      for (List<IN> l : sorted) {
        System.out.println("<sentence id=" + numSentences + " k=" + n + " logProb=" + kBest.getCount(l) + " prob="
            + Math.exp(kBest.getCount(l)) + '>');
        writeAnswers(l, printWriter, readerAndWriter);
        System.out.println("</sentence>");
        n++;
      }
      numSentences++;
    }

    long millis = timer.stop();
    double wordspersec = numWords / (((double) millis) / 1000);
    NumberFormat nf = new DecimalFormat("0.00"); // easier way!
    System.err.println(this.getClass().getName() + " tagged " + numWords + " words in " + numSentences
        + " documents at " + nf.format(wordspersec) + " words per second.");
  }
205523
public void createFeatures(List<String> features, String[] tokens) {

    for (DocumentFeatureGenerator generator : generators) {
      generator.createFeatures(features, tokens);
    }
  }
573642
void addDocuments(final Document[] docs) throws IOException
   {
      final IndexWriter writer = getIndexWriter();

      IOException ioExc = null;
      try
      {
         for (Document doc : docs)
         {
            try
            {
               writer.addDocument(doc);
            }
            catch (Throwable e) //NOSONAR
            {
               if (ioExc == null)
               {
                  if (e instanceof IOException)
                  {
                     ioExc = (IOException)e;
                  }
                  else
                  {
                     ioExc = Util.createIOException(e);
                  }
               }

               log.warn("Exception while inverting document", e);
            }
         }
      }
      finally
      {
         invalidateSharedReader();
      }

      if (ioExc != null)
      {
         throw ioExc;
      }
   }
468484
public List<Pair<int[][][], int[]>> documentsToDataAndLabelsList(Collection<List<IN>> documents) {
    int numDatums = 0;

    List<Pair<int[][][], int[]>> docList = new ArrayList<Pair<int[][][], int[]>>();
    for (List<IN> doc : documents) {
      Pair<int[][][], int[]> docPair = documentToDataAndLabels(doc);
      docList.add(docPair);
      numDatums += doc.size();
    }

    System.err.println("numClasses: " + classIndex.size() + ' ' + classIndex);
    System.err.println("numDocuments: " + docList.size());
    System.err.println("numDatums: " + numDatums);
    System.err.println("numFeatures: " + featureIndex.size());
    return docList;
  }
505789
@Override
  public void printProbsDocument(List<IN> document) {

    Pair<int[][][], int[]> p = documentToDataAndLabels(document);
    int[][][] data = p.first();

    // FactorTable[] factorTables =
    // CRFLogConditionalObjectiveFunction.getCalibratedCliqueTree(weights, data,
    // labelIndices, classIndex.size());
    CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(),
        classIndex, flags.backgroundSymbol);

    // for (int i = 0; i < factorTables.length; i++) {
    for (int i = 0; i < cliqueTree.length(); i++) {
      IN wi = document.get(i);
      System.out.print(wi.get(CoreAnnotations.TextAnnotation.class) + '\t');
      for (Iterator<String> iter = classIndex.iterator(); iter.hasNext();) {
        String label = iter.next();
        int index = classIndex.indexOf(label);
        // double prob = Math.pow(Math.E, factorTables[i].logProbEnd(index));
        double prob = cliqueTree.prob(i, index);
        System.out.print(label + '=' + prob);
        if (iter.hasNext()) {
          System.out.print("\t");
        } else {
          System.out.print("\n");
        }
      }
    }
  }
1124328
protected void println(Object... objects) throws IOException {
        for (Object object : objects) {
            pageContext.getOut().println(object);
        }
    }
1216484
public void setClassifiers(java.util.Collection<Classifier> classifiers) {
        if (classifiers == null) {
            this.classifiers = null;
            return;
        }

        this.classifiers = new java.util.ArrayList<Classifier>(classifiers);
    }
1073164
public static void main(String[] args) throws IOException, ClassNotFoundException {
    CoNLLDocumentReaderAndWriter f = new CoNLLDocumentReaderAndWriter();
    f.init(new SeqClassifierFlags());
    int numDocs = 0;
    int numTokens = 0;
    int numEntities = 0;
    String lastAnsBase = "";
    for (Iterator<List<CoreLabel>> it = f.getIterator(new FileReader(args[0])); it.hasNext(); ) {
      List<CoreLabel> doc = it.next();
      numDocs++;
      for (CoreLabel fl : doc) {
        // System.out.println("FL " + (++i) + " was " + fl);
        if (fl.word().equals(BOUNDARY)) {
          continue;
        }
        String ans = fl.get(AnswerAnnotation.class);
        String ansBase;
        String ansPrefix;
        String[] bits = ans.split("-");
        if (bits.length == 1) {
          ansBase = bits[0];
          ansPrefix = "";
        } else {
          ansBase = bits[1];
          ansPrefix = bits[0];
        }
        numTokens++;
        if (ansBase.equals("O")) {
        } else if (ansBase.equals(lastAnsBase)) {
          if (ansPrefix.equals("B")) {
            numEntities++;
          }
        } else {
          numEntities++;
        }
      }
    }
    System.out.println("File " + args[0] + " has " + numDocs + " documents, " +
            numTokens + " (non-blank line) tokens and " +
            numEntities + " entities.");
  }
1310286
protected void runStatements(Reader reader, PrintStream out)
            throws IOException {
        log.debug("runStatements()");
        StringBuilder txt = new StringBuilder();
        String line = "";
        BufferedReader in = new BufferedReader(reader);

        while ((line = in.readLine()) != null) {
            line = getProject().replaceProperties(line);
            if (line.indexOf("--") >= 0) {
                txt.append("\n");
            }
        }
        // Catch any statements not followed by ;
        if (!txt.toString().equals("")) {
            execGroovy(txt.toString(), out);
        }
    }
334417
public void process(RandomAccessFileOrArray rf) throws DocumentException, IOException
    {
        String line;
        boolean isMetrics = false;
        while ((line = rf.readLine()) != null)
        {
            StringTokenizer tok = new StringTokenizer(line, " ,\n\r\t\f");
            if (!tok.hasMoreTokens())
                continue;
            String ident = tok.nextToken();
            if (ident.equals("FontName"))
                FontName = tok.nextToken("\u00ff").substring(1);
            else if (ident.equals("FullName"))
                FullName = tok.nextToken("\u00ff").substring(1);
            else if (ident.equals("FamilyName"))
                FamilyName = tok.nextToken("\u00ff").substring(1);
            else if (ident.equals("Weight"))
                Weight = tok.nextToken("\u00ff").substring(1);
            else if (ident.equals("ItalicAngle"))
                ItalicAngle = Float.parseFloat(tok.nextToken());
            else if (ident.equals("IsFixedPitch"))
                IsFixedPitch = tok.nextToken().equals("true");
            else if (ident.equals("CharacterSet"))
                CharacterSet = tok.nextToken("\u00ff").substring(1);
            else if (ident.equals("FontBBox"))
            {
                llx = (int)Float.parseFloat(tok.nextToken());
                lly = (int)Float.parseFloat(tok.nextToken());
                urx = (int)Float.parseFloat(tok.nextToken());
                ury = (int)Float.parseFloat(tok.nextToken());
            }
            else if (ident.equals("UnderlinePosition"))
                UnderlinePosition = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("UnderlineThickness"))
                UnderlineThickness = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("EncodingScheme"))
                EncodingScheme = tok.nextToken("\u00ff").substring(1);
            else if (ident.equals("CapHeight"))
                CapHeight = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("XHeight"))
                XHeight = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("Ascender"))
                Ascender = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("Descender"))
                Descender = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("StdHW"))
                StdHW = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("StdVW"))
                StdVW = (int)Float.parseFloat(tok.nextToken());
            else if (ident.equals("StartCharMetrics"))
            {
                isMetrics = true;
                break;
            }
        }
        if (!isMetrics)
            throw new DocumentException("Missing StartCharMetrics in " + fileName);
        while ((line = rf.readLine()) != null)
        {
            StringTokenizer tok = new StringTokenizer(line);
            if (!tok.hasMoreTokens())
                continue;
            String ident = tok.nextToken();
            if (ident.equals("EndCharMetrics"))
            {
                isMetrics = false;
                break;
            }
            Integer C = Integer.valueOf(-1);
            Integer WX = Integer.valueOf(250);
            String N = "";
            int B[] = null;

            tok = new StringTokenizer(line, ";");
            while (tok.hasMoreTokens())
            {
                StringTokenizer tokc = new StringTokenizer(tok.nextToken());
                if (!tokc.hasMoreTokens())
                    continue;
                ident = tokc.nextToken();
                if (ident.equals("C"))
                    C = Integer.valueOf(tokc.nextToken());
                else if (ident.equals("WX"))
                    WX = Integer.valueOf((int)Float.parseFloat(tokc.nextToken()));
                else if (ident.equals("N"))
                    N = tokc.nextToken();
                else if (ident.equals("B")) {
                    B = new int[]{Integer.parseInt(tokc.nextToken()), 
                                         Integer.parseInt(tokc.nextToken()),
                                         Integer.parseInt(tokc.nextToken()),
                                         Integer.parseInt(tokc.nextToken())};
                }
            }
            Object metrics[] = new Object[]{C, WX, N, B};
            if (C.intValue() >= 0)
                CharMetrics.put(C, metrics);
            CharMetrics.put(N, metrics);
        }
        if (isMetrics)
            throw new DocumentException("Missing EndCharMetrics in " + fileName);
        if (!CharMetrics.containsKey("nonbreakingspace")) {
            Object[] space = (Object[])CharMetrics.get("space");
            if (space != null)
                CharMetrics.put("nonbreakingspace", space);
        }
        while ((line = rf.readLine()) != null)
        {
            StringTokenizer tok = new StringTokenizer(line);
            if (!tok.hasMoreTokens())
                continue;
            String ident = tok.nextToken();
            if (ident.equals("EndFontMetrics"))
                return;
            if (ident.equals("StartKernPairs"))
            {
                isMetrics = true;
                break;
            }
        }
        if (!isMetrics)
            throw new DocumentException("Missing EndFontMetrics in " + fileName);
        while ((line = rf.readLine()) != null)
        {
            StringTokenizer tok = new StringTokenizer(line);
            if (!tok.hasMoreTokens())
                continue;
            String ident = tok.nextToken();
            if (ident.equals("KPX"))
            {
                String first = tok.nextToken();
                String second = tok.nextToken();
                Integer width = Integer.valueOf((int)Float.parseFloat(tok.nextToken()));
                Object relates[] = (Object[])KernPairs.get(first);
                if (relates == null)
                    KernPairs.put(first, new Object[]{second, width});
                else
                {
                    int n = relates.length;
                    Object relates2[] = new Object[n + 2];
                    System.arraycopy(relates, 0, relates2, 0, n);
                    relates2[n] = second;
                    relates2[n + 1] = width;
                    KernPairs.put(first, relates2);
                }
            }
            else if (ident.equals("EndKernPairs"))
            {
                isMetrics = false;
                break;
            }
        }
        if (isMetrics)
            throw new DocumentException("Missing EndKernPairs in " + fileName);
        rf.close();
    }
265792
@Override
    public void startedPart(Map headers) {
        if (_docReader != null)
            throw new IllegalStateException("_docReader is already defined");
        Log.v(TAG, "%s: Starting new document; headers =%s", this, headers);
        _docReader = new MultipartDocumentReader(db);
        _docReader.setHeaders(headers);
        _docReader.startedPart(headers);
    }
641835
public void createDocumentClassificationFactory(
      final TrainingParameters params) throws IOException {
    final String featureDescription = DocumentFeatureDescriptor
        .createDocumentFeatureDescriptor(params);
    System.err.println(featureDescription);
    final byte[] featureGeneratorBytes = featureDescription
        .getBytes(Charset.forName("UTF-8"));
    final Map<String, Object> resources = DocumentModelResources
        .loadDocumentResources(params);
    //TODO add default feature generator
    setDocumentClassifierFactory(
        DocumentClassifierFactory.create(DocumentClassifierFactory.class.getName(),
            featureGeneratorBytes, resources));
  }
915256
public void setDocuments(java.util.Collection<DocumentMetadata> documents) {
        if (documents == null) {
            this.documents = null;
            return;
        }

        this.documents = new java.util.ArrayList<DocumentMetadata>(documents);
    }
539058
private void prepare() throws IOException, DocumentException, PrintingException {
		if (baos == null) {
			baos = new ByteArrayOutputStream(); // let it grow as much as needed
		}
		baos.reset();
		boolean resize = false;
		if (page.getConstraint().getWidth() == 0 || page.getConstraint().getHeight() == 0) {
			resize = true;
		}
		// Create a document in the requested ISO scale.
		Document document = new Document(page.getBounds(), 0, 0, 0, 0);
		PdfWriter writer;
		writer = PdfWriter.getInstance(document, baos);

		// Render in correct colors for transparent rasters
		writer.setRgbTransparencyBlending(true);

		// The mapView is not scaled to the document, we assume the mapView
		// has the right ratio.

		// Write document title and metadata
		document.open();
		PdfContext context = new PdfContext(writer);
		context.initSize(page.getBounds());
		// first pass of all children to calculate size
		page.calculateSize(context);
		if (resize) {
			// we now know the bounds of the document
			// round 'm up and restart with a new document
			int width = (int) Math.ceil(page.getBounds().getWidth());
			int height = (int) Math.ceil(page.getBounds().getHeight());
			page.getConstraint().setWidth(width);
			page.getConstraint().setHeight(height);

			document = new Document(new Rectangle(width, height), 0, 0, 0, 0);
			writer = PdfWriter.getInstance(document, baos);
			// Render in correct colors for transparent rasters
			writer.setRgbTransparencyBlending(true);

			document.open();
			baos.reset();
			context = new PdfContext(writer);
			context.initSize(page.getBounds());
		}
		// int compressionLevel = writer.getCompressionLevel(); // For testing
		// writer.setCompressionLevel(0);

		// Actual drawing
		document.addTitle("Geomajas");
		// second pass to layout
		page.layout(context);
		// finally render (uses baos)
		page.render(context);

		document.add(context.getImage());
		// Now close the document
		document.close();
	}
501083
@Override
  public void train(Collection<List<IN>> docs, DocumentReaderAndWriter<IN> readerAndWriter) {
    Timing timer = new Timing();
    timer.start();
    makeAnswerArraysAndTagIndex(docs);
    long elapsedMs = timer.stop();
    System.err.println("Time to convert docs to feature indices: " + Timing.toSecondsString(elapsedMs) + " seconds");
    if (flags.exportFeatures != null) {
      timer.start();
      CRFFeatureExporter<IN> featureExporter = new CRFFeatureExporter<IN>(this);
      featureExporter.printFeatures(flags.exportFeatures, docs);
      elapsedMs = timer.stop();
      System.err.println("Time to export features: " + Timing.toSecondsString(elapsedMs) + " seconds");
    }
    for (int i = 0; i <= flags.numTimesPruneFeatures; i++) {
      timer.start();
      Pair<int[][][][], int[][]> dataAndLabels = documentsToDataAndLabels(docs);
      elapsedMs = timer.stop();
      System.err.println("Time to convert docs to data/labels: " + Timing.toSecondsString(elapsedMs) + " seconds");

      Evaluator[] evaluators = null;
      if (flags.evaluateIters > 0) {
        List<Evaluator> evaluatorList = new ArrayList<Evaluator>();
        evaluatorList.add(new MemoryEvaluator());
        if (flags.evaluateTrain) {
          CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Train set", this);
          List<Pair<int[][][], int[]>> trainDataAndLabels = new ArrayList<Pair<int[][][], int[]>>();
          for (int j = 0; j < dataAndLabels.first().length; j++) {
            Pair<int[][][], int[]> p = new Pair<int[][][], int[]>(dataAndLabels.first()[j], dataAndLabels.second()[j]);
            trainDataAndLabels.add(p);
          }
          crfEvaluator.setTestData(docs, trainDataAndLabels);
          crfEvaluator.setEvalCmd(flags.evalCmd);
          evaluatorList.add(crfEvaluator);
        }
        if (flags.testFile != null) {
          CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set (" + flags.testFile + ")",
              this);
          ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter);
          List<Pair<int[][][], int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank);
          crfEvaluator.setTestData(testObjBank, testDataAndLabels);
          crfEvaluator.setEvalCmd(flags.evalCmd);
          evaluatorList.add(crfEvaluator);
        }
        if (flags.testFiles != null) {
          String[] testFiles = flags.testFiles.split(",");
          for (String testFile : testFiles) {
            CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set ("
                + testFile + ")", this);
            ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(testFile, readerAndWriter);
            List<Pair<int[][][], int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank);
            crfEvaluator.setTestData(testObjBank, testDataAndLabels);
            crfEvaluator.setEvalCmd(flags.evalCmd);
            evaluatorList.add(crfEvaluator);
          }
        }
        evaluators = new Evaluator[evaluatorList.size()];
        evaluatorList.toArray(evaluators);
      }

      if (flags.numTimesPruneFeatures == i) {
        docs = null; // hopefully saves memory
      }
      // save feature index to disk and read in later
      File featIndexFile = null;

      if (flags.saveFeatureIndexToDisk) {
        try {
          System.err.println("Writing feature index to temporary file.");
          featIndexFile = IOUtils.writeObjectToTempFile(featureIndex, "featIndex" + i + ".tmp");
          featureIndex = null;
        } catch (IOException e) {
          throw new RuntimeException("Could not open temporary feature index file for writing.");
        }
      }

      // first index is the number of the document
      // second index is position in the document also the index of the
      // clique/factor table
      // third index is the number of elements in the clique/window thase
      // features are for (starting with last element)
      // fourth index is position of the feature in the array that holds them
      // element in data[i][j][k][m] is the index of the mth feature occurring
      // in position k of the jth clique of the ith document
      int[][][][] data = dataAndLabels.first();
      // first index is the number of the document
      // second index is the position in the document
      // element in labels[i][j] is the index of the correct label (if it
      // exists) at position j in document i
      int[][] labels = dataAndLabels.second();

      if (flags.loadProcessedData != null) {
        List processedData = loadProcessedData(flags.loadProcessedData);
        if (processedData != null) {
          // enlarge the data and labels array
          int[][][][] allData = new int[data.length + processedData.size()][][][];
          int[][] allLabels = new int[labels.length + processedData.size()][];
          System.arraycopy(data, 0, allData, 0, data.length);
          System.arraycopy(labels, 0, allLabels, 0, labels.length);
          // add to the data and labels array
          addProcessedData(processedData, allData, allLabels, data.length);
          data = allData;
          labels = allLabels;
        }
      }

      if (flags.useFloat) {
        CRFLogConditionalObjectiveFloatFunction func = new CRFLogConditionalObjectiveFloatFunction(data, labels,
            featureIndex, windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
        func.crfType = flags.crfType;

        QNMinimizer minimizer;
        if (flags.interimOutputFreq != 0) {
          FloatFunction monitor = new ResultStoringFloatMonitor(flags.interimOutputFreq, flags.serializeTo);
          minimizer = new QNMinimizer(monitor);
        } else {
          minimizer = new QNMinimizer();
        }

        if (i == 0) {
          minimizer.setM(flags.QNsize);
        } else {
          minimizer.setM(flags.QNsize2);
        }

        float[] initialWeights;
        if (flags.initialWeights == null) {
          initialWeights = func.initial();
        } else {
          try {
            System.err.println("Reading initial weights from file " + flags.initialWeights);
            DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(
                flags.initialWeights))));
            initialWeights = Convert.readFloatArr(dis);
          } catch (IOException e) {
            throw new RuntimeException("Could not read from float initial weight file " + flags.initialWeights);
          }
        }
        System.err.println("numWeights: " + initialWeights.length);
        float[] weights = minimizer.minimize(func, (float) flags.tolerance, initialWeights);
        this.weights = ArrayMath.floatArrayToDoubleArray(func.to2D(weights));

      } else {

        /*double[] estimate = null;

        if(flags.estimateInitial){
          int[][][][] approxData = new int[data.length/100][][][];
          int[][] approxLabels = new int[data.length/100][];

          Random generator = new Random(1);
          for(int k=0;k<approxData.length; k++){
            int thisInd = generator.nextInt(data.length);
            approxData[k] = data[ thisInd];
            approxLabels[k] = labels[ thisInd];
          }

          CRFLogConditionalObjectiveFunction approxFunc = new CRFLogConditionalObjectiveFunction(approxData, approxLabels, featureIndex, windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
          approxFunc.crfType = flags.crfType;
          minimizer = new QNMinimizer(10);

          if (flags.initialWeights == null) {
            estimate = approxFunc.initial();
          } else {
            try {
              System.err.println("Reading initial weights from file " + flags.initialWeights);
              DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(flags.initialWeights))));
              estimate = Convert.readDoubleArr(dis);
            } catch (IOException e) {
              throw new RuntimeException("Could not read from double initial weight file " + flags.initialWeights);
            }
          }

          estimate = minimizer.minimize(approxFunc, 1e-2, estimate);

        }
        */
        CRFLogConditionalObjectiveFunction func = new CRFLogConditionalObjectiveFunction(data, labels, featureIndex,
            windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
        func.crfType = flags.crfType;
        if (evaluators != null) {
          for (Evaluator eval : evaluators) {
            if (eval instanceof CRFClassifierEvaluator) {
              ((CRFClassifierEvaluator) eval).setHelperFunction(func);
            }
          }
        }
        Minimizer minimizer = getMinimizer(i, evaluators);

        double[] initialWeights;
        if (flags.initialWeights == null) {
          initialWeights = func.initial();
        } else {
          try {
            System.err.println("Reading initial weights from file " + flags.initialWeights);
            DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(
                flags.initialWeights))));
            initialWeights = Convert.readDoubleArr(dis);
          } catch (IOException e) {
            throw new RuntimeException("Could not read from double initial weight file " + flags.initialWeights);
          }
        }
        System.err.println("numWeights: " + initialWeights.length);

        if (flags.testObjFunction) {
          StochasticDiffFunctionTester tester = new StochasticDiffFunctionTester(func);
          if (tester.testSumOfBatches(initialWeights, 1e-4)) {
            System.err.println("Testing complete... exiting");
            System.exit(1);
          } else {
            System.err.println("Testing failed....exiting");
            System.exit(1);
          }

        }
        double[] weights = minimizer.minimize(func, flags.tolerance, initialWeights);
        this.weights = func.to2D(weights);
      }

      // save feature index to disk and read in later
      if (flags.saveFeatureIndexToDisk) {
        try {
          System.err.println("Reading temporary feature index file.");
          featureIndex = (Index<String>) IOUtils.readObjectFromFile(featIndexFile);
        } catch (Exception e) {
          throw new RuntimeException("Could not open temporary feature index file for reading.");
        }
      }

      if (i != flags.numTimesPruneFeatures) {
        dropFeaturesBelowThreshold(flags.featureDiffThresh);
        System.err.println("Removing features with weight below " + flags.featureDiffThresh + " and retraining...");
      }

    }
  }
563404
protected function processDocuments(array $documents)
    {
        $processingInstructions = $this->configuration->getIndexFieldProcessingInstructionsConfiguration();
        if (count($processingInstructions) > 0) {
            $service = GeneralUtility::makeInstance(Service::class);
            $service->processDocuments($documents, $processingInstructions);
        }
    }
196758
public void run()
    {
        Object objMessage = null;
        String[] files = this.getFileList();
        if (files == null)
            return;
        
        for (String strFile : files)
        {
            m_reader = this.getReader(strFile);
        
            try {
                while (true)
                {
                    objMessage = m_reader.readObject();
                    if (objMessage == null)
                        break;  // EOF = Done
                    
                    BaseBuffer buffer = new VectorBuffer((Vector)objMessage, BaseBuffer.PHYSICAL_FIELDS | BaseBuffer.MODIFIED_ONLY);
                    buffer.setHeaderCount(3);
                    String strTrxType = buffer.getHeader().toString();
                    String strTableName = buffer.getHeader().toString();
                    String strKey = buffer.getHeader().toString();
                    
                    try {
                        Record record = this.getRecord(strTableName);
                        if (record == null)
                        {
                        	ClassInfo recClassInfo = (ClassInfo)this.getMainRecord();
                            recClassInfo.addNew();
                            recClassInfo.setKeyArea(ClassInfo.CLASS_NAME_KEY);
                            recClassInfo.getField(ClassInfo.CLASS_NAME).setString(strTableName);
                            if (recClassInfo.seek(DBConstants.EQUALS))
                            {
                                strTableName = recClassInfo.getPackageName(null) + '.' + strTableName;
                                record = Record.makeRecordFromClassName(strTableName, this);
                                this.disableAllListeners(record);
                                record.setAutoSequence(false);
                            }
                            else
                            {
                                if (!NOTRX.equalsIgnoreCase(strTrxType))
                                    System.out.println("Error - table not found: " + strTableName);
                                continue;
                            }
                        }
                        record.addNew();
                        record.getCounterField().setString(strKey);
                        if (ProxyConstants.ADD.equalsIgnoreCase(strTrxType))
                        {
                            buffer.bufferToFields(record, DBConstants.DISPLAY, DBConstants.READ_MOVE);
                            record.add();
                        }
                        else if (ProxyConstants.SET.equalsIgnoreCase(strTrxType))
                        {
                            if (record.seek(DBConstants.EQUALS))
                            {
                                record.edit();
                                buffer.bufferToFields(record, DBConstants.DISPLAY, DBConstants.READ_MOVE);
                                record.set();
                            }
                            else
                                System.out.println("Error - record not found: " + strTableName + ", key: " + strKey);
                        }
                        else if (ProxyConstants.REMOVE.equalsIgnoreCase(strTrxType))
                        {
                            if (record.seek(DBConstants.EQUALS))
                            {
                                record.edit();
                                record.remove();
                            }
                            else
                                System.out.println("Error - record not found: " + strTableName + ", key: " + strKey);
                        }
                    } catch (DBException e) {
                        e.printStackTrace();
                        System.out.println("Error - record: " + strTableName + ", key: " + strKey + " trx type: " + strTrxType);
                    }
                    
                    System.out.println("trxType: " + strTrxType + " record: " + strTableName + " key: " + strKey);
                }
                m_reader.close();
                m_reader = null;
            
            } catch (EOFException e) {
                // Ok - Done
            } catch (IOException e) {
                e.printStackTrace();
            } catch (ClassNotFoundException e) {
                e.printStackTrace();
            }
        }
    }