Validate term with distance.

@param term
the term
@param termVector
the term vector
@return true, if successful
@throws IOException
Signals that an I/O exception has occurred.
500250
private static boolean validateTermWithDistance(BytesRef term,
      ComponentTermVector termVector) throws IOException {
    if (termVector.distances == null || termVector.distances.isEmpty()) {
      return true;
    } else {
      // first check maximum for all distances
      for (SubComponentDistance item : termVector.distances) {
        if (item.maximum == null) {
          continue;
        } else {
          if (!item.getDistance().validateMaximum(term)) {
            return false;
          }
        }
      }
      // then check minimum for all distances
      for (SubComponentDistance item : termVector.distances) {
        if (item.minimum == null) {
          continue;
        } else {
          if (!item.getDistance().validateMinimum(term)) {
            return false;
          }
        }
      }
      // try {
      // System.out.println("ACCEPT: " + term.utf8ToString()+"
      // ("+termVector.distances.size()+")");
      // for (SubComponentDistance item : termVector.distances) {
      // System.out.println(
      // item.hashCode() + "\t" +
      // term.utf8ToString()+"\t"+item.getDistance().getClass().getName()+"\t"+item.getDistance().minimum+"\t"+item.getDistance().maximum
      // + "\t" +
      // item.getDistance().validate(term)+"\t"+item.getDistance().compute(term));
      // }
      // } catch (Exception e) {
      // System.out.println("PROBLEEM: "+e.getMessage());
      // }
      return true;
    }
  }
933886
protected void finishAdding()
    {
        noMoreAdding = true;
        
        workSpace = null;
        storageSpace = null;
        wordCounts = null;
        
        int finalLength = currentLength.get();
        int[] frqs = new int[finalLength];
        for(Map.Entry<Integer, AtomicInteger> entry : termDocumentFrequencys.entrySet())
            frqs[entry.getKey()] = entry.getValue().get();
        for(SparseVector vec : vectors)
        {
            //Make sure all the vectors have the same length
            vec.setLength(finalLength);
        }
        weighting.setWeight(vectors, IntList.view(frqs, finalLength));
        
        System.out.println("Final Length: " + finalLength);
        for(SparseVector vec : vectors)
        {
            //Unlike normal index functions, WordWeighting needs to use the vector to do some set up first
            weighting.applyTo(vec);
        }
    }
1279366
private void processSpace() throws IOException {
        LOGGER.info("generating reflective vectors");
        compressedDocumentsWriter.close();
        int numDocuments = documentCounter.get();
        termToIndexVector.clear();
        indexToTerm = new String[termToIndex.size()];
        for (Map.Entry<String,Integer> e : termToIndex.entrySet())
            indexToTerm[e.getValue()] = e.getKey();

        // Read in the compressed version of the corpus, re-processing each
        // document to build up the document vectors
        DataInputStream corpusReader = new DataInputStream(
            new BufferedInputStream(new FileInputStream(compressedDocuments)));

        // Set up the concurrent data structures so we can reprocess the
        // documents concurrently using a work queue
        final BlockingQueue<Runnable> workQueue =
            new LinkedBlockingQueue<Runnable>();
        for (int i = 0; i < Runtime.getRuntime().availableProcessors(); ++i) {
            Thread t = new WorkerThread(workQueue);
            t.start();
        }
        final Semaphore documentsRerocessed = new Semaphore(0);         

        for (int d = 0; d < numDocuments; ++d) {
            final int docId = d;

            // This value already has any filtered tokens taken into account,
            // i.e. in only counts those tokens that remain after filtering
            int tokensInDoc = corpusReader.readInt();
            // Read in the document
            final int[] doc = new int[tokensInDoc];
            for (int i = 0; i < tokensInDoc; ++i)
                doc[i] = corpusReader.readInt();

            workQueue.offer(new Runnable() {
                    public void run() {
                        // This method creates the document vector and then adds
                        // that document vector with the reflective semantic
                        // vector for each word occurring in the document
                        LOGGER.fine("reprocessing doc #" + docId);
                        processIntDocument(docToVector.get(docId), doc);
                        documentsRerocessed.release();
                    }
                });
        }
        corpusReader.close();

        // Wait until all the documents have been processed
        try {
            documentsRerocessed.acquire(numDocuments);
        } catch (InterruptedException ie) {
            throw new Error("interrupted while waiting for documents to " +
                            "finish reprocessing", ie);
        }        
        LOGGER.fine("finished reprocessing all documents");

    }
703403
protected boolean is4YearTerm(Term<?> term) {

        if(!(term instanceof DateTerm.YearTerm)) {
            return false;
        }

        DateTerm.YearTerm yearTerm = (DateTerm.YearTerm)term;
        if(yearTerm.getFormat().length() == 4) {
            return true;
        }

        return false;

    }
481887
public boolean filter(Term term) {

        if (!stop.isEmpty() && (stop.contains(term.getName()))) {
            return true;
        }

        if (!natureStop.isEmpty() && (natureStop.contains(term.natrue().natureStr))) {
            return true;
        }

        if (!regexList.isEmpty()) {
            for (Pattern stopwordPattern : regexList) {
                if (stopwordPattern.matcher(term.getName()).matches()) {
                    return true;
                }
            }
        }

        return false;
    }
111725
public static boolean isVector(IntBuffer shapeInfo) {
        int rank = Shape.rank(shapeInfo);
        if (rank > 2 || rank < 1)
            return false;
        else {
            int len = Shape.length(shapeInfo);
            IntBuffer shape = Shape.shapeOf(shapeInfo);
            return shape.get(0) == len || shape.get(1) == len;
        }
    }
265866
private void parseTag() throws IOException {
        if (dataPos == dataSize)
            return;
        if (isEOC(data[dataPos]) && (data[dataPos + 1] == 0)) {
            int numOfEncapsulatedLenBytes = 0;
            Object elem = null;
            int index;
            for (index = ndefsList.size()-1; index >= 0; index--) {
                // Determine the first element in the vector that does not
                // have a matching EOC
                elem = ndefsList.get(index);
                if (elem instanceof Integer) {
                    break;
                } else {
                    numOfEncapsulatedLenBytes += ((byte[])elem).length - 3;
                }
            }
            if (index < 0) {
                throw new IOException("EOC does not have matching " +
                                      "indefinite-length tag");
            }
            int sectionLen = dataPos - ((Integer)elem).intValue() +
                             numOfEncapsulatedLenBytes;
            byte[] sectionLenBytes = getLengthBytes(sectionLen);
            ndefsList.set(index, sectionLenBytes);
            unresolved--;

            // Add the number of bytes required to represent this section
            // to the total number of length bytes,
            // and subtract the indefinite-length tag (1 byte) and
            // EOC bytes (2 bytes) for this section
            numOfTotalLenBytes += (sectionLenBytes.length - 3);
        }
        dataPos++;
    }
1227941
public static String readString(InputStream is, String term) throws IOException {
        return readString(new Utf8StreamReader(is), term);
    }
511940
public static Boolean isWithinDistance(Geometry geomA, Geometry geomB, Double distance) {
        if(geomA == null||geomB == null){
            return null;
        }
        return geomA.isWithinDistance(geomB, distance);
    }
1177122
public boolean filter(Term term) {

		if (stop.size() > 0 && (stop.contains(term.getName()))) {
			return true;
		}

		if (natureStop.size() > 0 && (natureStop.contains(term.natrue().natureStr))) {
			return true;
		}

		if (regexList.size() > 0) {
			for (Pattern stopWordPattern : regexList) {
				if (stopWordPattern.matcher(term.getName()).matches()) {
					return true;
				}
			}
		}

		return false;
	}
439165
protected void indexVectorInternal(double[] vector) throws Exception {
		if (vector.length != vectorLength) {
			throw new Exception("The dimensionality of the vector is wrong!");
		}
		// append the persistent index
		appendPersistentIndex(vector);
		// append the ram-based index
		if (loadIndexInMemory) {
			vectorsList.add(vector);
		}
	}
950734
public static void writeWord2VecModel(Word2Vec vectors, OutputStream stream) throws IOException {
        ZipOutputStream zipfile = new ZipOutputStream(new BufferedOutputStream(new CloseShieldOutputStream(stream)));

        ZipEntry syn0 = new ZipEntry("syn0.txt");
        zipfile.putNextEntry(syn0);

        // writing out syn0
        File tempFileSyn0 = DL4JFileUtils.createTempFile("word2vec", "0");
        File tempFileSyn1 = DL4JFileUtils.createTempFile("word2vec", "1");
        File tempFileSyn1Neg = DL4JFileUtils.createTempFile("word2vec", "n");
        File tempFileCodes = DL4JFileUtils.createTempFile("word2vec", "h");
        File tempFileHuffman = DL4JFileUtils.createTempFile("word2vec", "h");
        File tempFileFreqs = DL4JFileUtils.createTempFile("word2vec", "f");
        tempFileSyn0.deleteOnExit();
        tempFileSyn1.deleteOnExit();
        tempFileSyn1Neg.deleteOnExit();
        tempFileFreqs.deleteOnExit();
        tempFileCodes.deleteOnExit();
        tempFileHuffman.deleteOnExit();

        try {
            writeWordVectors(vectors.lookupTable(), tempFileSyn0);

            FileUtils.copyFile(tempFileSyn0, zipfile);

            // writing out syn1
            INDArray syn1 = ((InMemoryLookupTable<VocabWord>) vectors.getLookupTable()).getSyn1();

            if (syn1 != null)
                try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileSyn1))) {
                    for (int x = 0; x < syn1.rows(); x++) {
                        INDArray row = syn1.getRow(x);
                        StringBuilder builder = new StringBuilder();
                        for (int i = 0; i < row.length(); i++) {
                            builder.append(row.getDouble(i)).append(" ");
                        }
                        writer.println(builder.toString().trim());
                    }
                }

            ZipEntry zSyn1 = new ZipEntry("syn1.txt");
            zipfile.putNextEntry(zSyn1);

            FileUtils.copyFile(tempFileSyn1, zipfile);

            // writing out syn1
            INDArray syn1Neg = ((InMemoryLookupTable<VocabWord>) vectors.getLookupTable()).getSyn1Neg();

            if (syn1Neg != null)
                try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileSyn1Neg))) {
                    for (int x = 0; x < syn1Neg.rows(); x++) {
                        INDArray row = syn1Neg.getRow(x);
                        StringBuilder builder = new StringBuilder();
                        for (int i = 0; i < row.length(); i++) {
                            builder.append(row.getDouble(i)).append(" ");
                        }
                        writer.println(builder.toString().trim());
                    }
                }

            ZipEntry zSyn1Neg = new ZipEntry("syn1Neg.txt");
            zipfile.putNextEntry(zSyn1Neg);

            FileUtils.copyFile(tempFileSyn1Neg, zipfile);


            ZipEntry hC = new ZipEntry("codes.txt");
            zipfile.putNextEntry(hC);

            // writing out huffman tree
            try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileCodes))) {
                for (int i = 0; i < vectors.getVocab().numWords(); i++) {
                    VocabWord word = vectors.getVocab().elementAtIndex(i);
                    StringBuilder builder = new StringBuilder(encodeB64(word.getLabel())).append(" ");
                    for (int code : word.getCodes()) {
                        builder.append(code).append(" ");
                    }

                    writer.println(builder.toString().trim());
                }
            }

            FileUtils.copyFile(tempFileCodes, zipfile);

            ZipEntry hP = new ZipEntry("huffman.txt");
            zipfile.putNextEntry(hP);

            // writing out huffman tree
            try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileHuffman))) {
                for (int i = 0; i < vectors.getVocab().numWords(); i++) {
                    VocabWord word = vectors.getVocab().elementAtIndex(i);
                    StringBuilder builder = new StringBuilder(encodeB64(word.getLabel())).append(" ");
                    for (int point : word.getPoints()) {
                        builder.append(point).append(" ");
                    }

                    writer.println(builder.toString().trim());
                }
            }

            FileUtils.copyFile(tempFileHuffman, zipfile);

            ZipEntry hF = new ZipEntry("frequencies.txt");
            zipfile.putNextEntry(hF);

            // writing out word frequencies
            try (PrintWriter writer = new PrintWriter(new FileWriter(tempFileFreqs))) {
                for (int i = 0; i < vectors.getVocab().numWords(); i++) {
                    VocabWord word = vectors.getVocab().elementAtIndex(i);
                    StringBuilder builder = new StringBuilder(encodeB64(word.getLabel())).append(" ")
                            .append(word.getElementFrequency()).append(" ")
                            .append(vectors.getVocab().docAppearedIn(word.getLabel()));

                    writer.println(builder.toString().trim());
                }
            }

            FileUtils.copyFile(tempFileFreqs, zipfile);

            ZipEntry config = new ZipEntry("config.json");
            zipfile.putNextEntry(config);
            //log.info("Current config: {}", vectors.getConfiguration().toJson());
            try (ByteArrayInputStream bais = new ByteArrayInputStream(vectors.getConfiguration().toJson().getBytes(StandardCharsets.UTF_8))) {
                IOUtils.copy(bais, zipfile);
            }

            zipfile.flush();
            zipfile.close();
        } finally {
            for (File f : new File[]{tempFileSyn0, tempFileSyn1, tempFileSyn1Neg, tempFileCodes, tempFileHuffman, tempFileFreqs}) {
                try {
                    f.delete();
                } catch (Exception e) {
                    //Ignore, is temp file
                }
            }
        }
    }
867607
private static void check(Vector a, Vector b) {
        if (a.length() != b.length())
            throw new IllegalArgumentException(
                    "input vector lengths do not match");
    }
575566
@Override
    public void recognition(Term[] terms) {
        this.terms = terms;
        List<Term> termList = recogntion_();
        for (Term term2 : termList) {
            TermUtil.insertTerm(terms, term2, InsertTermType.SCORE_ADD_SORT);
        }
    }
758491
@Override
    public void writeVectorInt(Vector<Integer> vector) {
        log.debug("writeVectorInt: {}", vector);
        writeAMF3();
        buf.put(AMF3.TYPE_VECTOR_INT);
        if (hasReference(vector)) {
            putInteger(getReferenceId(vector) << 1);
            return;
        }
        storeReference(vector);
        putInteger(vector.size() << 1 | 1);
        buf.put((byte) 0x00);
        for (Integer v : vector) {
            buf.putInt(v);
        }
        // debug
        if (log.isDebugEnabled()) {
            int pos = buf.position();
            buf.position(0);
            StringBuilder sb = new StringBuilder();
            HexDump.dumpHex(sb, buf.array());
            log.debug("\n{}", sb);
            buf.position(pos);
        }
    }
404656
@Override
  public boolean validateMaximum(BytesRef term) {
    if (maximum == null) {
      return true;
    } else {
      double[][] state = _start();
      char ch1;
      char ch2 = 0x00;
      int i = term.offset + prefixOffset;
      for (; i < term.length; i++) {
        ch1 = (char) term.bytes[i];
        if (ch1 == 0x00) {
          break;
        }
        state = _step(state, ch1, ch2);
        if (!_can_match(state)) {
          return false;
        }
        ch2 = ch1;
      }
      return _is_match(state);
    }
  }
1109716
public void addTerm(final IndexTerm term) {
        int i = 0;
        final int termNum = termList.size();

        for (; i < termNum; i++) {
            final IndexTerm indexTerm = termList.get(i);
            if (indexTerm.equals(term)) {
                return;
            }

            // Add targets when same term name and same term key
            if (indexTerm.getTermFullName().equals(term.getTermFullName())
                    && indexTerm.getTermKey().equals(term.getTermKey())) {
                indexTerm.addTargets(term.getTargetList());
                indexTerm.addSubTerms(term.getSubTerms());
                break;
            }
        }

        if (i == termNum) {
            termList.add(term);
        }
    }