Runs stages 2 - 7 which remain static regardless of the BEL document
input.

@param pedantic the flag for pedantic-ness
@param document the {@link Document BEL document}
762527
private void runCommonStages(boolean pedantic, Document document) {
        if (!stage2(document)) {
            if (pedantic) {
                bail(NAMESPACE_RESOLUTION_FAILURE);
            }
        }

        if (!stage3(document)) {
            if (pedantic) {
                bail(SYMBOL_VERIFICATION_FAILURE);
            }
        }

        if (!stage4(document)) {
            if (pedantic) {
                bail(SEMANTIC_VERIFICATION_FAILURE);
            }
        }

        ProtoNetwork pn = stage5(document);

        if (!stage6(document, pn)) {
            if (pedantic) {
                bail(STATEMENT_EXPANSION_FAILURE);
            }
        }

        if (!stage7(pn, document)) {
            if (pedantic) {
                bail(PROTO_NETWORK_SAVE_FAILURE);
            }
        }
    }
508357
public void processDocument(BufferedReader document) throws IOException {
        if (wordToTransformedVector != null) {
            throw new IllegalStateException("Cannot add new documents to a " +
                "GenericWordSpace whose vectors have been transformed");
        }

        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // prefetch the first windowSize words 
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i)
            nextWords.offer(documentTokens.next());
        
        while (!nextWords.isEmpty()) {
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                (semanticFilter.isEmpty() || semanticFilter.contains(focusWord))
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);
            
            if (calculateSemantics) {
                SparseIntegerVector focusSemantics = 
                    getSemanticVector(focusWord);

                // Keep track of the relative position of the focus word in case
                // word ordering is being used.
                int position = -prevWords.size(); // first word is furthest
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when word order is taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        position++;
                        continue;
                    }
                    
                    int dimension = basisMapping.getDimension(
                        new Duple<String,Integer>(word, position));
                    synchronized(focusSemantics) {
                        focusSemantics.add(dimension, 1);
                    }
                    position++;
                }
            
                // Repeat for the words in the forward window.
                position = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when word order is taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++position;
                        continue;
                    }
                
                    int dimension = basisMapping.getDimension(
                        new Duple<String,Integer>(word, position));
                    synchronized(focusSemantics) {
                        focusSemantics.add(dimension, 1);
                    }
                    position++;
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);
            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
292673
protected void _format(SarlAgent agent, IFormattableDocument document) {
		formatAnnotations(agent, document, XbaseFormatterPreferenceKeys.newLineAfterClassAnnotations);
		formatModifiers(agent, document);

		final ISemanticRegionsFinder regionFor = this.textRegionExtensions.regionFor(agent);
		document.append(regionFor.keyword(this.keywords.getAgentKeyword()), ONE_SPACE);

		document.surround(regionFor.keyword(this.keywords.getExtendsKeyword()), ONE_SPACE);
		document.format(agent.getExtends());

		formatBody(agent, document);
	}
525320
private boolean stage2(final Document document) {
        beginStage(PHASE1_STAGE2_HDR, "2", NUM_PHASES);
        final StringBuilder bldr = new StringBuilder();

        Collection<Namespace> namespaces = document.getNamespaceMap().values();
        final int docNSCount = namespaces.size();

        bldr.append("Compiling ");
        bldr.append(docNSCount);
        bldr.append(" namespace");
        if (docNSCount > 1) {
            bldr.append("s");
        }
        bldr.append(" for ");
        bldr.append(document.getName());
        stageOutput(bldr.toString());

        boolean success = true;
        long t1 = currentTimeMillis();
        try {
            p1.stage2NamespaceCompilation(document);
        } catch (ResourceDownloadError e) {
            success = false;
            stageError(e.getUserFacingMessage());
        } catch (IndexingFailure e) {
            success = false;
            stageError(e.getUserFacingMessage());
        }

        long t2 = currentTimeMillis();
        bldr.setLength(0);
        markTime(bldr, t1, t2);
        markEndStage(bldr);
        stageOutput(bldr.toString());
        return success;
    }
101392
public void writeDocument(OutputStream os, MediaType mt, Document document) {
        ProvFormat format = mimeTypeRevMap.get(mt.toString());
        writeDocument(os, format, document);
    }
600642
private void printErrors(final List<Span> falsePositives,
      final List<Span> falseNegatives, final String doc) {
    this.printStream.println("False positives: {");
    for (final Span span : falsePositives) {
      this.printStream.println(span.getCoveredText(doc));
    }
    this.printStream.println("} False negatives: {");
    for (final Span span : falseNegatives) {
      this.printStream.println(span.getCoveredText(doc));
    }
    this.printStream.println("}\n");
  }
375213
protected void processClass(ClassDoc doc) {
        defaultProcess(doc, true);
        for ( MemberDoc member : doc.fields() ) {
            processMember(member);
        }
        for ( MemberDoc member : doc.constructors() ) {
            processMember(member);
        }
        for ( MemberDoc member : doc.methods() ) {
            processMember(member);
        }
        if ( doc instanceof AnnotationTypeDoc ) {
            for ( MemberDoc member : ((AnnotationTypeDoc)doc).elements() ) {
                processMember(member);
            }
        }
    }
769834
public void setRtfDocument(RtfDocument doc) {
        super.setRtfDocument(doc);
        if(document != null) {
            this.colorNumber = document.getDocumentHeader().getColorNumber(this);
        }
    }
305663
void remove_document(Document doc)
    {
        for (Document<K> document : documents_)
        {
            if (document.equals(doc))
            {
                remove_document(doc);
                return;
            }
        }
    }
428425
public void validate(Document doc) {
    	try {
    		doValidate(doc);
    	}
        catch (Exception e) {
            throw new PluginException("Error validating document: " + e, e);
        }
    }
505789
@Override
  public void printProbsDocument(List<IN> document) {

    Pair<int[][][], int[]> p = documentToDataAndLabels(document);
    int[][][] data = p.first();

    // FactorTable[] factorTables =
    // CRFLogConditionalObjectiveFunction.getCalibratedCliqueTree(weights, data,
    // labelIndices, classIndex.size());
    CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(),
        classIndex, flags.backgroundSymbol);

    // for (int i = 0; i < factorTables.length; i++) {
    for (int i = 0; i < cliqueTree.length(); i++) {
      IN wi = document.get(i);
      System.out.print(wi.get(CoreAnnotations.TextAnnotation.class) + '\t');
      for (Iterator<String> iter = classIndex.iterator(); iter.hasNext();) {
        String label = iter.next();
        int index = classIndex.indexOf(label);
        // double prob = Math.pow(Math.E, factorTables[i].logProbEnd(index));
        double prob = cliqueTree.prob(i, index);
        System.out.print(label + '=' + prob);
        if (iter.hasNext()) {
          System.out.print("\t");
        } else {
          System.out.print("\n");
        }
      }
    }
  }
1053909
private void processFiles(File[] files) {
        phaseOutput(format("=== %s ===", getApplicationName()));
        phaseOutput(format("Compiling %d BEL Document(s)", files.length));

        boolean pedantic = withPedantic();
        boolean verbose = withVerbose();

        int validated = 0;

        for (int i = 1; i <= files.length; i++) {
            File file = files[i - 1];

            if (verbose) {
                phaseOutput("Compiling " + i + " of " + files.length
                        + " BEL Document(s)");
            }

            // validate document
            Document document = stage1(file);
            if (document == null) {
                if (pedantic) {
                    bail(VALIDATION_FAILURE);
                }
                continue;
            }
            validated++;

            // run stages 2 - 7
            runCommonStages(pedantic, document);
        }

        if (validated == 0) {
            bail(NO_VALID_DOCUMENTS);
        }
    }
319530
final void initDocument(int documentNumber)
        {
                // save masked DTM document handle
                m_docHandle = documentNumber<<DOCHANDLE_SHIFT;

                // Initialize the doc -- no parent, no next-sib
                nodes.writeSlot(0,DOCUMENT_NODE,-1,-1,0);
                // wait for the first startElement to create the doc root node
                done = false;
        }
717141
private void classifyNoSeq(List<IN> document) {
    if (flags.useReverse) {
      Collections.reverse(document);
    }

    if (flags.lowerNewgeneThreshold) {
      // Used to raise recall for task 1B
      System.err.println("Using NEWGENE threshold: " + flags.newgeneThreshold);
      for (int i = 0, docSize = document.size(); i < docSize; i++) {
        CoreLabel wordInfo = document.get(i);
        Datum<String, String> d = makeDatum(document, i, featureFactory);
        Counter<String> scores = classifier.scoresOf(d);
        //String answer = BACKGROUND;
        String answer = flags.backgroundSymbol;
        // HN: The evaluation of scoresOf seems to result in some
        // kind of side effect.  Specifically, the symptom is that
        // if scoresOf is not evaluated at every position, the
        // answers are different
        if ("NEWGENE".equals(wordInfo.get(GazAnnotation.class))) {
          for (String label : scores.keySet()) {
            if ("G".equals(label)) {
              System.err.println(wordInfo.word() + ':' + scores.getCount(label));
              if (scores.getCount(label) > flags.newgeneThreshold) {
                answer = label;
              }
            }
          }
        }
        wordInfo.set(AnswerAnnotation.class, answer);
      }
    } else {
      for (int i = 0, listSize = document.size(); i < listSize; i++) {
        String answer = classOf(document, i);
        CoreLabel wordInfo = document.get(i);
        //System.err.println("XXX answer for " +
        //        wordInfo.word() + " is " + answer);
        wordInfo.set(AnswerAnnotation.class, answer);
      }
      if (flags.justify && (classifier instanceof LinearClassifier)) {
        LinearClassifier<String, String> lc = (LinearClassifier<String, String>) classifier;
        for (int i = 0, lsize = document.size(); i < lsize; i++) {
          CoreLabel lineInfo = document.get(i);
          System.err.print("@@ Position " + i + ": ");
          System.err.println(lineInfo.word() + " chose " + lineInfo.get(AnswerAnnotation.class));
          lc.justificationOf(makeDatum(document, i, featureFactory));
        }
      }
    }
    if (flags.useReverse) {
      Collections.reverse(document);
    }
  }
488004
public void startDocument() {
    if (encoding != null) {
      write(writer, "<?xml version=\"1.0\" encoding=\"");
      write(writer, encoding);
      write(writer, "\" standalone=\"yes\"?>" + NL);
    }
    level = 0;
  }
1316580
protected synchronized void processConfDoc(Document doc) {
        Element rootElement = doc.getDocumentElement();

        if ("true".equalsIgnoreCase(getAttrValue(rootElement, "use-query-string"))) setUseQueryString(true);
        if ("true".equalsIgnoreCase(getAttrValue(rootElement, "use-context"))) {
            log.debug("use-context set to true");
            setUseContext(true);
        }
        setDecodeUsing(getAttrValue(rootElement, "decode-using"));
        setDefaultMatchType(getAttrValue(rootElement, "default-match-type"));

        NodeList rootElementList = rootElement.getChildNodes();
        for (int i = 0; i < rootElementList.getLength(); i++) {
            Node node = rootElementList.item(i);

            if (node.getNodeType() == Node.ELEMENT_NODE &&
                    ((Element) node).getTagName().equals("rule")) {
                Element ruleElement = (Element) node;
                // we have a rule node
                NormalRule rule = new NormalRule();

                processRuleBasics(ruleElement, rule);
                procesConditions(ruleElement, rule);
                processRuns(ruleElement, rule);

                Node toNode = ruleElement.getElementsByTagName("to").item(0);
                rule.setTo(getNodeValue(toNode));
                rule.setToType(getAttrValue(toNode, "type"));
                rule.setToContextStr(getAttrValue(toNode, "context"));
                rule.setToLast(getAttrValue(toNode, "last"));
                rule.setQueryStringAppend(getAttrValue(toNode, "qsappend"));
                if ("true".equalsIgnoreCase(getAttrValue(toNode, "encode"))) rule.setEncodeToUrl(true);

                processSetAttributes(ruleElement, rule);

                addRule(rule);

            } else if (node.getNodeType() == Node.ELEMENT_NODE &&
                    ((Element) node).getTagName().equals("class-rule")) {
                Element ruleElement = (Element) node;

                ClassRule classRule = new ClassRule();
                if ("false".equalsIgnoreCase(getAttrValue(ruleElement, "enabled"))) classRule.setEnabled(false);
                if ("false".equalsIgnoreCase(getAttrValue(ruleElement, "last"))) classRule.setLast(false);
                classRule.setClassStr(getAttrValue(ruleElement, "class"));
                classRule.setMethodStr(getAttrValue(ruleElement, "method"));

                addRule(classRule);

            } else if (node.getNodeType() == Node.ELEMENT_NODE &&
                    ((Element) node).getTagName().equals("outbound-rule")) {

                Element ruleElement = (Element) node;
                // we have a rule node
                OutboundRule rule = new OutboundRule();

                processRuleBasics(ruleElement, rule);
                if ("true".equalsIgnoreCase(getAttrValue(ruleElement, "encodefirst"))) rule.setEncodeFirst(true);

                procesConditions(ruleElement, rule);
                processRuns(ruleElement, rule);

                Node toNode = ruleElement.getElementsByTagName("to").item(0);
                rule.setTo(getNodeValue(toNode));
                rule.setToLast(getAttrValue(toNode, "last"));
                if ("false".equalsIgnoreCase(getAttrValue(toNode, "encode"))) rule.setEncodeToUrl(false);

                processSetAttributes(ruleElement, rule);

                addOutboundRule(rule);

            } else if (node.getNodeType() == Node.ELEMENT_NODE &&
                    ((Element) node).getTagName().equals("catch")) {

                Element catchXMLElement = (Element) node;
                // we have a rule node
                CatchElem catchElem = new CatchElem();

                catchElem.setClassStr(getAttrValue(catchXMLElement, "class"));

                processRuns(catchXMLElement, catchElem);

                catchElems.add(catchElem);

            }
        }
        docProcessed = true;
    }
311951
public void setRtfDocument(RtfDocument doc) {
        super.setRtfDocument(doc);
        for(int i = 0; i < this.chunks.size(); i++) {
            ((RtfBasicElement) this.chunks.get(i)).setRtfDocument(this.document);
        }
    }