处理文本

@param text
@param processor
174834
public void parseText(char[] text, IHit<V> processor)
    {
        int position = 1;
        int currentState = 0;
        for (char c : text)
        {
            currentState = getState(currentState, c);
            int[] hitArray = output[currentState];
            if (hitArray != null)
            {
                for (int hit : hitArray)
                {
                    processor.hit(position - l[hit], position, v[hit]);
                }
            }
            ++position;
        }
    }
1191939
public Document parseString(String text) throws SAXException {
        try {
            return parse(new InputSource(new StringReader(text)));
        } catch (IOException e) {
            throw new RuntimeException("unexpected world exception while reading memory stream", e);
        }
    }
1124822
public void setAsText(String text) throws IllegalArgumentException {
        if (text == null || text.equals("")) {
            setValue(null);
            return;
        }
        Enum<?> value = Enum.valueOf(enumClass, text);
        setValue(value);
    }
196209
public void logWarning(@Nullable final String text) {
    if (text != null || this.preprocessorLogger != null) {
      this.preprocessorLogger.warning(text);
    }
  }
309328
public void logInfo(@Nullable final String text) {
    if (text != null && this.preprocessorLogger != null) {
      this.preprocessorLogger.info(text);
    }
  }
761676
public Phrase process(String text) {
        int fsize = fonts.size();
        if (fsize == 0)
            throw new IndexOutOfBoundsException("No font is defined.");
        char cc[] = text.toCharArray();
        int len = cc.length;
        StringBuffer sb = new StringBuffer();
        Font font = null;
        int lastidx = -1;
        Phrase ret = new Phrase();
        for (int k = 0; k < len; ++k) {
            char c = cc[k];
            if (c == '\n' || c == '\r') {
                sb.append(c);
                continue;
            }
            if (Utilities.isSurrogatePair(cc, k)) {
                int u = Utilities.convertToUtf32(cc, k);
                for (int f = 0; f < fsize; ++f) {
                    font = (Font)fonts.get(f);
                    if (font.getBaseFont().charExists(u)) {
                        if (lastidx != f) {
                            if (sb.length() > 0 && lastidx != -1) {
                                Chunk ck = new Chunk(sb.toString(), (Font)fonts.get(lastidx));
                                ret.add(ck);
                                sb.setLength(0);
                            }
                            lastidx = f;
                        }
                        sb.append(c);
                        sb.append(cc[++k]);
                        break;
                    }
                }
            }
            else {
                for (int f = 0; f < fsize; ++f) {
                    font = (Font)fonts.get(f);
                    if (font.getBaseFont().charExists(c)) {
                        if (lastidx != f) {
                            if (sb.length() > 0 && lastidx != -1) {
                                Chunk ck = new Chunk(sb.toString(), (Font)fonts.get(lastidx));
                                ret.add(ck);
                                sb.setLength(0);
                            }
                            lastidx = f;
                        }
                        sb.append(c);
                        break;
                    }
                }
            }
        }
        if (sb.length() > 0) {
            Chunk ck = new Chunk(sb.toString(), (Font)fonts.get(lastidx == -1 ? 0 : lastidx));
            ret.add(ck);
        }
        return ret;
    }
225957
@Override
    public void setText(String text) {
        if (text == null) {
            if (headerElem != null) {
                removeChild(headerElem);
                headerElem = null;
            }
            return;
        }

        if (headerElem == null) {
            headerElem = Document.get().createHElement(3);
            attachChild(headerElem);
        }
        headerElem.setInnerText(text);
    }
948749
public static void processSentences(
			CMMClassifier<CoreLabel> cmm, PrintStream out, String text) {
		
		if (inputType == inputTypes.PARAGRAPH) { // split in multiple sentences
			if (outputSeparators) out.println("<p>");
			LinkedList<LinkedList<Word>> sentences = Splitting.tokenizeSentences(LVMorphologyReaderAndWriter.getAnalyzer(), text, sentencelengthcap);
			for (LinkedList<Word> sentence : sentences) 
				outputSentence(cmm, out, LVMorphologyReaderAndWriter.analyzeSentence2(sentence) );
			if (outputSeparators) 
				out.println("</p>");
			else 
				out.println();
		} else outputSentence(cmm, out, LVMorphologyReaderAndWriter.analyzeSentence(text) ); // just a single sentence for other types
	}

	/**
	 * Outputs the tagged sentence according to the outputType set in this class
	 * @param cmm - the tagger, needed to retrieve tagger features if they are requested
	 * @param out - a stream to output the data
	 * @param sentence - actual tokens to be output
	 */
	public static void outputSentence(CMMClassifier<CoreLabel> cmm,
			PrintStream out, List<CoreLabel> sentence) {
		if (outputSeparators) out.println("<s>");

        if (outputType != outputTypes.lowercasedText && outputType != outputTypes.analyzerOptions) { //FIXME - a separate flag would be better
            sentence = cmm.classify(sentence); // runs the actual morphotagging system
        }

		switch (outputType) {
		case JSON:
			out.println( output_JSON(sentence));
			break;
		case CONLL_X:
			out.println( output_CONLL(sentence, cmm));
			break;
		case XML:
			try {
				output_XML(sentence, out);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			break;
		case VISL_CG:
			out.println( output_VISL(sentence));
			break;
		case lemmatizedText:
			out.println( output_lemmatized(sentence));
			break;
        case lowercasedText:
            out.println( output_lowercased(sentence));
            break;
        case analyzerOptions:
            out.println( output_analyzer(sentence));
            break;
		default:
			out.println( output_separated(sentence));	    
		}
		if (outputSeparators) out.println("</s>");
		out.flush();
	}	
	
	private static String output_JSON(List<CoreLabel> tokens) {		
		LinkedList<String> tokenJSON = new LinkedList<String>();
		
		for (CoreLabel word : tokens) {
			String token = word.getString(TextAnnotation.class);
			if (token.contains("<s>")) continue;
			Word analysis = word.get(LVMorphologyAnalysis.class);
			Wordform maxwf = analysis.getMatchingWordform(word.getString(AnswerAnnotation.class), false);
			if (mini_tag) maxwf.removeNonlexicalAttributes();
			if (maxwf != null)
				tokenJSON.add(String.format("{\"Word\":\"%s\",\"Tag\":\"%s\",\"Lemma\":\"%s\"}", JSONValue.escape(token), JSONValue.escape(maxwf.getTag()), JSONValue.escape(maxwf.getValue(AttributeNames.i_Lemma))));
			else 
				tokenJSON.add(String.format("{\"Word\":\"%s\",\"Tag\":\"-\",\"Lemma\":\"%s\"}", JSONValue.escape(token), JSONValue.escape(token)));			
		}
		
		String s = formatJSON(tokenJSON).toString();

		return s;
	}
	
	private static void output_XML(List<CoreLabel> tokens, PrintStream straume) throws IOException {
		PrintWriter w = new PrintWriter(straume);
		for (CoreLabel word : tokens) {
			String token = word.getString(TextAnnotation.class);
			if (token.contains("<s>")) continue;
			Word analysis = word.get(LVMorphologyAnalysis.class);
			Wordform maxwf = analysis.getMatchingWordform(word.getString(AnswerAnnotation.class), false);
			if (mini_tag) maxwf.removeNonlexicalAttributes();
			maxwf.addAttribute("Tag", maxwf.getTag());
			maxwf.toXML(w);
//			if (maxwf != null)
//				tokenJSON.add(String.format("{\"Word\":\"%s\",\"Tag\":\"%s\",\"Lemma\":\"%s\"}", JSONValue.escape(token), JSONValue.escape(maxwf.getTag()), JSONValue.escape(maxwf.getValue(AttributeNames.i_Lemma))));
//			else 
//				tokenJSON.add(String.format("{\"Word\":\"%s\",\"Tag\":\"-\",\"Lemma\":\"%s\"}", JSONValue.escape(token), JSONValue.escape(token)));			
		}		
		w.flush();
	}

	private static String output_CONLL(List<CoreLabel> tokens, CMMClassifier<CoreLabel> cmm){
		StringBuilder s = new StringBuilder();

		int counter = 1;
		for (CoreLabel word : tokens) {
			String token = word.getString(TextAnnotation.class);
			if (token.contains("<s>")) continue;
			token = token.replace(' ', '_');
			
			s.append(Integer.toString(counter));
			s.append('\t');
			s.append(token);
			s.append('\t');
			Word analysis = word.get(LVMorphologyAnalysis.class);
			Wordform mainwf = analysis.getMatchingWordform(word.getString(AnswerAnnotation.class), false); 
			if (mainwf != null) {
				String lemma = mainwf.getValue(AttributeNames.i_Lemma);
				lemma = lemma.replace(' ', '_');
				String answer = word.getString(AnswerAnnotation.class);
				if (answer.trim().isEmpty()) answer = "_"; // no empty tag
				s.append(lemma);
				s.append('\t');
				s.append(answer);
				s.append('\t');
				s.append(mainwf.getTag());
				s.append('\t');

				// Feature atribūtu filtri
				if (mini_tag) mainwf.removeNonlexicalAttributes();
				if (LETAfeatures) {
					addLETAfeatures(mainwf);
					// mainwf.removeAttribute(AttributeNames.i_SourceLemma); FIXME - atvasinātiem vārdiem šis var būt svarīgs, atpriedekļotas lemmas..
					mainwf.removeTechnicalAttributes();
				}
				
				s.append(mainwf.pipeDelimitedEntries()); // Pievienojam vārda fīčas
				
				if (features) { // visas fīčas, ko lietoja trenējot
					Datum<String, String> d = cmm.makeDatum(tokens, counter, cmm.featureFactory);
					for (String feature : d.asFeatures()) {
						s.append(feature.substring(0, feature.length()-2).replace(' ', '_')); // noņeam trailing |C kas tām fīčām tur ir
						s.append('|');
					}
				}
				s.deleteCharAt(s.length()-1); // noņemam peedeejo | separatoru, kas ir lieks
				s.append('\t');
				
			} else {
				s.append(token); 
				s.append("\t_\t_\t_\t");
			}
			if (saveColumns) {
				s.append(word.getString(ExtraColumnAnnotation.class));
			} else {
				String syntax = word.getString(ParentAnnotation.class);
				if (syntax != null) {
					s.append(syntax);
				}
				else s.append("_\t_\t_\t_");
			}
			s.append(eol);
			counter++;
		}
		
		return s.toString();
	}
	
	private static String output_lemmatized(List<CoreLabel> tokens){
		StringBuilder s = new StringBuilder();
		
		for (CoreLabel word : tokens) {
			String token = word.getString(TextAnnotation.class);
			if (token.contains("<s>")) continue;
			token = token.replace(' ', '_');
			
			Word analysis = word.get(LVMorphologyAnalysis.class);
			Wordform mainwf = analysis.getMatchingWordform(word.getString(AnswerAnnotation.class), false); 
			if (mainwf != null && !token.isEmpty()) {
				String lemma = mainwf.getValue(AttributeNames.i_Lemma);
				if (saveCase && Character.isUpperCase(token.charAt(0))) lemma = lemma.substring(0,1).toUpperCase() + lemma.substring(1);
				if (!saveCase) lemma=lemma.toLowerCase();
				lemma = lemma.replace(' ', '_');
				s.append(lemma);
				s.append(' ');
				
			} else {
				System.err.println("Empty lemma");
			}
		}
		return s.toString().trim();
	}

    private static String output_lowercased(List<CoreLabel> tokens){
        StringBuilder s = new StringBuilder();

        for (CoreLabel word : tokens) {
            String token = word.getString(TextAnnotation.class);
            if (token.contains("<s>")) continue;
            token = token.replace(' ', '_').toLowerCase();
            s.append(token);
            s.append(' ');
        }
        return s.toString().trim();
    }

    private static String output_analyzer(List<CoreLabel> tokens){
        StringBuilder s = new StringBuilder();

        for (CoreLabel word : tokens) {
            String token = word.getString(TextAnnotation.class);
            if (token.contains("<s>")) continue;
            token = token.replace(' ', '_');
            s.append(token);
            Word analysis = word.get(LVMorphologyAnalysis.class);
            for (Wordform wf : analysis.wordforms) {
                s.append(token_separator);
                s.append(wf.getTag());
            }
        }
        return s.toString().trim();
    }

	private static void addLETAfeatures(Wordform wf) {
		String lemma = wf.getValue(AttributeNames.i_Lemma);
				
		if (wf.isMatchingStrong(AttributeNames.i_PartOfSpeech, AttributeNames.i_Number)) {
			String numbercode = lemma.replaceAll("\\d", "0"); // uzskatam ka nav atšķirības starp skaitļiem ja ciparu skaits vienāds
			wf.addAttribute("LETA_lemma", numbercode);
		} else if (wf.isMatchingStrong(AttributeNames.i_CapitalLetters, AttributeNames.v_FirstUpper) && Dictionary.dict("surnames").contains(lemma))
			wf.addAttribute("LETA_lemma", "_surname_");
		else if (Dictionary.dict("vocations").contains(lemma))
			wf.addAttribute("LETA_lemma", "_vocation_");
		else if (Dictionary.dict("relations").contains(lemma))
			wf.addAttribute("LETA_lemma", "_relationship_");
		else if (Dictionary.dict("partijas").contains(lemma))
			wf.addAttribute("LETA_lemma", "_party_"); // TODO - nočekot kā visā procesā sanāk ar case-sensitivity, te tas ir svarīgi
		else if (Dictionary.dict("months").contains(lemma)) // TODO - te būtu jāčeko, lai personvārdi Marts un Jūlijs te neapēdas, ja ir ar lielo burtu ne teikuma sākumā 
			wf.addAttribute("LETA_lemma", "_month_"); 
		else if (Dictionary.dict("common_lemmas").contains(lemma)) 
			wf.addAttribute("LETA_lemma", lemma);
		else wf.addAttribute("LETA_lemma", "_rare_");		
	}

	// VISL CG format, as described in http://beta.visl.sdu.dk/cg3/chunked/streamformats.html#stream-vislcg
	private static String output_VISL(List<CoreLabel> tokens) {		
		StringBuilder s = new StringBuilder();
		
		for (CoreLabel word : tokens) {
			String token = word.getString(TextAnnotation.class);
			if (token.contains("<s>")) continue;
			
			token.replaceAll("\"", "\\\""); // VISL (seems to) require to escape quotes in their format. Possibly other escaping needs, not sure from their docs.
			
			s.append(String.format("\"<%s>\"\n", token)); // <"They"> from the example
			
			Word analysis = word.get(LVMorphologyAnalysis.class);
			Wordform maxwf = analysis.getMatchingWordform(word.getString(AnswerAnnotation.class), false);
			for (Wordform wf : analysis.wordforms) { // output the "cohort" in VISL-CG terms
				String lemma = wf.getValue(AttributeNames.i_Lemma);
				//Ad-hoc ... removing 'bookkeeping' attributes that seem useless for CG
				wf.removeTechnicalAttributes();
				wf.removeAttribute(AttributeNames.i_Lemma);
				wf.removeAttribute(AttributeNames.i_SourceLemma);
				
				lemma.replaceAll("\"", "\\\"");
				s.append(String.format("\t\"%s\" ", lemma)); // <"They"> from the example
				s.append(wf.getTag());
				s.append(" ");
				AttributeValues minimum = new AttributeValues(wf);
				minimum.removeNonlexicalAttributes();
				s.append(minimum.getTag());
				s.append(" ");
				for (Entry<String, String> entry : wf.entrySet()) { // visi attributevalue paariishi
					String key = entry.getKey();
					String value = entry.getValue();
					// For attributes with distinctive value names (like parts of speech) skip the attribute name for readability in CG
					if ((!key.equalsIgnoreCase(AttributeNames.i_PartOfSpeech) && 
						!key.equalsIgnoreCase(AttributeNames.i_Case) && 
						!key.equalsIgnoreCase(AttributeNames.i_Number) &&
						!key.equalsIgnoreCase(AttributeNames.i_Gender) &&
						!key.equalsIgnoreCase(AttributeNames.i_NounType) &&
						!key.equalsIgnoreCase(AttributeNames.i_Izteiksme) &&
						!key.equalsIgnoreCase(AttributeNames.i_VerbType) &&
						!key.equalsIgnoreCase(AttributeNames.i_Laiks) &&
						!key.equalsIgnoreCase(AttributeNames.i_Transitivity) &&
						!key.equalsIgnoreCase(AttributeNames.i_Declension) &&						
						!key.equalsIgnoreCase(AttributeNames.i_Definiteness) &&
						!key.equalsIgnoreCase(AttributeNames.i_Lokaamiiba) &&
						!key.equalsIgnoreCase(AttributeNames.i_AdjectiveType) &&
						!key.equalsIgnoreCase(AttributeNames.i_SaikljaTips) &&
						!key.equalsIgnoreCase(AttributeNames.i_Uzbuuve) &&
						!key.equalsIgnoreCase(AttributeNames.i_PieturziimesTips) &&
						!key.equalsIgnoreCase(AttributeNames.i_Voice) &&
						!key.equalsIgnoreCase(AttributeNames.i_VvTips)
						) || (value.equalsIgnoreCase(AttributeNames.v_NA) &&
								!key.equalsIgnoreCase(AttributeNames.i_Anafora) &&
								!key.equalsIgnoreCase(AttributeNames.i_Laiks)
						)) {
						 s.append(key.replace(' ', '_')); 
						 s.append('=');
					}
					 s.append(value.replace(' ', '_'));
					 s.append(' ');
				}
				s.append(eol);
			}
		}
		
		s.append("\"<<s>>\"");
		
		return s.toString();
	}
	
	private static String output_separated(List<CoreLabel> tokens){
		StringBuilder s = new StringBuilder();
		
		for (CoreLabel word : tokens) {
			String token = word.getString(TextAnnotation.class);
			if (token.contains("<s>")) continue;
            Word analysis = word.get(LVMorphologyAnalysis.class);
            Wordform mainwf = analysis.getMatchingWordform(word.getString(AnswerAnnotation.class), false);

            if (s.length()>0) s.append(token_separator);
            if (whitespaceMarker && mainwf.isMatchingStrong(AttributeNames.i_WhitespaceBefore, "")) {
                s.append("<g />");
                s.append(token_separator);
            }

            if (outputType == outputTypes.MOSES) token = token.replace(' ', '_');
			s.append(token);
			s.append(field_separator);

			if (mainwf != null) {
				if (mini_tag) mainwf.removeNonlexicalAttributes();
				s.append(mainwf.getTag());
				s.append(field_separator);
				String lemma = mainwf.getValue(AttributeNames.i_Lemma);
				if (outputType == outputTypes.MOSES) lemma = lemma.replace(' ', '_');
				s.append(lemma);
			} else s.append(field_separator); 
			/*
			mainwf = word.get(LVMorphologyAnalysisBest.class);
			if (mainwf != null) {
				s.append("Single-token suggestion:\t");
				s.append(mainwf.getTag());
				s.append("\t");
				s.append(mainwf.getValue(AttributeNames.i_Lemma));
				s.append("\t");
			}
			s.append("\n");
			if (all_options)
					s.append(word.toTabSep(statistics, probabilities));
			else s.append(word.toTabSepsingle(statistics)); */
		}
		
		tokens = null;
		return s.toString();
	}
	
	private static StringBuilder formatJSON(Collection<String> tags) {
		Iterator<String> i = tags.iterator();
		StringBuilder out = new StringBuilder("[");
		while (i.hasNext()) {
			out.append(i.next());
			if (i.hasNext()) out.append(", ");
		}
		out.append("]");
		return out;
	}
	
	
	public static List<List<CoreLabel>> readCONLL(BufferedReader in) throws IOException {
		String s;
	    List<CoreLabel> sentence = new LinkedList<CoreLabel>();
	    List<List<CoreLabel>> result = new LinkedList<List<CoreLabel>>();
	    
	    CoreLabel stag = new CoreLabel();
		stag.set(TextAnnotation.class, "<s>");
		sentence.add(stag);
	    
	    while ((s = in.readLine()) != null) {
	    	if (s.trim().length() > 0) {
	    		String[] fields = s.split("\t");
	    		String token = fields[1];
	    		if (!token.equalsIgnoreCase("_")) token = token.replace('_', ' ');
	    		String extraColumns = "";
	    		if (saveColumns) {
	    			for (int field_i = 6; field_i < fields.length; field_i++) extraColumns += fields[field_i] + "\t";
	    			extraColumns.trim();
	    		}	    		
	    		String syntax = "";
	    		if (fields.length >= 10) syntax = fields[6] + "\t" + fields[7] + "\t" + fields[8] + "\t" + fields[9];

	    		CoreLabel word = new CoreLabel();
				word.set(TextAnnotation.class, token);
				word.set(ParentAnnotation.class, syntax);
				word.set(ExtraColumnAnnotation.class, extraColumns);
	    		sentence.add(word);
	    	} else {
	    		stag = new CoreLabel();
	    		stag.set(TextAnnotation.class, "<s>");
	    		sentence.add(stag);
	    		
	    		result.add(LVMorphologyReaderAndWriter.analyzeLabels(sentence));
	    		
	    		sentence = new LinkedList<CoreLabel>();
	    		stag = new CoreLabel();
	    		stag.set(TextAnnotation.class, "<s>");
	    		sentence.add(stag);
	    	}
	    }
	    if (sentence.size() > 0) {
	    	stag = new CoreLabel();
			stag.set(TextAnnotation.class, "<s>");
			sentence.add(stag);
	    	result.add(LVMorphologyReaderAndWriter.analyzeLabels(sentence));
	    }
	    		
		return result;
	}

}
291367
public void addValue(String text) {
        Token[] tokens = TokenParser.makeTokens(text, isTokenize());
        addValue(tokens);
    }
376103
public void setText(final String TEXT) {
        if (null == text) {
            _text = TEXT;
        } else {
            text.set(TEXT);
        }
    }
661483
public void print(String text) {
        try {
            out.write(text);
        } catch(IOException ioe) {
            throw new GroovyRuntimeException(ioe);
        }
    }
222822
public void process(Iterator<String> text) {
        String nextToken = null, curToken = null;
        // Base case for the next token buffer to ensure we always have two
        // valid tokens present
        if (text.hasNext())
            nextToken = text.next();
        while (text.hasNext()) {
            curToken = nextToken;
            nextToken = text.next();
            // Only process bigrams where the two tokens weren't excluded by the
            // token filter
            if (!(excludeToken(curToken) || excludeToken(nextToken)))
                processBigram(curToken, nextToken);
        }
    }
491565
public void sendTextToNextAlert(String text) {
		Alert alert = driver.switchTo().alert();

		alert.sendKeys(text);
		alert.accept();
	}
704243
public NerResultSet process(final String text) {
        // program pipeline: (Text) ------> (Phrases without classified) ------> (Phrases classified)
        //                         extractor                            classifier
        final NerResultSet rs = extractor.process(text);
        classifier.process(rs);
        if (tracing) trace = classifier.trace;
        return rs;
    }
928913
public void writeToStdIn(String text, boolean newLine) {
        if ( outputStream == null) {
            outputStream = new BufferedOutputStream(process.getOutputStream());
            outputWriter = new BufferedWriter(new OutputStreamWriter(outputStream));
        }

        try {
            outputWriter.write(text);
            if ( newLine ) {
                outputWriter.newLine();
            }
            outputWriter.flush();
            outputStream.flush();
        } catch (IOException e) {
            getLog().debug("Error when writing to process in for " + this, e);
            ChorusAssert.fail("IOException when writing line to process");
        }
    }
816940
public Object parseText(String text) {
        if (text == null || "".equals(text)) {
            throw new IllegalArgumentException("Text must not be null");
        }
        return createParser().parse(text);
    }
871969
public void setText (String text) {
		if (content instanceof VisLabel) {
			((VisLabel) content).setText(text);
		} else {
			setContent(new VisLabel(text));
		}
		pack();
	}