Segment sentences.

@param infile the infile
@param outfile the outfile
@param senSegmenter the sen segmenter
519745
private static void senSegmentFile(String infile, String outfile, JVnSenSegmenter senSegmenter ){
    	try{
    	 BufferedReader in = new BufferedReader(new InputStreamReader(
                 new FileInputStream(infile), "UTF-8"));
         BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
                 new FileOutputStream(outfile), "UTF-8"));

         String para = "", line = "", text = "";
         while ((line = in.readLine()) != null){                    
             if (!line.equals("")){
                 if (line.charAt(0) == '#'){
                     //skip comment line
                     text += line + "\n";
                     continue;
                 }
                 
                 para = senSegmenter.senSegment(line).trim();                                        
                 text += para.trim() + "\n\n";
             }
             else{
                 //blank line
                 text += "\n";                        
             }
         }
         text = text.trim();
         
         out.write(text);
         out.newLine();
         
         in.close();
         out.close();
    	}
    	catch (Exception e){
    		System.out.println("Error in sensegment file " + infile);
    	}
    }
356116
public void freeSegment(SegmentKelp segment)
    throws IOException
  {
    // System.out.println("FREE: " + segment);
    SegmentMeta segmentMeta = findSegmentMeta(segment.length());
    
    segmentMeta.remove(segment);
    
    segment.close();
    
    segmentMeta.addFree(segment.extent());
    
    // System.out.println("  FREE: " + segmentMeta.getFreeCount() + " " + segment.getLength());
  }
331493
public void append(LogSegment segment) {
        while (true) {
            List<LogSegment> curr = contents.get();
            List<LogSegment> updated = new ArrayList<LogSegment>(curr);
            updated.add(segment);
            if (contents.compareAndSet(curr, updated)) {
                return;
            }
        }
    }
1144856
private void compactEntries(Segment segment, Segment compactSegment) {
    for (long i = segment.firstIndex(); i <= segment.lastIndex(); i++) {
      checkEntry(i, segment, compactSegment);
    }
  }
262493
private void compactGroup(List<Segment> segments, List<OffsetPredicate> predicates, Segment compactSegment) {
    // Iterate through all segments being compacted and write entries to a single compact segment.
    for (int i = 0; i < segments.size(); i++) {
      compactSegment(segments.get(i), predicates.get(i), compactSegment);
    }
  }
315923
public String transform(CharSequence content, File inputFile) {
		final ParsingContext rootContextForReplacements = new ParsingContext();
		initializeContext(rootContextForReplacements);
		CharSequence rawContent = preProcessing(content);
		Stage stage = Stage.first();
		do {
			final ContentParserInterceptor interceptor = new ContentParserInterceptor();
			// Reset the lineno because it is not reset between the different stages.
			rootContextForReplacements.setLineNo(1);
			final boolean hasChanged = parse(rawContent, inputFile, 0, stage, rootContextForReplacements, interceptor);
			if (hasChanged) {
				rawContent = interceptor.getResult();
			}
			stage = stage.next();
		} while (stage != null);
		return postProcessing(rawContent);
	}
964640
private void setBreakpoint(File file, int line, String condition)
			throws Exception
	{
		if (file == null)
		{
			file = interpreter.getDefaultFile();
		}

		if (file == null || file.getPath().equals("?"))
		{
			Set<File> files = interpreter.getSourceFiles();

			if (files.size() > 1)
			{
				println("Assuming file " + file.getPath());
			} else if (files.isEmpty())
			{
				println("No files defined");
				return;
			}

			file = files.iterator().next();
		}

		PStm stmt = interpreter.findStatement(file, line);

		if (stmt == null)
		{
			PExp exp = interpreter.findExpression(file, line);

			if (exp == null)
			{
				println("No breakable expressions or statements at " + file
						+ ":" + line);
			} else
			{
				interpreter.clearBreakpoint(BreakpointManager.getBreakpoint(exp).number);
				Breakpoint bp = interpreter.setBreakpoint(exp, condition);
				println("Created " + bp);
				println(interpreter.getSourceLine(bp.location));
			}
		} else
		{
			interpreter.clearBreakpoint(BreakpointManager.getBreakpoint(stmt).number);
			Breakpoint bp = interpreter.setBreakpoint(stmt, condition);
			println("Created " + bp);
			println(interpreter.getSourceLine(bp.location));
		}
	}
1085237
private void compactSegment(Segment segment, OffsetPredicate predicate, Segment compactSegment) {
    for (long i = segment.firstIndex(); i <= segment.lastIndex(); i++) {
      checkEntry(i, segment, predicate, compactSegment);
    }
  }
979322
public void setOutfile(final File outfile) {
    //
    // if file name was empty, skip link step
    //
    if (outfile == null || outfile.toString().length() > 0) {
      this.outFile = outfile;
    }
  }
37827
public void classifyAndWriteAnswers(String testFile, OutputStream outStream,
                                      DocumentReaderAndWriter<IN> readerWriter)
    throws IOException
  {
    ObjectBank<List<IN>> documents =
      makeObjectBankFromFile(testFile, readerWriter);
    PrintWriter pw = IOUtils.encodedOutputStreamPrintWriter(outStream, flags.outputEncoding, true);
    classifyAndWriteAnswers(documents, pw, readerWriter);
  }
1320969
private void initSegs() throws IOException {
        int loaded = 0;
        File[] segFiles = listSegmentFiles();
        if (segFiles.length == 0) {
            return;
        }

        try {
            for (int i = 0; i < segFiles.length; i++) {
                File segFile = segFiles[i];
                int segId = Integer.parseInt(segFile.getName().substring(0, segFile.getName().indexOf('.')));
                if (segId != i) {
                    throw new IOException("Segment file " + i + ".seg missing");
                }

                if (getMeta().hasSegmentInService(segId)) {
                    // Always load a live segment as READ_ONLY
                    Segment s = getSegmentFactory().createSegment(segId, segFile, _segFileSizeMB, Segment.Mode.READ_ONLY);
                    s.incrLoadSize(getMeta().getSegmentLoadSize(segId));
                    _segList.add(s);
                    loaded++;
                } else {
                    // Segment is not live and is free for reuse
                    _segList.add(null);
                }
            }
        } catch (IOException e) {
            _log.error(e.getMessage());
            
            clearInternal(false /* DO NOT CLEAR META */);
            throw e;
        }

        _log.info("loaded: " + loaded + "/" + segFiles.length);
    }
39828
public static void convert(File infile, File xslfile, File outfile) {
		try {
			// Create transformer factory
			TransformerFactory factory = TransformerFactory.newInstance();

			// Use the factory to create a template containing the xsl file
			Templates template = factory.newTemplates(new StreamSource(
					new FileInputStream(xslfile)));

			// Use the template to create a transformer
			Transformer xformer = template.newTransformer();
			
			// passing 2 parameters
			String branch = outfile.getParentFile().getCanonicalPath().substring(root.length());
			branch = branch.replace(File.separatorChar, '/');
			StringBuffer path = new StringBuffer();
			for (int i = 0; i < branch.length(); i++) {
				if (branch.charAt(i) == '/') path.append("/..");
			}
			
			xformer.setParameter("branch", branch);
			xformer.setParameter("root", path.toString());

			// Prepare the input and output files
			Source source = new StreamSource(new FileInputStream(infile));
			Result result = new StreamResult(new FileOutputStream(outfile));

			// Apply the xsl file to the source file and write the result to the
			// output file
			xformer.transform(source, result);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
376546
protected void segmentAfterRule(String sentence, String normalized, List<String> wordList)
    {
        if (!enableRuleBasedSegment)
        {
            segmenter.segment(sentence, normalized, wordList);
            return;
        }
        int start = 0;
        int end = start;
        byte preType = typeTable[normalized.charAt(end)];
        byte curType;
        while (++end < normalized.length())
        {
            curType = typeTable[normalized.charAt(end)];
            if (curType != preType)
            {
                if (preType == CharType.CT_NUM)
                {
                    // 浮点数识别
                    if ("，,．.".indexOf(normalized.charAt(end)) != -1)
                    {
                        if (end + 1 < normalized.length())
                        {
                            if (typeTable[normalized.charAt(end + 1)] == CharType.CT_NUM)
                            {
                                continue;
                            }
                        }
                    }
                    else if ("年月日时分秒".indexOf(normalized.charAt(end)) != -1)
                    {
                        preType = curType; // 交给统计分词
                        continue;
                    }
                }
                pushPiece(sentence, normalized, start, end, preType, wordList);
                start = end;
            }
            preType = curType;
        }
        if (end == normalized.length())
            pushPiece(sentence, normalized, start, end, preType, wordList);
    }
1178342
public Segment toSegment()
    {
        return new Segment()
        {
            @Override
            protected List<Term> segSentence(char[] sentence)
            {
                List<String> wordList = segment(new String(sentence));
                List<Term> termList = new LinkedList<Term>();
                for (String word : wordList)
                {
                    termList.add(new Term(word, null));
                }
                return termList;
            }
        }.enableCustomDictionary(false);
    }
408871
private void parseFragment(String fragment, boolean query) {
    ChunkTokenizer tokenizer = new ChunkTokenizer(fragment);

    while (tokenizer.hasNext()) {
      /* check to see if we have an expression or a literal */
      String chunk = tokenizer.next();

      if (chunk.startsWith("{")) {
        /* it's an expression, defer encoding until resolution */
        FragmentType type = (query) ? FragmentType.QUERY : FragmentType.PATH_SEGMENT;

        Expression expression = Expressions.create(chunk, type);
        if (expression == null) {
          this.templateChunks.add(Literal.create(encode(chunk, query)));
        } else {
          this.templateChunks.add(expression);
        }
      } else {
        /* it's a literal, pct-encode it */
        this.templateChunks.add(Literal.create(encode(chunk, query)));
      }
    }
  }
297406
public final void entryRuleAlternatives() throws RecognitionException {
        try {
            // InternalXtext.g:321:1: ( ruleAlternatives EOF )
            // InternalXtext.g:322:1: ruleAlternatives EOF
            {
             before(grammarAccess.getAlternativesRule()); 
            pushFollow(FollowSets000.FOLLOW_1);
            ruleAlternatives();

            state._fsp--;

             after(grammarAccess.getAlternativesRule()); 
            match(input,EOF,FollowSets000.FOLLOW_2); 

            }

        }
        catch (RecognitionException re) {
            reportError(re);
            recover(input,re);
        }
        finally {
        }
        return ;
    }
267957
@GuardedBy("readers")
    private void releaseSegmentsIfNeeded() throws ReaderNotInReaderGroupException {
        releaseSealedSegments();
        Segment segment = groupState.findSegmentToReleaseIfRequired();
        if (segment != null) {
            log.info("{} releasing segment {}", this, segment);
            EventSegmentReader reader = readers.stream().filter(r -> r.getSegmentId().equals(segment)).findAny().orElse(null);
            if (reader != null) {
                if (groupState.releaseSegment(segment, reader.getOffset(), getLag())) {
                    readers.remove(reader);
                    reader.close();
                }
            }
        }
    }