读取CoNLL文件，创建索引

@param conllPath
@param labeled
@param lowercased
@param clusterFile
@return
@throws Exception
174425
public static IndexMaps createIndices(String conllPath, boolean labeled, boolean lowercased, String clusterFile) throws IOException
    {
        HashMap<String, Integer> wordMap = new HashMap<String, Integer>();
        HashMap<Integer, Integer> labels = new HashMap<Integer, Integer>();
        HashMap<String, Integer> clusterMap = new HashMap<String, Integer>();
        HashMap<Integer, Integer> cluster4Map = new HashMap<Integer, Integer>();
        HashMap<Integer, Integer> cluster6Map = new HashMap<Integer, Integer>();

        String rootString = "ROOT";

        wordMap.put("ROOT", 0);
        labels.put(0, 0);

        // 所有label的id必须从零开始并且连续
        BufferedReader reader = new BufferedReader(new FileReader(conllPath));
        String line;
        while ((line = reader.readLine()) != null)
        {
            String[] args = line.trim().split("\t");
            if (args.length > 7)
            {
                String label = args[7];
                int head = Integer.parseInt(args[6]);
                if (head == 0)
                    rootString = label;

                if (!labeled)
                    label = "~";
                else if (label.equals("_"))
                    label = "-";

                if (!wordMap.containsKey(label))
                {
                    labels.put(wordMap.size(), labels.size());
                    wordMap.put(label, wordMap.size());
                }
            }
        }

        reader = new BufferedReader(new FileReader(conllPath));
        while ((line = reader.readLine()) != null)
        {
            String[] cells = line.trim().split("\t");
            if (cells.length > 7)
            {
                String pos = cells[3];
                if (!wordMap.containsKey(pos))
                {
                    wordMap.put(pos, wordMap.size());
                }
            }
        }

        if (clusterFile.length() > 0)
        {
            reader = new BufferedReader(new FileReader(clusterFile));
            while ((line = reader.readLine()) != null)
            {
                String[] cells = line.trim().split("\t");
                if (cells.length > 2)
                {
                    String cluster = cells[0];
                    String word = cells[1];
                    String prefix4 = cluster.substring(0, Math.min(4, cluster.length()));
                    String prefix6 = cluster.substring(0, Math.min(6, cluster.length()));
                    int clusterId = wordMap.size();

                    if (!wordMap.containsKey(cluster))
                    {
                        clusterMap.put(word, wordMap.size());
                        wordMap.put(cluster, wordMap.size());
                    }
                    else
                    {
                        clusterId = wordMap.get(cluster);
                        clusterMap.put(word, clusterId);
                    }

                    int pref4Id = wordMap.size();
                    if (!wordMap.containsKey(prefix4))
                    {
                        wordMap.put(prefix4, wordMap.size());
                    }
                    else
                    {
                        pref4Id = wordMap.get(prefix4);
                    }

                    int pref6Id = wordMap.size();
                    if (!wordMap.containsKey(prefix6))
                    {
                        wordMap.put(prefix6, wordMap.size());
                    }
                    else
                    {
                        pref6Id = wordMap.get(prefix6);
                    }

                    cluster4Map.put(clusterId, pref4Id);
                    cluster6Map.put(clusterId, pref6Id);
                }
            }
        }

        reader = new BufferedReader(new FileReader(conllPath));
        while ((line = reader.readLine()) != null)
        {
            String[] cells = line.trim().split("\t");
            if (cells.length > 7)
            {
                String word = cells[1];
                if (lowercased)
                    word = word.toLowerCase();
                if (!wordMap.containsKey(word))
                {
                    wordMap.put(word, wordMap.size());
                }
            }
        }

        return new IndexMaps(wordMap, labels, rootString, cluster4Map, cluster6Map, clusterMap);
    }
1124117
private void readCrlf() throws IOException {
        final int crsymbol = this.origin.read();
        final int lfsymbol = this.origin.read();
        if (crsymbol != '\r' || lfsymbol != '\n') {
            throw new IOException(
                String.format(
                    "%s %d%s%d",
                    "CRLF expected at end of chunk: ",
                    crsymbol,
                    "/",
                    lfsymbol
                )
            );
        }
    }
545267
public List<String> readFile(Reader reader) throws Exception {
        try {
            if (characteristics.doIgnoreCase()) {
                if (characteristics.aloneOnLine()) {
                    // Batch separator is located alone on a line
                    // beginning at first position. Ignore case.
                    //
                    // Example with separator "GO":
                    //
                    //   create index blipp ( blopp )
                    //   go
                    //
                    // -or-
                    //
                    //   CREATE INDEX blipp ( blopp )
                    //   GO
                    //
                    return readFileIgnoreCaseAloneOnLine(reader);
                } else {
                    // Batch separator may be located anywhere on a
                    // line. Ignore case.
                    //
                    // Example with separator ";":
                    //
                    //   CREATE INDEX blipp ( blopp );
                    //
                    // Beware of case (separator "GO"):
                    //
                    //   CREATE TABLE blipp (
                    //     pogopop INTEGER
                    //   )
                    //
                    // which will accept "go" in pogopop as separator.
                    //
                    // Since this is a "dangerous" case we do not support it!
                    throw new Exception("Batch separator detection scheme not implemented");
                }
            } else {
                if (characteristics.aloneOnLine()) {
                    // Batch separator is located alone on a line
                    // beginning at first position. Consider case.
                    //
                    // Example with separator "GO":
                    //
                    //   CREATE INDEX blipp ( blopp )
                    //   GO
                    //
                    // is handled while
                    //
                    //   create index blipp ( blopp )
                    //   go
                    //
                    // is not handled
                    //
                    return readFileConsiderCaseAloneOnLine(reader);
                } else {
                    // Batch separator may be located anywhere on a
                    // line. Consider case.
                    //
                    // Example with separator ";":
                    //
                    //   CREATE INDEX blipp ( blopp );
                    //
                    // If we follow our coding standards (keeping SQL uppercased
                    // and identifiers/names lowercased) the following
                    // is also handled correctly (separator "GO"):
                    //
                    //   CREATE TABLE blipp (
                    //     pogopop INTEGER
                    //   )
                    //   GO
                    //
                    // which will -NOT- accidentaly accept "go" in pogopop as separator.
                    // In the latter case it would be better still to use the alone-on-line
                    // model.
                    //
                    return readFileConsiderCase(reader);
                }
            }
        } catch (Exception e) {
            throw new Exception("Failed to read file: " + e.getMessage());

        } finally {
            if (options.debug) System.out.println("Done reading");
        }
    }
1093317
public void configureWebApp() throws Exception
    {
        Set tlds = new HashSet();
        
        // Find tld's from web.xml
        // When the XMLConfigurator (or other configurator) parsed the web.xml,
        // It should have created aliases for all TLDs.  So search resources aliases
        // for aliases ending in tld
        if (_context.getResourceAliases()!=null)
        {
            Iterator iter=_context.getResourceAliases().values().iterator();
            while(iter.hasNext())
            {
                String location = (String)iter.next();
                if (location!=null && location.toLowerCase().endsWith(".tld"))
                {
                    if (!location.startsWith("/"))
                        location="/WEB-INF/"+location;
                    Resource l=_context.getBaseResource().addPath(location);
                    tlds.add(l);
                }
            }
        }
        
        // Look for any tlds in WEB-INF directly.
        if (_context.getWebInf()!=null)
        {
            String[] contents = _context.getWebInf().list();
            for (int i=0;i<contents.length;i++)
            {
                if (contents[i]!=null && contents[i].toLowerCase().endsWith(".tld"))
                {
                    Resource l=_context.getWebInf().addPath(contents[i]);
                    tlds.add(l);
                }
                
            }

            // Look for any tlds in the META-INF of included jars
            Resource lib=_context.getWebInf().addPath("lib/");
            if (lib.exists() && lib.isDirectory())
            {
                contents = lib.list();
                for (int i=0;i<contents.length;i++)
                {
                    if (contents[i]!=null && contents[i].toLowerCase().endsWith(".jar"))
                    {
                        Resource l=lib.addPath(contents[i]);
                        Resource meta=Resource.newResource("jar:"+l+"!/META-INF/");
                        if (meta.exists())
                        {
                            String[] meta_contents=meta.list();
                            
                            for (int j=0;j<meta_contents.length;j++)
                            {
                                if (meta_contents[j]!=null && meta_contents[j].toLowerCase().endsWith(".tld"))
                                {
                                    Resource t=meta.addPath(meta_contents[j]);
                                    tlds.add(t);
                                }
                            }
                        }
                    }
                }
            }
        }
        
        // Create a TLD parser
        XmlParser parser = new XmlParser(false);
        
        parser.redirectEntity("web-jsptaglibrary_1_1.dtd",WebApplicationContext.class.getResource("/javax/servlet/jsp/resources/web-jsptaglibrary_1_1.dtd"));
        parser.redirectEntity("web-jsptaglibrary_1_2.dtd",WebApplicationContext.class.getResource("/javax/servlet/jsp/resources/web-jsptaglibrary_1_2.dtd"));
        parser.redirectEntity("web-jsptaglibrary_2_0.xsd",WebApplicationContext.class.getResource("/javax/servlet/jsp/resources/web-jsptaglibrary_2_0.xsd"));
        parser.setXpath("/taglib/listener/listener-class");
        // Parse all the discovered TLDs
        Iterator iter = tlds.iterator();
        while (iter.hasNext())
        {
            try
            {
                Resource tld = (Resource)iter.next();
                if (log.isDebugEnabled()) log.debug("TLD="+tld);
                
                XmlParser.Node root = parser.parse(tld.getURL());
                
                for (int i=0;i<root.size();i++)
                {
                    Object o=root.get(i);
                    if (o instanceof XmlParser.Node)
                    {
                        XmlParser.Node node = (XmlParser.Node)o;
                        if ("listener".equals(node.getTag()))
                        {
                            String className=node.getString("listener-class",false,true);
                            if (log.isDebugEnabled()) log.debug("listener="+className);
                            
                            try
                            {
                                Class listenerClass=getWebApplicationContext().loadClass(className);
                                EventListener l=(EventListener)listenerClass.newInstance();
                                _context.addEventListener(l);
                            }
                            catch(Exception e)
                            {
                                log.warn("Could not instantiate listener "+className,e);
                            }
                        }
                    }
                }
            }
            catch(Exception e)
            {
                log.warn(e);
            }
        }
    }
1216521
@Override
    protected void doHandle(CommandContext ctx) throws CommandLineException {

        final BatchManager batchManager = ctx.getBatchManager();

        final boolean list = l.isPresent(ctx.getParsedCommandLine());
        final String path = file.getValue(ctx.getParsedCommandLine());
        final String name = this.name.getValue(ctx.getParsedCommandLine());

        if(list) {
            if(path != null || name != null) {
                throw new CommandFormatException("-l is exclusive, neither --file nor name can appear with -l.");
            }

            final Set<String> heldbackNames = batchManager.getHeldbackNames();
            if(!heldbackNames.isEmpty()) {
                List<String> names = new ArrayList<String>(heldbackNames.size());
                for (String heldbackName : heldbackNames) {
                    names.add(heldbackName == null ? "<unnamed>" : heldbackName);
                }
                Collections.sort(names);
                for (String heldbackName : names) {
                    ctx.printLine(heldbackName);
                }
            }
            return;
        }

        if(batchManager.isBatchActive()) {
            throw new CommandLineException("Can't start a new batch while in batch mode.");
        }

        if(path != null) {
            if(name != null) {
                throw new CommandFormatException("Either --file or name argument can be specified at a time.");
            }

            final File f = new File(path);
            if(!f.exists()) {
                throw new CommandLineException("File " + f.getAbsolutePath() + " does not exist.");
            }

            final File currentDir = ctx.getCurrentDir();
            final File baseDir = f.getParentFile();
            if(baseDir != null) {
                ctx.setCurrentDir(baseDir);
            }

            try (BufferedReader reader = new BufferedReader(new FileReader(f))) {
                String line = reader.readLine();
                batchManager.activateNewBatch();
                final Batch batch = batchManager.getActiveBatch();
                while (line != null) {
                    line = line.trim();
                    if (!line.isEmpty() && line.charAt(0) != '#') {
                        batch.add(ctx.toBatchedCommand(line));
                    }
                    line = reader.readLine();
                }
            } catch(IOException e) {
                batchManager.discardActiveBatch();
                throw new CommandLineException("Failed to read file " + f.getAbsolutePath(), e);
            } catch(CommandFormatException e) {
                batchManager.discardActiveBatch();
                throw new CommandLineException("Failed to create batch from " + f.getAbsolutePath(), e);
            } finally {
                if (baseDir != null) {
                    ctx.setCurrentDir(currentDir);
                }
            }
            return;
        }

        boolean activated;
        if(batchManager.isHeldback(name)) {
            activated = batchManager.activateHeldbackBatch(name);
            if (activated) {
                final String msg = name == null ? "Re-activated batch" : "Re-activated batch '" + name + "'";
                ctx.printLine(msg);
                List<BatchedCommand> batch = batchManager.getActiveBatch().getCommands();
                if (!batch.isEmpty()) {
                    for (int i = 0; i < batch.size(); ++i) {
                        BatchedCommand cmd = batch.get(i);
                        ctx.printLine("#" + (i + 1) + ' ' + cmd.getCommand());
                    }
                }
            }
        } else if(name != null) {
            throw new CommandLineException("'" + name + "' not found among the held back batches.");
        } else {
            activated = batchManager.activateNewBatch();
        }

        if(!activated) {
            // that's more like illegal state
            throw new CommandLineException("Failed to activate batch.");
        }
    }
766608
public static ClusterConfigration parseConfiguration(String line) {
    String[] lines = line.trim().split("(?:\\r?\\n)");
    if (lines.length < 2) {
      throw new IllegalArgumentException("Incorrect config response:" + line);
    }
    String configversion = lines[0];
    String nodeListStr = lines[1];
    if (!ByteUtils.isNumber(configversion)) {
      throw new IllegalArgumentException(
          "Invalid configversion: " + configversion + ", it should be a number.");
    }
    String[] nodeStrs = nodeListStr.split("(?:\\s)+");
    int version = Integer.parseInt(configversion);
    List<CacheNode> nodeList = new ArrayList<CacheNode>(nodeStrs.length);
    for (String nodeStr : nodeStrs) {
      if (nodeStr.equals("")) {
        continue;
      }

      int firstDelimiter = nodeStr.indexOf(DELIMITER);
      int secondDelimiter = nodeStr.lastIndexOf(DELIMITER);
      if (firstDelimiter < 1 || firstDelimiter == secondDelimiter) {
        throw new IllegalArgumentException(
            "Invalid server ''" + nodeStr + "'' in response:  " + line);
      }
      String hostName = nodeStr.substring(0, firstDelimiter).trim();
      String ipAddress = nodeStr.substring(firstDelimiter + 1, secondDelimiter).trim();
      String portNum = nodeStr.substring(secondDelimiter + 1).trim();
      int port = Integer.parseInt(portNum);
      nodeList.add(new CacheNode(hostName, ipAddress, port));
    }

    return new ClusterConfigration(version, nodeList);

  }
897442
private void readCRLF() throws IOException {
        int cr = in.read();
        int lf = in.read();
        if ((cr != '\r') || (lf != '\n')) {
            throw new IOException(
                "CRLF expected at end of chunk: " + cr + "/" + lf);
        }
    }
982561
public final void mCOLON() throws RecognitionException {
		try {
			int _type = COLON;
			int _channel = DEFAULT_TOKEN_CHANNEL;
			// src/main/resources/org/drools/compiler/lang/dsl/DSLMap.g:318:7: ( ':' )
			// src/main/resources/org/drools/compiler/lang/dsl/DSLMap.g:318:9: ':'
			{
			match(':'); if (state.failed) return;
			}

			state.type = _type;
			state.channel = _channel;
		}
		finally {
			// do for sure before leaving
		}
	}
1010698
protected void setTldFile(Resource fileTld, int dialect) throws TagLibException {
	if (dialect == CFMLEngine.DIALECT_BOTH) {
	    setTldFile(fileTld, CFMLEngine.DIALECT_CFML);
	    setTldFile(fileTld, CFMLEngine.DIALECT_LUCEE);
	    return;
	}

	TagLib[] tlds = dialect == CFMLEngine.DIALECT_CFML ? cfmlTlds : luceeTlds;

	if (fileTld == null) return;
	this.tldFile = fileTld;
	String key;
	Map<String, TagLib> map = new HashMap<String, TagLib>();
	// First fill existing to set
	for (int i = 0; i < tlds.length; i++) {
	    key = getKey(tlds[i]);
	    map.put(key, tlds[i]);
	}

	TagLib tl;

	// now overwrite with new data
	if (fileTld.isDirectory()) {
	    Resource[] files = fileTld.listResources(new ExtensionResourceFilter(new String[] { "tld", "tldx" }));
	    for (int i = 0; i < files.length; i++) {
		try {
		    tl = TagLibFactory.loadFromFile(files[i], getIdentification());
		    key = getKey(tl);
		    if (!map.containsKey(key)) map.put(key, tl);
		    else overwrite(map.get(key), tl);
		}
		catch (TagLibException tle) {
		    SystemOut.printDate(out, "can't load tld " + files[i]);
		    tle.printStackTrace(getErrWriter());
		}

	    }
	}
	else if (fileTld.isFile()) {
	    tl = TagLibFactory.loadFromFile(fileTld, getIdentification());
	    key = getKey(tl);
	    if (!map.containsKey(key)) map.put(key, tl);
	    else overwrite(map.get(key), tl);
	}

	// now fill back to array
	tlds = new TagLib[map.size()];
	if (dialect == CFMLEngine.DIALECT_CFML) cfmlTlds = tlds;
	else luceeTlds = tlds;

	int index = 0;
	Iterator<TagLib> it = map.values().iterator();
	while (it.hasNext()) {
	    tlds[index++] = it.next();
	}
    }
1174921
public static void load(String dir, List<String[]> sentences, List<PennTreebankPOS[]> labels) {
        List<File> files = new ArrayList<>();
        walkin(new File(dir), files);

        for (File file : files) {
            try {
                FileInputStream stream = new FileInputStream(file);
                BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
                String line = null;
                List<String> sent = new ArrayList<>();
                List<PennTreebankPOS> label = new ArrayList<>();
                while ((line = reader.readLine()) != null) {
                    line = line.trim();
                    if (line.isEmpty()) {
                        if (!sent.isEmpty()) {
                            sentences.add(sent.toArray(new String[sent.size()]));
                            labels.add(label.toArray(new PennTreebankPOS[label.size()]));
                            sent.clear();
                            label.clear();
                        }
                    } else if (!line.startsWith("===") && !line.startsWith("*x*")) {
                        String[] words = line.split("\\s");
                        for (String word : words) {
                            String[] w = word.split("/");
                            if (w.length == 2) {
                                sent.add(w[0]);
                                
                                int pos = w[1].indexOf('|');
                                String tag = pos == -1 ? w[1] : w[1].substring(0, pos);
                                if (tag.equals("PRP$R")) tag = "PRP$";
                                if (tag.equals("JJSS")) tag = "JJS";
                                label.add(PennTreebankPOS.getValue(tag));
                            }
                        }
                    }
                }
                
                if (!sent.isEmpty()) {
                    sentences.add(sent.toArray(new String[sent.size()]));
                    labels.add(label.toArray(new PennTreebankPOS[label.size()]));
                    sent.clear();
                    label.clear();
                }
                
                reader.close();
            } catch (Exception e) {
                logger.error("Failed to load training data {}", file, e);
            }
        }
    }
1113719
public void watchPath(DisconfCoreProcessor disconfCoreMgr, DisConfCommonModel disConfCommonModel, String keyName,
                          DisConfigTypeEnum disConfigTypeEnum, String value) throws Exception {

        // 新建
        String monitorPath = makeMonitorPath(disConfigTypeEnum, disConfCommonModel, keyName, value);

        // 进行监控
        NodeWatcher nodeWatcher =
                new NodeWatcher(disconfCoreMgr, monitorPath, keyName, disConfigTypeEnum, new DisconfSysUpdateCallback(),
                        debug);
        nodeWatcher.monitorMaster();
    }
132312
public static void createGobblinHelixCluster(
      String zkConnectionString,
      String clusterName,
      boolean overwrite) {
    ClusterSetup clusterSetup = new ClusterSetup(zkConnectionString);
    // Create the cluster and overwrite if it already exists
    clusterSetup.addCluster(clusterName, overwrite);
    // Helix 0.6.x requires a configuration property to have the form key=value.
    String autoJoinConfig = ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN + "=true";
    clusterSetup.setConfig(HelixConfigScope.ConfigScopeProperty.CLUSTER, clusterName, autoJoinConfig);
  }
814037
public static String getPath(String relativePath, String backupDirCanonicalPath) throws MalformedURLException
   {
      String path = "file:" + backupDirCanonicalPath + "/" + relativePath;

      URL urlPath = new URL(resolveFileURL(path));

      return urlPath.getFile();
   }
123237
public static LabelAwareSentenceIterator createWithPath(String path) throws Exception {
        return new LabelAwareUimaSentenceIterator(null, path,
                        new UimaResource(AnalysisEngineFactory.createEngine(AnalysisEngineFactory
                                        .createEngineDescription(TokenizerAnnotator.getDescription(),
                                                        SentenceAnnotator.getDescription()))));
    }
374428
public static void compress(String srcFilePath, String destFilePath) throws Exception {
		File src = new File(srcFilePath);
		if (!src.exists()) {
			throw new RuntimeException(srcFilePath + "不存在");
		}
		File zipFile = new File(destFilePath);
		ZipOutputStream zos=null;
		try {
			FileOutputStream fos = new FileOutputStream(zipFile);
			CheckedOutputStream cos = new CheckedOutputStream(fos, new CRC32());
			zos = new ZipOutputStream(cos);
			String baseDir = "";
			compressbyType(src, zos, baseDir);
		}finally {
			if(zos!=null)
			{
				zos.close();
			}
		}
	}
413207
static void loadHadoopClustersProps(String filename) {
    // read the property file
    // populate the map
    Properties prop = new Properties();
    if (StringUtils.isBlank(filename)) {
      filename = Constants.HRAVEN_CLUSTER_PROPERTIES_FILENAME;
    }
    try {
      //TODO : property file to be moved out from resources into config dir
      InputStream inp = Cluster.class.getResourceAsStream("/" + filename);
      if (inp == null) {
        LOG.error(filename
            + " for mapping clusters to cluster identifiers in hRaven does not exist");
        return;
      }
      prop.load(inp);
      Set<String> hostnames = prop.stringPropertyNames();
      for (String h : hostnames) {
        CLUSTERS_BY_HOST.put(h, prop.getProperty(h));
      }
    } catch (IOException e) {
      // An ExceptionInInitializerError will be thrown to indicate that an
      // exception occurred during evaluation of a static initializer or the
      // initializer for a static variable.
      throw new ExceptionInInitializerError(" Could not load properties file " + filename
          + " for mapping clusters to cluster identifiers in hRaven");
    }
  }
888203
public static void compressList(String srcFilePath, String destFilePath) throws Exception {
		File src = new File(srcFilePath);
		if (!src.exists() || !src.isDirectory()) {
			throw new RuntimeException(srcFilePath + "不存在");
		}
		File zipFile = new File(destFilePath);
		ZipOutputStream zos=null;
		try {
			FileOutputStream fos = new FileOutputStream(zipFile);
			CheckedOutputStream cos = new CheckedOutputStream(fos, new CRC32());
			zos = new ZipOutputStream(cos);
			String baseDir = "";
			File[] files = src.listFiles();
			for (File file : files) {
				compressbyType(file, zos, baseDir);
			}			
		} finally {
			if(zos!=null)
			{
				zos.close();
			}
		}
	}