Read a collection {@link Token}s from a given file.

@param tokenFilePath the token file path
@param configuration a {@link Configuration} object carrying Hadoop configuration properties
@return a collection of {@link Token}s
@throws IOException
171468
public static Collection<Token<? extends TokenIdentifier>> readTokensFromFile(Path tokenFilePath,
      Configuration configuration) throws IOException {
    return Credentials.readTokenStorageFile(tokenFilePath, configuration).getAllTokens();
  }
75792
def get_token(filename=TOKEN_PATH, envvar=TOKEN_ENVVAR):
    
    if os.path.isfile(filename):
        with open(filename) as token_file:
            token = token_file.readline().strip()

    else:
        token = os.environ.get(envvar)

        if not token:
            raise ValueError("No token found.\n"
                             "{} file doesn't exist.\n{} environment variable is not set.".format(filename, envvar))

    return token
353720
public Boolean loadPropertiesFile(String path) throws IOException
    {
        Properties prop = new Properties();
        InputStream input = null;

        try {

            input = new FileInputStream(path);
            prop.load(input);

            for(String key : prop.stringPropertyNames()) {
                String value = prop.getProperty(key);
                configs.put(key, value);
            }

            return true;
        } catch (IOException ex) {
            return false;
        }
    }
207240
public void read(String filePath) throws IOException{


		String line;
		ArrayList<String> words = new ArrayList<String>();
		ArrayList<String> markP = new ArrayList<String>();
		ArrayList<String> typeP = new ArrayList<String>();
		ArrayList<String> markC = new ArrayList<String>();
		ArrayList<String> typeC = new ArrayList<String>();
		if(filePath == null)
			return;
		File file = new File(filePath);
		BufferedReader reader = null;
		entityType = new HashSet<String>();

		//按行读取文件内容，一次读一整行
		reader = new BufferedReader(new InputStreamReader(new FileInputStream(file),"UTF-8"));

		//从文件中提取实体并存入队列中

		while ((line = reader.readLine()) != null) {
			line = line.trim();
			if(line.equals("")){

				newextract(words, markP, typeP, markC, typeC);


			}else{
				//判断实体,实体开始的边界为B-***或者S-***，结束的边界为E-***或N（O）或空白字符或B-***
				//predict
				String[] toks = line.split("\\s+");

				int ci = 1;
				int cj=2;

				if(toks.length>3){//如果列数大于三，默认取最后两列
					ci=toks.length-2;
					cj=toks.length-1;
				}else if(toks.length<3){
					System.out.println(line);
					return;
				}

				String[] marktype = getMarkType(toks[ci]);
				words.add(toks[0]);
				markP.add(marktype[0]);
				typeP.add(marktype[1]);
				entityType.add(marktype[1]);	

				//correct
				marktype = getMarkType(toks[cj]);
				markC.add(marktype[0]);
				typeC.add(marktype[1]);	
				entityType.add(marktype[1]);	

			}
		}
		reader.close();
		if(words.size()>0){
			newextract(words, markP, typeP, markC, typeC);
		}
		//从entityPs和entityCs提取正确估计的实体，存到entityCinPs，并更新mpc中的统计信息
		extractCInPEntity();
	}
396874
private static Token createThreadContextToken(final String configuration) {
		if (configuration == null) {
			InternalLogger.log(Level.ERROR, "\"{context}\" requires a key");
			return new PlainTextToken("");
		} else {
			int splitIndex = configuration.indexOf(',');
			String key = splitIndex == -1 ? configuration.trim() : configuration.substring(0, splitIndex).trim();
			if (key.isEmpty()) {
				InternalLogger.log(Level.ERROR, "\"{context}\" requires a key");
				return new PlainTextToken("");
			} else {
				String defaultValue = splitIndex == -1 ? null : configuration.substring(splitIndex + 1).trim();
				return defaultValue == null ? new ThreadContextToken(key) : new ThreadContextToken(key, defaultValue);
			}
		}
	}
415295
public Configuration fromFile(URL path) throws ConfigurationLoadException {
    PropertiesConfiguration configuration = setupConfiguration(new PropertiesConfiguration());
    configuration.setURL(path);
    try {
      configuration.load();
      return configuration;
    } catch (ConfigurationException e) {
      throw new ConfigurationLoadException(
          "Encountered a problem reading the provided configuration file \"" + path + "\"!", e);
    }
  }
164778
@SuppressWarnings({"unchecked"})
    public <T> T read(File jsonFile, Configuration configuration) throws IOException {
        notNull(jsonFile, "json file can not be null");
        isTrue(jsonFile.exists(), "json file does not exist");
        notNull(configuration, "jsonProvider can not be null");

        FileInputStream fis = null;
        try {
            fis = new FileInputStream(jsonFile);
            return read(fis, configuration);
        } finally {
            Utils.closeQuietly(fis);
        }
    }
771972
static public Tokenizer getTokenizer(String filename) throws IOException {
    if(filename.toLowerCase().endsWith("xml")) {
      return new XmlTokenizer(filename);
    } else {
      return new BinaryTokenizer(filename);
    }
  }
184659
public Token read(Token t) throws IOException {
    if (t.e == EditsElement.OPCODE) {
      // read the offset first
      LongToken offsetToken = new LongToken(EditsElement.OFFSET);
      offsetToken.fromString(getNextElementsValue(EditsElement.OFFSET.name()));
      t.offset = offsetToken.value;
    }
    t.fromString(getNextElementsValue(t.getEditsElement().toString()));
    return t;
  }
378074
public void load(String path,String tag) throws IOException{
		if(path == null) return;
		dict = MyCollection.loadTSet(path);
		maxLen = 0;
		TObjectHashIterator<String> it = dict.iterator();
		while(it.hasNext()){
			String k = it.next();
			if(k.length()>maxLen){
				maxLen = k.length();
			}
		}		
		name = tag;		
	}
1117
public void setClasspath(String classpath) {
        this.classpath = new LinkedList<String>();
        StringTokenizer tokenizer = new StringTokenizer(classpath, File.pathSeparator);
        while (tokenizer.hasMoreTokens()) {
            this.classpath.add(tokenizer.nextToken());
        }
    }
860955
@Override
   public TokenStream tokenStream(String fieldName, Reader reader)
   {
      if (indexingConfig != null)
      {
         Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName);
         if (propertyAnalyzer != null)
         {
            return propertyAnalyzer.tokenStream(fieldName, reader);
         }
      }
      return defaultAnalyzer.tokenStream(fieldName, reader);
   }
339455
private static Properties load(String configFilePath)
	{
		Properties props = new Properties();
		BufferedInputStream fis = null;
		try {
			File configFile = new File(configFilePath);
	        fis = new BufferedInputStream(new FileInputStream(configFile));
	        props.load(fis);
        }
        catch(IOException e){
        	System.err.println("Could not load configuration file "+configFilePath);
        }
        finally{
			if(fis!=null){
			    try {
					fis.close();
				}
				catch (IOException e) {
		        	System.err.println("Error closing file stream of configuration file "+configFilePath);
				}
			}
        }
        return props;
	}
654397
static Iterator<Token> tokenize(ConfigOrigin origin, Reader input, ConfigSyntax flavor) {
        return new TokenIterator(origin, input, flavor != ConfigSyntax.JSON);
    }
192968
public static Set<Path> getPaths(InputFormat<?, ?> inputFormat, Path location) throws IOException {
    JobConf jobConf = new JobConf(getHadoopConfiguration());

    Set<Path> paths = Sets.newHashSet();

    FileInputFormat.addInputPaths(jobConf, location.toString());
    InputSplit[] splits = inputFormat.getSplits(jobConf, 1000);
    for (InputSplit split : splits) {
      if (!(split instanceof FileSplit)) {
        throw new IOException("Not a file split. Found " + split.getClass().getName());
      }
      FileSplit fileSplit = (FileSplit) split;
      paths.add(fileSplit.getPath());
    }

    return paths;
  }
271544
@Override
  public MtasTokenCollection createTokenCollection(Reader reader)
      throws MtasParserException, MtasConfigException {
    AtomicInteger position = new AtomicInteger(0);
    Integer unknownAncestors = 0;

    Map<String, Set<Integer>> idPositions = new HashMap<>();
    Map<String, Integer[]> idOffsets = new HashMap<>();

    Map<String, Map<Integer, Set<String>>> updateList = createUpdateList();
    Map<String, List<MtasParserObject>> currentList = createCurrentList();

    tokenCollection = new MtasTokenCollection();
    MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
    try (MtasBufferedReader br = new MtasBufferedReader(reader)) {
      String line;
      int currentOffset;
      int previousOffset = br.getPosition();
      MtasParserType tmpCurrentType;
      MtasParserObject currentObject;
      Pattern groupPattern = Pattern.compile("^<([^\\/>]+)\\/>$");
      Pattern groupStartPattern = Pattern
          .compile("^<([^>\\/\\s][^>\\s]*)(|\\s[^>]+)>$");
      Pattern groupEndPattern = Pattern.compile("^<\\/([^>\\s]+)>$");
      Pattern attributePattern = Pattern.compile("([^\\s]+)=\"([^\"]*)\"");
      while ((line = br.readLine()) != null) {
        currentOffset = br.getPosition();
        // group
        if (line.trim().matches("^<[^>]*>$")) {
          Matcher matcherGroupStart = groupStartPattern.matcher(line.trim());
          Matcher matcherGroupEnd = groupEndPattern.matcher(line.trim());
          Matcher matcherGroup = groupPattern.matcher(line.trim());
          if (matcherGroup.find()) {
            // full group, ignore
          } else if (matcherGroupStart.find()) {
            // start group
            // System.out.println("Start "+matcherGroupStart.group(1)+" -
            // "+matcherGroupStart.group(2));
            if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                && (currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION).isEmpty())
                && (tmpCurrentType = groupTypes
                    .get(matcherGroupStart.group(1))) != null) {
              currentObject = new MtasParserObject(tmpCurrentType);
              currentObject.setUnknownAncestorNumber(unknownAncestors);
              currentObject.setRealOffsetStart(previousOffset);
              String attributeText = matcherGroupStart.group(2).trim();
              if (!attributeText.equals("")) {
                Matcher matcherAttribute = attributePattern
                    .matcher(attributeText);
                currentObject.objectAttributes = new HashMap<String, String>();
                while (matcherAttribute.find()) {
                  currentObject.objectAttributes.put(matcherAttribute.group(1),
                      matcherAttribute.group(2));
                }
              }
              if (!prevalidateObject(currentObject, currentList)) {
                unknownAncestors++;
              } else {
                currentList.get(MAPPING_TYPE_GROUP).add(currentObject);
                unknownAncestors = 0;
              }
            }
          } else if (matcherGroupEnd.find()) {
            // end group
            if (!currentList.get(MAPPING_TYPE_GROUP).isEmpty()) {
              if ((tmpCurrentType = groupTypes
                  .get(matcherGroupEnd.group(1))) != null) {
                currentObject = currentList.get(MAPPING_TYPE_GROUP)
                    .remove(currentList.get(MAPPING_TYPE_GROUP).size() - 1);
                assert unknownAncestors == 0 : "error in administration "
                    + currentObject.getType().getName();
                // ignore text: should not occur
                currentObject.setRealOffsetEnd(currentOffset - 1);
                idPositions.put(currentObject.getId(),
                    currentObject.getPositions());
                idOffsets.put(currentObject.getId(), currentObject.getOffset());
                currentObject.updateMappings(idPositions, idOffsets);
                unknownAncestors = currentObject.getUnknownAncestorNumber();
                computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                    currentList, updateList);
              }
            }
          }
        } else {
          if ((currentList.get(MAPPING_TYPE_RELATION).isEmpty())
              && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION).isEmpty())
              && (currentList.get(MAPPING_TYPE_WORD).isEmpty())
              && (currentList.get(MAPPING_TYPE_WORD_ANNOTATION).isEmpty())
              && (wordType != null)) {
            // start word
            currentObject = new MtasParserObject(wordType);
            currentObject.setOffsetStart(previousOffset);
            currentObject.setRealOffsetStart(previousOffset);
            currentObject.setUnknownAncestorNumber(unknownAncestors);
            if (!prevalidateObject(currentObject, currentList)) {
              unknownAncestors++;
            } else {
              int p = position.getAndIncrement();
              currentObject.addPosition(p);
              currentList.get(MAPPING_TYPE_WORD).add(currentObject);
              unknownAncestors = 0;
            }
            if ((currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION).isEmpty())
                && (!currentList.get(MAPPING_TYPE_WORD).isEmpty())) {
              // start and finish word annotations
              String[] items = line.split("\t");
              for (int i = 0; i < items.length; i++) {
                if ((tmpCurrentType = wordAnnotationTypes.get(i)) != null) {
                  // start word annotation
                  currentObject = new MtasParserObject(tmpCurrentType);
                  currentObject.setRealOffsetStart(previousOffset);
                  currentObject.addPositions(currentList.get(MAPPING_TYPE_WORD)
                      .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
                      .getPositions());
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                        .add(currentObject);
                    unknownAncestors = 0;
                  }
                  // finish word annotation
                  if (unknownAncestors > 0) {
                    unknownAncestors--;
                  } else {
                    currentObject = currentList
                        .get(MAPPING_TYPE_WORD_ANNOTATION).remove(
                            currentList.get(MAPPING_TYPE_WORD_ANNOTATION).size()
                                - 1);
                    assert unknownAncestors == 0 : "error in administration "
                        + currentObject.getType().getName();
                    currentObject.setText(items[i]);
                    currentObject.setRealOffsetEnd(currentOffset - 1);
                    idPositions.put(currentObject.getId(),
                        currentObject.getPositions());
                    idOffsets.put(currentObject.getId(),
                        currentObject.getOffset());
                    // offset always null, so update later with word (should be
                    // possible)
                    if ((currentObject.getId() != null)
                        && (!currentList.get(MAPPING_TYPE_WORD).isEmpty())) {
                      currentList.get(MAPPING_TYPE_WORD)
                          .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
                          .addUpdateableIdWithOffset(currentObject.getId());
                    }
                    currentObject.updateMappings(idPositions, idOffsets);
                    unknownAncestors = currentObject.getUnknownAncestorNumber();
                    computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                        currentList, updateList);
                  }
                }
              }
            }
            // finish word
            if (unknownAncestors > 0) {
              unknownAncestors--;
            } else {
              currentObject = currentList.get(MAPPING_TYPE_WORD)
                  .remove(currentList.get(MAPPING_TYPE_WORD).size() - 1);
              assert unknownAncestors == 0 : "error in administration "
                  + currentObject.getType().getName();
              currentObject.setText(null);
              currentObject.setOffsetEnd(currentOffset - 1);
              currentObject.setRealOffsetEnd(currentOffset - 1);
              // update ancestor groups with position and offset
              for (MtasParserObject currentGroup : currentList
                  .get(MAPPING_TYPE_GROUP)) {
                currentGroup.addPositions(currentObject.getPositions());
                currentGroup.addOffsetStart(currentObject.getOffsetStart());
                currentGroup.addOffsetEnd(currentObject.getOffsetEnd());
              }
              idPositions.put(currentObject.getId(),
                  currentObject.getPositions());
              idOffsets.put(currentObject.getId(), currentObject.getOffset());
              currentObject.updateMappings(idPositions, idOffsets);
              unknownAncestors = currentObject.getUnknownAncestorNumber();
              computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                  currentList, updateList);
            }
          }
        }
        previousOffset = br.getPosition();
      }
    } catch (IOException e) {
      log.debug(e);
      throw new MtasParserException(e.getMessage());
    }
    // update tokens with offset
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_OFFSET).entrySet()) {
      for (String refId : updateItem.getValue()) {
        Integer[] refOffset = idOffsets.get(refId);
        if (refOffset != null) {
          tokenCollection.get(updateItem.getKey()).addOffset(refOffset[0],
              refOffset[1]);
        }
      }
    }
    // update tokens with position
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_POSITION).entrySet()) {
      for (String refId : updateItem.getValue()) {
        MtasToken token = tokenCollection.get(updateItem.getKey());
        token.addPositions(idPositions.get(refId));
      }
    }
    // final check
    tokenCollection.check(autorepair, makeunique);
    return tokenCollection;
  }
290224
public void init(Configuration conf, Path generatedModel) throws IOException, InterruptedException {
		FileSystem fileSystem = FileSystem.get(conf);
		for(Category category : Category.values()) {
			wordCountPerCategory.put(category, new HashMap<String, Integer>()); // init token count
		}
		// Use a HashSet to calculate the total vocabulary size
		Set<String> vocabulary = new HashSet<String>();
		// Read tuples from generate job
		for(FileStatus fileStatus : fileSystem.globStatus(generatedModel)) {
      TupleFile.Reader reader = new TupleFile.Reader(fileSystem, conf, fileStatus.getPath());
      Tuple tuple = new Tuple(reader.getSchema());
			while(reader.next(tuple)) {
				// Read Tuple
				Integer count = (Integer) tuple.get("count");
				Category category = (Category) tuple.get("category");
				String word = tuple.get("word").toString();
				vocabulary.add(word);
				tokensPerCategory.put(category, MapUtils.getInteger(tokensPerCategory, category, 0) + count);
				wordCountPerCategory.get(category).put(word, count);
			}
			reader.close();
		}
		V = vocabulary.size();
	}