Factory function that creates a Tokenizer object, the input format
is set based on filename (*.xml is XML, otherwise binary)

@param filename input filename
771972
static public Tokenizer getTokenizer(String filename) throws IOException {
    if(filename.toLowerCase().endsWith("xml")) {
      return new XmlTokenizer(filename);
    } else {
      return new BinaryTokenizer(filename);
    }
  }
1285093
public static InputStream loadFile(String fileName) {
        logger.entering(fileName);
        ClassLoader loader = Thread.currentThread().getContextClassLoader();
        InputStream iStream = loader.getResourceAsStream(fileName);
        if (iStream == null) {
            try {
                iStream = new FileInputStream(fileName);
            } catch (FileNotFoundException e) { // NOSONAR
                // Gobbling the checked exception here and doing nothing with it
                // because we are explicitly checking if the inputstream is null
                // and then throwing a runtime exception
            }
        }
        if (iStream == null) {
            throw new IllegalArgumentException("[" + fileName + "] is not a valid resource");
        }
        logger.exiting();
        return iStream;
    }
105643
@Override
    public XMLStreamReader2 createXMLStreamReader(File f)
        throws XMLStreamException
    {
        /* true for auto-close, since caller has no access to the underlying
         * input stream created from the File
         */
        return createSR(f, false, true);
    }
520666
public String getFileTokenUrl(String filename) {
        try {
            String baseUrl = conf.get(PROP_HTTP_TOKEN_BASE_URL, "");
            String secretKey = conf.get(PROP_HTTP_SECRET_KEY, "");
            if (Strings.isBlank(filename) || Strings.isBlank(secretKey) || Strings.isBlank(baseUrl)) {
                log.info("[FastdfsService] filename & http_token_base_url & http_secret_key must not empty");
                return "";
            }
            int pos = filename.indexOf(FILENAME_SEPERATOR);
            String tokenFilename = filename.substring(pos + 1);
            long ts = Times.getTS();
            StringBuilder sb = new StringBuilder();
            sb.append(baseUrl);
            if (!baseUrl.endsWith(FILENAME_SEPERATOR))
                sb.append(FILENAME_SEPERATOR);
            sb.append(filename);
            sb.append("?token=");
            sb.append(getToken(tokenFilename, ts, secretKey));
            sb.append("&ts=");
            sb.append(ts);
            return sb.toString();
        } catch (Exception e) {
            log.error(e);
        }
        return "";
    }
1010232
@Override
  public MtasTokenCollection createTokenCollection(Reader reader)
      throws MtasParserException, MtasConfigException {
    Boolean hasRoot = rootTag == null ? true : false;
    Boolean parsingContent = contentTag == null ? true : false;
    String textContent = null;
    Integer unknownAncestors = 0;
    Integer lastOffset = 0;

    AtomicInteger position = new AtomicInteger(0);
    Map<String, Set<Integer>> idPositions = new HashMap<>();
    Map<String, Integer[]> idOffsets = new HashMap<>();

    Map<String, Map<Integer, Set<String>>> updateList = createUpdateList();
    Map<String, List<MtasParserObject>> currentList = createCurrentList();
    Map<String, Map<String, String>> variables = createVariables();

    tokenCollection = new MtasTokenCollection();
    MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
    XMLInputFactory factory = XMLInputFactory.newInstance();
    try {      
      XMLStreamReader streamReader = factory.createXMLStreamReader(reader);
      QName qname;
      try {
        int event = streamReader.getEventType();
        MtasParserType<?> currentType;
        MtasParserType<?> tmpCurrentType;
        MtasParserType<?> tmpVariableType;
        MtasParserObject currentObject = null;
        MtasParserObject variableObject = null;
        while (true) {
          switch (event) {
          case XMLStreamConstants.START_DOCUMENT:
            log.debug("start of document");
            String encodingScheme = streamReader.getCharacterEncodingScheme();
            if (encodingScheme == null) {
              //ignore for now
              log.info("No encodingScheme found, assume utf-8");
              //throw new MtasParserException("No encodingScheme found");              
            } else if (!encodingScheme.equalsIgnoreCase("utf-8")) {
              throw new MtasParserException(
                  "XML not UTF-8 encoded but '" + encodingScheme + "'");
            }
            break;
          case XMLStreamConstants.END_DOCUMENT:
            log.debug("end of document");
            break;
          case XMLStreamConstants.SPACE:
            // set offset (end of start-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            break;
          case XMLStreamConstants.START_ELEMENT:
            // get data
            qname = streamReader.getName();
            // check for rootTag
            if (!hasRoot) {
              if (qname.equals(getQName(rootTag))) {
                hasRoot = true;
              } else {
                throw new MtasParserException("No " + rootTag);
              }
              // parse content
            } else {
              if ((tmpVariableType = variableTypes.get(qname)) != null) {
                variableObject = new MtasParserObject(tmpVariableType);
                collectAttributes(variableObject, streamReader);
                computeVariablesFromObject(variableObject, currentList,
                    variables);
              }
              if (parsingContent) {
                // check for relation : not within word, not within
                // groupAnnotation
                if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (tmpCurrentType = relationTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_RELATION).add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for relation annotation: not within word, but within
                  // relation
                } else if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (!currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (tmpCurrentType = relationAnnotationTypes
                        .get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_RELATION_ANNOTATION)
                        .add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for group: not within word, not within relation, not
                  // within groupAnnotation
                } else if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (tmpCurrentType = groupTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_GROUP).add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for group annotation: not within word, not within
                  // relation, but within group
                } else if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (!currentList.get(MAPPING_TYPE_GROUP).isEmpty())
                    && (tmpCurrentType = groupAnnotationTypes
                        .get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for word: not within relation, not within
                  // groupAnnotation, not within word, not within wordAnnotation
                } else if ((currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_WORD_ANNOTATION).isEmpty())
                    && (tmpCurrentType = wordTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setOffsetStart(lastOffset);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentObject.addPosition(position.getAndIncrement());
                    currentList.get(MAPPING_TYPE_WORD).add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for word annotation: not within relation, not within
                  // groupAnnotation, but within word
                } else if ((currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (!currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (tmpCurrentType = wordAnnotationTypes
                        .get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.addPositions(currentList.get(MAPPING_TYPE_WORD)
                      .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
                      .getPositions());
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                        .add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for references: within relation
                } else if (!currentList.get(MAPPING_TYPE_RELATION).isEmpty()
                    && (tmpCurrentType = refTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_REF).add(currentObject);
                    unknownAncestors = 0;
                    // add reference to ancestor relations
                    for (MtasParserObject currentRelation : currentList
                        .get(MAPPING_TYPE_RELATION)) {
                      currentRelation.addRefId(currentObject
                          .getAttribute(currentType.getRefAttributeName()));
                      // register mapping for relation (for recursive relations)
                      SortedSet<String> keyMapList;
                      if (currentRelation.getId() != null) {
                        if (relationKeyMap
                            .containsKey(currentRelation.getId())) {
                          keyMapList = relationKeyMap
                              .get(currentRelation.getId());
                        } else {
                          keyMapList = new TreeSet<>();
                          relationKeyMap.put(currentRelation.getId(),
                              keyMapList);
                        }
                        keyMapList.add(currentObject
                            .getAttribute(currentType.getRefAttributeName()));
                      }
                    }
                  }
                } else {
                  unknownAncestors++;
                }
                // check for start content
              } else if (qname.equals(getQName(contentTag))) {
                parsingContent = true;
                // unexpected
              } else if (!allowNonContent) {
                throw new MtasParserException(
                    "Unexpected " + qname.getLocalPart() + " in document");
              }
            }
            // set offset (end of start-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            break;
          case XMLStreamConstants.END_ELEMENT:
            // set offset (end of end-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            // get data
            qname = streamReader.getName();
            // parse content
            if (parsingContent) {
              if (unknownAncestors > 0) {
                unknownAncestors--;
                // check for reference: because otherwise currentList should
                // contain no references
              } else if (!currentList.get(MAPPING_TYPE_REF).isEmpty()) {
                if ((currentType = refTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_REF)
                      .remove(currentList.get(MAPPING_TYPE_REF).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  // ignore text and realOffset: not relevant
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for wordAnnotation: because otherwise currentList
                // should contain no wordAnnotations
              } else if (!currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                  .isEmpty()) {
                if ((currentType = wordAnnotationTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                      .remove(
                          currentList.get(MAPPING_TYPE_WORD_ANNOTATION).size()
                              - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  // offset always null, so update later with word (should be
                  // possible)
                  if ((currentObject.getId() != null)
                      && (!currentList.get(MAPPING_TYPE_WORD).isEmpty())) {
                    currentList.get(MAPPING_TYPE_WORD)
                        .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
                        .addUpdateableIdWithOffset(currentObject.getId());
                  }
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for word: because otherwise currentList should contain
                // no words
              } else if (!currentList.get(MAPPING_TYPE_WORD).isEmpty()) {
                if ((currentType = wordTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_WORD)
                      .remove(currentList.get(MAPPING_TYPE_WORD).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setOffsetEnd(lastOffset);
                  currentObject.setRealOffsetEnd(lastOffset);
                  // update ancestor groups with position and offset
                  for (MtasParserObject currentGroup : currentList
                      .get(MAPPING_TYPE_GROUP)) {
                    currentGroup.addPositions(currentObject.getPositions());
                    currentGroup.addOffsetStart(currentObject.getOffsetStart());
                    currentGroup.addOffsetEnd(currentObject.getOffsetEnd());
                  }
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for group annotation: because otherwise currentList
                // should contain no groupAnnotations
              } else if (!currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                  .isEmpty()) {
                if ((currentType = groupAnnotationTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                      .remove(
                          currentList.get(MAPPING_TYPE_GROUP_ANNOTATION).size()
                              - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for relation annotation
              } else if (!currentList.get(MAPPING_TYPE_RELATION_ANNOTATION)
                  .isEmpty()) {
                if ((currentType = relationAnnotationTypes
                    .get(qname)) != null) {
                  currentObject = currentList
                      .get(MAPPING_TYPE_RELATION_ANNOTATION).remove(currentList
                          .get(MAPPING_TYPE_RELATION_ANNOTATION).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for relation
              } else if (!currentList.get(MAPPING_TYPE_RELATION).isEmpty()) {
                if ((currentType = relationTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_RELATION).remove(
                      currentList.get(MAPPING_TYPE_RELATION).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  // ignore text: should not occur
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for group
              } else if (!currentList.get(MAPPING_TYPE_GROUP).isEmpty()) {
                if ((currentType = groupTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_GROUP)
                      .remove(currentList.get(MAPPING_TYPE_GROUP).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  // ignore text: should not occur
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  unknownAncestors--;
                }
              } else if (qname.equals(getQName("text"))) {
                parsingContent = false;
                assert unknownAncestors == 0 : "error in administration unknownAncestors";
                assert currentList.get(MAPPING_TYPE_REF)
                    .isEmpty() : "error in administration references";
                assert currentList.get(MAPPING_TYPE_GROUP)
                    .isEmpty() : "error in administration groups";
                assert currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                    .isEmpty() : "error in administration groupAnnotations";
                assert currentList.get(MAPPING_TYPE_WORD)
                    .isEmpty() : "error in administration words";
                assert currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                    .isEmpty() : "error in administration wordAnnotations";
                assert currentList.get(MAPPING_TYPE_RELATION)
                    .isEmpty() : "error in administration relations";
                assert currentList.get(MAPPING_TYPE_RELATION_ANNOTATION)
                    .isEmpty() : "error in administration relationAnnotations";
              }
            }
            // forget text
            textContent = null;
            break;
          case XMLStreamConstants.CHARACTERS:
            // set offset (end of start-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            // check for text
            if (streamReader.hasText()) {
              textContent = streamReader.getText();
            }
            if (currentObject != null && unknownAncestors.equals(0)) {
              currentObject.addText(textContent);
            }
            break;
          default:
            break;
          }
          if (!streamReader.hasNext()) {
            break;
          }
          event = streamReader.next();
        }
      } finally {
        streamReader.close();
      }
      // final checks
      assert unknownAncestors == 0 : "error in administration unknownAncestors";
      assert hasRoot : "no " + rootTag;
    } catch (XMLStreamException e) {
      log.debug(e);      
      throw new MtasParserException("No valid XML: " + e.getMessage());
    }

    // update tokens with variable
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_VARIABLE).entrySet()) {
      MtasToken token = tokenCollection.get(updateItem.getKey());
      String encodedPrefix = token.getPrefix();
      String encodedPostfix = token.getPostfix();
      token.setValue(decodeAndUpdateWithVariables(encodedPrefix, encodedPostfix,
          variables));
    }
    // update tokens with offset
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_OFFSET).entrySet()) {
      Set<String> refIdList = new HashSet<>();
      for (String refId : updateItem.getValue()) {
        if (idPositions.containsKey(refId)) {
          refIdList.add(refId);
        }
        if (relationKeyMap.containsKey(refId)) {
          refIdList.addAll(recursiveCollect(refId, relationKeyMap, 10));
        }
      }
      for (String refId : refIdList) {
        Integer[] refOffset = idOffsets.get(refId);
        Integer tokenId = updateItem.getKey();
        if (tokenId != null && refOffset != null) {
          MtasToken token = tokenCollection.get(tokenId);
          token.addOffset(refOffset[0], refOffset[1]);
        }
      }
    }
    // update tokens with position
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_POSITION).entrySet()) {
      HashSet<String> refIdList = new HashSet<>();
      for (String refId : updateItem.getValue()) {
        if (idPositions.containsKey(refId)) {
          refIdList.add(refId);
        }
        if (relationKeyMap.containsKey(refId)) {
          refIdList.addAll(recursiveCollect(refId, relationKeyMap, 10));
        }
      }
      for (String refId : refIdList) {
        Set<Integer> refPositions = idPositions.get(refId);
        Integer tokenId = updateItem.getKey();
        if (tokenId != null && refPositions != null) {
          MtasToken token = tokenCollection.get(tokenId);
          token.addPositions(refPositions);
        }
      }
    }
    // final check
    tokenCollection.check(autorepair, makeunique);
    return tokenCollection;
  }
901610
public static File fileFromString(String fileName) {
        try {
            return new File(new URI(fileName).getPath());
        } catch (URISyntaxException ex ) {
            // Not a valid uri
            return new File(fileName);
        }
    }
678772
static void createConfigFile(String xmlName, Resource configFile) throws IOException {
	configFile.createFile(true);
	createFileFromResource("/resource/config/" + xmlName + ".xml", configFile.getAbsoluteResource());
    }
1345160
def NewFile(self, filename, encoding, options):
        
        ret = libxml2mod.xmlReaderNewFile(self._o, filename, encoding, options)
        return ret
906621
private void processFile(String fileName)
      throws ValidationException, ParserConfigurationException, SAXException, IOException,
          SchedulerException, ClassNotFoundException, ParseException, XPathException {

    prepForProcessing();

    logger.info("Parsing XML file: " + fileName);
    InputSource is = new InputSource(getInputStream(fileName));
    // is.setSystemId(systemId);

    process(is);

    maybeThrowValidationException();
  }
1315485
public Parser<Token> token(String tokenName) {
    return Parsers.token(InternalFunctors.tokenWithSameValue(word(tokenName)));
  }
625224
public synchronized void setFilename(String filename)
    {
        if (filename!=null)
        {
            filename=filename.trim();
            if (filename.length()==0)
                filename=null;
        }
        
        if (isStarted() && _filename!=null && filename==null)
            _out=null;
        
        _reopen=isStarted() &&
            ((_filename==null && filename!=null)||
             (_filename!=null && !_filename.equals(filename)));
        _filename=filename;

        if (!isStarted() && _filename!=null)
            _out=null;
    }
559096
public File inFileNamed(String filenamePattern)
    {
        if (filenamePattern != null && !filenamePattern.isEmpty())
        {
            f.setFilenamePattern(new RegexParameterizedPatternParser(filenamePattern));
        }
        return f;
    }
58759
public static BufferedReader asBufferedReader(String fileName) throws IOException {
		Validate.notBlank(fileName, "filename is blank");
		return asBufferedReader(getPath(fileName));
	}
1284725
public void load(String filename) throws FileNotFoundException, SAXException, IOException, ParserConfigurationException {
    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
    DocumentBuilder builder = factory.newDocumentBuilder();
    loadMessages(builder.parse(new CSFileInputStream(filename)));
  }
1077928
public CharSequence readSource(JavaFileObject filename) {
        try {
            inputFiles.add(filename);
            return filename.getCharContent(false);
        } catch (IOException e) {
            log.error("error.reading.file", filename, JavacFileManager.getMessage(e));
            return null;
        }
    }
200216
public static File toFile(final String filename) {
        if (filename == null) {
            return null;
        }
        String f;
        try {
            f = URLDecoder.decode(filename, UTF8);
        } catch (final UnsupportedEncodingException e) {
            throw new RuntimeException(e);
        }
        f = f.replace(WINDOWS_SEPARATOR, File.separator).replace(UNIX_SEPARATOR, File.separator);
        return new File(f);
    }
979394
public static XElement parseXML(String fileName) throws XMLStreamException {
        try (InputStream in = new FileInputStream(fileName)) {
            return parseXML(in);
        } catch (IOException ex) {
            throw new XMLStreamException(ex);
        }
    }