This file can be any file type supported by the DataReader. E.g. for the OSMReader it is the
OSM xml (.osm), a compressed xml (.osm.zip or .osm.gz) or a protobuf file (.pbf)
9059
public GraphHopper setDataReaderFile(String dataReaderFileStr) {
        ensureNotLoaded();
        if (isEmpty(dataReaderFileStr))
            throw new IllegalArgumentException("Data reader file cannot be empty.");

        dataReaderFile = dataReaderFileStr;
        return this;
    }
158396
private Document toXML(Collection<ArchiveEntry> files) throws Exception
    {
        Document doc = new DOMDocument();

        Element packageElement = new DOMElement("package");
        doc.setRootElement(packageElement);

        Element infoElement = new DOMElement("infos");
        packageElement.add(infoElement);
        addInfoElements(infoElement);

        Element filesElement = new DOMElement(FILES_TAG);
        packageElement.add(filesElement);
        addFileElements(files, filesElement);

        return doc;
    }
1137769
static KAFDocument load(File file) throws IOException, JDOMException, KAFNotValidException {
	SAXBuilder builder = new SAXBuilder();
	Document document = (Document) builder.build(file);
	Element rootElem = document.getRootElement();
	return DOMToKAF(document);
    }
907892
def data_file(self):
        """"""

        if self._data_file is None:
            fileobj = _SubFile(self._fileobj, self.data_offset)

            if self.headers["archive_compression"] == b"xz":
                if not getattr(sys.modules[__name__], 'lzma', False):
                    raise NoLZMAModuleError('lzma module not present')
                self._data_file = lzma.LZMAFile(fileobj)
            else:
                self._data_file = gzip.GzipFile(fileobj=fileobj)

        return self._data_file
1118565
function (filename) {
            var contents = fs.readFileSync(filename, 'utf-8');
            if(contents) {
                //Windows is the BOM. Skip the Byte Order Mark.
                contents = contents.substring(contents.indexOf('<'));
            }
            return new et.ElementTree(et.XML(contents));
        }
1204965
def parse_meta(filename, data):
    
    if "." not in filename:
        raise MetaParsingException(
            "Can't recognize type of your metadata ('%s')!" % filename
        )

    suffix = filename.rsplit(".", 1)[1].lower()

    if suffix not in SUPPORTED_FILES:
        raise MetaParsingException("Can't parse file of type '%s'!" % suffix)

    fp = validator.FieldParser()
    for key, val in SUPPORTED_FILES[suffix](data).items():
        fp.process(key, val)

    return fp.get_epublication()
40288
function(file, type) {
    var promise = new Parse.Promise();

    if (typeof(FileReader) === "undefined") {
      return Parse.Promise.error(new Parse.Error(
          Parse.Error.FILE_READ_ERROR,
          "Attempted to use a FileReader on an unsupported browser."));
    }

    var reader = new FileReader();
    reader.onloadend = function() {
      if (reader.readyState !== 2) {
        promise.reject(new Parse.Error(
            Parse.Error.FILE_READ_ERROR,
            "Error reading file."));
        return;
      }

      var dataURL = reader.result;
      var matches = /^data:([^;]*);base64,(.*)$/.exec(dataURL);
      if (!matches) {
        promise.reject(new Parse.Error(
            Parse.ERROR.FILE_READ_ERROR,
            "Unable to interpret data URL: " + dataURL));
        return;
      }

      promise.resolve(matches[2], type || matches[1]);
    };
    reader.readAsDataURL(file);
    return promise;
  }
677963
function layerPBFReader(tag, layer, pbf) {
  if (tag === 15) {
    layer.version = pbf.readVarint();
  } else if (tag === 1) {
    layer.name = pbf.readString();
  } else if (tag === 5) {
    layer.extent = pbf.readVarint();
  } else if (tag === 2) {
    layer.features.push(pbf.pos);
  } else if (tag === 3) {
    layer.keys.push(pbf.readString());
  } else if (tag === 4) {
    let value = null;
    const end = pbf.readVarint() + pbf.pos;
    while (pbf.pos < end) {
      tag = pbf.readVarint() >> 3;
      value = tag === 1 ? pbf.readString() :
        tag === 2 ? pbf.readFloat() :
          tag === 3 ? pbf.readDouble() :
            tag === 4 ? pbf.readVarint64() :
              tag === 5 ? pbf.readVarint() :
                tag === 6 ? pbf.readSVarint() :
                  tag === 7 ? pbf.readBoolean() : null;
    }
    layer.values.push(value);
  }
}
629170
private String getMIMEType(String path) {
        String extension = getFileExtension(path);

        if (extension == null) {
            return FILE_TRANSFER_BINARY_MIME;
        }

        if ("zip".equals(extension) || "ear".equals(extension) || "war".equals(extension) || "jar".equals(extension) || "eba".equals(extension)) {
            return FILE_TRANSFER_ZIP_MIME;

        } else if ("pax".equals(extension)) {
            return FILE_TRANSFER_PAX_MIME;

        } else if ("gz".equals(extension) || "gzip".equals(extension)) {

            return FILE_TRANSFER_GZIP_MIME;

        } else if ("tar".equals(extension)) {
            return FILE_TRANSFER_TAR_MIME;

        } else if ("txt".equals(extension) || "log".equals(extension) || "trace".equals(extension) || "properties".equals(extension)) {
            return FILE_TRANSFER_TEXT_MIME;

        } else if ("xml".equals(extension) || "xslt".equals(extension) || "xsl".equals(extension)) {
            return FILE_TRANSFER_XML_MIME;

        } else if ("html".equals(extension) || "htm".equals(extension)) {
            return FILE_TRANSFER_HTML_MIME;

        } else {
            return FILE_TRANSFER_BINARY_MIME;
        }
    }
929661
private static void extractXmpData(@NotNull final Metadata metadata, @NotNull SequentialReader reader) throws IOException
    {
        byte[] bytes = readUntil(reader, "<?xpacket end=\"w\"?>".getBytes());
        String xmp = new String(bytes, Charsets.UTF_8);
        new XmpReader().extract(xmp, metadata);
    }
1037220
@Override
	public byte readDataType() {
		// prevent the handling of an empty Object
		if (buf.hasRemaining()) {
			do {
				// get the data type
				currentDataType = buf.get();
				log.trace("Data type: {}", currentDataType);
				switch (currentDataType) {
				case AMF.TYPE_NULL:
				case AMF.TYPE_UNDEFINED:
					return DataTypes.CORE_NULL;
				case AMF.TYPE_NUMBER:
					return DataTypes.CORE_NUMBER;
				case AMF.TYPE_BOOLEAN:
					return DataTypes.CORE_BOOLEAN;
				case AMF.TYPE_STRING:
				case AMF.TYPE_LONG_STRING:
					return DataTypes.CORE_STRING;
				case AMF.TYPE_CLASS_OBJECT:
				case AMF.TYPE_OBJECT:
					return DataTypes.CORE_OBJECT;
				case AMF.TYPE_MIXED_ARRAY:
					return DataTypes.CORE_MAP;
				case AMF.TYPE_ARRAY:
					return DataTypes.CORE_ARRAY;
				case AMF.TYPE_DATE:
					return DataTypes.CORE_DATE;
				case AMF.TYPE_XML:
					return DataTypes.CORE_XML;
				case AMF.TYPE_REFERENCE:
					return DataTypes.OPT_REFERENCE;
				case AMF.TYPE_UNSUPPORTED:
				case AMF.TYPE_MOVIECLIP:
				case AMF.TYPE_RECORDSET:
					// These types are not handled by core datatypes
					// So add the amf mask to them, this way the deserializer
					// will call back to readCustom, we can then handle or
					// return null
					return (byte) (currentDataType + DataTypes.CUSTOM_AMF_MASK);
				case AMF.TYPE_AMF3_OBJECT:
					log.debug("Switch to AMF3");
					return DataTypes.CORE_SWITCH;
				}
			} while (hasMoreProperties());
			log.trace("No more data types available");
			return DataTypes.CORE_END_OBJECT;
		}
		// empty object, may as well be null
		return DataTypes.CORE_NULL;
	}
957417
void readFile(final Reference ref, final URI parseFile) throws DITAOTException {
        currentFile = ref.filename;
        assert currentFile.isAbsolute();
        final URI src = parseFile != null ? parseFile : currentFile;
        assert src.isAbsolute();
        final URI rel = tempFileNameScheme.generateTempFileName(currentFile);
        outputFile = new File(job.tempDirURI.resolve(rel));
        final File outputDir = outputFile.getParentFile();
        if (!outputDir.exists() && !outputDir.mkdirs()) {
            logger.error("Failed to create output directory " + outputDir.getAbsolutePath());
            return;
        }
        validateMap = Collections.emptyMap();
        defaultValueMap = Collections.emptyMap();
        logger.info("Processing " + currentFile + " to " + outputFile.toURI());
        final String[] params = { currentFile.toString() };

        // Verify stub for current file is in Job
        final FileInfo fi = job.getFileInfo(currentFile);
        if (fi == null) {
            final FileInfo stub = new FileInfo.Builder()
                    .src(currentFile)
                    .uri(rel)
                    .result(currentFile)
                    .isInput(currentFile.equals(rootFile))
                    .build();
            job.add(stub);
        }

//        InputSource in = null;
        Result out = null;
        try {
            final TransformerFactory tf = TransformerFactory.newInstance();
            final SAXTransformerFactory stf = (SAXTransformerFactory) tf;
            final TransformerHandler serializer = stf.newTransformerHandler();

            XMLReader parser = getXmlReader(ref.format);
            XMLReader xmlSource = parser;
            for (final XMLFilter f: getProcessingPipe(currentFile)) {
                f.setParent(xmlSource);
                f.setEntityResolver(CatalogUtils.getCatalogResolver());
                xmlSource = f;
            }

            try {
                final LexicalHandler lexicalHandler = new DTDForwardHandler(xmlSource);
                parser.setProperty("http://xml.org/sax/properties/lexical-handler", lexicalHandler);
                parser.setFeature("http://xml.org/sax/features/lexical-handler", true);
            } catch (final SAXNotRecognizedException e) {}

//            in = new InputSource(src.toString());
            out = new StreamResult(new FileOutputStream(outputFile));
            serializer.setResult(out);
            xmlSource.setContentHandler(serializer);
            xmlSource.parse(src.toString());

            if (listFilter.isValidInput()) {
                processParseResult(currentFile);
                categorizeCurrentFile(ref);
            } else if (!currentFile.equals(rootFile)) {
                logger.warn(MessageUtils.getMessage("DOTJ021W", params).toString());
                failureList.add(currentFile);
            }
        } catch (final RuntimeException e) {
            throw e;
        } catch (final SAXParseException sax) {
            final Exception inner = sax.getException();
            if (inner != null && inner instanceof DITAOTException) {
                throw (DITAOTException) inner;
            }
            if (currentFile.equals(rootFile)) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ012F", params).toString() + ": " + sax.getMessage(), sax);
            } else if (processingMode == Mode.STRICT) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + sax.getMessage(), sax);
            } else {
                logger.error(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + sax.getMessage(), sax);
            }
            failureList.add(currentFile);
        } catch (final FileNotFoundException e) {
            if (!exists(currentFile)) {
                if (currentFile.equals(rootFile)) {
                    throw new DITAOTException(MessageUtils.getMessage("DOTA069F", params).toString(), e);
                } else if (processingMode == Mode.STRICT) {
                    throw new DITAOTException(MessageUtils.getMessage("DOTX008E", params).toString(), e);
                } else {
                    logger.error(MessageUtils.getMessage("DOTX008E", params).toString());
                }
            } else if (currentFile.equals(rootFile)) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ078F", params).toString() + " Cannot load file: " + e.getMessage(), e);
            } else if (processingMode == Mode.STRICT) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ079E", params).toString() + " Cannot load file: " + e.getMessage(), e);
            } else {
                logger.error(MessageUtils.getMessage("DOTJ079E", params).toString() + " Cannot load file: " + e.getMessage());
            }
            failureList.add(currentFile);
        } catch (final Exception e) {
            if (currentFile.equals(rootFile)) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ012F", params).toString() + ": " + e.getMessage(),  e);
            } else if (processingMode == Mode.STRICT) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + e.getMessage(), e);
            } else {
                logger.error(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + e.getMessage(), e);
            }
            failureList.add(currentFile);
        } finally {
            if (out != null) {
                try {
                    close(out);
                } catch (final IOException e) {
                    logger.error(e.getMessage(), e) ;
                }
            }
            if (failureList.contains(currentFile)) {
                FileUtils.deleteQuietly(outputFile);
            }
        }

        if (!listFilter.isValidInput() && currentFile.equals(rootFile)) {
            if (validate) {
                // stop the build if all content in the input file was filtered out.
                throw new DITAOTException(MessageUtils.getMessage("DOTJ022F", params).toString());
            } else {
                // stop the build if the content of the file is not valid.
                throw new DITAOTException(MessageUtils.getMessage("DOTJ034F", params).toString());
            }
        }

        doneList.add(currentFile);
        listFilter.reset();
        keydefFilter.reset();

    }
869280
function MarcXmlReader(options) {
  if (!(this instanceof MarcXmlReader)) {
    return new MarcXmlReader(options);
  }

  // File with records in XML format.
  this.recordsFile = null;
  // Flag is true when next() can be performed.
  this.readyToRead = false;

  // XML parser.
  this.parser = null;
  // Buffer for reading records from input file.
  this.bufferSize = 4096;
  this.buffer = new Buffer(this.bufferSize);
  // List of parsed records.
  this.records = null;

  // Encoding conversion stream.
  this.decoder = null;

  // File options.
  options = options || {};
  this.options = {
    // MARC format variation (MARC21, UNIMARC).
    format: (options.format || 'UNIMARC').toUpperCase(),
    // Input data encoding.
    encoding: options.encoding || null,
    // Permissive mode (ignore minor errors).
    permissive: options.permissive || false
  }
}
1166707
public static SimpleDataSet loadArffFile(Reader input, DataStore store) 
    {
        DataStore list = store.emptyClone();
        DoubleList weights = new DoubleList();
        
        BufferedReader br = new BufferedReader(input);
        
        int numOfVars = 0;
        int numReal = 0;
        List<Boolean> isReal = new ArrayList<>();
        List<String> variableNames = new ArrayList<>();
        List<HashMap<String, Integer>> catVals = new  ArrayList<>();
        String line = null;
        CategoricalData[] categoricalData = null;
        try
        {
            boolean atData = false;
            while( (line = br.readLine()) != null )
            {
                if(line.startsWith("%") || line.trim().isEmpty())
                    continue;///Its a comment, skip
                
                line = line.trim();
                
                if(line.startsWith("@") && !atData)
                {
                    line = line.substring(1).toLowerCase();
                    
                    
                    if(line.toLowerCase().startsWith("data"))
                    {
                        categoricalData = new CategoricalData[numOfVars-numReal];
                        
                        int k = 0;
                        for(int i = 0; i < catVals.size(); i++)
                        {
                            if(catVals.get(i) != null)
                            {
                                categoricalData[k] = new CategoricalData(catVals.get(i).size());
                                categoricalData[k].setCategoryName(variableNames.get(i));
                                for(Entry<String, Integer> entry : catVals.get(i).entrySet())
                                    categoricalData[k].setOptionName(entry.getKey(), entry.getValue());
                                k++;
                            }
                        }
                        
                        //prept to start reading in data
                        list.setNumNumeric(numReal);
                        list.setCategoricalDataInfo(categoricalData);
                        atData = true;
                        continue;
                    }
                    else if(!line.toLowerCase().startsWith("attribute"))
                        continue;
                    numOfVars++;
                    line = line.substring("attribute".length()).trim();//Remove the space, it could be multiple spaces
                    
                    String variableName = null;
                    line = line.replace("\t", " ");
                    if(line.startsWith("'"))
                    {
                        Pattern p = Pattern.compile("'.+?'");
                        Matcher m = p.matcher(line);
                        m.find();
                        variableName = nameTrim(m.group());
                                
                        line = line.replaceFirst("'.+?'", "placeHolder");
                    }
                    else
                        variableName = nameTrim(line.trim().replaceAll("\\s+.*", ""));
                    variableNames.add(variableName);
                    String[] tmp = line.split("\\s+", 2);
                    
                    
                    if(tmp[1].trim().equals("real") || tmp[1].trim().equals("numeric") || tmp[1].trim().startsWith("integer"))
                    {
                        numReal++;
                        isReal.add(true);
                        catVals.add(null);
                    }
                    else//Not correct, but we arent supporting anything other than real and categorical right now
                    {
                        isReal.add(false);
                        String cats = tmp[1].replace("{", "").replace("}", "").trim();
                        if(cats.endsWith(","))
                            cats = cats.substring(0, cats.length()-1);
                        String[] catValsRaw =  cats.split(",");
                        HashMap<String, Integer> tempMap = new HashMap<String, Integer>();
                        for(int i = 0; i < catValsRaw.length; i++)
                        {
                            catValsRaw[i] = nameTrim(catValsRaw[i]);
                            tempMap.put(catValsRaw[i], i);
                        }
                        catVals.add(tempMap);
                    }
                }
                else if(atData && !line.isEmpty())
                {
                    double weight = 1.0;
                    String[] tmp = line.split(",");
                    if(tmp.length != isReal.size())
                    {
                        String s = tmp[isReal.size()];
                        if(tmp.length == isReal.size()+1)//{#} means the # is the weight
                        {
                            if(!s.matches("\\{\\d+(\\.\\d+)?\\}"))
                                throw new RuntimeException("extra column must indicate a data point weigh in the form of \"{#}\", instead bad token " + s + " was found");
                            weight = Double.parseDouble(s.substring(1, s.length()-1));
                        }
                        else
                        {
                            throw new RuntimeException("Column had " + tmp.length + " values instead of " + isReal.size());
                        }
                    }
                        
                    DenseVector vec = new DenseVector(numReal);
                    
                    int[] cats = new int[numOfVars - numReal];
                    int k = 0;//Keeping track of position in cats
                    for(int i  = 0; i < isReal.size(); i++)
                    {
                        String val_string = tmp[i].trim();
                        if (isReal.get(i))
                            if (val_string.equals("?"))//missing value, indicated by NaN
                                vec.set(i - k, Double.NaN);
                            else
                                vec.set(i - k, Double.parseDouble(val_string));
                        else//Categorical
                        {
                            tmp[i] = nameTrim(tmp[i]).trim().toLowerCase();
                            if(tmp[i].equals("?"))//missing value, indicated by -1
                                cats[k++] = -1;
                            else
                                cats[k++] = catVals.get(i).get(tmp[i]);
                        }
                    }
                    
                    list.addDataPoint(new DataPoint(vec, cats, categoricalData)); 
                    weights.add(weight);
                }
            }
        }
        catch (IOException ex)
        {
            
        }
        
        SimpleDataSet dataSet =  new SimpleDataSet(list);
        for(int i = 0; i < weights.size(); i++)
            dataSet.setWeight(i, weights.getD(i));
        int k = 0;
        for (int i = 0; i < isReal.size(); i++)
            if (isReal.get(i))
                dataSet.setNumericName(variableNames.get(i), k++);
        
        return dataSet;
    }
559149
def create_from_xml(sourcefileTag, get_value_from_logfile, listOfColumns,
                        correct_only, log_zip_cache, columns_relevant_for_diff,
                        result_file_or_url):
        '''
        
        '''

        def read_logfile_lines(log_file):
            if not log_file:
                return []
            log_file_url = Util.make_url(log_file)
            url_parts = urllib.parse.urlparse(log_file_url, allow_fragments=False)
            log_zip_path = os.path.dirname(url_parts.path) + ".zip"
            log_zip_url = urllib.parse.urlunparse((url_parts.scheme, url_parts.netloc,
                log_zip_path, url_parts.params, url_parts.query, url_parts.fragment))
            path_in_zip = urllib.parse.unquote(
                os.path.relpath(url_parts.path, os.path.dirname(log_zip_path)))
            if log_zip_url.startswith("file:///") and not log_zip_path.startswith("/"):
                # Replace file:/// with file: for relative paths,
                # otherwise opening fails.
                log_zip_url = "file:" + log_zip_url[8:]

            try:
                with Util.open_url_seekable(log_file_url, 'rt') as logfile:
                    return logfile.readlines()
            except IOError as unused_e1:
                try:
                    if log_zip_url not in log_zip_cache:
                        log_zip_cache[log_zip_url] = zipfile.ZipFile(
                            Util.open_url_seekable(log_zip_url, 'rb'))
                    log_zip = log_zip_cache[log_zip_url]

                    try:
                        with io.TextIOWrapper(log_zip.open(path_in_zip)) as logfile:
                            return logfile.readlines()
                    except KeyError:
                        logging.warning("Could not find logfile '%s' in archive '%s'.",
                                        log_file, log_zip_url)
                        return []

                except IOError as unused_e2:
                    logging.warning("Could not find logfile '%s' nor log archive '%s'.",
                                    log_file, log_zip_url)
                    return []

        status = Util.get_column_value(sourcefileTag, 'status', '')
        category = Util.get_column_value(sourcefileTag, 'category', result.CATEGORY_MISSING)
        score = result.score_for_task(
            sourcefileTag.get('properties', '').split(), category, status)
        logfileLines = None

        values = []

        for column in listOfColumns: # for all columns that should be shown
            value = None # default value
            if column.title.lower() == 'status':
                value = status

            elif not correct_only or category == result.CATEGORY_CORRECT:
                if not column.pattern or column.href:
                    # collect values from XML
                    value = Util.get_column_value(sourcefileTag, column.title)

                else: # collect values from logfile
                    if logfileLines is None: # cache content
                        logfileLines = read_logfile_lines(sourcefileTag.get('logfile'))

                    value = get_value_from_logfile(logfileLines, column.pattern)

            if column.title.lower() == 'score' and value is None and score is not None:
                # If no score column exists in the xml, take the internally computed score,
                # if available
                value = str(score)
            values.append(value)

        sourcefiles = sourcefileTag.get('files')
        if sourcefiles:
            if sourcefiles.startswith('['):
                sourcefileList = [s.strip() for s in sourcefiles[1:-1].split(',') if s.strip()]
                sourcefiles_exist = True if sourcefileList else False
            else:
                raise AssertionError('Unknown format for files tag:')
        else:
            sourcefiles_exist = False

        return RunResult(get_task_id(sourcefileTag, result_file_or_url if sourcefiles_exist else None),
                         status, category, score,
                         sourcefileTag.get('logfile'), listOfColumns, values,
                         columns_relevant_for_diff, sourcefiles_exist=sourcefiles_exist)
448109
@Override
    protected BomFile parseBomFile(String jsonText, String localFileName) {
        JSONObject json = new JSONObject(jsonText);
        String name = json.getString(Constants.NAME);
        String version = getVersion(json, localFileName);
        Map dependencies = getDependenciesFromJson(json, Constants.DEPENDENCIES);
        Map optionalDependencies = getDependenciesFromJson(json, OPTIONAL_DEPENDENCIES);
        String fileName = getFilename(name, version);
        String sha1 = Constants.EMPTY_STRING;
        String resolved = null;
        if(json.has(RESOLVED)) {
            resolved = json.getString(RESOLVED);
        }

        // optional fields for packageJson parser
        if (json.has(SHA1)) {
            sha1 = json.getString(SHA1);
        } else {
            logger.debug("shasum not found in file {}", localFileName);
        }

        RegistryType registryType = getRegistryType(resolved);

        BomFile bom = new BomFile(name, version, sha1, fileName, localFileName, dependencies, optionalDependencies, resolved, registryType);
        return bom;
    }
822199
def from_zipfile(cls, path, filename, encoding, dialect, fields, converters):
        """"""

        stream = ZipReader(path, filename).readlines(encoding)
        return cls(stream, dialect, fields, converters)