This should be used for creating streams to read file metadata, e.g. the footer, not for
data in columns.
6309
public static InStream create(String name, FSDataInputStream file, long streamOffset,
      int streamLength, CompressionCodec codec, int bufferSize) throws IOException {

    return create(name, file, streamOffset, streamLength, codec, bufferSize, true, 1);
  }
181263
@Override
  void readStreamBlob(InputStream is, 
                      byte[] buffer, int offset, RowCursor cursor)
    throws IOException
  {
    int len = BitsUtil.readInt(is);

    if (len == 0) {
      BitsUtil.writeInt(buffer, offset + offset(), 0);
      return;
    }
    
    try (BlobOutputStream blob = openOutputStream(cursor)) {
      while (len > 0) {
        boolean isNext = (len & BLOB_CONT_MASK) != 0;
        len &= ~BLOB_CONT_MASK;
        
        blob.writeFromStream(is, len);
        
        if (isNext) {
          len = BitsUtil.readInt(is);
        }
        else {
          len = 0;
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
      
      throw e;
    }
    
    if (len < 0) {
      throw new IOException(L.l("Unexpected end of file while reading {0}", this));
    }

    //cursor.setBlob(getIndex(), blob);
  }
1131749
function footerHTML( footer ) {
  var content = [];
  content.push( '<div class="footer-cnt"></div>' );
  if ( footer != null ) {
    // footer links
    if ( footer.linkList != null ) {
      content.push( '<div class="footer-links">' );
      for ( var i = 0; i < footer.linkList.length; i++ ) {
        if ( footer.linkList[i] ) {
        var lnk = footer.linkList[i];
        if ( lnk.target != null ) {
          content.push( '<a href="'+ $.i18n( lnk.url )+'" target="'+lnk.target+'">'+ $.i18n( lnk.text )+'</a>   ' );
        } else {
          content.push( '<a href="'+ $.i18n( lnk.url )+'">'+ $.i18n( lnk.text )+'</a>   ' );          
        }
        }
      }
      content.push( "</div>" );
    }
    if ( footer.copyrightText != null ) {
      content.push( '<div class="copyright-div">'+ $.i18n( footer.copyrightText ) +'</div>' );
    } else {
      content.push( '<div class="copyright-div">&copy; MH 2018</div>' );
    }
    
    // header hooks
    if ( footer.modules != null ) {
      for ( var i = 0; i < footer.modules.length; i++ ) {
        if ( footer.modules[i] ) {
        var fMod =  footer.modules[i];
        log( 'footerHTML', "addFooterHtml "+ fMod.type );
        var hook = getHookMethod( "addFooterHtml", fMod.type );
        if ( hook != "" ) {
          log( 'footerHTML', hook+"( ... )");
          content.push( '<div id="'+fMod.id+'" class="'+fMod.type+'"></div>' );
          
          if ( fMod.moduleConfig != null) {
            moduleConfig[ fMod.id ] = fMod.moduleConfig;
          }
          
          if ( fMod.param == null ) {
            fMod.param = {};
          }
          fMod.param.get = getUrlGETparams();
          
          hookCalls.push( hook+'( "'+fMod.id+'", "'+fMod.type+'", '+JSON.stringify(fMod.param)+' )'  );
        }
        }
      }
    }
  }
  $( '#footer' ).html( content.join("\n") );
}
1261831
public static Metadata fromPomProperties(InputStream is) {
        Metadata metadata = new Metadata();
        BufferedReader input = new BufferedReader(new InputStreamReader(is));
        try {
            String line;
            while ((line = input.readLine()) != null) {
                if (line.startsWith("#"))
                    continue;
                String[] property = line.trim().split("=");
                if (property.length == 2)
                    metadata.put(property[0], property[1]);
            }
        } catch (IOException e) {
            // Problems? Too bad!
        }
        return metadata;
    }
545088
private void pullTopicMetadata(final AbstractPipelineInput input, final Collection<FileInfo> fis) throws DITAOTException {
        // Pull metadata (such as navtitle) into the map from the referenced topics
        final File styleFile = new File(input.getAttribute(ANT_INVOKER_EXT_PARAM_STYLE));
        for (final FileInfo f : fis) {
            final File inputFile = new File(job.tempDir, f.file.getPath());
            final File tmp = new File(inputFile.getAbsolutePath() + ".tmp" + Long.toString(System.currentTimeMillis()));
            if (!tmp.getParentFile().exists() && !tmp.getParentFile().mkdirs()) {
                throw new DITAOTException("Failed to create directory " + tmp.getParent());
            }
            logger.info("Processing " + inputFile.toURI());
            logger.debug("Processing " + inputFile.toURI() + " to " + tmp.toURI());

            Source source = null;
            Result result = null;
            try {
                source = new StreamSource(inputFile.toURI().toString());
                result = new StreamResult(tmp);

                logger.info("Loading stylesheet " + styleFile);
                final TransformerFactory tf = TransformerFactory.newInstance();
                final CatalogResolver xmlCatalog = CatalogUtils.getCatalogResolver();
                tf.setURIResolver(xmlCatalog);
                final Transformer t = withLogger(tf.newTransformer(new StreamSource(styleFile)), logger);
                final URIResolver resolver;
                if (Configuration.DEBUG) {
                    resolver = new DebugURIResolver(xmlCatalog);
                } else {
                    resolver = xmlCatalog;
                }
                t.setURIResolver(resolver);
                for (Entry<String, String> e : input.getAttributes().entrySet()) {
                    logger.debug("Set parameter " + e.getKey() + " to '" + e.getValue() + "'");
                    t.setParameter(e.getKey(), e.getValue());
                }

                t.transform(source, result);
            } catch (final TransformerConfigurationException e) {
                throw new RuntimeException("Failed to compile stylesheet '" + styleFile.toURI() + "': " + e.getMessage(), e);
            } catch (final Exception e) {
                throw new DITAOTException("Failed to transform document: " + e.getMessage(), e);
            } finally {
                try {
                    XMLUtils.close(source);
                } catch (final IOException e) {
                    // NOOP
                }
                try {
                    XMLUtils.close(result);
                } catch (final IOException e) {
                    // NOOP
                }
            }
            try {
                logger.debug("Moving " + tmp.toURI() + " to " + inputFile.toURI());
                if (!inputFile.delete()) {
                    throw new IOException("Failed to to delete input file " + inputFile.toURI());
                }
                if (!tmp.renameTo(inputFile)) {
                    throw new IOException("Failed to to replace input file " + inputFile.toURI());
                }
            } catch (final IOException e) {
                throw new DITAOTException("Failed to replace document: " + e.getMessage(), e);
            } finally {
                logger.debug("Remove " + tmp.toURI());
                FileUtils.delete(tmp);
            }
        }
    }
476811
public Metadata readNextMetadata() throws IOException {
        Metadata metadata = null;
        
        boolean isLast = (bitStream.readRawUInt(Metadata.STREAM_METADATA_IS_LAST_LEN) != 0);
        int type = bitStream.readRawUInt(Metadata.STREAM_METADATA_TYPE_LEN);
        int length = bitStream.readRawUInt(Metadata.STREAM_METADATA_LENGTH_LEN);
        
        if (type == Metadata.METADATA_TYPE_STREAMINFO) {
        	metadata = streamInfo = new StreamInfo(bitStream, length, isLast);
            pcmProcessors.processStreamInfo((StreamInfo)metadata);
        } else if (type == Metadata.METADATA_TYPE_SEEKTABLE) {
        	metadata = seekTable = new SeekTable(bitStream, length, isLast);
        } else if (type == Metadata.METADATA_TYPE_APPLICATION) {
            metadata = new Application(bitStream, length, isLast);
        } else if (type == Metadata.METADATA_TYPE_PADDING) {
            metadata = new Padding(bitStream, length, isLast);
        } else if (type == Metadata.METADATA_TYPE_VORBIS_COMMENT) {
            metadata = vorbisComment = new VorbisComment(bitStream, length, isLast);
        } else if (type == Metadata.METADATA_TYPE_CUESHEET) {
	    metadata = new CueSheet(bitStream, length, isLast);
	} else if (type == Metadata.METADATA_TYPE_PICTURE) {
	    metadata = new Picture(bitStream, length, isLast);
        } else {
            metadata = new Unknown(bitStream, length, isLast);
        }
        frameListeners.processMetadata(metadata);
        //if (isLast) state = DECODER_SEARCH_FOR_FRAME_SYNC;
        return metadata;
    }
137926
private static void writeFooter(BufferedWriter out, String type) throws IOException
    {
        out.write(PEM_END + type + DASHES);
        out.newLine();
    }
312976
def default_stream_factory(
    total_content_length, filename, content_type, content_length=None
):
    """"""
    max_size = 1024 * 500
    if SpooledTemporaryFile is not None:
        return SpooledTemporaryFile(max_size=max_size, mode="wb+")
    if total_content_length is None or total_content_length > max_size:
        return TemporaryFile("wb+")
    return BytesIO()
251918
def read_data_page(file_obj, schema_helper, page_header, column_metadata,
                   dictionary):
    
    daph = page_header.data_page_header
    raw_bytes = _read_page(file_obj, page_header, column_metadata)
    io_obj = io.BytesIO(raw_bytes)
    vals = []
    debug_logging = logger.isEnabledFor(logging.DEBUG)

    if debug_logging:
        logger.debug("  definition_level_encoding: %s",
                     _get_name(parquet_thrift.Encoding, daph.definition_level_encoding))
        logger.debug("  repetition_level_encoding: %s",
                     _get_name(parquet_thrift.Encoding, daph.repetition_level_encoding))
        logger.debug("  encoding: %s", _get_name(parquet_thrift.Encoding, daph.encoding))

    # definition levels are skipped if data is required.
    definition_levels = None
    num_nulls = 0
    max_definition_level = -1
    if not schema_helper.is_required(column_metadata.path_in_schema[-1]):
        max_definition_level = schema_helper.max_definition_level(
            column_metadata.path_in_schema)
        bit_width = encoding.width_from_max_int(max_definition_level)
        if debug_logging:
            logger.debug("  max def level: %s   bit_width: %s",
                         max_definition_level, bit_width)
        if bit_width == 0:
            definition_levels = [0] * daph.num_values
        else:
            definition_levels = _read_data(io_obj,
                                           daph.definition_level_encoding,
                                           daph.num_values,
                                           bit_width)[:daph.num_values]

        # any thing that isn't at max definition level is a null.
        num_nulls = len(definition_levels) - definition_levels.count(max_definition_level)
        if debug_logging:
            logger.debug("  Definition levels: %s", len(definition_levels))

    # repetition levels are skipped if data is at the first level.
    repetition_levels = None  # pylint: disable=unused-variable
    if len(column_metadata.path_in_schema) > 1:
        max_repetition_level = schema_helper.max_repetition_level(
            column_metadata.path_in_schema)
        bit_width = encoding.width_from_max_int(max_repetition_level)
        repetition_levels = _read_data(io_obj,
                                       daph.repetition_level_encoding,
                                       daph.num_values,
                                       bit_width)

    # NOTE: The repetition levels aren't yet used.
    if daph.encoding == parquet_thrift.Encoding.PLAIN:
        read_values = encoding.read_plain(io_obj, column_metadata.type, daph.num_values - num_nulls)
        schema_element = schema_helper.schema_element(column_metadata.path_in_schema[-1])
        read_values = convert_column(read_values, schema_element) \
            if schema_element.converted_type is not None else read_values
        if definition_levels:
            itr = iter(read_values)
            vals.extend([next(itr) if level == max_definition_level else None for level in definition_levels])
        else:
            vals.extend(read_values)
        if debug_logging:
            logger.debug("  Values: %s, nulls: %s", len(vals), num_nulls)

    elif daph.encoding == parquet_thrift.Encoding.PLAIN_DICTIONARY:
        # bit_width is stored as single byte.
        bit_width = struct.unpack(b"<B", io_obj.read(1))[0]
        if debug_logging:
            logger.debug("bit_width: %d", bit_width)

        dict_values_bytes = io_obj.read()
        dict_values_io_obj = io.BytesIO(dict_values_bytes)
        # read_values stores the bit-packed values. If there are definition levels and the data contains nulls,
        # the size of read_values will be less than daph.num_values
        read_values = []
        while dict_values_io_obj.tell() < len(dict_values_bytes):
            read_values.extend(encoding.read_rle_bit_packed_hybrid(
                dict_values_io_obj, bit_width, len(dict_values_bytes)))

        if definition_levels:
            itr = iter(read_values)
            # add the nulls into a new array, values, but using the definition_levels data.
            values = [dictionary[next(itr)] if level == max_definition_level else None for level in definition_levels]
        else:
            values = [dictionary[v] for v in read_values]

        # there can be extra values on the end of the array because the last bit-packed chunk may be zero-filled.
        if len(values) > daph.num_values:
            values = values[0: daph.num_values]
        vals.extend(values)

        if debug_logging:
            logger.debug("  Read %s values using PLAIN_DICTIONARY encoding and definition levels show %s nulls",
                         len(vals), num_nulls)

    else:
        raise ParquetFormatException("Unsupported encoding: %s",
                                     _get_name(parquet_thrift.Encoding, daph.encoding))
    return vals
188703
function createMetaRecordStream(metaFilePaths, types) {
  const metaRecordStream = combinedStream.create();

  metaFilePaths.forEach((metaFilePath) => {
    metaRecordStream.append( (next) => {
      logger.info( `Loading ${path.basename(metaFilePath)} records from ${path.dirname(metaFilePath)}` );
      next(createOneMetaRecordStream(metaFilePath));
    });
  });

  return metaRecordStream;
}
304493
def WriteFileFooter(self):
    """"""

    if self.cur_file_size != self.cur_info.size:
      raise IOError("Incorrect file size: st_size=%d, but written %d bytes." %
                    (self.cur_info.size, self.cur_file_size))

    # TODO(user):pytype: BLOCKSIZE/NUL constants are not visible to type
    # checker.
    # pytype: disable=module-attr
    blocks, remainder = divmod(self.cur_file_size, tarfile.BLOCKSIZE)
    if remainder > 0:
      self._tar_fd.fileobj.write(tarfile.NUL * (tarfile.BLOCKSIZE - remainder))
      blocks += 1
    self._tar_fd.offset += blocks * tarfile.BLOCKSIZE
    # pytype: enable=module-attr

    self._ResetState()

    return self._stream.GetValueAndReset()
1233741
def _read_page(file_obj, page_header, column_metadata):
    """"""
    bytes_from_file = file_obj.read(page_header.compressed_page_size)
    codec = column_metadata.codec
    if codec is not None and codec != parquet_thrift.CompressionCodec.UNCOMPRESSED:
        if column_metadata.codec == parquet_thrift.CompressionCodec.SNAPPY:
            raw_bytes = snappy.decompress(bytes_from_file)
        elif column_metadata.codec == parquet_thrift.CompressionCodec.GZIP:
            io_obj = io.BytesIO(bytes_from_file)
            with gzip.GzipFile(fileobj=io_obj, mode='rb') as file_data:
                raw_bytes = file_data.read()
        else:
            raise ParquetFormatException(
                "Unsupported Codec: {0}".format(codec))
    else:
        raw_bytes = bytes_from_file

    if logger.isEnabledFor(logging.DEBUG):
        logger.debug(
            "Read page with compression type %s. Bytes %d -> %d",
            _get_name(parquet_thrift.CompressionCodec, codec),
            page_header.compressed_page_size,
            page_header.uncompressed_page_size)
    assert len(raw_bytes) == page_header.uncompressed_page_size, \
        "found {0} raw bytes (expected {1})".format(
            len(raw_bytes),
            page_header.uncompressed_page_size)
    return raw_bytes
1035438
private void read(InputStream is, Table table) throws IOException
   {
      byte[] buffer = new byte[m_definition.getPageSize()];
      while (true)
      {
         int bytesRead = is.read(buffer);
         if (bytesRead == -1)
         {
            break;
         }

         if (bytesRead != buffer.length)
         {
            throw new IOException("Unexpected end of file");
         }

         readPage(buffer, table);
      }
   }
1078366
function createReadStream(wofConfig, types, wofAdminRecords) {
  const wofRoot = wofConfig.datapath;
  const metaFilePaths = getMetaFilePaths(wofRoot, types);

  // Select correct stream between meta and SQLite based on config and do specialized stuff
  const stream = wofConfig.sqlite === true ?
    createSQLiteRecordStream(getSqliteFilePaths(wofRoot, types), wofConfig.importPlace)
      .pipe(toJSONStream.create()) :
    createMetaRecordStream(metaFilePaths, types)
      .pipe(isNotNullIslandRelated.create())
      .pipe(loadJSON.create(wofRoot, wofConfig.missingFilesAreFatal));

  // All the pipeline is the same for both meta and SQLite streams
  return stream
  .pipe(recordHasIdAndProperties.create())
  .pipe(isActiveRecord.create())
  .pipe(extractFields.create())
  .pipe(recordHasName.create())
  .pipe(through2.obj(function(wofRecord, enc, callback) {
    // store admin records in memory to traverse the heirarchy
    if (wofRecord.place_type !== 'venue' && wofRecord.place_type !== 'postalcode') {
      wofAdminRecords[wofRecord.id] = wofRecord;
    }

    callback(null, wofRecord);
  }));
}
529636
def dump_metadata(filename, show_row_group_metadata, out=sys.stdout):
    
    def println(value):
        """Write a new line containing `value` to `out`."""
        out.write(value + "\n")
    footer = read_footer(filename)
    println("File Metadata: {0}".format(filename))
    println("  Version: {0}".format(footer.version))
    println("  Num Rows: {0}".format(footer.num_rows))
    println("  k/v metadata: ")
    if footer.key_value_metadata:
        for item in footer.key_value_metadata:
            println("    {0}={1}".format(item.key, item.value))
    else:
        println("    (none)")
    println("  schema: ")
    for element in footer.schema:
        println("    {name} ({type}): length={type_length}, "
                "repetition={repetition_type}, "
                "children={num_children}, "
                "converted_type={converted_type}".format(
                    name=element.name,
                    type=parquet_thrift.Type._VALUES_TO_NAMES[element.type]  # pylint: disable=protected-access
                    if element.type else None,
                    type_length=element.type_length,
                    repetition_type=_get_name(parquet_thrift.FieldRepetitionType,
                                              element.repetition_type),
                    num_children=element.num_children,
                    converted_type=element.converted_type))
    if show_row_group_metadata:
        println("  row groups: ")
        for row_group in footer.row_groups:
            num_rows = row_group.num_rows
            size_bytes = row_group.total_byte_size
            println(
                "  rows={num_rows}, bytes={bytes}".format(num_rows=num_rows,
                                                          bytes=size_bytes))
            println("    chunks:")
            for col_group in row_group.columns:
                cmd = col_group.meta_data
                println("      type={type} file_offset={offset} "
                        "compression={codec} "
                        "encodings={encodings} path_in_schema={path_in_schema} "
                        "num_values={num_values} uncompressed_bytes={raw_bytes} "
                        "compressed_bytes={compressed_bytes} "
                        "data_page_offset={data_page_offset} "
                        "dictionary_page_offset={dictionary_page_offset}".format(
                            type=_get_name(parquet_thrift.Type, cmd.type),
                            offset=col_group.file_offset,
                            codec=_get_name(parquet_thrift.CompressionCodec, cmd.codec),
                            encodings=",".join(
                                [_get_name(
                                    parquet_thrift.Encoding, s) for s in cmd.encodings]),
                            path_in_schema=cmd.path_in_schema,
                            num_values=cmd.num_values,
                            raw_bytes=cmd.total_uncompressed_size,
                            compressed_bytes=cmd.total_compressed_size,
                            data_page_offset=cmd.data_page_offset,
                            dictionary_page_offset=cmd.dictionary_page_offset))
                with open(filename, 'rb') as file_obj:
                    offset = _get_offset(cmd)
                    file_obj.seek(offset, 0)
                    values_read = 0
                    println("      pages: ")
                    while values_read < num_rows:
                        page_header = _read_page_header(file_obj)
                        # seek past current page.
                        file_obj.seek(page_header.compressed_page_size, 1)
                        daph = page_header.data_page_header
                        type_ = _get_name(parquet_thrift.PageType, page_header.type)
                        raw_bytes = page_header.uncompressed_page_size
                        num_values = None
                        if page_header.type == parquet_thrift.PageType.DATA_PAGE:
                            num_values = daph.num_values
                            values_read += num_values
                        if page_header.type == parquet_thrift.PageType.DICTIONARY_PAGE:
                            pass

                        encoding_type = None
                        def_level_encoding = None
                        rep_level_encoding = None
                        if daph:
                            encoding_type = _get_name(parquet_thrift.Encoding, daph.encoding)
                            def_level_encoding = _get_name(
                                parquet_thrift.Encoding, daph.definition_level_encoding)
                            rep_level_encoding = _get_name(
                                parquet_thrift.Encoding, daph.repetition_level_encoding)

                        println("        page header: type={type} "
                                "uncompressed_size={raw_bytes} "
                                "num_values={num_values} encoding={encoding} "
                                "def_level_encoding={def_level_encoding} "
                                "rep_level_encoding={rep_level_encoding}".format(
                                    type=type_,
                                    raw_bytes=raw_bytes,
                                    num_values=num_values,
                                    encoding=encoding_type,
                                    def_level_encoding=def_level_encoding,
                                    rep_level_encoding=rep_level_encoding))
643435
private void encodeFooter(final FacesContext context, final ResponseWriter responseWriter, final Sheet sheet)
                throws IOException {
        // footer
        final UIComponent footer = sheet.getFacet("footer");
        if (footer != null) {
            responseWriter.startElement("div", null);
            responseWriter.writeAttribute("class", "ui-datatable-footer ui-widget-header ui-corner-bottom", null);
            footer.encodeAll(context);
            responseWriter.endElement("div");
        }
    }
867164
private DataSet doDelimitedFile(final Reader dataSource, final boolean createMDFromFile) throws IOException {
        if (dataSource == null) {
            throw new IllegalArgumentException("dataSource is null");
        }
        final DefaultDataSet ds = new DefaultDataSet(getPzMetaData(), this);
        try (BufferedReader br = new BufferedReader(dataSource)) {
            // gather the conversion properties
            ds.setPZConvertProps(ParserUtils.loadConvertProperties());

            boolean processedFirst = false;
            /** loop through each line in the file */
            String line = null;
            int estimatedColCount = FPConstants.SPLITLINE_SIZE_INIT;
            while ((line = fetchNextRecord(br, getQualifier(), getDelimiter())) != null) {
                // check to see if the user has elected to skip the first record
                if (!processedFirst && isIgnoreFirstRecord()) {
                    processedFirst = true;
                    continue;
                } else if (!processedFirst && createMDFromFile) {
                    processedFirst = true;
                    setPzMetaData(ParserUtils.getPZMetaDataFromFile(line, delimiter, qualifier, this, isAddSuffixToDuplicateColumnNames()));
                    ds.setMetaData(getPzMetaData());
                    continue;
                }
                // column values

                // check number of Qualifier, if ODD number --> Incorrect!!!
                if (oddNumberOfQualifier(line, getQualifier())) {
                    addError(ds, "Odd number of Qualifier characters", lineCount, 1, isStoreRawDataToDataError() ? line : null);
                    continue;
                }

                List<String> columns = ParserUtils.splitLine(line, getDelimiter(), getQualifier(), estimatedColCount, isPreserveLeadingWhitespace(),
                        isPreserveTrailingWhitespace());
                final String mdkey = ParserUtils.getCMDKeyForDelimitedFile(getPzMetaData(), columns);
                final List<ColumnMetaData> metaData = ParserUtils.getColumnMetaData(mdkey, getPzMetaData());
                final int columnCount = metaData.size();
                estimatedColCount = columnCount;

                if (columns.size() > columnCount) {
                    // Incorrect record length on line log the error. Line
                    // will not be included in the dataset log the error
                    if (isIgnoreExtraColumns()) {
                        // user has chosen to ignore the fact that we have too many columns in the data from
                        // what the mapping has described. sublist the array to remove un-needed columns
                        columns = columns.subList(0, columnCount);
                        addError(ds, "Flatpack truncated line to correct number of columns", lineCount, 1, isStoreRawDataToDataError() ? line : null);
                    } else {
                        addError(ds, "Too many columns expected: " + columnCount + " Flatpack got: " + columns.size(), lineCount, 2,
                                isStoreRawDataToDataError() ? line : null);
                        continue;
                    }
                } else if (columns.size() < columnCount) {
                    if (isHandlingShortLines()) {
                        // We can pad this line out
                        while (columns.size() < columnCount) {
                            columns.add("");
                        }

                        // log a warning
                        addError(ds, "Flatpack padded line to correct number of columns", lineCount, 1, isStoreRawDataToDataError() ? line : null);

                    } else {
                        addError(ds, "Too few columns expected: " + columnCount + " only got: " + columns.size(), lineCount, 2,
                                isStoreRawDataToDataError() ? line : null);
                        continue;
                    }
                }

                final Row row = new Row();
                row.setMdkey(mdkey.equals(FPConstants.DETAIL_ID) ? null : mdkey); // try
                // to limit the memory use
                row.setCols(columns);
                row.setRowNumber(lineCount);
                if (isFlagEmptyRows()) {
                    // user has elected to have the parser flag rows that are empty
                    row.setEmpty(ParserUtils.isListElementsEmpty(columns));
                }
                if (isStoreRawDataToDataSet()) {
                    // user told the parser to keep a copy of the raw data in the row
                    // WARNING potential for high memory usage here
                    row.setRawData(line);
                }

                // add the row to the array
                ds.addRow(row);

            }
        } finally {
            closeReaders();
        }
        return ds;
    }