Read trn data.

@param dataFile the data file
174402
public void readTrnData(String dataFile) {
	if (cpStr2Int != null) {
	    cpStr2Int.clear();
	} else {
	    cpStr2Int = new HashMap();
	}
	
	if (cpInt2Str != null) {
	    cpInt2Str.clear();	    
	} else {
	    cpInt2Str = new HashMap();
	}
	
	if (lbStr2Int != null) {
	    lbStr2Int.clear();
	} else {
	    lbStr2Int = new HashMap();
	}
	
	if (lbInt2Str != null) {
	    lbInt2Str.clear();
	} else {
	    lbInt2Str = new HashMap();
	}
	
	if (trnData != null) {
	    trnData.clear();
	} else {
	    trnData = new ArrayList();
	}

	// open data file	
	BufferedReader fin = null;
	
	try {
	    fin = new BufferedReader(new InputStreamReader(new FileInputStream(dataFile), "UTF-8"));
	//    BufferedWriter flog = new BufferedWriter(new OutputStreamWriter(
//	/	new FileOutputStream((new File(dataFile)).getParent() + File.separator + "log.txt"), "UTF-8"));
	    
	    System.out.println("Reading training data ...");
	    
	    String line;
	    while ((line = fin.readLine()) != null) {
		StringTokenizer strTok = new StringTokenizer(line, " \t\r\n");
		int len = strTok.countTokens();
				
		if (len <= 1) {
		    // skip this invalid line
		    continue;
		}
		
		List strCps = new ArrayList();
		for (int i = 0; i < len - 1; i++) {
		    strCps.add(strTok.nextToken());
		}
		
		String labelStr = strTok.nextToken();
//		
//		String [] tags = {"N", "Np", "Nc", "Nu", "V", "A", "P", "L", "M", 
//		"R", "E", "C", "I", "T", "U", "Y", "X", "LBKT", "RBKT"};
//
//
//		//System.out.println("--" + labelStr);
//		//if (!StringUtils.isSign(labelStr)){
//			boolean flag = false;
//			for (String tag : tags){
//				if (labelStr.equalsIgnoreCase(tag)){
//					flag = true;
//				}
//			}
//			
//			if (!flag){
//			//	flog.write(line + "\n");
//				//System.out.println("--" + labelStr);
//			}
//		//}
		
		List intCps = new ArrayList();
		
		for (int i = 0; i < strCps.size(); i++) {	
		    String cpStr = (String)strCps.get(i);		    
		    Integer cpInt = (Integer)cpStr2Int.get(cpStr);		    
		    if (cpInt != null) {
			intCps.add(cpInt);
		    } else {
			intCps.add(new Integer(cpStr2Int.size()));
			cpStr2Int.put(cpStr, new Integer(cpStr2Int.size()));
			cpInt2Str.put(new Integer(cpInt2Str.size()), cpStr);
		    }
		}
		
		Integer labelInt = (Integer)lbStr2Int.get(labelStr);
		if (labelInt == null) {
		    labelInt = new Integer(lbStr2Int.size());
		    
//		    System.out.println("hey:" + labelStr);
//		    flog.write(labelStr + "\t" + line + "\n");
		    lbStr2Int.put(labelStr, labelInt);
		    lbInt2Str.put(labelInt, labelStr);
		}
		
		int[] cps = new int[intCps.size()];
		for (int i = 0; i < cps.length; i++) {
		    cps[i] = ((Integer)intCps.get(i)).intValue();
		}
		
		Observation obsr = new Observation(labelInt.intValue(), cps);
		
		// add this observation to the data
		trnData.add(obsr);
	    }
	    
	    System.out.println("Reading " + Integer.toString(trnData.size()) +
			" training data examples completed!");
	  // flog.close();
	
	} catch (IOException e) {
	    System.out.println(e.toString());
	    return;
	}

	option.numCps = cpStr2Int.size();
	option.numLabels = lbStr2Int.size();	
	option.numTrainExps = trnData.size();	
    }
1008685
static Map<String, String> readFasta(final File inputFastqFile) throws IOException, NoSuchElementException, BioException{
    	
    	BufferedReader reader = null;
    	HashMap<String, String> fasta = new HashMap<String, String>();
        
    	try{
    		 reader = reader(inputFastqFile);
    		 for (SequenceIterator sequences = SeqIOTools.readFastaDNA(reader); sequences.hasNext(); ) 
    		{
                 Sequence sequence = sequences.nextSequence();
                 String sequenceId = sequence.getName();
                 String fastaSeq   = sequence.seqString();
                 fasta.put(sequenceId, fastaSeq);
             }
    		
    	}finally {
            try {
            	reader.close();
            }
            catch (Exception e) {
            	 e.printStackTrace();
            	 System.exit(1);
            }
        }
    
        
        return fasta;
    	
    }
893118
public static function mapExtensionToType($dataFile) {
        $ext = substr(strrchr($dataFile, '.'), 1);
        $ext = strtolower($ext);
        switch ($ext) {
            case 'xls':
            case 'xlsx':
                return 'Excel';
            break;
            case 'json':
                return 'Json';
            break;
            default:
                throw new Garp_Content_Import_Exception("Could not find importer for type $ext");
            break;
        }
    }
202872
private void readStatus(boolean emptyForMissing, final Path statusFile) {
    this.status = null;
    if (Files.exists(statusFile)) {
      // try some number of times if dsMsg is null
      for (int i = 1; i <= 3; i++) {
        this.status = Status.spinRead(baseName, statusFile);
        if (this.status.dsMsg != null) break;
      }
    }
    if (this.status == null && emptyForMissing) {
      this.status = Status.create(baseName, Status.SHUTDOWN, 0, statusFile);
    }
  }
867164
private DataSet doDelimitedFile(final Reader dataSource, final boolean createMDFromFile) throws IOException {
        if (dataSource == null) {
            throw new IllegalArgumentException("dataSource is null");
        }
        final DefaultDataSet ds = new DefaultDataSet(getPzMetaData(), this);
        try (BufferedReader br = new BufferedReader(dataSource)) {
            // gather the conversion properties
            ds.setPZConvertProps(ParserUtils.loadConvertProperties());

            boolean processedFirst = false;
            /** loop through each line in the file */
            String line = null;
            int estimatedColCount = FPConstants.SPLITLINE_SIZE_INIT;
            while ((line = fetchNextRecord(br, getQualifier(), getDelimiter())) != null) {
                // check to see if the user has elected to skip the first record
                if (!processedFirst && isIgnoreFirstRecord()) {
                    processedFirst = true;
                    continue;
                } else if (!processedFirst && createMDFromFile) {
                    processedFirst = true;
                    setPzMetaData(ParserUtils.getPZMetaDataFromFile(line, delimiter, qualifier, this, isAddSuffixToDuplicateColumnNames()));
                    ds.setMetaData(getPzMetaData());
                    continue;
                }
                // column values

                // check number of Qualifier, if ODD number --> Incorrect!!!
                if (oddNumberOfQualifier(line, getQualifier())) {
                    addError(ds, "Odd number of Qualifier characters", lineCount, 1, isStoreRawDataToDataError() ? line : null);
                    continue;
                }

                List<String> columns = ParserUtils.splitLine(line, getDelimiter(), getQualifier(), estimatedColCount, isPreserveLeadingWhitespace(),
                        isPreserveTrailingWhitespace());
                final String mdkey = ParserUtils.getCMDKeyForDelimitedFile(getPzMetaData(), columns);
                final List<ColumnMetaData> metaData = ParserUtils.getColumnMetaData(mdkey, getPzMetaData());
                final int columnCount = metaData.size();
                estimatedColCount = columnCount;

                if (columns.size() > columnCount) {
                    // Incorrect record length on line log the error. Line
                    // will not be included in the dataset log the error
                    if (isIgnoreExtraColumns()) {
                        // user has chosen to ignore the fact that we have too many columns in the data from
                        // what the mapping has described. sublist the array to remove un-needed columns
                        columns = columns.subList(0, columnCount);
                        addError(ds, "Flatpack truncated line to correct number of columns", lineCount, 1, isStoreRawDataToDataError() ? line : null);
                    } else {
                        addError(ds, "Too many columns expected: " + columnCount + " Flatpack got: " + columns.size(), lineCount, 2,
                                isStoreRawDataToDataError() ? line : null);
                        continue;
                    }
                } else if (columns.size() < columnCount) {
                    if (isHandlingShortLines()) {
                        // We can pad this line out
                        while (columns.size() < columnCount) {
                            columns.add("");
                        }

                        // log a warning
                        addError(ds, "Flatpack padded line to correct number of columns", lineCount, 1, isStoreRawDataToDataError() ? line : null);

                    } else {
                        addError(ds, "Too few columns expected: " + columnCount + " only got: " + columns.size(), lineCount, 2,
                                isStoreRawDataToDataError() ? line : null);
                        continue;
                    }
                }

                final Row row = new Row();
                row.setMdkey(mdkey.equals(FPConstants.DETAIL_ID) ? null : mdkey); // try
                // to limit the memory use
                row.setCols(columns);
                row.setRowNumber(lineCount);
                if (isFlagEmptyRows()) {
                    // user has elected to have the parser flag rows that are empty
                    row.setEmpty(ParserUtils.isListElementsEmpty(columns));
                }
                if (isStoreRawDataToDataSet()) {
                    // user told the parser to keep a copy of the raw data in the row
                    // WARNING potential for high memory usage here
                    row.setRawData(line);
                }

                // add the row to the array
                ds.addRow(row);

            }
        } finally {
            closeReaders();
        }
        return ds;
    }
375030
public function set_data($datavalue, $datakey = null) {
        $this->datavalue = $datavalue;

        // We must clear the datakey when changing the datavalue.
        $this->set_data_key($datakey);
    }
666114
@Override
    public String[] readLine(int line, boolean readResult) throws TechnicalException {
        String sqlRequest;
        try {
            final Path file = Paths.get(dataInPath + scenarioName + ".sql");
            sqlRequest = new String(Files.readAllBytes(file), Charset.forName(Constants.DEFAULT_ENDODING));
            sqlSanitized4readOnly(sqlRequest);
        } catch (final IOException e) {
            throw new TechnicalException(Messages.getMessage(TechnicalException.TECHNICAL_ERROR_MESSAGE) + e.getMessage(), e);
        }
        try (Connection connection = getConnection(); PreparedStatement statement = connection.prepareStatement(sqlRequest); ResultSet rs = statement.executeQuery();) {
            final String[] ret = readResult ? new String[columns.size()] : new String[columns.size() - 1];
            if (line == 0) {
                for (int i = 0; i < ret.length; i++) {
                    ret[i] = columns.get(i);
                }
            } else {
                while (rs.next() && rs.getRow() < line) {
                }
                for (int i = 1; i <= ret.length; i++) {
                    ret[i - 1] = rs.getString(i);
                }
            }
            return ret;
        } catch (final SQLException e) {
            logger.debug("In DBDataProvider, this catch aims for testing the end of provided data. DBDataProvider.readLine({}, {})", line, readResult, e);
            return null;
        }
    }
980254
public void loadFrom(String modelfile) throws IOException,
	ClassNotFoundException {
		ObjectInputStream in = new ObjectInputStream(new BufferedInputStream(
				new GZIPInputStream(new FileInputStream(modelfile))));
		templets = (TempletGroup) in.readObject();
		cl = (Linear) in.readObject();
		in.close();
	}
1302053
@Override
	public List<Sentence> readString(String dataStr) {
		// TODO Auto-generated method stub
		dataStr = dataStr + "$";
		String  [] lines = dataStr.split("\\n");
		
		List<Sentence> data = new ArrayList<Sentence>();
		Sentence sent = new Sentence();
		for (String line : lines){
			System.out.println(".");
			if (line.isEmpty() || line.equalsIgnoreCase("$")){								
				data.add(sent);
				sent = new Sentence();				
			}
			else{
				StringTokenizer tk = new StringTokenizer(line, "\t");
				if (tk.countTokens() != 2) continue;
				else {
					String token = tk.nextToken();
					String tag = tk.nextToken();
					TWord tw = new TWord(token, tag);
					sent.addTWord(tw);
				}
			}
		}
		return data;
	}
570040
protected LinkedHashMap<Pattern, String> load(File propertyFile) {
    LinkedHashMap<Pattern, String> result = new LinkedHashMap<Pattern, String>();
    BufferedReader reader = null;
    try {
      reader =
          new BufferedReader(new InputStreamReader(new FileInputStream(propertyFile), "UTF-8"));
      String line;
      while ((line = reader.readLine()) != null) {
        if (line.length() == 0) {
          continue;
        }
        char firstChar = line.charAt(0);
        if (firstChar == '#' || firstChar == '!') {
          continue;
        }
        int pos = line.indexOf('=');
        if (pos < 0) {
          continue;
        }
        String key = line.substring(0, pos);
        String value = line.substring(pos + 1, line.length());
        int pos2 = line.indexOf('@');
        if (pos2 == 0) {
          key = key.substring(1);
        } else if (pos2 > 0) {
          key = Pattern.quote(key.substring(0, pos2)) + key.substring(pos2);
        }
        result.put(Pattern.compile(key), value);
      }
    } catch (IOException e) {
      throw new GenException(Message.DOMAGEN9001, e, e);
    } finally {
      IOUtil.close(reader);
    }
    return result;
  }
148657
private function loadData($dataFile)
    {
        $fh          = fopen($dataFile, 'r');
        $keys        = fgetcsv($fh);
        $columnCount = count($keys);

        $rows = [];

        while (!feof($fh)) {
            $row = fgetcsv($fh);

            if ($row === false) {
                break;
            }

            if (count($row) != $columnCount) {
                echo "\n$dataFile: problem with row " . implode(', ', $row) . "\n";
            }
            $rows[] = array_combine($keys, $row);
        }

        fclose($fh);

        return $rows;
    }
1244874
private String loadDrlFile(final Reader drl) throws IOException {
        final StringBuilder buf = new StringBuilder();
        final BufferedReader input = new BufferedReader( drl );
        String line;
        while ( (line = input.readLine()) != null ) {
            buf.append( line );
            buf.append( nl );
        }
        return buf.toString();
    }
1195821
public void readData(BufferedReader in) throws IOException {
    final String SEEN = "SEEN";
    String line;
    int lineNum = 1;
    // all lines have one tagging with raw count per line
    line = in.readLine();
    Pattern p = Pattern.compile("^smooth\\[([0-9])\\] = (.*)$");
    while (line != null && line.length() > 0) {
      try {
        Matcher m = p.matcher(line);
        if (m.matches()) {
          int i = Integer.parseInt(m.group(1));
          smooth[i] = Double.parseDouble(m.group(2));
        } else {
          // split on spaces, quote with doublequote, and escape with backslash
          String[] fields = StringUtils.splitOnCharWithQuoting(line, ' ', '\"', '\\');
          // System.out.println("fields:\n" + fields[0] + "\n" + fields[1] +
          // "\n" + fields[2] + "\n" + fields[3] + "\n" + fields[4]);
          boolean seen = fields[3].equals(SEEN);
          addTagging(seen, new IntTaggedWord(fields[2], fields[0], wordIndex, tagIndex), Double.parseDouble(fields[4]));
        }
      } catch (RuntimeException e) {
        throw new IOException("Error on line " + lineNum + ": " + line, e);
      }
      lineNum++;
      line = in.readLine();
    }
    initRulesWithWord();
  }
1207219
public void setDataSource(DataSource dataSource) {
        mDataSource = dataSource;
        mDriverClassName = null;
        mURL = null;
        mUsername = null;
        mPassword = null;
    }
374381
private void readUncompressedFPRowByNumber( ByteBuffer rowdata, int rn, RandomAccessFile thefile, int typeBytes )
            throws IOException, DataFormatException {
        int datanumber = fileWindow.getCols() * typeBytes;
        thefile.seek((rn * datanumber));

        thefile.read(rowdata.array());
    }
923693
private void readCompressedFPRowByNumber( ByteBuffer rowdata, int rn, long[] adrows, RandomAccessFile thefile, int typeBytes )
            throws DataFormatException, IOException {
        int offset = (int) (adrows[rn + 1] - adrows[rn]);
        /*
         * The fact that the file is compressed does not mean that the row is compressed. If the
         * first byte is 0 (49), then the row is compressed, otherwise (first byte = 48) the row has
         * to be read in simple XDR uncompressed format.
         */
        byte[] tmp = new byte[offset - 1];
        thefile.seek(adrows[rn]);
        int firstbyte = (thefile.read() & 0xff);
        if (firstbyte == 49) {
            /* The row is compressed. */
            // thefile.seek((long) adrows[rn] + 1);
            thefile.read(tmp, 0, offset - 1);
            Inflater decompresser = new Inflater();
            decompresser.setInput(tmp, 0, tmp.length);
            decompresser.inflate(rowdata.array());
            decompresser.end();
        } else if (firstbyte == 48) {
            /* The row is NOT compressed */
            // thefile.seek((long) (adrows[rn]));
            // if (thefile.read() == 48)
            // {
            // thefile.seek((long) (adrows[rn] + 1));
            thefile.read(rowdata.array(), 0, offset - 1);
            // }
        }
    }
9059
public GraphHopper setDataReaderFile(String dataReaderFileStr) {
        ensureNotLoaded();
        if (isEmpty(dataReaderFileStr))
            throw new IllegalArgumentException("Data reader file cannot be empty.");

        dataReaderFile = dataReaderFileStr;
        return this;
    }