/*
///////////////////////////////////////////////////
Internal methods, loading input data
///////////////////////////////////////////////////
512913
protected boolean initialLoad(int minimum)
        throws IOException
    {
        mInputPtr = 0;
        mInputEnd = 0;

        while (mInputEnd < minimum) {
            int count = mIn.read(mCharBuffer, mInputEnd,
                                 mCharBuffer.length - mInputEnd);
            if (count < 1) {
                return false;
            }
            mInputEnd += count;
        }
        return true;
    }
1091166
@Override
    public Optional<TemplateContentData> load(String responseTemplateName, Input input) throws TemplateLoaderException {
        TemplateEnumerator templateEnumerator = templateEnumeratorSupplier.apply(responseTemplateName, input);
        while (templateEnumerator.hasNext()) {
            String templateName = (String) templateEnumerator.next();
            String templatePath = buildCompletePath(templateName);
            try {
                URI templateUri = getResourceURI(templatePath);
                if (templateUri != null) {
                    String templateIdentifier = templateUri.toString();
                    TemplateContentData templateContentData = templateCache.get(templateIdentifier);
                    if (templateContentData == null) {
                        try (InputStream inputStream = getTemplateAsStream(templatePath)) {
                            byte[] templateByteArray = new byte[inputStream.available()];
                            inputStream.read(templateByteArray);
                            templateContentData = TemplateContentData.builder()
                                    .withIdentifier(templateIdentifier)
                                    .withTemplateContent(templateByteArray)
                                    .withTemplateBaseDir(directoryPath)
                                    .build();
                        } catch (IOException e) {
                            String message = String.format("Fail to read template file: %s with error: %s", templatePath, e.getMessage());
                            LOGGER.error(message);
                            throw new TemplateLoaderException(message);
                        }
                        templateCache.put(templateIdentifier, templateContentData);
                    }
                    return Optional.of(templateContentData);
                }
            } catch (URISyntaxException e) {
                String message = String.format("Cannot get valid URI for template file path: %s with error: %s", templatePath, e.getMessage());
                LOGGER.error(message);
                throw new TemplateLoaderException(message);
            }
        }
        String message = String.format("Cannot find template file: %s given directory path: %s and file extension: %s, returning empty.",
                responseTemplateName, directoryPath, fileExtension);
        LOGGER.warn(message);
        return Optional.empty();
    }
579716
@Override
  public void readFields(DataInput in) throws IOException {
    super.readFields(in);
    int numEntries = getPartialListing().length;
    blockLocations = new LocatedBlocks[numEntries];
    for (int i=0; i<numEntries; i++) {
      blockLocations[i] = new LocatedBlocks();
      blockLocations[i].readFields(in);
    }
  }
1166707
public static SimpleDataSet loadArffFile(Reader input, DataStore store) 
    {
        DataStore list = store.emptyClone();
        DoubleList weights = new DoubleList();
        
        BufferedReader br = new BufferedReader(input);
        
        int numOfVars = 0;
        int numReal = 0;
        List<Boolean> isReal = new ArrayList<>();
        List<String> variableNames = new ArrayList<>();
        List<HashMap<String, Integer>> catVals = new  ArrayList<>();
        String line = null;
        CategoricalData[] categoricalData = null;
        try
        {
            boolean atData = false;
            while( (line = br.readLine()) != null )
            {
                if(line.startsWith("%") || line.trim().isEmpty())
                    continue;///Its a comment, skip
                
                line = line.trim();
                
                if(line.startsWith("@") && !atData)
                {
                    line = line.substring(1).toLowerCase();
                    
                    
                    if(line.toLowerCase().startsWith("data"))
                    {
                        categoricalData = new CategoricalData[numOfVars-numReal];
                        
                        int k = 0;
                        for(int i = 0; i < catVals.size(); i++)
                        {
                            if(catVals.get(i) != null)
                            {
                                categoricalData[k] = new CategoricalData(catVals.get(i).size());
                                categoricalData[k].setCategoryName(variableNames.get(i));
                                for(Entry<String, Integer> entry : catVals.get(i).entrySet())
                                    categoricalData[k].setOptionName(entry.getKey(), entry.getValue());
                                k++;
                            }
                        }
                        
                        //prept to start reading in data
                        list.setNumNumeric(numReal);
                        list.setCategoricalDataInfo(categoricalData);
                        atData = true;
                        continue;
                    }
                    else if(!line.toLowerCase().startsWith("attribute"))
                        continue;
                    numOfVars++;
                    line = line.substring("attribute".length()).trim();//Remove the space, it could be multiple spaces
                    
                    String variableName = null;
                    line = line.replace("\t", " ");
                    if(line.startsWith("'"))
                    {
                        Pattern p = Pattern.compile("'.+?'");
                        Matcher m = p.matcher(line);
                        m.find();
                        variableName = nameTrim(m.group());
                                
                        line = line.replaceFirst("'.+?'", "placeHolder");
                    }
                    else
                        variableName = nameTrim(line.trim().replaceAll("\\s+.*", ""));
                    variableNames.add(variableName);
                    String[] tmp = line.split("\\s+", 2);
                    
                    
                    if(tmp[1].trim().equals("real") || tmp[1].trim().equals("numeric") || tmp[1].trim().startsWith("integer"))
                    {
                        numReal++;
                        isReal.add(true);
                        catVals.add(null);
                    }
                    else//Not correct, but we arent supporting anything other than real and categorical right now
                    {
                        isReal.add(false);
                        String cats = tmp[1].replace("{", "").replace("}", "").trim();
                        if(cats.endsWith(","))
                            cats = cats.substring(0, cats.length()-1);
                        String[] catValsRaw =  cats.split(",");
                        HashMap<String, Integer> tempMap = new HashMap<String, Integer>();
                        for(int i = 0; i < catValsRaw.length; i++)
                        {
                            catValsRaw[i] = nameTrim(catValsRaw[i]);
                            tempMap.put(catValsRaw[i], i);
                        }
                        catVals.add(tempMap);
                    }
                }
                else if(atData && !line.isEmpty())
                {
                    double weight = 1.0;
                    String[] tmp = line.split(",");
                    if(tmp.length != isReal.size())
                    {
                        String s = tmp[isReal.size()];
                        if(tmp.length == isReal.size()+1)//{#} means the # is the weight
                        {
                            if(!s.matches("\\{\\d+(\\.\\d+)?\\}"))
                                throw new RuntimeException("extra column must indicate a data point weigh in the form of \"{#}\", instead bad token " + s + " was found");
                            weight = Double.parseDouble(s.substring(1, s.length()-1));
                        }
                        else
                        {
                            throw new RuntimeException("Column had " + tmp.length + " values instead of " + isReal.size());
                        }
                    }
                        
                    DenseVector vec = new DenseVector(numReal);
                    
                    int[] cats = new int[numOfVars - numReal];
                    int k = 0;//Keeping track of position in cats
                    for(int i  = 0; i < isReal.size(); i++)
                    {
                        String val_string = tmp[i].trim();
                        if (isReal.get(i))
                            if (val_string.equals("?"))//missing value, indicated by NaN
                                vec.set(i - k, Double.NaN);
                            else
                                vec.set(i - k, Double.parseDouble(val_string));
                        else//Categorical
                        {
                            tmp[i] = nameTrim(tmp[i]).trim().toLowerCase();
                            if(tmp[i].equals("?"))//missing value, indicated by -1
                                cats[k++] = -1;
                            else
                                cats[k++] = catVals.get(i).get(tmp[i]);
                        }
                    }
                    
                    list.addDataPoint(new DataPoint(vec, cats, categoricalData)); 
                    weights.add(weight);
                }
            }
        }
        catch (IOException ex)
        {
            
        }
        
        SimpleDataSet dataSet =  new SimpleDataSet(list);
        for(int i = 0; i < weights.size(); i++)
            dataSet.setWeight(i, weights.getD(i));
        int k = 0;
        for (int i = 0; i < isReal.size(); i++)
            if (isReal.get(i))
                dataSet.setNumericName(variableNames.get(i), k++);
        
        return dataSet;
    }
706252
private static boolean loadDataMultipart(ByteBuf undecodedChunk, String delimiter, HttpData httpData) {
        if (!undecodedChunk.hasArray()) {
            return loadDataMultipartStandard(undecodedChunk, delimiter, httpData);
        }
        final SeekAheadOptimize sao = new SeekAheadOptimize(undecodedChunk);
        final int startReaderIndex = undecodedChunk.readerIndex();
        final int delimeterLength = delimiter.length();
        int index = 0;
        int lastRealPos = sao.pos;
        byte prevByte = HttpConstants.LF;
        boolean delimiterFound = false;
        while (sao.pos < sao.limit) {
            final byte nextByte = sao.bytes[sao.pos++];
            // Check the delimiter
            if (prevByte == HttpConstants.LF && nextByte == delimiter.codePointAt(index)) {
                index++;
                if (delimeterLength == index) {
                    delimiterFound = true;
                    break;
                }
                continue;
            }
            lastRealPos = sao.pos;
            if (nextByte == HttpConstants.LF) {
                index = 0;
                lastRealPos -= (prevByte == HttpConstants.CR)? 2 : 1;
            }
            prevByte = nextByte;
        }
        if (prevByte == HttpConstants.CR) {
            lastRealPos--;
        }
        final int lastPosition = sao.getReadPosition(lastRealPos);
        final ByteBuf content = undecodedChunk.copy(startReaderIndex, lastPosition - startReaderIndex);
        try {
            httpData.addContent(content, delimiterFound);
        } catch (IOException e) {
            throw new ErrorDataDecoderException(e);
        }
        undecodedChunk.readerIndex(lastPosition);
        return delimiterFound;
    }
378089
@Override
	public void load(UnitDaoFactory unitDaoFactory) {
		BufferedReader reader = new BufferedReader(new InputStreamReader(fileInputStream));
		BufferedReaderIterator iterator = new BufferedReaderIterator(reader);

		StringBuilder currentQuery = new StringBuilder();
		while (iterator.hasNext()) {
			String line = iterator.next().trim();

			if (!isCommentOrEmptyLine(line)) {
				currentQuery.append(line).append(" ");
			}

			if (isEndOfQuery(line)) {
				unitDaoFactory.getUnitDao().execute(currentQuery.toString());
				currentQuery = new StringBuilder();
			}
		}
	}
994039
protected void loadGrams(DataInputStream input) throws IOException {
    gramSet = new HashSet();
    try {
      while(true) {
        int gramLength=input.readInt();
      
        int[] words = new int[gramLength];
        for (int wi=0;wi<gramLength;wi++) {
          words[wi]=input.readInt();
        }
        gramSet.add(new NGram(words));
      }
    }
    catch(EOFException e) {
      
    }
  }
146972
public void readFields(DataInput in) throws IOException {
    version = in.readLong();
    dir = Text.readString(in);
    gen = in.readLong();
  }
676684
@Override
  public void readFields(DataInput in) throws IOException {
    this.filename = Text.readString(in);
    this.jobid = Text.readString(in);
    this.isJobConfFile = in.readBoolean();
    this.isJobHistoryFile = in.readBoolean();
  }
1293417
public void readData(DataInput din) throws IOException {
        reference = din.readUnsignedShort();
        andMask = din.readUnsignedShort();
        orMask = din.readUnsignedShort();
    }
774094
public void build(String... inputFile) throws Exception {
		System.out.println("read file ..."); 
				
		
		for(String file:inputFile){
			LinkedList<Integer> n1gramList = new LinkedList<Integer>();
			for(int i=0;i<n;i++){
				n1gramList.add(-1);
			}
			Scanner scanner = new Scanner(new FileInputStream(file), "utf-8");
			while(scanner.hasNext()) { 
				String s = scanner.next();
				int idx = wordDict.lookupIndex(s);
				n1gramList.add(idx);
				n1gramList.remove();
				assert n1gramList.size()==n;
				
				String[] grams = getNgram(n1gramList);
				String n1gram = grams[0];
				String ngram = grams[1];
				
				int n1gramidx = n1gramDict.lookupIndex(n1gram);
				if(n1gramidx==n1gramCount.size())
					n1gramCount.add(1);
				else if(n1gramidx>n1gramCount.size()){
					throw new Exception();
				}
				else{
					int count = n1gramCount.get(n1gramidx);
					n1gramCount.set(n1gramidx, count+1);
				}

				
				int ngramidx = ngramDict.lookupIndex(ngram);
				
				if(ngramidx==prob.size())
					prob.add(1);
				else if(ngramidx>prob.size()){
					throw new Exception();
				}
				else{
					float count = prob.get(ngramidx);
					prob.set(ngramidx, count+1);
				}
				
				
				
				if(index.contains(ngramidx))
					assert(index.get(ngramidx) == n1gramidx);
				else
					index.put(ngramidx, n1gramidx);

			}
			scanner.close();
		}
		
		ngramDict.setStopIncrement(true);
		n1gramDict.setStopIncrement(true);
		wordDict.setStopIncrement(true);
		System.out.println("词表大小" + wordDict.size());

	}
521935
protected Data init(Data data) {
	if (JSON_ARRAY == null) JSON_ARRAY = getFLF(data, "_literalArray");
	if (JSON_STRUCT == null) JSON_STRUCT = getFLF(data, "_literalStruct");
	if (GET_STATIC_SCOPE == null) GET_STATIC_SCOPE = getFLF(data, "_getStaticScope");
	if (GET_SUPER_STATIC_SCOPE == null) GET_SUPER_STATIC_SCOPE = getFLF(data, "_getSuperStaticScope");
	return data;
    }
839013
@Override
    public void readExternal(ObjectInput in)
        throws IOException, ClassNotFoundException {

        byte header = in.readByte();

        switch ((header & 0xFF) >> 4) {
            case DATE_TYPE:
                this.obj = this.readDate(in, header);
                break;
            case TIME_TYPE:
                this.obj = this.readTime(in);
                break;
            case WEEKMODEL_TYPE:
                this.obj = this.readWeekmodel(in, header);
                break;
            case MOMENT_TYPE:
                this.obj = this.readMoment(in, header);
                break;
            case MACHINE_TIME_TYPE:
                this.obj = this.readMachineTime(in, header);
                break;
            case DURATION_TYPE:
                this.obj = this.readDuration(in, header);
                break;
            case DAY_PERIOD_TYPE:
                this.obj = this.readDayPeriod(in, header);
                break;
            case TIMESTAMP_TYPE:
                this.obj = this.readTimestamp(in, header);
                break;
            default:
                throw new StreamCorruptedException("Unknown serialized type.");
        }

    }
273165
public void load(InputStream in) {
    try {
      PropertiesConfiguration config = new PropertiesConfiguration();
      // disabled to prevent accumulo classpath value from being shortened
      config.setDelimiterParsingDisabled(true);
      config.load(in);
      ((CompositeConfiguration) internalConfig).addConfiguration(config);
    } catch (ConfigurationException e) {
      throw new IllegalArgumentException(e);
    }
  }
880001
private static void addInputPath(Job job, Iterable<String> pathStrings, Class<? extends InputFormat> inputFormatClass)
  {
    Configuration conf = job.getConfiguration();
    StringBuilder inputFormats = new StringBuilder(
        StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS))
    );

    String[] paths = Iterables.toArray(pathStrings, String.class);
    for (int i = 0; i < paths.length - 1; i++) {
      if (inputFormats.length() > 0) {
        inputFormats.append(',');
      }
      inputFormats.append(paths[i]).append(';').append(inputFormatClass.getName());
    }
    if (inputFormats.length() > 0) {
      conf.set(MultipleInputs.DIR_FORMATS, inputFormats.toString());
    }
    // add last one separately for possible initialization in MultipleInputs
    MultipleInputs.addInputPath(job, new Path(paths[paths.length - 1]), inputFormatClass);
  }
729316
function load_data ($input = null) {
		$data = null;
		$file = null;
		if ( $input === null ) {
			$file = $this->file;
		} elseif ( file_exists($input) ) {
			$file = $input;
		} else {
			$data = $input;
		}
		if ( !empty($data) || $data = $this->_rfile($file) ) {
			if ( $this->file != $file ) $this->file = $file;
			if ( preg_match('/\.php$/i', $file) && preg_match('/<\?.*?\?>(.*)/ims', $data, $strip) ) {
				$data = ltrim($strip[1]);
			}
			if ( $this->convert_encoding ) $data = iconv($this->input_encoding, $this->output_encoding, $data);
			if ( substr($data, -1) != "\n" ) $data .= "\n";
			$this->file_data = &$data;
			return true;
		}
		return false;
	}
1282940
public static void fill(InterleavedI16 input, int value) {

		for (int y = 0; y < input.height; y++) {
			int index = input.getStartIndex() + y * input.getStride();
			int end = index + input.width*input.numBands;
			Arrays.fill(input.data,index,end, (short)value);
		}
	}