Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.

@return A Flink InputFormat that wraps the Hadoop FileInputFormat.
9002
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
			org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath) throws IOException {
		return readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance());
	}
1266139
@Pure
	public static URL generateShapeFileIndexFromShapeFile(File shapeFile) throws IOException {
		final File shxFile = FileSystem.replaceExtension(shapeFile, "." + ShapeFileIndexFilter.EXTENSION_SHX); //$NON-NLS-1$

		try (ShapeFileReader<Object> shpReader = new ShapeFileReader<>(shapeFile, new NullFactory())) {

			try (ShapeFileIndexWriter shxWriter = new ShapeFileIndexWriter(
					shxFile,
					shpReader.getShapeElementType(),
					shpReader.getBoundsFromHeader())) {
				int offset = shpReader.getFileReadingPosition();
				int newOffset;
				int shpLength;

				while (shpReader.read() != null) {
					newOffset = shpReader.getFileReadingPosition();
					// byte length - file header length - record header length
					shpLength = newOffset - offset;
					shxWriter.write(shpLength);
					offset = newOffset;
				}
			}
		}

		return shxFile.toURI().toURL();
	}
1016136
public static HadoopCompat getCompat() {
        String ver = VersionInfo.getVersion();

        log.debug("Read Hadoop VersionInfo string {}", ver);

        final String pkgName = HadoopCompatLoader.class.getPackage().getName();
        final String className;

        if (ver.startsWith("1.")) {
            className = pkgName + ".h1.Hadoop1Compat";
        } else {
            className = pkgName + ".h2.Hadoop2Compat";
        }

        log.debug("Attempting to load class {} and instantiate with nullary constructor", className);
        try {
            Constructor<?> ctor = Class.forName(className).getConstructor();
            log.debug("Invoking constructor {}", ctor);
            return (HadoopCompat)ctor.newInstance();
        } catch (NoSuchMethodException e) {
            throw new RuntimeException(e);
        } catch (SecurityException e) {
            throw new RuntimeException(e);
        } catch (ClassNotFoundException e) {
            throw new RuntimeException(e);
        } catch (InstantiationException e) {
            throw new RuntimeException(e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException(e);
        } catch (IllegalArgumentException e) {
            throw new RuntimeException(e);
        } catch (InvocationTargetException e) {
            throw new RuntimeException(e);
        }
    }
519460
public DataWriterBuilder<S, D> writeInFormat(WriterOutputFormat format) {
    this.format = format;
    log.debug("writeInFormat : {}", this.format);
    return this;
  }
985732
public static Map<String, Object> read(StreamingInput input)
		throws IOException
	{
		Token next = input.peek();
		if(next == null) return Collections.emptyMap();

		switch(next)
		{
			case LIST_START:
			case LIST_END:
			case OBJECT_END:
			case VALUE:
			case NULL:
				throw new SerializationException("Error in configuration file, should be in in key, value format");
		}

		return readMap(input);
	}
533041
public String transform(File inputFile) {
		final String content;
		try (FileReader reader = new FileReader(inputFile)) {
			content = read(reader);
		} catch (IOException exception) {
			reportError(Messages.SarlDocumentationParser_0, exception);
			return null;
		}
		return transform(content, inputFile);
	}
1205773
public  InputStream getInputStream(File f)
	throws IOException
	{

		// use the magic numbers to determine the compression type,
		// use file extension only as 2nd choice

		int magic = 0;


		InputStream test = getInputStreamFromFile(f);
		magic = getMagicNumber(test);
		test.close();


		InputStream inputStream = null;

		String fileName = f.getName();

		if (magic == UncompressInputStream.LZW_MAGIC ) {
			// a Z compressed file
			return openCompressedFile(f);
		}

		else if (magic == GZIP_MAGIC ) {
			return openGZIPFile(f);
		}

		else if ( fileName.endsWith(".gz")) {
			return openGZIPFile(f);
		}

		else if ( fileName.endsWith(".zip")){

			ZipFile zipfile = new ZipFile(f);

			// stream to first entry is returned ...
			ZipEntry entry;
			Enumeration<? extends ZipEntry> e = zipfile.entries();
			if ( e.hasMoreElements()){
				entry = e.nextElement();
				inputStream = zipfile.getInputStream(entry);
			} else {
				throw new IOException ("Zip file has no entries");
			}

		}

		else if ( fileName.endsWith(".jar")) {

			JarFile jarFile = new JarFile(f);

			// stream to first entry is returned
			JarEntry entry;
			Enumeration<JarEntry> e = jarFile.entries();
			if ( e.hasMoreElements()){
				entry = e.nextElement();
				inputStream = jarFile.getInputStream(entry);
			} else {
				throw new IOException ("Jar file has no entries");
			}
		}

		else if ( fileName.endsWith(".Z")) {
			// unix compressed
			return openCompressedFile(f);

		}

		else {

			// no particular extension found, assume that it is an uncompressed file
			inputStream = getInputStreamFromFile(f);
		}

		return inputStream;
	}
960285
public static StreamShardHandle convertToStreamShardHandle(StreamShardMetadata streamShardMetadata) {
		Shard shard = new Shard();
		shard.withShardId(streamShardMetadata.getShardId());
		shard.withParentShardId(streamShardMetadata.getParentShardId());
		shard.withAdjacentParentShardId(streamShardMetadata.getAdjacentParentShardId());

		HashKeyRange hashKeyRange = new HashKeyRange();
		hashKeyRange.withStartingHashKey(streamShardMetadata.getStartingHashKey());
		hashKeyRange.withEndingHashKey(streamShardMetadata.getEndingHashKey());
		shard.withHashKeyRange(hashKeyRange);

		SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
		sequenceNumberRange.withStartingSequenceNumber(streamShardMetadata.getStartingSequenceNumber());
		sequenceNumberRange.withEndingSequenceNumber(streamShardMetadata.getEndingSequenceNumber());
		shard.withSequenceNumberRange(sequenceNumberRange);

		return new StreamShardHandle(streamShardMetadata.getStreamName(), shard);
	}
242433
public static ObjectStream<SequenceLabelSample> getSequenceStream(
      final String inputData, final String clearFeatures,
      final String aCorpusFormat) throws IOException {
    ObjectStream<SequenceLabelSample> samples = null;
    if (aCorpusFormat.equalsIgnoreCase("conll03")) {
      final ObjectStream<String> nameStream = IOUtils
          .readFileIntoMarkableStreamFactory(inputData);
      samples = new CoNLL03Format(clearFeatures, nameStream);
    } else if (aCorpusFormat.equalsIgnoreCase("conll02")) {
      final ObjectStream<String> nameStream = IOUtils
          .readFileIntoMarkableStreamFactory(inputData);
      samples = new CoNLL02Format(clearFeatures, nameStream);
    } else if (aCorpusFormat.equalsIgnoreCase("tabulated")) {
      final ObjectStream<String> nameStream = IOUtils
          .readFileIntoMarkableStreamFactory(inputData);
      samples = new TabulatedFormat(clearFeatures, nameStream);
    } else if (aCorpusFormat.equalsIgnoreCase("lemmatizer")) {
      final ObjectStream<String> seqStream = IOUtils
          .readFileIntoMarkableStreamFactory(inputData);
      samples = new LemmatizerFormat(clearFeatures, seqStream);
    } else {
      System.err.println("Test set corpus format not valid!!");
      System.exit(1);
    }
    return samples;
  }
281371
public List<File> writeSortedChunks(Iterator<T> input) throws IOException {
        List<File> result = new ArrayList<File>();
        while (input.hasNext()) {
            File chunkFile = writeSortedChunk(input);
            result.add(chunkFile);
        }
        if (debugMerge) {
            System.out.printf("Chunks %d (chunkSize=%d, maxOpenFiles=%d)\n", result.size(), config.chunkSize, config.maxOpenFiles);
        }
        return result;
    }
1004440
@Override
	public <I> StreamConsumer<I> newInput(Function<I, K> keyFunction, Reducer<K, I, O, A> reducer) {
		return super.newInput(keyFunction, reducer);
	}
1204804
public static <K, V> Map<K, V> createHashMapAdapter(int expectedMapSize) {
        final int initialCapacity = calculateInitialCapacity(expectedMapSize);
        return new HashMapAdapter<K, V>(initialCapacity, HASHMAP_DEFAULT_LOAD_FACTOR);
    }
320261
public HadoopJarStepConfig toHadoopJarStepConfig() {
    List<String> args = new ArrayList<String>();

    if (reducer == null) {
      hadoopConfig.put("mapred.reduce.tasks", "0");
    }

    for (Map.Entry<String, String> entry : hadoopConfig.entrySet()) {
      args.add("-D");
      args.add(entry.getKey() + "=" + entry.getValue());
    }

    for (String input : inputs) {
      args.add("-input");
      args.add(input);
    }

    if (output != null) {
      args.add("-output");
      args.add(output);
    }

    if (mapper != null) {
      args.add("-mapper");
      args.add(mapper);
    }

    if (reducer != null) {
      args.add("-reducer");
      args.add(reducer);
    }

    return new HadoopJarStepConfig()
      .withJar("/home/hadoop/contrib/streaming/hadoop-streaming.jar")
      .withArgs(args);
  }
567336
public StreamMessageHeader createHeader(Object input) throws ConvertFailException
    {
        StreamMessageHeader header = new StreamMessageHeader();
        header.setType(getType());

        return header;
    }
1168796
public static FormatFactory factory()
    {

        return new FormatFactory()
        {
            @Override
            public Format getFormat(String name, String args, Locale locale)
            {
                Map<String, Format> mt = FormatTables.get(name);
                Preconditions.checkArgument(mt != null, "There's no format instance for [%s]", name);

                Format m = mt.get((args == null || args.length() < 1) ? FormatNames.DEFAULT : args);
                Preconditions.checkArgument(m != null, "There's no signature in [%s] for the given args [%s]", name,
                        args);

                return ((ConfigurableFormat) m).withLocale(locale);
            }
        };

    }
920066
private Configuration generateClusterConfiguration(Configuration configuration) {
		final Configuration resultConfiguration = new Configuration(Preconditions.checkNotNull(configuration));

		final String webTmpDir = configuration.getString(WebOptions.TMP_DIR);
		final File uniqueWebTmpDir = new File(webTmpDir, "flink-web-" + UUID.randomUUID());

		resultConfiguration.setString(WebOptions.TMP_DIR, uniqueWebTmpDir.getAbsolutePath());

		return resultConfiguration;
	}
1317861
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) {
		// select the name (or create a default one)
		String name = this.name != null ? this.name : this.format.toString();
		GenericDataSinkBase<T> sink = new GenericDataSinkBase<T>(this.format, new UnaryOperatorInformation<T, Nothing>(this.type, new NothingTypeInfo()), name);
		// set input
		sink.setInput(input);
		// set dop
		if(this.dop > 0) {
			// use specified dop
			sink.setDegreeOfParallelism(this.dop);
		} else {
			// if no dop has been specified, use dop of input operator to enable chaining
			sink.setDegreeOfParallelism(input.getDegreeOfParallelism());
		}
		
		return sink;
	}