Get the source URL and store it to a destination file path

@param input input stream
@param destinationFilePath destination
@throws UpdateException on error
765933
public static void updateFileFromInputStream(final InputStream input, final String destinationFilePath) throws
        UpdateException {
        update(new FileStreamUpdater(input), new File(destinationFilePath));
    }
544449
public HttpMove createMoveMethod(final String sourcePath, final String destinationPath) {
        return new HttpMove(repositoryURL + sourcePath, repositoryURL + destinationPath);
    }
1008781
public void setSourceUrl(java.util.Collection<StringFilter> sourceUrl) {
        if (sourceUrl == null) {
            this.sourceUrl = null;
            return;
        }

        this.sourceUrl = new java.util.ArrayList<StringFilter>(sourceUrl);
    }
407673
private static InputStream inputStreamFromFile(String relativePathPrefix, String src) throws IOException {
        File file = getFile(relativePathPrefix, src);
        if (!file.exists()) {
            if (src.length() > 0 && src.charAt(0) == File.separatorChar) {
                file = getFile(relativePathPrefix, src.substring(1));
            }
            if (!file.exists()) {
                file = null;
            }
        } else if (!file.isFile() || !file.canRead()) {
            file = null;
        }
        if (file != null) {
            return new FileInputStream(file);
        }
        return null;
    }
48208
public static <E extends Exception> void copy(File srcFile, File destDir, final boolean preserveFileDate,
            final Try.BiPredicate<? super File, ? super File, E> filter) throws UncheckedIOException, E {
        if (!srcFile.exists()) {
            throw new IllegalArgumentException("The source file doesn't exist: " + srcFile.getAbsolutePath());
        }

        if (destDir.exists()) {
            if (destDir.isFile()) {
                throw new IllegalArgumentException("The destination file must be directory: " + destDir.getAbsolutePath());
            }
        } else {
            if (!destDir.mkdirs()) {
                throw new UncheckedIOException(new IOException("Failed to create destination directory: " + destDir.getAbsolutePath()));
            }
        }

        if (destDir.canWrite() == false) {
            throw new UncheckedIOException(new IOException("Destination '" + destDir + "' cannot be written to"));
        }

        String destCanonicalPath = null;
        String srcCanonicalPath = null;
        try {
            srcFile = srcFile.getCanonicalFile();
            destDir = destDir.getCanonicalFile();
            destCanonicalPath = destDir.getCanonicalPath();
            srcCanonicalPath = srcFile.getCanonicalPath();
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }

        if (srcFile.isDirectory()) {
            if (destCanonicalPath.startsWith(srcCanonicalPath) && (destCanonicalPath.length() == srcCanonicalPath.length()
                    || destCanonicalPath.charAt(srcCanonicalPath.length()) == '/' || destCanonicalPath.charAt(srcCanonicalPath.length()) == '\\')) {
                throw new IllegalArgumentException(
                        "Failed to copy due to the target directory: " + destCanonicalPath + " is in or same as the source directory: " + srcCanonicalPath);
            }

            try {
                doCopyDirectory(srcFile, destDir, preserveFileDate, filter);
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        } else {
            File destFile = null;

            try {
                if (destDir.getCanonicalPath().equals(srcFile.getParentFile().getCanonicalPath())) {
                    destFile = new File(destDir, "Copy of " + srcFile.getName());
                } else {
                    destFile = new File(destDir, srcFile.getName());
                }
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }

            doCopyFile(srcFile, destFile, preserveFileDate);
        }
    }
1304747
@Override
    public InputStream get(String path) throws IOException {
        InputStream stream = _cl.getResourceAsStream(rewritePath(path));
        if (stream == null) {
            throw new FileNotFoundException("Not found in classpath: " + path);
        } else {
            return stream;
        }
    }
243640
public static InputStream asStream(String generalPath) throws IOException {
		if (StringUtils.startsWith(generalPath, CLASSPATH_PREFIX)) {
			String resourceName = StringUtils.substringAfter(generalPath, CLASSPATH_PREFIX);
			return ResourceUtil.asStream(resourceName);
		}

		try {
			// try URL
			return FileUtil.asInputStream(getFileByURL(new URL(generalPath)));
		} catch (MalformedURLException ex) {
			// no URL -> treat as file path
			return FileUtil.asInputStream(generalPath);
		}
	}
1150374
public static void updateFileFromUrl(final String sourceUrl, final String destinationFilePath,
                                         final String username, final String password,
                                         final URLFileUpdaterFactory factory) throws
        UpdateException {
        String tusername = username;
        String tpassword = password;
        URL url;
        try {
            url = new URL(sourceUrl);
            if (null == username && null == password && null != url.getUserInfo()) {
                //try to extract userinfo from URL
                final String userInfo = url.getUserInfo();
                final String[] split = userInfo.split(":", 2);
                if (2 == split.length) {
                    tusername = split[0];
                    tpassword = split[1];
                    url = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getFile());
                }
            }
        } catch (MalformedURLException e) {
            throw new UpdateException(e);
        }
        final File destinationFile = new File(destinationFilePath);
        final long mtime = destinationFile.exists() ? destinationFile.lastModified() : 0;

        update(factory.fileUpdaterFromURL(url, tusername, tpassword), destinationFile);
        if (destinationFile.lastModified() > mtime) {
            logger.info("updated file: " + destinationFile.getAbsolutePath());
        } else {
            logger.info("file already up to date: " + destinationFile.getAbsolutePath());
        }

    }
1075410
protected static String addFileNameToTargetPath(String sourcePath, String targetPath) {
        if (targetPath.endsWith("/")) {
            String filename = Paths.get(sourcePath).getFileName().toString();
            return targetPath + filename;
        }
        return targetPath;
    }
363216
public static void download(String entranceURL, String type, String path, boolean containEntrance, boolean justOriginalName){
        int timeout = 300000;
        if(!entranceURL.endsWith("/")){
            entranceURL += "/";
        }
        Set<String> urls = new HashSet<>();
        boolean ok = false;
        int limit=0;
        while (!ok && (limit++) < 3) {
            try {
                System.out.println("【"+type+"】*** connect " + entranceURL);
                for (Element element : Jsoup.connect(entranceURL).timeout(timeout).get().select("a")) {
                    String href = element.attr("href").trim();
                    if (!href.startsWith("http")) {
                        if(href.startsWith("/")){
                            href = "http://www.bbc.co.uk" + href;
                        }else{
                            href = entranceURL + href;
                        }
                    }
                    if (href.startsWith(entranceURL) && (!href.equals(entranceURL) || containEntrance) ) {
                        urls.add(href);
                    }
                }
                ok = true;
            } catch (Exception e) {
                System.out.println(e.getMessage() + " retry...");
            }
        }
        AtomicInteger i = new AtomicInteger(1);
        Set<String> resources = new HashSet<>();
        urls.stream().sorted().forEach(url -> {
            boolean success = false;
            int times=0;
            while (!success && (times++) < 3) {
                try {
                    System.out.println(i.get() + "、connect " + url);
                    for (Element element : Jsoup.connect(url).timeout(timeout).get().select("a")) {
                        String href = element.attr("href").trim();
                        //只下载mp3、mp4、wav和pdf文件
                        if (href.endsWith(".mp3") || href.endsWith(".wav") || href.endsWith(".mp4") || href.endsWith(".pdf")) {
                            if (!href.startsWith("http")) {
                                if(href.startsWith("/")){
                                    href = "http://www.bbc.co.uk" + href;
                                }else{
                                    String[] attr = url.split("/");
                                    href = url.substring(0, url.length()-attr[attr.length-1].length()) + href;
                                }
                            }
                            resources.add(href);
                        }
                    }
                    i.incrementAndGet();
                    success = true;
                } catch (Exception e) {
                    System.out.println(e.getMessage() + " retry...");
                }
            }
        });
        AtomicInteger j = new AtomicInteger(1);
        count += resources.size();
        resources.stream().sorted().forEach(resource -> {
            boolean success = false;
            int times=0;
            while (!success && (times++) < 3) {
                try {
                    //提取文件名称
                    String[] attr = resource.split("/");
                    String fileName = attr[attr.length - 2] + "_" + attr[attr.length - 1].replace(attr[attr.length - 2], "");
                    if(attr[attr.length - 1].endsWith(attr[attr.length - 2])){
                        fileName = attr[attr.length - 1];
                    }
                    fileName = fileName.replace("_download", "");
                    if(justOriginalName){
                        fileName = attr[attr.length - 1];
                    }
                    System.out.println(resources.size() + "/" + j.get() + "、find resource: " + resource);
                    //确保本地路径存储
                    Path dir = Paths.get(path, type);
                    if (!Files.exists(dir)) {
                        //不存在则新建
                        dir.toFile().mkdirs();
                    }
                    //保存文件的完整本地路径
                    Path out = Paths.get(path, type, fileName);
                    //如果文件存在则表示之前已经下载过，本次不用下载
                    //因为BBC的访问不稳定，所以可能需要执行程序多次才能完整下载完毕，所以这里要处理已存在文件的问题
                    if (!Files.exists(out)) {
                        //下载文件
                        Connection.Response response = Jsoup.connect(resource).maxBodySize(0).ignoreContentType(true).timeout(timeout).execute();
                        //将文件保存到本地
                        Files.write(out, response.bodyAsBytes());
                        System.out.println(resources.size() + "/" + j.get() + "、save resource to: " + out);
                    } else {
                        System.out.println(resources.size() + "/" + j.get() + "、resource exist, don't need to download");
                    }
                    j.incrementAndGet();
                    success = true;
                } catch (Exception e) {
                    System.out.println(e.getMessage() + " retry...");
                }
            }
        });
    }
568346
public void setXsdInput(final String xsdInputPath) {
        if (xsdInputPath == null) {
            throw (new IllegalArgumentException(
                    "You must provide a source folder or file"));
        }
        File xsdInput = new File(xsdInputPath);
        if (xsdInput.exists()) {
            if (xsdInput.isDirectory() && xsdInput.list().length == 0) {
                throw new IllegalArgumentException("Folder '" + xsdInputPath
                        + "' is empty");
            }
        } else {
            throw new IllegalArgumentException("Input file or folder '"
                    + xsdInputPath + "' not found");
        }
        this.xsdInput = xsdInput;
    }
127391
def download_file(self, file_url, destination_folder=None):
        
        if not destination_folder:
            destination_folder = constants.Files.DOWNLOADS_FOLDER
        page_utils._download_file_to(file_url, destination_folder)
924073
public void updateStreamPullUrl(UpdateStreamPullUrlRequest request) {
        checkNotNull(request, "The parameter request should NOT be null.");

        checkStringNotEmpty(request.getDomain(), "Domain should NOT be empty.");
        checkStringNotEmpty(request.getApp(), "App should NOT be empty");
        checkStringNotEmpty(request.getStream(), "Stream should NOT be empty.");
        checkStringNotEmpty(request.getPullUrl(), "PullUrl should NOT be empty.");

        InternalRequest internalRequest = createRequest(HttpMethodName.PUT, request, LIVE_DOMAIN,
                request.getDomain(), LIVE_APP, request.getApp(), LIVE_STREAM, request.getStream());
        internalRequest.addParameter("pullUrl", request.getPullUrl());
        invokeHttpClient(internalRequest, AbstractBceResponse.class);
    }
35577
static void copyResources(URL resourceUrl, File targetPath) throws IOException, URISyntaxException {
        if (resourceUrl == null) {
            return;
        }

        URLConnection urlConnection = resourceUrl.openConnection();

        /**
         * Copy resources either from inside jar or from project folder.
         */
        if (urlConnection instanceof JarURLConnection) {
            copyJarResourceToPath((JarURLConnection) urlConnection, targetPath);
        } else if (VFS_PROTOCOL.equals(resourceUrl.getProtocol())) {
            VirtualFile virtualFileOrFolder = VFS.getChild(resourceUrl.toURI());
            copyFromWarToFolder(virtualFileOrFolder, targetPath);
        } else {
            File file = new File(resourceUrl.getPath());
            if (file.isDirectory()) {
                for (File resourceFile : FileUtils.listFiles(file, null, true)) {
                    int index = resourceFile.getPath().lastIndexOf(targetPath.getName()) + targetPath.getName().length();
                    File targetFile = new File(targetPath, resourceFile.getPath().substring(index));
                    if (!targetFile.exists() || targetFile.length() != resourceFile.length()) {
                        if (resourceFile.isFile()) {
                            FileUtils.copyFile(resourceFile, targetFile);
                        }
                    }
                }
            } else {
                if (!targetPath.exists() || targetPath.length() != file.length()) {
                    FileUtils.copyFile(file, targetPath);
                }
            }
        }
    }
440064
@Override
    public String getInputFilePath(Map<String, Object> properties)
    {
        String path = (String) properties.get(SparkPropertiesConstants.HDFS_INPUT_FILE_PATH);
        if (path == null || path.isEmpty())
        {
            throw new KunderaException(
                    "Please set the path of inputfile while creating EntityManager using the property" + "\""
                            + SparkPropertiesConstants.HDFS_INPUT_FILE_PATH + "\".");
        }

        return path;
    }
1258617
public static long copyUrl(OutputStream dst, URL src) throws IOException {
		long copied;

		try (InputStream srcStream = src.openStream()) {
			copied = copyStream(dst, srcStream);
		}
		return copied;
	}
445119
public static InputStream getUrlPictureStream(String urlPath) throws IOException {
		URL url  = new URL(urlPath);
		return url.openConnection().getInputStream();
	}