Implement the import action (upload files from the current machine to
HDFS)

@param object
@throws SftpException
@throws JSchException
@throws InvocationTargetException
@throws InterruptedException
777379
private void uploadFilesToDFS(IStructuredSelection selection)
      throws InvocationTargetException, InterruptedException {

    // Ask the user which files to upload
    FileDialog dialog =
        new FileDialog(Display.getCurrent().getActiveShell(), SWT.OPEN
            | SWT.MULTI);
    dialog.setText("Select the local files to upload");
    dialog.open();

    List<File> files = new ArrayList<File>();
    for (String fname : dialog.getFileNames())
      files.add(new File(dialog.getFilterPath() + File.separator + fname));

    // TODO enable upload command only when selection is exactly one folder
    List<DFSFolder> folders = filterSelection(DFSFolder.class, selection);
    if (folders.size() >= 1)
      uploadToDFS(folders.get(0), files);
  }

  /**
   * Implement the import action (upload directory from the current machine
   * to HDFS)
   * 
   * @param object
   * @throws SftpException
   * @throws JSchException
   * @throws InvocationTargetException
   * @throws InterruptedException
   */
  private void uploadDirectoryToDFS(IStructuredSelection selection)
      throws InvocationTargetException, InterruptedException {

    // Ask the user which local directory to upload
    DirectoryDialog dialog =
        new DirectoryDialog(Display.getCurrent().getActiveShell(), SWT.OPEN
            | SWT.MULTI);
    dialog.setText("Select the local file or directory to upload");

    String dirName = dialog.open();
    final File dir = new File(dirName);
    List<File> files = new ArrayList<File>();
    files.add(dir);

    // TODO enable upload command only when selection is exactly one folder
    final List<DFSFolder> folders =
        filterSelection(DFSFolder.class, selection);
    if (folders.size() >= 1)
      uploadToDFS(folders.get(0), files);

  }

  private void uploadToDFS(final DFSFolder folder, final List<File> files)
      throws InvocationTargetException, InterruptedException {

    PlatformUI.getWorkbench().getProgressService().busyCursorWhile(
        new IRunnableWithProgress() {
          public void run(IProgressMonitor monitor)
              throws InvocationTargetException {

            int work = 0;
            for (File file : files)
              work += computeUploadWork(file);

            monitor.beginTask("Uploading files to distributed file system",
                work);

            for (File file : files) {
              try {
                folder.upload(monitor, file);

              } catch (IOException ioe) {
                ioe.printStackTrace();
                MessageDialog.openError(null,
                    "Upload files to distributed file system",
                    "Upload failed.\n" + ioe);
              }
            }

            monitor.done();

            // Update the UI
            folder.doRefresh();
          }
        });
  }

  private void reconnect(IStructuredSelection selection) {
    for (DFSLocation location : filterSelection(DFSLocation.class, selection))
      location.reconnect();
  }

  private void disconnect(IStructuredSelection selection) {
    if (selection.size() != 1)
      return;

    Object first = selection.getFirstElement();
    if (!(first instanceof DFSLocationsRoot))
      return;

    DFSLocationsRoot root = (DFSLocationsRoot) first;
    root.disconnect();
    root.refresh();
  }

  /**
   * Implements the Download action from HDFS to the current machine
   * 
   * @param object
   * @throws SftpException
   * @throws JSchException
   * @throws InterruptedException
   * @throws InvocationTargetException
   */
  private void downloadFromDFS(IStructuredSelection selection)
      throws InvocationTargetException, InterruptedException {

    // Ask the user where to put the downloaded files
    DirectoryDialog dialog =
        new DirectoryDialog(Display.getCurrent().getActiveShell());
    dialog.setText("Copy to local directory");
    dialog.setMessage("Copy the selected files and directories from the "
        + "distributed filesystem to a local directory");
    String directory = dialog.open();

    if (directory == null)
      return;

    final File dir = new File(directory);
    if (!dir.exists())
      dir.mkdirs();

    if (!dir.isDirectory()) {
      MessageDialog.openError(null, "Download to local file system",
          "Invalid directory location: \"" + dir + "\"");
      return;
    }

    final List<DFSPath> paths = filterSelection(DFSPath.class, selection);

    PlatformUI.getWorkbench().getProgressService().busyCursorWhile(
        new IRunnableWithProgress() {
          public void run(IProgressMonitor monitor)
              throws InvocationTargetException {

            int work = 0;
            for (DFSPath path : paths)
              work += path.computeDownloadWork();

            monitor
                .beginTask("Downloading files to local file system", work);

            for (DFSPath path : paths) {
              if (monitor.isCanceled())
                return;
              try {
                path.downloadToLocalDirectory(monitor, dir);
              } catch (Exception e) {
                // nothing we want to do here
                e.printStackTrace();
              }
            }

            monitor.done();
          }
        });
  }
899820
@Override
    @Transactional
    public void importData(ExternalPermissionOwner externalPermissionOwner) {
        final String name = externalPermissionOwner.getName();
        final String fname = externalPermissionOwner.getFname();
        final IPermissionOwner permissionOwner =
                this.permissionOwnerDao.getOrCreatePermissionOwner(name, fname);

        final String desc = externalPermissionOwner.getDesc();
        permissionOwner.setDescription(desc);

        for (final ExternalActivity externalActivity : externalPermissionOwner.getActivities()) {
            final String activityName = externalActivity.getName();
            final String activityFname = externalActivity.getFname();
            final String targetProvider = externalActivity.getTargetProvider();

            final IPermissionActivity permissionActivity =
                    this.permissionOwnerDao.getOrCreatePermissionActivity(
                            permissionOwner, activityName, activityFname, targetProvider);

            final String activityDesc = externalActivity.getDesc();
            permissionActivity.setDescription(activityDesc);
        }

        this.permissionOwnerDao.saveOwner(permissionOwner);
    }
990661
private void doExport(final String fileName) throws FrameworkException {

		// collect all nodes etc that belong to the frontend (including files)
		// and export them to the given output file
		final Set<RelationshipInterface> rels = new LinkedHashSet<>();
		final Set<NodeInterface> nodes        = new LinkedHashSet<>();
		final Set<String> filePaths           = new LinkedHashSet<>();
		final App app                         = StructrApp.getInstance();

		try (final Tx tx = app.tx()) {

			// collect folders that are marked for export
			for (final Folder folder : app.nodeQuery(Folder.class).and(StructrApp.key(Folder.class, "includeInFrontendExport"), true).getAsList()) {

				collectDataRecursively(app, folder, nodes, rels, filePaths);
			}

			// collect pages (including files, shared components etc.)
			for (final Page page : app.nodeQuery(Page.class).getAsList()) {

				collectDataRecursively(app, page, nodes, rels, filePaths);
			}

			SyncCommand.exportToFile(fileName, nodes, rels, filePaths, true);

			tx.success();
		}
	}
815257
private void writeObject(ObjectOutputStream out) throws IOException {
		out.writeUTF(hadoopOutputFormatName);
		jobConf.write(out);
		out.writeObject(converter);
		out.writeObject(fileOutputCommitterWrapper);
	}
11184
private void loadEntireJsonObject(JSONObject json) throws JSONException {
    Iterator<?> it = json.keys();
    while (it.hasNext()) {
      // This key is something like core-site.xml or hdfs-site.xml
      Object obj = it.next();
      if (!(obj instanceof String)) {
        LOG.warn("Object not instance of string : " + obj + " skipping");
        continue;
      }
      String key = (String) obj;
      JSONObject partition = json.getJSONObject(key);
      loadJsonResource(partition, properties, key);
    }
  }
187373
String copyFile(
            final ExecutionContext executionContext,
            final File file,
            final InputStream input,
            final String content,
            final INodeEntry node,
            String destination
    ) throws FileCopierException
    {
        final String pluginname = getProvider().getName();
        final Map<String, Map<String, String>> localDataContext = createScriptDataContext(
            executionContext.getFramework(),
            executionContext.getFrameworkProject(),
            executionContext.getDataContext());
        //add node context data
        localDataContext.put("node", DataContextUtils.nodeData(node));

        Description pluginDesc = getDescription();
        //load config.* property values in from project or framework scope
        final Map<String, Map<String, String>> finalDataContext;
        try {
            finalDataContext = loadConfigData(executionContext, loadInstanceDataFromNodeAttributes(node, pluginDesc), localDataContext, pluginDesc,
                                              ServiceNameConstants.FileCopier
            );
        } catch (ConfigurationException e) {
            throw new FileCopierException("[" + pluginname + "]: "+e.getMessage(),
                    StepFailureReason.ConfigurationFailure, e);
        }

        final File srcFile =
                null != file ?
                        file :
                        BaseFileCopier.writeTempFile(executionContext, null, input, content);

        String destFilePath = destination;
        if (null == destFilePath) {

            String identity = null!=executionContext.getDataContext() && null!=executionContext.getDataContext().get("job")?
                              executionContext.getDataContext().get("job").get("execid"):null;
            destFilePath = BaseFileCopier.generateRemoteFilepathForNode(
                    node,
                    executionContext.getFramework().getFrameworkProjectMgr().getFrameworkProject(executionContext.getFrameworkProject()),
                    executionContext.getFramework(),
                    (null != file ? file.getName() : "dispatch-script"),
                    null,
                    identity
            );
        }
        //put file in a directory
        if (null != destFilePath && destFilePath.endsWith("/")) {
            destFilePath += srcFile.getName();
        }
        //add some more data context values to allow templatized script-copy attribute
        final HashMap<String, String> scptexec = new HashMap<>();
        //set up the data context to include the local temp file
        scptexec.put("file", srcFile.getAbsolutePath());
        scptexec.put("destination", null != destFilePath ? destFilePath : "");
        finalDataContext.put("file-copy", scptexec);
        Map<String, Map<String, String>> fileCopyContext =
                DataContextUtils.addContext(
                        "file-copy",
                        scptexec,
                        null
                );

        final ExecArgList execArgList = createScriptArgsList(fileCopyContext);
        final String localNodeOsFamily=getFramework().createFrameworkNode().getOsFamily();

        executionContext.getExecutionListener().log(3,
                                                    "[" +
                                                    getProvider().getName() +
                                                    "] executing: " +
                                                    execArgList.asFlatStringList()
        );

        final ByteArrayOutputStream captureSysOut = new ByteArrayOutputStream();
        try {
            final int result = getScriptExecHelper().runLocalCommand(
                    localNodeOsFamily,
                    execArgList,
                    finalDataContext,
                    null,
                    captureSysOut,
                    System.err
            );
            executionContext.getExecutionListener().log(3, "[" + pluginname + "]: result code: " + result);
            if(result!=0){
                throw new FileCopierException("[" + pluginname + "]: external script failed with exit code: " + result,
                                              NodeStepFailureReason.NonZeroResultCode);
            }
        } catch (IOException e) {
            throw new FileCopierException("[" + pluginname + "] " + e.getMessage(), StepFailureReason.IOFailure);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new FileCopierException("[" + pluginname + "] " + e.getMessage(), StepFailureReason.Interrupted);
        }finally {
            if(null == file) {
                if (!ScriptfileUtils.releaseTempFile(srcFile)) {
                    executionContext.getExecutionListener().log(
                            Constants.WARN_LEVEL,
                            "Unable to remove local temp file: " + srcFile.getAbsolutePath()
                    );
                }
            }
        }


        //load string of output from outputstream
        final String output = captureSysOut.toString();
        if (null == output || output.length() < 1) {
            if (null != destFilePath) {
                return destFilePath;
            }
            throw new FileCopierException("[" + pluginname + "]: No output from external script",
                                          ScriptPluginFailureReason.ScriptPluginFileCopierOutputMissing
            );
        }
        //TODO: require any specific format for the data?
        //look for first line of output
        final String[] split1 = output.split("(\\r?\\n)");
        if (split1.length < 1) {
            if (null != destFilePath) {
                return destFilePath;
            }
            throw new FileCopierException("[" + pluginname + "]: No output from external script",
                                          ScriptPluginFailureReason.ScriptPluginFileCopierOutputMissing);
        }
        final String remotefilepath = split1[0];

        executionContext.getExecutionListener().log(3, "[" + pluginname + "]: result filepath: " + remotefilepath);

        return remotefilepath;
    }
248905
public void addImport(String type) throws IOException {
    Matcher matcher = TYPE_PATTERN.matcher(type);
    if (!matcher.matches()) {
      throw new IllegalArgumentException(type);
    }
    if (importedTypes.put(type, matcher.group(1)) != null) {
      throw new IllegalArgumentException(type);
    }
    out.write("import ");
    out.write(type);
    out.write(";\n");
  }
1130031
@Override
    public ZipImporter importFrom(final File file) throws ArchiveImportException {
        return importFrom(file, Filters.includeAll());
    }
112482
public static void copyToHDFS(String localFile, String hdfsFile) throws IOException {

    String hadoopConfPath; 

    if (Environment.getProperty("hadoop.conf.path") == null)
      hadoopConfPath = "../../../conf";
    else
      hadoopConfPath = Environment.getProperty("hadoop.conf.path");

    // Read the configuration for the Hadoop environment
    Configuration hadoopConf = new Configuration();
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-default.xml"));
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-site.xml"));

    // System.out.println(hadoopConf.get("hadoop.tmp.dir"));
    // System.out.println(hadoopConf.get("fs.default.name"));
    FileSystem fs = FileSystem.get(hadoopConf);

    // HadoopDFS deals with Path
    Path inFile = new Path("file://" + localFile);
    Path outFile = new Path(hadoopConf.get("fs.default.name") + hdfsFile);

     // Read from and write to new file
    Environment.logInfo("Uploading to HDFS (file " + outFile + ") ...");
    fs.copyFromLocalFile(false, inFile, outFile);
  }
1265311
@Override
    public ZipImporter importFrom(File file, Filter<ArchivePath> filter) throws ArchiveImportException {
        Validate.notNull(file, "File must be specified");
        if (file.isDirectory()) {
            throw new IllegalArgumentException("File to import as ZIP must not be a directory: "
                + file.getAbsolutePath());
        }
        Validate.notNull(filter, "Filter must be specified");

        final ZipFile zipFile;
        try {
            zipFile = new ZipFile(file);
        } catch (final IOException ioe) {
            throw new ArchiveImportException("Could not obtain ZIP File from File", ioe);
        }

        // Delegate
        return this.importFrom(zipFile, filter);
    }
1057657
public void importArtifacts(SessionProvider sp, InputStream in) throws RepositoryException, FileNotFoundException
   {
      LOG.info("Extract repository to temporary folder");
      String path = System.getProperty("java.io.tmpdir") + File.separator + "maven2";
      File temporaryFolder = new File(getUniqueFilename(path));
      if (!temporaryFolder.mkdir())
      {
         throw new FileNotFoundException("Cannot create temporary folder");
      }
      ZipEntry entry;
      ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(in));
      try
      {
         while ((entry = zipIn.getNextEntry()) != null)
         {

            if (!(entry.isDirectory() && new File(temporaryFolder + File.separator + entry.getName()).mkdir()))
            {
               int count;
               byte data[] = new byte[BUFFER];
               File file = new File(temporaryFolder + File.separator + entry.getName());

               FileUtils.touch(file);

               FileOutputStream fos = new FileOutputStream(file);
               BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER);
               while ((count = zipIn.read(data, 0, BUFFER)) != -1)
               {
                  dest.write(data, 0, count);
               }
               dest.flush();
               dest.close();
            }

         }
      }
      catch (IOException e)
      {
         LOG.error("Cannot get zip entry from stream", e);
      }
      finally
      {
         IOUtils.closeQuietly(zipIn);
         IOUtils.closeQuietly(in);
      }

      // main part - copy to JCR from temp folder
      // use uploading artifacts from local folder.

      importArtifacts(sp, temporaryFolder);
      try
      {
         FileUtils.deleteDirectory(temporaryFolder);
      }
      catch (IOException e)
      {
         LOG.error("Cannot remove temporary folder", e);
      }

   }
243347
def stage_import_from_file(self, fd, filename='upload.gz'):
        
        schema = ImportSchema()
        resp = self.service.post(self.base,
                                 files={'file': (filename, fd)})
        return self.service.decode(schema, resp)
1075920
protected void handleUpdatedObject(final Object object) {
		runWithWriteLock(new Runnable() {
			public void run() {
				int index = getFinalEventList().indexOf(object);
				if (index >= 0) {
					getFinalEventList().set(index, object);
				}
			}
		});
	}
1099378
private GridFSInputFile createGFSInputFile(GridFS gfs, Object entity, Field f)
    {
        Object obj = PropertyAccessorHelper.getObject(entity, f);
        GridFSInputFile gridFSInputFile = null;
        if (f.getType().isAssignableFrom(byte[].class))
            gridFSInputFile = gfs.createFile((byte[]) obj);
        else if (f.getType().isAssignableFrom(File.class))
        {
            try
            {
                gridFSInputFile = gfs.createFile((File) obj);
            }
            catch (IOException e)
            {
                log.error("Error while creating GridFS file for \"" + f.getName() + "\". Caused by: ", e);
                throw new KunderaException("Error while creating GridFS file for \"" + f.getName() + "\". Caused by: ",
                        e);
            }
        }
        else
            new UnsupportedOperationException(f.getType().getSimpleName() + " is unsupported Lob object");
        return gridFSInputFile;
    }
469513
public static void importModel() throws Exception {
    // Upload file to H2O
    HttpClient client = new HttpClient();
    PostMethod post = new PostMethod(URL + "/Upload.json?key=" + JSON_FILE.getName());
    Part[] parts = { new FilePart(JSON_FILE.getName(), JSON_FILE) };
    post.setRequestEntity(new MultipartRequestEntity(parts, post.getParams()));
    if( 200 != client.executeMethod(post) )
      throw new RuntimeException("Request failed: " + post.getStatusLine());
    post.releaseConnection();

    // Parse the key into a model
    GetMethod get = new GetMethod(URL + "/2/ImportModel.json?" //
        + "destination_key=MyImportedNeuralNet&" //
        + "type=NeuralNetModel&" //
        + "json=" + JSON_FILE.getName());
    if( 200 != client.executeMethod(get) )
      throw new RuntimeException("Request failed: " + get.getStatusLine());
    get.releaseConnection();
  }
489787
private <RequestT, ResponseT> ResponseT uploadObject(final InputStream originalStream,
                                                         final File file,
                                                         final ObjectMetadata metadata,
                                                         final ProgressListener listener,
                                                         final Request<RequestT> request,
                                                         final S3DataSource originalRequest,
                                                         final boolean skipServerSideValidation,
                                                         final boolean skipClientSideValidationPerRequest,
                                                         final UploadObjectStrategy<RequestT, ResponseT> uploadStrategy,
                                                         final boolean setContentTypeIfNotProvided) {

        InputStream input = getInputStream(originalStream, file, metadata, request,
                                           skipServerSideValidation, setContentTypeIfNotProvided);

        final ObjectMetadata returnedMetadata;
        MD5DigestCalculatingInputStream md5DigestStream = null;
        try {
            if (metadata.getContentMD5() == null && !skipClientSideValidationPerRequest) {
                /*
                 * If the user hasn't set the content MD5, then we don't want to buffer the whole
                 * stream in memory just to calculate it. Instead, we can calculate it on the fly
                 * and validate it with the returned ETag from the object upload.
                 */
                input = md5DigestStream = new MD5DigestCalculatingInputStream(input);
            }

            populateRequestMetadata(request, metadata);
            request.setContent(input);
            publishProgress(listener, ProgressEventType.TRANSFER_STARTED_EVENT);
            try {
                returnedMetadata = uploadStrategy.invokeServiceCall(request);
            } catch (Throwable t) {
                publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
                throw failure(t);
            }
        } finally {
            cleanupDataSource(originalRequest, file, originalStream, input, log);
        }

        String contentMd5 = metadata.getContentMD5();
        if (md5DigestStream != null) {
            contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
        }

        final String etag = returnedMetadata.getETag();
        if (contentMd5 != null && !skipMd5CheckStrategy.skipClientSideValidationPerPutResponse(returnedMetadata)) {
            byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5);
            byte[] serverSideHash = BinaryUtils.fromHex(etag);

            if (!Arrays.equals(clientSideHash, serverSideHash)) {
                publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
                throw new SdkClientException(
                    "Unable to verify integrity of data upload. Client calculated content hash (contentMD5: " + contentMd5
                    + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by Amazon S3.  "
                    + "You may need to delete the data stored in Amazon S3. (metadata.contentMD5: " + metadata.getContentMD5()
                    + ", md5DigestStream: " + md5DigestStream
                    + uploadStrategy.md5ValidationErrorSuffix()
                    + ")");
            }
        }

        publishProgress(listener, ProgressEventType.TRANSFER_COMPLETED_EVENT);

        return uploadStrategy.createResult(returnedMetadata, contentMd5);
    }
1062334
@Override
    public I importFrom(final InputStream stream) throws ArchiveImportException {
        return importFrom(stream, Filters.includeAll());
    }