:param job_homedir: explicit value for home directory, used for testing purposes
    :rtype: string
    :returns: path to output directory

    Returns the output directory, where all outputs are created, and
    uploaded from
11665
def get_output_dir(job_homedir=None):
    '''
    
    '''
    if job_homedir is not None:
        home_dir = job_homedir
    else:
        home_dir = environ.get('HOME')
    odir = os.path.join(home_dir, 'out')
    return odir
676790
def job_runner(self):
        # We recommend that you define a subclass, override this method and set up your own config
        
        outputs = luigi.task.flatten(self.output())
        for output in outputs:
            if not isinstance(output, luigi.contrib.hdfs.HdfsTarget):
                warnings.warn("Job is using one or more non-HdfsTarget outputs" +
                              " so it will be run in local mode")
                return LocalJobRunner()
        else:
            return DefaultHadoopJobRunner()
725186
def _dump(self, out_dir=''):
        """"""
        with self.no_unpicklable_properties():
            self.job_file = os.path.join(out_dir, 'job-instance.pickle')
            if self.__module__ == '__main__':
                d = pickle.dumps(self)
                module_name = os.path.basename(sys.argv[0]).rsplit('.', 1)[0]
                d = d.replace('(c__main__', "(c" + module_name)
                open(self.job_file, "w").write(d)
            else:
                pickle.dump(self, open(self.job_file, "w"))
142248
def get_default_home_dir():
    
    ding0_dir = str(cfg_ding0.get('config',
                                  'config_dir'))
    return os.path.join(os.path.expanduser('~'), ding0_dir)
660537
def set_up_dirs(proc_name, output_dir=None, work_dir=None, log_dir=None):
    
    output_dir = safe_mkdir(adjust_path(output_dir or join(os.getcwd(), proc_name)), 'output_dir')
    debug('Saving results into ' + output_dir)

    work_dir = safe_mkdir(work_dir or join(output_dir, 'work'), 'working directory')
    info('Using work directory ' + work_dir)

    log_fpath = set_up_log(log_dir or safe_mkdir(join(work_dir, 'log')), proc_name + '.log')

    return output_dir, work_dir, log_fpath
1085997
def get_job_model(self):
        
        if not self.job:
            raise Exception('Job not loaded yet. Use load(id) first.')

        return JobModel(self.job_id, self.job, self.home_config['storage_dir'])
1071892
def set_workdir(self, workdir, chroot=False):
        
        if not chroot and hasattr(self, "workdir") and self.workdir != workdir:
            raise ValueError("self.workdir != workdir: %s, %s" % (self.workdir,  workdir))

        # Directories with (input|output|temporary) data.
        self.workdir = os.path.abspath(workdir)
        self.indir = Directory(os.path.join(self.workdir, "indata"))
        self.outdir = Directory(os.path.join(self.workdir, "outdata"))
        self.tmpdir = Directory(os.path.join(self.workdir, "tmpdata"))
        self.wdir = Directory(self.workdir)
452202
private void markOutputDirSuccessful(JobContext context)
  throws IOException {
    if (outputPath != null) {
      FileSystem fileSys = outputPath.getFileSystem(context.getConfiguration());
      if (fileSys.exists(outputPath)) {
        // create a file in the folder to mark it
        Path filePath = new Path(outputPath, SUCCEEDED_FILE_NAME);
        fileSys.create(filePath).close();
      }
    }
  }
792239
def collect_results(working_folder):
    
    results = {} 
    working_folder = Path(working_folder)
    output_folders = working_folder/'output_folders'

    for rec_name in os.listdir(output_folders):
        if not os.path.isdir(output_folders / rec_name):
            continue
        # print(rec_name)
        results[rec_name] = {}
        for sorter_name in os.listdir(output_folders / rec_name):
            # print('  ', sorter_name)
            output_folder = output_folders / rec_name / sorter_name
            #~ print(output_folder)
            if not os.path.isdir(output_folder):
                continue
            SorterClass = sorter_dict[sorter_name]
            results[rec_name][sorter_name] = SorterClass.get_result_from_folder(output_folder)

    return results
804663
def read_job(self, job_id, checkout=False):
        
        self.job_id = job_id

        commit = self.get_head_commit()
        self.logger.debug('Job ref points to ' + commit)
        self.command_exec(['read-tree', self.ref_head])

        if checkout:
            self.logger.debug('Working directory in ' + self.work_tree)

            # make sure we have checked out all files we have added until now. Important for simple models,
            # so we have the actual model.py and dataset scripts.
            if os.path.exists(self.work_tree):
                shutil.rmtree(self.work_tree)

            os.makedirs(self.work_tree)

            # make the working tree reflect exactly the tree of ref_head.
            # since we removed the dir before, we have exactly the tree of the reference
            # '--', '.' is important to not update HEAD
            self.command_exec(['--work-tree', self.work_tree, 'checkout', self.ref_head, '--', '.'])
1159851
def dump_all_outputs(self, job, target, abspaths=None):
        
        import os
        import shutil
        from pathlib import Path

        root = Path(native_str(target))
        true_outputs = job.get_output()

        if abspaths or len(true_outputs) < self.BULK_OUTPUT_FILE_THRESHOLD:
            return super().dump_all_outputs(job, root, abspaths)

        stagingdir = root / Path(native_str(job.workingdir)).name
        workdir = job.get_directory(job.workingdir)
        if not root.is_dir():
            root.mkdir(parents=False)
        if stagingdir.exists():
            if PY2:
                raise IOError('Path % exists' % stagingdir)
            else:
                raise FileExistsError(stagingdir)
        workdir.put(str(root))
        assert stagingdir.is_dir()
        assert root in stagingdir.parents

        for pathstr in true_outputs:
            if os.path.isabs(pathstr):
                continue
            destpath = root / pathstr
            currpath = stagingdir / pathstr
            if not destpath.parent.is_dir():
                destpath.parent.mkdir(parents=True)
            currpath.rename(destpath)

        shutil.rmtree(str(stagingdir))
21040
def job_file(self):
        
        job_file_name = '%s.job' % (self.name)
        job_file_path = os.path.join(self.initial_dir, job_file_name)
        self._job_file = job_file_path
        return self._job_file
686867
def mkdir(self, src, extra_args=[]):
        ''''''
        return self._getStdOutCmd([self._hadoop_cmd, 'fs', '-mkdir'] + extra_args + [self._full_hdfs_path(src)], True)
1194169
def validate_save_dir(self):
        
        # checking if working directory exists
        print 'Working dir: {} '.format(self.workdir)
        if not os.path.isdir(self.workdir):
            print 'Creating working directory {}'.format(self.workdir)
            os.mkdir(self.workdir)
        # actual target directory for VM
        target_directory = os.path.join(self.workdir, self.vm_obj.config.instanceUuid)
        print 'Target dir: {}'.format(target_directory)
        if not os.path.isdir(target_directory):
            print 'Creating target dir {}'.format(target_directory)
            os.mkdir(target_directory)
        return target_directory
157506
def check_for_new_files_in_home(self):
        
        if not self._user:
            return None
        try:
            created_files = set(self._listdir(self._home_dir)).difference(self._home_dir_content)
        except (subprocess.CalledProcessError, IOError):
            # Probably home directory does not exist
            created_files = []
        if created_files:
            logging.warning('The tool created the following files in %s, '
                            'this may influence later runs:\n\t%s',
                            self._home_dir, '\n\t'.join(created_files))
        return created_files
1081708
def get_output_dir(self, nb):
        """"""
        self.package_dir, self.package_name = self.get_package_dir_name(nb)

        return join(self.package_dir, self.package_name)
1223741
def _createWorkingDir(rootdir,input):
    
    # extract rootname from input
    rootname = input[:input.find('_')]
    newdir = os.path.join(rootdir,rootname)
    if not os.path.exists(newdir):
        os.mkdir(newdir)
    return newdir