Downloads a sample from CGHub via GeneTorrent, then uses S3AM to transfer it to S3

    input_args: dict        Dictionary of input arguments
    analysis_id: str        An analysis ID for a sample in CGHub
505685
def download_and_transfer_sample(job, sample, inputs):

    
    analysis_id = sample[0]
    work_dir = job.fileStore.getLocalTempDir()
    folder_path = os.path.join(work_dir, os.path.basename(analysis_id))
    # Acquire genetorrent key and download sample
    shutil.copy(inputs['genetorrent_key'], os.path.join(work_dir, 'cghub.key'))
    parameters = ['-vv', '-c', 'cghub.key', '-d', analysis_id]
    docker_call(job=job, tool='quay.io/ucsc_cgl/genetorrent:3.8.7--9911761265b6f08bc3ef09f53af05f56848d805b',
                work_dir=work_dir, parameters=parameters)
    try:
        sample = glob.glob(os.path.join(folder_path, '*tar*'))[0]
    except KeyError as e:
        print 'No tarfile found inside of folder: '.format(e)
        raise
    # Upload sample to S3AM
    key_path = inputs['ssec']
    if sample.endswith('gz'):
        sample_name = analysis_id + '.tar.gz'
        shutil.move(sample, os.path.join(work_dir, sample_name))
    else:
        sample_name = analysis_id + '.tar'
        shutil.move(sample, os.path.join(work_dir, sample_name))
    # Parse s3_dir to get bucket and s3 path
    s3_dir = inputs['s3_dir']
    bucket_name = s3_dir.lstrip('/').split('/')[0]
    base_url = 'https://s3-us-west-2.amazonaws.com/'
    url = os.path.join(base_url, bucket_name, sample_name)
    # Generate keyfile for upload
    with open(os.path.join(work_dir, 'temp.key'), 'wb') as f_out:
        f_out.write(generate_unique_key(key_path, url))
    # Upload to S3 via S3AM
    s3am_command = ['s3am',
                    'upload',
                    '--sse-key-file', os.path.join(work_dir, 'temp.key'),
                    'file://{}'.format(os.path.join(work_dir, sample_name)),
                    's3://' + bucket_name + '/']
    subprocess.check_call(s3am_command)
628800
def download_and_transfer_sample(job, input_args, samples):
    
    if len(samples) > 1:
        a = samples[len(samples)/2:]
        b = samples[:len(samples)/2]
        job.addChildJobFn(download_and_transfer_sample, input_args, a, disk='30G')
        job.addChildJobFn(download_and_transfer_sample, input_args, b, disk='30G')
    else:
        analysis_id = samples[0]
        work_dir = job.fileStore.getLocalTempDir()
        sudo = input_args['sudo']
        # Acquire dbgap_key
        shutil.copy(input_args['dbgap_key'], os.path.join(work_dir, 'dbgap.ngc'))
        # Call to fastq-dump to pull down SRA files and convert to fastq
        if input_args['single_end']:
            parameters = [analysis_id]
        else:
            parameters = ['--split-files', analysis_id]
        docker_call(tool='quay.io/ucsc_cgl/fastq-dump:2.5.7--4577a6c1a3c94adaa0c25dd6c03518ee610433d1',
                    work_dir=work_dir, tool_parameters=parameters, sudo=sudo)
        # Collect files and encapsulate into a tarball
        shutil.rmtree(os.path.join(work_dir, 'sra'))
        sample_name = analysis_id + '.tar.gz'
        if input_args['single_end']:
            r = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*.f*'))]
            tarball_files(work_dir, tar_name=sample_name, files=r)
        else:
            r1 = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*_1*'))]
            r2 = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*_2*'))]
            tarball_files(work_dir, tar_name=sample_name, files=r1 + r2)
        # Parse s3_dir to get bucket and s3 path
        key_path = input_args['ssec']
        s3_dir = input_args['s3_dir']
        bucket_name = s3_dir.lstrip('/').split('/')[0]
        base_url = 'https://s3-us-west-2.amazonaws.com/'
        url = os.path.join(base_url, bucket_name, sample_name)
        # Generate keyfile for upload
        with open(os.path.join(work_dir, 'temp.key'), 'wb') as f_out:
            f_out.write(generate_unique_key(key_path, url))
        # Upload to S3 via S3AM
        s3am_command = ['s3am',
                        'upload',
                        '--sse-key-file', os.path.join(work_dir, 'temp.key'),
                        'file://{}'.format(os.path.join(work_dir, sample_name)),
                        's3://' + bucket_name + '/']
        subprocess.check_call(s3am_command)
584414
def analyze(self, handle, filename):
        
        # multipart post files.
        files = {"file" : (filename, handle)}

        # ensure the handle is at offset 0.
        handle.seek(0)

        response = self._request("/submit/file", method='POST', files=files)

        try:
            if response.status_code == 201:
                # good response
                return response.json()['job_id']
            else:
                raise sandboxapi.SandboxError("api error in analyze: {r}".format(r=response.content.decode('utf-8')))
        except (ValueError, KeyError) as e:
            raise sandboxapi.SandboxError("error in analyze: {e}".format(e=e))
338405
function analyze (fileSourceData) {
    var analysis = new Analysis();
        
    for (let sample of fileSourceData) {
        if (saysHelloWorld(sample.text)) {
            analysis.addError({
                line: sample.line,
                message: 'At least try to look like you didn\'t copy the demo code!'
            });
        }
    }

    return analysis;
}
1086638
def runGetBiosample(self, id_):
        
        compoundId = datamodel.BiosampleCompoundId.parse(id_)
        dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
        biosample = dataset.getBiosample(id_)
        return self.runGetRequest(biosample)
1069420
def EnableFreeAPIKeyRateLimit(self):
    
    self._analyzer.hashes_per_batch = 4
    self._analyzer.wait_after_analysis = 60
    self._analysis_queue_timeout = self._analyzer.wait_after_analysis + 1
679160
def run_analysis(args):
  
  import google.datalab.bigquery as bq
  if args.bigquery_table:
    table = bq.Table(args.bigquery_table)
    schema_list = table.schema._bq_schema
  else:
    schema_list = json.loads(
        file_io.read_file_to_string(args.schema_file).decode())
    table = bq.ExternalDataSource(
        source=args.input_file_pattern,
        schema=bq.Schema(schema_list))

  # Check the schema is supported.
  for col_schema in schema_list:
    col_type = col_schema['type'].lower()
    if col_type != 'string' and col_type != 'integer' and col_type != 'float':
      raise ValueError('Schema contains an unsupported type %s.' % col_type)

  run_numerical_analysis(table, schema_list, args)
  run_categorical_analysis(table, schema_list, args)

  # Save a copy of the schema to the output location.
  file_io.write_string_to_file(
      os.path.join(args.output_dir, SCHEMA_FILE),
      json.dumps(schema_list, indent=2, separators=(',', ': ')))
375717
def do_action_to_ancestors(analysis_request, transition_id):
    
    parent_ar = analysis_request.getParentAnalysisRequest()
    if parent_ar:
        do_action_for(parent_ar, transition_id)
645036
def add_analysis(self, analysis):
        
        analysis_object = api.get_object(analysis)
        result = analysis_object.getResult()
        results_range = analysis_object.getResultsRange()
        range_result = results_range.get('result', None)
        range_min = results_range.get('min', None)
        range_max = results_range.get('max', None)
        # All them must be floatable
        for value in [result, range_result, range_min, range_max]:
            if not api.is_floatable(value):
                return
        cap_date = analysis_object.getResultCaptureDate()
        cap_date = api.is_date(cap_date) and \
                   cap_date.strftime('%Y-%m-%d %I:%M %p') or ''
        if not cap_date:
            return

        # Create json
        ref_sample_id = analysis_object.getSample().getId()
        as_keyword = analysis_object.getKeyword()
        as_name = analysis_object.Title()
        as_ref = '{} ({})'.format(as_name, as_keyword)
        as_rows = self.analyses_dict.get(as_ref, {})
        an_rows = as_rows.get(ref_sample_id, [])
        an_rows.append({
            'date': cap_date,
            'target': api.to_float(range_result),
            'upper': api.to_float(range_max),
            'lower': api.to_float(range_min),
            'result': api.to_float(result),
            'unit': analysis_object.getUnit(),
            'id': api.get_uid(analysis_object)
        })
        as_rows[ref_sample_id] = an_rows
        self.analyses_dict[as_ref] = as_rows
189644
def reindex_request(analysis, idxs=None):
    
    if not IRequestAnalysis.providedBy(analysis) or \
            IDuplicateAnalysis.providedBy(analysis):
        # Analysis not directly bound to an Analysis Request. Do nothing
        return

    n_idxs = ['assigned_state', 'getDueDate']
    n_idxs = idxs and list(set(idxs + n_idxs)) or n_idxs
    request = analysis.getRequest()
    ancestors = [request] + request.getAncestors(all_ancestors=True)
    for ancestor in ancestors:
        push_reindex_to_actions_pool(ancestor, n_idxs)
1241072
def run_analysis(self, argv):
        """"""
        args = self._parser.parse_args(argv)

        if not HAVE_ST:
            raise RuntimeError(
                "Trying to run fermipy analysis, but don't have ST")

        if is_null(args.skydirs):
            skydir_dict = None
        else:
            skydir_dict = load_yaml(args.skydirs)

        gta = GTAnalysis(args.config,
                         logging={'verbosity': 3},
                         fileio={'workdir_regex': '\.xml$|\.npy$'})
        #gta.setup(overwrite=False)
        gta.load_roi(args.roi_baseline)
        gta.print_roi()

        basedir = os.path.dirname(args.config)
        # This should be a no-op, b/c it was done in the baseline analysis

        for profile in args.profiles:
            if skydir_dict is None:
                skydir_keys = [None]
            else:
                skydir_keys = sorted(skydir_dict.keys())

            for skydir_key in skydir_keys:
                if skydir_key is None:
                    pkey, psrc_name, pdict = build_profile_dict(basedir, profile)
                else:
                    skydir_val = skydir_dict[skydir_key]
                    pkey, psrc_name, pdict = build_profile_dict(basedir, profile)
                    pdict['ra'] = skydir_val['ra']
                    pdict['dec'] = skydir_val['dec']
                    pkey += "_%06i" % skydir_key

                outfile = "sed_%s.fits" % pkey

                # Add the source and get the list of correlated soruces
                correl_dict, test_src_name = add_source_get_correlated(gta, psrc_name, 
                                                                       pdict, correl_thresh=0.25, 
                                                                       non_null_src=args.non_null_src)

                # Write the list of correlated sources
                correl_yaml = os.path.join(basedir, "correl_%s.yaml" % pkey)
                write_yaml(correl_dict, correl_yaml)

                gta.free_sources(False)
                for src_name in correl_dict.keys():
                    gta.free_source(src_name, pars='norm')

                # build the SED
                if args.non_null_src:
                    gta.update_source(test_src_name, reoptimize=True)
                    gta.write_roi("base_%s"% pkey, make_plots=False)
                gta.sed(test_src_name, prefix=pkey, outfile=outfile, make_plots=args.make_plots)

                # remove the source
                gta.delete_source(test_src_name)
                # put the ROI back to how it was
                gta.load_xml(args.roi_baseline)

        return gta
878630
def get_experiment_fn(args):
  

  def get_experiment(output_dir):
    # Merge schema, input features, and transforms.
    train_config = util.merge_metadata(args.preprocess_output_dir,
                                       args.transforms_file)

    # Get the model to train.
    estimator = util.get_estimator(output_dir, train_config, args)

    # Save a copy of the scehma and input to the model folder.
    schema_file = os.path.join(args.preprocess_output_dir, util.SCHEMA_FILE)

    # Make list of files to save with the trained model.
    additional_assets = {'features.json': args.transforms_file,
                         util.SCHEMA_FILE: schema_file}
    if util.is_classification_model(args.model_type):
      target_name = train_config['target_column']
      vocab_file_name = util.CATEGORICAL_ANALYSIS % target_name
      vocab_file_path = os.path.join(
          args.preprocess_output_dir, vocab_file_name)
      assert file_io.file_exists(vocab_file_path)
      additional_assets[vocab_file_name] = vocab_file_path

    export_strategy_target = util.make_export_strategy(
        train_config=train_config,
        args=args,
        keep_target=True,
        assets_extra=additional_assets)
    export_strategy_notarget = util.make_export_strategy(
        train_config=train_config,
        args=args,
        keep_target=False,
        assets_extra=additional_assets)

    input_reader_for_train = get_reader_input_fn(
        train_config=train_config,
        preprocess_output_dir=args.preprocess_output_dir,
        model_type=args.model_type,
        data_paths=args.train_data_paths,
        batch_size=args.train_batch_size,
        shuffle=True,
        num_epochs=args.num_epochs)

    input_reader_for_eval = get_reader_input_fn(
        train_config=train_config,
        preprocess_output_dir=args.preprocess_output_dir,
        model_type=args.model_type,
        data_paths=args.eval_data_paths,
        batch_size=args.eval_batch_size,
        shuffle=False,
        num_epochs=1)

    return tf.contrib.learn.Experiment(
        estimator=estimator,
        train_input_fn=input_reader_for_train,
        eval_input_fn=input_reader_for_eval,
        train_steps=args.max_steps,
        export_strategies=[export_strategy_target, export_strategy_notarget],
        min_eval_frequency=args.min_eval_frequency,
        eval_steps=None,
    )

  # Return a function to create an Experiment.
  return get_experiment
567632
def analysis_set_properties(object_id, input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/%s/setProperties' % object_id, input_params, always_retry=always_retry, **kwargs)
360768
def create_analysis(name, kernel, src_dir, scaffold_name):
    """"""

    # analysis folder
    folder = os.path.join(os.getcwd(), 'analyses', name)
    if not os.path.exists(folder):
        os.makedirs(folder)
    else:
        log.warning('Analysis folder {} already exists.'.format(folder))

    # copy all other files
    for f in os.listdir(src_dir):
        if f in ('__pycache__',) or \
           any(f.endswith(ending) for ending in ('.pyc',)):
            continue
        copy_scaffold_file(os.path.join(src_dir, f),
                           os.path.join(folder, f),
                           name, scaffold_name)
377252
def save_analysis(self, filepath=None, **kwargs):
        
        # We pass a copy of the analysis attribute dictionary.
        dict_obj = deepcopy(self.analysis_output)
        # If no filepath is provided we create one.
        if filepath is None:
            filepath = "_".join(
                (str(self.system_id), "pywindow_analysis")
            )
            filepath = '/'.join((os.getcwd(), filepath))
        # Dump the dictionary to json file.
        Output().dump2json(dict_obj, filepath, default=to_list, **kwargs)
        return
820702
def analysis_describe(object_id, input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/%s/describe' % object_id, input_params, always_retry=always_retry, **kwargs)
642330
def copy_analysis_dict( analysis ):
    '''  '''
    assert isinstance(analysis, dict), "(!) Input 'analysis' should be a dict!"
    new_dict = { POSTAG: analysis[POSTAG],\
                 ROOT: analysis[ROOT],\
                 FORM: analysis[FORM],\
                 CLITIC: analysis[CLITIC],\
                 ENDING: analysis[ENDING] }
    if LEMMA in analysis:
       new_dict[LEMMA] = analysis[LEMMA]
    if ROOT_TOKENS in analysis:
       new_dict[ROOT_TOKENS] = analysis[ROOT_TOKENS]
    return new_dict