Prepare HMET data for simulation
165727
def prepare_hmet(self):
        
        if self._prepare_lsm_hmet:
            netcdf_file_path = None
            hmet_ascii_output_folder = None
            if self.output_netcdf:
                netcdf_file_path = '{0}_hmet.nc'.format(self.project_manager.name)
                if self.hotstart_minimal_mode:
                    netcdf_file_path = '{0}_hmet_hotstart.nc'.format(self.project_manager.name)
            else:
                hmet_ascii_output_folder = 'hmet_data_{0}to{1}'
                if self.hotstart_minimal_mode:
                    hmet_ascii_output_folder += "_hotstart"

            self.event_manager.prepare_hmet_lsm(self.lsm_data_var_map_array,
                                                hmet_ascii_output_folder,
                                                netcdf_file_path)
            self.simulation_modified_input_cards += ["HMET_NETCDF",
                                                     "HMET_ASCII"]
        else:
            log.info("HMET preparation skipped due to missing parameters ...")
223220
def sample(config, samples):
  """"""
  url = get_api_path('sample.json')
  multiple_files = []
  images = [s['image'] for s in samples]
  labels = [s['label'] for s in samples]
  for image in images:
    multiple_files.append(('images', (image, open(image, 'rb'), 'image/png')))
  headers=get_headers(no_content_type=True)
  headers["config"]= json.dumps(config, cls=HCEncoder)
  headers["labels"]= json.dumps(labels)
  print("With headers", headers)

  try:
      r = requests.post(url, files=multiple_files, headers=headers, timeout=30)
      return r.text
  except requests.exceptions.RequestException:
      e = sys.exc_info()[0]
      print("Error while calling hyperchamber - ", e)
      return None
701393
def prepare_runtime(self, runtime_dir, data):
        """"""
        # Copy over Python process runtime (resolwe.process).
        import resolwe.process as runtime_package

        src_dir = os.path.dirname(inspect.getsourcefile(runtime_package))
        dest_package_dir = os.path.join(runtime_dir, PYTHON_RUNTIME_DIRNAME, 'resolwe', 'process')
        shutil.copytree(src_dir, dest_package_dir)
        os.chmod(dest_package_dir, 0o755)

        # Write python source file.
        source = data.process.run.get('program', '')
        program_path = os.path.join(runtime_dir, PYTHON_PROGRAM_FILENAME)
        with open(program_path, 'w') as file:
            file.write(source)
        os.chmod(program_path, 0o755)

        # Write serialized inputs.
        inputs = copy.deepcopy(data.input)
        hydrate_input_references(inputs, data.process.input_schema)
        hydrate_input_uploads(inputs, data.process.input_schema)
        inputs_path = os.path.join(runtime_dir, PYTHON_INPUTS_FILENAME)

        # XXX: Skip serialization of LazyStorageJSON. We should support
        # LazyStorageJSON in Python processes on the new communication protocol
        def default(obj):
            """Get default value."""
            class_name = obj.__class__.__name__
            if class_name == 'LazyStorageJSON':
                return ''

            raise TypeError(f'Object of type {class_name} is not JSON serializable')

        with open(inputs_path, 'w') as file:
            json.dump(inputs, file, default=default)

        # Generate volume maps required to expose needed files.
        volume_maps = {
            PYTHON_RUNTIME_DIRNAME: PYTHON_RUNTIME_VOLUME,
            PYTHON_PROGRAM_FILENAME: PYTHON_PROGRAM_VOLUME,
            PYTHON_INPUTS_FILENAME: PYTHON_INPUTS_VOLUME,
        }

        return volume_maps
1220050
def load(self, dtype_out_time, dtype_out_vert=False, region=False,
             plot_units=False, mask_unphysical=False):
        """"""
        msg = ("Loading data from disk for object={0}, dtype_out_time={1}, "
               "dtype_out_vert={2}, and region="
               "{3}".format(self, dtype_out_time, dtype_out_vert, region))
        logging.info(msg + ' ({})'.format(ctime()))
        # Grab from the object if its there.
        try:
            data = self.data_out[dtype_out_time]
        except (AttributeError, KeyError):
            # Otherwise get from disk.  Try scratch first, then archive.
            try:
                data = self._load_from_disk(dtype_out_time, dtype_out_vert,
                                            region=region)
            except IOError:
                data = self._load_from_tar(dtype_out_time, dtype_out_vert)
        # Copy the array to self.data_out for ease of future access.
        self._update_data_out(data, dtype_out_time)
        # Apply desired plotting/cleanup methods.
        if mask_unphysical:
            data = self.var.mask_unphysical(data)
        if plot_units:
            data = self.var.to_plot_units(data, dtype_vert=dtype_out_vert)
        return data
844439
def make_data(n,m):
    """"""
    I = range(1,n+1)
    J = range(1,m+1)
    x,y,w = {},{},{}
    for i in I:
        x[i] = random.randint(0,100)
        y[i] = random.randint(0,100)
        w[i] = random.randint(1,5)
    return I,J,x,y,w
81106
def _initialise_data(self, memmap_dir=None):
        """"""

        if self.number_values == 0:
            pass
        elif self.data_type.nptype is None:
            self._data = []
        else:
            if memmap_dir:
                memmap_file = tempfile.NamedTemporaryFile(
                    mode='w+b', prefix="nptdms_", dir=memmap_dir)
                self._data = np.memmap(
                    memmap_file.file,
                    mode='w+',
                    shape=(self.number_values,),
                    dtype=self.data_type.nptype)
            else:
                self._data = np.zeros(
                    self.number_values, dtype=self.data_type.nptype)
            self._data_insert_position = 0
        if self._data is not None:
            log.debug("Allocated %d sample slots for %s", len(self._data),
                      self.path)
        else:
            log.debug("Allocated no space for %s", self.path)
529886
def preprocess_data():
    

    print("Importing dataset from disk...")
    path = 'enron1/ham/'
    ham1 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
            for f in os.listdir(path) if os.path.isfile(path + f)]
    path = 'enron1/spam/'
    spam1 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
             for f in os.listdir(path) if os.path.isfile(path + f)]
    path = 'enron2/ham/'
    ham2 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
            for f in os.listdir(path) if os.path.isfile(path + f)]
    path = 'enron2/spam/'
    spam2 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
             for f in os.listdir(path) if os.path.isfile(path + f)]

    # Merge and create labels
    emails = ham1 + spam1 + ham2 + spam2
    y = np.array([-1] * len(ham1) + [1] * len(spam1) +
                 [-1] * len(ham2) + [1] * len(spam2))

    # Words count, keep only frequent words
    count_vect = CountVectorizer(decode_error='replace', stop_words='english',
                                 min_df=0.001)
    X = count_vect.fit_transform(emails)

    print('Vocabulary size: %d' % X.shape[1])

    # Shuffle
    perm = np.random.permutation(X.shape[0])
    X, y = X[perm, :], y[perm]

    # Split train and test
    split = 500
    X_train, X_test = X[-split:, :], X[:-split, :]
    y_train, y_test = y[-split:], y[:-split]

    print("Labels in trainset are {:.2f} spam : {:.2f} ham".format(
        np.mean(y_train == 1), np.mean(y_train == -1)))

    return X_train, y_train, X_test, y_test
810582
def prepare_data():
  """"""
  ((x_train, y_train), (x_test, y_test)) = DATASET.load_data()
  x_train = x_train.astype("float32")
  x_test = x_test.astype("float32")
  x_train /= 255.0
  x_test /= 255.0
  return ((x_train, y_train), (x_test, y_test))
1106861
def makebunches(data, commdct):
    """"""
    bunchdt = {}
    ddtt, dtls = data.dt, data.dtls
    for obj_i, key in enumerate(dtls):
        key = key.upper()
        bunchdt[key] = []
        objs = ddtt[key]
        for obj in objs:
            bobj = makeabunch(commdct, obj, obj_i)
            bunchdt[key].append(bobj)
    return bunchdt
999395
def prepare(args):
    
    from jcvi.utils.table import write_csv
    from jcvi.formats.base import write_file
    from jcvi.formats.fastq import guessoffset, readlen

    p = OptionParser(prepare.__doc__ + FastqNamings)
    p.add_option("--corr", default=False, action="store_true",
                 help="Extra parameters for corrected data [default: %default]")
    p.add_option("--norun", default=False, action="store_true",
                 help="Don't write `run.sh` script [default: %default]")
    p.add_option("--ploidy", default="2", choices=("1", "2"),
                 help="Ploidy [default: %default]")
    p.set_cpus()
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    organism_name = args[0]
    project_name = "".join(x[0] for x in organism_name.split()).upper()
    fnames = sorted(glob("*.fastq*") if len(args) == 1 else args[1:])
    for x in fnames:
        assert op.exists(x), "File `{0}` not found.".format(x)

    groupheader = "group_name library_name file_name".split()
    libheader = "library_name project_name organism_name type paired "\
        "frag_size frag_stddev insert_size insert_stddev read_orientation "\
        "genomic_start genomic_end".split()
    groups_33 = []
    groups_64 = []
    libs = []
    for file_name in fnames:
        offset = guessoffset([file_name])
        group_name = op.basename(file_name).split(".")[0]
        library_name = "-".join(group_name.split("-")[:2])

        # Handle paired files and convert to wildcard
        if ".1." in file_name:
            file_name = file_name.replace(".1.", ".?.")
        elif ".2." in file_name:
            continue

        groupscontents = groups_64 if offset == 64 else groups_33
        groupscontents.append((group_name, library_name, file_name))
        if library_name not in libs:
            libs.append(library_name)

    libcontents = []
    for library_name in libs:
        L = Library(library_name)
        size = L.size
        stddev = L.stddev
        type = L.type
        paired = L.paired
        read_orientation = L.read_orientation

        size = size or ""
        stddev = stddev or ""
        frag_size = size if type == "fragment" else ""
        frag_stddev = stddev if type == "fragment" else ""
        insert_size = size if type != "fragment" else ""
        insert_stddev = stddev if type != "fragment" else ""
        genomic_start, genomic_end = "", ""
        libcontents.append((library_name, project_name, organism_name, type, \
            paired, frag_size, frag_stddev, insert_size, insert_stddev, \
            read_orientation, genomic_start, genomic_end))

    for groups, csvfile in ((groups_33, "in_groups_33.csv"), \
                            (groups_64, "in_groups_64.csv"), \
                            (groups_33 + groups_64, "in_groups.csv")):
        if not groups:
            continue
        write_csv(groupheader, groups, filename=csvfile, tee=True)
        logging.debug("`{0}` created (# of groups = {1}).".\
            format(csvfile, len(groups)))

    write_csv(libheader, libcontents, filename="in_libs.csv", tee=True)
    logging.debug("`in_libs.csv` created (# of libs = {0}).".\
        format(len(libcontents)))

    runfile = "run.sh"

    # ALLPATHS stalls on reads over 250bp <https://www.biostars.org/p/122091/>
    max_rd_len = max(readlen([f]) for f in fnames)
    extra = "CLOSE_UNIPATH_GAPS=False " if max_rd_len > 200 else ""
    if opts.corr:
        extra += "FE_NUM_CYCLES=1 EC_K=28 FE_QUAL_CEIL_RADIUS=0"
        extra += " REMOVE_DODGY_READS_FRAG=False FE_MAX_KMER_FREQ_TO_MARK=1"

    if not opts.norun:
        contents = ALLPATHSRUN.format(opts.ploidy, opts.cpus, extra)
        write_file(runfile, contents)
1240585
def prep_bam_inputs(out_dir, sample, call_file, bam_file):
    
    base = utils.splitext_plus(os.path.basename(bam_file))[0]
    with open(call_file) as in_handle:
        for cur_hla in (x.strip() for x in in_handle):
            out_file = os.path.join(utils.safe_makedir(os.path.join(out_dir, base)),
                                    "%s.type.%s.filtered.bam" % (base, cur_hla))
            if not os.path.exists(out_file):
                cmd = ["samtools", "view", "-b","-o", out_file, bam_file, cur_hla]
                subprocess.check_call(cmd)
1227329
def load_data():
    ''''''
    boston = load_boston()
    X_train, X_test, y_train, y_test = train_test_split(boston.data, boston.target, random_state=99, test_size=0.25)
    #normalize data
    ss_X = StandardScaler()
    ss_y = StandardScaler()

    X_train = ss_X.fit_transform(X_train)
    X_test = ss_X.transform(X_test)
    y_train = ss_y.fit_transform(y_train[:, None])[:,0]
    y_test = ss_y.transform(y_test[:, None])[:,0]

    return X_train, X_test, y_train, y_test
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
119644
def prepare(data):
    """"""
    sha = data.get("sha")
    tree = data.get("tree")
    return {"sha": sha, "tree": tree}
357997
def prepare_config(data: dict) -> dict:
        """"""
        data_copy = deepcopy(data)
        # handle single sample cases with 'unknown' phenotype
        if len(data_copy['samples']) == 1:
            if data_copy['samples'][0]['phenotype'] == 'unknown':
                LOG.info("setting 'unknown' phenotype to 'unaffected'")
                data_copy['samples'][0]['phenotype'] = 'unaffected'
        # set the mother/father to '0' if they are not set for a sample
        for sample_data in data_copy['samples']:
            sample_data['mother'] = sample_data.get('mother') or '0'
            sample_data['father'] = sample_data.get('father') or '0'
            if sample_data['analysis_type'] == 'wgs' and sample_data.get('capture_kit') is None:
                sample_data['capture_kit'] = DEFAULT_CAPTURE_KIT
        return data_copy
126603
def _organize_calls(out_file, hla_base, data):
    
    hla_truth = get_hla_truthset(data)
    sample = dd.get_sample_name(data)
    with file_transaction(data, out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            writer = csv.writer(out_handle)
            writer.writerow(["sample", "locus", "mismatches", "options", "alleles", "p-groups", "expected",
                             "validates"])
            for genotype_file in glob.glob("%s.HLA-*.gt" % (hla_base)):
                hla_locus = os.path.basename(genotype_file).replace(
                        "%s.HLA-" % os.path.basename(hla_base), "").replace(".gt", "")
                with open(genotype_file) as in_handle:
                    total_options = set([])
                    for i, line in enumerate(in_handle):
                        _, aone, atwo, m = line.split("\t")[:4]
                        pgroups = (hla_groups.hla_protein(aone, data), hla_groups.hla_protein(atwo, data))
                        if i == 0:
                            call_alleles = [aone, atwo]
                            call_pgroups = pgroups
                            mismatches = m
                        total_options.add(pgroups)
                    if len(total_options) > 0:
                        truth_alleles = tz.get_in([sample, hla_locus], hla_truth, [])
                        writer.writerow([sample, hla_locus, mismatches, len(total_options),
                                         ";".join(call_alleles), ";".join(call_pgroups),
                                         ";".join(truth_alleles), matches_truth(call_alleles, truth_alleles, data)])
    return out_file
199308
def sample_dynamic_data(params):
    
    model_key = os.path.basename(params['model']).replace('.txt', '')
    if 'writedir' not in params or params['writedir'] is None:
        params['writedir'] = settings.writedir + model_key + '_sim'
    if not os.path.exists(params['writedir']): os.makedirs(params['writedir'])
    readwrite.write_params(params['writedir'] + '/params.txt', params)
    # init variables
    writedir = params['writedir']
    tmax = params['tmax']
    branching = params['branching']
    noiseObs = params['noiseObs']
    noiseDyn = params['noiseDyn']
    nrRealizations = params['nrRealizations']
    step = params['step']  # step size for saving the figure

    nrSamples = 1  # how many files?
    maxRestarts = 1000
    maxNrSamples = 1

    # simple vector auto regressive process or
    # hill kinetics process simulation
    if 'krumsiek11' not in model_key:
        # create instance, set seed
        grnsim = GRNsim(model=model_key, params=params)
        nrOffEdges_list = np.zeros(nrSamples)
        for sample in range(nrSamples):
            # random topology / for a given edge density
            if 'hill' not in model_key:
                Coupl = np.array(grnsim.Coupl)
                for sampleCoupl in range(10):
                    nrOffEdges = 0
                    for gp in range(grnsim.dim):
                        for g in range(grnsim.dim):
                            # only consider off-diagonal edges
                            if g != gp:
                                Coupl[gp, g] = 0.7 if np.random.rand() < 0.4 else 0
                                nrOffEdges += 1 if Coupl[gp, g] > 0 else 0
                            else:
                                Coupl[gp, g] = 0.7
                    # check that the coupling matrix does not have eigenvalues
                    # greater than 1, which would lead to an exploding var process
                    if max(sp.linalg.eig(Coupl)[0]) < 1:
                        break
                nrOffEdges_list[sample] = nrOffEdges
                grnsim.set_coupl(Coupl)
            # init type
            real = 0
            X0 = np.random.rand(grnsim.dim)
            Xsamples = []
            for restart in range(nrRealizations+maxRestarts):
                # slightly break symmetry in initial conditions
                if 'toggleswitch' in model_key:
                    X0 = (np.array([0.8 for i in range(grnsim.dim)])
                          + 0.01*np.random.randn(grnsim.dim))
                X = grnsim.sim_model(tmax=tmax, X0=X0,
                                     noiseDyn=noiseDyn)
                # check branching
                check = True
                if branching:
                    check, Xsamples = _check_branching(X, Xsamples, restart)
                if check:
                    real += 1
                    grnsim.write_data(X[::step], dir=writedir,
                                      noiseObs=noiseObs,
                                      append=(False if restart==0 else True),
                                      branching=branching,
                                      nrRealizations=nrRealizations)
                # append some zeros
                if 'zeros' in writedir and real == 2:
                    grnsim.write_data(noiseDyn*np.random.randn(500,3), dir=writedir,
                                      noiseObs=noiseObs,
                                      append=(False if restart==0 else True),
                                      branching=branching,
                                      nrRealizations=nrRealizations)
                if real >= nrRealizations:
                    break
        if False:
            logg.info('mean nr of offdiagonal edges',nrOffEdges_list.mean(),
                      'compared to total nr',grnsim.dim*(grnsim.dim-1)/2.)

    # more complex models
    else:
        initType = 'random'

        dim = 11
        step = 5

        grnsim = GRNsim(dim=dim,initType=initType,model=model_key,params=params)
        curr_nrSamples = 0
        Xsamples = []
        for sample in range(maxNrSamples):
            # choose initial conditions such that branchings result
            if initType == 'branch':
                X0mean = grnsim.branch_init_model1(tmax)
                if X0mean is None:
                    grnsim.set_coupl()
                    continue
            real = 0
            for restart in range(nrRealizations+maxRestarts):
                if initType == 'branch':
                    # vary initial conditions around mean
                    X0 = X0mean + (0.05*np.random.rand(dim) - 0.025*np.ones(dim))
                else:
                    # generate random initial conditions within [0.3,0.7]
                    X0 = 0.4*np.random.rand(dim)+0.3
                if model_key in [5,6]:
                    X0 = np.array([0.3,0.3,0,0,0,0])
                if model_key in [7,8,9,10]:
                    X0 = 0.6*np.random.rand(dim)+0.2
                    X0[2:] = np.zeros(4)
                if 'krumsiek11' in model_key:
                    X0 = np.zeros(dim)
                    X0[grnsim.varNames['Gata2']] = 0.8
                    X0[grnsim.varNames['Pu.1']] = 0.8
                    X0[grnsim.varNames['Cebpa']] = 0.8
                    X0 += 0.001*np.random.randn(dim)
                    if False:
                        switch_gene = restart - (nrRealizations - dim)
                        if switch_gene >= dim:
                            break
                        X0[switch_gene] = 0 if X0[switch_gene] > 0.1 else 0.8
                X = grnsim.sim_model(tmax,X0=X0,
                                     noiseDyn=noiseDyn,
                                     restart=restart)
                # check branching
                check = True
                if branching:
                    check, Xsamples = _check_branching(X,Xsamples,restart)
                if check:
                    real += 1
                    grnsim.write_data(X[::step],dir=writedir,noiseObs=noiseObs,
                                     append=(False if restart==0 else True),
                                     branching=branching,
                                     nrRealizations=nrRealizations)
                if real >= nrRealizations:
                    break
    import glob
    # load the last simulation file
    filename = glob.glob(writedir + '/sim*.txt')[-1]
    logg.info('reading simulation results', filename)
    adata = readwrite._read(filename, first_column_names=True,
                            suppress_cache_warning=True)
    adata.uns['tmax_write'] = tmax/step
    return adata