Create objects to store data and metadata for each sample. Also, perform necessary file manipulations
764327
def objectprep(self):
        """"""
        # Move the files to subfolders and create objects
        self.runmetadata = createobject.ObjectCreation(self)
        if self.extension == 'fastq':
            # To streamline the CLARK process, decompress and combine .gz and paired end files as required
            logging.info('Decompressing and combining .fastq files for CLARK analysis')
            fileprep.Fileprep(self)
        else:
            logging.info('Using .fasta files for CLARK analysis')
            for sample in self.runmetadata.samples:
                sample.general.combined = sample.general.fastqfiles[0]
944214
def _add_versions(samples):
    
    samples[0]["versions"] = {"tools": programs.write_versions(samples[0]["dirs"], samples[0]["config"]),
                              "data": provenancedata.write_versions(samples[0]["dirs"], samples)}
    return samples
437728
def populate_schema_objects(self, schema, obj_type):
        """"""
        metadata = self.dbmetadata[obj_type]
        schema = schema or self.dbname

        try:
            objects = metadata[schema].keys()
        except KeyError:
            # schema doesn't exist
            objects = []

        return objects
460500
def command_create_tables(self, meta_name=None, verbose=False):
        '''
        
        '''

        def _create_metadata_tables(metadata):
            for table in metadata.sorted_tables:
                if verbose:
                    print(self._schema(table))
                else:
                    print('  '+table.name)
                engine = self.session.get_bind(clause=table)
                metadata.create_all(bind=engine, tables=[table])

        if isinstance(self.metadata, MetaData):
            print('Creating tables...')
            _create_metadata_tables(self.metadata)
        else:
            for current_meta_name, metadata in self.metadata.items():
                if meta_name not in (current_meta_name, None):
                    continue
                print('Creating tables for {}...'.format(current_meta_name))
                _create_metadata_tables(metadata)
444253
def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=32, val_bs:int=None, pad_idx=1,
               pad_first=True, device:torch.device=None, no_check:bool=False, backwards:bool=False, **dl_kwargs) -> DataBunch:
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        collate_fn = partial(pad_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)
        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs)
        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)
        dataloaders = [train_dl]
        for ds in datasets[1:]:
            lengths = [len(t) for t in ds.x.items]
            sampler = SortSampler(ds.x, key=lengths.__getitem__)
            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))
        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)
1008266
def _generate_examples(self, archive_paths, objects_getter, bboxes_getter,
                         prefixes=None):
    """"""
    trainable_classes = set(
        self.info.features['objects_trainable']['label'].names)
    for i, archive_path in enumerate(archive_paths):
      prefix = prefixes[i] if prefixes else None
      objects = objects_getter(prefix)
      bboxes = bboxes_getter(prefix)
      logging.info('Opening archive %s ...', archive_path)
      archive = tfds.download.iter_archive(
          archive_path, tfds.download.ExtractMethod.TAR_STREAM)
      for fpath, fobj in archive:
        fname = os.path.basename(fpath)
        image_id = int(os.path.splitext(fname)[0], 16)
        image_objects = [obj._asdict() for obj in objects.get(image_id, [])]
        image_bboxes = [bbox._asdict() for bbox in bboxes.get(image_id, [])]
        image_objects_trainable = [
            obj for obj in image_objects if obj['label'] in trainable_classes
        ]
        yield {
            'image': _resize_image_if_necessary(
                fobj, target_pixels=self.builder_config.target_pixels),
            'image/filename': fname,
            'objects': image_objects,
            'objects_trainable': image_objects_trainable,
            'bobjects': image_bboxes,
        }
1371766
def skeleton(files, metadata, sqlite_extensions):
    ""
    if os.path.exists(metadata):
        click.secho(
            "File {} already exists, will not over-write".format(metadata),
            bg="red",
            fg="white",
            bold=True,
            err=True,
        )
        sys.exit(1)
    app = Datasette(files, sqlite_extensions=sqlite_extensions)
    databases = {}
    for database_name, info in app.inspect().items():
        databases[database_name] = {
            "title": None,
            "description": None,
            "description_html": None,
            "license": None,
            "license_url": None,
            "source": None,
            "source_url": None,
            "queries": {},
            "tables": {
                table_name: {
                    "title": None,
                    "description": None,
                    "description_html": None,
                    "license": None,
                    "license_url": None,
                    "source": None,
                    "source_url": None,
                    "units": {},
                }
                for table_name in (info.get("tables") or {})
            },
        }
    open(metadata, "w").write(
        json.dumps(
            {
                "title": None,
                "description": None,
                "description_html": None,
                "license": None,
                "license_url": None,
                "source": None,
                "source_url": None,
                "databases": databases,
            },
            indent=4,
        )
    )
    click.echo("Wrote skeleton to {}".format(metadata))
813204
def write_metadata(self, fp):
        
        fp.attrs['model'] = self.name
        fp.attrs['variable_params'] = list(self.variable_params)
        fp.attrs['sampling_params'] = list(self.sampling_params)
        fp.write_kwargs_to_attrs(fp.attrs, static_params=self.static_params)
1202143
def create_data_and_metadata_from_data(self, data: numpy.ndarray, intensity_calibration: CalibrationModule.Calibration=None, dimensional_calibrations: typing.List[CalibrationModule.Calibration]=None, metadata: dict=None, timestamp: str=None) -> DataAndMetadata.DataAndMetadata:
        
        return self.create_data_and_metadata(numpy.copy(data), intensity_calibration, dimensional_calibrations, metadata, timestamp)
372023
def save_sampleset(self, f, name):
        ''''''
        f.create_dataset(name + '_encoder_x', data=self.encoder_x)
        f.create_dataset(name + '_decoder_x', data=self.decoder_x)
        f.create_dataset(name + '_decoder_y', data=self.decoder_y)
299481
def write_sampler_metadata(self, sampler):
        
        super(EmceePTFile, self).write_sampler_metadata(sampler)
        self[self.sampler_group].attrs["betas"] = sampler.betas
880272
def main(self):
        
        logging.info('Preparing metadata')
        # If this script is run as part of a pipeline, the metadata objects will already exist
        if not self.metadata:
            self.filer()
        else:
            self.objectprep()
        # Use the number of metadata objects to calculate the number of cores to use per sample in multi-threaded
        # methods with sequence calls to multi-threaded applications
        try:
            self.threads = int(self.cpus / len(self.metadata)) if self.cpus / len(
                self.metadata) > 1 else 1
        except (TypeError, ZeroDivisionError):
            self.threads = self.cpus
        logging.info('Reading and formatting primers')
        self.primers()
        logging.info('Baiting .fastq files against primers')
        self.bait()
        logging.info('Baiting .fastq files against previously baited .fastq files')
        self.doublebait()
        logging.info('Assembling contigs from double-baited .fastq files')
        self.assemble_amplicon_spades()
        logging.info('Creating BLAST database')
        self.make_blastdb()
        logging.info('Running BLAST analyses')
        self.blastnthreads()
        logging.info('Parsing BLAST results')
        self.parseblast()
        logging.info('Clearing amplicon files from previous iterations')
        self.ampliconclear()
        logging.info('Creating reports')
        self.reporter()
131604
def write_metadata(self, fp):
        
        super(BaseDataModel, self).write_metadata(fp)
        fp.write_stilde(self.data)
161222
def populate_metadata(model, MetadataClass):
    
    content_type = ContentType.objects.get_for_model(model)
    for instance in model.objects.all():
        create_metadata_instance(MetadataClass, instance)
528458
def makeblastdb(self):
        
        # Iterate through the samples to set the bait file.
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                # Remove the file extension
                db = os.path.splitext(sample[self.analysistype].baitfile)[0]
                # Add '.nhr' for searching below
                nhr = '{}.nhr'.format(db)
                # Check for already existing database files
                if not os.path.isfile(str(nhr)):
                    # Create the databases
                    command = 'makeblastdb -in {} -parse_seqids -max_file_sz 2GB -dbtype nucl -out {}'\
                        .format(sample[self.analysistype].baitfile, db)
                    out, err = run_subprocess(command)
                    write_to_logfile(command,
                                     command,
                                     self.logfile, sample.general.logout, sample.general.logerr,
                                     sample[self.analysistype].logout, sample[self.analysistype].logerr)
                    write_to_logfile(out,
                                     err,
                                     self.logfile, sample.general.logout, sample.general.logerr,
                                     sample[self.analysistype].logout, sample[self.analysistype].logerr)
104148
def generate_metadata(self):
        # type: (Descriptor) -> dict
        
        genmeta = {}
        encmeta = {}
        # page align md5
        if (self.must_compute_md5 and
                self._ase.mode == blobxfer.models.azure.StorageModes.Page):
            aligned = blobxfer.util.page_align_content_length(self._offset)
            diff = aligned - self._offset
            if diff > 0:
                with self._hasher_lock:
                    self.md5.update(b'\0' * diff)
        # generate encryption metadata
        if self._ase.is_encrypted:
            if self.must_compute_md5:
                md5digest = blobxfer.util.base64_encode_as_string(
                    self.md5.digest())
            else:
                md5digest = None
            if self.hmac is not None:
                hmacdigest = blobxfer.util.base64_encode_as_string(
                    self.hmac.digest())
            else:
                hmacdigest = None
            encmeta = self._ase.encryption_metadata.convert_to_json_with_mac(
                md5digest, hmacdigest)
        # generate file attribute metadata
        if self._store_file_attr and not self.local_path.use_stdin:
            merged = blobxfer.models.metadata.generate_fileattr_metadata(
                self.local_path, genmeta)
            if merged is not None:
                genmeta = merged
        # generate vectored io metadata
        if self.local_path.view.mode == VectoredIoDistributionMode.Stripe:
            merged = blobxfer.models.metadata.\
                generate_vectored_io_stripe_metadata(self.local_path, genmeta)
            if merged is not None:
                genmeta = merged
        if len(encmeta) > 0:
            metadata = encmeta
        else:
            metadata = {}
        if len(genmeta) > 0:
            metadata[blobxfer.models.metadata.JSON_KEY_BLOBXFER_METADATA] = \
                json.dumps(genmeta, ensure_ascii=False, sort_keys=True)
        if len(metadata) == 0:
            return None
        return metadata
973915
def get_samples_data(self):
        
        for obj in self.get_samples_from_request():
            yield {
                "obj": obj,
                "id": api.get_id(obj),
                "uid": api.get_uid(obj),
                "title": api.get_title(obj),
                "path": api.get_path(obj),
                "url": api.get_url(obj),
                "sample_type": obj.getSampleTypeTitle(),
                "client_title": obj.getClientTitle(),
                "date": ulocalized_time(obj.created(), long_format=True),
            }