Bulk create database records from all csv files found within a directory.
512500
def load_all_csvs_to_model(path, model, field_names=None, delimiter=None, batch_len=10000,
                           dialect=None, num_header_rows=1, mode='rUb',
                           strip=True, clear=False, dry_run=True, ignore_errors=True,
                           sort_files=True, recursive=False, ext='',
                           min_mod_date=None, max_mod_date=None,
                           verbosity=2):
    """"""
    if min_mod_date and isinstance(min_mod_date, basestring):
        min_mod_date = parse_date(min_mod_date)
    if isinstance(min_mod_date, datetime.date):
        min_mod_date = datetime.datetime.combine(min_mod_date, datetime.datetime.min.time())

    if max_mod_date and isinstance(max_mod_date, basestring):
        max_mod_date = parse_date(max_mod_date)
    if isinstance(max_mod_date, datetime.date):
        max_mod_date = datetime.datetime.combine(max_mod_date, datetime.datetime.min.time())

    path = path or './'
    batch_len = batch_len or 1000
    if verbosity > 0:
        if dry_run:
            print 'DRY_RUN: actions will not modify the database.'
        else:
            print 'THIS IS NOT A DRY RUN, THESE ACTIONS WILL MODIFY THE DATABASE!!!!!!!!!'

    if clear:
        clear_model(model, dry_run=dry_run, verbosity=verbosity)

    file_dicts = util.find_files(path, ext=ext, level=None if recursive else 0, verbosity=verbosity)
    file_bytes = reduce(lambda a,b: a+b['size'], file_dicts, 0)

    if sort_files:
        file_dicts = sorted(file_dicts, key=itemgetter('path'))
    if verbosity > 1:
        print file_dicts
    if verbosity > 0:
        widgets = [pb.Counter(), '/%d bytes for all files: ' % file_bytes, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
        i, pbar = 0, pb.ProgressBar(widgets=widgets, maxval=file_bytes)
        pbar.start()
    N = 0
    file_bytes_done = 0
    for meta in file_dicts:
        if (min_mod_date and meta['modified'] < min_mod_date) or (max_mod_date and meta['modified'] > max_mod_date):
            if verbosity > 1:
                print("Skipping {0} because it's mdate is not between {1} and {2}".format(meta['path'], min_mod_date, max_mod_date))
            continue
        if verbosity > 0:
            print
            print 'Loading "%s"...' % meta['path']
        N += load_csv_to_model(path=meta['path'], model=model, field_names=field_names, delimiter=delimiter, batch_len=batch_len, 
                               dialect=dialect, num_header_rows=num_header_rows, mode=mode,
                               strip=strip, clear=False, dry_run=dry_run, 
                               ignore_errors=ignore_errors, verbosity=verbosity)
        if verbosity > 0:
            file_bytes_done += meta['size']
            pbar.update(file_bytes_done)
    if verbosity > 0:
        pbar.finish()
    return N
974490
def bulk_submit(cls, exports, copy_only=False, api=None):
        
        if not exports:
            raise SbgError('Exports are required')

        api = api or cls._API

        items = []
        for export in exports:
            file_ = Transform.to_file(export.get('file'))
            volume = Transform.to_volume(export.get('volume'))
            location = Transform.to_location(export.get('location'))
            properties = export.get('properties', {})
            overwrite = export.get('overwrite', False)

            item = {
                'source': {
                    'file': file_
                },
                'destination': {
                    'volume': volume,
                    'location': location
                },
                'properties': properties,
                'overwrite': overwrite
            }

            items.append(item)

        data = {'items': items}
        params = {'copy_only': copy_only}

        response = api.post(
            url=cls._URL['bulk_create'], params=params, data=data
        )
        return ExportBulkRecord.parse_records(response=response, api=api)
1107169
def bulk_create(self, objs, *args, **kwargs):
        """"""
        if hasattr(self.model, 'save_prep'):
            # Method from AbstractBaseModel. If the model class doesn't
            # subclass AbstractBaseModel, then don't call this.
            self.model.save_prep(instance_or_instances=objs)

        return super(CommonManager, self).bulk_create(objs=objs,
                                                      *args,
                                                      **kwargs)
1204592
def batch(args):
    
    from jcvi.apps.grid import MakeManager

    p = OptionParser(batch.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 2:
        sys.exit(not p.print_help())

    cdsfile = args[0]
    anchors = args[1:]
    workdirs = [".".join(op.basename(x).split(".")[:2]) for x in anchors]
    for wd in workdirs:
        mkdir(wd)

    mm = MakeManager()
    for wd, ac in zip(workdirs, anchors):
        pairscdsfile = wd + ".cds.fasta"
        cmd = "python -m jcvi.apps.ks prepare {} {} -o {}".\
                format(ac, cdsfile, pairscdsfile)
        mm.add((ac, cdsfile), pairscdsfile, cmd)
        ksfile = wd + ".ks"
        cmd = "python -m jcvi.apps.ks calc {} -o {} --workdir {}".\
                format(pairscdsfile, ksfile, wd)
        mm.add(pairscdsfile, ksfile, cmd)
    mm.write()
346121
def create_archive(archive, filenames, verbosity=0, program=None, interactive=True):
    """"""
    util.check_new_filename(archive)
    util.check_archive_filelist(filenames)
    if verbosity >= 0:
        util.log_info("Creating %s ..." % archive)
    res = _create_archive(archive, filenames, verbosity=verbosity,
                          interactive=interactive, program=program)
    if verbosity >= 0:
        util.log_info("... %s created." % archive)
    return res
889403
def csv_to_sqlite(
    input_filename,
    output_filename,
    samples=None,
    dialect=None,
    batch_size=10000,
    encoding="utf-8",
    callback=None,
    force_types=None,
    chunk_size=8388608,
    table_name="table1",
    schema=None,
):
    ""

    # TODO: automatically detect encoding if encoding == `None`
    # TODO: should be able to specify fields
    # TODO: if table_name is "2019" the final name will be "field_2019" - must
    #       be "table_2019"
    # TODO: if schema is provided and the names are in uppercase, this function
    #       will fail

    if dialect is None:  # Get a sample to detect dialect
        fobj = open_compressed(input_filename, mode="rb")
        sample = fobj.read(chunk_size)
        dialect = rows.plugins.csv.discover_dialect(sample, encoding=encoding)
    elif isinstance(dialect, six.text_type):
        dialect = csv.get_dialect(dialect)

    if schema is None:  # Identify data types
        fobj = open_compressed(input_filename, encoding=encoding)
        data = list(islice(csv.DictReader(fobj, dialect=dialect), samples))
        schema = rows.import_from_dicts(data).fields
        if force_types is not None:
            schema.update(force_types)

    # Create lazy table object to be converted
    # TODO: this lazyness feature will be incorported into the library soon so
    #       we can call here `rows.import_from_csv` instead of `csv.reader`.
    reader = csv.reader(
        open_compressed(input_filename, encoding=encoding), dialect=dialect
    )
    header = make_header(next(reader))  # skip header
    table = rows.Table(fields=OrderedDict([(field, schema[field]) for field in header]))
    table._rows = reader

    # Export to SQLite
    return rows.export_to_sqlite(
        table,
        output_filename,
        table_name=table_name,
        batch_size=batch_size,
        callback=callback,
    )
1327451
def create_db(self, app):
        """"""
        assert 'ZODB_STORAGE' in app.config, \
               'ZODB_STORAGE not configured'
        storage = app.config['ZODB_STORAGE']
        if isinstance(storage, basestring):
            factory, dbargs = zodburi.resolve_uri(storage)
        elif isinstance(storage, tuple):
            factory, dbargs = storage
        else:
            factory, dbargs = storage, {}
        return DB(factory(), **dbargs)
1211310
def _create_sample_file(data, out_dir):
    """"""
    sample_file = os.path.join(out_dir, "sample_file.txt")
    with open(sample_file, 'w') as outh:
        for sample in data:
            outh.write(sample[0]["clean_fastq"] + "\n")
    return sample_file
54076
def write_installed_files(self, paths, prefix, dry_run=False):
        
        prefix = os.path.join(prefix, '')
        base = os.path.dirname(self.path)
        base_under_prefix = base.startswith(prefix)
        base = os.path.join(base, '')
        record_path = self.get_distinfo_file('RECORD')
        logger.info('creating %s', record_path)
        if dry_run:
            return None
        with CSVWriter(record_path) as writer:
            for path in paths:
                if os.path.isdir(path) or path.endswith(('.pyc', '.pyo')):
                    # do not put size and hash, as in PEP-376
                    hash_value = size = ''
                else:
                    size = '%d' % os.path.getsize(path)
                    with open(path, 'rb') as fp:
                        hash_value = self.get_hash(fp.read())
                if path.startswith(base) or (base_under_prefix and
                                             path.startswith(prefix)):
                    path = os.path.relpath(path, base)
                writer.writerow((path, hash_value, size))

            # add the RECORD file itself
            if record_path.startswith(base):
                record_path = os.path.relpath(record_path, base)
            writer.writerow((record_path, '', ''))
        return record_path
1155912
def create_new_csv(samples, args):
    """"""
    out_fn = os.path.splitext(args.csv)[0] + "-merged.csv"
    logger.info("Preparing new csv: %s" % out_fn)
    with file_transaction(out_fn) as tx_out:
        with open(tx_out, 'w') as handle:
            handle.write(_header(args.csv))
            for s in samples:
                sample_name = s['name'] if isinstance(s['out_file'], list) else os.path.basename(s['out_file'])
                handle.write("%s,%s,%s\n" % (sample_name, s['name'], ",".join(s['anno'])))
528458
def makeblastdb(self):
        
        # Iterate through the samples to set the bait file.
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                # Remove the file extension
                db = os.path.splitext(sample[self.analysistype].baitfile)[0]
                # Add '.nhr' for searching below
                nhr = '{}.nhr'.format(db)
                # Check for already existing database files
                if not os.path.isfile(str(nhr)):
                    # Create the databases
                    command = 'makeblastdb -in {} -parse_seqids -max_file_sz 2GB -dbtype nucl -out {}'\
                        .format(sample[self.analysistype].baitfile, db)
                    out, err = run_subprocess(command)
                    write_to_logfile(command,
                                     command,
                                     self.logfile, sample.general.logout, sample.general.logerr,
                                     sample[self.analysistype].logout, sample[self.analysistype].logerr)
                    write_to_logfile(out,
                                     err,
                                     self.logfile, sample.general.logout, sample.general.logerr,
                                     sample[self.analysistype].logout, sample[self.analysistype].logerr)
901132
def make_csv(data):
    
    # Convert sub-dicts to news cols
    for entry in data:
        rval = entry
        if isinstance(data, dict):
            rval = data[entry]
        todel = []
        toadd = {}
        for key, val in rval.items():
            if isinstance(val, dict):
                for key2, val2 in val.items():
                    toadd[str(key) + "[" + str(key2) + "]"] = val2
                todel.append(key)
        for k in todel:
            del rval[k]
        for k, v in toadd.items():
            rval[k] = v

    # Convert everything to CSV
    columns = set()
    output = [[]]
    if isinstance(data, dict):
        output[0].append("id")
        for entry in data:
            for col in data[entry]:
                columns.add(col)
    else:
        for entry in data:
            for col in entry:
                columns.add(col)

    columns = sorted(columns)

    for col in columns:
        output[0].append(col)

    if isinstance(data, dict):
        for entry in data:
            new_output = [str(entry)]
            for col in columns:
                new_output.append(str(data[entry][col]) if col in data[entry] else "")
            output.append(new_output)
    else:
        for entry in data:
            new_output = []
            for col in columns:
                new_output.append(str(entry[col]) if col in entry else "")
            output.append(new_output)

    csv_string = io.StringIO()
    csv_writer = UnicodeWriter(csv_string)
    for row in output:
        csv_writer.writerow(row)
    csv_string.seek(0)
    web.header('Content-Type', 'text/csv; charset=utf-8')
    web.header('Content-disposition', 'attachment; filename=export.csv')
    return csv_string.read()
414881
def csv(self, client_id):
        """"""
        logging.info('Exporting CSV file [{}]'.format(client_id))
        query = (
            db.session.query(Query)
            .filter_by(client_id=client_id)
            .one()
        )

        rejected_tables = security_manager.rejected_datasources(
            query.sql, query.database, query.schema)
        if rejected_tables:
            flash(
                security_manager.get_table_access_error_msg('{}'.format(rejected_tables)))
            return redirect('/')
        blob = None
        if results_backend and query.results_key:
            logging.info(
                'Fetching CSV from results backend '
                '[{}]'.format(query.results_key))
            blob = results_backend.get(query.results_key)
        if blob:
            logging.info('Decompressing')
            json_payload = utils.zlib_decompress_to_string(blob)
            obj = json.loads(json_payload)
            columns = [c['name'] for c in obj['columns']]
            df = pd.DataFrame.from_records(obj['data'], columns=columns)
            logging.info('Using pandas to convert to CSV')
            csv = df.to_csv(index=False, **config.get('CSV_EXPORT'))
        else:
            logging.info('Running a query to turn into CSV')
            sql = query.select_sql or query.executed_sql
            df = query.database.get_df(sql, query.schema)
            # TODO(bkyryliuk): add compression=gzip for big files.
            csv = df.to_csv(index=False, **config.get('CSV_EXPORT'))
        response = Response(csv, mimetype='text/csv')
        response.headers['Content-Disposition'] = f'attachment; filename={query.name}.csv'
        logging.info('Ready to return response')
        return response
668369
def build_docs(directory):
    """"""
    os.chdir(directory)
    process = subprocess.Popen(["make", "html"], cwd=directory)
    process.communicate()
95991
def _collect_data(directory):
  
  # Returns:
  data_files = []
  transcripts = [
      filename for filename in os.listdir(directory)
      if filename.endswith(".csv")
  ]
  for transcript in transcripts:
    transcript_path = os.path.join(directory, transcript)
    with open(transcript_path, "r") as transcript_file:
      transcript_reader = csv.reader(transcript_file)
      # skip header
      _ = next(transcript_reader)
      for transcript_line in transcript_reader:
        media_name, label = transcript_line[0:2]
        filename = os.path.join(directory, media_name)
        data_files.append((media_name, filename, label))
  return data_files
1349035
def bulk_get(cls, files, api=None):
        
        api = api or cls._API
        file_ids = [Transform.to_file(file_) for file_ in files]
        data = {'file_ids': file_ids}

        logger.info('Getting files in bulk.')
        response = api.post(url=cls._URL['bulk_get'], data=data)
        return FileBulkRecord.parse_records(response=response, api=api)
769390
def LoadFile(f, table_name, conn):
  """"""
  reader = csv.reader(f)
  header = next(reader)

  columns = []
  for n in header:
    n = n.replace(' ', '')
    n = n.replace('-', '_')
    columns.append(n)

  create_columns = []
  column_types = {}
  for n in columns:
    if n in column_types:
      create_columns.append("%s %s" % (n, column_types[n]))
    else:
      create_columns.append("%s INTEGER" % (n))

  c = conn.cursor()
  try:
    c.execute("CREATE TABLE %s (%s)" % (table_name, ",".join(create_columns)))
  except sqlite.OperationalError:
    # Likely table exists
    print("table %s already exists?" % (table_name))
    for create_column in create_columns:
      try:
        c.execute("ALTER TABLE %s ADD COLUMN %s" % (table_name, create_column))
      except sqlite.OperationalError:
        # Likely it already exists
        print("column %s already exists in %s?" % (create_column, table_name))

  placeholders = ",".join(["?"] * len(columns))
  insert_values = "INSERT INTO %s (%s) VALUES (%s)" % (table_name, ",".join(columns), placeholders)

  #c.execute("BEGIN TRANSACTION;")
  for row in reader:
    if row:
      if len(row) < len(columns):
        row.extend([None] * (len(columns) - len(row)))
      c.execute(insert_values, row)
  #c.execute("END TRANSACTION;")
  conn.commit()