Read clusters from file and save to model file.
11251
def load(ctx, input, output):
    """"""
    log.debug('chemdataextractor.cluster.load')
    import pickle
    click.echo('Reading %s' % input.name)
    clusters = {}
    for line in input.readlines():
        cluster, word, freq = line.split()
        clusters[word] = cluster
    pickle.dump(clusters, output, protocol=pickle.HIGHEST_PROTOCOL)
133552
def load_csv_to_model(path, model, field_names=None, delimiter=None, batch_len=10000, 
                      dialect=None, num_header_rows=1, mode='rUb',
                      strip=True, clear=False, dry_run=True, ignore_errors=True, verbosity=2):
    ''''''

    reader_kwargs = {}
    errors = collections.Counter()
    if delimiter or dialect:
        reader_kwargs['dialect'] = dialect or 'excel'
        if delimiter:
            reader_kwargs['delimiter'] = delimiter
    reader_kwargs['delimiter'] = str(reader_kwargs['delimiter'][0])
    delimiter = reader_kwargs['delimiter']

    path = path or './'
    if not delimiter:
        for d in ',', '|', '\t', ';':
            try:
                return load_csv_to_model(path=path, model=model, field_names=field_names, delimiter=d, batch_len=batch_len,
                                         dialect=dialect, num_header_rows=num_header_rows,
                                         strip=strip, clear=clear, dry_run=dry_run, ignore_errors=ignore_errors, verbosity=verbosity)
            except:
                pass
        return None

    if clear:
        clear_model(model, dry_run=dry_run, verbosity=verbosity)

    M = 0
    with open(path, mode) as f:
        reader = csv.reader(f, **reader_kwargs)
        header_rows = []
        i = 0
        while len(header_rows) < num_header_rows and i < 100:
            row = reader.next()
            i += 1
            if not row or any(compiled_regex.match(row[0]) for compiled_regex in header_rows_to_ignore):
                if verbosity > 1:
                    print 'IGNORED: %r' % row
            else:
                header_rows += [row]
        if verbosity > 2:
            print 'HEADER: %r' % header_rows
        if verbosity > 0:
            N = count_lines(path, mode) - i + 10  # + 10 fudge factor in case multiple newlines in a single csv row
            widgets = [pb.Counter(), '/%d lines: ' % N, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
            i, pbar = 0, pb.ProgressBar(widgets=widgets, maxval=N).start()
        if verbosity > 3:
            print 'Generating all the batches before iterating may take a while...'
        for batch_num, batch_of_rows in enumerate(util.generate_batches(reader, batch_len)):
            if verbosity > 2:
                print 
                print i
            batch_of_objects = []
            for j, row in enumerate(batch_of_rows):
                if verbosity > 3:
                    print j, row
                if not row or all(not el for el in row):
                    if verbosity > 2:
                        print 'IGNORED: %r' % row
                    continue
                if verbosity or not ignore_errors:
                    M = M or len(row)
                    if len(row) != M:
                        print 'ERROR importing row #%d in batch_num=%d which is row #%d overall. The row had %d columns, but previous rows had %d.' % (j + 1, batch_num + 1, i + j + 1, len(row), M)
                        print 'Erroneously parsed row:'
                        print repr(row)
                        
                        if not ignore_errors:
                            raise ValueError('ERROR importing row #%d which had %d columns, but previous rows had %d.' % (i + j + 1, len(row), M))
                try:
                    obj, row_errors = django_object_from_row(row, model=model, field_names=field_names, strip=strip, verbosity=verbosity)
                    batch_of_objects += [obj]
                    errors += row_errors
                except:
                    if verbosity > 0:
                        print 'Error importing row #%d' % (i + j + 1)
                        print_exc()
                    if not ignore_errors:
                        raise
                if verbosity > 0:
                    try:
                        pbar.update(i + j)
                    except:
                        print_exc()
                        if not ignore_errors:
                            raise
            i += len(batch_of_rows)
            if not dry_run:
                model.objects.bulk_create(batch_of_objects)
            elif verbosity > 0:
                print "DRY_RUN: NOT bulk creating batch of %d records in %r" % (len(batch_of_objects), model)
        if verbosity > 0:
            pbar.finish()
    return i
408436
def _fromfile(self):
        
        fd = self.parent._fd
        byte_order = self.parent.byte_order
        offset_size = self.parent.offset_size

        fmt = {4: 'I', 8: 'Q'}[offset_size]
        offset = struct.unpack(byte_order + fmt, fd.read(offset_size))[0]
        if not offset:
            raise StopIteration()

        # read standard tags
        tags = self.tags
        fd.seek(offset)
        fmt, size = {4: ('H', 2), 8: ('Q', 8)}[offset_size]
        try:
            numtags = struct.unpack(byte_order + fmt, fd.read(size))[0]
        except Exception:
            warnings.warn("corrupted page list")
            raise StopIteration()

        for _ in range(numtags):
            tag = TIFFtag(self.parent)
            tags[tag.name] = tag

        # read LSM info subrecords
        if self.is_lsm:
            pos = fd.tell()
            for name, reader in CZ_LSM_INFO_READERS.items():
                try:
                    offset = self.cz_lsm_info["offset_"+name]
                except KeyError:
                    continue
                if not offset:
                    continue
                fd.seek(offset)
                try:
                    setattr(self, "cz_lsm_"+name, reader(fd, byte_order))
                except ValueError:
                    pass
            fd.seek(pos)
1296771
def read(filename):
    
    # The format is specified at
    # <http://www.intes.de>.

    cells = {}
    meshio_to_permas_type = {
        "vertex": (1, "PLOT1"),
        "line": (2, "PLOTL2"),
        "triangle": (3, "TRIA3"),
        "quad": (4, "QUAD4"),
        "tetra": (4, "TET4"),
        "hexahedron": (8, "HEXE8"),
        "wedge": (6, "PENTA6"),
        "pyramid": (5, "PYRA5"),
    }

    opener = gzip.open if filename.endswith(".gz") else open

    with opener(filename, "rb") as f:
        while True:
            line = f.readline().decode("utf-8")
            if not line or re.search("\\$END STRUCTURE", line):
                break
            for meshio_type, permas_ele in meshio_to_permas_type.items():
                num_nodes = permas_ele[0]
                permas_type = permas_ele[1]

                if re.search("\\$ELEMENT TYPE = {}".format(permas_type), line):
                    while True:
                        line = f.readline().decode("utf-8")
                        if not line or line.startswith("!"):
                            break
                        data = numpy.array(line.split(), dtype=int)
                        if meshio_type in cells:
                            cells[meshio_type].append(data[-num_nodes:])
                        else:
                            cells[meshio_type] = [data[-num_nodes:]]

            if re.search("\\$COOR", line):
                points = []
                while True:
                    line = f.readline().decode("utf-8")
                    if not line or line.startswith("!"):
                        break
                    for r in numpy.array(line.split(), dtype=float)[1:]:
                        points.append(r)

    points = numpy.array(points)
    points = numpy.reshape(points, newshape=(len(points) // 3, 3))
    for key in cells:
        # Subtract one to account for the fact that python indices
        # are 0-based.
        cells[key] = numpy.array(cells[key], dtype=int) - 1

    return Mesh(points, cells)
1161423
def _to_file_format(self, iamc_index):
        """"""
        df = self.timeseries(iamc_index=iamc_index).reset_index()
        df = df.rename(columns={c: str(c).title() for c in df.columns})
        return df
1238012
def create_or_update_cluster(config_file, override_min_workers,
                             override_max_workers, no_restart, restart_only,
                             yes, override_cluster_name):
    """"""
    config = yaml.load(open(config_file).read())
    if override_min_workers is not None:
        config["min_workers"] = override_min_workers
    if override_max_workers is not None:
        config["max_workers"] = override_max_workers
    if override_cluster_name is not None:
        config["cluster_name"] = override_cluster_name
    config = _bootstrap_config(config)
    get_or_create_head_node(config, config_file, no_restart, restart_only, yes,
                            override_cluster_name)
1332331
def from_file(self, filename):
        """"""
        f = open(filename, 'rb')
        while True:
            data = f.read(10480)
            if not data:
                break
            self.update(data)
        f.close()
1209607
def read_lines_from_file(cls_name, filename):
  """"""
  with tf.io.gfile.GFile(filename, "rb") as f:
    lines = [tf.compat.as_text(line)[:-1] for line in f]
  header_line = "%s%s" % (_HEADER_PREFIX, cls_name)
  if lines[0] != header_line:
    raise ValueError("File {fname} does not seem to have been created from "
                     "{name}.save_to_file.".format(
                         fname=filename, name=cls_name))
  metadata_dict = json.loads(lines[1][len(_METADATA_PREFIX):])
  return lines[2:], metadata_dict
1008090
def _annotate(args, setclus):
    """"""
    logger.info("Creating bed file")
    bedfile = generate_position_bed(setclus)
    a = pybedtools.BedTool(bedfile, from_string=True)
    beds = []
    logger.info("Annotating clusters")
    if hasattr(args, 'list_files'):
        beds = args.list_files.split(",")
        for filebed in beds:
            logger.info("Using %s " % filebed)
            db = os.path.basename(filebed)
            b = pybedtools.BedTool(filebed)
            c = a.intersect(b, wo=True)
            setclus = anncluster(c, setclus, db, args.type_ann, args.feature_id)
    return setclus
72532
def score_file(filename):
  """"""
  # Prepare model.
  hparams = create_hparams()
  encoders = registry.problem(FLAGS.problem).feature_encoders(FLAGS.data_dir)
  has_inputs = "inputs" in encoders

  # Prepare features for feeding into the model.
  if has_inputs:
    inputs_ph = tf.placeholder(dtype=tf.int32)  # Just length dimension.
    batch_inputs = tf.reshape(inputs_ph, [1, -1, 1, 1])  # Make it 4D.
  targets_ph = tf.placeholder(dtype=tf.int32)  # Just length dimension.
  batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1])  # Make it 4D.
  if has_inputs:
    features = {"inputs": batch_inputs, "targets": batch_targets}
  else:
    features = {"targets": batch_targets}

  # Prepare the model and the graph when model runs on features.
  model = registry.model(FLAGS.model)(hparams, tf.estimator.ModeKeys.EVAL)
  _, losses = model(features)
  saver = tf.train.Saver()

  with tf.Session() as sess:
    # Load weights from checkpoint.
    if FLAGS.checkpoint_path is None:
      ckpts = tf.train.get_checkpoint_state(FLAGS.output_dir)
      ckpt = ckpts.model_checkpoint_path
    else:
      ckpt = FLAGS.checkpoint_path
    saver.restore(sess, ckpt)
    # Run on each line.
    with tf.gfile.Open(filename) as f:
      lines = f.readlines()
    results = []
    for line in lines:
      tab_split = line.split("\t")
      if len(tab_split) > 2:
        raise ValueError("Each line must have at most one tab separator.")
      if len(tab_split) == 1:
        targets = tab_split[0].strip()
      else:
        targets = tab_split[1].strip()
        inputs = tab_split[0].strip()
      # Run encoders and append EOS symbol.
      targets_numpy = encoders["targets"].encode(
          targets) + [text_encoder.EOS_ID]
      if has_inputs:
        inputs_numpy = encoders["inputs"].encode(inputs) + [text_encoder.EOS_ID]
      # Prepare the feed.
      if has_inputs:
        feed = {inputs_ph: inputs_numpy, targets_ph: targets_numpy}
      else:
        feed = {targets_ph: targets_numpy}
      # Get the score.
      np_loss = sess.run(losses["training"], feed)
      results.append(np_loss)
  return results
891701
def main():
    
    parser = argparse.ArgumentParser(
        description='Performs clustering from the command-line. Calls JVM start/stop automatically.')
    parser.add_argument("-j", metavar="classpath", dest="classpath", help="additional classpath, jars/directories")
    parser.add_argument("-X", metavar="heap", dest="heap", help="max heap size for jvm, e.g., 512m")
    parser.add_argument("-t", metavar="train", dest="train", required=True, help="training set file")
    parser.add_argument("-T", metavar="test", dest="test", help="test set file")
    parser.add_argument("-d", metavar="outmodel", dest="outmodel", help="model output file name")
    parser.add_argument("-l", metavar="inmodel", dest="inmodel", help="model input file name")
    parser.add_argument("-p", metavar="attributes", dest="attributes", help="attribute range")
    parser.add_argument("-x", metavar="num folds", dest="numfolds", help="number of folds")
    parser.add_argument("-s", metavar="seed", dest="seed", help="seed value for randomization")
    parser.add_argument("-c", metavar="class index", dest="classindex", help="1-based class attribute index")
    parser.add_argument("-g", metavar="graph", dest="graph", help="graph output file (if supported)")
    parser.add_argument("clusterer", help="clusterer classname, e.g., weka.clusterers.SimpleKMeans")
    parser.add_argument("option", nargs=argparse.REMAINDER, help="additional clusterer options")
    parsed = parser.parse_args()
    jars = []
    if parsed.classpath is not None:
        jars = parsed.classpath.split(os.pathsep)
    params = []
    if parsed.train is not None:
        params.extend(["-t", parsed.train])
    if parsed.test is not None:
        params.extend(["-T", parsed.test])
    if parsed.outmodel is not None:
        params.extend(["-d", parsed.outmodel])
    if parsed.inmodel is not None:
        params.extend(["-l", parsed.inmodel])
    if parsed.attributes is not None:
        params.extend(["-p", parsed.attributes])
    if parsed.numfolds is not None:
        params.extend(["-x", parsed.numfolds])
    if parsed.seed is not None:
        params.extend(["-s", parsed.seed])
    if parsed.classindex is not None:
        params.extend(["-c", parsed.classindex])
    if parsed.graph is not None:
        params.extend(["-g", parsed.graph])

    jvm.start(jars, max_heap_size=parsed.heap, packages=True)

    logger.debug("Commandline: " + join_options(sys.argv[1:]))

    try:
        clusterer = Clusterer(classname=parsed.clusterer)
        if len(parsed.option) > 0:
            clusterer.options = parsed.option
        print(ClusterEvaluation.evaluate_clusterer(clusterer, params))
    except Exception as e:
        print(e)
    finally:
        jvm.stop()
1211367
def from_file(self, fname):
        """"""
        f = open(fname, "rb")
        data = f.read()
        self.update(data)
        f.close()
592496
def load_hdf(cls, filename, path=''):
        
        with pd.HDFStore(filename) as store:
            models = []
            types = []
            for k in store.keys():
                m = re.search('/(\S+)/stars', k)
                if m:
                    models.append(m.group(1))
                    types.append(store.get_storer(m.group(0)).attrs.poptype)
        poplist = []
        for m,t in zip(models,types):
            poplist.append(t().load_hdf(filename, path='{}/{}'.format(path,m)))

        return cls(poplist)
1119661
def _create_clusters(seqL, bam_file, args):
    
    clus_obj = []
    cluster_file = op.join(args.out, "cluster.bed")
    if not os.path.exists(op.join(args.out, 'list_obj.pk')):
        if not file_exists(cluster_file):
            logger.info("Parsing aligned file")
            logger.info("Merging sequences")
            bedtools = os.path.join(os.path.dirname(sys.executable), "bedtools")
            bedtools = bedtools if os.path.exists(bedtools) else "bedtools"
            parse_cmd = "awk '{i=i+1;print $1\"\\t\"$2\"\\t\"$3\"\\t\"$4\"\\t\"i\"\\t\"$6}'"
            cmd = "{bedtools} bamtobed -i {bam_file} | {parse_cmd} | {bedtools} cluster -s -d 20 -i - > {cluster_file}"
            do.run(cmd.format(**locals()))
        c = pybedtools.BedTool(cluster_file)
        logger.info("Creating clusters")
        clus_obj = detect_clusters(c, seqL, args.min_seqs, args.non_un_gl)
        with open(op.join(args.out, 'list_obj.pk'), 'wb') as output:
            pickle.dump(clus_obj, output, pickle.HIGHEST_PROTOCOL)
    else:
        logger.info("Loading previous clusters")
        with open(op.join(args.out, 'list_obj.pk'), 'rb') as input:
            clus_obj = pickle.load(input)
    # bedfile = pybedtools.BedTool(generate_position_bed(clus_obj), from_string=True)
    # seqs_2_loci = bedfile.intersect(pybedtools.BedTool(aligned_bed, from_string=True), wo=True, s=True)
    # seqs_2_position = add_seqs_position_to_loci(seqs_2_loci, seqL)
    logger.info("%s clusters found" % (len(clus_obj.clusid)))
    return clus_obj
953377
def _read(self, filename):
        """"""
        data = {}
        parsers = [
            FirstDataParser(), CoordinateParser(), EnergyGradParser(),
            SkipApproxHessian(), HessianParser(), MassParser(),
        ]
        with open(filename) as f:
            while True:
                line = f.readline()
                if line == "":
                    break
                # at each line, a parsers checks if it has to process a piece of
                # file. If that happens, the parser gets control over the file
                # and reads as many lines as it needs to collect data for some
                # attributes.
                for parser in parsers:
                    if parser.test(line, data):
                        parser.read(line, f, data)
                        break
        self.__dict__.update(data)
152168
def join(input_files, output_file):
    '''
    
    '''

    # get feature collections
    final_features = []
    for file in input_files:
        with open(file) as f:
            feat_collection = geojson.load(f)
            final_features += feat_collection['features']

    feat_collection['features'] = final_features

    # write to output file
    with open(output_file, 'w') as f:
        geojson.dump(feat_collection, f)
586703
public void save(String modelFile, Set<Map.Entry<String, Integer>> featureIdSet, final double ratio, boolean text) throws IOException
    {
        float[] parameter = this.parameter;
        this.compress(ratio, 1e-3f);

        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(IOUtil.newOutputStream(modelFile)));
        save(out);
        out.close();

        if (text)
        {
            BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(IOUtil.newOutputStream(modelFile + ".txt"), "UTF-8"));
            TagSet tagSet = featureMap.tagSet;
            for (Map.Entry<String, Integer> entry : featureIdSet)
            {
                bw.write(entry.getKey());
                if (featureIdSet.size() == parameter.length)
                {
                    bw.write("\t");
                    bw.write(String.valueOf(parameter[entry.getValue()]));
                }
                else
                {
                    for (int i = 0; i < tagSet.size(); ++i)
                    {
                        bw.write("\t");
                        bw.write(String.valueOf(parameter[entry.getValue() * tagSet.size() + i]));
                    }
                }
                bw.newLine();
            }
            bw.close();
        }
    }