Yield features from DataBC WFS
10158
def get_features(
    dataset,
    query=None,
    crs="epsg:4326",
    bounds=None,
    sortby=None,
    pagesize=10000,
    max_workers=5,
):
    
    param_dicts = define_request(dataset, query, crs, bounds, sortby, pagesize)

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        for result in executor.map(make_request, param_dicts):
            for feature in result:
                yield feature
946734
def to_dataset_files(self, local_path=None, remote_path=None):
        
        return FrameToGMQL.to_dataset_files(self, path_local=local_path, path_remote=remote_path)
1173281
def _cldf2wld(dataset):
    """"""
    header = [f for f in dataset.dataset.lexeme_class.fieldnames() if f != 'ID']
    D = {0: ['lid'] + [h.lower() for h in header]}
    for idx, row in enumerate(dataset.objects['FormTable']):
        row = deepcopy(row)
        row['Segments'] = ' '.join(row['Segments'])
        D[idx + 1] = [row['ID']] + [row[h] for h in header]
    return D
238781
def auto_data_specs(self):
        
        for spec in self.study_class.data_specs():
            if spec.name not in self._name_map:
                yield spec
921402
def dataset_walker(datasets):
    
    for dataset in datasets:
        yield dataset, None
        for anc_ds in dataset.attrs.get('ancillary_variables', []):
            try:
                anc_ds.attrs
                yield anc_ds, dataset
            except AttributeError:
                continue
7835
def dataset_generator(filepath,
                      dataset,
                      chunk_size=1,
                      start_idx=None,
                      end_idx=None):
  """"""
  encoder = dna_encoder.DNAEncoder(chunk_size=chunk_size)
  with h5py.File(filepath, "r") as h5_file:
    # Get input keys from h5_file
    src_keys = [s % dataset for s in ["%s_in", "%s_na", "%s_out"]]
    src_values = [h5_file[k] for k in src_keys]
    inp_data, mask_data, out_data = src_values
    assert len(set([v.len() for v in src_values])) == 1

    if start_idx is None:
      start_idx = 0
    if end_idx is None:
      end_idx = inp_data.len()

    for i in range(start_idx, end_idx):
      if i % 100 == 0:
        print("Generating example %d for %s" % (i, dataset))
      inputs, mask, outputs = inp_data[i], mask_data[i], out_data[i]
      ex_dict = to_example_dict(encoder, inputs, mask, outputs)
      # Original data has one output for every 128 input bases. Ensure that the
      # ratio has been maintained given the chunk size and removing EOS.
      assert (len(ex_dict["inputs"]) - 1) == ((
          128 // chunk_size) * ex_dict["targets_shape"][0])
      yield ex_dict
1362724
def get_wd_data(self):
        
        wait = wx.BusyInfo('Reading in data from current working directory, please wait...')
        #wx.Yield()
        print('-I- Read in any available data from working directory')
        self.contribution = cb.Contribution(self.WD, dmodel=self.data_model)
        del wait
1370169
def _resolve_features(self):
        """"""

        for parentid in self.featsbyparent:
            parent = self.featsbyid[parentid]
            for child in self.featsbyparent[parentid]:
                parent.add_child(child, rangecheck=self.strict)

        # Replace top-level multi-feature reps with a pseudo-feature
        for n, record in enumerate(self.records):
            if not isinstance(record, Feature):
                continue
            if not record.is_multi:
                continue
            assert record.multi_rep == record
            newrep = sorted(record.siblings + [record])[0]
            if newrep != record:
                for sib in sorted(record.siblings + [record]):
                    sib.multi_rep = newrep
                    if sib != newrep:
                        newrep.add_sibling(sib)
                record.siblings = None
            parent = newrep.pseudoify()
            self.records[n] = parent

        if not self.assumesorted:
            for seqid in self.inferred_regions:
                if seqid not in self.declared_regions:
                    seqrange = self.inferred_regions[seqid]
                    srstring = '##sequence-region {:s} {:d} {:d}'.format(
                        seqid, seqrange.start + 1, seqrange.end
                    )
                    seqregion = Directive(srstring)
                    self.records.append(seqregion)

        for record in sorted(self.records):
            yield record
        self._reset()
439566
def ParseMultiple(self, result_dicts):
    """"""
    for result_dict in result_dicts:
      wmi_dict = result_dict.ToDict()

      try:
        creator_sid_bytes = bytes(wmi_dict["CreatorSID"])
        wmi_dict["CreatorSID"] = BinarySIDtoStringSID(creator_sid_bytes)
      except ValueError:
        # We recover from corrupt SIDs by outputting it raw as a string
        wmi_dict["CreatorSID"] = compatibility.Repr(wmi_dict["CreatorSID"])
      except KeyError:
        pass

      for output_type in self.output_types:
        anomalies = []

        output = rdfvalue.RDFValue.classes[output_type]()
        for k, v in iteritems(wmi_dict):
          try:
            output.Set(k, v)
          except AttributeError as e:
            # Skip any attribute we don't know about
            anomalies.append("Unknown field %s, with value %s" % (k, v))
          except ValueError as e:
            anomalies.append("Invalid value %s for field %s: %s" % (v, k, e))

        # Yield anomalies first to help with debugging
        if anomalies:
          yield rdf_anomaly.Anomaly(
              type="PARSER_ANOMALY",
              generated_by=self.__class__.__name__,
              finding=anomalies)

        # Raise if the parser generated no output but there were fields.
        if wmi_dict and not output:
          raise ValueError("Non-empty dict %s returned empty output." %
                           wmi_dict)

        yield output
794728
def data_collector(iterable, def_buf_size=5242880):
    
    buf = b''
    for data in iterable:
        buf += data
        if len(buf) >= def_buf_size:
            output = buf[:def_buf_size]
            buf = buf[def_buf_size:]
            yield output
    if len(buf) > 0:
        yield buf
1331592
def _iterOutFiles(self):
        
        out = StringIO()
        self.callgrind(out, relative_path=True)
        yield (
            'cachegrind.out.pprofile',
            out.getvalue(),
            'application/x-kcachegrind',
        )
        for name, lines in self.iterSource():
            lines = ''.join(lines)
            if lines:
                if isinstance(lines, unicode):
                    lines = lines.encode('utf-8')
                yield (
                    os.path.normpath(
                        os.path.splitdrive(name)[1]
                    ).lstrip(_ALLSEP),
                    lines,
                    'text/x-python',
                )
        sql_name_template = 'query_%%0%ii-%%i_hits_%%6fs.sql' % len(
            str(len(self.sql_dict)),
        )
        for index, (query, time_list) in enumerate(
            sorted(
                self.sql_dict.iteritems(),
                key=lambda x: (sum(x[1]), len(x[1])),
                reverse=True,
            ),
        ):
            yield (
                sql_name_template % (
                    index,
                    len(time_list),
                    sum(time_list),
                ),
                b'\n'.join(b'-- %10.6fs' % x for x in time_list) + b'\n' + query,
                'application/sql',
            )
        if self.zodb_dict:
            yield (
                'ZODB_setstate.txt',
                '\n\n'.join(
                    (
                        '%s (%fs)\n' % (
                            db_name,
                            sum(sum(x) for x in oid_dict.itervalues()),
                        )
                    ) + '\n'.join(
                        '%s (%i): %s' % (
                            oid.encode('hex'),
                            len(time_list),
                            ', '.join('%fs' % x for x in time_list),
                        )
                        for oid, time_list in oid_dict.iteritems()
                    )
                    for db_name, oid_dict in self.zodb_dict.iteritems()
                ),
                'text/plain',
            )
        if self.traverse_dict:
            yield (
                'unrestrictedTraverse_pathlist.txt',
                tabulate(
                    ('self', 'path', 'hit', 'total duration'),
                    sorted(
                        (
                            (context, path, len(duration_list), sum(duration_list))
                            for (context, path), duration_list in self.traverse_dict.iteritems()
                        ),
                        key=lambda x: x[3],
                        reverse=True,
                    ),
                ),
                'text/plain',
            )
1148818
def list_records(self, file_const=None):
        """"""
        for r in self._dataset.files:
            if file_const and r.minor_type != file_const:
                continue
            yield self.instance_from_name(r.path)
647105
def _generate_examples(self, data_file):
    """"""
    target_language = self.builder_config.target_language

    with tf.io.gfile.GFile(data_file) as f:
      for i, line in enumerate(f):
        line_parts = line.strip().split("\t")
        if len(line_parts) != 2:
          raise ValueError(("Wrong data format in line {}. The line '{}' does "
                            "not have exactly one delimiter.").format(i, line))
        source, target = line_parts[0].strip(), line_parts[1].strip()

        yield {"en": source, target_language: target}
936130
def _build_pcollection(self, pipeline, filepaths, language):
    """"""

    beam = tfds.core.lazy_imports.apache_beam

    def _extract_content(filepath):
      """Extracts article content from a single WikiMedia XML file."""
      logging.info("generating examples from = %s", filepath)
      with tf.io.gfile.GFile(filepath) as f:
        for _, elem in etree.iterparse(f, events=("end",)):
          if not elem.tag.endswith("page"):
            continue
          namespace = elem.tag[:-4]
          title = elem.find("./{0}title".format(namespace)).text
          ns = elem.find("./{0}ns".format(namespace)).text

          # Filter pages that are not in the "main" namespace.
          if ns != "0":
            continue

          raw_content = elem.find(
              "./{0}revision/{0}text".format(namespace)).text
          elem.clear()

          # Filter redirects.
          if raw_content is None or raw_content.lower().startswith("#redirect"):
            beam.metrics.Metrics.counter(language, "filtered-redirects").inc()
            continue

          beam.metrics.Metrics.counter(language, "extracted-examples").inc()
          yield (title, raw_content)

    def _clean_content(inputs):
      """Cleans raw wikicode to extract text."""
      title, raw_content = inputs
      try:
        text = _parse_and_clean_wikicode(raw_content)
      except (
          tfds.core.lazy_imports.mwparserfromhell.parser.ParserError) as e:
        beam.metrics.Metrics.counter(language, "parser-error").inc()
        logging.error("mwparserfromhell ParseError: %s", e)
        return

      beam.metrics.Metrics.counter(language, "cleaned-examples").inc()

      yield {
          "title": title,
          "text": text
      }

    return (
        pipeline
        | beam.Create(filepaths)
        | beam.FlatMap(_extract_content)
        | beam.FlatMap(_clean_content)
    )
426529
def _convert_dataset_files(value):
    """"""
    output = {}
    for k, v in value.items():
        inst = DatasetFile.from_jsonld(v)
        output[inst.path] = inst
    return output
170549
def window_iterator(data, width):
    
    start = 0
    while start < len(data):
        yield data[start:start+width]
        start += width
164662
def export_datasources(print_stdout, datasource_file,
                       back_references, include_defaults):
    """"""
    data = dict_import_export.export_to_dict(
        session=db.session,
        recursive=True,
        back_references=back_references,
        include_defaults=include_defaults)
    if print_stdout or not datasource_file:
        yaml.safe_dump(data, stdout, default_flow_style=False)
    if datasource_file:
        logging.info('Exporting datasources to %s', datasource_file)
        with open(datasource_file, 'w') as data_stream:
            yaml.safe_dump(data, data_stream, default_flow_style=False)