Creates data pipeline and adds definition
761407
def create_datapipeline(self):
        """"""
        utils.banner("Creating Data Pipeline")
        dpobj = datapipeline.AWSDataPipeline(app=self.app, env=self.env, region=self.region, prop_path=self.json_path)
        dpobj.create_datapipeline()
        dpobj.set_pipeline_definition()
        if self.configs[self.env].get('datapipeline').get('activate_on_deploy'):
            dpobj.activate_pipeline()
738495
def build_definitions(dpath="sqlalchemy_models/_definitions.json"):
    
    command.run(AlsoChildrenWalker, module=todo, outdir="sqlalchemy_models", definition_name="_definitions.json",
                relation_decision=RelationDesicion())
    pass  # skip due to unpatched issue in alchemyjsonschema run()
    definitions = ""
    for line in open(dpath, 'r'):
        definitions += line
    definitions = json.loads(definitions)
    newdef = open(dpath.replace("_def", "def"), 'w+')
    for name in definitions['definitions']:
        for mod in [um, em, wm, bm]:
            if hasattr(mod, name):
                model = getattr(mod, name)
                for attr in definitions['definitions'][name]['properties']:
                    if hasattr(getattr(model, attr).property, 'columns') and \
                            isinstance(getattr(model, attr).property.columns[0].type, LedgerAmount):
                        definitions['definitions'][name]['properties'][attr]['type'] = "number"
    newdef.write(json.dumps(definitions, indent=2))
    newdef.close()
1243918
def add_pipeline(subparsers):
    """"""
    pipeline_parser = subparsers.add_parser(
        'pipeline', help=add_pipeline.__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    pipeline_parser.set_defaults(func=pipeline_parser.print_help)

    pipeline_subparsers = pipeline_parser.add_subparsers(title='Pipelines')

    pipeline_full_parser = pipeline_subparsers.add_parser(
        'app', help=runner.prepare_app_pipeline.__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    pipeline_full_parser.set_defaults(func=runner.prepare_app_pipeline)

    pipeline_onetime_parser = pipeline_subparsers.add_parser(
        'onetime', help=runner.prepare_onetime_pipeline.__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    pipeline_onetime_parser.set_defaults(func=runner.prepare_onetime_pipeline)
    add_env(pipeline_onetime_parser)
802283
def append_pipeline(self, name, *args, **kwargs) -> Pipeline:
        
        return self.pipeline.append(name, *args, **kwargs)
1296051
def prepare_shell_data(self, shells, key, entry):
        """"""
        if self.can_process_shell(entry):
            if key in ['python']:
                entry['type'] = key

            if 'with' in entry and isinstance(entry['with'], str):
                rendered_with = ast.literal_eval(render(entry['with'],
                                                        variables=self.pipeline.variables,
                                                        model=self.pipeline.model,
                                                        env=self.get_merged_env(include_os=True)))
            elif 'with' in entry:
                rendered_with = entry['with']
            else:
                rendered_with = ['']

            for item in rendered_with:
                shells.append({
                    'id': self.next_task_id,
                    'creator': key,
                    'entry': entry,
                    'model': self.pipeline.model,
                    'env': self.get_merged_env(),
                    'item': item,
                    'dry_run': self.pipeline.options.dry_run,
                    'debug': self.pipeline.options.debug,
                    'strict': self.pipeline.options.strict,
                    'variables': self.pipeline.variables,
                    'temporary_scripts_path': self.pipeline.options.temporary_scripts_path})
                self.next_task_id += 1
1226906
Pipeline createPipeline(Language lang, Language motherTongue, TextChecker.QueryParams params, UserConfig userConfig)
    throws Exception { // package-private for mocking
    Pipeline lt = new Pipeline(lang, params.altLanguages, motherTongue, cache, userConfig);
    lt.setMaxErrorsPerWordRate(config.getMaxErrorsPerWordRate());
    if (config.getLanguageModelDir() != null) {
      lt.activateLanguageModelRules(config.getLanguageModelDir());
    }
    if (config.getWord2VecModelDir () != null) {
      lt.activateWord2VecModelRules(config.getWord2VecModelDir());
    }
    if (config.getRulesConfigFile() != null) {
      configureFromRulesFile(lt, lang);
    } else {
      configureFromGUI(lt, lang);
    }
    if (params.useQuerySettings) {
      Tools.selectRules(lt, new HashSet<>(params.disabledCategories), new HashSet<>(params.enabledCategories),
        new HashSet<>(params.disabledRules), new HashSet<>(params.enabledRules), params.useEnabledOnly);
    }
    if (pool != null) {
      lt.setupFinished();
    }
    return lt;
  }
483644
def _diff(old_pipeline_definition, new_pipeline_definition):
    '''
    
    '''
    old_pipeline_definition.pop('ResponseMetadata', None)
    new_pipeline_definition.pop('ResponseMetadata', None)

    diff = salt.utils.data.decode(difflib.unified_diff(
        salt.utils.json.dumps(old_pipeline_definition, indent=4).splitlines(True),
        salt.utils.json.dumps(new_pipeline_definition, indent=4).splitlines(True),
    ))
    return ''.join(diff)
1266369
function _pipeline (callback) {
  const that = this;
  if (typeof callback === 'function') {
    // output=file/buffer
    if (this._isStreamInput()) {
      // output=file/buffer, input=stream
      this.on('finish', function () {
        that._flattenBufferIn();
        sharp.pipeline(that.options, callback);
      });
    } else {
      // output=file/buffer, input=file/buffer
      sharp.pipeline(this.options, callback);
    }
    return this;
  } else if (this.options.streamOut) {
    // output=stream
    if (this._isStreamInput()) {
      // output=stream, input=stream
      if (this.streamInFinished) {
        this._flattenBufferIn();
        sharp.pipeline(this.options, function (err, data, info) {
          if (err) {
            that.emit('error', err);
          } else {
            that.emit('info', info);
            that.push(data);
          }
          that.push(null);
        });
      } else {
        this.on('finish', function () {
          that._flattenBufferIn();
          sharp.pipeline(that.options, function (err, data, info) {
            if (err) {
              that.emit('error', err);
            } else {
              that.emit('info', info);
              that.push(data);
            }
            that.push(null);
          });
        });
      }
    } else {
      // output=stream, input=file/buffer
      sharp.pipeline(this.options, function (err, data, info) {
        if (err) {
          that.emit('error', err);
        } else {
          that.emit('info', info);
          that.push(data);
        }
        that.push(null);
      });
    }
    return this;
  } else {
    // output=promise
    if (this._isStreamInput()) {
      // output=promise, input=stream
      return new Promise(function (resolve, reject) {
        that.on('finish', function () {
          that._flattenBufferIn();
          sharp.pipeline(that.options, function (err, data, info) {
            if (err) {
              reject(err);
            } else {
              if (that.options.resolveWithObject) {
                resolve({ data: data, info: info });
              } else {
                resolve(data);
              }
            }
          });
        });
      });
    } else {
      // output=promise, input=file/buffer
      return new Promise(function (resolve, reject) {
        sharp.pipeline(that.options, function (err, data, info) {
          if (err) {
            reject(err);
          } else {
            if (that.options.resolveWithObject) {
              resolve({ data: data, info: info });
            } else {
              resolve(data);
            }
          }
        });
      });
    }
  }
}
936178
def make_prediction_pipeline(pipeline, args):
  

  # DF bug: DF does not work with unicode strings
  predicted_values, errors = (
      pipeline |
      'Read CSV Files' >>
      beam.io.ReadFromText(str(args.predict_data),
                           strip_trailing_newlines=True) |
      'Batch Input' >>
      beam.ParDo(EmitAsBatchDoFn(args.batch_size)) |
      'Run TF Graph on Batches' >>
      beam.ParDo(RunGraphDoFn(args.trained_model_dir)).with_outputs('errors', main='main'))

  ((predicted_values, errors) |
   'Format and Save' >>
   FormatAndSave(args))
208284
private void buildDefinition(MarkupDocBuilder markupDocBuilder, String definitionName, Model model) {
        if (logger.isDebugEnabled()) {
            logger.debug("Definition processed : '{}'", definitionName);
        }
        if (config.isSeparatedDefinitionsEnabled()) {
            MarkupDocBuilder defDocBuilder = copyMarkupDocBuilder(markupDocBuilder);
            applyDefinitionComponent(defDocBuilder, definitionName, model);
            Path definitionFile = context.getOutputPath().resolve(definitionDocumentNameResolver.apply(definitionName));
            defDocBuilder.writeToFileWithoutExtension(definitionFile, StandardCharsets.UTF_8);
            if (logger.isDebugEnabled()) {
                logger.debug("Separate definition file produced : '{}'", definitionFile);
            }

            definitionRef(markupDocBuilder, definitionName);

        } else {
            applyDefinitionComponent(markupDocBuilder, definitionName, model);
        }
    }
283169
def preprocess(train_dataset, output_dir, eval_dataset, checkpoint, pipeline_option):
    """"""

    import apache_beam as beam
    import google.datalab.utils
    from . import _preprocess

    if checkpoint is None:
      checkpoint = _util._DEFAULT_CHECKPOINT_GSURL

    job_name = ('preprocess-image-classification-' +
                datetime.datetime.now().strftime('%y%m%d-%H%M%S'))

    staging_package_url = _util.repackage_to_staging(output_dir)
    tmpdir = tempfile.mkdtemp()
    # suppress DataFlow warnings about wheel package as extra package.
    original_level = logging.getLogger().getEffectiveLevel()
    logging.getLogger().setLevel(logging.ERROR)
    try:
      # Workaround for DataFlow 2.0, which doesn't work well with extra packages in GCS.
      # Remove when the issue is fixed and new version of DataFlow is included in Datalab.
      extra_packages = [staging_package_url, _TF_GS_URL, _PROTOBUF_GS_URL]
      local_packages = [os.path.join(tmpdir, os.path.basename(p))
                        for p in extra_packages]
      for source, dest in zip(extra_packages, local_packages):
        file_io.copy(source, dest, overwrite=True)

      options = {
          'staging_location': os.path.join(output_dir, 'tmp', 'staging'),
          'temp_location': os.path.join(output_dir, 'tmp'),
          'job_name': job_name,
          'project': _util.default_project(),
          'extra_packages': local_packages,
          'teardown_policy': 'TEARDOWN_ALWAYS',
          'no_save_main_session': True
      }
      if pipeline_option is not None:
        options.update(pipeline_option)

      opts = beam.pipeline.PipelineOptions(flags=[], **options)
      p = beam.Pipeline('DataflowRunner', options=opts)
      _preprocess.configure_pipeline(p, train_dataset, eval_dataset,
                                     checkpoint, output_dir, job_name)
      job_results = p.run()
    finally:
      shutil.rmtree(tmpdir)
      logging.getLogger().setLevel(original_level)

    if (_util.is_in_IPython()):
      import IPython
      dataflow_url = 'https://console.developers.google.com/dataflow?project=%s' % \
                     _util.default_project()
      html = 'Job "%s" submitted.' % job_name
      html += '<p>Click <a href="%s" target="_blank">here</a> to track preprocessing job. <br/>' \
          % dataflow_url
      IPython.display.display_html(html, raw=True)
    return google.datalab.utils.DataflowJob(job_results)
677573
def log_pipeline(self, pl):
        
        from datetime import datetime
        from ambry.etl.pipeline import CastColumns

        self.build_fs.makedir('pipeline', allow_recreate=True)

        try:
            ccp = pl[CastColumns]
            caster_code = ccp.pretty_code
        except Exception as e:
            caster_code = str(e)

        templ = u("""
Pipeline     : {}
run time     : {}
phase        : {}
source name  : {}
source table : {}
dest table   : {}
========================================================
{}

Pipeline Headers
================
{}

Caster Code
===========
{}

""")
        try:
            v = templ.format(pl.name, str(datetime.now()), pl.phase, pl.source_name, pl.source_table,
                             pl.dest_table, unicode(pl), pl.headers_report(), caster_code)
        except UnicodeError as e:
            v = ''
            self.error('Faled to write pipeline log for pipeline {} '.format(pl.name))

        path = os.path.join('pipeline', pl.phase + '-' + pl.file_name + '.txt')

        self.build_fs.makedir(os.path.dirname(path), allow_recreate=True, recursive=True)
        # LazyFS should handled differently because of:
        # TypeError: lazy_fs.setcontents(..., encoding='utf-8') got an unexpected keyword argument 'encoding'
        if isinstance(self.build_fs, LazyFS):
            self.build_fs.wrapped_fs.setcontents(path, v, encoding='utf8')
        else:
            self.build_fs.setcontents(path, v, encoding='utf8')
1236420
def receive(self, data, api_context):
        """"""
        self.log.debug(f"Putting data on the pipeline: {data}")
        result = {
            "api_contexts": self.api_contexts,
            "api_context": api_context,
            "strategy": dict(),  # Shared strategy data
            "result": data,
            "log_level": api_context["log_level"],
        }
        self.strat.execute(self.strategy_context_schema().load(result).data)
1163169
public void newDataSet(DataSetProviderType type,
                           RemoteCallback<DataSetDef> callback) throws Exception {
        dataSetDefServices.call(callback).createDataSetDef(type);
    }
775958
def _full_pipeline(self):
        """"""
        options = self._pipeline_options()
        full_pipeline = [{'$changeStream': options}]
        full_pipeline.extend(self._pipeline)
        return full_pipeline
621361
function loadSpecification(defPath, data) {
  const resolvedPath = path.resolve(defPath);

  const extName = path.extname(resolvedPath);
  const loader = LOADERS[extName];

  // Check whether the definition file is actually a usable file
  if (loader === undefined) {
    throw new Error('Definition file should be .js, .json, .yml or .yaml');
  }

  const swaggerDefinition = loader(data, resolvedPath);
  return swaggerDefinition;
}
867949
def build_source(self, stage, source, ps, force=False):
        """"""
        from ambry.bundle.process import call_interval

        assert source.is_processable, source.name

        if source.state == self.STATES.BUILT and not force:
            ps.update(message='Source {} already built'.format(source.name), state='skipped')
            return

        pl = self.pipeline(source, ps=ps)

        source.state = self.STATES.BUILDING

        # Doing this before hand to get at least some information about the pipline,
        # in case there is an error during the run. It will get overwritten with more information
        # after successful run
        self.log_pipeline(pl)

        try:

            source_name = source.name  # In case the source drops out of the session, which is does.
            s_vid = source.vid

            ps.update(message='Running pipeline {}'.format(pl.name), s_vid=s_vid, item_type='rows', item_count=0)

            @call_interval(5)
            def run_progress_f(sink_pipe, rows):
                (n_records, rate) = sink_pipe.report_progress()
                if n_records > 0:
                    ps.update(message='Running pipeline {}: rate: {}'
                              .format(pl.name, rate),
                              s_vid=s_vid, item_type='rows', item_count=n_records)


            pl.run(callback=run_progress_f)

            # Run the final routines at the end of the pipelin
            for f in pl.final:
                ps.update(message='Run final routine: {}'.format(f.__name__))
                f(pl)

            ps.update(message='Finished building source')

        except:
            self.log_pipeline(pl)
            raise

        self.commit()

        try:
            partitions = list(pl[ambry.etl.PartitionWriter].partitions)
            ps.update(message='Finalizing segment partition',
                      item_type='partitions', item_total=len(partitions), item_count=0)
            for i, p in enumerate(partitions):

                ps.update(message='Finalizing segment partition {}'.format(p.name), item_count=i, p_vid=p.vid)

                try:
                    p.finalize()
                except AttributeError:
                    print(self.table(p.table_name))
                    raise

                # FIXME Shouldn't need to do this commit, but without it, some stats get added multiple
                # times, causing an error later. Probably could be avoided by adding the stats to the
                # collection in the dataset

                self.commit()

        except IndexError:
            self.error("Pipeline didn't have a PartitionWriters, won't try to finalize")

        self.log_pipeline(pl)
        source.state = self.STATES.BUILT

        self.commit()

        return source.name