Parse a workflow into cwl2wdl style dictionary.
508409
def _wf_to_dict(wf):
    
    inputs, outputs = _get_wf_inout(wf)
    out = {"name": _id_to_name(wf.tool["id"]).replace("-", "_"), "inputs": inputs,
           "outputs": outputs, "steps": [], "subworkflows": [],
           "requirements": []}
    for step in wf.steps:
        inputs, outputs = _get_step_inout(step)
        inputs, scatter = _organize_step_scatter(step, inputs)
        if isinstance(step.embedded_tool, cwltool.workflow.Workflow):
            wf_def = _wf_to_dict(step.embedded_tool)
            out["subworkflows"].append({"id": wf_def["name"], "definition": wf_def,
                                        "inputs": inputs, "outputs": outputs, "scatter": scatter})
        else:
            task_def = _tool_to_dict(step.embedded_tool)
            out["steps"].append({"task_id": task_def["name"], "task_definition": task_def,
                                 "inputs": inputs, "outputs": outputs, "scatter": scatter})
    return out
410039
def parse_workflow_call_body(self, i):
        
        io_map = OrderedDict()

        if isinstance(i, wdl_parser.Terminal):
            return i.source_string # no io mappings; represents just a blank call
        elif isinstance(i, wdl_parser.Ast):
            if i.name == 'CallBody':
                declarations = self.parse_workflow_call_body_declarations(i.attr("declarations")) # have not seen this used
                io_map = self.parse_workflow_call_body_io(i.attr('io'))
            else:
                raise NotImplementedError
        elif isinstance(i, wdl_parser.AstList):
            raise NotImplementedError

        return io_map
428073
def to_dict(self):
        
        return {
            'name': self.name,
            'id': self.id,
            'type': self.type,
            'workflow_id': self.workflow_id,
            'queue': self.queue,
            'start_time': self.start_time,
            'arguments': self.arguments,
            'acknowledged': self.acknowledged,
            'func_name': self.func_name,
            'hostname': self.hostname,
            'worker_name': self.worker_name,
            'worker_pid': self.worker_pid,
            'routing_key': self.routing_key
        }
460263
def load_workflow(self, workflow_id):
        
        with switch_db(WorkflowDefinitionModel, db_alias='hyperstream'):
            workflow_definition = WorkflowDefinitionModel.objects.get(workflow_id=workflow_id)
            if not workflow_definition:
                logging.warn("Attempted to load workflow with id {}, but not found".format(workflow_id))

            workflow = Workflow(
                workflow_id=workflow_id,
                name=workflow_definition.name,
                description=workflow_definition.description,
                owner=workflow_definition.owner,
                online=workflow_definition.online,
                monitor=workflow_definition.monitor
            )

            for n in workflow_definition.nodes:
                workflow.create_node(
                    stream_name=n.stream_name,
                    channel=self.channel_manager.get_channel(n.channel_id),
                    plates=[self.plate_manager.plates[p] for p in n.plate_ids])

            for f in workflow_definition.factors:
                source_nodes = [workflow.nodes[node_id] for node_id in f.sources] if f.sources else []
                sink_nodes = [workflow.nodes[node_id] for node_id in f.sinks] if f.sinks else []
                alignment_node = workflow.nodes[f.alignment_node] if f.alignment_node else None
                splitting_node = workflow.nodes[f.splitting_node] if f.splitting_node else None
                output_plate = f.output_plate

                parameters = Tool.parameters_from_model(f.tool.parameters)
                # tool = dict(name=f.tool.name, parameters=parameters)
                tool = self.channel_manager.get_tool(f.tool.name, parameters, version=None)

                if f.factor_type == "Factor":
                    if len(sink_nodes) != 1:
                        raise ValueError(
                            "Standard factors should have a single sink node, received {}"
                            .format(len(sink_nodes)))

                    if splitting_node is not None:
                        raise ValueError(
                            "Standard factors do not support splitting nodes")

                    if output_plate is not None:
                        raise ValueError(
                            "Standard factors do not support output plates")

                    workflow.create_factor(
                        tool=tool,
                        sources=source_nodes,
                        sink=sink_nodes[0],
                        alignment_node=alignment_node
                    )

                elif f.factor_type == "MultiOutputFactor":
                    if len(source_nodes) > 1:
                        raise ValueError(
                            "MultiOutputFactor factors should have at most one source node, received {}"
                            .format(len(source_nodes)))

                    if len(sink_nodes) != 1:
                        raise ValueError(
                            "MultiOutputFactor factors should have a single sink node, received {}"
                            .format(len(sink_nodes)))

                    if alignment_node is not None:
                        raise ValueError(
                            "MultiOutputFactor does not support alignment nodes")

                    if output_plate is not None:
                        raise ValueError(
                            "MultiOutputFactor does not support output plates")

                    workflow.create_multi_output_factor(
                        tool=tool,
                        source=source_nodes[0] if source_nodes else None,
                        splitting_node=splitting_node,
                        sink=sink_nodes[0]
                    )

                elif f.factor_type == "NodeCreationFactor":
                    if len(source_nodes) > 1:
                        raise ValueError(
                            "NodeCreationFactor factors should no more than one source node, received {}"
                            .format(len(source_nodes)))

                    if len(sink_nodes) != 0:
                        raise ValueError(
                            "NodeCreationFactor factors should not have sink nodes"
                            .format(len(sink_nodes)))

                    if output_plate is None:
                        raise ValueError(
                            "NodeCreationFactor requires an output plate definition")

                    workflow.create_node_creation_factor(
                        tool=tool,
                        source=source_nodes[0] if source_nodes else None,
                        output_plate=output_plate.to_mongo().to_dict(),
                        plate_manager=self.plate_manager
                    )

                else:
                    raise NotImplementedError("Unsupported factor type {}".format(f.factor_type))

            self.add_workflow(workflow, False)
            return workflow
310451
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'workspaces') and self.workspaces is not None:
            _dict['workspaces'] = [x._to_dict() for x in self.workspaces]
        if hasattr(self, 'pagination') and self.pagination is not None:
            _dict['pagination'] = self.pagination._to_dict()
        return _dict
469656
def get_worfklow_spec(self):
        
        # TODO: convert from in-process to redis based caching
        if self.current.workflow_name not in self.workflow_spec_cache:
            # path = self.find_workflow_path()
            # spec_package = InMemoryPackager.package_in_memory(self.current.workflow_name, path)
            # spec = BpmnSerializer().deserialize_workflow_spec(spec_package)

            try:
                self.current.wf_object = BPMNWorkflow.objects.get(name=self.current.workflow_name)
            except ObjectDoesNotExist:
                self.current.wf_object = BPMNWorkflow.objects.get(name='not_found')
                self.current.task_data['non-existent-wf'] = self.current.workflow_name
                self.current.workflow_name = 'not_found'
            xml_content = self.current.wf_object.xml.body
            spec = ZopsSerializer().deserialize_workflow_spec(xml_content, self.current.workflow_name)

            spec.wf_id = self.current.wf_object.key
            self.workflow_spec_cache[self.current.workflow_name] = spec
        return self.workflow_spec_cache[self.current.workflow_name]
258760
def to_obj(self, wd=False, pack=False, relpath=None):
        
        self._closed()

        obj = CommentedMap()
        obj['cwlVersion'] = 'v1.0'
        obj['class'] = 'Workflow'
        try:
            obj['doc'] = self.documentation
        except (AttributeError, ValueError):
            pass
        try:
            obj['label'] = self.label
        except (AttributeError, ValueError):
            pass
        if self._has_requirements():
            obj['requirements'] = []
        if self.has_workflow_step:
            obj['requirements'].append(
                {'class': 'SubworkflowFeatureRequirement'})
        if self.has_scatter_requirement:
            obj['requirements'].append({'class': 'ScatterFeatureRequirement'})
        if self.has_multiple_inputs:
            obj['requirements'].append(
                {'class': 'MultipleInputFeatureRequirement'})
        obj['inputs'] = self.wf_inputs
        obj['outputs'] = self.wf_outputs

        steps_obj = CommentedMap()
        for key in self.wf_steps:
            steps_obj[key] = self.wf_steps[key].to_obj(relpath=relpath,
                                                       pack=pack,
                                                       wd=wd)
        obj['steps'] = steps_obj

        return obj
543355
def get_version(extension, workflow_file):
    ''''''
    if extension == 'py' and two_seven_compatible(workflow_file):
        return '2.7'
    elif extension == 'cwl':
        return yaml.load(open(workflow_file))['cwlVersion']
    else:  # Must be a wdl file.
        # Borrowed from https://github.com/Sage-Bionetworks/synapse-orchestrator/blob/develop/synorchestrator/util.py#L142
        try:
            return [l.lstrip('version') for l in workflow_file.splitlines() if 'version' in l.split(' ')][0]
        except IndexError:
            return 'draft-2'
791698
def _preprocess(project_dict):
        
        handlers = {
            ('archive',): _list_if_none,
            ('on-run-start',): _list_if_none_or_string,
            ('on-run-end',): _list_if_none_or_string,
        }

        for k in ('models', 'seeds'):
            handlers[(k,)] = _dict_if_none
            handlers[(k, 'vars')] = _dict_if_none
            handlers[(k, 'pre-hook')] = _list_if_none_or_string
            handlers[(k, 'post-hook')] = _list_if_none_or_string
        handlers[('seeds', 'column_types')] = _dict_if_none

        def converter(value, keypath):
            if keypath in handlers:
                handler = handlers[keypath]
                return handler(value)
            else:
                return value

        return deep_map(converter, project_dict)
661971
def _cwlvar_to_wdl(var):
    
    if isinstance(var, (list, tuple)):
        return [_cwlvar_to_wdl(x) for x in var]
    elif isinstance(var, dict):
        assert var.get("class") == "File", var
        # XXX handle secondary files
        return var.get("path") or var["value"]
    else:
        return var
81447
def _to_cwldata(key, val, get_retriever):
    
    out = []
    if isinstance(val, dict):
        if len(val) == 2 and "base" in val and "indexes" in val:
            if len(val["indexes"]) > 0 and val["base"] == val["indexes"][0]:
                out.append(("%s__indexes" % key, _item_to_cwldata(val["base"], get_retriever)))
            else:
                out.append((key, _to_cwlfile_with_indexes(val, get_retriever)))
        # Dump shared nested keys like resources as a JSON string
        elif key in workflow.ALWAYS_AVAILABLE or key in workflow.STRING_DICT:
            out.append((key, _item_to_cwldata(json.dumps(val), get_retriever)))
        elif key in workflow.FLAT_DICT:
            flat = []
            for k, vs in val.items():
                if not isinstance(vs, (list, tuple)):
                    vs = [vs]
                for v in vs:
                    flat.append("%s:%s" % (k, v))
            out.append((key, _item_to_cwldata(flat, get_retriever)))
        else:
            remain_val = {}
            for nkey, nval in val.items():
                cur_nkey = "%s__%s" % (key, nkey)
                cwl_nval = _item_to_cwldata(nval, get_retriever)
                if isinstance(cwl_nval, dict):
                    out.extend(_to_cwldata(cur_nkey, nval, get_retriever))
                elif key in workflow.ALWAYS_AVAILABLE:
                    remain_val[nkey] = nval
                else:
                    out.append((cur_nkey, cwl_nval))
            if remain_val:
                out.append((key, json.dumps(remain_val, sort_keys=True, separators=(',', ':'))))
    else:
        out.append((key, _item_to_cwldata(val, get_retriever)))
    return out
594291
def s(text, flow_expression, workitem)

      if text.is_a?(String)

        literal_sub(
          Rufus.dsub(text, dict_class.new(flow_expression, workitem)),
          flow_expression,
          workitem)

      elsif text.is_a?(Array)

        text.collect { |e| s(e, flow_expression, workitem) }

      elsif text.is_a?(Hash)

        text.remap { |(k, v), h|
          h[s(k, flow_expression, workitem)] = s(v, flow_expression, workitem)
        }

      else

        text
      end
    end
5395
def get_parsed_context(context_arg):
    """"""
    assert context_arg, ("pipeline must be invoked with context arg set. For "
                         "this json parser you're looking for something "
                         "like: "
                         "pypyr pipelinename './myjsonfile.json'")
    logger.debug("starting")
    # open the json file on disk so that you can initialize the dictionary
    logger.debug(f"attempting to open file: {context_arg}")
    with open(context_arg) as json_file:
        payload = json.load(json_file)

    logger.debug(f"json file loaded into context. Count: {len(payload)}")
    logger.debug("done")
    return payload
695539
def _organize_tools_on(data, is_cwl):
    
    # want tools_on: [gvcf] if joint calling specified in CWL
    if is_cwl:
        if tz.get_in(["algorithm", "jointcaller"], data):
            val = tz.get_in(["algorithm", "tools_on"], data)
            if not val:
                val = []
            if not isinstance(val, (list, tuple)):
                val = [val]
            if "gvcf" not in val:
                val.append("gvcf")
            data["algorithm"]["tools_on"] = val
    return data
452576
def get_job_logs_from_workflow(workflow_id):
    """"""
    query_result = (
        db.session.query(
            models.CrawlerJob.logs,
        )
        .join(
            models.CrawlerWorkflowObject,
            models.CrawlerJob.job_id == models.CrawlerWorkflowObject.job_id,
        )
        .filter(models.CrawlerWorkflowObject.object_id == workflow_id)
        .one_or_none()
    )

    if query_result is None:
        click.secho(
            (
                "Workflow %s was not found, maybe it's not a crawl workflow?" %
                workflow_id
            ),
            fg='yellow',
        )
        sys.exit(1)

    _show_file(
        file_path=query_result[0],
        header_name='Log',
    )
269848
def Parse(self, stat, knowledge_base):
    """"""
    value = stat.registry_data.GetValue()
    if not value:
      raise parser.ParseError("Invalid value for key %s" % stat.pathspec.path)
    value = artifact_utils.ExpandWindowsEnvironmentVariables(
        value, knowledge_base)
    if value:
      yield rdfvalue.RDFString(value)
1309364
def get_workflow(self):
        """"""
        extra_context = self.get_initial()
        entry_point = self.request.GET.get("step", None)
        workflow = self.workflow_class(self.request,
                                       context_seed=extra_context,
                                       entry_point=entry_point)
        return workflow