CWL target for quantitation.

    XXX Needs to be split and parallelized by expression caller, with merging
    of multiple calls.
2808
def quantitate(data):
    
    data = to_single_data(to_single_data(data))
    data = generate_transcript_counts(data)[0][0]
    data["quant"] = {}
    if "sailfish" in dd.get_expression_caller(data):
        data = to_single_data(sailfish.run_sailfish(data)[0])
        data["quant"]["tsv"] = data["sailfish"]
        data["quant"]["hdf5"] = os.path.join(os.path.dirname(data["sailfish"]), "abundance.h5")
    if ("kallisto" in dd.get_expression_caller(data) or "pizzly" in dd.get_fusion_caller(data, [])):
        data = to_single_data(kallisto.run_kallisto_rnaseq(data)[0])
        data["quant"]["tsv"] = os.path.join(data["kallisto_quant"], "abundance.tsv")
        data["quant"]["hdf5"] = os.path.join(data["kallisto_quant"], "abundance.h5")
    if (os.path.exists(os.path.join(data["kallisto_quant"], "fusion.txt"))):
        data["quant"]["fusion"] = os.path.join(data["kallisto_quant"], "fusion.txt")
    else:
        data["quant"]["fusion"] = None
    if "salmon" in dd.get_expression_caller(data):
        data = to_single_data(salmon.run_salmon_reads(data)[0])
        data["quant"]["tsv"] = data["salmon"]
        data["quant"]["hdf5"] = os.path.join(os.path.dirname(data["salmon"]), "abundance.h5")
    return [[data]]
1100426
@Procedure(mode = Mode.WRITE)
    @Description("apoc.periodic.iterate('statement returning items', 'statement per item', {batchSize:1000,iterateList:true,parallel:false,params:{},concurrency:50,retries:0}) YIELD batches, total - run the second statement for each item returned by the first statement. Returns number of batches and total processed rows")
    public Stream<BatchAndTotalResult> iterate(
            @Name("cypherIterate") String cypherIterate,
            @Name("cypherAction") String cypherAction,
            @Name("config") Map<String,Object> config) {

        long batchSize = Util.toLong(config.getOrDefault("batchSize", 10000));
        int concurrency = Util.toInteger(config.getOrDefault("concurrency", 50));
        boolean parallel = Util.toBoolean(config.getOrDefault("parallel", false));
        boolean iterateList = Util.toBoolean(config.getOrDefault("iterateList", true));
        long retries = Util.toLong(config.getOrDefault("retries", 0)); // todo sleep/delay or push to end of batch to try again or immediate ?
        Map<String,Object> params = (Map)config.getOrDefault("params", Collections.emptyMap());
        int failedParams = Util.toInteger(config.getOrDefault("failedParams", -1));
        try (Result result = db.execute(slottedRuntime(cypherIterate),params)) {
            Pair<String,Boolean> prepared = prepareInnerStatement(cypherAction, iterateList, result.columns(), "_batch");
            String innerStatement = prepared.first();
            iterateList=prepared.other();
            log.info("starting batching from `%s` operation using iteration `%s` in separate thread", cypherIterate,cypherAction);
            return iterateAndExecuteBatchedInSeparateThread((int)batchSize, parallel, iterateList, retries, result, (p) -> db.execute(innerStatement, merge(params, p)).close(), concurrency, failedParams);
        }
    }
848059
def main(self):
        
        # Find the target files
        self.targets()
        kmer = 15 if self.analysistype == 'GDCS' else 17
        # Use bbduk to bait the FASTQ reads matching the target sequences
        self.bait(maskmiddle='t', k=kmer)
        # If desired, use bbduk to bait the target sequences with the previously baited FASTQ files
        if self.revbait:
            self.reversebait(maskmiddle='t', k=kmer)
        # Run the bowtie2 read mapping module
        self.mapping()
        # Use samtools to index the sorted bam file
        self.indexing()
        # Parse the results
        # self.parsing()
        self.parsebam()
1162838
public void process( FastQueue<AssociatedIndex> matches , int numSource , int numDestination ) {

		if( checkSource ) {
			if( checkDestination ) {
				processSource(matches, numSource, firstPass);
				processDestination(firstPass,numDestination,pruned);
			} else {
				processSource(matches, numSource, pruned);
			}
		} else if( checkDestination ) {
			processDestination(matches,numDestination,pruned);
		} else {
			// well this was pointless, just return the input set
			pruned = matches;
		}
	}
1223310
ArrayCreationExpr juggleArrayCreation(TokenRange range, List<TokenRange> levelRanges, Type type, NodeList<Expression> dimensions, List<NodeList<AnnotationExpr>> arrayAnnotations, ArrayInitializerExpr arrayInitializerExpr) {
        NodeList<ArrayCreationLevel> levels = new NodeList<>();

        for (int i = 0; i < arrayAnnotations.size(); i++) {
            levels.add(new ArrayCreationLevel(levelRanges.get(i), dimensions.get(i), arrayAnnotations.get(i)));
        }
        return new ArrayCreationExpr(range, type, levels, arrayInitializerExpr);
    }
538294
private String performSplitting(String source, String splitString, String indexString)
        throws TransformationOperationException {
        try {
            Integer index = Integer.parseInt(indexString);
            return source.split(splitString)[index];
        } catch (NumberFormatException e) {
            throw new TransformationOperationException("The given result index parameter is not a number");
        } catch (IndexOutOfBoundsException e) {
            throw new TransformationOperationException(
                "The split result does not have that much results for the index parameter");
        }
    }
1054402
def _upstream_analysis(self, compile_contexts, classpath_entries):
    """"""
    # Reorganize the compile_contexts by class directory.
    compile_contexts_by_directory = {}
    for compile_context in compile_contexts.values():
      compile_context = self.select_runtime_context(compile_context)
      compile_contexts_by_directory[compile_context.classes_dir.path] = compile_context
    # If we have a compile context for the target, include it.
    for entry in classpath_entries:
      path = entry.path
      if not path.endswith('.jar'):
        compile_context = compile_contexts_by_directory.get(path)
        if not compile_context:
          self.context.log.debug('Missing upstream analysis for {}'.format(path))
        else:
          yield compile_context.classes_dir.path, compile_context.analysis_file
707833
private String performSplitting(String source, String splitString, String indexString)
        throws TransformationOperationException {
        Integer index = parseIntString(indexString, false, 0);
        Matcher matcher = generateMatcher(splitString, source);
        for (int i = 0; i <= index; i++) {
            matcher.find();
        }
        String value = matcher.group();
        if (value == null) {
            getLogger().warn("No result for given regex and index. The empty string will be taken instead.");
            value = "";
        }
        return value;
    }
666663
function Bformat_analyser(audioCtx)
{
    this.initialized = false;
    
    this.ctx = audioCtx;
    this.fftSize = 2048;
    this.analysers = new Array(4);
    this.analBuffers = new Array(4);
    // Input and output nodes
    this.in = this.ctx.createChannelSplitter(4);
    this.out = this.ctx.createChannelMerger(4);
    // Initialize analyzer buffers
    for (var i=0; i<4; i++) {
        this.analysers[i] = this.ctx.createAnalyser();
        this.analysers[i].fftSize = this.fftSize;
        this.analysers[i].smoothingTimeConstant = 0;
        this.analBuffers[i] = new Float32Array(this.fftSize);
    }
    // Create connections
    for (var i=0; i<4; i++) {
        this.in.connect(this.out, i, i);
        this.in.connect(this.analysers[i], i, 0);
    }
    
    this.initialized = true;
}
1093759
public void analyze(Analyzer analyzer, boolean allowFlow)
    throws Exception
  {
    _pendingTargets = new IntArray();
    _completedTargets = new IntArray();
    
    analyzeImpl(analyzer, allowFlow, _pendingTargets, _completedTargets);
  }
410995
StatementDMQL compileMergeStatement(RangeVariable[] outerRanges) {

        boolean[]     insertColumnCheckList;
        int[]         insertColumnMap = null;
        int[]         updateColumnMap = null;
        int[]         baseUpdateColumnMap;
        Table         table;
        RangeVariable targetRange;
        RangeVariable sourceRange;
        Expression    mergeCondition;
        HsqlArrayList updateList        = new HsqlArrayList();
        Expression[]  updateExpressions = null;
        HsqlArrayList insertList        = new HsqlArrayList();
        Expression    insertExpression  = null;

        read();
        readThis(Tokens.INTO);

        targetRange = readSimpleRangeVariable(StatementTypes.MERGE);
        table       = targetRange.rangeTable;

        readThis(Tokens.USING);

        sourceRange = readTableOrSubquery();

        // parse ON search conditions
        readThis(Tokens.ON);

        mergeCondition = XreadBooleanValueExpression();

        if (mergeCondition.getDataType() != Type.SQL_BOOLEAN) {
            throw Error.error(ErrorCode.X_42568);
        }

        RangeVariable[] fullRangeVars   = new RangeVariable[] {
            sourceRange, targetRange
        };
        RangeVariable[] sourceRangeVars = new RangeVariable[]{ sourceRange };
        RangeVariable[] targetRangeVars = new RangeVariable[]{ targetRange };

        // parse WHEN clause(s) and convert lists to arrays
        insertColumnMap       = table.getColumnMap();
        insertColumnCheckList = table.getNewColumnCheckList();

        OrderedHashSet updateColNames = new OrderedHashSet();
        OrderedHashSet insertColNames = new OrderedHashSet();

        readMergeWhen(insertColNames, updateColNames, insertList, updateList,
                      targetRangeVars, sourceRange);

        if (insertList.size() > 0) {
            int colCount = insertColNames.size();

            if (colCount != 0) {
                insertColumnMap = table.getColumnIndexes(insertColNames);
                insertColumnCheckList =
                    table.getColumnCheckList(insertColumnMap);
            }

            insertExpression = (Expression) insertList.get(0);

            setParameterTypes(insertExpression, table, insertColumnMap);
        }

        if (updateList.size() > 0) {
            updateExpressions = new Expression[updateList.size()];

            updateList.toArray(updateExpressions);

            updateColumnMap = table.getColumnIndexes(updateColNames);
        }

        if (updateExpressions != null) {
            Table baseTable = table.getBaseTable();

            baseUpdateColumnMap = updateColumnMap;

            if (table != baseTable) {
                baseUpdateColumnMap = new int[updateColumnMap.length];

                ArrayUtil.projectRow(table.getBaseTableColumnMap(),
                                     updateColumnMap, baseUpdateColumnMap);
            }

            resolveUpdateExpressions(table, sourceRangeVars, updateColumnMap,
                                     updateExpressions, outerRanges);
        }

        HsqlList unresolved = null;

        unresolved = mergeCondition.resolveColumnReferences(fullRangeVars,
                null);

        ExpressionColumn.checkColumnsResolved(unresolved);
        mergeCondition.resolveTypes(session, null);

        if (mergeCondition.isParam()) {
            mergeCondition.dataType = Type.SQL_BOOLEAN;
        }

        if (mergeCondition.getDataType() != Type.SQL_BOOLEAN) {
            throw Error.error(ErrorCode.X_42568);
        }

        RangeVariableResolver resolver =
            new RangeVariableResolver(fullRangeVars, mergeCondition,
                                      compileContext);

        resolver.processConditions();

        fullRangeVars = resolver.rangeVariables;

        if (insertExpression != null) {
            unresolved =
                insertExpression.resolveColumnReferences(sourceRangeVars,
                    unresolved);

            ExpressionColumn.checkColumnsResolved(unresolved);
            insertExpression.resolveTypes(session, null);
        }

        StatementDMQL cs = new StatementDML(session, fullRangeVars,
                                            insertColumnMap, updateColumnMap,
                                            insertColumnCheckList,
                                            mergeCondition, insertExpression,
                                            updateExpressions, compileContext);

        return cs;
    }
551995
@Override
    public <T extends Aggregate> void exec(Batch<T> batch) {
        //profilingHookIn(batch);

        IntPointer pointer = (IntPointer) getPointer(batch);

        int maxTypes = 5;

        int maxIntArrays = batch.getSample().maxIntArrays();

        int maxArraySize = batch.getSample().maxIntArraySize();


        int indexPos = maxTypes * Batch.getBatchLimit();
        int intArraysPos = indexPos + (batch.getSample().maxIndexArguments() * Batch.getBatchLimit());
        int realPos = (intArraysPos + (maxIntArrays * maxArraySize * Batch.getBatchLimit()))
                / (Nd4j.dataType() == DataType.DOUBLE ? 2 : 1);
        int argsPos = (realPos + ((batch.getSample().maxRealArguments() * Batch.getBatchLimit())))
                / (Nd4j.dataType() == DataType.DOUBLE ? 1 : 2);
        int shapesPos = argsPos + (batch.getSample().maxArguments() * Batch.getBatchLimit());
        DataType dataType = null;
        for (int i = 0; i < batch.getNumAggregates(); i++) {
            T op = batch.getAggregates().get(i);

            if (i == 0)
                dataType = op.getArguments().get(0).dataType();

            // put num arguments
            int idx = i * maxTypes;
            pointer.put(idx, op.getArguments().size());
            pointer.put(idx + 1, op.getShapes().size());
            pointer.put(idx + 2, op.getIndexingArguments().size());
            pointer.put(idx + 3, op.getRealArguments().size());
            pointer.put(idx + 4, op.getIntArrayArguments().size());


            // putting indexing arguments
            for (int e = 0; e < op.getIndexingArguments().size(); e++) {
                idx = indexPos + i * batch.getSample().maxIndexArguments();
                pointer.put(idx + e, op.getIndexingArguments().get(e));
            }

            // putting intArray values
            int bsize = maxIntArrays * maxArraySize;
            for (int e = 0; e < op.getIntArrayArguments().size(); e++) {
                int step = (i * bsize) + (e * maxArraySize);
                if (op.getIntArrayArguments().get(e) != null)
                    for (int x = 0; x < op.getIntArrayArguments().get(e).length; x++) {
                        idx = intArraysPos + step + x;
                        pointer.put(idx, op.getIntArrayArguments().get(e)[x]);
                    }
            }

            // TODO: variable datatype should be handled here
            // putting real arguments

            switch (dataType){
                case FLOAT:
                    FloatPointer fPtr = new FloatPointer(pointer);
                    for (int e = 0; e < op.getRealArguments().size(); e++) {
                        idx = realPos + i * op.maxRealArguments();
                        fPtr.put(idx + e, op.getRealArguments().get(e).floatValue());
                    }
                    break;
                case DOUBLE:
                    DoublePointer dPtr = new DoublePointer(pointer);
                    for (int e = 0; e < op.getRealArguments().size(); e++) {
                        idx = realPos + (i * op.maxRealArguments());
                        dPtr.put(idx + e, op.getRealArguments().get(e).doubleValue());
                    }
                    break;
                default:
                    throw new ND4JIllegalArgumentException("Only FLOAT and DOUBLE datatypes are supported");
            }

            if (extraz.get() == null)
                extraz.set(new PointerPointer(32));

            // putting arguments pointers

            PointerPointer ptrPtr = new PointerPointer(pointer);//extraz.get().put(pointer);

            for (int e = 0; e < op.getArguments().size(); e++) {
                idx = argsPos + i * batch.getSample().maxArguments();

                if (op.getArguments().get(e) != null) {
                    ptrPtr.put(idx + e, op.getArguments().get(e).data().addressPointer());
                }
            }


            // putting shape pointers
            for (int e = 0; e < op.getShapes().size(); e++) {
                idx = shapesPos + i * batch.getSample().maxShapes();

                if (op.getShapes().get(e) != null)
                    ptrPtr.put(idx + e, op.getShapes().get(e).addressPointer());
            }
        }

        loop.execAggregateBatch(null, batch.getNumAggregates(), batch.opNum(),
                    batch.getSample().maxArguments(), batch.getSample().maxShapes(),
                    batch.getSample().maxIntArrays(), batch.getSample().maxIntArraySize(),
                    batch.getSample().maxIndexArguments(), batch.getSample().maxRealArguments(), pointer, FlatBuffersMapper.getDataTypeAsByte(dataType));

    }
672573
protected void analyzeImpl(Analyzer analyzer, boolean allowFlow,
                             IntArray pendingTargets,
                             IntArray completedTargets)
    throws Exception
  {
    setOffset(0);
    
    pendingTargets.add(0);

    ArrayList<CodeAttribute.ExceptionItem> exns;
    exns = getExceptions();

    for (int i = 0; i < exns.size(); i++) {
      CodeAttribute.ExceptionItem exn = exns.get(i);

      pendingTargets.add(exn.getHandler());
    }
    
    analyze(analyzer, allowFlow, pendingTargets, completedTargets);
  }
985662
def writeExpression(self, rnaQuantificationId, quantfilename):
        
        isNormalized = self._isNormalized
        units = self._units
        with open(quantfilename, "r") as quantFile:
            quantificationReader = csv.reader(quantFile, delimiter=b"\t")
            header = next(quantificationReader)
            expressionLevelColNum = self.setColNum(
                                        header, self._expressionLevelCol)
            nameColNum = self.setColNum(header, self._nameCol)
            countColNum = self.setColNum(header, self._countCol, -1)
            confColLowNum = self.setColNum(header, self._confColLow, -1)
            confColHiNum = self.setColNum(header, self._confColHi, -1)
            expressionId = 0
            for expression in quantificationReader:
                expressionLevel = expression[expressionLevelColNum]
                name = expression[nameColNum]
                rawCount = 0.0
                if countColNum != -1:
                    rawCount = expression[countColNum]
                confidenceLow = 0.0
                confidenceHi = 0.0
                score = 0.0
                if confColLowNum != -1 and confColHiNum != -1:
                    confidenceLow = float(expression[confColLowNum])
                    confidenceHi = float(expression[confColHiNum])
                    score = (confidenceLow + confidenceHi)/2

                datafields = (expressionId, rnaQuantificationId, name,
                              expressionLevel, isNormalized, rawCount, score,
                              units, confidenceLow, confidenceHi)
                self._db.addExpression(datafields)
                expressionId += 1
            self._db.batchAddExpression()
275688
function mergeScan(accumulator, seed, concurrent) {
    if (concurrent === void 0) {
        concurrent = Number.POSITIVE_INFINITY;
    }
    return function (source) { return source.lift(new MergeScanOperator(accumulator, seed, concurrent)); };
}
980200
def combine_calls(*args):
    
    if len(args) == 3:
        is_cwl = False
        batch_id, samples, data = args
        caller_names, vrn_files = _organize_variants(samples, batch_id)
    else:
        is_cwl = True
        samples = [utils.to_single_data(x) for x in args]
        samples = [cwlutils.unpack_tarballs(x, x) for x in samples]
        data = samples[0]
        batch_id = data["batch_id"]
        caller_names = data["variants"]["variantcallers"]
        vrn_files = data["variants"]["calls"]
    logger.info("Ensemble consensus calls for {0}: {1}".format(
        batch_id, ",".join(caller_names)))
    edata = copy.deepcopy(data)
    base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble", batch_id))
    if any([vcfutils.vcf_has_variants(f) for f in vrn_files]):
        # Decompose multiallelic variants and normalize
        passonly = not tz.get_in(["config", "algorithm", "ensemble", "use_filtered"], edata, False)
        vrn_files = [normalize.normalize(f, data, passonly=passonly, rerun_effects=False, remove_oldeffects=True,
                                         nonrefonly=True,
                                         work_dir=utils.safe_makedir(os.path.join(base_dir, c)))
                     for c, f in zip(caller_names, vrn_files)]
        if "classifiers" not in (dd.get_ensemble(edata) or {}):
            callinfo = _run_ensemble_intersection(batch_id, vrn_files, caller_names, base_dir, edata)
        else:
            config_file = _write_config_file(batch_id, caller_names, base_dir, edata)
            callinfo = _run_ensemble(batch_id, vrn_files, config_file, base_dir,
                                     dd.get_ref_file(edata), edata)
            callinfo["vrn_file"] = vcfutils.bgzip_and_index(callinfo["vrn_file"], data["config"])
        # After decomposing multiallelic variants and normalizing, re-evaluate effects
        ann_ma_file, _ = effects.add_to_vcf(callinfo["vrn_file"], data)
        if ann_ma_file:
            callinfo["vrn_file"] = ann_ma_file

        edata["config"]["algorithm"]["variantcaller"] = "ensemble"
        edata["vrn_file"] = callinfo["vrn_file"]
        edata["ensemble_bed"] = callinfo["bed_file"]
        callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
    else:
        out_vcf_file = os.path.join(base_dir, "{0}-ensemble.vcf".format(batch_id))
        vcfutils.write_empty_vcf(out_vcf_file, samples=[dd.get_sample_name(d) for d in samples])
        callinfo = {"variantcaller": "ensemble",
                    "vrn_file": vcfutils.bgzip_and_index(out_vcf_file, data["config"]),
                    "bed_file": None}
    if is_cwl:
        callinfo["batch_samples"] = data["batch_samples"]
        callinfo["batch_id"] = batch_id
        return [{"ensemble": callinfo}]
    else:
        return [[batch_id, callinfo]]
746992
public function optimize(array $expression, Call $call, CompilationContext $context)
    {
        if (!isset($expression['parameters']) || 2 != \count($expression['parameters'])) {
            throw new CompilerException("'hash_equals' requires two parameters", $expression);
        }

        /*
         * Process the expected symbol to be returned
         */
        $call->processExpectedReturn($context);

        $symbolVariable = $call->getSymbolVariable(true, $context);
        if ($symbolVariable->isNotVariableAndString()) {
            throw new CompilerException('Returned values by functions can only be assigned to variant variables', $expression);
        }

        $context->headersManager->add('kernel/string');

        /**
         * Process parameters.
         */
        $resolvedParams = $call->getReadOnlyResolvedParams($expression['parameters'], $context, $expression);

        if ($call->mustInitSymbolVariable()) {
            $symbolVariable->initVariant($context);
        }

        return new CompiledExpression('bool', 'zephir_hash_equals('.$resolvedParams[0].', '.$resolvedParams[1].')', $expression);
    }