Check if a GenomicsDB output is incomplete and we should regenerate.

    Works around current inability to move GenomicsDB outputs and support
    transactional directories.
500223
def _incomplete_genomicsdb(dbdir):
    
    for test_file in ["callset.json", "vidmap.json", "genomicsdb_array/genomicsdb_meta.json"]:
        if not os.path.exists(os.path.join(dbdir, test_file)):
            return True
    return False
447446
def _fix_genotypes_object(self, genotypes, variant_info):
        """"""
        # Checking the name (if there were duplications)
        if self.has_index and variant_info.name != genotypes.variant.name:
            if not variant_info.name.startswith(genotypes.variant.name):
                raise ValueError("Index file not synced with IMPUTE2 file")
            genotypes.variant.name = variant_info.name

        # Trying to set multi-allelic information
        if self.has_index and self._index_has_location:
            # Location was in the index, so we can automatically set the
            # multi-allelic state of the genotypes
            genotypes.multiallelic = variant_info.multiallelic

        else:
            # Location was not in the index, so we check one marker before and
            # after the one we found
            logging.warning("Multiallelic variants are not detected on "
                            "unindexed files.")
614791
def check_recovery(working_dir):
    
    recovery_start_block, recovery_end_block = get_recovery_range(working_dir)
    if recovery_start_block is not None and recovery_end_block is not None:
        local_current_block = virtualchain_hooks.get_last_block(working_dir)
        if local_current_block <= recovery_end_block:
            return True

        # otherwise, we're outside the recovery range and we can clear it
        log.debug('Chain state is at block {}, and is outside the recovery window {}-{}'.format(local_current_block, recovery_start_block, recovery_end_block))
        clear_recovery_range(working_dir)
        return False

    else:
        # not recovering
        return False
228263
def _read_missing_lob_data_from_db(self, readoffset, readlength):
        """"""
        logger.debug('Reading missing lob data from db. Offset: %d, readlength: %d' % (readoffset, readlength))
        lob_data = self._make_read_lob_request(readoffset, readlength)

        # make sure we really got as many items (not bytes!) as requested:
        enc_lob_data = self._decode_lob_data(lob_data)
        assert readlength == len(enc_lob_data), 'expected: %d, received; %d' % (readlength, len(enc_lob_data))

        # jump to end of data, and append new and properly decoded data to it:
        # import pdb;pdb.set_trace()
        self.data.seek(0, SEEK_END)
        self.data.write(enc_lob_data)
        self._current_lob_length = len(self.data.getvalue())
416011
def check(self, init_if_missing=False):
        
        for d in self.path_overrides + [self.inputdir]:            
            if os.path.exists(d):
                self.allpaths.append(d)
        if len(self.allpaths) > 0:
            return(True)

        if init_if_missing:
            os.makedirs(self.inputdir)
            return True
1099301
def await_flush_completion(self, timeout=None):
        
        try:
            for batch in self._incomplete.all():
                log.debug('Waiting on produce to %s',
                          batch.produce_future.topic_partition)
                if not batch.produce_future.wait(timeout=timeout):
                    raise Errors.KafkaTimeoutError('Timeout waiting for future')
                if not batch.produce_future.is_done:
                    raise Errors.UnknownError('Future not done')

                if batch.produce_future.failed():
                    log.warning(batch.produce_future.exception)
        finally:
            self._flushes_in_progress.decrement()
1276232
public boolean isEveryOwnedOutputSpent(TransactionBag transactionBag) {
        for (TransactionOutput output : outputs) {
            if (output.isAvailableForSpending() && output.isMineOrWatched(transactionBag))
                return false;
        }
        return true;
    }
175772
def apply_quality_control_checks(
        seq,
        check_gen9_seqs=True,
        check_short_length=True,
        check_local_gc_content=True,
        check_global_gc_content=True):
    

    seq = seq.upper()
    failure_reasons = []
    
    # Minimum length
    if check_short_length:
        if len(seq) < min_gene_length:
            failure_reasons.append('minimum_length: Sequence is %d bp long and needs to be at least %d bp'%(len(seq),min_gene_length))

    # Maximum length
    if len(seq) > max_gene_length:
        failure_reasons.append('maximum_length: Sequence is %d bp long and needs to be shorter than %d bp'%(len(seq),max_gene_length))

    # Gen9 restricted sequences
    if check_gen9_seqs:
        for site in reserved_restriction_sites:
            pattern = dna.dna_to_re(site)
            reverse_site = dna.reverse_complement(site)
            reverse_pattern = dna.dna_to_re(reverse_site)

            if pattern.match(seq):
                failure_reasons.append('gen9_restricted_sequences: Reserved sequence %s is present'%(site))
            if reverse_pattern.match(seq):
                failure_reasons.append('gen9_restricted_sequences: Reverse-complement of reserved sequence %s is present'%(site))

    # Global GC content
    if check_global_gc_content:
        gc_content = dna.gc_content(seq)

        if gc_content < global_gc_content_min:
            failure_reasons.append('global_gc_content_min: Global GC content is %.3f%% and must be at least %.3f%%'%(gc_content,global_gc_content_min))

        if gc_content > global_gc_content_max:
            failure_reasons.append('global_gc_content_max: Global GC content is %.3f%% and must be less than %.3f%%'%(gc_content,global_gc_content_max))

    # Local GC content (windows)
    if check_local_gc_content:
        windows = [seq]

        if local_gc_window_size < len(seq):
            windows = dna.sliding_window(seq, local_gc_window_size)

        for seq_window in windows:
            lgc_content = dna.gc_content(seq_window)

            if lgc_content < local_gc_content_min:
                failure_reasons.append('local_gc_content_min: Local GC content is %.3f%% and must be at least %.3f%%'%(lgc_content,local_gc_content_min))
                break
            if lgc_content > local_gc_content_max:
                failure_reasons.append('local_gc_content_max: Local GC content is %.3f%% and must be less than %.3f%%'%(lgc_content,local_gc_content_max))
                break

    # Homopolymers
    for base in dna.dna_bases:
        homopolymer = base * homopolymer_max_lengths[base]
        if homopolymer in seq:
            failure_reasons.append('max_%s_homopolymer: %s'%(
                base.lower(), dna.case_highlight(seq,a_homopolymer)))

    # Make sure all the checks passed.
    if failure_reasons:
        intro = "The given sequence fails following Gen9 design guidelines:"
        raise ValueError('\n'.join([intro] + failure_reasons))
751401
def _run_genotype_gvcfs_genomicsdb(genomics_db, region, out_file, data):
    
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            broad_runner = broad.runner_from_config(data["config"])
            params = ["-T", "GenotypeGVCFs",
                      "--variant", "gendb://%s" % genomics_db,
                      "-R", dd.get_ref_file(data),
                      "--output", tx_out_file,
                      "-L", bamprep.region_to_gatk(region)]
            params += ["-ploidy", str(ploidy.get_ploidy([data], region))]
            # Avoid slow genotyping runtimes with improved quality score calculation in GATK4
            # https://gatkforums.broadinstitute.org/gatk/discussion/11471/performance-troubleshooting-tips-for-genotypegvcfs/p1
            params += ["--use-new-qual-calculator"]
            resources = config_utils.get_resources("gatk", data["config"])
            params += [str(x) for x in resources.get("options", [])]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.run_gatk(params, memscale=memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])
334886
function(result) {
        var errors, db;
        errors = [];

        if ('undefined' === typeof NDDB) {
            errors.push('NDDB not found.');
        }

        if ('undefined' === typeof JSUS) {
            errors.push('JSUS not found.');
        }

        if ('undefined' === typeof J) {
            errors.push('J not found.');
        }

        if ('undefined' === typeof node.window) {
            errors.push('node.window not found.');
        }

        if ('undefined' === typeof W) {
            errors.push('W not found.');
        }

        if ('undefined' === typeof node.widgets) {
            errors.push('node.widgets not found.');
        }

        if ('undefined' !== typeof NDDB) {
            try {
                db = new NDDB();
            }
            catch(e) {
                errors.push('An error occurred manipulating the NDDB object: ' +
                            e.message);
            }
        }

        // We need to test node.Stager because it will be used in other tests.
        if ('undefined' === typeof node.Stager) {
            errors.push('node.Stager not found.');
        }

        return { success: !errors.length, errors: errors };
    }
304549
def kma(inputfile_1, out_path, databases, db_path_kma, min_cov=0.6,
            threshold=0.9, kma_path="cge/kma/kma", sample_name="",
            inputfile_2=None, kma_mrs=None, kma_gapopen=None,
            kma_gapextend=None, kma_penalty=None, kma_reward=None, kma_pm=None,
            kma_fpm=None, kma_memmode=False, kma_nanopore=False, debug=False,
            kma_add_args=None):
        
        threshold = threshold * 100
        min_cov = min_cov * 100

        kma_results = dict()
        kma_results["excluded"] = dict()

        if(sample_name):
           sample_name = "_" + sample_name

        # Initiate output dicts.
        gene_align_sbjct = {}
        gene_align_query = {}
        gene_align_homo = {}

        for db in databases:
            kma_db = db_path_kma + "/" + db
            kma_outfile = out_path + "/kma_" + db + sample_name
            kma_cmd = ("%s -t_db %s -o %s -e 1.0" % (kma_path,
                       kma_db, kma_outfile))
            if(inputfile_2 is not None):
                kma_cmd += " -ipe " + inputfile_1 + " " + inputfile_2
            else:
                kma_cmd += " -i " + inputfile_1
            if(kma_mrs is not None):
                kma_cmd += " -mrs " + str(kma_mrs)
            if(kma_gapopen is not None):
                kma_cmd += " -gapopen " + str(kma_gapopen)
            if(kma_gapextend is not None):
                kma_cmd += " -gapextend " + str(kma_gapextend)
            if(kma_penalty is not None):
                kma_cmd += " -penalty " + str(kma_penalty)
            if(kma_reward is not None):
                kma_cmd += " -reward " + str(kma_reward)
            if(kma_pm is not None):
                kma_cmd += " -pm " + kma_pm
            if(kma_fpm is not None):
                kma_cmd += " -fpm " + kma_fpm
            if (kma_memmode):
                kma_cmd += " -mem_mode "
            if (kma_nanopore):
                kma_cmd += " -bcNano "
                kma_cmd += " -mp 20 "
            if (kma_add_args is not None):
                kma_cmd += " " + kma_add_args + " "

            # kma output files
            align_filename = kma_outfile + ".aln"
            res_filename = kma_outfile + ".res"

            # If .res file exists then skip mapping
            if os.path.isfile(res_filename) and os.access(res_filename, os.R_OK):
                print("Found " + res_filename + " skipping DB.")
            else:
                # Call KMA
                if(debug):
                    print("KMA cmd: " + kma_cmd)

                process = subprocess.Popen(kma_cmd, shell=True,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)
                out, err = process.communicate()

            kma_results[db] = 'No hit found'

            # Open res file
            try:
                res_file = open(res_filename, "r")
                header = res_file.readline()
            except IOError as error:
                sys.exit("Error: KMA did not run as expected.\n" +
                         "KMA finished with the following response:" +
                         "\n{}\n{}".format(out.decode("utf-8"),
                                           err.decode("utf-8")))

            for line in res_file:

                if kma_results[db] == 'No hit found':
                    kma_results[db] = dict()
                    # kma_results[db]["excluded"] = dict()
                    # continue

                data = [data.strip() for data in line.split("\t")]
                gene = data[0]

                sbjct_len = int(data[3])
                sbjct_ident = float(data[4])
                coverage = float(data[5])
                depth = float(data[-3])
                q_value = float(data[-2])
                p_value = float(data[-1])


                if gene not in kma_results[db]:
                    hit = gene
                else:
                    hit = gene + "_" + str(len(kma_results[db][gene]) + 1)

                exclude_reasons = []

                if(coverage < min_cov or sbjct_ident < threshold):
                    exclude_reasons.append(coverage)
                    exclude_reasons.append(sbjct_ident)

                if(exclude_reasons):
                    # kma_results[db]["excluded"][hit] = exclude_reasons
                    kma_results["excluded"][hit] = exclude_reasons

                kma_results[db][hit] = dict()
                kma_results[db][hit]['sbjct_length'] = sbjct_len
                kma_results[db][hit]["perc_coverage"] = coverage
                kma_results[db][hit]["sbjct_string"] = []
                kma_results[db][hit]["query_string"] = []
                kma_results[db][hit]["homo_string"] = []
                kma_results[db][hit]["sbjct_header"] = gene
                kma_results[db][hit]["perc_ident"] = sbjct_ident
                kma_results[db][hit]["query_start"] = "NA"
                kma_results[db][hit]["query_end"] = "NA"
                kma_results[db][hit]["contig_name"] = "NA"
                kma_results[db][hit]["HSP_length"] = ""
                kma_results[db][hit]["cal_score"] = q_value
                kma_results[db][hit]["depth"] = depth
                kma_results[db][hit]["p_value"] = p_value
            res_file.close()


            if kma_results[db] == 'No hit found':
                continue

            # Open align file
            with open(align_filename, "r") as align_file:
                hit_no = dict()
                gene = ""
                # Parse through alignments
                for line in align_file:
                    # Skip empty lines
                    if(not line.strip()):
                        continue

                    # Check when a new gene alignment start
                    if line.startswith("#"):
                        gene = line[1:].strip()

                        if gene not in hit_no:
                            hit_no[gene] = str(1)
                        else:
                            hit_no[gene] += str(int(hit_no[gene]) + 1)

                    else:
                        # Check if gene one of the user specified genes
                        if hit_no[gene] == '1':
                            hit = gene
                        else:
                            hit = gene + "_" + hit_no[gene]

                        if hit in kma_results[db]:
                            line_data = line.split("\t")[-1].strip()
                            if line.startswith("template"):
                                kma_results[db][hit]["sbjct_string"] += (
                                    [line_data])
                            elif line.startswith("query"):
                                kma_results[db][hit]["query_string"] += (
                                    [line_data])
                            else:
                                kma_results[db][hit]["homo_string"] += (
                                    [line_data])
                        else:
                            print(hit + " not in results: ", kma_results)

            # concatinate all sequences lists and find subject start
            # and subject end

            gene_align_sbjct[db] = {}
            gene_align_query[db] = {}
            gene_align_homo[db] = {}

            for hit in kma_results[db]:
                # if(hit == "excluded"):
                # continue
                align_sbjct = "".join(kma_results[db][hit]['sbjct_string'])
                align_query = "".join(kma_results[db][hit]['query_string'])
                align_homo = "".join(kma_results[db][hit]['homo_string'])

                # Extract only aligned sequences
                start = re.search("^-*(\w+)", align_query).start(1)
                end = re.search("\w+(-*)$", align_query).start(1)

                kma_results[db][hit]['sbjct_string'] = align_sbjct[start:end]
                kma_results[db][hit]['query_string'] = align_query[start:end]
                kma_results[db][hit]['homo_string'] = align_homo[start:end]

                # Save align start and stop positions relative to
                # subject sequence
                kma_results[db][hit]['sbjct_start'] = start + 1
                kma_results[db][hit]["sbjct_end"] = end + 1
                kma_results[db][hit]["HSP_length"] = end - start

                # Count gaps in the alignment
                kma_results[db][hit]["gaps"] = (
                    kma_results[db][hit]['sbjct_string'].count("-") +
                    kma_results[db][hit]['query_string'].count("-"))

                # Save sequences covering the entire subject sequence
                # in seperate variables
                gene_align_sbjct[db][hit] = align_sbjct
                gene_align_query[db][hit] = align_query
                gene_align_homo[db][hit] = align_homo

        return FinderResult(kma_results, gene_align_sbjct, gene_align_query,
                            gene_align_homo)
1077732
public synchronized void regenerateReport() {
    // Generate the configuration map from master and worker configuration records
    Map<PropertyKey, Map<Optional<String>, List<String>>> confMap = generateConfMap();

    // Update the errors and warnings configuration
    Map<Scope, List<InconsistentProperty>> confErrors = new HashMap<>();
    Map<Scope, List<InconsistentProperty>> confWarns = new HashMap<>();

    for (Map.Entry<PropertyKey, Map<Optional<String>, List<String>>> entry : confMap.entrySet()) {
      if (entry.getValue().size() >= 2) {
        PropertyKey key = entry.getKey();
        InconsistentProperty inconsistentProperty = new InconsistentProperty()
            .setName(key.getName()).setValues(entry.getValue());
        Scope scope = key.getScope().equals(Scope.ALL) ? Scope.SERVER : key.getScope();
        if (entry.getKey().getConsistencyLevel().equals(ConsistencyCheckLevel.ENFORCE)) {
          confErrors.putIfAbsent(scope, new ArrayList<>());
          confErrors.get(scope).add(inconsistentProperty);
        } else {
          confWarns.putIfAbsent(scope, new ArrayList<>());
          confWarns.get(scope).add(inconsistentProperty);
        }
      }
    }

    // Update configuration status
    ConfigStatus status = confErrors.values().stream().anyMatch(a -> a.size() > 0)
        ? ConfigStatus.FAILED : confWarns.values().stream().anyMatch(a -> a.size() > 0)
        ? ConfigStatus.WARN : ConfigStatus.PASSED;

    if (!status.equals(mConfigCheckReport.getConfigStatus())) {
      logConfigReport();
    }

    mConfigCheckReport = new ConfigCheckReport(confErrors, confWarns, status);
  }
456205
def check(repair = false)
      # Determine size of the data blobs file.
      data_file_size = File.exist?(@blobs_file_name) ?
        File.size(@blobs_file_name) : 0

      next_start = 0
      prev_entry = nil
      @entries.each do |entry|
        # Entries should never overlap
        if prev_entry && next_start > entry[START]
          PEROBS.log.fatal
                "#{@dir}: Index entries are overlapping\n" +
                "ID: #{'%016X' % prev_entry[ID]}  " +
                "Start: #{prev_entry[START]}  " +
                "Bytes: #{prev_entry[BYTES]}\n" +
                "ID: #{'%016X' % entry[ID]}  Start: #{entry[START]}  " +
                "Bytes: #{entry[BYTES]}"
        end
        next_start = entry[START] + entry[BYTES]

        # Entries must fit within the data file
        if next_start > data_file_size
          PEROBS.log.fatal
                "#{@dir}: Entry for ID #{'%016X' % entry[ID]} " +
                "goes beyond 'data' file " +
                "size (#{data_file_size})\n" +
                "ID: #{'%016X' % entry[ID]}  Start: #{entry[START]}  " +
                "Bytes: #{entry[BYTES]}"
        end

        prev_entry = entry
      end

      true
    end
512611
function finishProcessing(err) {
      if (err) return cb(err);
      db.eval(hmz, 2, 'processing_domains', 'expiring_domains',
        domain, next);
    }
918674
def transition_complete(self, pipeline_key):
    
    def txn():
      pipeline_record = db.get(pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to mark pipeline ID "%s" as complete but it does not exist.',
            pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status not in (
          _PipelineRecord.WAITING, _PipelineRecord.RUN):
        logging.warning(
            'Tried to mark pipeline ID "%s" as complete, found bad state: %s',
            pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()

      pipeline_record.status = _PipelineRecord.DONE
      pipeline_record.finalized_time = self._gettime()
      pipeline_record.put()

    db.run_in_transaction(txn)
1013012
function CheckpointPlugin () {
    if (!db.getDiskSize) {
      throw new Error(
        'This plugin depends on pouchdb-size plugin. ' +
        'Please add pouchdb-size plugin befor this one.'
      );
    }

    var changes = db.changes({
      live: true,
      since: 'now'
    })
      .on('error', function (error) {
        logger.error(
          'Error while fetching db changes: \'%s\', stack: \'%s\'',
          String(error), error.stack
        );
        changes.cancel();
        db.emit('error', error);
      })
      .on('change', executeOnce(checkDBSize, delay));

    db.on('destroyed', function () {
      destroyed = true;
      changes.cancel();
    });
  }
1058486
private boolean shouldVerifyCompletenessForDataset(Dataset dataset, List<Pattern> blacklist,
    List<Pattern> whitelist) {
    boolean renamingRequired = this.state.getPropAsBoolean(COMPACTION_RENAME_SOURCE_DIR_ENABLED, DEFAULT_COMPACTION_RENAME_SOURCE_DIR_ENABLED);

    LOG.info ("Should verify completeness with renaming source dir : " + renamingRequired);

    return !datasetAlreadyCompacted(this.fs, dataset, renamingRequired)
        && DatasetFilterUtils.survived(dataset.getName(), blacklist, whitelist);
  }