Loop through all of the files that we previously fetched from git,
        creating the disease-phenotype association.
        :param limit:
        :return:
170875
def process_all_common_disease_files(self, limit=None):
        
        LOG.info("Iterating over all common disease files")
        common_file_count = 0
        total_processed = ""  # stopgap gill we fix common-disease files
        unpadded_doids = ""   # stopgap gill we fix common-disease files
        for ingest in self.files:
            if ingest[:5] == 'common':
                common_file_count += 1
                raw = self.files[ingest]['file']
                total_processed += self.process_common_disease_file(
                    raw, unpadded_doids, limit)
            if not self.test_mode and limit is not None and total_processed > limit:
                break
        LOG.info("Finished iterating over all common disease files.")
        return
1204702
def all_modified_files_coverage
      unless @project.nil?
        all_modified_files_coverage ||= begin
          modified_files = git.modified_files.nil? ? [] : git.modified_files
          added_files = git.added_files.nil? ? [] : git.added_files
          all_changed_files = modified_files | added_files
          @project.coverage_files.select do |file|
            all_changed_files.include? file.source_file_pathname_relative_to_repo_root.to_s
          end
        end

        all_modified_files_coverage
      end
    end
680648
def get_changed_files(include_staged=False):
  

  process = subprocess.Popen(['git', 'status', '--porcelain'],
    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  stdout, __ = process.communicate()
  if process.returncode != 0:
    raise ValueError(stdout)
  files = []
  for line in stdout.decode().split('\n'):
    if not line or line.startswith('#'): continue
    assert line[2] == ' '
    if not include_staged and line[1] == ' ': continue
    files.append(line[3:])
  return files
60193
def _parse_patient_phenotypes(self, file, limit=None):
        
        model = Model(self.graph)
        line_counter = 0
        reader = csv.reader(file, delimiter="\t")
        for row in reader:
            (patient_id, hpo_curie, present) = row
            patient_curie = ':{0}'.format(patient_id)
            if patient_id == 'Patient':  # skip header
                line_counter += 1
                continue

            model.addPerson(patient_curie, patient_id)

            self.graph.addTriple(
                patient_curie, self.globaltt['has phenotype'], self.globaltt['disease'])
            if present == 'yes':
                self.graph.addTriple(
                    patient_curie, self.globaltt['has phenotype'], hpo_curie)

            line_counter += 1
            if not self.test_mode and limit is not None \
                    and line_counter >= limit:
                break
47141
def populate
      # Display progressbar with the number of files as countdown
      progressbar = init_progressbar(current_files.count)

      # Extract the blame history from all checked in files
      current_files.each do |file|
        progressbar.increment

        # Skip this file if non wanted type
        next unless check_file?(file)

        # -w ignore whitespaces (defined in @wopt)
        # -M detect moved or copied lines.
        # -p procelain mode (parsed by BlameParser)
        execute("git #{git_directory_params} blame #{encoding_opt} -p -M #{default_params} #{commit_range.to_s} #{@wopt} -- '#{file}'") do |result|
          BlameParser.new(result.to_s).parse.each do |row|
            next if row[:boundary]

            email = get(row, :author, :mail)
            name = get(row, :author, :name)

            # Create or find user
            author = author_by_email(email, name)

            # Get author by name and increase the number of loc by 1
            author.inc(:loc, get(row, :num_lines))

            # Store the files and authors together
            associate_file_with_author(author, file)
          end
        end
      end

      # Get repository summery and update each author accordingly
      execute("git #{git_directory_params} shortlog #{encoding_opt} #{default_params} -se #{commit_range.to_s}") do |result|
        result.to_s.split("\n").map do |line|
          _, commits, name, email = line.match(/(\d+)\s+(.+)\s+<(.+?)>/).to_a
          author = author_by_email(email)

          author.name = name

          author.update({
            raw_commits: commits.to_i,
            raw_files: files_from_author(author).count,
            files_list: files_from_author(author)
          })
        end
      end

      progressbar.finish
    end
408896
def parse(self, limit=None):
        
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)

        sgd_file = '/'.join((self.rawdir, self.files['sgd_phenotype']['file']))
        columns = [
            'Feature Name', 'Feature Type', 'Gene Name', 'SGDID', 'Reference',
            'Experiment Type', 'Mutant Type', 'Allele', 'Strain Background',
            'Phenotype', 'Chemical', 'Condition', 'Details', 'Reporter']
        sgd_df = pd.read_csv(sgd_file, sep='\t', names=columns)
        records = sgd_df.to_dict(orient='records')
        for index, assoc in enumerate(records):
            if isinstance(assoc['Gene Name'], str):
                if limit is not None and index > limit:
                    break
                self.make_association(assoc)

        return
1313585
function _loop(dir) {
            /* eventually try every python file */
            var pdir = fs.readdirSync(dir);

            pdir.forEach(function(v, i) {
              var stat = fs.statSync(dir + '/' + v);
              if (stat.isFile()) {
                // only process Makefiles and .mk targets.
                if (v !== 'Makefile' && path.extname(v) !== '.mk') {
                  return;
                }

                _log('patching ' + v);

                /* patch the file */
                var py = fs.readFileSync(dir + '/' + v, {
                  encoding: 'utf8',
                });
                py = py.replace(/([a-z]|\/)*python(\w|)/gm, isPy); // this is definently needed
                fs.writeFileSync(dir + '/' + v, py, {
                  encoding: 'utf8',
                }); // write to file
              } else if (stat.isDirectory()) {
                // must be dir?
                // skip tests because we don't need them here
                if (v !== 'test') {
                  _loop(dir + '/' + v);
                }
              }
            });
          }
376304
def doc_children(self, doctype, limiters=[]):
        
        result = []
        for doc in self.docstring:
            if len(limiters) == 0 or doc.doctype in limiters:
                result.extend(doc.children(doctype))

        return result
753452
def limit(self, limit):
        
        query = self._copy()
        query._limit = limit
        return query
784287
def _process_feature_pub(self, limit):
        

        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph

        raw = '/'.join((self.rawdir, 'feature_pub'))
        LOG.info("processing feature_pub")

        line_counter = 0

        with open(raw, 'r') as f:
            filereader = csv.reader(f, delimiter='\t', quotechar='\"')
            f.readline()  # read the header row; skip
            for line in filereader:
                (feature_pub_id, feature_id, pub_id) = line
                # 1440    3175682 62137
                # 2       3160606 99159

                feature_key = feature_id
                if self.test_mode and not (
                        int(feature_key)
                        in self.test_keys['gene'] + self.test_keys['allele'] and
                        int(pub_id) in self.test_keys['pub']):
                    continue
                if feature_key not in self.idhash['feature']:
                    continue
                feature_id = self.idhash['feature'][feature_key]
                pub_key = pub_id
                pub_id = self.idhash['publication'][pub_key]

                graph.addTriple(pub_id, self.globaltt['mentions'], feature_id)

                line_counter += 1

                if not self.test_mode and limit is not None and line_counter > limit:
                    break

        return
984064
def current_files
      if commit_range.is_range?
        execute("git #{git_directory_params} -c diff.renames=0 -c diff.renameLimit=1000 diff -M -C -c --name-only --ignore-submodules=all --diff-filter=AM #{encoding_opt} #{default_params} #{commit_range.to_s}") do |result|
          filter_files(result.to_s.split(/\n/))
        end
      else
        submodules = current_submodules
        execute("git #{git_directory_params} ls-tree -r #{commit_range.to_s} --name-only") do |result|
          filter_files(result.to_s.split(/\n/).select { |f| !submodules.index(f) })
        end
      end
    end
510527
def files(self, paths, access=None, extensions=None,
              minsize=None, recursion=False):
        
        self.__init__()
        self.access = access
        self.filetype = 'file'
        self.extensions = extensions
        self.minsize = minsize
        self.paths = paths
        self.recursion = recursion

        return _sorter(self._generator_file())
72754
def full_file_list_with_sequence(scan_path):
    
    
    file_list = []
    path = os.path.abspath(scan_path)
    
    for root, dirs, files in os.walk(path):
        if len(files) != 0 and not '.svn' in root and not '.git' in root:
            try:
                sc = sequential.SequentialFolder(str(root))
                if sc.sequence:
                    file_list.append(sc)
                    continue
            except Exception as e:
                pass
                
            for f in files:
                file_list.append(os.path.join(root, f))
    return file_list
800909
def process_mgi_relationship_transgene_genes(self, limit=None):
        
        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph
        LOG.info("getting transgene genes")
        raw = '/'.join((self.rawdir, 'mgi_relationship_transgene_genes'))
        geno = Genotype(graph)
        col = [
            'rel_key', 'allele_key', 'allele_id', 'allele_label', 'category_key',
            'category_name', 'property_key', 'property_name', 'gene_num'
        ]
        with open(raw, 'r', encoding="utf8") as csvfile:
            filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
            header = next(filereader)
            if header != col:
                LOG.error('expected columns:  %s\n\tBut got:\n%s', col, header)
            for row in filereader:
                # rel_key,
                allele_key = int(row[col.index('allele_key')])
                allele_id = row[col.index('allele_id')]
                # allele_label,
                # category_key,
                # category_name,
                # property_key,
                # property_name,
                gene_num = int(row[col.index('gene_num')])

                if self.test_mode and allele_key not in self.test_keys.get('allele')\
                        and gene_num not in self.test_ids:
                    continue

                gene_id = 'NCBIGene:' + str(gene_num)

                # geno.addParts(gene_id, allele_id, self.globaltt['has_variant_part'])
                seqalt_id = self.idhash['seqalt'].get(allele_key)
                if seqalt_id is None:
                    seqalt_id = allele_id
                geno.addSequenceDerivesFrom(seqalt_id, gene_id)

                if not self.test_mode and limit is not None and \
                        filereader.line_num > limit:
                    break

        return
231010
def fetch_all_data(self, limit=50000):
        
        # Query text
        query = text(
            """
            SELECT domain, entity_id, state, last_changed
            FROM states
            WHERE
                state NOT IN ('unknown', 'unavailable')
            ORDER BY last_changed DESC
            LIMIT :limit
            """
        )

        try:
            print("Querying the database, this could take a while")
            response = self.perform_query(query, limit=limit)
            master_df = pd.DataFrame(response.fetchall())
            print("master_df created successfully.")
            self._master_df = master_df.copy()
            self.parse_all_data()
        except:
            raise ValueError("Error querying the database.")
1336550
def parse(self, limit=None):
        

        files_to_download, ftp = self._get_file_list(
            self.files['anat_entity']['path'],
            self.files['anat_entity']['pattern'])
        for dlname in files_to_download:
            localfile = '/'.join((self.rawdir, dlname))
            with gzip.open(localfile, 'rt', encoding='ISO-8859-1') as fh:
                LOG.info("Processing %s", localfile)
                self._parse_gene_anatomy(fh, limit)
        return
676393
def mergeall(filename, snrmin, snrmax, bdfdir):
    

    filename = os.path.abspath(filename)
    bignumber = 500

    if os.path.exists(filename):
        scans = ps.read_scans(filename, bdfdir=bdfdir)
        scanlist = sorted(scans.keys())
    else:
        logger.warn('Could not find file {0}. Estimating scans from available files.'.format(filename))
        filelist = glob.glob(os.path.join(os.path.dirname(filename), '*{0}_sc*pkl'.format(os.path.basename(filename))))
        try:
            scanlist = sorted(set([int(fn.rstrip('.pkl').split('_sc')[1].split('seg')[0]) for fn in filelist]))
        except IndexError:
            logger.warn('Could not parse filenames for scans. Looking over big range.')
            scanlist = range(bignumber)

    logger.info('Merging over scans {0}'.format(scanlist))

    for scan in scanlist:
        pc.merge_segments(filename, scan)
    pc.merge_scans(os.path.dirname(filename), os.path.basename(filename), scanlist, snrmin=snrmin, snrmax=snrmax)