Retrieve all the referenced files, deserialize, then merge them together
507407
def _get_values(profile=None):
    '''
    
    '''
    profile = profile or {}
    serializers = salt.loader.serializers(__opts__)

    ret = {}
    for fname in profile.get('files', []):
        try:
            with salt.utils.files.flopen(fname) as yamlfile:
                contents = serializers.yaml.deserialize(yamlfile)
                ret = salt.utils.dictupdate.merge(
                    ret, contents, **profile.get('merge', {}))
        except IOError:
            log.error("File '%s' not found ", fname)
        except TypeError as exc:
            log.error("Error deserializing sdb file '%s': %s", fname, exc)
    return ret
1326038
def serialize_objects_and_references
      obj_dir = {}
      objid_cache = {}
      # create a dictionary for referenced objects (no value resolution at this point)
      # at the same time, delete duplicates and old versions when objects have multiple versions
      @parsed.uniq!
      @parsed.length.times do |i|
        o = @parsed[i]
        objid_cache[o.object_id] = i
        tmp_key = [o[:indirect_reference_id], o[:indirect_generation_number]]
        if tmp_found = obj_dir[tmp_key]
          tmp_found.clear
          @parsed[objid_cache[tmp_found.object_id]] = nil
        end
        obj_dir[tmp_key] = o
      end
      @parsed.compact!
      objid_cache.clear

      should_resolve = [@parsed, @root_object]
      while should_resolve.count > 0
        obj = should_resolve.pop
        if obj.is_a?(Hash)
          obj.keys.each do |k|
            o = obj[k]
            if o.is_a?(Hash)
              if o[:is_reference_only]
                if o[:indirect_reference_id].nil?
                  o = nil
                else
                  o[:referenced_object] = obj_dir[[o[:indirect_reference_id], o[:indirect_generation_number]]]
                  warn "Couldn't connect reference for #{o}" if o[:referenced_object].nil? && (o[:indirect_reference_id] + o[:indirect_generation_number] != 0)
                  o.delete :indirect_reference_id
                  o.delete :indirect_generation_number
                  o = (o[:referenced_object] && o[:referenced_object][:indirect_without_dictionary]) || o
                end
                obj[k] = o
              else
                should_resolve << o
              end
            elsif o.is_a?(Array)
              should_resolve << o
            end
          end
        elsif obj.is_a?(Array)
          obj.map! do |o|
            if o.is_a?(Hash)
              if o[:is_reference_only]
                if o[:indirect_reference_id].nil?
                  o = nil
                else
                  o[:referenced_object] = obj_dir[[o[:indirect_reference_id], o[:indirect_generation_number]]]
                  warn "Couldn't connect reference for #{o}" if o[:referenced_object].nil?
                  o.delete :indirect_reference_id
                  o.delete :indirect_generation_number
                  o = (o[:referenced_object] && o[:referenced_object][:indirect_without_dictionary]) || o
                end
              else
                should_resolve << o
              end
            elsif o.is_a?(Array)
              should_resolve << o
            end
            o
          end
        end
      end
    end
618351
def get_files(self, retrieve=False):

		'''
		
		'''

		if self.exists and hasattr(self.rdf.triples, 'pcdm') and hasattr(self.rdf.triples.pcdm, 'hasFile'):
			files = [ self.repo.parse_uri(uri) for uri in self.rdf.triples.pcdm.hasFile ]

			# return
			return files

		else:
			return []
275832
def merge_files(configuration, locale, fail_if_missing=True):
    
    for target, sources in configuration.generate_merge.items():
        merge(configuration, locale, target, sources, fail_if_missing)
1001185
function getFiles(paths) {
  var result = fileUtil.getAllFiles(paths);

  var files = result.map(function (item) {
    return item.relativePath;
    // return path.join(item.basePath, item.relativePath);
  });

  return files;
}
285787
def merge_required_files(dirnames, out_dir):
    
    # The list of files to merge
    fn_to_merge = ("steps_summary.tex", "excluded_markers.txt",
                   "excluded_samples.txt")

    # Merging the files
    for fn in fn_to_merge:
        o_fn = os.path.join(out_dir, fn)
        with open(o_fn, "w") as o_file:
            for dn in dirnames:
                i_fn = os.path.join(dn, fn)
                with open(i_fn, "r") as i_file:
                    o_file.write(i_file.read())

    # Merging the result summary file
    o_fn = os.path.join(out_dir, "results_summary.txt")
    with open(o_fn, "w") as o_file:
        for i, dn in enumerate(dirnames):
            i_fn = os.path.join(dn, "results_summary.txt")
            with open(i_fn, "r") as i_file:
                if i != 0:
                    # We skip the first 4 lines (file descriptions)
                    [i_file.readline() for i in range(4)]
                o_file.write(i_file.read())

    # Merging the graphic paths file
    graphic_paths = set()
    for dn in dirnames:
        fn = os.path.join(dn, "graphic_paths.txt")
        if os.path.isfile(fn):
            with open(fn, "r") as i_file:
                graphic_paths.update({
                    os.path.join(dn, path)
                    for path in i_file.read().splitlines()
                })
    if len(graphic_paths) > 0:
        with open(os.path.join(out_dir, "graphic_paths.txt"), "w") as o_file:
            for path in sorted(graphic_paths):
                print >>o_file, os.path.relpath(path, out_dir)
597246
def parse_references(cls, filename):
        
        references = set()
        for line in open(filename):
            matched = cls.RE_REF.match(line)
            if matched:
                reference = matched.group('path')
                reference_base = os.path.splitext(reference)[0]
                references.add(reference_base)
        return references
143668
def GetAllFiles(self):
    
    for filepath in self._files:
      yield filepath

    for dirpath in self._dirs:
      for filepath in ArtifactRegistrySources._GetDirYamlFiles(dirpath):
        if filepath in self._files:
          continue
        yield filepath
1036227
def __deserialize_file(self, response):
        
        fd, path = tempfile.mkstemp(dir=self.configuration.temp_folder_path)
        os.close(fd)
        os.remove(path)

        content_disposition = response.getheader("Content-Disposition")
        if content_disposition:
            filename = re.search(r'filename=[\'"]?([^\'"\s]+)[\'"]?',
                                 content_disposition).group(1)
            path = os.path.join(os.path.dirname(path), filename)

        with open(path, "wb") as f:
            f.write(response.data)

        return path
372445
def _add_secondary_if_exists(secondary, out, get_retriever):
    
    secondary = [_file_local_or_remote(y, get_retriever) for y in secondary]
    secondary = [z for z in secondary if z]
    if secondary:
        out["secondaryFiles"] = [{"class": "File", "path": f} for f in secondary]
    return out
1069908
function getDirNamesForDuplicateFiles(files) {
        // Must have at least two files in list for this to make sense
        if (files.length <= 1) {
            return [];
        }

        // First collect paths from the list of files and fill map with them
        var map = {}, filePaths = [], displayPaths = [];
        files.forEach(function (file, index) {
            var fp = file.fullPath.split("/");
            fp.pop(); // Remove the filename itself
            displayPaths[index] = fp.pop();
            filePaths[index] = fp;

            if (!map[displayPaths[index]]) {
                map[displayPaths[index]] = [index];
            } else {
                map[displayPaths[index]].push(index);
            }
        });

        // This function is used to loop through map and resolve duplicate names
        var processMap = function (map) {
            var didSomething = false;
            _.forEach(map, function (arr, key) {
                // length > 1 means we have duplicates that need to be resolved
                if (arr.length > 1) {
                    arr.forEach(function (index) {
                        if (filePaths[index].length !== 0) {
                            displayPaths[index] = filePaths[index].pop() + "/" + displayPaths[index];
                            didSomething = true;

                            if (!map[displayPaths[index]]) {
                                map[displayPaths[index]] = [index];
                            } else {
                                map[displayPaths[index]].push(index);
                            }
                        }
                    });
                }
                delete map[key];
            });
            return didSomething;
        };

        var repeat;
        do {
            repeat = processMap(map);
        } while (repeat);

        return displayPaths;
    }
820593
def files(
            self):
        
        relatedFilesResultsList = []
        relatedFilesResultsList[:] = [dict(l)
                                      for l in self.relatedFilesResultsList]
        return relatedFilesResultsList
1077399
def merge_vcf_files(infiles, ref_seqs, outfile, threads=1):
    ''''''
    vars_dict = vcf_file_read.vcf_files_to_dict_of_vars(infiles, ref_seqs, threads=threads)
    _dict_of_vars_to_vcf_file(vars_dict, outfile)
917073
private void mergeFiles(List<File> inputFiles, OutputStream outputStream)
      throws IOException {
    long startTime = System.nanoTime();

    //Merge files
    for (File f : inputFiles) {
      if (f.exists()) {
        FileInputStream fileInputStream = new FileInputStream(f);
        BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream);
        try {
          LOGGER.log(Level.INFO, "Merging {0} size={1}", new Object[]{f.getName(), f.length()});

          byte[] buffer = new byte[8192];
          int length;
          while ((length = bufferedInputStream.read(buffer)) > 0) {
            outputStream.write(buffer, 0, length);
          }
        } finally {
          bufferedInputStream.close();
          fileInputStream.close();
        }
      } else {
        LOGGER.log(Level.INFO, "Skip merging file {0} because it doesn't exist", f.getName());
      }
    }

    LOGGER.log(Level.INFO, "Time to merge {0} s", ((System.nanoTime() - startTime) / 1000000000.0));
  }
177408
def merge_extras(items, config):
    
    final = {}
    for extra_name in items[0]["disambiguate"].keys():
        in_files = []
        for data in items:
            in_files.append(data["disambiguate"][extra_name])
        out_file = "%s-allmerged%s" % os.path.splitext(in_files[0])
        if in_files[0].endswith(".bam"):
            merged_file = merge.merge_bam_files(in_files, os.path.dirname(out_file), items[0],
                                                out_file=out_file)
        else:
            assert extra_name == "summary", extra_name
            merged_file = _merge_summary(in_files, out_file, items[0])
        final[extra_name] = merged_file
    out = []
    for data in items:
        data["disambiguate"] = final
        out.append([data])
    return out
1167066
def getfiles(qfiles, dirname, names):
    """"""
    for name in names:
        fullname = os.path.join(dirname, name)
        if os.path.isfile(fullname) and \
            fullname.endswith('.cf') or \
                fullname.endswith('.post'):
            qfiles.put(fullname)
329168
def _fetchFilesFromFolder(self, target, recursive):
        '''
            
        '''
        directory_items = os.walk(target)

        # If recursive is false, fetch only the first tuple
        if not recursive:
            directory_items = [next(directory_items)]

        targets = []
        for dir_name, folders, files in directory_items:
            for f in files:
                targets.append(os.path.join(dir_name, f))

        return targets