Create a .tar.gz file from a directory
 Done in pure ruby to ensure portability
515451
def compress_directory(dir)
      File.open archive_path(dir), 'wb' do |oarchive_path|
        Zlib::GzipWriter.wrap oarchive_path do |gz|
          Gem::Package::TarWriter.new gz do |tar|
            Find.find "#{builds_dir}/"+dir do |f|
              relative_path = f.sub "#{builds_dir}/", ""
              mode = File.stat(f).mode
              size = File.stat(f).size
              
              if File.directory? f
                tar.mkdir relative_path, mode
              else
                tar.add_file_simple relative_path, mode, size do |tio|
                  File.open f, 'rb' do |rio|
                    while buffer = rio.read(BLOCKSIZE_TO_READ)
                      tio.write buffer
                    end
                  end
                end
              end
              
            end
          end
        end
      end
      
    end
1274107
def create_archive(
    source: Path,
    target: Path,
    interpreter: str,
    main: str,
    compressed: bool = True
) -> None:
    
    # Check that main has the right format.
    mod, sep, fn = main.partition(":")
    mod_ok = all(part.isidentifier() for part in mod.split("."))
    fn_ok = all(part.isidentifier() for part in fn.split("."))
    if not (sep == ":" and mod_ok and fn_ok):
        raise zipapp.ZipAppError("Invalid entry point: " + main)

    main_py = MAIN_TEMPLATE.format(module=mod, fn=fn)

    with maybe_open(target, "wb") as fd:
        # write shebang
        write_file_prefix(fd, interpreter)

        # determine compression
        compression = zipfile.ZIP_DEFLATED if compressed else zipfile.ZIP_STORED

        # create zipapp
        with zipfile.ZipFile(fd, "w", compression=compression) as z:
            for child in source.rglob("*"):

                # skip compiled files
                if child.suffix == '.pyc':
                    continue

                arcname = child.relative_to(source)
                z.write(str(child), str(arcname))

            # write main
            z.writestr("__main__.py", main_py.encode("utf-8"))

    # make executable
    # NOTE on windows this is no-op
    target.chmod(target.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
118595
def create_tar_archive(self):
        
        #file filter
        EXCLUDE_FILES = glob.glob(os.path.join(self.savefolder, 'cells'))
        EXCLUDE_FILES += glob.glob(os.path.join(self.savefolder,
                                                'populations', 'subsamples'))
        EXCLUDE_FILES += glob.glob(os.path.join(self.savefolder,
                                                'raw_nest_output'))

        def filter_function(tarinfo):
            print(tarinfo.name)
            if len([f for f in EXCLUDE_FILES if os.path.split(tarinfo.name)[-1]
                    in os.path.split(f)[-1]]) > 0 or \
               len([f for f in EXCLUDE_FILES if os.path.split(tarinfo.path)[-1]
                    in os.path.split(f)[-1]]) > 0:
                print('excluding %s' % tarinfo.name)

                return None
            else:
                return tarinfo

        if RANK == 0:
            print('creating archive %s' % (self.savefolder + '.tar'))
            #open file
            f = tarfile.open(self.savefolder + '.tar', 'w')
            #avoid adding files to repo as /scratch/$USER/hybrid_model/...
            arcname = os.path.split(self.savefolder)[-1]

            f.add(name=self.savefolder,
                  arcname=arcname,
                  filter=filter_function)
            f.close()

        #resync
        COMM.Barrier()
959790
def extract_tar(fileobj):
    """"""
    import time

    archive = tarfile.open(fileobj=fileobj)
    filenames = [info.name for info in archive.getmembers() if info.isfile()]

    for src_name, dst_name in filter_tzfiles(filenames):
        mtime = archive.getmember(src_name).mtime
        modified = tuple(time.gmtime(mtime)[:6])
        bytes = archive.extractfile(src_name).read()

        yield dst_name, modified, bytes
1214517
def archive(sha, file = nil, opts = {})
      opts[:format] ||= 'zip'

      if opts[:format] == 'tgz'
        opts[:format] = 'tar'
        opts[:add_gzip] = true
      end

      if !file
        tempfile = Tempfile.new('archive')
        file = tempfile.path
        # delete it now, before we write to it, so that Ruby doesn't delete it
        # when it finalizes the Tempfile.
        tempfile.close!
      end

      arr_opts = []
      arr_opts << "--format=#{opts[:format]}" if opts[:format]
      arr_opts << "--prefix=#{opts[:prefix]}" if opts[:prefix]
      arr_opts << "--remote=#{opts[:remote]}" if opts[:remote]
      arr_opts << sha
      arr_opts << '--' << opts[:path] if opts[:path]
      command('archive', arr_opts, true, (opts[:add_gzip] ? '| gzip' : '') + " > #{escape file}")
      return file
    end
13545
def create_tar_file(source_files, target=None):
    
    if target:
        filename = target
    else:
        _, filename = tempfile.mkstemp()

    with tarfile.open(filename, mode='w:gz') as t:
        for sf in source_files:
            # Add all files from the directory into the root of the directory structure of the tar
            t.add(sf, arcname=os.path.basename(sf))
    return filename
1305600
def make_light_tarfile(self, name=None):
        """"""
        name = os.path.basename(self.workdir) + "-light.tar.gz" if name is None else name
        return self.make_tarfile(name=name, exclude_dirs=["outdata", "indata", "tmpdata"])
32310
def compile_binary(source):
    
    cmd = 'make sense'
    slink = '/usr/local/bin/chkrootkit'
    target = '/usr/local/chkrootkit/chkrootkit'
    # Tar Extraction
    t = tarfile.open(source, 'r')
    t.extractall(TMPDIR)
    if isinstance(t.getnames(), list):
        extract_dir = t.getnames()[0].split('/')[0]
        os.chdir(TMPDIR + '/' + extract_dir)
        logger.info('make output: \n%s' % subprocess.getoutput(cmd))
        # move directory in place
        os.rename(TMPDIR + '/' + extract_dir, 'usr/local/chkrootkit')
        # create symlink to binary in directory
        os.symlink(target, slink)
        return True
    return False
655805
def _dir_tar_out(self):
        """"""
        return os.path.join(self.proj.tar_direc_out, self.proj.name,
                            self.model.name, self.run.name)
951451
def create_tarfile(files: List[str], tar_path: str) -> None:
    """"""
    with tarfile.open(tar_path, "w:gz") as tar:
        for f in files:
            tar.add(f)
955737
def generate_tar_files(directory_list):
    """"""
    
    tar_file_list = []

    for directory in directory_list:
        if dir_exists(directory):
            _generate_tar(directory)                  # create the tar archive
            tar_file_list.append(directory + '.tar')  # append the tar archive filename to the returned tar_file_list list
        else:
            stderr("The directory '" + directory + "' does not exist and a tar archive could not be created from it.", exit=1)            

    return tar_file_list
1363325
def create_tarfile(self):
        
        floyd_logger.info("Compressing data...")
        # Show progress bar (file_compressed/file_to_compress)
        self.__compression_bar = ProgressBar(expected_size=self.__files_to_compress, filled_char='=')

        # Auxiliary functions
        def dfilter_file_counter(tarinfo):
            """
            Dummy filter function used to track the progression at file levels.
            """
            self.__compression_bar.show(self.__files_compressed)
            self.__files_compressed += 1
            return tarinfo

        def warn_purge_exit(info_msg, filename, progress_bar, exit_msg):
            """
            Warn the user that's something went wrong,
            remove the tarball and provide an exit message.
            """
            progress_bar.done()
            floyd_logger.info(info_msg)
            rmtree(os.path.dirname(filename))
            sys.exit(exit_msg)

        try:
            # Define the default signal handler for catching: Ctrl-C
            signal.signal(signal.SIGINT, signal.default_int_handler)
            with tarfile.open(self.filename, "w:gz") as tar:
                tar.add(self.source_dir, arcname=os.path.basename(self.source_dir), filter=dfilter_file_counter)
            self.__compression_bar.done()
        except (OSError, IOError) as e:
            # OSError: [Errno 13] Permission denied
            if e.errno == errno.EACCES:
                self.source_dir = os.getcwd() if self.source_dir == '.' else self.source_dir  # Expand cwd
                warn_purge_exit(info_msg="Permission denied. Removing compressed data...",
                                filename=self.filename,
                                progress_bar=self.__compression_bar,
                                exit_msg=("Permission denied. Make sure to have read permission "
                                          "for all the files and directories in the path: %s")
                                % (self.source_dir))
            # OSError: [Errno 28] No Space Left on Device (IOError on python2.7)
            elif e.errno == errno.ENOSPC:
                dir_path = os.path.dirname(self.filename)
                warn_purge_exit(info_msg="No space left. Removing compressed data...",
                                filename=self.filename,
                                progress_bar=self.__compression_bar,
                                exit_msg=("No space left when compressing your data in: %s.\n"
                                          "Make sure to have enough space before uploading your data.")
                                % (os.path.abspath(dir_path)))

        except KeyboardInterrupt:  # Purge tarball on Ctrl-C
            warn_purge_exit(info_msg="Ctrl-C signal detected: Removing compressed data...",
                            filename=self.filename,
                            progress_bar=self.__compression_bar,
                            exit_msg="Stopped the data upload gracefully.")
579117
def gzip
      @assets.map! do |asset|
        gzip_content = Zlib::Deflate.deflate(asset.content)
        [
          asset,
          ::JekyllAssetPipeline::Asset
            .new(gzip_content, "#{asset.filename}.gz", asset.dirname)
        ]
      end.flatten!
    end
736109
def tar_gzip_dir(directory, destination, base=None):
    """"""
    dest_file = tarfile.open(destination, 'w:gz')
    abs_dir_path = os.path.abspath(directory)
    base_name = abs_dir_path + "/"
    base = base or os.path.basename(directory)
    for path, dirnames, filenames in os.walk(abs_dir_path):
        rel_path = path[len(base_name):]
        dir_norm_path = os.path.join(base, rel_path)
        dir_tar_info = dest_file.gettarinfo(name=path)
        dir_tar_info.name = dir_norm_path
        dest_file.addfile(dir_tar_info)
        for filename in filenames:
            actual_path = os.path.join(path, filename)
            norm_path = os.path.join(base, rel_path, filename)
            tar_info = dest_file.gettarinfo(name=actual_path)
            tar_info.name = norm_path
            new_file = open(actual_path, 'rb')
            dest_file.addfile(tar_info, new_file)
    dest_file.close()
1164175
def _tar_and_copy(src_dir, target_dir):
  """"""
  src_dir = src_dir.rstrip("/")
  target_dir = target_dir.rstrip("/")
  tmp_dir = tempfile.gettempdir().rstrip("/")
  src_base = os.path.basename(src_dir)
  shell_run(
      "tar --exclude=.git -zcf {tmp_dir}/{src_base}.tar.gz -C {src_dir} .",
      src_dir=src_dir,
      src_base=src_base,
      tmp_dir=tmp_dir)
  final_destination = "%s/%s.tar.gz" % (target_dir, src_base)
  shell_run(
      ("gsutil cp {tmp_dir}/{src_base}.tar.gz "
       "{final_destination}"),
      tmp_dir=tmp_dir,
      src_base=src_base,
      final_destination=final_destination)
  return final_destination
534650
def open_tar(path_or_file, *args, **kwargs):
  
  (path, fileobj) = ((path_or_file, None) if isinstance(path_or_file, string_types)
                     else (None, path_or_file))  # TODO(#6071): stop using six.string_types
                                                 # This should only accept python3 `str`, not byte strings.
  with closing(TarFile.open(path, *args, fileobj=fileobj, **kwargs)) as tar:
    yield tar
120614
def extract_native_client_tarball(dir):
    
    '''
    assert_valid_dir(dir)

    target_tarball = os.path.join(dir, 'native_client.tar.xz')
    if os.path.isfile(target_tarball) and os.stat(target_tarball).st_size == 0:
        return

    subprocess.check_call(['pixz', '-d', 'native_client.tar.xz'], cwd=dir)
    subprocess.check_call(['tar', 'xf', 'native_client.tar'], cwd=dir)
    os.unlink(os.path.join(dir, 'native_client.tar'))
    open(target_tarball, 'w').close()