Migrate old minion and master pki file paths to new ones.
16495
def migrate_paths(opts):
    '''
    
    '''
    oldpki_dir = os.path.join(syspaths.CONFIG_DIR, 'pki')

    if not os.path.exists(oldpki_dir):
        # There's not even a pki directory, don't bother migrating
        return

    newpki_dir = opts['pki_dir']

    if opts['default_include'].startswith('master'):
        keepers = ['master.pem',
                   'master.pub',
                   'syndic_master.pub',
                   'minions',
                   'minions_pre',
                   'minions_rejected',
                   ]
        if not os.path.exists(newpki_dir):
            os.makedirs(newpki_dir)
        for item in keepers:
            oi_path = os.path.join(oldpki_dir, item)
            ni_path = os.path.join(newpki_dir, item)
            if os.path.exists(oi_path) and not os.path.exists(ni_path):
                shutil.move(oi_path, ni_path)

    if opts['default_include'].startswith('minion'):
        keepers = ['minion_master.pub',
                   'minion.pem',
                   'minion.pub',
                   ]
        if not os.path.exists(newpki_dir):
            os.makedirs(newpki_dir)
        for item in keepers:
            oi_path = os.path.join(oldpki_dir, item)
            ni_path = os.path.join(newpki_dir, item)
            if os.path.exists(oi_path) and not os.path.exists(ni_path):
                shutil.move(oi_path, ni_path)
248860
def upgrade(self, only):
        """"""
        repositories = self.meta.repositories
        if only:
            repositories = only
        for repo in repositories:
            changelogs = "{0}{1}{2}".format(self.log_path, repo,
                                            "/ChangeLog.txt")
            if os.path.isfile(changelogs):
                os.remove(changelogs)
            if os.path.isdir(self.lib_path + "{0}_repo/".format(repo)):
                for f in (os.listdir(self.lib_path + "{0}_repo/".format(
                        repo))):
                    files = "{0}{1}_repo/{2}".format(self.lib_path, repo, f)
                    if os.path.isfile(files):
                        os.remove(files)
                    elif os.path.isdir(files):
                        shutil.rmtree(files)
        Update().repository(only)
1346120
def MigrateArtifacts():
  """"""

  # First, delete all existing artifacts in REL_DB.
  artifacts = data_store.REL_DB.ReadAllArtifacts()
  if artifacts:
    logging.info("Deleting %d artifacts from REL_DB.", len(artifacts))
    for artifact in data_store.REL_DB.ReadAllArtifacts():
      data_store.REL_DB.DeleteArtifact(Text(artifact.name))
  else:
    logging.info("No artifacts found in REL_DB.")

  artifacts = artifact_registry.REGISTRY.GetArtifacts(
      reload_datastore_artifacts=True)

  logging.info("Found %d artifacts in AFF4.", len(artifacts))

  # Only migrate user-created artifacts.
  artifacts = list(filter(_IsCustom, artifacts))

  logging.info("Migrating %d user-created artifacts.", len(artifacts))

  for artifact in artifacts:
    _MigrateArtifact(artifact)
879987
def migrate_to_multiple_pub_keys()
      all_pub_keys = users_public_keys()
      base_path    = @configuration.keydir_path

      puts "Starting migration of PUB keys from old format to new! (This can take a while)"
      all_pub_keys.each do |pub_key_path|
        # skip git pub or already migrated key
        unless pub_key_path.match(/.*git.pub$/) || pub_key_path.include?('@')
          file_name = extract_file_name(pub_key_path,base_path,:pub)
          pub_content = gitolite_admin_repo().file_content(pub_key_path)

          # delete_user
          remove_file(pub_key_path, "Migrating user ('#{file_name}') to new annotation, temporary removing user")

          # create user
          create_user_depracated(file_name, pub_content)
        end
      end
      puts "End migration of pub keys"
      require 'pp'
      pp users_public_keys
      puts "--------------- END ---------------"
    end
1332519
def _check_minions_directories(pki_dir):
    '''
    
    '''
    minions_accepted = os.path.join(pki_dir, salt.key.Key.ACC)
    minions_pre = os.path.join(pki_dir, salt.key.Key.PEND)
    minions_rejected = os.path.join(pki_dir, salt.key.Key.REJ)
    minions_denied = os.path.join(pki_dir, salt.key.Key.DEN)

    return minions_accepted, minions_pre, minions_rejected, minions_denied
1077779
def move(self, key, folder):
        
        # Basically this is a sophisticated __delitem__
        # We need the path so we can make it in the new folder
        path, host, flags = self._exists(key)
        self._invalidate_cache()

        # Now, move the message file to the new folder
        newpath = joinpath(
            folder.base, 
            folder.get_name(), 
            "cur",     # we should probably move it to new if it's in new
            basename(path)
            )
        self.filesystem.rename(path, newpath)
        # And update the caches in the new folder
        folder._invalidate_cache()
811133
def cmd_merge(self):
        """"""
        from peewee_migrate.router import Router, LOGGER

        LOGGER.setLevel('DEBUG')
        LOGGER.propagate = 0

        router = Router(self.database,
                        migrate_dir=self.app.config['PEEWEE_MIGRATE_DIR'],
                        migrate_table=self.app.config['PEEWEE_MIGRATE_TABLE'])

        router.merge()
1219771
def remove_key(pki_dir, id_):
    '''
    
    '''
    key = os.path.join(pki_dir, 'minions', id_)
    if os.path.isfile(key):
        os.remove(key)
        log.debug('Deleted \'%s\'', key)
1281943
def copy_file_to_master(
        file_path,
        remote_path='.',
        username=None,
        key_path=None
):
    

    return copy_file(shakedown.master_ip(), file_path, remote_path, username, key_path)
1074248
def move(self, new_path, replication=None):
        
        if not new_path.startswith('/'):
            new_path = self._normpath(self.dirname + '/' + new_path)
        else:
            new_path = self._normpath(new_path)
        if new_path == self.path:
            raise ValueError('New path should be different from the original one.')
        update_def = self.UpdateRequestXML(path=new_path)
        if replication:
            update_def.replication = replication
        headers = {
            'Content-Type': 'application/xml',
            'x-odps-volume-fs-path': self.path,
        }
        self._client.put(self.parent.resource(), params={'meta': ''}, headers=headers, data=update_def.serialize())
        self._del_cache(self.path)
        self.path = new_path
        self.reload()
289575
def get_versioned_files(cls):
        """"""
        files = cls._git_ls_files()
        submodules = cls._list_submodules()
        for subdir in submodules:
            subdir = os.path.relpath(subdir).replace(os.path.sep, '/')
            files += add_prefix_to_each(subdir, cls._git_ls_files(subdir))
        return add_directories(files)
1046937
def replace(self, source, dest):
        """"""
        for i, broker in enumerate(self.replicas):
            if broker == source:
                self.replicas[i] = dest
                return
971676
def rewrite_input_paths(self):
        
        for local_path, remote_path in self.file_renames.items():
            self.job_inputs.rewrite_paths(local_path, remote_path)
144530
def move(ctx, client, sources, destination):
    """"""
    from renku.api._git import _expand_directories

    dst = Path(destination)

    def fmt_path(path):
        """Format path as relative to the client path."""
        return str(Path(path).absolute().relative_to(client.path))

    files = {
        fmt_path(source): fmt_path(file_or_dir)
        for file_or_dir in sources
        for source in _expand_directories((file_or_dir, ))
    }

    def fmt_dst(path):
        """Build a destination path for a source path."""
        return str(dst / os.path.relpath(path, start=files[path]))

    destinations = {source: fmt_dst(source) for source in files}

    # 1. Check .gitignore.
    ignored = client.find_ignored_paths(*destinations.values())
    if ignored:
        click.echo(WARNING + 'Renamed files match .gitignore.\n')
        if click.confirm(
            'Do you want to edit ".gitignore" now?', default=False
        ):
            click.edit(filename=str(client.path / '.gitignore'))

    # 2. Update dataset metadata files.
    with progressbar(
        client.datasets.items(),
        item_show_func=lambda item: str(item[1].short_id) if item else '',
        label='Updating dataset metadata',
        width=0,
    ) as bar:
        for (path, dataset) in bar:
            renames = {}

            for file in dataset.files:
                filepath = fmt_path(os.path.normpath(str(path.parent / file)))

                if filepath in files:
                    renames[file] = os.path.relpath(
                        destinations[filepath], start=str(path.parent)
                    )

            if renames:
                dataset = dataset.rename_files(
                    lambda key: renames.get(key, key)
                )

                dataset.to_yaml()

    # 3. Manage .gitattributes for external storage.
    tracked = tuple(
        path for path, attr in client.find_attr(*files).items()
        if attr.get('filter') == 'lfs'
    )
    client.untrack_paths_from_storage(*tracked)
    existing = client.find_attr(*tracked)
    if existing:
        click.echo(WARNING + 'There are custom .gitattributes.\n')
        if click.confirm(
            'Do you want to edit ".gitattributes" now?', default=False
        ):
            click.edit(filename=str(client.path / '.gitattributes'))

    client.track_paths_in_storage(*(destinations[path] for path in tracked))

    # 4. Handle symlinks.
    dst.parent.mkdir(parents=True, exist_ok=True)

    for source, target in destinations.items():
        src = Path(source)
        if src.is_symlink():
            Path(target).parent.mkdir(parents=True, exist_ok=True)
            Path(target).symlink_to(
                os.path.relpath(
                    str(src.resolve()), start=os.path.dirname(target)
                )
            )
            src.unlink()
            del files[source]

    # Finally move the files.
    final_sources = list(set(files.values()))
    if final_sources:
        run(['git', 'mv'] + final_sources + [destination], check=True)
1301211
def _copy_module(old):
    """"""
    saved = sys.modules.pop(old.__name__, None)
    new = __import__(old.__name__)
    sys.modules[old.__name__] = saved
    return new
165388
def run(self):
        """"""
        old_inplace, self.inplace = self.inplace, 0
        _build_ext.run(self)
        self.inplace = old_inplace
        if old_inplace:
            self.copy_extensions_to_source()
704678
def clone(self, repo, ref, deps=()):
        """"""

        if os.path.isdir(repo):
            repo = os.path.abspath(repo)

        def clone_strategy(directory):
            env = git.no_git_env()

            def _git_cmd(*args):
                cmd_output('git', *args, cwd=directory, env=env)

            _git_cmd('init', '.')
            _git_cmd('remote', 'add', 'origin', repo)

            try:
                self._shallow_clone(ref, _git_cmd)
            except CalledProcessError:
                self._complete_clone(ref, _git_cmd)

        return self._new_repo(repo, ref, deps, clone_strategy)