// EndBackup is part of the backupstorage.BackupHandle interface.
771553
func (bh *S3BackupHandle) EndBackup(ctx context.Context) error {
	if bh.readOnly {
		return fmt.Errorf("EndBackup cannot be called on read-only backup")
	}
	bh.waitGroup.Wait()
	return bh.errors.Error()
}
1279846
func (dc *Config) Backend(base backend.B) backend.B {
	return &caching.Backend{
		B:           base,
		FailOnError: !dc.FailOpen,
		CacheGet:    dc.cacheGet,
	}
}
557039
func (f *Facade) Backup(ctx datastore.Context, w io.Writer, excludes []string, snapshotSpacePercent int, backupFilename string) error {
	defer ctx.Metrics().Stop(ctx.Metrics().Start("Facade.Backup"))
	// Do not DFSLock here, ControlPlaneDao does that
	stime := time.Now()
	message := fmt.Sprintf("started backup at %s", stime.UTC())
	plog.WithField("excludes", excludes).Info("Started backup")
	alog := f.auditLogger.Message(ctx, "Started Backup").
		Action(audit.Backup).
		WithFields(logrus.Fields{
				"starttime": stime.UTC().Format("2006-01-02-150405"),
				"backupfile": backupFilename})
	alog.Succeeded()
	alog = f.auditLogger.Message(ctx, "Completed Backup").
		Action(audit.Backup)
	templates, images, err := f.GetServiceTemplatesAndImages(ctx)
	if err != nil {
		plog.WithError(err).Debug("Could not get service templates and images")
		return alog.Error(err)
	}
	plog.WithField("elapsed", time.Since(stime)).Info("Loaded templates and their images")
	pools, err := f.GetResourcePools(ctx)
	if err != nil {
		plog.WithError(err).Debug("Could not get resource pools")
		return alog.Error(err)
	}
	plog.WithField("elapsed", time.Since(stime)).Info("Loaded resource pools")
	tenants, err := f.GetTenantIDs(ctx)
	if err != nil {
		plog.WithError(err).Debug("Could not get tenants")
		return alog.Error(err)
	}
	snapshots := make([]string, len(tenants))
	snapshotExcludes := map[string][]string{}
	for i, tenant := range tenants {
		tenantLogger := plog.WithField("tenant", tenant)
		tag := fmt.Sprintf("backup-%s-%s", tenant, stime)
		snapshot, err := f.Snapshot(ctx, tenant, message, []string{tag}, snapshotSpacePercent)
		if err != nil {
			tenantLogger.WithError(err).Debug("Could not snapshot tenant")
			return alog.Error(err)
		}

		defer func(tenant, snapshot, tag string) {
			if err := f.DeleteSnapshot(ctx, snapshot); err != nil {
				tenantLogger.WithError(err).Warn("Could not delete snapshot; untagging for consumption by TTL")
				if _, err := f.RemoveSnapshotTag(ctx, tenant, tag); err != nil {
					tenantLogger.WithError(err).Error("Could not untag snapshot.  Snapshot must be deleted manually!")
				}
			}
		}(tenant, snapshot, tag)

		snapshots[i] = snapshot
		snapshotExcludes[snapshot] = append(excludes, f.getExcludedVolumes(ctx, tenant)...)
		tenantLogger.WithField("snapshot", snapshot).Info("Created a snapshot for tenant")
	}
	plog.WithField("elapsed", time.Since(stime)).Info("Loaded tenants")
	data := dfs.BackupInfo{
		Templates:        templates,
		BaseImages:       images,
		Pools:            pools,
		Snapshots:        snapshots,
		SnapshotExcludes: snapshotExcludes,
		Timestamp:        stime,
		BackupVersion:    1,
	}
	plog.WithField("data", data).Info("Calling dfs.Backup")
	if err := f.dfs.Backup(data, w); err != nil {
		plog.WithError(err).Debug("Could not backup")
		return alog.Error(err)
	}
	duration := time.Since(stime)
	plog.WithField("duration", duration).Info("Completed backup")
	alog.WithFields(logrus.Fields{
				"backupfile": backupFilename,
				"elasped": fmt.Sprintf("%fsec", duration.Seconds()),
			}).Succeeded()
	return nil
}
613559
func (bs *CephBackupStorage) Close() error {
	bs.mu.Lock()
	defer bs.mu.Unlock()

	if bs._client != nil {
		// a new client the next time one is needed.
		bs._client = nil
	}
	return nil
}
1370222
func (fbs *FileBackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) {
	// Make sure the directory exists.
	p := path.Join(*FileBackupStorageRoot, dir)
	if err := os.MkdirAll(p, os.ModePerm); err != nil {
		return nil, err
	}

	// Create the subdirectory for this named backup.
	p = path.Join(p, name)
	if err := os.Mkdir(p, os.ModePerm); err != nil {
		return nil, err
	}

	return &FileBackupHandle{
		fbs:      fbs,
		dir:      dir,
		name:     name,
		readOnly: false,
	}, nil
}
1091068
func (be *BuiltinBackupEngine) backupFile(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fe *FileEntry, name string, hookExtraEnv map[string]string) (err error) {
	// Open the source file for reading.
	var source *os.File
	source, err = fe.open(cnf, true)
	if err != nil {
		return err
	}
	defer source.Close()

	fi, err := source.Stat()
	if err != nil {
		return err
	}

	logger.Infof("Backing up file: %v", fe.Name)
	// Open the destination file for writing, and a buffer.
	wc, err := bh.AddFile(ctx, name, fi.Size())
	if err != nil {
		return vterrors.Wrapf(err, "cannot add file: %v,%v", name, fe.Name)
	}
	defer func(name, fileName string) {
		if rerr := wc.Close(); rerr != nil {
			if err != nil {
				// We already have an error, just log this one.
				logger.Errorf2(rerr, "failed to close file %v,%v", name, fe.Name)
			} else {
				err = rerr
			}
		}
	}(name, fe.Name)
	dst := bufio.NewWriterSize(wc, writerBufferSize)

	// Create the hasher and the tee on top.
	hasher := newHasher()
	writer := io.MultiWriter(dst, hasher)

	// Create the external write pipe, if any.
	var pipe io.WriteCloser
	var wait hook.WaitFunc
	if *backupStorageHook != "" {
		h := hook.NewHook(*backupStorageHook, []string{"-operation", "write"})
		h.ExtraEnv = hookExtraEnv
		pipe, wait, _, err = h.ExecuteAsWritePipe(writer)
		if err != nil {
			return vterrors.Wrapf(err, "'%v' hook returned error", *backupStorageHook)
		}
		writer = pipe
	}

	// Create the gzip compression pipe, if necessary.
	var gzip *pgzip.Writer
	if *backupStorageCompress {
		gzip, err = pgzip.NewWriterLevel(writer, pgzip.BestSpeed)
		if err != nil {
			return vterrors.Wrap(err, "cannot create gziper")
		}
		gzip.SetConcurrency(*backupCompressBlockSize, *backupCompressBlocks)
		writer = gzip
	}

	// Copy from the source file to writer (optional gzip,
	// optional pipe, tee, output file and hasher).
	_, err = io.Copy(writer, source)
	if err != nil {
		return vterrors.Wrap(err, "cannot copy data")
	}

	// Close gzip to flush it, after that all data is sent to writer.
	if gzip != nil {
		if err = gzip.Close(); err != nil {
			return vterrors.Wrap(err, "cannot close gzip")
		}
	}

	// Close the hook pipe if necessary.
	if pipe != nil {
		if err := pipe.Close(); err != nil {
			return vterrors.Wrap(err, "cannot close hook pipe")
		}
		stderr, err := wait()
		if stderr != "" {
			logger.Infof("'%v' hook returned stderr: %v", *backupStorageHook, stderr)
		}
		if err != nil {
			return vterrors.Wrapf(err, "'%v' returned error", *backupStorageHook)
		}
	}

	// Flush the buffer to finish writing on destination.
	if err = dst.Flush(); err != nil {
		return vterrors.Wrapf(err, "cannot flush destination: %v", name)
	}

	// Save the hash.
	fe.Hash = hasher.HashString()
	return nil
}
50397
func (bs *CephBackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) {
	c, err := bs.client()
	if err != nil {
		return nil, err
	}
	// ceph bucket name
	bucket := alterBucketName(dir)

	// List prefixes that begin with dir (i.e. list subdirs).
	var subdirs []string
	searchPrefix := objName(dir, "")

	doneCh := make(chan struct{})
	for object := range c.ListObjects(bucket, searchPrefix, false, doneCh) {
		if object.Err != nil {
			_, err := c.BucketExists(bucket)
			if err != nil {
				return nil, nil
			}
			return nil, object.Err
		}
		subdir := strings.TrimPrefix(object.Key, searchPrefix)
		subdir = strings.TrimSuffix(subdir, "/")
		subdirs = append(subdirs, subdir)
	}

	// Backups must be returned in order, oldest first.
	sort.Strings(subdirs)

	result := make([]backupstorage.BackupHandle, 0, len(subdirs))
	for _, subdir := range subdirs {
		result = append(result, &CephBackupHandle{
			client:   c,
			bs:       bs,
			dir:      dir,
			name:     subdir,
			readOnly: true,
		})
	}
	return result, nil
}
1329127
func GetBackupStorage() (BackupStorage, error) {
	bs, ok := BackupStorageMap[*BackupStorageImplementation]
	if !ok {
		return nil, fmt.Errorf("no registered implementation of BackupStorage")
	}
	return bs, nil
}
1300153
func (bs *S3BackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) {
	log.Infof("StartBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, *bucket)
	c, err := bs.client()
	if err != nil {
		return nil, err
	}

	return &S3BackupHandle{
		client:   c,
		bs:       bs,
		dir:      dir,
		name:     name,
		readOnly: false,
	}, nil
}
4462
func (fbs *FileBackupStorage) RemoveBackup(ctx context.Context, dir, name string) error {
	p := path.Join(*FileBackupStorageRoot, dir, name)
	return os.RemoveAll(p)
}
346321
func (s *DefaultStorageFactory) Backends() []Backend {
	servers := sets.NewString(s.StorageConfig.Transport.ServerList...)

	for _, overrides := range s.Overrides {
		servers.Insert(overrides.etcdLocation...)
	}

	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	if len(s.StorageConfig.Transport.CertFile) > 0 && len(s.StorageConfig.Transport.KeyFile) > 0 {
		cert, err := tls.LoadX509KeyPair(s.StorageConfig.Transport.CertFile, s.StorageConfig.Transport.KeyFile)
		if err != nil {
			klog.Errorf("failed to load key pair while getting backends: %s", err)
		} else {
			tlsConfig.Certificates = []tls.Certificate{cert}
		}
	}
	if len(s.StorageConfig.Transport.CAFile) > 0 {
		if caCert, err := ioutil.ReadFile(s.StorageConfig.Transport.CAFile); err != nil {
			klog.Errorf("failed to read ca file while getting backends: %s", err)
		} else {
			caPool := x509.NewCertPool()
			caPool.AppendCertsFromPEM(caCert)
			tlsConfig.RootCAs = caPool
			tlsConfig.InsecureSkipVerify = false
		}
	}

	backends := []Backend{}
	for server := range servers {
		backends = append(backends, Backend{
			Server: server,
			// We can't share TLSConfig across different backends to avoid races.
			// For more details see: http://pr.k8s.io/59338
			TLSConfig: tlsConfig.Clone(),
		})
	}
	return backends
}
695345
func (bs *S3BackupStorage) RemoveBackup(ctx context.Context, dir, name string) error {
	log.Infof("RemoveBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, *bucket)

	c, err := bs.client()
	if err != nil {
		return err
	}

	query := &s3.ListObjectsV2Input{
		Bucket: bucket,
		Prefix: objName(dir, name),
	}

	for {
		objs, err := c.ListObjectsV2(query)
		if err != nil {
			return err
		}

		objIds := make([]*s3.ObjectIdentifier, 0, len(objs.Contents))
		for _, obj := range objs.Contents {
			objIds = append(objIds, &s3.ObjectIdentifier{
				Key: obj.Key,
			})
		}

		quiet := true // return less in the Delete response
		out, err := c.DeleteObjects(&s3.DeleteObjectsInput{
			Bucket: bucket,
			Delete: &s3.Delete{
				Objects: objIds,
				Quiet:   &quiet,
			},
		})

		if err != nil {
			return err
		}

		for _, objError := range out.Errors {
			return fmt.Errorf(objError.String())
		}

		if objs.NextContinuationToken == nil {
			break
		}

		query.ContinuationToken = objs.NextContinuationToken
	}

	return nil
}
310368
func (t *tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}
201414
func (c *ServicedCli) initBackup() {
	c.app.Commands = append(
		c.app.Commands,
		cli.Command{
			Name:        "backup",
			Usage:       "Dump all templates and services to a tgz file",
			Description: "serviced backup DIRPATH",
			Action:      c.cmdBackup,
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "exclude",
					Value: &cli.StringSlice{},
					Usage: "Subdirectory of the tenant volume to exclude from backup",
				},
				cli.BoolFlag{
					Name: "check",
					Usage: "check space, but do not do backup",
				},
				cli.BoolFlag{
					Name: "force",
					Usage: "attempt backup even if space check fails",
				},
			},
		},
		cli.Command{
			Name:        "restore",
			Usage:       "Restore templates and services from a tgz file",
			Description: "serviced restore FILEPATH",
			Action:      c.cmdRestore,
		},
	)
}
614195
func (dao *ControlPlaneDao) AsyncBackup(backupRequest model.BackupRequest, filename *string) (err error) {
	ctx := datastore.Get()
	if len(backupRequest.Username) > 0 {
		ctx.SetUser(backupRequest.Username)
	}
	dfslocker := dao.facade.DFSLock(ctx)
	dfslocker.Lock("backup")
	inprogress.Reset()
	dfslocker.Unlock()
	go dao.Backup(backupRequest, filename)
	return
}
672023
func (builder *StoreBucketTypePropsCommandBuilder) WithBackend(backend string) *StoreBucketTypePropsCommandBuilder {
	builder.props.Backend = []byte(backend)
	return builder
}
1055040
func (s *NfsInfo) GetPersistanceBackup() (dumper PersistanceBackup, err error) {
	return NewNFSBackup(s.User, s.Pass, s.Ip, s.SSHPrivateKey, s.RemoteArchivePath, s.BackupType)
}