// rollingRestart restarts a service one instance at a time and waits for it to reach pass health checks.
170376
func (f *Facade) rollingRestart(ctx datastore.Context, svc *service.Service, timeout time.Duration, cancel <-chan interface{}) error {
	logger := plog.WithFields(log.Fields{
		"serviceid":   svc.ID,
		"servicename": svc.Name,
	})

	// Run through and set all instances to "Pending Restart"
	for instanceID := 0; instanceID < svc.Instances; instanceID++ {
		err := f.zzk.UpdateInstanceCurrentState(ctx, svc.PoolID, svc.ID, instanceID, service.StatePendingRestart)
		if err != nil {
			logger.WithError(err).Debug("Failed to update instance current state to pending restart")
			return err
		}
	}

	// Keep track of which instances haven't been restarted yet, so we can revert the current state if we exit
	// prematurely
	nextInstance := 0
	defer func() {
		for instance := nextInstance; instance < svc.Instances; instance++ {
			err := f.zzk.UpdateInstanceCurrentState(ctx, svc.PoolID, svc.ID, instance, service.StateRunning)
			if err != nil {
				logger.WithField("instance", instance).WithError(err).Error("Failed to revert instance current state to started")
			}
		}
	}()

	// Build the service health object to use for getting instance health
	svch := service.BuildServiceHealth(*svc)
	var punctualLock sync.RWMutex
	punctualInstances := 0

	for instanceID := 0; instanceID < svc.Instances; instanceID++ {
		ilogger := logger.WithField("instance", instanceID)
		ilogger.Debug("Restarting instance")

		select {
		case <-cancel:
			return nil
		default:
		}

		// Set up the timeout
		cancelWait := make(chan struct{})
		done := make(chan struct{})
		timer := time.NewTimer(timeout)
		go func() {
			select {
			case <-timer.C:
				ilogger.Warn("Timeout waiting for instance to restart")
			case <-done:
				punctualLock.Lock()
				punctualInstances++
				punctualLock.Unlock()
			case <-cancel:
				ilogger.Debug("Rolling restart cancelled")
			}
			timer.Stop()
			close(cancelWait)
		}()

		// Before we restart, check the current instance's container ID
		state, err := f.zzk.GetServiceState(ctx, svc.PoolID, svc.ID, instanceID)
		if err != nil {
			close(done)
			logger.WithError(err).Debug("Failed to get service's current container ID")
			return err
		}

		oldContainer := state.ContainerID

		if err = f.zzk.RestartInstance(ctx, svc.PoolID, svc.ID, instanceID); err != nil {
			close(done)
			logger.WithError(err).Debug("Failed to restart instance")
			return err
		}

		// Increment nextInstance now, since we have started the restart process for this instance
		nextInstance++

		// Wait for the instance's containerID to change
		checkContainer := func(s *zkservice.State, exists bool) bool {
			if !exists {
				return true
			}
			if s.ContainerID != "" && s.ContainerID != oldContainer {
				return service.InstanceCurrentState(s.Status) == service.StateRunning
			}
			return false
		}
		if err = f.zzk.WaitInstance(ctx, svc, instanceID, checkContainer, cancelWait); err != nil {
			close(done)
			logger.WithError(err).Debug("Failed to wait on instance")
			return err
		}

		// Check if we're ready to move on to the next instance on an interval
		ready := func() bool {
			// I don't care if it is the last instance
			if instanceID == svc.Instances-1 {
				return true
			}

			// Wait for health checks to pass
			statuses := f.getInstanceHealth(svch, instanceID)
			for key, status := range statuses {
				ilogger.WithFields(log.Fields{
					"status": status,
					"key":    key,
				}).Debug("Got health status for instance")
				if status != health.OK {
					return false
				}
			}

			return true
		}

		hctimer := time.NewTimer(500 * time.Millisecond)

		// See if we have timed out
		cancelled := false
		select {
		case <-cancelWait:
			cancelled = true
		default:
		}

		for !cancelled && !ready() {
			ilogger.Debug("Instance not ready yet, checking again in 500 ms")
			select {
			case <-cancelWait:
				cancelled = true
			case <-hctimer.C:
				hctimer.Stop()
				hctimer.Reset(500 * time.Millisecond)
			}
		}
		hctimer.Stop()
		ilogger.Debug("Done restarting instance")
		close(done)
		<-cancelWait
	}

	punctualLock.RLock()
	instances := punctualInstances
	punctualLock.RUnlock()
	if instances == svc.Instances {
		f.SetServicesCurrentState(ctx, service.SVCCSRunning, svc.ID)
	}

	return nil
}
1110692
func (c *ServicedCli) cmdSnapshotRollback(ctx *cli.Context) {
	args := ctx.Args()
	if len(args) < 1 {
		fmt.Printf("Incorrect Usage.\n\n")
		cli.ShowCommandHelp(ctx, "rollback")
		return
	}

	if err := c.driver.Rollback(args[0], ctx.Bool("force-restart")); err != nil {
		fmt.Fprintln(os.Stderr, err)
		c.exit(1)
	} else {
		fmt.Println(args[0])
	}
}
54396
func (s *HlsMediaStoreSettings) SetRestartDelay(v int64) *HlsMediaStoreSettings {
	s.RestartDelay = &v
	return s
}
60026
func startRollingBackRotation(ca services.CertAuthority) error {
	rotation := ca.GetRotation()

	// Rollback always sets rotation to manual mode.
	rotation.Mode = services.RotationModeManual

	signingKeys := ca.GetSigningKeys()
	checkingKeys := ca.GetCheckingKeys()
	keyPairs := ca.GetTLSKeyPairs()

	// Rotation sets the first key to be the new key
	// and keep only public keys/certs for the new CA.
	signingKeys = [][]byte{signingKeys[1]}
	checkingKeys = [][]byte{checkingKeys[1]}

	// Keep the new certificate as trusted
	// as during the rollback phase, both types of clients may be present in the cluster.
	keyPairs = []services.TLSKeyPair{keyPairs[1], services.TLSKeyPair{Cert: keyPairs[0].Cert}}
	rotation.State = services.RotationStateInProgress
	rotation.Phase = services.RotationPhaseRollback

	ca.SetSigningKeys(signingKeys)
	ca.SetCheckingKeys(checkingKeys)
	ca.SetTLSKeyPairs(keyPairs)
	ca.SetRotation(rotation)
	return nil
}
1014174
func (rhf RollbackHookFunc) Rollback(context map[string]interface{},
	serviceName string,
	awsSession *session.Session,
	noop bool,
	logger *logrus.Logger) error {
	rhf(context,
		serviceName,
		awsSession,
		noop,
		logger)
	return nil
}
655219
func (c *Server) Restart() error {
	c.Lock()
	defer c.Unlock()
	if err := c.hostsDeny(); err != nil {
		return err
	}
	if err := c.hostsAllow(); err != nil {
		return err
	}
	if err := c.writeExports(); err != nil {
		return err
	}
	if err := restart(); err != nil {
		return err
	}
	c.cleanupBindMounts()
	return nil
}
335749
func WithRestartPolicy(policy string) func(c *TestContainerConfig) {
	return func(c *TestContainerConfig) {
		c.HostConfig.RestartPolicy.Name = policy
	}
}
949490
func PsRestart(appID, target string) error {
	c, appID, err := load(appID)

	if err != nil {
		return err
	}

	psType := ""
	psNum := -1

	if target != "" {
		if strings.Contains(target, ".") {
			parts := strings.Split(target, ".")
			psType = parts[0]
			psNum, err = strconv.Atoi(parts[1])

			if err != nil {
				return err
			}
		} else {
			psType = target
		}
	}

	fmt.Printf("Restarting processes... but first, %s!\n", drinkOfChoice())
	startTime := time.Now()
	quit := progress()

	_, err = ps.Restart(c, appID, psType, psNum)

	quit <- true
	<-quit

	if err != nil {
		return err
	}

	fmt.Printf("done in %ds\n", int(time.Since(startTime).Seconds()))

	processes, count, err := ps.List(c, appID, c.ResponseLimit)

	if err != nil {
		return err
	}

	printProcesses(appID, processes, count)
	return nil
}
819102
func (s *HlsWebdavSettings) SetRestartDelay(v int64) *HlsWebdavSettings {
	s.RestartDelay = &v
	return s
}
184689
func NewRestartFrameArgs(callFrameID CallFrameID) *RestartFrameArgs {
	args := new(RestartFrameArgs)
	args.CallFrameID = callFrameID
	return args
}
1150527
func ProjectRestart(p project.APIProject, c *cli.Context) error {
	err := p.Restart(context.Background(), c.Int("timeout"), c.Args()...)
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	return nil
}
566583
func NewRollingUpdater(namespace string, rcClient coreclient.ReplicationControllersGetter, podClient coreclient.PodsGetter, sc scaleclient.ScalesGetter) *RollingUpdater {
	updater := &RollingUpdater{
		rcClient:    rcClient,
		podClient:   podClient,
		scaleClient: sc,
		ns:          namespace,
	}
	// Inject real implementations.
	updater.scaleAndWait = updater.scaleAndWaitWithScaler
	updater.getOrCreateTargetController = updater.getOrCreateTargetControllerWithClient
	updater.getReadyPods = updater.readyPods
	updater.cleanup = updater.cleanupWithClients
	updater.nowFn = metav1.Now
	return updater
}
294669
func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.CloudInstanceGroup, cluster *api.Cluster, instanceGroups *api.InstanceGroupList) error {
	if len(groups) == 0 {
		glog.Info("Cloud Instance Group length is zero. Not doing a rolling-update.")
		return nil
	}

	var resultsMutex sync.Mutex
	results := make(map[string]error)

	masterGroups := make(map[string]*cloudinstances.CloudInstanceGroup)
	nodeGroups := make(map[string]*cloudinstances.CloudInstanceGroup)
	bastionGroups := make(map[string]*cloudinstances.CloudInstanceGroup)
	for k, group := range groups {
		switch group.InstanceGroup.Spec.Role {
		case api.InstanceGroupRoleNode:
			nodeGroups[k] = group
		case api.InstanceGroupRoleMaster:
			masterGroups[k] = group
		case api.InstanceGroupRoleBastion:
			bastionGroups[k] = group
		default:
			return fmt.Errorf("unknown group type for group %q", group.InstanceGroup.ObjectMeta.Name)
		}
	}

	// Upgrade bastions first; if these go down we can't see anything
	{
		var wg sync.WaitGroup

		for k, bastionGroup := range bastionGroups {
			wg.Add(1)
			go func(k string, group *cloudinstances.CloudInstanceGroup) {
				resultsMutex.Lock()
				results[k] = fmt.Errorf("function panic bastions")
				resultsMutex.Unlock()

				defer wg.Done()

				g, err := NewRollingUpdateInstanceGroup(c.Cloud, group)
				if err == nil {
					err = g.RollingUpdate(c, cluster, instanceGroups, true, c.BastionInterval, c.ValidationTimeout)
				}

				resultsMutex.Lock()
				results[k] = err
				resultsMutex.Unlock()
			}(k, bastionGroup)
		}

		wg.Wait()
	}

	// Do not continue update if bastion(s) failed
	for _, err := range results {
		if err != nil {
			return fmt.Errorf("bastion not healthy after update, stopping rolling-update: %q", err)
		}
	}

	// Upgrade masters next
	{
		// We run master nodes in series, even if they are in separate instance groups
		// typically they will be in separate instance groups, so we can force the zones,
		// and we don't want to roll all the masters at the same time.  See issue #284

		for _, group := range masterGroups {
			g, err := NewRollingUpdateInstanceGroup(c.Cloud, group)
			if err == nil {
				err = g.RollingUpdate(c, cluster, instanceGroups, false, c.MasterInterval, c.ValidationTimeout)
			}

			// Do not continue update if master(s) failed, cluster is potentially in an unhealthy state
			if err != nil {
				return fmt.Errorf("master not healthy after update, stopping rolling-update: %q", err)
			}
		}
	}

	// Upgrade nodes, with greater parallelism
	{
		var wg sync.WaitGroup

		// We run nodes in series, even if they are in separate instance groups
		// typically they will not being separate instance groups. If you roll the nodes in parallel
		// you can get into a scenario where you can evict multiple statefulset pods from the same
		// statefulset at the same time. Further improvements needs to be made to protect from this as
		// well.

		wg.Add(1)

		go func() {
			for k := range nodeGroups {
				resultsMutex.Lock()
				results[k] = fmt.Errorf("function panic nodes")
				resultsMutex.Unlock()
			}

			defer wg.Done()

			for k, group := range nodeGroups {
				g, err := NewRollingUpdateInstanceGroup(c.Cloud, group)
				if err == nil {
					err = g.RollingUpdate(c, cluster, instanceGroups, false, c.NodeInterval, c.ValidationTimeout)
				}

				resultsMutex.Lock()
				results[k] = err
				resultsMutex.Unlock()

				// TODO: Bail on error?
			}
		}()

		wg.Wait()
	}

	for _, err := range results {
		if err != nil {
			return err
		}
	}

	glog.Infof("Rolling update completed for cluster %q!", c.ClusterName)
	return nil
}
307714
func (session *SafeSession) SetRollback() {
	if session == nil || session.Session == nil || !session.Session.InTransaction {
		return
	}
	session.mu.Lock()
	defer session.mu.Unlock()
	session.mustRollback = true
}
1311852
func (mp *Proxy) Rollback(ctx context.Context, session *ProxySession) error {
	return mp.doRollback(ctx, session)
}
122829
func (si *SolrInterface) Rollback() (*SolrUpdateResponse, error) {
	return si.Update(M{"rollback": M{}}, nil)
}
192816
func (s *ReleaseServer) RollbackRelease(c ctx.Context, req *services.RollbackReleaseRequest) (*services.RollbackReleaseResponse, error) {
	s.Log("preparing rollback of %s", req.Name)
	currentRelease, targetRelease, err := s.prepareRollback(req)
	if err != nil {
		return nil, err
	}

	if !req.DryRun {
		s.Log("creating rolled back release for %s", req.Name)
		if err := s.env.Releases.Create(targetRelease); err != nil {
			return nil, err
		}
	}
	s.Log("performing rollback of %s", req.Name)
	res, err := s.performRollback(currentRelease, targetRelease, req)
	if err != nil {
		return res, err
	}

	if !req.DryRun {
		s.Log("updating status for rolled back release for %s", req.Name)
		if err := s.env.Releases.Update(targetRelease); err != nil {
			return res, err
		}
	}

	return res, nil
}