// monitorContainer tracks the running state of the container.
// runs when the container dies
169903
func (a *HostAgent) monitorContainer(logger *log.Entry, ctr *docker.Container) <-chan time.Time {
	ev := make(chan time.Time, 1)
	ctr.OnEvent(docker.Die, func(_ string) {
		someSlice := strings.Split(ctr.Name, "-")
		serviceID := someSlice[0]
		instanceID, err := strconv.ParseInt(someSlice[1], 10, 0)
		a.setInstanceState(serviceID, int(instanceID), service.StateStopped)
		defer close(ev)
		dctr, err := ctr.Inspect()
		if err != nil {
			logger.WithError(err).Error("Could not look up container")
			ev <- time.Now()
			return
		}

		logger.WithFields(log.Fields{
			"terminated": dctr.State.FinishedAt,
			"exitcode":   dctr.State.ExitCode,
		}).Debug("Container exited")

		if dctr.State.ExitCode != 0 || log.GetLevel() == log.DebugLevel {
			dockerLogsToFile(ctr.ID, 1000)
		}

		if err := ctr.Delete(true); err != nil {
			logger.WithError(err).Warn("Could not delete container")
		}

		// just in case something unusual happened
		if !dctr.State.FinishedAt.IsZero() {
			ev <- dctr.State.FinishedAt
		} else {
			ev <- time.Now()
		}
		return
	})
	return ev
}
143861
func (r *runtimeVM) PauseContainer(c *Container) error {
	logrus.Debug("runtimeVM.PauseContainer() start")
	defer logrus.Debug("runtimeVM.PauseContainer() end")

	// Lock the container
	c.opLock.Lock()
	defer c.opLock.Unlock()

	if _, err := r.task.Pause(r.ctx, &task.PauseRequest{
		ID: c.ID(),
	}); err != nil {
		return errdefs.FromGRPC(err)
	}

	return nil
}
1316496
func (a *HostAgent) RestartContainer(cancel <-chan interface{}, serviceID string, instanceID int) error {
	logger := plog.WithFields(log.Fields{
		"serviceid":  serviceID,
		"instanceid": instanceID,
	})

	// look up the container to get the image
	ctrName := fmt.Sprintf("%s-%d", serviceID, instanceID)
	ctr, err := docker.FindContainer(ctrName)
	if err == docker.ErrNoSuchContainer {
		// container has been deleted so we will pull when the container starts
		// again.
		logger.Debug("Container not found")
		return nil
	} else if err != nil {
		logger.WithError(err).Debug("Could not look up container")
		return err
	}

	if !ctr.IsRunning() {
		// container has stopped, so we will pull when the container starts
		// again; the event monitor will handle the stopped container
		logger.Debug("Container stopped")
		return nil
	}

	go func() {
		a.setInstanceState(serviceID, instanceID, service.StatePulling)
		for {
			// relentlessly try to pull the image
			_, _, err := a.pullImage(logger, cancel, ctr.Config.Image)
			if err != nil {
				logger.WithError(err).Debug("Could not pull the service image")
				// wait 5 seconds and try again
				select {
				case <-time.After(5 * time.Second):
				case <-cancel:
					logger.Info("Cancelled image pull")
					return
				}
				continue
			}
			logger.Debug("Pulled image")
			break
		}

		a.setInstanceState(serviceID, instanceID, service.StateStopping)
		// set the container to stop; ctr.Stop() stops the container by
		// container id and not name, so if the container was stopped or
		// deleted before the pull is successful, then this will just be a
		// no-op.  The restart of the container is handled by the delegate once
		// it is notified that the container has stopped.
		if err := ctr.Stop(45 * time.Second); err != nil {
			logger.WithError(err).Debug("Could not stop container")
		}
	}()

	return nil
}
537351
func (r *Runtime) ExecContainer(c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error {
	impl, err := r.RuntimeImpl(c)
	if err != nil {
		return err
	}

	return impl.ExecContainer(c, cmd, stdin, stdout, stderr, tty, resize)
}
802135
func (ws *watcherState) handlingContainerID(id string) bool {
	ws.Lock()
	defer ws.Unlock()

	_, handled := ws.events[id]
	return handled
}
490431
func (c *Calcium) ControlContainer(ctx context.Context, IDs []string, t string) (chan *types.ControlContainerMessage, error) {
	ch := make(chan *types.ControlContainerMessage)

	go func() {
		defer close(ch)
		wg := sync.WaitGroup{}
		for _, ID := range IDs {
			container, err := c.GetContainer(ctx, ID)
			if err != nil {
				ch <- &types.ControlContainerMessage{
					ContainerID: ID,
					Error:       err,
				}
				continue
			}

			wg.Add(1)
			go func(container *types.Container) {
				defer wg.Done()
				var err error
				var message []*bytes.Buffer
				defer func() {
					if err == nil {
						log.Infof("[ControlContainer] Control container %s %s", container.ID, t)
						log.Infof("[ControlContainer] Output:\n %s", string(types.HookOutput(message)))
					}
					ch <- &types.ControlContainerMessage{
						ContainerID: container.ID,
						Error:       err,
						Hook:        message,
					}
				}()

				containerInfo, err := container.Inspect(ctx)
				if err != nil {
					return
				}

				switch t {
				case cluster.ContainerStop:
					message, err = c.doStopContainer(ctx, container, containerInfo, false)
					return
				case cluster.ContainerStart:
					message, err = c.doStartContainer(ctx, container, containerInfo)
					return
				case cluster.ContainerRestart:
					message, err = c.doStopContainer(ctx, container, containerInfo, false)
					if err != nil {
						return
					}
					m2, e2 := c.doStartContainer(ctx, container, containerInfo)
					message = append(message, m2...)
					if e2 != nil {
						err = e2
					}
					return
				default:
					err = types.ErrUnknownControlType
				}
			}(container)
		}
		wg.Wait()
	}()

	return ch, nil
}
1074329
func (r *runtimeVM) StartContainer(c *Container) error {
	logrus.Debug("runtimeVM.startContainer() start")
	defer logrus.Debug("runtimeVM.startContainer() end")

	// Lock the container
	c.opLock.Lock()
	defer c.opLock.Unlock()

	if err := r.start(r.ctx, c.ID(), ""); err != nil {
		return err
	}

	// Spawn a goroutine waiting for the container to terminate. Once it
	// happens, the container status is retrieved to be updated.
	go func() {
		r.wait(r.ctx, c.ID(), "")
		r.UpdateContainerStatus(c)
	}()

	return nil
}
1263302
func (s *State) Monitor() monitor.Monitor {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.monitor
}
1369051
func (d *Daemon) initHealthMonitor(c *container.Container) {
	// If no healthcheck is setup then don't init the monitor
	if getProbe(c) == nil {
		return
	}

	// This is needed in case we're auto-restarting
	d.stopHealthchecks(c)

	if h := c.State.Health; h != nil {
		h.SetStatus(types.Starting)
		h.FailingStreak = 0
	} else {
		h := &container.Health{}
		h.SetStatus(types.Starting)
		c.State.Health = h
	}

	d.updateHealthMonitor(c)
}
435248
func PodContainerRunning(containerName string) watchtools.ConditionFunc {
	return func(event watch.Event) (bool, error) {
		switch event.Type {
		case watch.Deleted:
			return false, errors.NewNotFound(schema.GroupResource{Resource: "pods"}, "")
		}
		switch t := event.Object.(type) {
		case *corev1.Pod:
			switch t.Status.Phase {
			case corev1.PodRunning, corev1.PodPending:
			case corev1.PodFailed, corev1.PodSucceeded:
				return false, kubectl.ErrPodCompleted
			default:
				return false, nil
			}
			for _, s := range t.Status.ContainerStatuses {
				if s.Name != containerName {
					continue
				}
				if s.State.Terminated != nil {
					return false, ErrContainerTerminated
				}
				return s.State.Running != nil, nil
			}
			for _, s := range t.Status.InitContainerStatuses {
				if s.Name != containerName {
					continue
				}
				if s.State.Terminated != nil {
					return false, ErrContainerTerminated
				}
				return s.State.Running != nil, nil
			}
			return false, nil
		}
		return false, nil
	}
}
1359404
func (c *Client) KillContainer(opts KillContainerOptions) error {
	path := "/containers/" + opts.ID + "/kill" + "?" + queryString(opts)
	resp, err := c.do("POST", path, doOptions{context: opts.Context})
	if err != nil {
		e, ok := err.(*Error)
		if !ok {
			return err
		}
		switch e.Status {
		case http.StatusNotFound:
			return &NoSuchContainer{ID: opts.ID}
		case http.StatusConflict:
			return &ContainerNotRunning{ID: opts.ID}
		default:
			return err
		}
	}
	resp.Body.Close()
	return nil
}
351059
func (r *runtimeVM) UpdateContainer(c *Container, res *rspec.LinuxResources) error {
	logrus.Debug("runtimeVM.updateContainer() start")
	defer logrus.Debug("runtimeVM.updateContainer() end")

	// Lock the container
	c.opLock.Lock()
	defer c.opLock.Unlock()

	// Convert resources into protobuf Any type
	any, err := typeurl.MarshalAny(res)
	if err != nil {
		return err
	}

	if _, err := r.task.Update(r.ctx, &task.UpdateTaskRequest{
		ID:        c.ID(),
		Resources: any,
	}); err != nil {
		return errdefs.FromGRPC(err)
	}

	return nil
}
488502
func (r *runtimeOCI) StartContainer(c *Container) error {
	c.opLock.Lock()
	defer c.opLock.Unlock()

	if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr,
		r.path, rootFlag, r.root, "start", c.id); err != nil {

		return err
	}
	c.state.Started = time.Now()
	return nil
}
719050
func (f *FakeDockerClient) StopContainer(id string, timeout time.Duration) error {
	f.Lock()
	defer f.Unlock()
	f.appendCalled(CalledDetail{name: "stop"})
	if err := f.popError("stop"); err != nil {
		return err
	}
	f.appendContainerTrace("Stopped", id)
	// Container status should be Updated before container moved to ExitedContainerList
	f.updateContainerStatus(id, StatusExitedPrefix)
	var newList []dockertypes.Container
	for _, container := range f.RunningContainerList {
		if container.ID == id {
			// The newest exited container should be in front. Because we assume so in GetPodStatus()
			f.ExitedContainerList = append([]dockertypes.Container{container}, f.ExitedContainerList...)
			continue
		}
		newList = append(newList, container)
	}
	f.RunningContainerList = newList
	container, ok := f.ContainerMap[id]
	if !ok {
		container = convertFakeContainer(&FakeContainer{
			ID:         id,
			Name:       id,
			Running:    false,
			StartedAt:  time.Now().Add(-time.Second),
			FinishedAt: time.Now(),
		})
	} else {
		container.State.FinishedAt = dockerTimestampToString(f.Clock.Now())
		container.State.Running = false
	}
	f.ContainerMap[id] = container
	f.normalSleep(200, 50, 50)
	return nil
}
454946
func monitor(d *Daemon, c *container.Container, stop chan struct{}, probe probe) {
	probeTimeout := timeoutWithDefault(c.Config.Healthcheck.Timeout, defaultProbeTimeout)
	probeInterval := timeoutWithDefault(c.Config.Healthcheck.Interval, defaultProbeInterval)
	for {
		select {
		case <-stop:
			logrus.Debugf("Stop healthcheck monitoring for container %s (received while idle)", c.ID)
			return
		case <-time.After(probeInterval):
			logrus.Debugf("Running health check for container %s ...", c.ID)
			startTime := time.Now()
			ctx, cancelProbe := context.WithTimeout(context.Background(), probeTimeout)
			results := make(chan *types.HealthcheckResult, 1)
			go func() {
				healthChecksCounter.Inc()
				result, err := probe.run(ctx, d, c)
				if err != nil {
					healthChecksFailedCounter.Inc()
					logrus.Warnf("Health check for container %s error: %v", c.ID, err)
					results <- &types.HealthcheckResult{
						ExitCode: -1,
						Output:   err.Error(),
						Start:    startTime,
						End:      time.Now(),
					}
				} else {
					result.Start = startTime
					logrus.Debugf("Health check for container %s done (exitCode=%d)", c.ID, result.ExitCode)
					results <- result
				}
				close(results)
			}()
			select {
			case <-stop:
				logrus.Debugf("Stop healthcheck monitoring for container %s (received while probing)", c.ID)
				cancelProbe()
				// Wait for probe to exit (it might take a while to respond to the TERM
				// signal and we don't want dying probes to pile up).
				<-results
				return
			case result := <-results:
				handleProbeResult(d, c, result, stop)
				// Stop timeout
				cancelProbe()
			case <-ctx.Done():
				logrus.Debugf("Health check for container %s taking too long", c.ID)
				handleProbeResult(d, c, &types.HealthcheckResult{
					ExitCode: -1,
					Output:   fmt.Sprintf("Health check exceeded timeout (%v)", probeTimeout),
					Start:    startTime,
					End:      time.Now(),
				}, stop)
				cancelProbe()
				// Wait for probe to exit (it might take a while to respond to the TERM
				// signal and we don't want dying probes to pile up).
				<-results
			}
		}
	}
}
378172
func (h *healthChecker) runHealthCheck() {
	t := h.s.now()
	err := checkStorageHealth(h.s.storage, h.s.now)
	passed := h.s.now().Sub(t)
	if err != nil {
		h.s.logger.Errorf("Storage health check failed: %v", err)
	}

	// Make sure to only hold the mutex to access the fields, and not while
	// we're querying the storage object.
	h.mu.Lock()
	h.err = err
	h.passed = passed
	h.mu.Unlock()
}
1236342
func (c *client) terminateContainer(ctr *container) error {
	const terminateTimeout = time.Minute * 5
	ctr.terminateInvoked = true
	err := ctr.hcsContainer.Terminate()

	if hcsshim.IsPending(err) {
		err = ctr.hcsContainer.WaitTimeout(terminateTimeout)
	} else if hcsshim.IsAlreadyStopped(err) {
		err = nil
	}

	if err != nil {
		c.logger.WithError(err).WithField("container", ctr.id).
			Debug("failed to terminate container")
		return err
	}

	return nil
}