// removeTerminatedContainer returns the specified container but with
// the stats of the terminated containers removed.
354429
func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi.Container {
	containerMap := make(map[containerID][]*runtimeapi.Container)
	// Sort order by create time
	sort.Slice(containers, func(i, j int) bool {
		return containers[i].CreatedAt < containers[j].CreatedAt
	})
	for _, container := range containers {
		refID := containerID{
			podRef:        buildPodRef(container.Labels),
			containerName: kubetypes.GetContainerName(container.Labels),
		}
		containerMap[refID] = append(containerMap[refID], container)
	}

	result := make([]*runtimeapi.Container, 0)
	for _, refs := range containerMap {
		if len(refs) == 1 {
			result = append(result, refs[0])
			continue
		}
		found := false
		for i := 0; i < len(refs); i++ {
			if refs[i].State == runtimeapi.ContainerState_CONTAINER_RUNNING {
				found = true
				result = append(result, refs[i])
			}
		}
		if !found {
			result = append(result, refs[len(refs)-1])
		}
	}
	return result
}
491206
func (c *Calcium) ReplaceContainer(ctx context.Context, opts *types.ReplaceOptions) (chan *types.ReplaceContainerMessage, error) {
	if len(opts.IDs) == 0 {
		oldContainers, err := c.ListContainers(ctx, &types.ListContainersOptions{
			Appname: opts.Name, Entrypoint: opts.Entrypoint.Name, Nodename: opts.Nodename,
		})
		if err != nil {
			return nil, err
		}
		for _, container := range oldContainers {
			opts.IDs = append(opts.IDs, container.ID)
		}
	}
	ch := make(chan *types.ReplaceContainerMessage)
	go func() {
		defer close(ch)
		// 并发控制
		step := opts.Count
		wg := sync.WaitGroup{}
		defer wg.Wait()
		for index, ID := range opts.IDs {
			container, containerJSON, containerLock, err := c.doLockAndGetContainer(ctx, ID)
			if err != nil {
				log.Errorf("[ReplaceContainer] Get container %s failed %v", ID, err)
				continue
			}
			if opts.Podname != "" && container.Podname != opts.Podname {
				log.Warnf("[ReplaceContainer] Skip not in pod container %s", container.ID)
				c.doUnlock(containerLock, container.ID)
				continue
			}

			log.Infof("[ReplaceContainer] Replace old container %s", container.ID)
			wg.Add(1)

			go func(replaceOpts types.ReplaceOptions, container *types.Container, containerJSON *enginetypes.VirtualizationInfo, containerLock lock.DistributedLock, index int) {
				defer wg.Done()
				defer c.doUnlock(containerLock, container.ID)
				// 使用复制之后的配置
				// 停老的，起新的
				replaceOpts.Memory = container.Memory
				replaceOpts.CPUQuota = container.Quota
				replaceOpts.SoftLimit = container.SoftLimit
				// 覆盖 podname 如果做全量更新的话
				replaceOpts.Podname = container.Podname
				// 继承网络配置
				if replaceOpts.NetworkInherit {
					log.Infof("[ReplaceContainer] Inherit old container network configuration mode %v", containerJSON.Networks)
					replaceOpts.NetworkMode = ""
					replaceOpts.Networks = containerJSON.Networks
				}

				createMessage, removeMessage, err := c.doReplaceContainer(
					ctx, container, containerJSON, &replaceOpts, index,
				)
				ch <- &types.ReplaceContainerMessage{
					Create: createMessage,
					Remove: removeMessage,
					Error:  err,
				}
				if err != nil {
					log.Errorf("[ReplaceContainer] Replace and remove failed %v, old container restarted", err)
					return
				}
				log.Infof("[ReplaceContainer] Replace and remove success %s", container.ID)
				// 传 opts 的值，产生一次复制
			}(*opts, container, containerJSON, containerLock, index)
			if (index+1)%step == 0 {
				wg.Wait()
			}
		}
	}()

	return ch, nil
}
1107022
func (c *Client) StopContainer(id string, timeout uint) error {
	return c.stopContainer(id, timeout, doOptions{})
}
1049573
func (l *location) RemoveContainer(id string) error {
	return l.client.ContainerDelete(id)
}
538396
func (p *podContainerDeletor) deleteContainersInPod(filterContainerID string, podStatus *kubecontainer.PodStatus, removeAll bool) {
	containersToKeep := p.containersToKeep
	if removeAll {
		containersToKeep = 0
		filterContainerID = ""
	}

	for _, candidate := range getContainersToDeleteInPod(filterContainerID, podStatus, containersToKeep) {
		select {
		case p.worker <- candidate.ID:
		default:
			klog.Warningf("Failed to issue the request to remove container %v", candidate.ID)
		}
	}
}
388160
func (p *PriorityQueue) DeleteNominatedPodIfExists(pod *v1.Pod) {
	p.lock.Lock()
	p.nominatedPods.delete(pod)
	p.lock.Unlock()
}
1065572
func (s *LoadBalancerService) RemoveFromGlobalLoadBalancerRule(p *RemoveFromGlobalLoadBalancerRuleParams) (*RemoveFromGlobalLoadBalancerRuleResponse, error) {
	resp, err := s.cs.newRequest("removeFromGlobalLoadBalancerRule", p.toURLValues())
	if err != nil {
		return nil, err
	}

	var r RemoveFromGlobalLoadBalancerRuleResponse
	if err := json.Unmarshal(resp, &r); err != nil {
		return nil, err
	}

	// If we have a async client, we need to wait for the async result
	if s.cs.async {
		b, err := s.cs.GetAsyncJobResult(r.JobID, s.cs.timeout)
		if err != nil {
			if err == AsyncTimeoutErr {
				return &r, err
			}
			return nil, err
		}

		if err := json.Unmarshal(b, &r); err != nil {
			return nil, err
		}
	}

	return &r, nil
}
14466
func (ds *dockerService) RemoveContainer(_ context.Context, r *runtimeapi.RemoveContainerRequest) (*runtimeapi.RemoveContainerResponse, error) {
	ds.performPlatformSpecificContainerForContainer(r.ContainerId)

	// Ideally, log lifecycle should be independent of container lifecycle.
	// However, docker will remove container log after container is removed,
	// we can't prevent that now, so we also clean up the symlink here.
	err := ds.removeContainerLogSymlink(r.ContainerId)
	if err != nil {
		return nil, err
	}
	err = ds.client.RemoveContainer(r.ContainerId, dockertypes.ContainerRemoveOptions{RemoveVolumes: true, Force: true})
	if err != nil {
		return nil, fmt.Errorf("failed to remove container %q: %v", r.ContainerId, err)
	}
	return &runtimeapi.RemoveContainerResponse{}, nil
}
2727
func (self *StandardFilterer) RemoveFilter(filter Filter) {
	if self.filters.SetContains(filter) {
		self.filters.SetRemove(filter)
	}
}
820505
func (kl *Kubelet) podIsTerminated(pod *v1.Pod) bool {
	// Check the cached pod status which was set after the last sync.
	status, ok := kl.statusManager.GetPodStatus(pod.UID)
	if !ok {
		// If there is no cached status, use the status from the
		// apiserver. This is useful if kubelet has recently been
		// restarted.
		status = pod.Status
	}
	return status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded || (pod.DeletionTimestamp != nil && notRunning(status.ContainerStatuses))
}
1096758
func (f *Facade) RemoveServiceTemplate(ctx datastore.Context, id string) error {
	defer ctx.Metrics().Stop(ctx.Metrics().Start("Facade.RemoveServiceTemplate"))
	alog := f.auditLogger.Message(ctx, "Removing Service Template").
		Action(audit.Remove).ID(id).Type(servicetemplate.GetType())
	_, err := f.templateStore.Get(ctx, id)
	logger := plog.WithFields(logrus.Fields{
		"templateid": id,
	})
	if err != nil {
		logger.WithError(err).Error("Unable to find template")
		return alog.Error(fmt.Errorf("Unable to find template: %s", id))
	}

	// CC-3673 - LogFilters are NOT removed to avoid breaking any deployed services that might be using those filters

	logger.Info("Removed template")
	if err := f.templateStore.Delete(ctx, id); err != nil {
		return alog.Error(err)
	}

	go LogstashContainerReloader(ctx, f)
	alog.Succeeded()
	return nil
}
1359404
func (c *Client) KillContainer(opts KillContainerOptions) error {
	path := "/containers/" + opts.ID + "/kill" + "?" + queryString(opts)
	resp, err := c.do("POST", path, doOptions{context: opts.Context})
	if err != nil {
		e, ok := err.(*Error)
		if !ok {
			return err
		}
		switch e.Status {
		case http.StatusNotFound:
			return &NoSuchContainer{ID: opts.ID}
		case http.StatusConflict:
			return &ContainerNotRunning{ID: opts.ID}
		default:
			return err
		}
	}
	resp.Body.Close()
	return nil
}
936385
func (b *KubeletBuilder) usesContainerizedMounter() bool {
	switch b.Distribution {
	case distros.DistributionContainerOS:
		return true
	default:
		return false
	}
}
164859
func (kl *Kubelet) cleanUpContainersInPod(podID types.UID, exitedContainerID string) {
	if podStatus, err := kl.podCache.Get(podID); err == nil {
		removeAll := false
		if syncedPod, ok := kl.podManager.GetPodByUID(podID); ok {
			// generate the api status using the cached runtime status to get up-to-date ContainerStatuses
			apiPodStatus := kl.generateAPIPodStatus(syncedPod, podStatus)
			// When an evicted or deleted pod has already synced, all containers can be removed.
			removeAll = eviction.PodIsEvicted(syncedPod.Status) || (syncedPod.DeletionTimestamp != nil && notRunning(apiPodStatus.ContainerStatuses))
		}
		kl.containerDeletor.deleteContainersInPod(exitedContainerID, podStatus, removeAll)
	}
}
268279
func (r *Runtime) StopContainer(ctx context.Context, c *Container, timeout int64) error {
	impl, err := r.RuntimeImpl(c)
	if err != nil {
		return err
	}

	return impl.StopContainer(ctx, c, timeout)
}
63785
func (c *containerManager) RemoveAll(stdout io.Writer) {
	for containerID := range c.tmpContainers {
		if err := c.removeContainer(containerID, stdout); err != nil {
			return
		}
		delete(c.tmpContainers, containerID)
		fmt.Fprintf(stdout, "Removing intermediate container %s\n", stringid.TruncateID(containerID))
	}
}
1331974
func (e *ClientExecutor) removeContainer(id string) error {
	e.Client.StopContainer(id, 0)
	err := e.Client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            id,
		RemoveVolumes: true,
		Force:         true,
	})
	if _, ok := err.(*docker.NoSuchContainer); err != nil && !ok {
		return fmt.Errorf("unable to cleanup container: %v", err)
	}
	return nil
}