yms-daemon/internal/backendupdate/container.go

package backendupdate

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/distribution/reference"

	"yms-daemon/internal/backendexecutor"
	"yms-daemon/internal/containerengine"
	"yms-daemon/internal/deploymentconfig"
	"yms-daemon/internal/hostnginx"
	"yms-daemon/internal/transaction"
)

// inputTypeContainerImage 标识以容器镜像方式执行后端更新的输入类型。
const inputTypeContainerImage = "container-image"

// inputTypeContainerRestart 标识容器后端同版本重启的输入类型，用于事务恢复时区分
// 普通更新与重启，从而在重启时复用本地镜像而不重新拉取。
const inputTypeContainerRestart = "container-restart"

// defaultContainerStopGraceSeconds 停止旧容器时，发送停止信号后到强制终止前的默认等待秒数，
// 当 backend.stop_grace_seconds 未配置时使用。该值覆盖容器创建时设置的 9000 秒 StopTimeout，
// 避免旧容器因优雅停机超时长期无法退出。
const defaultContainerStopGraceSeconds = 30

// persistedContainerRequest 容器后端更新请求的持久化形态，以 JSON 存入事务
// 记录。事务在提交、重试或补偿时依赖这些字段重建执行上下文。
type persistedContainerRequest struct {
	// InputType 标识输入来源类型。
	InputType string `json:"inputType"`
	// ImageReference 用户提供的镜像引用（带标签）。
	ImageReference string `json:"imageReference"`
	// ImmutableReference 解析出的不可变镜像引用（带摘要）。
	ImmutableReference string `json:"immutableReference"`
	// ImageDigest 解析出的镜像仓库摘要。
	ImageDigest string `json:"imageDigest"`
	// Platform 镜像的精确平台信息。
	Platform containerengine.Platform `json:"platform"`
	// TargetPort 本次更新要切换到的目标端口。
	TargetPort int `json:"targetPort"`
	// TargetContainer 目标槽位的容器名。
	TargetContainer string `json:"targetContainer"`
	// PreviousPort 更新前 Nginx 上游指向的端口。
	PreviousPort int `json:"previousPort"`
	// PreviousContainer 更新前活动槽位的容器名，首次安装时为空。
	PreviousContainer string `json:"previousContainer"`
	// TargetHealthEndpoint 目标容器的健康检查端点。
	TargetHealthEndpoint string `json:"targetHealthEndpoint"`
	// StartLog 表示是否在启动阶段转发容器日志。
	StartLog bool `json:"startLog"`
	// GatewayBeforePath 切换前 Nginx 配置快照的文件路径。
	GatewayBeforePath string `json:"gatewayBeforePath"`
	// GatewayAfterPath 切换后 Nginx 配置快照的文件路径。
	GatewayAfterPath string `json:"gatewayAfterPath"`
	// GatewayReceiptPath Nginx 切换成功后的回执文件路径。
	GatewayReceiptPath string `json:"gatewayReceiptPath"`
}

// UpdateContainerImage 拉取一个开发镜像，冻结其仓库摘要，并更新非活动的 Docker
// 后端槽位。
//
// 参数 imageReference 是必须带标签且不含多余空白的精确镜像引用；startLog 表示
// 是否转发容器启动日志；report 用于回传实时进度，可为 nil。返回值为本次更新
// 对应的事务记录以及错误。若已存在同镜像的活动事务则复用续跑。
func (u *Updater) UpdateContainerImage(ctx context.Context, imageReference string, startLog bool, report ProgressReporter) (transaction.Transaction, error) {
	if u.containerExecutor == nil || u.engine == nil {
		return transaction.Transaction{}, errors.New("container backend updater is not configured")
	}
	if strings.TrimSpace(imageReference) != imageReference || imageReference == "" {
		return transaction.Transaction{}, errors.New("exact container image reference is required")
	}
	if err := u.validateContainerInputs(); err != nil {
		return transaction.Transaction{}, err
	}

	active, err := u.store.ActiveTransaction(ctx)
	if err == nil {
		var request persistedContainerRequest
		if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil {
			return active, decodeErr
		}
		if active.Service != serviceBackend || request.InputType != inputTypeContainerImage {
			return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
		}
		if request.ImageReference != imageReference {
			return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
		}
		return u.runContainerUpdate(ctx, active, request, false, report)
	}
	if !errors.Is(err, transaction.ErrNotFound) {
		return transaction.Transaction{}, err
	}

	reportProgress(report, Progress{Message: "Pulling backend image " + imageReference})
	resolved, err := u.pullAndResolveImage(ctx, imageReference)
	if err != nil {
		return transaction.Transaction{}, err
	}
	reportProgress(report, Progress{Message: "Backend image resolved: " + resolved.ImmutableReference})

	before, err := u.gateway.Read()
	if err != nil {
		return transaction.Transaction{}, err
	}
	activeSlot, err := u.config.Backend.SlotForPort(before.ActivePort)
	if err != nil {
		return transaction.Transaction{}, err
	}
	deployment, deploymentErr := u.store.BackendContainerDeployment(ctx)
	hasDeployment := deploymentErr == nil
	if deploymentErr != nil && !errors.Is(deploymentErr, transaction.ErrNotFound) {
		return transaction.Transaction{}, deploymentErr
	}
	hasHistory, err := u.store.HasCommittedBackendContainerTransactionHistory(ctx)
	if err != nil {
		return transaction.Transaction{}, err
	}
	targetPort, targetSlot, previousContainer, err := u.resolveContainerSlots(
		ctx,
		before.ActivePort,
		activeSlot,
		deployment,
		hasDeployment,
		hasHistory,
	)
	if err != nil {
		return transaction.Transaction{}, err
	}
	afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort)
	if err != nil {
		return transaction.Transaction{}, err
	}
	transactionID := rand.Text()
	transactionRoot := filepath.Join(u.workRoot, transactionID)
	if err := os.MkdirAll(transactionRoot, 0o750); err != nil {
		return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err)
	}
	beforePath := filepath.Join(transactionRoot, "gateway.before.conf")
	afterPath := filepath.Join(transactionRoot, "gateway.after.conf")
	if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil {
		return transaction.Transaction{}, err
	}
	if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil {
		return transaction.Transaction{}, err
	}
	request := persistedContainerRequest{
		InputType: inputTypeContainerImage, ImageReference: imageReference,
		ImmutableReference: resolved.ImmutableReference, ImageDigest: resolved.Digest, Platform: resolved.Platform,
		TargetPort: targetPort, TargetContainer: targetSlot.ContainerName,
		PreviousPort: before.ActivePort, PreviousContainer: previousContainer,
		TargetHealthEndpoint: targetSlot.HealthEndpoint,
		StartLog:             startLog,
		GatewayBeforePath:    beforePath, GatewayAfterPath: afterPath,
		GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"),
	}
	content, err := json.Marshal(request)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("encode container backend update request: %w", err)
	}
	record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{
		ID: transactionID, IdempotencyKey: serviceBackend + ":container:" + resolved.Digest,
		Source: sourceLocalCLI, Service: serviceBackend, Request: content,
	})
	if err != nil {
		return transaction.Transaction{}, err
	}
	if !created {
		if err := decodeContainerRequest(record.Request, &request); err != nil {
			return record, err
		}
	}
	return u.runContainerUpdate(ctx, record, request, created, report)
}

// RestartContainer 使用当前已提交部署记录的镜像执行一次同版本零停机轮转：它复用本地
// 镜像（不重新拉取），把同一镜像部署到非活动槽位，健康检查通过后切换流量并停止旧容器。
//
// 参数 ctx 用于控制整个重启过程的取消；report 用于回传实时进度，可为 nil。返回值为本次
// 重启对应的事务记录以及错误。若不存在已提交部署记录、现场与部署记录不一致，或存在其他
// 未完成事务，则拒绝新建重启；存在未完成的重启事务时复用续跑。
func (u *Updater) RestartContainer(ctx context.Context, report ProgressReporter) (transaction.Transaction, error) {
	if u.containerExecutor == nil || u.engine == nil {
		return transaction.Transaction{}, errors.New("container backend updater is not configured")
	}
	if err := u.validateContainerInputs(); err != nil {
		return transaction.Transaction{}, err
	}

	deployment, err := u.store.BackendContainerDeployment(ctx)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("container backend restart requires a committed deployment: %w", err)
	}

	active, err := u.store.ActiveTransaction(ctx)
	if err == nil {
		var request persistedContainerRequest
		if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil {
			return active, decodeErr
		}
		if active.Service != serviceBackend || request.InputType != inputTypeContainerRestart {
			return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
		}
		return u.runContainerUpdate(ctx, active, request, false, report)
	}
	if !errors.Is(err, transaction.ErrNotFound) {
		return transaction.Transaction{}, err
	}

	before, err := u.gateway.Read()
	if err != nil {
		return transaction.Transaction{}, err
	}
	if before.ActivePort != deployment.ActivePort {
		return transaction.Transaction{}, fmt.Errorf("container backend restart requires gateway active port %d, got %d", deployment.ActivePort, before.ActivePort)
	}

	activeContainer, err := u.engine.InspectContainer(ctx, deployment.ContainerName)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("inspect active backend container %s: %w", deployment.ContainerName, err)
	}
	if activeContainer.ID != deployment.ContainerID {
		return transaction.Transaction{}, fmt.Errorf("active backend container %s identity does not match committed deployment", deployment.ContainerName)
	}
	if !activeContainer.Running || activeContainer.Dead {
		return transaction.Transaction{}, fmt.Errorf("active backend container %s is not running", deployment.ContainerName)
	}

	image, err := u.engine.InspectImage(ctx, activeContainer.ImageReference)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("inspect active backend image %s: %w", activeContainer.ImageReference, err)
	}
	if image.Platform.OS == "" || image.Platform.Architecture == "" {
		return transaction.Transaction{}, errors.New("active backend image does not report an exact platform")
	}

	targetPort := otherPort(deployment.ActivePort)
	targetSlot, err := u.config.Backend.SlotForPort(targetPort)
	if err != nil {
		return transaction.Transaction{}, err
	}
	previousSlot, err := u.config.Backend.SlotForPort(deployment.ActivePort)
	if err != nil {
		return transaction.Transaction{}, err
	}
	afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort)
	if err != nil {
		return transaction.Transaction{}, err
	}
	transactionID := rand.Text()
	transactionRoot := filepath.Join(u.workRoot, transactionID)
	if err := os.MkdirAll(transactionRoot, 0o750); err != nil {
		return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err)
	}
	beforePath := filepath.Join(transactionRoot, "gateway.before.conf")
	afterPath := filepath.Join(transactionRoot, "gateway.after.conf")
	if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil {
		return transaction.Transaction{}, err
	}
	if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil {
		return transaction.Transaction{}, err
	}
	request := persistedContainerRequest{
		InputType: inputTypeContainerRestart, ImageReference: activeContainer.ImageReference,
		ImmutableReference: activeContainer.ImageReference, ImageDigest: deployment.ImageDigest, Platform: image.Platform,
		TargetPort: targetPort, TargetContainer: targetSlot.ContainerName,
		PreviousPort: deployment.ActivePort, PreviousContainer: previousSlot.ContainerName,
		TargetHealthEndpoint: targetSlot.HealthEndpoint,
		StartLog:             true,
		GatewayBeforePath:    beforePath, GatewayAfterPath: afterPath,
		GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"),
	}
	content, err := json.Marshal(request)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("encode container backend restart request: %w", err)
	}
	record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{
		ID: transactionID, IdempotencyKey: serviceBackend + ":restart:" + rand.Text(),
		Source: sourceLocalCLI, Service: serviceBackend, Request: content,
	})
	if err != nil {
		return transaction.Transaction{}, err
	}
	if !created {
		if err := decodeContainerRequest(record.Request, &request); err != nil {
			return record, err
		}
	}
	return u.runContainerUpdate(ctx, record, request, created, report)
}

// validateContainerInputs 校验容器后端的配置源文件与临时目录满足部署前置条件：
// 配置源必须是直接普通文件，临时目录必须是直接目录。
func (u *Updater) validateContainerInputs() error {
	configInfo, err := os.Lstat(u.containerConfigSource)
	if err != nil {
		return fmt.Errorf("inspect backend configuration %s: %w", u.containerConfigSource, err)
	}
	if !configInfo.Mode().IsRegular() || configInfo.Mode()&os.ModeSymlink != 0 {
		return fmt.Errorf("backend configuration is not a direct regular file: %s", u.containerConfigSource)
	}
	tmpInfo, err := os.Lstat(u.containerTmpSource)
	if err != nil {
		return fmt.Errorf("inspect backend temporary directory %s: %w", u.containerTmpSource, err)
	}
	if !tmpInfo.IsDir() || tmpInfo.Mode()&os.ModeSymlink != 0 {
		return fmt.Errorf("backend temporary path is not a direct directory: %s", u.containerTmpSource)
	}
	return nil
}

// resolveContainerSlots 在选择目标槽位前，核对已提交的部署、网关以及两个精确
// 容器名是否一致。全新安装没有部署记录、没有已提交的容器事务历史，也没有槽位
// 容器，会从非路由槽位启动，这样只有健康检查通过后才会暴露流量。
//
// 返回值依次为目标端口、目标槽位以及上一容器名。activePort 为当前路由端口，
// activeSlot 为当前路由槽位，deployment 为已提交的部署记录，hasDeployment 与
// hasHistory 分别表示是否存在部署记录与容器事务历史。
func (u *Updater) resolveContainerSlots(
	ctx context.Context,
	activePort int,
	activeSlot deploymentconfig.BackendSlot,
	deployment transaction.BackendContainerDeployment,
	hasDeployment bool,
	hasHistory bool,
) (int, deploymentconfig.BackendSlot, string, error) {
	inactivePort := otherPort(activePort)
	inactiveSlot, err := u.config.Backend.SlotForPort(inactivePort)
	if err != nil {
		return 0, deploymentconfig.BackendSlot{}, "", err
	}
	active, activeFound, err := u.inspectSlotContainer(ctx, activeSlot.ContainerName, "active")
	if err != nil {
		return 0, deploymentconfig.BackendSlot{}, "", err
	}
	inactive, inactiveFound, err := u.inspectSlotContainer(ctx, inactiveSlot.ContainerName, "inactive")
	if err != nil {
		return 0, deploymentconfig.BackendSlot{}, "", err
	}
	if err := validateCommittedContainer(deployment, activePort, activeSlot, active, activeFound, hasDeployment); err != nil {
		return 0, deploymentconfig.BackendSlot{}, "", err
	}
	if activeFound {
		return selectRunningContainerSlot(inactivePort, activeSlot, inactiveSlot, active, inactive, inactiveFound)
	}
	if hasHistory {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing on a server with committed backend container transaction history", activeSlot.ContainerName)
	}
	if inactiveFound && inactive.Running && !inactive.Dead {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing but inactive container %s is running", activeSlot.ContainerName, inactiveSlot.ContainerName)
	}
	return inactivePort, inactiveSlot, "", nil
}

func (u *Updater) inspectSlotContainer(ctx context.Context, name string, role string) (containerengine.Container, bool, error) {
	container, err := u.engine.InspectContainer(ctx, name)
	if errors.Is(err, containerengine.ErrNotFound) {
		return containerengine.Container{}, false, nil
	}
	if err != nil {
		return containerengine.Container{}, false, fmt.Errorf("inspect %s backend container %s: %w", role, name, err)
	}
	return container, true, nil
}

func validateCommittedContainer(deployment transaction.BackendContainerDeployment, activePort int, activeSlot deploymentconfig.BackendSlot, active containerengine.Container, found bool, hasDeployment bool) error {
	if !hasDeployment {
		return nil
	}
	if deployment.ActivePort != activePort {
		return fmt.Errorf("committed backend container port %d does not match gateway active port %d", deployment.ActivePort, activePort)
	}
	if deployment.ContainerName != activeSlot.ContainerName {
		return fmt.Errorf("committed backend container %s does not match gateway slot container %s", deployment.ContainerName, activeSlot.ContainerName)
	}
	if !found {
		return fmt.Errorf("committed active backend container %s is missing", activeSlot.ContainerName)
	}
	if active.ID != deployment.ContainerID {
		return fmt.Errorf("active backend container %s identity does not match committed deployment", activeSlot.ContainerName)
	}
	return nil
}

func selectRunningContainerSlot(inactivePort int, activeSlot, inactiveSlot deploymentconfig.BackendSlot, active, inactive containerengine.Container, inactiveFound bool) (int, deploymentconfig.BackendSlot, string, error) {
	if !active.Running || active.Dead {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is not running", activeSlot.ContainerName)
	}
	if inactiveFound && inactive.Running && !inactive.Dead {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inactive backend container %s is unexpectedly running", inactiveSlot.ContainerName)
	}
	return inactivePort, inactiveSlot, activeSlot.ContainerName, nil
}

// resolvedImage 拉取并解析后得到的镜像信息。
type resolvedImage struct {
	// ImmutableReference 带摘要的不可变镜像引用。
	ImmutableReference string
	// Digest 镜像仓库摘要。
	Digest string
	// Platform 镜像的精确平台信息。
	Platform containerengine.Platform
}

// pullAndResolveImage 解析镜像引用、确保其为带标签引用，拉取镜像并检查其平台
// 信息，随后从仓库摘要中解析出唯一的不可变引用。若仓库存在多个匹配摘要则报错，
// 以保证后续部署所用的引用是确定且唯一的。
func (u *Updater) pullAndResolveImage(ctx context.Context, imageReference string) (resolvedImage, error) {
	named, err := parseTaggedImageReference(imageReference)
	if err != nil {
		return resolvedImage{}, fmt.Errorf("parse container image reference: %w", err)
	}
	if err := u.engine.Ping(ctx); err != nil {
		return resolvedImage{}, err
	}
	if err := u.engine.PullImage(ctx, imageReference); err != nil {
		return resolvedImage{}, err
	}
	image, err := u.engine.InspectImage(ctx, imageReference)
	if err != nil {
		return resolvedImage{}, fmt.Errorf("inspect pulled backend image: %w", err)
	}
	if err := validateImagePlatform(image.Platform); err != nil {
		return resolvedImage{}, err
	}
	repository := reference.TrimNamed(named).Name()
	immutableReference, digest, err := resolveRepositoryDigest(image.RepoDigests, repository)
	if err != nil {
		return resolvedImage{}, err
	}
	return resolvedImage{ImmutableReference: immutableReference, Digest: digest, Platform: image.Platform}, nil
}

func parseTaggedImageReference(imageReference string) (reference.Named, error) {
	named, err := reference.ParseNormalizedNamed(imageReference)
	if err != nil {
		return nil, err
	}
	if _, ok := named.(reference.Tagged); !ok {
		return nil, errors.New("--container-image requires a tag-qualified image reference")
	}
	return named, nil
}

func validateImagePlatform(platform containerengine.Platform) error {
	if platform.OS == "" || platform.Architecture == "" {
		return errors.New("pulled backend image does not report an exact platform")
	}
	return nil
}

func resolveRepositoryDigest(values []string, repository string) (string, string, error) {
	matches := make(map[string]string)
	for _, value := range values {
		digested, err := reference.ParseNormalizedNamed(value)
		if err != nil {
			return "", "", fmt.Errorf("parse pulled repository digest %q: %w", value, err)
		}
		withDigest, ok := digested.(reference.Digested)
		if !ok || reference.TrimNamed(digested).Name() != repository {
			continue
		}
		matches[withDigest.Digest().String()] = value
	}
	if len(matches) == 0 {
		return "", "", fmt.Errorf("pulled backend image has no repository digest for %s", repository)
	}
	if len(matches) != 1 {
		return "", "", fmt.Errorf("pulled backend image has multiple repository digests for %s", repository)
	}
	for digest, immutableReference := range matches {
		return immutableReference, digest, nil
	}
	return "", "", errors.New("repository digest resolution produced no result")
}

// runContainerUpdate 根据事务当前状态执行容器后端更新的核心流程：启动目标容器，
// 再执行切换与提交。created 表示本调用是否新建了事务；report 用于回传进度。执行
// 失败时若事务尚未进入失败态则触发容器补偿。
func (u *Updater) runContainerUpdate(ctx context.Context, record transaction.Transaction, request persistedContainerRequest, created bool, report ProgressReporter) (transaction.Transaction, error) {
	message := "Resuming container backend transaction " + record.ID
	if created {
		message = "Created container backend transaction " + record.ID
	}
	reportProgress(report, Progress{TransactionID: record.ID, State: record.State, Message: message})
	if record.State.Terminal() {
		return terminalResult(record)
	}
	imageAcquisition := backendexecutor.ImageAcquisitionPull
	if request.InputType == inputTypeContainerRestart {
		imageAcquisition = backendexecutor.ImageAcquisitionPresent
	}
	executorRequest := backendexecutor.Request{
		ImageAcquisition: imageAcquisition,
		ImageReference:   request.ImmutableReference, DisplayImageReference: request.ImageReference,
		ExpectedImageDigest: request.ImageDigest, Platform: request.Platform,
		ContainerName: request.TargetContainer, Port: request.TargetPort,
		PortEnvironmentKey: deploymentconfig.ContainerPortEnvironment,
		ConfigSource:       u.containerConfigSource, ConfigTarget: u.containerConfigTarget,
		TmpSource: u.containerTmpSource, TmpTarget: u.containerTmpTarget,
		ConfigEnvironmentKey: deploymentconfig.ContainerConfigEnvironment,
		ConfigLocation:       deploymentconfig.ContainerConfigLocation,
		RestartPolicy:        containerengine.RestartPolicy{Name: "no"},
		HealthEndpoint:       request.TargetHealthEndpoint,
		StartLog:             request.StartLog,
		LogReporter: func(line string) {
			reportProgress(report, Progress{TransactionID: record.ID, State: transaction.StateStarting, Message: "CONTAINER LOG  " + line})
		},
	}
	switch record.State {
	case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting:
		if err := u.containerExecutor.Run(ctx, record.ID, executorRequest); err != nil {
			current, readErr := u.store.Transaction(ctx, record.ID)
			if readErr != nil || current.State == transaction.StateFailed {
				return u.currentWithError(ctx, record.ID, errors.Join(err, readErr))
			}
			before, beforeErr := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
			after, afterErr := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
			if snapshotErr := errors.Join(beforeErr, afterErr); snapshotErr != nil {
				return u.currentWithError(ctx, record.ID, errors.Join(err, snapshotErr))
			}
			rollbackErr := u.rollbackContainer(ctx, record.ID, request, before, after, err)
			return u.currentWithError(ctx, record.ID, rollbackErr)
		}
	case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, transaction.StateRollingBack:
	default:
		return u.currentWithError(ctx, record.ID, fmt.Errorf("container backend update cannot resume transaction %s in state %s", record.ID, record.State))
	}
	if err := u.switchAndCommitContainer(ctx, record.ID, request, report); err != nil {
		return u.currentWithError(ctx, record.ID, err)
	}
	return u.store.Transaction(ctx, record.ID)
}

// switchAndCommitContainer 执行容器后端的流量切换与提交：切换 Nginx 上游到目标
// 端口、进入排空阶段、确认目标容器运行且镜像引用匹配，最后提交容器部署记录。
// 切换失败时触发容器补偿。
func (u *Updater) switchAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error {
	before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
	if err != nil {
		return err
	}
	after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
	if err != nil {
		return err
	}
	for {
		record, err := u.store.Transaction(ctx, transactionID)
		if err != nil {
			return err
		}
		switch record.State {
		case transaction.StateSwitching:
			if err := u.switchContainerTraffic(ctx, transactionID, request, before, after, report); err != nil {
				return err
			}
		case transaction.StateVerifying:
			if err := u.markContainerDraining(ctx, transactionID, request); err != nil {
				return err
			}
		case transaction.StateDraining:
			return u.drainAndCommitContainer(ctx, transactionID, request, report)
		case transaction.StateCommitted:
			return nil
		case transaction.StateRollingBack:
			return u.rollbackContainer(ctx, transactionID, request, before, after, errors.New("resuming container backend rollback"))
		default:
			return fmt.Errorf("container backend commit cannot continue transaction %s in state %s", transactionID, record.State)
		}
	}
}

func (u *Updater) switchContainerTraffic(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, report ProgressReporter) error {
	reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateSwitching, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)})
	operation := &gatewayOperation{controller: u.gateway, before: before, after: after, receiptPath: request.GatewayReceiptPath}
	if _, err := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewaySwitchIntent(request), operation); err != nil {
		return u.rollbackContainer(ctx, transactionID, request, before, after, err)
	}
	_, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy container slot")
	return err
}

func (u *Updater) markContainerDraining(ctx context.Context, transactionID string, request persistedContainerRequest) error {
	message := "container backend switch verified"
	if request.PreviousContainer != "" {
		message += "; previous container draining"
	}
	_, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, message)
	return err
}

func (u *Updater) drainAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error {
	if request.PreviousContainer != "" {
		reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateDraining, Message: fmt.Sprintf("Draining previous backend container for %s", u.drain)})
		if err := waitContext(ctx, u.drain); err != nil {
			return err
		}
		operation := &containerStopOperation{engine: u.engine, name: request.PreviousContainer, stopGraceSeconds: u.containerStopGraceSeconds}
		if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousContainerIntent(request), operation); err != nil {
			return err
		}
	}
	target, err := u.engine.InspectContainer(ctx, request.TargetContainer)
	if err != nil {
		return fmt.Errorf("inspect target backend container before commit: %w", err)
	}
	if !target.Running || target.Dead || target.ID == "" {
		return fmt.Errorf("target backend container %s is not running with an exact identity", request.TargetContainer)
	}
	if target.ImageReference != request.ImmutableReference && target.ImageReference != request.ImageReference {
		return fmt.Errorf("target backend container %s image reference does not match the transaction", request.TargetContainer)
	}
	_, err = u.store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{ActivePort: request.TargetPort, ContainerName: request.TargetContainer, ImageDigest: request.ImageDigest, ContainerID: target.ID}, "container backend update committed")
	if err == nil {
		reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Container backend update committed"})
	}
	return err
}

// rollbackContainer 执行容器后端补偿：恢复 Nginx 上游、停止被补偿的目标容器，并
// 将事务迁入 RolledBack 状态。cause 为触发补偿的原始错误，会与补偿过程中的错误
// 合并后返回。
func (u *Updater) rollbackContainer(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error {
	record, err := u.store.Transaction(ctx, transactionID)
	if err != nil {
		return errors.Join(cause, err)
	}
	if record.State != transaction.StateRollingBack {
		if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "container backend compensation started"); err != nil {
			return errors.Join(cause, err)
		}
	}
	gateway := &gatewayOperation{controller: u.gateway, before: after, after: before, receiptPath: request.GatewayReceiptPath + ".restore"}
	_, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewayRestoreIntent(request), gateway)
	stop := &containerStopOperation{engine: u.engine, name: request.TargetContainer, stopGraceSeconds: u.containerStopGraceSeconds}
	_, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetContainerIntent(request), stop)
	if err := errors.Join(gatewayErr, stopErr); err != nil {
		return errors.Join(cause, err)
	}
	_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "container backend compensation completed")
	return errors.Join(cause, transitionErr)
}

// containerStopOperation 停止某个后端容器的事务操作。
type containerStopOperation struct {
	// engine 容器引擎。
	engine containerengine.Engine
	// name 待停止的容器名。
	name string
	// stopGraceSeconds 发送停止信号后到强制终止前的等待秒数。
	stopGraceSeconds int
}

// Apply 停止指定容器，若容器不存在则视为已满足（幂等成功）。
func (o *containerStopOperation) Apply(ctx context.Context) error {
	err := o.engine.StopContainer(ctx, o.name, o.stopGraceSeconds)
	if errors.Is(err, containerengine.ErrNotFound) {
		return nil
	}
	return err
}

// Inspect 检查容器状态：不存在或已停止视为已应用，仍运行视为未应用。
func (o *containerStopOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
	record, err := o.engine.InspectContainer(ctx, o.name)
	if errors.Is(err, containerengine.ErrNotFound) {
		return transaction.Inspection{Status: transaction.InspectionApplied}, nil
	}
	if err != nil {
		return transaction.Inspection{}, err
	}
	result, _ := json.Marshal(record)
	if !record.Running || record.Dead {
		return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
	}
	return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
}

// containerGatewaySwitchIntent 构造容器后端 Nginx 上游切换步骤的意图，记录切换
// 前后的端口。
func containerGatewaySwitchIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.container.gateway.switch", "switch host Nginx to container backend", struct {
		BeforePort int `json:"beforePort"`
		AfterPort  int `json:"afterPort"`
	}{request.PreviousPort, request.TargetPort})
}

// containerGatewayRestoreIntent 构造容器后端失败后恢复 Nginx 上游步骤的意图，记录
// 需恢复到的端口。
func containerGatewayRestoreIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.container.gateway.restore", "restore host Nginx after container backend failure", struct {
		Port int `json:"port"`
	}{request.PreviousPort})
}

// stopPreviousContainerIntent 构造排空后停止前一容器步骤的意图，记录容器名。
func stopPreviousContainerIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.previous-container.stop", "stop previous backend container after drain", struct {
		ContainerName string `json:"containerName"`
	}{request.PreviousContainer})
}

// stopTargetContainerIntent 构造停止被补偿容器步骤的意图，记录容器名。
func stopTargetContainerIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.target-container.stop", "stop compensated backend container", struct {
		ContainerName string `json:"containerName"`
	}{request.TargetContainer})
}

// decodeContainerRequest 将持久化的容器后端更新请求 JSON 反序列化到目标结构体，
// 并禁止出现未知字段。
func decodeContainerRequest(content json.RawMessage, request *persistedContainerRequest) error {
	decoder := json.NewDecoder(bytes.NewReader(content))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(request); err != nil {
		return fmt.Errorf("decode persisted container backend request: %w", err)
	}
	return nil
}

var _ transaction.Operation = (*containerStopOperation)(nil)