package backendupdate import ( "bytes" "context" "crypto/rand" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "github.com/distribution/reference" "yms-daemon/internal/backendexecutor" "yms-daemon/internal/containerengine" "yms-daemon/internal/deploymentconfig" "yms-daemon/internal/hostnginx" "yms-daemon/internal/transaction" ) // inputTypeContainerImage 标识以容器镜像方式执行后端更新的输入类型。 const inputTypeContainerImage = "container-image" // inputTypeContainerRestart 标识容器后端同版本重启的输入类型,用于事务恢复时区分 // 普通更新与重启,从而在重启时复用本地镜像而不重新拉取。 const inputTypeContainerRestart = "container-restart" // defaultContainerStopGraceSeconds 停止旧容器时,发送停止信号后到强制终止前的默认等待秒数, // 当 backend.stop_grace_seconds 未配置时使用。该值覆盖容器创建时设置的 9000 秒 StopTimeout, // 避免旧容器因优雅停机超时长期无法退出。 const defaultContainerStopGraceSeconds = 30 // persistedContainerRequest 容器后端更新请求的持久化形态,以 JSON 存入事务 // 记录。事务在提交、重试或补偿时依赖这些字段重建执行上下文。 type persistedContainerRequest struct { // InputType 标识输入来源类型。 InputType string `json:"inputType"` // ImageReference 用户提供的镜像引用(带标签)。 ImageReference string `json:"imageReference"` // ImmutableReference 解析出的不可变镜像引用(带摘要)。 ImmutableReference string `json:"immutableReference"` // ImageDigest 解析出的镜像仓库摘要。 ImageDigest string `json:"imageDigest"` // Platform 镜像的精确平台信息。 Platform containerengine.Platform `json:"platform"` // TargetPort 本次更新要切换到的目标端口。 TargetPort int `json:"targetPort"` // TargetContainer 目标槽位的容器名。 TargetContainer string `json:"targetContainer"` // PreviousPort 更新前 Nginx 上游指向的端口。 PreviousPort int `json:"previousPort"` // PreviousContainer 更新前活动槽位的容器名,首次安装时为空。 PreviousContainer string `json:"previousContainer"` // TargetHealthEndpoint 目标容器的健康检查端点。 TargetHealthEndpoint string `json:"targetHealthEndpoint"` // StartLog 表示是否在启动阶段转发容器日志。 StartLog bool `json:"startLog"` // GatewayBeforePath 切换前 Nginx 配置快照的文件路径。 GatewayBeforePath string `json:"gatewayBeforePath"` // GatewayAfterPath 切换后 Nginx 配置快照的文件路径。 GatewayAfterPath string `json:"gatewayAfterPath"` // GatewayReceiptPath Nginx 切换成功后的回执文件路径。 GatewayReceiptPath string `json:"gatewayReceiptPath"` } // UpdateContainerImage 拉取一个开发镜像,冻结其仓库摘要,并更新非活动的 Docker // 后端槽位。 // // 参数 imageReference 是必须带标签且不含多余空白的精确镜像引用;startLog 表示 // 是否转发容器启动日志;report 用于回传实时进度,可为 nil。返回值为本次更新 // 对应的事务记录以及错误。若已存在同镜像的活动事务则复用续跑。 func (u *Updater) UpdateContainerImage(ctx context.Context, imageReference string, startLog bool, report ProgressReporter) (transaction.Transaction, error) { if u.containerExecutor == nil || u.engine == nil { return transaction.Transaction{}, errors.New("container backend updater is not configured") } if strings.TrimSpace(imageReference) != imageReference || imageReference == "" { return transaction.Transaction{}, errors.New("exact container image reference is required") } if err := u.validateContainerInputs(); err != nil { return transaction.Transaction{}, err } active, err := u.store.ActiveTransaction(ctx) if err == nil { var request persistedContainerRequest if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil { return active, decodeErr } if active.Service != serviceBackend || request.InputType != inputTypeContainerImage { return active, &transaction.ActiveTransactionError{TransactionID: active.ID} } if request.ImageReference != imageReference { return active, &transaction.ActiveTransactionError{TransactionID: active.ID} } return u.runContainerUpdate(ctx, active, request, false, report) } if !errors.Is(err, transaction.ErrNotFound) { return transaction.Transaction{}, err } reportProgress(report, Progress{Message: "Pulling backend image " + imageReference}) resolved, err := u.pullAndResolveImage(ctx, imageReference) if err != nil { return transaction.Transaction{}, err } reportProgress(report, Progress{Message: "Backend image resolved: " + resolved.ImmutableReference}) before, err := u.gateway.Read() if err != nil { return transaction.Transaction{}, err } activeSlot, err := u.config.Backend.SlotForPort(before.ActivePort) if err != nil { return transaction.Transaction{}, err } deployment, deploymentErr := u.store.BackendContainerDeployment(ctx) hasDeployment := deploymentErr == nil if deploymentErr != nil && !errors.Is(deploymentErr, transaction.ErrNotFound) { return transaction.Transaction{}, deploymentErr } hasHistory, err := u.store.HasCommittedBackendContainerTransactionHistory(ctx) if err != nil { return transaction.Transaction{}, err } targetPort, targetSlot, previousContainer, err := u.resolveContainerSlots( ctx, before.ActivePort, activeSlot, deployment, hasDeployment, hasHistory, ) if err != nil { return transaction.Transaction{}, err } afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort) if err != nil { return transaction.Transaction{}, err } transactionID := rand.Text() transactionRoot := filepath.Join(u.workRoot, transactionID) if err := os.MkdirAll(transactionRoot, 0o750); err != nil { return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err) } beforePath := filepath.Join(transactionRoot, "gateway.before.conf") afterPath := filepath.Join(transactionRoot, "gateway.after.conf") if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil { return transaction.Transaction{}, err } if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil { return transaction.Transaction{}, err } request := persistedContainerRequest{ InputType: inputTypeContainerImage, ImageReference: imageReference, ImmutableReference: resolved.ImmutableReference, ImageDigest: resolved.Digest, Platform: resolved.Platform, TargetPort: targetPort, TargetContainer: targetSlot.ContainerName, PreviousPort: before.ActivePort, PreviousContainer: previousContainer, TargetHealthEndpoint: targetSlot.HealthEndpoint, StartLog: startLog, GatewayBeforePath: beforePath, GatewayAfterPath: afterPath, GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"), } content, err := json.Marshal(request) if err != nil { return transaction.Transaction{}, fmt.Errorf("encode container backend update request: %w", err) } record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{ ID: transactionID, IdempotencyKey: serviceBackend + ":container:" + resolved.Digest, Source: sourceLocalCLI, Service: serviceBackend, Request: content, }) if err != nil { return transaction.Transaction{}, err } if !created { if err := decodeContainerRequest(record.Request, &request); err != nil { return record, err } } return u.runContainerUpdate(ctx, record, request, created, report) } // RestartContainer 使用当前已提交部署记录的镜像执行一次同版本零停机轮转:它复用本地 // 镜像(不重新拉取),把同一镜像部署到非活动槽位,健康检查通过后切换流量并停止旧容器。 // // 参数 ctx 用于控制整个重启过程的取消;report 用于回传实时进度,可为 nil。返回值为本次 // 重启对应的事务记录以及错误。若不存在已提交部署记录、现场与部署记录不一致,或存在其他 // 未完成事务,则拒绝新建重启;存在未完成的重启事务时复用续跑。 func (u *Updater) RestartContainer(ctx context.Context, report ProgressReporter) (transaction.Transaction, error) { if u.containerExecutor == nil || u.engine == nil { return transaction.Transaction{}, errors.New("container backend updater is not configured") } if err := u.validateContainerInputs(); err != nil { return transaction.Transaction{}, err } deployment, err := u.store.BackendContainerDeployment(ctx) if err != nil { return transaction.Transaction{}, fmt.Errorf("container backend restart requires a committed deployment: %w", err) } active, err := u.store.ActiveTransaction(ctx) if err == nil { var request persistedContainerRequest if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil { return active, decodeErr } if active.Service != serviceBackend || request.InputType != inputTypeContainerRestart { return active, &transaction.ActiveTransactionError{TransactionID: active.ID} } return u.runContainerUpdate(ctx, active, request, false, report) } if !errors.Is(err, transaction.ErrNotFound) { return transaction.Transaction{}, err } before, err := u.gateway.Read() if err != nil { return transaction.Transaction{}, err } if before.ActivePort != deployment.ActivePort { return transaction.Transaction{}, fmt.Errorf("container backend restart requires gateway active port %d, got %d", deployment.ActivePort, before.ActivePort) } activeContainer, err := u.engine.InspectContainer(ctx, deployment.ContainerName) if err != nil { return transaction.Transaction{}, fmt.Errorf("inspect active backend container %s: %w", deployment.ContainerName, err) } if activeContainer.ID != deployment.ContainerID { return transaction.Transaction{}, fmt.Errorf("active backend container %s identity does not match committed deployment", deployment.ContainerName) } if !activeContainer.Running || activeContainer.Dead { return transaction.Transaction{}, fmt.Errorf("active backend container %s is not running", deployment.ContainerName) } image, err := u.engine.InspectImage(ctx, activeContainer.ImageReference) if err != nil { return transaction.Transaction{}, fmt.Errorf("inspect active backend image %s: %w", activeContainer.ImageReference, err) } if image.Platform.OS == "" || image.Platform.Architecture == "" { return transaction.Transaction{}, errors.New("active backend image does not report an exact platform") } targetPort := otherPort(deployment.ActivePort) targetSlot, err := u.config.Backend.SlotForPort(targetPort) if err != nil { return transaction.Transaction{}, err } previousSlot, err := u.config.Backend.SlotForPort(deployment.ActivePort) if err != nil { return transaction.Transaction{}, err } afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort) if err != nil { return transaction.Transaction{}, err } transactionID := rand.Text() transactionRoot := filepath.Join(u.workRoot, transactionID) if err := os.MkdirAll(transactionRoot, 0o750); err != nil { return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err) } beforePath := filepath.Join(transactionRoot, "gateway.before.conf") afterPath := filepath.Join(transactionRoot, "gateway.after.conf") if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil { return transaction.Transaction{}, err } if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil { return transaction.Transaction{}, err } request := persistedContainerRequest{ InputType: inputTypeContainerRestart, ImageReference: activeContainer.ImageReference, ImmutableReference: activeContainer.ImageReference, ImageDigest: deployment.ImageDigest, Platform: image.Platform, TargetPort: targetPort, TargetContainer: targetSlot.ContainerName, PreviousPort: deployment.ActivePort, PreviousContainer: previousSlot.ContainerName, TargetHealthEndpoint: targetSlot.HealthEndpoint, StartLog: true, GatewayBeforePath: beforePath, GatewayAfterPath: afterPath, GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"), } content, err := json.Marshal(request) if err != nil { return transaction.Transaction{}, fmt.Errorf("encode container backend restart request: %w", err) } record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{ ID: transactionID, IdempotencyKey: serviceBackend + ":restart:" + rand.Text(), Source: sourceLocalCLI, Service: serviceBackend, Request: content, }) if err != nil { return transaction.Transaction{}, err } if !created { if err := decodeContainerRequest(record.Request, &request); err != nil { return record, err } } return u.runContainerUpdate(ctx, record, request, created, report) } // validateContainerInputs 校验容器后端的配置源文件与临时目录满足部署前置条件: // 配置源必须是直接普通文件,临时目录必须是直接目录。 func (u *Updater) validateContainerInputs() error { configInfo, err := os.Lstat(u.containerConfigSource) if err != nil { return fmt.Errorf("inspect backend configuration %s: %w", u.containerConfigSource, err) } if !configInfo.Mode().IsRegular() || configInfo.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("backend configuration is not a direct regular file: %s", u.containerConfigSource) } tmpInfo, err := os.Lstat(u.containerTmpSource) if err != nil { return fmt.Errorf("inspect backend temporary directory %s: %w", u.containerTmpSource, err) } if !tmpInfo.IsDir() || tmpInfo.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("backend temporary path is not a direct directory: %s", u.containerTmpSource) } return nil } // resolveContainerSlots 在选择目标槽位前,核对已提交的部署、网关以及两个精确 // 容器名是否一致。全新安装没有部署记录、没有已提交的容器事务历史,也没有槽位 // 容器,会从非路由槽位启动,这样只有健康检查通过后才会暴露流量。 // // 返回值依次为目标端口、目标槽位以及上一容器名。activePort 为当前路由端口, // activeSlot 为当前路由槽位,deployment 为已提交的部署记录,hasDeployment 与 // hasHistory 分别表示是否存在部署记录与容器事务历史。 func (u *Updater) resolveContainerSlots( ctx context.Context, activePort int, activeSlot deploymentconfig.BackendSlot, deployment transaction.BackendContainerDeployment, hasDeployment bool, hasHistory bool, ) (int, deploymentconfig.BackendSlot, string, error) { inactivePort := otherPort(activePort) inactiveSlot, err := u.config.Backend.SlotForPort(inactivePort) if err != nil { return 0, deploymentconfig.BackendSlot{}, "", err } active, activeFound, err := u.inspectSlotContainer(ctx, activeSlot.ContainerName, "active") if err != nil { return 0, deploymentconfig.BackendSlot{}, "", err } inactive, inactiveFound, err := u.inspectSlotContainer(ctx, inactiveSlot.ContainerName, "inactive") if err != nil { return 0, deploymentconfig.BackendSlot{}, "", err } if err := validateCommittedContainer(deployment, activePort, activeSlot, active, activeFound, hasDeployment); err != nil { return 0, deploymentconfig.BackendSlot{}, "", err } if activeFound { return selectRunningContainerSlot(inactivePort, activeSlot, inactiveSlot, active, inactive, inactiveFound) } if hasHistory { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing on a server with committed backend container transaction history", activeSlot.ContainerName) } if inactiveFound && inactive.Running && !inactive.Dead { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing but inactive container %s is running", activeSlot.ContainerName, inactiveSlot.ContainerName) } return inactivePort, inactiveSlot, "", nil } func (u *Updater) inspectSlotContainer(ctx context.Context, name string, role string) (containerengine.Container, bool, error) { container, err := u.engine.InspectContainer(ctx, name) if errors.Is(err, containerengine.ErrNotFound) { return containerengine.Container{}, false, nil } if err != nil { return containerengine.Container{}, false, fmt.Errorf("inspect %s backend container %s: %w", role, name, err) } return container, true, nil } func validateCommittedContainer(deployment transaction.BackendContainerDeployment, activePort int, activeSlot deploymentconfig.BackendSlot, active containerengine.Container, found bool, hasDeployment bool) error { if !hasDeployment { return nil } if deployment.ActivePort != activePort { return fmt.Errorf("committed backend container port %d does not match gateway active port %d", deployment.ActivePort, activePort) } if deployment.ContainerName != activeSlot.ContainerName { return fmt.Errorf("committed backend container %s does not match gateway slot container %s", deployment.ContainerName, activeSlot.ContainerName) } if !found { return fmt.Errorf("committed active backend container %s is missing", activeSlot.ContainerName) } if active.ID != deployment.ContainerID { return fmt.Errorf("active backend container %s identity does not match committed deployment", activeSlot.ContainerName) } return nil } func selectRunningContainerSlot(inactivePort int, activeSlot, inactiveSlot deploymentconfig.BackendSlot, active, inactive containerengine.Container, inactiveFound bool) (int, deploymentconfig.BackendSlot, string, error) { if !active.Running || active.Dead { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is not running", activeSlot.ContainerName) } if inactiveFound && inactive.Running && !inactive.Dead { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inactive backend container %s is unexpectedly running", inactiveSlot.ContainerName) } return inactivePort, inactiveSlot, activeSlot.ContainerName, nil } // resolvedImage 拉取并解析后得到的镜像信息。 type resolvedImage struct { // ImmutableReference 带摘要的不可变镜像引用。 ImmutableReference string // Digest 镜像仓库摘要。 Digest string // Platform 镜像的精确平台信息。 Platform containerengine.Platform } // pullAndResolveImage 解析镜像引用、确保其为带标签引用,拉取镜像并检查其平台 // 信息,随后从仓库摘要中解析出唯一的不可变引用。若仓库存在多个匹配摘要则报错, // 以保证后续部署所用的引用是确定且唯一的。 func (u *Updater) pullAndResolveImage(ctx context.Context, imageReference string) (resolvedImage, error) { named, err := parseTaggedImageReference(imageReference) if err != nil { return resolvedImage{}, fmt.Errorf("parse container image reference: %w", err) } if err := u.engine.Ping(ctx); err != nil { return resolvedImage{}, err } if err := u.engine.PullImage(ctx, imageReference); err != nil { return resolvedImage{}, err } image, err := u.engine.InspectImage(ctx, imageReference) if err != nil { return resolvedImage{}, fmt.Errorf("inspect pulled backend image: %w", err) } if err := validateImagePlatform(image.Platform); err != nil { return resolvedImage{}, err } repository := reference.TrimNamed(named).Name() immutableReference, digest, err := resolveRepositoryDigest(image.RepoDigests, repository) if err != nil { return resolvedImage{}, err } return resolvedImage{ImmutableReference: immutableReference, Digest: digest, Platform: image.Platform}, nil } func parseTaggedImageReference(imageReference string) (reference.Named, error) { named, err := reference.ParseNormalizedNamed(imageReference) if err != nil { return nil, err } if _, ok := named.(reference.Tagged); !ok { return nil, errors.New("--container-image requires a tag-qualified image reference") } return named, nil } func validateImagePlatform(platform containerengine.Platform) error { if platform.OS == "" || platform.Architecture == "" { return errors.New("pulled backend image does not report an exact platform") } return nil } func resolveRepositoryDigest(values []string, repository string) (string, string, error) { matches := make(map[string]string) for _, value := range values { digested, err := reference.ParseNormalizedNamed(value) if err != nil { return "", "", fmt.Errorf("parse pulled repository digest %q: %w", value, err) } withDigest, ok := digested.(reference.Digested) if !ok || reference.TrimNamed(digested).Name() != repository { continue } matches[withDigest.Digest().String()] = value } if len(matches) == 0 { return "", "", fmt.Errorf("pulled backend image has no repository digest for %s", repository) } if len(matches) != 1 { return "", "", fmt.Errorf("pulled backend image has multiple repository digests for %s", repository) } for digest, immutableReference := range matches { return immutableReference, digest, nil } return "", "", errors.New("repository digest resolution produced no result") } // runContainerUpdate 根据事务当前状态执行容器后端更新的核心流程:启动目标容器, // 再执行切换与提交。created 表示本调用是否新建了事务;report 用于回传进度。执行 // 失败时若事务尚未进入失败态则触发容器补偿。 func (u *Updater) runContainerUpdate(ctx context.Context, record transaction.Transaction, request persistedContainerRequest, created bool, report ProgressReporter) (transaction.Transaction, error) { message := "Resuming container backend transaction " + record.ID if created { message = "Created container backend transaction " + record.ID } reportProgress(report, Progress{TransactionID: record.ID, State: record.State, Message: message}) if record.State.Terminal() { return terminalResult(record) } imageAcquisition := backendexecutor.ImageAcquisitionPull if request.InputType == inputTypeContainerRestart { imageAcquisition = backendexecutor.ImageAcquisitionPresent } executorRequest := backendexecutor.Request{ ImageAcquisition: imageAcquisition, ImageReference: request.ImmutableReference, DisplayImageReference: request.ImageReference, ExpectedImageDigest: request.ImageDigest, Platform: request.Platform, ContainerName: request.TargetContainer, Port: request.TargetPort, PortEnvironmentKey: deploymentconfig.ContainerPortEnvironment, ConfigSource: u.containerConfigSource, ConfigTarget: u.containerConfigTarget, TmpSource: u.containerTmpSource, TmpTarget: u.containerTmpTarget, ConfigEnvironmentKey: deploymentconfig.ContainerConfigEnvironment, ConfigLocation: deploymentconfig.ContainerConfigLocation, RestartPolicy: containerengine.RestartPolicy{Name: "no"}, HealthEndpoint: request.TargetHealthEndpoint, StartLog: request.StartLog, LogReporter: func(line string) { reportProgress(report, Progress{TransactionID: record.ID, State: transaction.StateStarting, Message: "CONTAINER LOG " + line}) }, } switch record.State { case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting: if err := u.containerExecutor.Run(ctx, record.ID, executorRequest); err != nil { current, readErr := u.store.Transaction(ctx, record.ID) if readErr != nil || current.State == transaction.StateFailed { return u.currentWithError(ctx, record.ID, errors.Join(err, readErr)) } before, beforeErr := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort) after, afterErr := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort) if snapshotErr := errors.Join(beforeErr, afterErr); snapshotErr != nil { return u.currentWithError(ctx, record.ID, errors.Join(err, snapshotErr)) } rollbackErr := u.rollbackContainer(ctx, record.ID, request, before, after, err) return u.currentWithError(ctx, record.ID, rollbackErr) } case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, transaction.StateRollingBack: default: return u.currentWithError(ctx, record.ID, fmt.Errorf("container backend update cannot resume transaction %s in state %s", record.ID, record.State)) } if err := u.switchAndCommitContainer(ctx, record.ID, request, report); err != nil { return u.currentWithError(ctx, record.ID, err) } return u.store.Transaction(ctx, record.ID) } // switchAndCommitContainer 执行容器后端的流量切换与提交:切换 Nginx 上游到目标 // 端口、进入排空阶段、确认目标容器运行且镜像引用匹配,最后提交容器部署记录。 // 切换失败时触发容器补偿。 func (u *Updater) switchAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error { before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort) if err != nil { return err } after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort) if err != nil { return err } for { record, err := u.store.Transaction(ctx, transactionID) if err != nil { return err } switch record.State { case transaction.StateSwitching: if err := u.switchContainerTraffic(ctx, transactionID, request, before, after, report); err != nil { return err } case transaction.StateVerifying: if err := u.markContainerDraining(ctx, transactionID, request); err != nil { return err } case transaction.StateDraining: return u.drainAndCommitContainer(ctx, transactionID, request, report) case transaction.StateCommitted: return nil case transaction.StateRollingBack: return u.rollbackContainer(ctx, transactionID, request, before, after, errors.New("resuming container backend rollback")) default: return fmt.Errorf("container backend commit cannot continue transaction %s in state %s", transactionID, record.State) } } } func (u *Updater) switchContainerTraffic(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, report ProgressReporter) error { reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateSwitching, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)}) operation := &gatewayOperation{controller: u.gateway, before: before, after: after, receiptPath: request.GatewayReceiptPath} if _, err := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewaySwitchIntent(request), operation); err != nil { return u.rollbackContainer(ctx, transactionID, request, before, after, err) } _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy container slot") return err } func (u *Updater) markContainerDraining(ctx context.Context, transactionID string, request persistedContainerRequest) error { message := "container backend switch verified" if request.PreviousContainer != "" { message += "; previous container draining" } _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, message) return err } func (u *Updater) drainAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error { if request.PreviousContainer != "" { reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateDraining, Message: fmt.Sprintf("Draining previous backend container for %s", u.drain)}) if err := waitContext(ctx, u.drain); err != nil { return err } operation := &containerStopOperation{engine: u.engine, name: request.PreviousContainer, stopGraceSeconds: u.containerStopGraceSeconds} if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousContainerIntent(request), operation); err != nil { return err } } target, err := u.engine.InspectContainer(ctx, request.TargetContainer) if err != nil { return fmt.Errorf("inspect target backend container before commit: %w", err) } if !target.Running || target.Dead || target.ID == "" { return fmt.Errorf("target backend container %s is not running with an exact identity", request.TargetContainer) } if target.ImageReference != request.ImmutableReference && target.ImageReference != request.ImageReference { return fmt.Errorf("target backend container %s image reference does not match the transaction", request.TargetContainer) } _, err = u.store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{ActivePort: request.TargetPort, ContainerName: request.TargetContainer, ImageDigest: request.ImageDigest, ContainerID: target.ID}, "container backend update committed") if err == nil { reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Container backend update committed"}) } return err } // rollbackContainer 执行容器后端补偿:恢复 Nginx 上游、停止被补偿的目标容器,并 // 将事务迁入 RolledBack 状态。cause 为触发补偿的原始错误,会与补偿过程中的错误 // 合并后返回。 func (u *Updater) rollbackContainer(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error { record, err := u.store.Transaction(ctx, transactionID) if err != nil { return errors.Join(cause, err) } if record.State != transaction.StateRollingBack { if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "container backend compensation started"); err != nil { return errors.Join(cause, err) } } gateway := &gatewayOperation{controller: u.gateway, before: after, after: before, receiptPath: request.GatewayReceiptPath + ".restore"} _, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewayRestoreIntent(request), gateway) stop := &containerStopOperation{engine: u.engine, name: request.TargetContainer, stopGraceSeconds: u.containerStopGraceSeconds} _, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetContainerIntent(request), stop) if err := errors.Join(gatewayErr, stopErr); err != nil { return errors.Join(cause, err) } _, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "container backend compensation completed") return errors.Join(cause, transitionErr) } // containerStopOperation 停止某个后端容器的事务操作。 type containerStopOperation struct { // engine 容器引擎。 engine containerengine.Engine // name 待停止的容器名。 name string // stopGraceSeconds 发送停止信号后到强制终止前的等待秒数。 stopGraceSeconds int } // Apply 停止指定容器,若容器不存在则视为已满足(幂等成功)。 func (o *containerStopOperation) Apply(ctx context.Context) error { err := o.engine.StopContainer(ctx, o.name, o.stopGraceSeconds) if errors.Is(err, containerengine.ErrNotFound) { return nil } return err } // Inspect 检查容器状态:不存在或已停止视为已应用,仍运行视为未应用。 func (o *containerStopOperation) Inspect(ctx context.Context) (transaction.Inspection, error) { record, err := o.engine.InspectContainer(ctx, o.name) if errors.Is(err, containerengine.ErrNotFound) { return transaction.Inspection{Status: transaction.InspectionApplied}, nil } if err != nil { return transaction.Inspection{}, err } result, _ := json.Marshal(record) if !record.Running || record.Dead { return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil } return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil } // containerGatewaySwitchIntent 构造容器后端 Nginx 上游切换步骤的意图,记录切换 // 前后的端口。 func containerGatewaySwitchIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.container.gateway.switch", "switch host Nginx to container backend", struct { BeforePort int `json:"beforePort"` AfterPort int `json:"afterPort"` }{request.PreviousPort, request.TargetPort}) } // containerGatewayRestoreIntent 构造容器后端失败后恢复 Nginx 上游步骤的意图,记录 // 需恢复到的端口。 func containerGatewayRestoreIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.container.gateway.restore", "restore host Nginx after container backend failure", struct { Port int `json:"port"` }{request.PreviousPort}) } // stopPreviousContainerIntent 构造排空后停止前一容器步骤的意图,记录容器名。 func stopPreviousContainerIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.previous-container.stop", "stop previous backend container after drain", struct { ContainerName string `json:"containerName"` }{request.PreviousContainer}) } // stopTargetContainerIntent 构造停止被补偿容器步骤的意图,记录容器名。 func stopTargetContainerIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.target-container.stop", "stop compensated backend container", struct { ContainerName string `json:"containerName"` }{request.TargetContainer}) } // decodeContainerRequest 将持久化的容器后端更新请求 JSON 反序列化到目标结构体, // 并禁止出现未知字段。 func decodeContainerRequest(content json.RawMessage, request *persistedContainerRequest) error { decoder := json.NewDecoder(bytes.NewReader(content)) decoder.DisallowUnknownFields() if err := decoder.Decode(request); err != nil { return fmt.Errorf("decode persisted container backend request: %w", err) } return nil } var _ transaction.Operation = (*containerStopOperation)(nil)