package backendupdate import ( "bytes" "context" "errors" "io" "log/slog" "net/http" "os" "path/filepath" "strings" "testing" "time" "yms-daemon/internal/backendexecutor" "yms-daemon/internal/containerengine" "yms-daemon/internal/deploymentconfig" "yms-daemon/internal/hostnginx" "yms-daemon/internal/transaction" ) // containerTestImage 容器更新测试使用的镜像引用,指向 harbor 仓库的一个具体 tag。 const containerTestImage = "harbor.ymswell.asia/ymswell/glory-ymswell:20260813-184902-a37bf50d-v1.1.8.1" // containerTestDigest 容器更新测试使用的镜像仓库摘要,用于与 containerTestImage 组合成不可变引用。 const containerTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot 验证在已存在运行中的 backend-8080 容器时, // 更新镜像会切换到非活跃槽位 backend-8081、提交部署并停止旧容器 backend-8080。 func TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot(t *testing.T) { ctx := context.Background() updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{ "backend-8080": { ID: "existing-backend-8080", Name: "backend-8080", Running: true, }, }, 0) record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil) if err != nil { t.Fatalf("update container backend: %v", err) } if record.State != transaction.StateCommitted || gateway.snapshot.ActivePort != 8081 { t.Fatalf("unexpected committed container update: record=%+v gateway=%+v", record, gateway.snapshot) } if engine.containers["backend-8080"].Running { t.Fatal("previous backend container is still running") } if len(engine.stopped) != 1 || engine.stopped[0] != "backend-8080" { t.Fatalf("unexpected stopped containers: %+v", engine.stopped) } assertCreatedContainerSpec(t, engine.lastCreateSpec) assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081") } // TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch 验证全新安装场景: // 目标容器 backend-8081 必须先在非路由槽位创建并通过健康检查,之后才切换网关; // 整个过程不得进入旧容器排水流程,也不得停止任何旧容器。 func TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, time.Hour) var progress []Progress gateway.beforeApply = func(snapshot hostnginx.Snapshot) error { target, found := engine.containers["backend-8081"] if !found || !target.Running || engine.healthChecks == 0 { return errors.New("gateway switch occurred before the target container passed health checking") } return nil } record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, func(item Progress) { progress = append(progress, item) }) if err != nil { t.Fatalf("first install container backend: %v", err) } if record.State != transaction.StateCommitted { t.Fatalf("unexpected committed first install: record=%+v", record) } if gateway.snapshot.ActivePort != 8081 || gateway.applyCount != 1 { t.Fatalf("first install did not switch once to the healthy inactive slot: gateway=%+v applies=%d", gateway.snapshot, gateway.applyCount) } target, found := engine.containers["backend-8081"] if !found || !target.Running || target.Dead { t.Fatalf("first install target container is not running: %+v", target) } if len(engine.stopped) != 0 { t.Fatalf("first install must not stop a previous container: %+v", engine.stopped) } if engine.healthChecks != 1 { t.Fatalf("unexpected first-install health check count: %d", engine.healthChecks) } for _, item := range progress { if strings.HasPrefix(item.Message, "Draining previous backend container") { t.Fatalf("first install entered previous-container drain: %+v", progress) } } assertCreatedContainerSpec(t, engine.lastCreateSpec) assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081") } // TestContainerUpdaterRejectsMissingActiveWithPresentInactive 验证当活跃容器 backend-8080 缺失 // 而非活跃容器 backend-8081 仍在运行时,更新会被拒绝并返回明确的错误信息。 func TestContainerUpdaterRejectsMissingActiveWithPresentInactive(t *testing.T) { updater, _, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{ "backend-8081": { ID: "unexpected-backend-8081", Name: "backend-8081", Running: true, }, }, 0) _, err := updater.UpdateContainerImage(context.Background(), containerTestImage, true, nil) if err == nil || !strings.Contains(err.Error(), "active backend container backend-8080 is missing but inactive container backend-8081 is running") { t.Fatalf("unexpected missing-active result: %v", err) } } // TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage 验证更新失败进入回滚后的恢复链路: // 首次更新因网关切换失败而进入回滚并停止目标容器,再次调用恢复同一回滚直至完成, // 最后重试同一镜像能够成功切换并提交。 func TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage(t *testing.T) { ctx := context.Background() updater, _, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0) failGateway := true gateway.beforeApply = func(hostnginx.Snapshot) error { if failGateway { return errors.New("host Nginx is unavailable") } return nil } rollingBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil) if err == nil || rollingBack.State != transaction.StateRollingBack { t.Fatalf("unexpected failed rollback result: record=%+v err=%v", rollingBack, err) } target, found := engine.containers["backend-8081"] if !found || target.Running { t.Fatalf("failed update target was not stopped: %+v", target) } failGateway = false rolledBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil) if err == nil || rolledBack.State != transaction.StateRolledBack { t.Fatalf("unexpected resumed rollback result: record=%+v err=%v", rolledBack, err) } committed, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil) if err != nil || committed.State != transaction.StateCommitted { t.Fatalf("retry same image after rollback: record=%+v err=%v", committed, err) } if gateway.snapshot.ActivePort != 8081 { t.Fatalf("retry did not switch to the healthy container: %+v", gateway.snapshot) } } // TestContainerUpdaterRejectsMissingCommittedContainer 验证当已提交部署记录中的活跃容器 backend-8080 丢失时, // 更新会将其识别为部署漂移并拒绝继续。 func TestContainerUpdaterRejectsMissingCommittedContainer(t *testing.T) { ctx := context.Background() updater, store, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0) transactionID := "committed-container-transaction" _, _, err := store.CreateTransaction(ctx, transaction.CreateRequest{ ID: transactionID, IdempotencyKey: "backend:container:previous", Source: sourceLocalCLI, Service: serviceBackend, Request: []byte(`{"inputType":"container-image"}`), }) if err != nil { t.Fatalf("create previous backend transaction: %v", err) } for _, state := range []transaction.State{ transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting, transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, } { if _, err := store.Transition(ctx, transactionID, state, "seed committed deployment"); err != nil { t.Fatalf("transition previous backend transaction to %s: %v", state, err) } } if _, err := store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{ ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "missing-backend-8080", }, "seed committed backend container deployment"); err != nil { t.Fatalf("commit previous backend deployment: %v", err) } _, err = updater.UpdateContainerImage(ctx, containerTestImage, true, nil) if err == nil || !strings.Contains(err.Error(), "committed active backend container backend-8080 is missing") { t.Fatalf("unexpected committed-container drift result: %v", err) } } // TestContainerUpdaterRestartsContainerWithLocalImage 验证 container 后端重启会复用当前已提交部署 // 记录的镜像:轮转到非活跃槽位、健康检查通过后切流并停止旧容器,且整个过程不重新拉取镜像。 func TestContainerUpdaterRestartsContainerWithLocalImage(t *testing.T) { ctx := context.Background() updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{ "backend-8080": { ID: "committed-backend-8080", Name: "backend-8080", Running: true, ImageReference: "harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest, }, }, 0) commitContainerDeployment(t, store, transaction.BackendContainerDeployment{ ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "committed-backend-8080", }) record, err := updater.RestartContainer(ctx, nil) if err != nil { t.Fatalf("restart container backend: %v", err) } if record.State != transaction.StateCommitted || gateway.snapshot.ActivePort != 8081 { t.Fatalf("unexpected committed container restart: record=%+v gateway=%+v", record, gateway.snapshot) } if engine.containers["backend-8080"].Running { t.Fatal("previous backend container is still running after restart") } if engine.pullCalls != 0 { t.Fatalf("restart must reuse the local image without pulling: pulls=%d", engine.pullCalls) } assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081") } // TestContainerUpdaterRestartRejectsWithoutDeployment 验证不存在已提交部署记录时,container 后端重启被拒绝。 func TestContainerUpdaterRestartRejectsWithoutDeployment(t *testing.T) { updater, _, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0) _, err := updater.RestartContainer(context.Background(), nil) if err == nil || !strings.Contains(err.Error(), "requires a committed deployment") { t.Fatalf("unexpected restart without deployment: %v", err) } } // TestContainerUpdaterRestartRejectsGatewayDrift 验证宿主 Nginx 活动端口与部署记录不一致时,container 后端重启被拒绝。 func TestContainerUpdaterRestartRejectsGatewayDrift(t *testing.T) { updater, store, _, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{ "backend-8080": { ID: "committed-backend-8080", Name: "backend-8080", Running: true, ImageReference: "harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest, }, }, 0) commitContainerDeployment(t, store, transaction.BackendContainerDeployment{ ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "committed-backend-8080", }) gateway.snapshot.ActivePort = 8081 _, err := updater.RestartContainer(context.Background(), nil) if err == nil || !strings.Contains(err.Error(), "requires gateway active port 8080, got 8081") { t.Fatalf("unexpected restart with gateway drift: %v", err) } } // commitContainerDeployment 通过合法的状态机推进写入一条 container 部署记录,供重启测试预置现场。 func commitContainerDeployment(t *testing.T, store *transaction.Store, deployment transaction.BackendContainerDeployment) { t.Helper() ctx := context.Background() transactionID := "seed-" + deployment.ContainerName _, _, err := store.CreateTransaction(ctx, transaction.CreateRequest{ ID: transactionID, IdempotencyKey: "backend:container:seed:" + deployment.ContainerName, Source: sourceLocalCLI, Service: serviceBackend, Request: []byte(`{"inputType":"container-image"}`), }) if err != nil { t.Fatalf("create seed backend transaction: %v", err) } for _, state := range []transaction.State{ transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting, transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, } { if _, err := store.Transition(ctx, transactionID, state, "seed committed deployment"); err != nil { t.Fatalf("transition seed backend transaction to %s: %v", state, err) } } if _, err := store.CommitBackendContainerDeployment(ctx, transactionID, deployment, "seed committed backend container deployment"); err != nil { t.Fatalf("commit seed backend deployment: %v", err) } } // newContainerUpdaterFixture 构造容器更新器的测试夹具,返回更新器、事务存储、假容器引擎和内存网关。 // containers 指定引擎初始存在的容器;drain 指定旧容器停止前的排水等待时长。 func newContainerUpdaterFixture( t *testing.T, containers map[string]containerengine.Container, drain time.Duration, ) (*Updater, *transaction.Store, *containerUpdateEngine, *memoryGateway) { t.Helper() ctx := context.Background() root := t.TempDir() store, err := transaction.OpenStore(ctx, filepath.Join(root, "transactions.db")) if err != nil { t.Fatalf("open transaction store: %v", err) } t.Cleanup(func() { if err := store.Close(); err != nil { t.Errorf("close transaction store: %v", err) } }) coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil))) if err != nil { t.Fatalf("create transaction coordinator: %v", err) } engine := &containerUpdateEngine{ image: containerengine.Image{ ID: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", RepoDigests: []string{"harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest}, Platform: containerengine.Platform{OS: "linux", Architecture: "amd64"}, }, containers: containers, } configSource := filepath.Join(root, "yms.yaml") if err := os.WriteFile(configSource, []byte("server: {}\n"), 0o600); err != nil { t.Fatalf("write backend configuration: %v", err) } tmpSource := filepath.Join(root, "tmp") if err := os.Mkdir(tmpSource, 0o755); err != nil { t.Fatalf("create backend temporary directory: %v", err) } client := &http.Client{Transport: containerUpdateRoundTripFunc(func(request *http.Request) (*http.Response, error) { engine.healthChecks++ return &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(bytes.NewBufferString(`{"status":"UP","components":{"ping":{"status":"UP"}}}`)), Request: request, }, nil })} executor, err := backendexecutor.New(store, coordinator, engine, client) if err != nil { t.Fatalf("create backend container executor: %v", err) } gateway := &memoryGateway{snapshot: hostnginx.Snapshot{Content: []byte(serverConfiguration8080), ActivePort: 8080}} updater := &Updater{ config: deploymentconfig.Config{ Daemon: deploymentconfig.Daemon{Environment: deploymentconfig.EnvironmentDev}, Backend: deploymentconfig.Backend{Type: deploymentconfig.BackendTypeContainer, Slot: deploymentconfig.BackendSlots{ Port8080: deploymentconfig.BackendSlot{ContainerName: "backend-8080", HealthEndpoint: "http://127.0.0.1:8080/yms/actuator/health"}, Port8081: deploymentconfig.BackendSlot{ContainerName: "backend-8081", HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health"}, }}, }, workRoot: filepath.Join(root, "work"), store: store, coordinator: coordinator, gateway: gateway, containerExecutor: executor, engine: engine, drain: drain, containerConfigSource: configSource, containerConfigTarget: deploymentconfig.ContainerConfigTarget, containerTmpSource: tmpSource, containerTmpTarget: deploymentconfig.ContainerTmpTarget, } return updater, store, engine, gateway } // assertCreatedContainerSpec 校验传入容器引擎的创建规格,确认目标容器名称、镜像引用、运行时约束、 // 环境变量和挂载点均符合容器更新契约。 func assertCreatedContainerSpec(t *testing.T, request containerengine.ContainerSpec) { t.Helper() if request.Name != "backend-8081" || request.ImageReference != containerTestImage { t.Fatalf("unexpected target container identity: %+v", request) } if request.NetworkMode != "host" || request.RestartPolicy.Name != "no" || request.User != "0:0" { t.Fatalf("unexpected target container runtime contract: %+v", request) } if len(request.Environment) != 2 || request.Environment[0] != "SERVER_PORT=8081" || request.Environment[1] != "SPRING_CONFIG_LOCATION=file:/app/config/yms.yaml" { t.Fatalf("unexpected target container environment: %+v", request.Environment) } if len(request.Mounts) != 2 || request.Mounts[0].Target != deploymentconfig.ContainerConfigTarget || !request.Mounts[0].ReadOnly || request.Mounts[1].Target != deploymentconfig.ContainerTmpTarget { t.Fatalf("unexpected target container mounts: %+v", request.Mounts) } } // assertCommittedContainerDeployment 校验事务存储中已提交的后端容器部署记录, // 确认其活跃端口、容器名称、容器 ID、镜像摘要和事务 ID 均与预期一致。 func assertCommittedContainerDeployment( t *testing.T, store *transaction.Store, transactionID string, port int, containerName string, containerID string, ) { t.Helper() deployment, err := store.BackendContainerDeployment(context.Background()) if err != nil { t.Fatalf("read committed backend container deployment: %v", err) } if deployment.ActivePort != port || deployment.ContainerName != containerName || deployment.ContainerID != containerID || deployment.ImageDigest != containerTestDigest || deployment.TransactionID != transactionID { t.Fatalf("unexpected committed backend container deployment: %+v", deployment) } } // serverConfiguration8080 测试用的 host Nginx 配置片段,其活跃后端端口为 8080。 const serverConfiguration8080 = `http { upstream yms-server { # yms-update managed upstream begin server 10.11.1.117:8080 max_fails=1 fail_timeout=2s; # server 10.11.1.117:8081 max_fails=1 fail_timeout=2s; # yms-update managed upstream end } } ` // containerUpdateRoundTripFunc 将普通函数适配为 http.RoundTripper, // 使测试可以用自定义逻辑响应后端的健康检查请求。 type containerUpdateRoundTripFunc func(*http.Request) (*http.Response, error) // RoundTrip 实现 http.RoundTripper 接口,直接委托给底层函数处理请求。 func (function containerUpdateRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return function(request) } // containerUpdateEngine 容器引擎的测试替身,在内存中模拟镜像与容器的生命周期, // 并记录停止、创建和健康检查等交互,供断言验证更新行为。 type containerUpdateEngine struct { image containerengine.Image containers map[string]containerengine.Container stopped []string lastCreateSpec containerengine.ContainerSpec healthChecks int pullCalls int } // Ping 返回 nil,模拟引擎连通性检查始终成功。 func (e *containerUpdateEngine) Ping(context.Context) error { return nil } // PullImage 记录一次拉取调用并返回 nil,模拟镜像拉取始终成功。 func (e *containerUpdateEngine) PullImage(context.Context, string) error { e.pullCalls++ return nil } // LoadImage 返回 nil,模拟从流加载镜像始终成功。 func (e *containerUpdateEngine) LoadImage(context.Context, io.Reader) error { return nil } // InspectImage 返回夹具预设的镜像信息,用于冻结镜像的平台与仓库摘要。 func (e *containerUpdateEngine) InspectImage(context.Context, string) (containerengine.Image, error) { return e.image, nil } // CreateContainer 依据规格在内存中登记一个新容器,记录最后一次创建规格并返回该容器。 func (e *containerUpdateEngine) CreateContainer(_ context.Context, spec containerengine.ContainerSpec) (containerengine.Container, error) { e.lastCreateSpec = spec record := containerengine.Container{ ID: "container-id-" + spec.Name, Name: spec.Name, ImageID: e.image.ID, ImageReference: spec.ImageReference, Platform: spec.Platform.OS + "/" + spec.Platform.Architecture, Status: "created", Environment: append([]string(nil), spec.Environment...), Labels: spec.Labels, NetworkMode: spec.NetworkMode, RestartPolicy: spec.RestartPolicy, Mounts: append([]containerengine.Mount(nil), spec.Mounts...), User: spec.User, StopTimeoutSeconds: spec.StopTimeoutSeconds, } e.containers[spec.Name] = record return record, nil } // StartContainer 将指定容器标记为运行中;若容器不存在则返回 containerengine.ErrNotFound。 func (e *containerUpdateEngine) StartContainer(_ context.Context, name string) error { record, found := e.containers[name] if !found { return containerengine.ErrNotFound } record.Running = true record.Status = "running" e.containers[name] = record return nil } // ContainerLogs 返回空的日志流,模拟容器日志读取。 func (e *containerUpdateEngine) ContainerLogs(context.Context, string) (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("")), nil } // StopContainer 记录被停止的容器名称并将该容器标记为已退出;若容器不存在则返回 containerengine.ErrNotFound。 func (e *containerUpdateEngine) StopContainer(_ context.Context, name string, _ int) error { e.stopped = append(e.stopped, name) record, found := e.containers[name] if !found { return containerengine.ErrNotFound } record.Running = false record.Status = "exited" e.containers[name] = record return nil } // InspectContainer 返回内存中的指定容器;若容器不存在则返回 containerengine.ErrNotFound。 func (e *containerUpdateEngine) InspectContainer(_ context.Context, name string) (containerengine.Container, error) { record, found := e.containers[name] if !found { return containerengine.Container{}, containerengine.ErrNotFound } return record, nil } // RemoveContainer 从内存中删除指定容器;若容器不存在则返回 containerengine.ErrNotFound。 func (e *containerUpdateEngine) RemoveContainer(_ context.Context, name string, _ bool) error { if _, found := e.containers[name]; !found { return containerengine.ErrNotFound } delete(e.containers, name) return nil } // Close 返回 nil,模拟引擎关闭无副作用。 func (e *containerUpdateEngine) Close() error { return nil } var _ containerengine.Engine = (*containerUpdateEngine)(nil)