Files

510 lines
22 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package backendupdate
import (
"bytes"
"context"
"errors"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"yms-daemon/internal/backendexecutor"
"yms-daemon/internal/containerengine"
"yms-daemon/internal/deploymentconfig"
"yms-daemon/internal/hostnginx"
"yms-daemon/internal/transaction"
)
// containerTestImage 容器更新测试使用的镜像引用,指向 harbor 仓库的一个具体 tag。
const containerTestImage = "harbor.ymswell.asia/ymswell/glory-ymswell:20260813-184902-a37bf50d-v1.1.8.1"
// containerTestDigest 容器更新测试使用的镜像仓库摘要,用于与 containerTestImage 组合成不可变引用。
const containerTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot 验证在已存在运行中的 backend-8080 容器时,
// 更新镜像会切换到非活跃槽位 backend-8081、提交部署并停止旧容器 backend-8080。
func TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot(t *testing.T) {
ctx := context.Background()
updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{
"backend-8080": {
ID: "existing-backend-8080", Name: "backend-8080", Running: true,
},
}, 0)
record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
if err != nil {
t.Fatalf("update container backend: %v", err)
}
if record.State != transaction.StateCommitted || gateway.snapshot.ActivePort != 8081 {
t.Fatalf("unexpected committed container update: record=%+v gateway=%+v", record, gateway.snapshot)
}
if engine.containers["backend-8080"].Running {
t.Fatal("previous backend container is still running")
}
if len(engine.stopped) != 1 || engine.stopped[0] != "backend-8080" {
t.Fatalf("unexpected stopped containers: %+v", engine.stopped)
}
assertCreatedContainerSpec(t, engine.lastCreateSpec)
assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
}
// TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch 验证全新安装场景:
// 目标容器 backend-8081 必须先在非路由槽位创建并通过健康检查,之后才切换网关;
// 整个过程不得进入旧容器排水流程,也不得停止任何旧容器。
func TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, time.Hour)
var progress []Progress
gateway.beforeApply = func(snapshot hostnginx.Snapshot) error {
target, found := engine.containers["backend-8081"]
if !found || !target.Running || engine.healthChecks == 0 {
return errors.New("gateway switch occurred before the target container passed health checking")
}
return nil
}
record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, func(item Progress) {
progress = append(progress, item)
})
if err != nil {
t.Fatalf("first install container backend: %v", err)
}
if record.State != transaction.StateCommitted {
t.Fatalf("unexpected committed first install: record=%+v", record)
}
if gateway.snapshot.ActivePort != 8081 || gateway.applyCount != 1 {
t.Fatalf("first install did not switch once to the healthy inactive slot: gateway=%+v applies=%d", gateway.snapshot, gateway.applyCount)
}
target, found := engine.containers["backend-8081"]
if !found || !target.Running || target.Dead {
t.Fatalf("first install target container is not running: %+v", target)
}
if len(engine.stopped) != 0 {
t.Fatalf("first install must not stop a previous container: %+v", engine.stopped)
}
if engine.healthChecks != 1 {
t.Fatalf("unexpected first-install health check count: %d", engine.healthChecks)
}
for _, item := range progress {
if strings.HasPrefix(item.Message, "Draining previous backend container") {
t.Fatalf("first install entered previous-container drain: %+v", progress)
}
}
assertCreatedContainerSpec(t, engine.lastCreateSpec)
assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
}
// TestContainerUpdaterRejectsMissingActiveWithPresentInactive 验证当活跃容器 backend-8080 缺失
// 而非活跃容器 backend-8081 仍在运行时,更新会被拒绝并返回明确的错误信息。
func TestContainerUpdaterRejectsMissingActiveWithPresentInactive(t *testing.T) {
updater, _, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{
"backend-8081": {
ID: "unexpected-backend-8081", Name: "backend-8081", Running: true,
},
}, 0)
_, err := updater.UpdateContainerImage(context.Background(), containerTestImage, true, nil)
if err == nil || !strings.Contains(err.Error(), "active backend container backend-8080 is missing but inactive container backend-8081 is running") {
t.Fatalf("unexpected missing-active result: %v", err)
}
}
// TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage 验证更新失败进入回滚后的恢复链路:
// 首次更新因网关切换失败而进入回滚并停止目标容器,再次调用恢复同一回滚直至完成,
// 最后重试同一镜像能够成功切换并提交。
func TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage(t *testing.T) {
ctx := context.Background()
updater, _, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
failGateway := true
gateway.beforeApply = func(hostnginx.Snapshot) error {
if failGateway {
return errors.New("host Nginx is unavailable")
}
return nil
}
rollingBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
if err == nil || rollingBack.State != transaction.StateRollingBack {
t.Fatalf("unexpected failed rollback result: record=%+v err=%v", rollingBack, err)
}
target, found := engine.containers["backend-8081"]
if !found || target.Running {
t.Fatalf("failed update target was not stopped: %+v", target)
}
failGateway = false
rolledBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
if err == nil || rolledBack.State != transaction.StateRolledBack {
t.Fatalf("unexpected resumed rollback result: record=%+v err=%v", rolledBack, err)
}
committed, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
if err != nil || committed.State != transaction.StateCommitted {
t.Fatalf("retry same image after rollback: record=%+v err=%v", committed, err)
}
if gateway.snapshot.ActivePort != 8081 {
t.Fatalf("retry did not switch to the healthy container: %+v", gateway.snapshot)
}
}
// TestContainerUpdaterRejectsMissingCommittedContainer 验证当已提交部署记录中的活跃容器 backend-8080 丢失时,
// 更新会将其识别为部署漂移并拒绝继续。
func TestContainerUpdaterRejectsMissingCommittedContainer(t *testing.T) {
ctx := context.Background()
updater, store, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
transactionID := "committed-container-transaction"
_, _, err := store.CreateTransaction(ctx, transaction.CreateRequest{
ID: transactionID, IdempotencyKey: "backend:container:previous", Source: sourceLocalCLI,
Service: serviceBackend, Request: []byte(`{"inputType":"container-image"}`),
})
if err != nil {
t.Fatalf("create previous backend transaction: %v", err)
}
for _, state := range []transaction.State{
transaction.StateValidating,
transaction.StatePrepared,
transaction.StateStarting,
transaction.StateSwitching,
transaction.StateVerifying,
transaction.StateDraining,
} {
if _, err := store.Transition(ctx, transactionID, state, "seed committed deployment"); err != nil {
t.Fatalf("transition previous backend transaction to %s: %v", state, err)
}
}
if _, err := store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{
ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "missing-backend-8080",
}, "seed committed backend container deployment"); err != nil {
t.Fatalf("commit previous backend deployment: %v", err)
}
_, err = updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
if err == nil || !strings.Contains(err.Error(), "committed active backend container backend-8080 is missing") {
t.Fatalf("unexpected committed-container drift result: %v", err)
}
}
// TestContainerUpdaterRestartsContainerWithLocalImage 验证 container 后端重启会复用当前已提交部署
// 记录的镜像:轮转到非活跃槽位、健康检查通过后切流并停止旧容器,且整个过程不重新拉取镜像。
func TestContainerUpdaterRestartsContainerWithLocalImage(t *testing.T) {
ctx := context.Background()
updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{
"backend-8080": {
ID: "committed-backend-8080", Name: "backend-8080", Running: true,
ImageReference: "harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest,
},
}, 0)
commitContainerDeployment(t, store, transaction.BackendContainerDeployment{
ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "committed-backend-8080",
})
record, err := updater.RestartContainer(ctx, nil)
if err != nil {
t.Fatalf("restart container backend: %v", err)
}
if record.State != transaction.StateCommitted || gateway.snapshot.ActivePort != 8081 {
t.Fatalf("unexpected committed container restart: record=%+v gateway=%+v", record, gateway.snapshot)
}
if engine.containers["backend-8080"].Running {
t.Fatal("previous backend container is still running after restart")
}
if engine.pullCalls != 0 {
t.Fatalf("restart must reuse the local image without pulling: pulls=%d", engine.pullCalls)
}
assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
}
// TestContainerUpdaterRestartRejectsWithoutDeployment 验证不存在已提交部署记录时,container 后端重启被拒绝。
func TestContainerUpdaterRestartRejectsWithoutDeployment(t *testing.T) {
updater, _, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
_, err := updater.RestartContainer(context.Background(), nil)
if err == nil || !strings.Contains(err.Error(), "requires a committed deployment") {
t.Fatalf("unexpected restart without deployment: %v", err)
}
}
// TestContainerUpdaterRestartRejectsGatewayDrift 验证宿主 Nginx 活动端口与部署记录不一致时,container 后端重启被拒绝。
func TestContainerUpdaterRestartRejectsGatewayDrift(t *testing.T) {
updater, store, _, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{
"backend-8080": {
ID: "committed-backend-8080", Name: "backend-8080", Running: true,
ImageReference: "harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest,
},
}, 0)
commitContainerDeployment(t, store, transaction.BackendContainerDeployment{
ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "committed-backend-8080",
})
gateway.snapshot.ActivePort = 8081
_, err := updater.RestartContainer(context.Background(), nil)
if err == nil || !strings.Contains(err.Error(), "requires gateway active port 8080, got 8081") {
t.Fatalf("unexpected restart with gateway drift: %v", err)
}
}
// commitContainerDeployment 通过合法的状态机推进写入一条 container 部署记录,供重启测试预置现场。
func commitContainerDeployment(t *testing.T, store *transaction.Store, deployment transaction.BackendContainerDeployment) {
t.Helper()
ctx := context.Background()
transactionID := "seed-" + deployment.ContainerName
_, _, err := store.CreateTransaction(ctx, transaction.CreateRequest{
ID: transactionID, IdempotencyKey: "backend:container:seed:" + deployment.ContainerName, Source: sourceLocalCLI,
Service: serviceBackend, Request: []byte(`{"inputType":"container-image"}`),
})
if err != nil {
t.Fatalf("create seed backend transaction: %v", err)
}
for _, state := range []transaction.State{
transaction.StateValidating,
transaction.StatePrepared,
transaction.StateStarting,
transaction.StateSwitching,
transaction.StateVerifying,
transaction.StateDraining,
} {
if _, err := store.Transition(ctx, transactionID, state, "seed committed deployment"); err != nil {
t.Fatalf("transition seed backend transaction to %s: %v", state, err)
}
}
if _, err := store.CommitBackendContainerDeployment(ctx, transactionID, deployment, "seed committed backend container deployment"); err != nil {
t.Fatalf("commit seed backend deployment: %v", err)
}
}
// newContainerUpdaterFixture 构造容器更新器的测试夹具,返回更新器、事务存储、假容器引擎和内存网关。
// containers 指定引擎初始存在的容器;drain 指定旧容器停止前的排水等待时长。
func newContainerUpdaterFixture(
t *testing.T,
containers map[string]containerengine.Container,
drain time.Duration,
) (*Updater, *transaction.Store, *containerUpdateEngine, *memoryGateway) {
t.Helper()
ctx := context.Background()
root := t.TempDir()
store, err := transaction.OpenStore(ctx, filepath.Join(root, "transactions.db"))
if err != nil {
t.Fatalf("open transaction store: %v", err)
}
t.Cleanup(func() {
if err := store.Close(); err != nil {
t.Errorf("close transaction store: %v", err)
}
})
coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("create transaction coordinator: %v", err)
}
engine := &containerUpdateEngine{
image: containerengine.Image{
ID: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
RepoDigests: []string{"harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest},
Platform: containerengine.Platform{OS: "linux", Architecture: "amd64"},
},
containers: containers,
}
configSource := filepath.Join(root, "yms.yaml")
if err := os.WriteFile(configSource, []byte("server: {}\n"), 0o600); err != nil {
t.Fatalf("write backend configuration: %v", err)
}
tmpSource := filepath.Join(root, "tmp")
if err := os.Mkdir(tmpSource, 0o755); err != nil {
t.Fatalf("create backend temporary directory: %v", err)
}
client := &http.Client{Transport: containerUpdateRoundTripFunc(func(request *http.Request) (*http.Response, error) {
engine.healthChecks++
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(bytes.NewBufferString(`{"status":"UP","components":{"ping":{"status":"UP"}}}`)),
Request: request,
}, nil
})}
executor, err := backendexecutor.New(store, coordinator, engine, client)
if err != nil {
t.Fatalf("create backend container executor: %v", err)
}
gateway := &memoryGateway{snapshot: hostnginx.Snapshot{Content: []byte(serverConfiguration8080), ActivePort: 8080}}
updater := &Updater{
config: deploymentconfig.Config{
Daemon: deploymentconfig.Daemon{Environment: deploymentconfig.EnvironmentDev},
Backend: deploymentconfig.Backend{Type: deploymentconfig.BackendTypeContainer, Slot: deploymentconfig.BackendSlots{
Port8080: deploymentconfig.BackendSlot{ContainerName: "backend-8080", HealthEndpoint: "http://127.0.0.1:8080/yms/actuator/health"},
Port8081: deploymentconfig.BackendSlot{ContainerName: "backend-8081", HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health"},
}},
},
workRoot: filepath.Join(root, "work"), store: store, coordinator: coordinator,
gateway: gateway, containerExecutor: executor, engine: engine, drain: drain,
containerConfigSource: configSource, containerConfigTarget: deploymentconfig.ContainerConfigTarget,
containerTmpSource: tmpSource, containerTmpTarget: deploymentconfig.ContainerTmpTarget,
}
return updater, store, engine, gateway
}
// assertCreatedContainerSpec 校验传入容器引擎的创建规格,确认目标容器名称、镜像引用、运行时约束、
// 环境变量和挂载点均符合容器更新契约。
func assertCreatedContainerSpec(t *testing.T, request containerengine.ContainerSpec) {
t.Helper()
if request.Name != "backend-8081" || request.ImageReference != containerTestImage {
t.Fatalf("unexpected target container identity: %+v", request)
}
if request.NetworkMode != "host" || request.RestartPolicy.Name != "no" || request.User != "0:0" {
t.Fatalf("unexpected target container runtime contract: %+v", request)
}
if len(request.Environment) != 2 || request.Environment[0] != "SERVER_PORT=8081" || request.Environment[1] != "SPRING_CONFIG_LOCATION=file:/app/config/yms.yaml" {
t.Fatalf("unexpected target container environment: %+v", request.Environment)
}
if len(request.Mounts) != 2 || request.Mounts[0].Target != deploymentconfig.ContainerConfigTarget || !request.Mounts[0].ReadOnly || request.Mounts[1].Target != deploymentconfig.ContainerTmpTarget {
t.Fatalf("unexpected target container mounts: %+v", request.Mounts)
}
}
// assertCommittedContainerDeployment 校验事务存储中已提交的后端容器部署记录,
// 确认其活跃端口、容器名称、容器 ID、镜像摘要和事务 ID 均与预期一致。
func assertCommittedContainerDeployment(
t *testing.T,
store *transaction.Store,
transactionID string,
port int,
containerName string,
containerID string,
) {
t.Helper()
deployment, err := store.BackendContainerDeployment(context.Background())
if err != nil {
t.Fatalf("read committed backend container deployment: %v", err)
}
if deployment.ActivePort != port || deployment.ContainerName != containerName || deployment.ContainerID != containerID || deployment.ImageDigest != containerTestDigest || deployment.TransactionID != transactionID {
t.Fatalf("unexpected committed backend container deployment: %+v", deployment)
}
}
// serverConfiguration8080 测试用的 host Nginx 配置片段,其活跃后端端口为 8080。
const serverConfiguration8080 = `http {
upstream yms-server {
# yms-update managed upstream begin
server 10.11.1.117:8080 max_fails=1 fail_timeout=2s;
# server 10.11.1.117:8081 max_fails=1 fail_timeout=2s;
# yms-update managed upstream end
}
}
`
// containerUpdateRoundTripFunc 将普通函数适配为 http.RoundTripper
// 使测试可以用自定义逻辑响应后端的健康检查请求。
type containerUpdateRoundTripFunc func(*http.Request) (*http.Response, error)
// RoundTrip 实现 http.RoundTripper 接口,直接委托给底层函数处理请求。
func (function containerUpdateRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return function(request)
}
// containerUpdateEngine 容器引擎的测试替身,在内存中模拟镜像与容器的生命周期,
// 并记录停止、创建和健康检查等交互,供断言验证更新行为。
type containerUpdateEngine struct {
image containerengine.Image
containers map[string]containerengine.Container
stopped []string
lastCreateSpec containerengine.ContainerSpec
healthChecks int
pullCalls int
}
// Ping 返回 nil,模拟引擎连通性检查始终成功。
func (e *containerUpdateEngine) Ping(context.Context) error { return nil }
// PullImage 记录一次拉取调用并返回 nil,模拟镜像拉取始终成功。
func (e *containerUpdateEngine) PullImage(context.Context, string) error {
e.pullCalls++
return nil
}
// LoadImage 返回 nil,模拟从流加载镜像始终成功。
func (e *containerUpdateEngine) LoadImage(context.Context, io.Reader) error { return nil }
// InspectImage 返回夹具预设的镜像信息,用于冻结镜像的平台与仓库摘要。
func (e *containerUpdateEngine) InspectImage(context.Context, string) (containerengine.Image, error) {
return e.image, nil
}
// CreateContainer 依据规格在内存中登记一个新容器,记录最后一次创建规格并返回该容器。
func (e *containerUpdateEngine) CreateContainer(_ context.Context, spec containerengine.ContainerSpec) (containerengine.Container, error) {
e.lastCreateSpec = spec
record := containerengine.Container{
ID: "container-id-" + spec.Name,
Name: spec.Name,
ImageID: e.image.ID,
ImageReference: spec.ImageReference,
Platform: spec.Platform.OS + "/" + spec.Platform.Architecture,
Status: "created",
Environment: append([]string(nil), spec.Environment...),
Labels: spec.Labels,
NetworkMode: spec.NetworkMode,
RestartPolicy: spec.RestartPolicy,
Mounts: append([]containerengine.Mount(nil), spec.Mounts...),
User: spec.User,
StopTimeoutSeconds: spec.StopTimeoutSeconds,
}
e.containers[spec.Name] = record
return record, nil
}
// StartContainer 将指定容器标记为运行中;若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) StartContainer(_ context.Context, name string) error {
record, found := e.containers[name]
if !found {
return containerengine.ErrNotFound
}
record.Running = true
record.Status = "running"
e.containers[name] = record
return nil
}
// ContainerLogs 返回空的日志流,模拟容器日志读取。
func (e *containerUpdateEngine) ContainerLogs(context.Context, string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("")), nil
}
// StopContainer 记录被停止的容器名称并将该容器标记为已退出;若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) StopContainer(_ context.Context, name string, _ int) error {
e.stopped = append(e.stopped, name)
record, found := e.containers[name]
if !found {
return containerengine.ErrNotFound
}
record.Running = false
record.Status = "exited"
e.containers[name] = record
return nil
}
// InspectContainer 返回内存中的指定容器;若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) InspectContainer(_ context.Context, name string) (containerengine.Container, error) {
record, found := e.containers[name]
if !found {
return containerengine.Container{}, containerengine.ErrNotFound
}
return record, nil
}
// RemoveContainer 从内存中删除指定容器;若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) RemoveContainer(_ context.Context, name string, _ bool) error {
if _, found := e.containers[name]; !found {
return containerengine.ErrNotFound
}
delete(e.containers, name)
return nil
}
// Close 返回 nil,模拟引擎关闭无副作用。
func (e *containerUpdateEngine) Close() error { return nil }
var _ containerengine.Engine = (*containerUpdateEngine)(nil)