yms-daemon/internal/backendupdate/container_test.go

package backendupdate

import (
	"bytes"
	"context"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"yms-daemon/internal/backendexecutor"
	"yms-daemon/internal/containerengine"
	"yms-daemon/internal/deploymentconfig"
	"yms-daemon/internal/hostnginx"
	"yms-daemon/internal/transaction"
)

// containerTestImage 容器更新测试使用的镜像引用，指向 harbor 仓库的一个具体 tag。
const containerTestImage = "harbor.ymswell.asia/ymswell/glory-ymswell:20260813-184902-a37bf50d-v1.1.8.1"

// containerTestDigest 容器更新测试使用的镜像仓库摘要，用于与 containerTestImage 组合成不可变引用。
const containerTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"

// TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot 验证在已存在运行中的 backend-8080 容器时，
// 更新镜像会切换到非活跃槽位 backend-8081、提交部署并停止旧容器 backend-8080。
func TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot(t *testing.T) {
	ctx := context.Background()
	updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{
		"backend-8080": {
			ID: "existing-backend-8080", Name: "backend-8080", Running: true,
		},
	}, 0)

	record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
	if err != nil {
		t.Fatalf("update container backend: %v", err)
	}
	if record.State != transaction.StateCommitted || gateway.snapshot.ActivePort != 8081 {
		t.Fatalf("unexpected committed container update: record=%+v gateway=%+v", record, gateway.snapshot)
	}
	if engine.containers["backend-8080"].Running {
		t.Fatal("previous backend container is still running")
	}
	if len(engine.stopped) != 1 || engine.stopped[0] != "backend-8080" {
		t.Fatalf("unexpected stopped containers: %+v", engine.stopped)
	}
	assertCreatedContainerSpec(t, engine.lastCreateSpec)
	assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
}

// TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch 验证全新安装场景：
// 目标容器 backend-8081 必须先在非路由槽位创建并通过健康检查，之后才切换网关；
// 整个过程不得进入旧容器排水流程，也不得停止任何旧容器。
func TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, time.Hour)
	var progress []Progress
	gateway.beforeApply = func(snapshot hostnginx.Snapshot) error {
		target, found := engine.containers["backend-8081"]
		if !found || !target.Running || engine.healthChecks == 0 {
			return errors.New("gateway switch occurred before the target container passed health checking")
		}
		return nil
	}

	record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, func(item Progress) {
		progress = append(progress, item)
	})
	if err != nil {
		t.Fatalf("first install container backend: %v", err)
	}
	if record.State != transaction.StateCommitted {
		t.Fatalf("unexpected committed first install: record=%+v", record)
	}
	if gateway.snapshot.ActivePort != 8081 || gateway.applyCount != 1 {
		t.Fatalf("first install did not switch once to the healthy inactive slot: gateway=%+v applies=%d", gateway.snapshot, gateway.applyCount)
	}
	target, found := engine.containers["backend-8081"]
	if !found || !target.Running || target.Dead {
		t.Fatalf("first install target container is not running: %+v", target)
	}
	if len(engine.stopped) != 0 {
		t.Fatalf("first install must not stop a previous container: %+v", engine.stopped)
	}
	if engine.healthChecks != 1 {
		t.Fatalf("unexpected first-install health check count: %d", engine.healthChecks)
	}
	for _, item := range progress {
		if strings.HasPrefix(item.Message, "Draining previous backend container") {
			t.Fatalf("first install entered previous-container drain: %+v", progress)
		}
	}
	assertCreatedContainerSpec(t, engine.lastCreateSpec)
	assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
}

// TestContainerUpdaterRejectsMissingActiveWithPresentInactive 验证当活跃容器 backend-8080 缺失
// 而非活跃容器 backend-8081 仍在运行时，更新会被拒绝并返回明确的错误信息。
func TestContainerUpdaterRejectsMissingActiveWithPresentInactive(t *testing.T) {
	updater, _, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{
		"backend-8081": {
			ID: "unexpected-backend-8081", Name: "backend-8081", Running: true,
		},
	}, 0)

	_, err := updater.UpdateContainerImage(context.Background(), containerTestImage, true, nil)
	if err == nil || !strings.Contains(err.Error(), "active backend container backend-8080 is missing but inactive container backend-8081 is running") {
		t.Fatalf("unexpected missing-active result: %v", err)
	}
}

// TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage 验证更新失败进入回滚后的恢复链路：
// 首次更新因网关切换失败而进入回滚并停止目标容器，再次调用恢复同一回滚直至完成，
// 最后重试同一镜像能够成功切换并提交。
func TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage(t *testing.T) {
	ctx := context.Background()
	updater, _, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
	failGateway := true
	gateway.beforeApply = func(hostnginx.Snapshot) error {
		if failGateway {
			return errors.New("host Nginx is unavailable")
		}
		return nil
	}

	rollingBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
	if err == nil || rollingBack.State != transaction.StateRollingBack {
		t.Fatalf("unexpected failed rollback result: record=%+v err=%v", rollingBack, err)
	}
	target, found := engine.containers["backend-8081"]
	if !found || target.Running {
		t.Fatalf("failed update target was not stopped: %+v", target)
	}

	failGateway = false
	rolledBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
	if err == nil || rolledBack.State != transaction.StateRolledBack {
		t.Fatalf("unexpected resumed rollback result: record=%+v err=%v", rolledBack, err)
	}

	committed, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
	if err != nil || committed.State != transaction.StateCommitted {
		t.Fatalf("retry same image after rollback: record=%+v err=%v", committed, err)
	}
	if gateway.snapshot.ActivePort != 8081 {
		t.Fatalf("retry did not switch to the healthy container: %+v", gateway.snapshot)
	}
}

// TestContainerUpdaterRejectsMissingCommittedContainer 验证当已提交部署记录中的活跃容器 backend-8080 丢失时，
// 更新会将其识别为部署漂移并拒绝继续。
func TestContainerUpdaterRejectsMissingCommittedContainer(t *testing.T) {
	ctx := context.Background()
	updater, store, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
	transactionID := "committed-container-transaction"
	_, _, err := store.CreateTransaction(ctx, transaction.CreateRequest{
		ID: transactionID, IdempotencyKey: "backend:container:previous", Source: sourceLocalCLI,
		Service: serviceBackend, Request: []byte(`{"inputType":"container-image"}`),
	})
	if err != nil {
		t.Fatalf("create previous backend transaction: %v", err)
	}
	for _, state := range []transaction.State{
		transaction.StateValidating,
		transaction.StatePrepared,
		transaction.StateStarting,
		transaction.StateSwitching,
		transaction.StateVerifying,
		transaction.StateDraining,
	} {
		if _, err := store.Transition(ctx, transactionID, state, "seed committed deployment"); err != nil {
			t.Fatalf("transition previous backend transaction to %s: %v", state, err)
		}
	}
	if _, err := store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{
		ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "missing-backend-8080",
	}, "seed committed backend container deployment"); err != nil {
		t.Fatalf("commit previous backend deployment: %v", err)
	}

	_, err = updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
	if err == nil || !strings.Contains(err.Error(), "committed active backend container backend-8080 is missing") {
		t.Fatalf("unexpected committed-container drift result: %v", err)
	}
}

// newContainerUpdaterFixture 构造容器更新器的测试夹具，返回更新器、事务存储、假容器引擎和内存网关。
// containers 指定引擎初始存在的容器；drain 指定旧容器停止前的排水等待时长。
func newContainerUpdaterFixture(
	t *testing.T,
	containers map[string]containerengine.Container,
	drain time.Duration,
) (*Updater, *transaction.Store, *containerUpdateEngine, *memoryGateway) {
	t.Helper()
	ctx := context.Background()
	root := t.TempDir()
	store, err := transaction.OpenStore(ctx, filepath.Join(root, "transactions.db"))
	if err != nil {
		t.Fatalf("open transaction store: %v", err)
	}
	t.Cleanup(func() {
		if err := store.Close(); err != nil {
			t.Errorf("close transaction store: %v", err)
		}
	})
	coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
	if err != nil {
		t.Fatalf("create transaction coordinator: %v", err)
	}
	engine := &containerUpdateEngine{
		image: containerengine.Image{
			ID:          "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
			RepoDigests: []string{"harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest},
			Platform:    containerengine.Platform{OS: "linux", Architecture: "amd64"},
		},
		containers: containers,
	}
	configSource := filepath.Join(root, "yms.yaml")
	if err := os.WriteFile(configSource, []byte("server: {}\n"), 0o600); err != nil {
		t.Fatalf("write backend configuration: %v", err)
	}
	tmpSource := filepath.Join(root, "tmp")
	if err := os.Mkdir(tmpSource, 0o755); err != nil {
		t.Fatalf("create backend temporary directory: %v", err)
	}
	client := &http.Client{Transport: containerUpdateRoundTripFunc(func(request *http.Request) (*http.Response, error) {
		engine.healthChecks++
		return &http.Response{
			StatusCode: http.StatusOK,
			Header:     make(http.Header),
			Body:       io.NopCloser(bytes.NewBufferString(`{"status":"UP","components":{"ping":{"status":"UP"}}}`)),
			Request:    request,
		}, nil
	})}
	executor, err := backendexecutor.New(store, coordinator, engine, client)
	if err != nil {
		t.Fatalf("create backend container executor: %v", err)
	}
	gateway := &memoryGateway{snapshot: hostnginx.Snapshot{Content: []byte(serverConfiguration8080), ActivePort: 8080}}
	updater := &Updater{
		config: deploymentconfig.Config{
			Daemon: deploymentconfig.Daemon{Environment: deploymentconfig.EnvironmentDev},
			Backend: deploymentconfig.Backend{Type: deploymentconfig.BackendTypeContainer, Slot: deploymentconfig.BackendSlots{
				Port8080: deploymentconfig.BackendSlot{ContainerName: "backend-8080", HealthEndpoint: "http://127.0.0.1:8080/yms/actuator/health"},
				Port8081: deploymentconfig.BackendSlot{ContainerName: "backend-8081", HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health"},
			}},
		},
		workRoot: filepath.Join(root, "work"), store: store, coordinator: coordinator,
		gateway: gateway, containerExecutor: executor, engine: engine, drain: drain,
		containerConfigSource: configSource, containerConfigTarget: deploymentconfig.ContainerConfigTarget,
		containerTmpSource: tmpSource, containerTmpTarget: deploymentconfig.ContainerTmpTarget,
	}
	return updater, store, engine, gateway
}

// assertCreatedContainerSpec 校验传入容器引擎的创建规格，确认目标容器名称、镜像引用、运行时约束、
// 环境变量和挂载点均符合容器更新契约。
func assertCreatedContainerSpec(t *testing.T, request containerengine.ContainerSpec) {
	t.Helper()
	if request.Name != "backend-8081" || request.ImageReference != "harbor.ymswell.asia/ymswell/glory-ymswell@"+containerTestDigest {
		t.Fatalf("unexpected target container identity: %+v", request)
	}
	if request.NetworkMode != "host" || request.RestartPolicy.Name != "no" || request.User != "0:0" {
		t.Fatalf("unexpected target container runtime contract: %+v", request)
	}
	if len(request.Environment) != 2 || request.Environment[0] != "SERVER_PORT=8081" || request.Environment[1] != "SPRING_CONFIG_LOCATION=file:/app/config/yms.yaml" {
		t.Fatalf("unexpected target container environment: %+v", request.Environment)
	}
	if len(request.Mounts) != 2 || request.Mounts[0].Target != deploymentconfig.ContainerConfigTarget || !request.Mounts[0].ReadOnly || request.Mounts[1].Target != deploymentconfig.ContainerTmpTarget {
		t.Fatalf("unexpected target container mounts: %+v", request.Mounts)
	}
}

// assertCommittedContainerDeployment 校验事务存储中已提交的后端容器部署记录，
// 确认其活跃端口、容器名称、容器 ID、镜像摘要和事务 ID 均与预期一致。
func assertCommittedContainerDeployment(
	t *testing.T,
	store *transaction.Store,
	transactionID string,
	port int,
	containerName string,
	containerID string,
) {
	t.Helper()
	deployment, err := store.BackendContainerDeployment(context.Background())
	if err != nil {
		t.Fatalf("read committed backend container deployment: %v", err)
	}
	if deployment.ActivePort != port || deployment.ContainerName != containerName || deployment.ContainerID != containerID || deployment.ImageDigest != containerTestDigest || deployment.TransactionID != transactionID {
		t.Fatalf("unexpected committed backend container deployment: %+v", deployment)
	}
}

// serverConfiguration8080 测试用的 host Nginx 配置片段，其活跃后端端口为 8080。
const serverConfiguration8080 = `http {
    upstream yms-server {
        # yms-update managed upstream begin
        server 10.11.1.117:8080 max_fails=1 fail_timeout=2s;
        # server 10.11.1.117:8081 max_fails=1 fail_timeout=2s;
        # yms-update managed upstream end
    }
}
`

// containerUpdateRoundTripFunc 将普通函数适配为 http.RoundTripper，
// 使测试可以用自定义逻辑响应后端的健康检查请求。
type containerUpdateRoundTripFunc func(*http.Request) (*http.Response, error)

// RoundTrip 实现 http.RoundTripper 接口，直接委托给底层函数处理请求。
func (function containerUpdateRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
	return function(request)
}

// containerUpdateEngine 容器引擎的测试替身，在内存中模拟镜像与容器的生命周期，
// 并记录停止、创建和健康检查等交互，供断言验证更新行为。
type containerUpdateEngine struct {
	image          containerengine.Image
	containers     map[string]containerengine.Container
	stopped        []string
	lastCreateSpec containerengine.ContainerSpec
	healthChecks   int
}

// Ping 返回 nil，模拟引擎连通性检查始终成功。
func (e *containerUpdateEngine) Ping(context.Context) error { return nil }

// PullImage 返回 nil，模拟镜像拉取始终成功。
func (e *containerUpdateEngine) PullImage(context.Context, string) error { return nil }

// LoadImage 返回 nil，模拟从流加载镜像始终成功。
func (e *containerUpdateEngine) LoadImage(context.Context, io.Reader) error { return nil }

// InspectImage 返回夹具预设的镜像信息，用于冻结镜像的平台与仓库摘要。
func (e *containerUpdateEngine) InspectImage(context.Context, string) (containerengine.Image, error) {
	return e.image, nil
}

// CreateContainer 依据规格在内存中登记一个新容器，记录最后一次创建规格并返回该容器。
func (e *containerUpdateEngine) CreateContainer(_ context.Context, spec containerengine.ContainerSpec) (containerengine.Container, error) {
	e.lastCreateSpec = spec
	record := containerengine.Container{
		ID:                 "container-id-" + spec.Name,
		Name:               spec.Name,
		ImageID:            e.image.ID,
		ImageReference:     spec.ImageReference,
		Platform:           spec.Platform.OS + "/" + spec.Platform.Architecture,
		Status:             "created",
		Environment:        append([]string(nil), spec.Environment...),
		Labels:             spec.Labels,
		NetworkMode:        spec.NetworkMode,
		RestartPolicy:      spec.RestartPolicy,
		Mounts:             append([]containerengine.Mount(nil), spec.Mounts...),
		User:               spec.User,
		StopTimeoutSeconds: spec.StopTimeoutSeconds,
	}
	e.containers[spec.Name] = record
	return record, nil
}

// StartContainer 将指定容器标记为运行中；若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) StartContainer(_ context.Context, name string) error {
	record, found := e.containers[name]
	if !found {
		return containerengine.ErrNotFound
	}
	record.Running = true
	record.Status = "running"
	e.containers[name] = record
	return nil
}

// ContainerLogs 返回空的日志流，模拟容器日志读取。
func (e *containerUpdateEngine) ContainerLogs(context.Context, string) (io.ReadCloser, error) {
	return io.NopCloser(strings.NewReader("")), nil
}

// StopContainer 记录被停止的容器名称并将该容器标记为已退出；若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) StopContainer(_ context.Context, name string) error {
	e.stopped = append(e.stopped, name)
	record, found := e.containers[name]
	if !found {
		return containerengine.ErrNotFound
	}
	record.Running = false
	record.Status = "exited"
	e.containers[name] = record
	return nil
}

// InspectContainer 返回内存中的指定容器；若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) InspectContainer(_ context.Context, name string) (containerengine.Container, error) {
	record, found := e.containers[name]
	if !found {
		return containerengine.Container{}, containerengine.ErrNotFound
	}
	return record, nil
}

// RemoveContainer 从内存中删除指定容器；若容器不存在则返回 containerengine.ErrNotFound。
func (e *containerUpdateEngine) RemoveContainer(_ context.Context, name string, _ bool) error {
	if _, found := e.containers[name]; !found {
		return containerengine.ErrNotFound
	}
	delete(e.containers, name)
	return nil
}

// Close 返回 nil，模拟引擎关闭无副作用。
func (e *containerUpdateEngine) Close() error { return nil }

var _ containerengine.Engine = (*containerUpdateEngine)(nil)