feat: backend container executor implement

2026-08-15 20:58:01 +08:00
parent b4e18c6f0c
commit fbce91d797
10 changed files with 1662 additions and 1 deletions
@@ -0,0 +1,437 @@
+package backendexecutor
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+	"slices"
+	"sync"
+	"testing"
+
+	"yms-daemon/internal/containerengine"
+	"yms-daemon/internal/transaction"
+)
+
+const (
+	testDigest      = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	testRepository  = "harbor.ymswell.asia/ymswell/glory-ymswell"
+	healthyResponse = `{"status":"UP","components":{"db":{"status":"UP"},"diskSpace":{"status":"UP"},"ping":{"status":"UP"},"redis":{"status":"UP"}}}`
+)
+
+func TestExecutorPreservesOpaqueImageTagsAndReachesSwitching(t *testing.T) {
+	tags := []string{
+		"20260814-093609-d7ed70f0-v1.1.8.1",
+		"20260814-093609-d7ed70f0",
+	}
+	for _, tag := range tags {
+		tag := tag
+		t.Run(tag, func(t *testing.T) {
+			ctx := context.Background()
+			store, coordinator := testTransactionKernel(t)
+			imageReference := testRepository + ":" + tag
+			request := testRequest(t, imageReference)
+			engine := newFakeEngine(request)
+			executor := testExecutor(t, store, coordinator, engine, healthyResponse)
+			record := createTransaction(t, store, "opaque-"+tag)
+
+			if err := executor.Run(ctx, record.ID, request); err != nil {
+				t.Fatalf("run backend executor: %v", err)
+			}
+			current, err := store.Transaction(ctx, record.ID)
+			if err != nil {
+				t.Fatalf("read completed preparation: %v", err)
+			}
+			if current.State != transaction.StateSwitching {
+				t.Fatalf("unexpected transaction state: %s", current.State)
+			}
+			pending, err := store.PendingSteps(ctx, record.ID)
+			if err != nil || len(pending) != 0 {
+				t.Fatalf("unexpected pending steps: steps=%+v err=%v", pending, err)
+			}
+
+			engine.mu.Lock()
+			if engine.lastCreateSpec.ImageReference != imageReference {
+				engine.mu.Unlock()
+				t.Fatalf("image reference changed: got %q want %q", engine.lastCreateSpec.ImageReference, imageReference)
+			}
+			if !slices.Contains(engine.lastCreateSpec.Environment, "SERVER_PORT=8081") {
+				engine.mu.Unlock()
+				t.Fatalf("missing explicit port environment: %+v", engine.lastCreateSpec.Environment)
+			}
+			initialCalls := engine.callCounts()
+			engine.mu.Unlock()
+
+			if err := executor.Run(ctx, record.ID, request); err != nil {
+				t.Fatalf("repeat executor at switching state: %v", err)
+			}
+			engine.mu.Lock()
+			repeatedCalls := engine.callCounts()
+			engine.mu.Unlock()
+			if repeatedCalls != initialCalls {
+				t.Fatalf("switching state repeated engine calls: before=%+v after=%+v", initialCalls, repeatedCalls)
+			}
+		})
+	}
+}
+
+func TestExecutorRecoversRecordedCreateIntentWithoutRepeatingCreate(t *testing.T) {
+	ctx := context.Background()
+	store, coordinator := testTransactionKernel(t)
+	request := testRequest(t, testRepository+":20260814-093609-d7ed70f0-v1.1.8.1")
+	engine := newFakeEngine(request)
+	engine.imageAvailable = true
+	engine.containers[request.ContainerName] = engine.containerFromSpec(containerSpec(request), false)
+	executor := testExecutor(t, store, coordinator, engine, healthyResponse)
+	record := createTransaction(t, store, "recover-create")
+	transitionToPrepared(t, store, record.ID)
+
+	loadStep := loadIntent(request)
+	if _, _, err := store.RecordStepIntent(ctx, record.ID, loadStep); err != nil {
+		t.Fatalf("record completed image intent: %v", err)
+	}
+	if _, err := store.CompleteStep(ctx, record.ID, loadStep.Key, transaction.StepSucceeded, []byte(`{"loaded":true}`), ""); err != nil {
+		t.Fatalf("complete image step: %v", err)
+	}
+	createStep := createIntent(request, engine.loadedImage.ID)
+	if _, _, err := store.RecordStepIntent(ctx, record.ID, createStep); err != nil {
+		t.Fatalf("record crash-window create intent: %v", err)
+	}
+
+	if err := executor.Run(ctx, record.ID, request); err != nil {
+		t.Fatalf("recover backend executor: %v", err)
+	}
+	engine.mu.Lock()
+	calls := engine.callCounts()
+	engine.mu.Unlock()
+	if calls.load != 0 || calls.create != 0 || calls.start != 1 {
+		t.Fatalf("unexpected recovery calls: %+v", calls)
+	}
+	current, err := store.Transaction(ctx, record.ID)
+	if err != nil || current.State != transaction.StateSwitching {
+		t.Fatalf("unexpected recovered state: record=%+v err=%v", current, err)
+	}
+}
+
+func TestExecutorMarksValidationFailureTerminal(t *testing.T) {
+	ctx := context.Background()
+	store, coordinator := testTransactionKernel(t)
+	request := testRequest(t, testRepository+":20260814-093609-d7ed70f0")
+	request.Port = 9090
+	engine := newFakeEngine(request)
+	executor := testExecutor(t, store, coordinator, engine, healthyResponse)
+	record := createTransaction(t, store, "invalid-port")
+
+	if err := executor.Run(ctx, record.ID, request); err == nil {
+		t.Fatal("expected validation failure")
+	}
+	current, err := store.Transaction(ctx, record.ID)
+	if err != nil || current.State != transaction.StateFailed {
+		t.Fatalf("unexpected failed transaction: record=%+v err=%v", current, err)
+	}
+}
+
+func TestExecutorKeepsPreparedStateForConflictingContainerInspection(t *testing.T) {
+	ctx := context.Background()
+	store, coordinator := testTransactionKernel(t)
+	request := testRequest(t, testRepository+":20260814-093609-d7ed70f0")
+	engine := newFakeEngine(request)
+	engine.imageAvailable = true
+	conflicting := engine.containerFromSpec(containerSpec(request), false)
+	conflicting.ImageID = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+	engine.containers[request.ContainerName] = conflicting
+	executor := testExecutor(t, store, coordinator, engine, healthyResponse)
+	record := createTransaction(t, store, "container-conflict")
+
+	err := executor.Run(ctx, record.ID, request)
+	var uncertain *transaction.UncertainStepError
+	if !errors.As(err, &uncertain) {
+		t.Fatalf("expected uncertain container step, got %v", err)
+	}
+	current, readErr := store.Transaction(ctx, record.ID)
+	if readErr != nil || current.State != transaction.StatePrepared {
+		t.Fatalf("conflict did not preserve prepared state: record=%+v err=%v", current, readErr)
+	}
+}
+
+func TestExecutorRejectsChangedRecoveryRequestWithoutChangingState(t *testing.T) {
+	ctx := context.Background()
+	store, coordinator := testTransactionKernel(t)
+	request := testRequest(t, testRepository+":20260814-093609-d7ed70f0-v1.1.8.1")
+	engine := newFakeEngine(request)
+	engine.imageAvailable = true
+	executor := testExecutor(t, store, coordinator, engine, healthyResponse)
+	record := createTransaction(t, store, "changed-recovery-request")
+	transitionToPrepared(t, store, record.ID)
+	if err := executor.prepare(ctx, record.ID, request); err != nil {
+		t.Fatalf("prepare original request: %v", err)
+	}
+	if _, err := store.Transition(ctx, record.ID, transaction.StateStarting, "test starting"); err != nil {
+		t.Fatalf("transition to starting: %v", err)
+	}
+
+	changed := request
+	changed.ImageReference = testRepository + ":20260814-093609-d7ed70f0"
+	err := executor.Run(ctx, record.ID, changed)
+	if !errors.Is(err, transaction.ErrStepConflict) {
+		t.Fatalf("expected persisted intent conflict, got %v", err)
+	}
+	current, readErr := store.Transaction(ctx, record.ID)
+	if readErr != nil || current.State != transaction.StateStarting {
+		t.Fatalf("changed recovery request altered transaction: record=%+v err=%v", current, readErr)
+	}
+}
+
+func TestImageMatchesRequiresDigestEvidenceAndExactPlatform(t *testing.T) {
+	t.Parallel()
+	platform := containerengine.Platform{OS: "linux", Architecture: "arm64"}
+	image := containerengine.Image{
+		ID:          "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+		RepoDigests: []string{testRepository + "@" + testDigest},
+		Platform:    platform,
+	}
+	matched, err := imageMatches(image, testDigest, platform)
+	if err != nil || !matched {
+		t.Fatalf("expected exact image match: matched=%t err=%v", matched, err)
+	}
+	image.RepoDigests = nil
+	matched, err = imageMatches(image, testDigest, platform)
+	if err != nil || matched {
+		t.Fatalf("image ID alone must not satisfy manifest digest: matched=%t err=%v", matched, err)
+	}
+	image.DescriptorDigest = testDigest
+	matched, err = imageMatches(image, testDigest, containerengine.Platform{OS: "linux", Architecture: "amd64"})
+	if err != nil || matched {
+		t.Fatalf("platform mismatch accepted: matched=%t err=%v", matched, err)
+	}
+}
+
+func testRequest(t *testing.T, imageReference string) Request {
+	t.Helper()
+	directory := t.TempDir()
+	archivePath := filepath.Join(directory, "backend-image.tar")
+	configPath := filepath.Join(directory, "yms.yaml")
+	if err := os.WriteFile(archivePath, []byte("image archive"), 0o600); err != nil {
+		t.Fatalf("write image archive: %v", err)
+	}
+	if err := os.WriteFile(configPath, []byte("server: {}\n"), 0o600); err != nil {
+		t.Fatalf("write backend configuration: %v", err)
+	}
+	return Request{
+		ArchivePath:         archivePath,
+		ImageReference:      imageReference,
+		ExpectedImageDigest: testDigest,
+		Platform:            containerengine.Platform{OS: "linux", Architecture: "amd64"},
+		ContainerName:       "explicit-backend-8081",
+		Port:                8081,
+		PortEnvironmentKey:  "SERVER_PORT",
+		ConfigSource:        configPath,
+		ConfigTarget:        "/app/config/application.yaml",
+		RestartPolicy:       containerengine.RestartPolicy{Name: "unless-stopped"},
+		HealthEndpoint:      "http://127.0.0.1:8081/yms/actuator/health",
+	}
+}
+
+func testTransactionKernel(t *testing.T) (*transaction.Store, *transaction.Coordinator) {
+	t.Helper()
+	store, err := transaction.OpenStore(context.Background(), filepath.Join(t.TempDir(), "transactions.db"))
+	if err != nil {
+		t.Fatalf("open transaction store: %v", err)
+	}
+	t.Cleanup(func() { _ = store.Close() })
+	coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	if err != nil {
+		t.Fatalf("create transaction coordinator: %v", err)
+	}
+	return store, coordinator
+}
+
+func testExecutor(t *testing.T, store *transaction.Store, coordinator *transaction.Coordinator, engine containerengine.Engine, body string) *Executor {
+	t.Helper()
+	client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+		return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(bytes.NewBufferString(body))}, nil
+	})}
+	executor, err := New(store, coordinator, engine, client)
+	if err != nil {
+		t.Fatalf("create backend executor: %v", err)
+	}
+	return executor
+}
+
+func createTransaction(t *testing.T, store *transaction.Store, suffix string) transaction.Transaction {
+	t.Helper()
+	record, _, err := store.CreateTransaction(context.Background(), transaction.CreateRequest{
+		ID:             "backend-" + suffix,
+		IdempotencyKey: "backend-request-" + suffix,
+		Source:         "test",
+		Service:        "backend",
+	})
+	if err != nil {
+		t.Fatalf("create backend transaction: %v", err)
+	}
+	return record
+}
+
+func transitionToPrepared(t *testing.T, store *transaction.Store, transactionID string) {
+	t.Helper()
+	ctx := context.Background()
+	if _, err := store.Transition(ctx, transactionID, transaction.StateValidating, "test validating"); err != nil {
+		t.Fatalf("transition to validating: %v", err)
+	}
+	if _, err := store.Transition(ctx, transactionID, transaction.StatePrepared, "test prepared"); err != nil {
+		t.Fatalf("transition to prepared: %v", err)
+	}
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+	return f(request)
+}
+
+type engineCalls struct {
+	ping    int
+	load    int
+	inspect int
+	create  int
+	start   int
+	remove  int
+}
+
+type fakeEngine struct {
+	mu             sync.Mutex
+	request        Request
+	loadedImage    containerengine.Image
+	imageAvailable bool
+	containers     map[string]containerengine.Container
+	lastCreateSpec containerengine.ContainerSpec
+	pingCalls      int
+	loadCalls      int
+	inspectCalls   int
+	createCalls    int
+	startCalls     int
+	removeCalls    int
+}
+
+func newFakeEngine(request Request) *fakeEngine {
+	return &fakeEngine{
+		request: request,
+		loadedImage: containerengine.Image{
+			ID:          "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+			RepoDigests: []string{testRepository + "@" + testDigest},
+			Platform:    request.Platform,
+		},
+		containers: make(map[string]containerengine.Container),
+	}
+}
+
+func (e *fakeEngine) Ping(context.Context) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.pingCalls++
+	return nil
+}
+
+func (e *fakeEngine) LoadImage(_ context.Context, input io.Reader) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.loadCalls++
+	if _, err := io.ReadAll(input); err != nil {
+		return err
+	}
+	e.imageAvailable = true
+	return nil
+}
+
+func (e *fakeEngine) InspectImage(context.Context, string) (containerengine.Image, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.inspectCalls++
+	if !e.imageAvailable {
+		return containerengine.Image{}, containerengine.ErrNotFound
+	}
+	return e.loadedImage, nil
+}
+
+func (e *fakeEngine) CreateContainer(_ context.Context, spec containerengine.ContainerSpec) (containerengine.Container, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.createCalls++
+	e.lastCreateSpec = spec
+	if _, exists := e.containers[spec.Name]; exists {
+		return containerengine.Container{}, errors.New("container name already exists")
+	}
+	record := e.containerFromSpec(spec, false)
+	e.containers[spec.Name] = record
+	return record, nil
+}
+
+func (e *fakeEngine) StartContainer(_ context.Context, name string) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.startCalls++
+	record, exists := e.containers[name]
+	if !exists {
+		return containerengine.ErrNotFound
+	}
+	record.Running = true
+	record.Status = "running"
+	e.containers[name] = record
+	return nil
+}
+
+func (e *fakeEngine) InspectContainer(_ context.Context, name string) (containerengine.Container, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	record, exists := e.containers[name]
+	if !exists {
+		return containerengine.Container{}, containerengine.ErrNotFound
+	}
+	return record, nil
+}
+
+func (e *fakeEngine) RemoveContainer(_ context.Context, name string, _ bool) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.removeCalls++
+	if _, exists := e.containers[name]; !exists {
+		return containerengine.ErrNotFound
+	}
+	delete(e.containers, name)
+	return nil
+}
+
+func (e *fakeEngine) Close() error { return nil }
+
+func (e *fakeEngine) containerFromSpec(spec containerengine.ContainerSpec, running bool) containerengine.Container {
+	return containerengine.Container{
+		ID:             "container-id-" + spec.Name,
+		Name:           spec.Name,
+		ImageID:        e.loadedImage.ID,
+		ImageReference: spec.ImageReference,
+		Platform:       spec.Platform.OS + "/" + spec.Platform.Architecture,
+		Running:        running,
+		Status:         "created",
+		Environment:    append([]string(nil), spec.Environment...),
+		NetworkMode:    spec.NetworkMode,
+		RestartPolicy:  spec.RestartPolicy,
+		Mounts:         append([]containerengine.Mount(nil), spec.Mounts...),
+	}
+}
+
+func (e *fakeEngine) callCounts() engineCalls {
+	return engineCalls{
+		ping:    e.pingCalls,
+		load:    e.loadCalls,
+		inspect: e.inspectCalls,
+		create:  e.createCalls,
+		start:   e.startCalls,
+		remove:  e.removeCalls,
+	}
+}
+
+var _ containerengine.Engine = (*fakeEngine)(nil)