feat: backend executor implement

This commit is contained in:
2026-08-16 17:12:06 +08:00
parent 390e0565d3
commit 7755da72e7
31 changed files with 2762 additions and 184 deletions
+109 -41
View File
@@ -24,31 +24,44 @@ import (
)
const (
healthPath = "/yms/actuator/health"
healthTimeout = 120 * time.Second
healthInterval = time.Second
hostNetworkMode = "host"
bindMountType = "bind"
stepLoadImage = "backend.image.load"
stepCreateContainer = "backend.container.create"
stepStartContainer = "backend.container.start"
stepCheckHealth = "backend.container.health"
healthPath = "/yms/actuator/health"
healthTimeout = 120 * time.Second
healthInterval = time.Second
containerStopTimeoutSeconds = 150 * 60
hostNetworkMode = "host"
bindMountType = "bind"
stepLoadImage = "backend.image.load"
stepPullImage = "backend.image.pull"
stepRemoveContainer = "backend.container.remove-inactive"
stepCreateContainer = "backend.container.create"
stepStartContainer = "backend.container.start"
stepCheckHealth = "backend.container.health"
)
const (
ImageAcquisitionLoad = "load"
ImageAcquisitionPull = "pull"
)
// Request contains exact values supplied by the update package and local deployment configuration.
// ImageReference is opaque: the executor never extracts meaning from its tag.
type Request struct {
ArchivePath string
ImageReference string
ExpectedImageDigest string
Platform containerengine.Platform
ContainerName string
Port int
PortEnvironmentKey string
ConfigSource string
ConfigTarget string
RestartPolicy containerengine.RestartPolicy
HealthEndpoint string
ImageAcquisition string
ArchivePath string
ImageReference string
ExpectedImageDigest string
Platform containerengine.Platform
ContainerName string
Port int
PortEnvironmentKey string
ConfigSource string
ConfigTarget string
TmpSource string
TmpTarget string
ConfigEnvironmentKey string
ConfigLocation string
RestartPolicy containerengine.RestartPolicy
HealthEndpoint string
}
// Executor drives the persisted transaction up to SWITCHING after the new container is healthy.
@@ -115,10 +128,10 @@ func (e *Executor) run(ctx context.Context, transactionID string, request Reques
case transaction.StateStarting:
// 重放 PREPARED 步骤会核对持久化意图,阻止恢复时换入另一组请求参数。
if err := e.prepare(ctx, transactionID, request); err != nil {
return e.failUnlessRecoverable(ctx, transactionID, err)
return err
}
if err := e.startAndCheck(ctx, transactionID, request); err != nil {
return e.failUnlessRecoverable(ctx, transactionID, err)
return err
}
if _, err := e.store.Transition(ctx, transactionID, transaction.StateSwitching, "backend container is healthy"); err != nil {
return err
@@ -144,12 +157,17 @@ func (e *Executor) validate(ctx context.Context, request Request) error {
if err := validateRequest(request); err != nil {
return err
}
if err := regularFile(request.ArchivePath, "image archive"); err != nil {
return err
if request.ImageAcquisition == ImageAcquisitionLoad {
if err := regularFile(request.ArchivePath, "image archive"); err != nil {
return err
}
}
if err := regularFile(request.ConfigSource, "backend configuration"); err != nil {
return err
}
if err := directDirectory(request.TmpSource, "backend temporary directory"); err != nil {
return err
}
if err := e.engine.Ping(ctx); err != nil {
return err
}
@@ -157,21 +175,27 @@ func (e *Executor) validate(ctx context.Context, request Request) error {
}
func (e *Executor) prepare(ctx context.Context, transactionID string, request Request) error {
loadOperation := &loadImageOperation{
engine: e.engine,
archivePath: request.ArchivePath,
imageReference: request.ImageReference,
expectedDigest: request.ExpectedImageDigest,
platform: request.Platform,
}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, loadIntent(request), loadOperation); err != nil {
return err
switch request.ImageAcquisition {
case ImageAcquisitionLoad:
operation := &loadImageOperation{engine: e.engine, archivePath: request.ArchivePath, imageReference: request.ImageReference, expectedDigest: request.ExpectedImageDigest, platform: request.Platform}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, loadIntent(request), operation); err != nil {
return err
}
case ImageAcquisitionPull:
operation := &pullImageOperation{engine: e.engine, imageReference: request.ImageReference, expectedDigest: request.ExpectedImageDigest, platform: request.Platform}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, pullIntent(request), operation); err != nil {
return err
}
}
image, err := e.engine.InspectImage(ctx, request.ImageReference)
if err != nil {
return fmt.Errorf("inspect prepared backend image: %w", err)
}
removeOperation := &removeContainerOperation{engine: e.engine, name: request.ContainerName}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, removeIntent(request), removeOperation); err != nil {
return err
}
createOperation := &createContainerOperation{
engine: e.engine,
expectedImage: image,
@@ -198,8 +222,17 @@ func (e *Executor) startAndCheck(ctx context.Context, transactionID string, requ
}
func validateRequest(request Request) error {
if !filepath.IsAbs(request.ArchivePath) {
return errors.New("image archive path must be absolute")
switch request.ImageAcquisition {
case ImageAcquisitionLoad:
if !filepath.IsAbs(request.ArchivePath) {
return errors.New("image archive path must be absolute")
}
case ImageAcquisitionPull:
if request.ArchivePath != "" {
return errors.New("pull image acquisition does not accept an archive path")
}
default:
return fmt.Errorf("unsupported image acquisition: %q", request.ImageAcquisition)
}
if request.ImageReference == "" || strings.TrimSpace(request.ImageReference) != request.ImageReference {
return errors.New("exact image reference is required")
@@ -222,6 +255,15 @@ func validateRequest(request Request) error {
if !filepath.IsAbs(request.ConfigSource) || !filepath.IsAbs(request.ConfigTarget) {
return errors.New("backend configuration source and target must be absolute paths")
}
if !filepath.IsAbs(request.TmpSource) || !filepath.IsAbs(request.TmpTarget) {
return errors.New("backend temporary source and target must be absolute paths")
}
if request.ConfigEnvironmentKey == "" || strings.Contains(request.ConfigEnvironmentKey, "=") || strings.TrimSpace(request.ConfigEnvironmentKey) != request.ConfigEnvironmentKey {
return errors.New("exact backend configuration environment key is required")
}
if request.ConfigLocation == "" || strings.TrimSpace(request.ConfigLocation) != request.ConfigLocation {
return errors.New("exact backend configuration location is required")
}
if err := validateRestartPolicy(request.RestartPolicy); err != nil {
return err
}
@@ -262,6 +304,17 @@ func regularFile(path, description string) error {
return nil
}
func directDirectory(path, description string) error {
info, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("inspect %s %s: %w", description, path, err)
}
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("%s is not a direct directory: %s", description, path)
}
return nil
}
func containerSpec(request Request) containerengine.ContainerSpec {
return containerengine.ContainerSpec{
Name: request.ContainerName,
@@ -269,18 +322,27 @@ func containerSpec(request Request) containerengine.ContainerSpec {
Platform: request.Platform,
Environment: []string{
request.PortEnvironmentKey + "=" + strconv.Itoa(request.Port),
request.ConfigEnvironmentKey + "=" + request.ConfigLocation,
},
NetworkMode: hostNetworkMode,
RestartPolicy: request.RestartPolicy,
Mounts: []containerengine.Mount{{
Type: bindMountType,
Source: request.ConfigSource,
Target: request.ConfigTarget,
ReadOnly: true,
}},
Mounts: []containerengine.Mount{
{Type: bindMountType, Source: request.ConfigSource, Target: request.ConfigTarget, ReadOnly: true},
{Type: bindMountType, Source: request.TmpSource, Target: request.TmpTarget},
},
User: "0:0",
StopTimeoutSeconds: containerStopTimeoutSeconds,
}
}
func pullIntent(request Request) transaction.StepIntent {
return intent(stepPullImage, "pull and verify backend image", struct {
ImageReference string `json:"imageReference"`
ImageDigest string `json:"imageDigest"`
Platform containerengine.Platform `json:"platform"`
}{request.ImageReference, request.ExpectedImageDigest, request.Platform})
}
func loadIntent(request Request) transaction.StepIntent {
return intent(stepLoadImage, "load and verify backend image", struct {
ArchivePath string `json:"archivePath"`
@@ -297,6 +359,12 @@ func createIntent(request Request, imageID string) transaction.StepIntent {
}{containerSpec(request), imageID})
}
func removeIntent(request Request) transaction.StepIntent {
return intent(stepRemoveContainer, "remove inactive backend container", struct {
ContainerName string `json:"containerName"`
}{request.ContainerName})
}
func startIntent(request Request) transaction.StepIntent {
return intent(stepStartContainer, "start inactive backend container", struct {
ContainerName string `json:"containerName"`
+70 -31
View File
@@ -59,7 +59,7 @@ func TestExecutorPreservesOpaqueImageTagsAndReachesSwitching(t *testing.T) {
engine.mu.Unlock()
t.Fatalf("image reference changed: got %q want %q", engine.lastCreateSpec.ImageReference, imageReference)
}
if !slices.Contains(engine.lastCreateSpec.Environment, "SERVER_PORT=8081") {
if !slices.Contains(engine.lastCreateSpec.Environment, "SERVER_PORT=8081") || !slices.Contains(engine.lastCreateSpec.Environment, "SPRING_CONFIG_LOCATION=file:/app/config/yms.yaml") {
engine.mu.Unlock()
t.Fatalf("missing explicit port environment: %+v", engine.lastCreateSpec.Environment)
}
@@ -97,6 +97,13 @@ func TestExecutorRecoversRecordedCreateIntentWithoutRepeatingCreate(t *testing.T
if _, err := store.CompleteStep(ctx, record.ID, loadStep.Key, transaction.StepSucceeded, []byte(`{"loaded":true}`), ""); err != nil {
t.Fatalf("complete image step: %v", err)
}
removeStep := removeIntent(request)
if _, _, err := store.RecordStepIntent(ctx, record.ID, removeStep); err != nil {
t.Fatalf("record completed remove intent: %v", err)
}
if _, err := store.CompleteStep(ctx, record.ID, removeStep.Key, transaction.StepSucceeded, []byte(`{"absent":true}`), ""); err != nil {
t.Fatalf("complete remove step: %v", err)
}
createStep := createIntent(request, engine.loadedImage.ID)
if _, _, err := store.RecordStepIntent(ctx, record.ID, createStep); err != nil {
t.Fatalf("record crash-window create intent: %v", err)
@@ -108,7 +115,7 @@ func TestExecutorRecoversRecordedCreateIntentWithoutRepeatingCreate(t *testing.T
engine.mu.Lock()
calls := engine.callCounts()
engine.mu.Unlock()
if calls.load != 0 || calls.create != 0 || calls.start != 1 {
if calls.load != 0 || calls.remove != 0 || calls.create != 0 || calls.start != 1 {
t.Fatalf("unexpected recovery calls: %+v", calls)
}
current, err := store.Transaction(ctx, record.ID)
@@ -135,7 +142,7 @@ func TestExecutorMarksValidationFailureTerminal(t *testing.T) {
}
}
func TestExecutorKeepsPreparedStateForConflictingContainerInspection(t *testing.T) {
func TestExecutorReplacesInactiveContainerWithConflictingImage(t *testing.T) {
ctx := context.Background()
store, coordinator := testTransactionKernel(t)
request := testRequest(t, testRepository+":20260814-093609-d7ed70f0")
@@ -147,14 +154,18 @@ func TestExecutorKeepsPreparedStateForConflictingContainerInspection(t *testing.
executor := testExecutor(t, store, coordinator, engine, healthyResponse)
record := createTransaction(t, store, "container-conflict")
err := executor.Run(ctx, record.ID, request)
var uncertain *transaction.UncertainStepError
if !errors.As(err, &uncertain) {
t.Fatalf("expected uncertain container step, got %v", err)
if err := executor.Run(ctx, record.ID, request); err != nil {
t.Fatalf("replace inactive container: %v", err)
}
current, readErr := store.Transaction(ctx, record.ID)
if readErr != nil || current.State != transaction.StatePrepared {
t.Fatalf("conflict did not preserve prepared state: record=%+v err=%v", current, readErr)
if readErr != nil || current.State != transaction.StateSwitching {
t.Fatalf("replacement did not reach switching: record=%+v err=%v", current, readErr)
}
engine.mu.Lock()
calls := engine.callCounts()
engine.mu.Unlock()
if calls.remove != 1 || calls.create != 1 {
t.Fatalf("inactive replacement calls mismatch: %+v", calls)
}
}
@@ -222,17 +233,22 @@ func testRequest(t *testing.T, imageReference string) Request {
t.Fatalf("write backend configuration: %v", err)
}
return Request{
ArchivePath: archivePath,
ImageReference: imageReference,
ExpectedImageDigest: testDigest,
Platform: containerengine.Platform{OS: "linux", Architecture: "amd64"},
ContainerName: "explicit-backend-8081",
Port: 8081,
PortEnvironmentKey: "SERVER_PORT",
ConfigSource: configPath,
ConfigTarget: "/app/config/application.yaml",
RestartPolicy: containerengine.RestartPolicy{Name: "unless-stopped"},
HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health",
ImageAcquisition: ImageAcquisitionLoad,
ArchivePath: archivePath,
ImageReference: imageReference,
ExpectedImageDigest: testDigest,
Platform: containerengine.Platform{OS: "linux", Architecture: "amd64"},
ContainerName: "explicit-backend-8081",
Port: 8081,
PortEnvironmentKey: "SERVER_PORT",
ConfigSource: configPath,
ConfigTarget: "/app/config/yms.yaml",
TmpSource: directory,
TmpTarget: "/home/yms/tmp",
ConfigEnvironmentKey: "SPRING_CONFIG_LOCATION",
ConfigLocation: "file:/app/config/yms.yaml",
RestartPolicy: containerengine.RestartPolicy{Name: "unless-stopped"},
HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health",
}
}
@@ -347,6 +363,14 @@ func (e *fakeEngine) LoadImage(_ context.Context, input io.Reader) error {
return nil
}
func (e *fakeEngine) PullImage(context.Context, string) error {
e.mu.Lock()
defer e.mu.Unlock()
e.loadCalls++
e.imageAvailable = true
return nil
}
func (e *fakeEngine) InspectImage(context.Context, string) (containerengine.Image, error) {
e.mu.Lock()
defer e.mu.Unlock()
@@ -384,6 +408,19 @@ func (e *fakeEngine) StartContainer(_ context.Context, name string) error {
return nil
}
func (e *fakeEngine) StopContainer(_ context.Context, name string) error {
e.mu.Lock()
defer e.mu.Unlock()
record, exists := e.containers[name]
if !exists {
return containerengine.ErrNotFound
}
record.Running = false
record.Status = "exited"
e.containers[name] = record
return nil
}
func (e *fakeEngine) InspectContainer(_ context.Context, name string) (containerengine.Container, error) {
e.mu.Lock()
defer e.mu.Unlock()
@@ -409,17 +446,19 @@ func (e *fakeEngine) Close() error { return nil }
func (e *fakeEngine) containerFromSpec(spec containerengine.ContainerSpec, running bool) containerengine.Container {
return containerengine.Container{
ID: "container-id-" + spec.Name,
Name: spec.Name,
ImageID: e.loadedImage.ID,
ImageReference: spec.ImageReference,
Platform: spec.Platform.OS + "/" + spec.Platform.Architecture,
Running: running,
Status: "created",
Environment: append([]string(nil), spec.Environment...),
NetworkMode: spec.NetworkMode,
RestartPolicy: spec.RestartPolicy,
Mounts: append([]containerengine.Mount(nil), spec.Mounts...),
ID: "container-id-" + spec.Name,
Name: spec.Name,
ImageID: e.loadedImage.ID,
ImageReference: spec.ImageReference,
Platform: spec.Platform.OS + "/" + spec.Platform.Architecture,
Running: running,
Status: "created",
Environment: append([]string(nil), spec.Environment...),
NetworkMode: spec.NetworkMode,
RestartPolicy: spec.RestartPolicy,
Mounts: append([]containerengine.Mount(nil), spec.Mounts...),
User: spec.User,
StopTimeoutSeconds: spec.StopTimeoutSeconds,
}
}
+57 -1
View File
@@ -23,6 +23,36 @@ type loadImageOperation struct {
platform containerengine.Platform
}
type pullImageOperation struct {
engine containerengine.Engine
imageReference string
expectedDigest string
platform containerengine.Platform
}
func (o *pullImageOperation) Apply(ctx context.Context) error {
return o.engine.PullImage(ctx, o.imageReference)
}
func (o *pullImageOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
image, err := o.engine.InspectImage(ctx, o.imageReference)
if errors.Is(err, containerengine.ErrNotFound) {
return transaction.Inspection{Status: transaction.InspectionNotApplied}, nil
}
if err != nil {
return transaction.Inspection{}, err
}
matches, err := imageMatches(image, o.expectedDigest, o.platform)
if err != nil {
return transaction.Inspection{}, err
}
result := resultJSON(image)
if !matches {
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
}
return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
}
func (o *loadImageOperation) Apply(ctx context.Context) error {
archive, err := os.Open(o.archivePath)
if err != nil {
@@ -62,6 +92,30 @@ type createContainerOperation struct {
spec containerengine.ContainerSpec
}
type removeContainerOperation struct {
engine containerengine.Engine
name string
}
func (o *removeContainerOperation) Apply(ctx context.Context) error {
err := o.engine.RemoveContainer(ctx, o.name, true)
if errors.Is(err, containerengine.ErrNotFound) {
return nil
}
return err
}
func (o *removeContainerOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
record, err := o.engine.InspectContainer(ctx, o.name)
if errors.Is(err, containerengine.ErrNotFound) {
return transaction.Inspection{Status: transaction.InspectionApplied}, nil
}
if err != nil {
return transaction.Inspection{}, err
}
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: containerResult(record)}, nil
}
func (o *createContainerOperation) Apply(ctx context.Context) error {
_, err := o.engine.CreateContainer(ctx, o.spec)
return err
@@ -168,7 +222,7 @@ func healthInspection(report healthcheck.ActuatorReport, ready bool, err error)
}
func containerMatches(record containerengine.Container, expectedImageID string, spec containerengine.ContainerSpec) bool {
if record.ImageID != expectedImageID || record.NetworkMode != spec.NetworkMode || record.RestartPolicy != spec.RestartPolicy {
if record.ImageID != expectedImageID || record.NetworkMode != spec.NetworkMode || record.RestartPolicy != spec.RestartPolicy || record.User != spec.User || record.StopTimeoutSeconds != spec.StopTimeoutSeconds {
return false
}
for _, expected := range spec.Environment {
@@ -195,6 +249,8 @@ func containerResult(record containerengine.Container) json.RawMessage {
}
var _ transaction.Operation = (*loadImageOperation)(nil)
var _ transaction.Operation = (*pullImageOperation)(nil)
var _ transaction.Operation = (*removeContainerOperation)(nil)
var _ transaction.Operation = (*createContainerOperation)(nil)
var _ transaction.Operation = (*startContainerOperation)(nil)
var _ transaction.Operation = (*healthOperation)(nil)