feat: backend executor implement

This commit is contained in:
2026-08-16 17:12:06 +08:00
parent 390e0565d3
commit 7755da72e7
31 changed files with 2762 additions and 184 deletions
+479
View File
@@ -0,0 +1,479 @@
package backendupdate
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/distribution/reference"
"yms-daemon/internal/backendexecutor"
"yms-daemon/internal/containerengine"
"yms-daemon/internal/deploymentconfig"
"yms-daemon/internal/hostnginx"
"yms-daemon/internal/transaction"
)
const inputTypeContainerImage = "container-image"
type persistedContainerRequest struct {
InputType string `json:"inputType"`
ImageReference string `json:"imageReference"`
ImmutableReference string `json:"immutableReference"`
ImageDigest string `json:"imageDigest"`
Platform containerengine.Platform `json:"platform"`
TargetPort int `json:"targetPort"`
TargetContainer string `json:"targetContainer"`
PreviousPort int `json:"previousPort"`
PreviousContainer string `json:"previousContainer"`
TargetHealthEndpoint string `json:"targetHealthEndpoint"`
GatewayBeforePath string `json:"gatewayBeforePath"`
GatewayAfterPath string `json:"gatewayAfterPath"`
GatewayReceiptPath string `json:"gatewayReceiptPath"`
}
// UpdateContainerImage pulls one development image, freezes its repository
// digest, and updates the inactive Docker backend slot.
func (u *Updater) UpdateContainerImage(ctx context.Context, imageReference string, report ProgressReporter) (transaction.Transaction, error) {
if u.containerExecutor == nil || u.engine == nil {
return transaction.Transaction{}, errors.New("container backend updater is not configured")
}
if strings.TrimSpace(imageReference) != imageReference || imageReference == "" {
return transaction.Transaction{}, errors.New("exact container image reference is required")
}
configInfo, err := os.Lstat(u.containerConfigSource)
if err != nil {
return transaction.Transaction{}, fmt.Errorf("inspect backend configuration %s: %w", u.containerConfigSource, err)
}
if !configInfo.Mode().IsRegular() || configInfo.Mode()&os.ModeSymlink != 0 {
return transaction.Transaction{}, fmt.Errorf("backend configuration is not a direct regular file: %s", u.containerConfigSource)
}
tmpInfo, err := os.Lstat(u.containerTmpSource)
if err != nil {
return transaction.Transaction{}, fmt.Errorf("inspect backend temporary directory %s: %w", u.containerTmpSource, err)
}
if !tmpInfo.IsDir() || tmpInfo.Mode()&os.ModeSymlink != 0 {
return transaction.Transaction{}, fmt.Errorf("backend temporary path is not a direct directory: %s", u.containerTmpSource)
}
active, err := u.store.ActiveTransaction(ctx)
if err == nil {
var request persistedContainerRequest
if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil {
return active, decodeErr
}
if active.Service != serviceBackend || request.InputType != inputTypeContainerImage {
return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
}
if request.ImageReference != imageReference {
return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
}
return u.runContainerUpdate(ctx, active, request, false, report)
}
if !errors.Is(err, transaction.ErrNotFound) {
return transaction.Transaction{}, err
}
reportProgress(report, Progress{Message: "Pulling backend image " + imageReference})
resolved, err := u.pullAndResolveImage(ctx, imageReference)
if err != nil {
return transaction.Transaction{}, err
}
reportProgress(report, Progress{Message: "Backend image resolved: " + resolved.ImmutableReference})
before, err := u.gateway.Read()
if err != nil {
return transaction.Transaction{}, err
}
activeSlot, err := u.config.Backend.SlotForPort(before.ActivePort)
if err != nil {
return transaction.Transaction{}, err
}
deployment, deploymentErr := u.store.BackendContainerDeployment(ctx)
hasDeployment := deploymentErr == nil
if deploymentErr != nil && !errors.Is(deploymentErr, transaction.ErrNotFound) {
return transaction.Transaction{}, deploymentErr
}
hasHistory, err := u.store.HasCommittedBackendContainerTransactionHistory(ctx)
if err != nil {
return transaction.Transaction{}, err
}
targetPort, targetSlot, previousContainer, err := u.resolveContainerSlots(
ctx,
before.ActivePort,
activeSlot,
deployment,
hasDeployment,
hasHistory,
)
if err != nil {
return transaction.Transaction{}, err
}
afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort)
if err != nil {
return transaction.Transaction{}, err
}
transactionID := rand.Text()
transactionRoot := filepath.Join(u.workRoot, transactionID)
if err := os.MkdirAll(transactionRoot, 0o750); err != nil {
return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err)
}
beforePath := filepath.Join(transactionRoot, "gateway.before.conf")
afterPath := filepath.Join(transactionRoot, "gateway.after.conf")
if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil {
return transaction.Transaction{}, err
}
if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil {
return transaction.Transaction{}, err
}
request := persistedContainerRequest{
InputType: inputTypeContainerImage, ImageReference: imageReference,
ImmutableReference: resolved.ImmutableReference, ImageDigest: resolved.Digest, Platform: resolved.Platform,
TargetPort: targetPort, TargetContainer: targetSlot.ContainerName,
PreviousPort: before.ActivePort, PreviousContainer: previousContainer,
TargetHealthEndpoint: targetSlot.HealthEndpoint,
GatewayBeforePath: beforePath, GatewayAfterPath: afterPath,
GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"),
}
content, err := json.Marshal(request)
if err != nil {
return transaction.Transaction{}, fmt.Errorf("encode container backend update request: %w", err)
}
record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{
ID: transactionID, IdempotencyKey: serviceBackend + ":container:" + resolved.Digest,
Source: sourceLocalCLI, Service: serviceBackend, Request: content,
})
if err != nil {
return transaction.Transaction{}, err
}
if !created {
if err := decodeContainerRequest(record.Request, &request); err != nil {
return record, err
}
}
return u.runContainerUpdate(ctx, record, request, created, report)
}
// resolveContainerSlots reconciles the committed deployment, gateway and both
// exact container names before selecting a target. A fresh installation has no
// deployment row, no committed container transaction history and no slot
// containers. It starts on the non-routed slot so traffic is exposed only after
// health passes.
func (u *Updater) resolveContainerSlots(
ctx context.Context,
activePort int,
activeSlot deploymentconfig.BackendSlot,
deployment transaction.BackendContainerDeployment,
hasDeployment bool,
hasHistory bool,
) (int, deploymentconfig.BackendSlot, string, error) {
inactivePort := otherPort(activePort)
inactiveSlot, err := u.config.Backend.SlotForPort(inactivePort)
if err != nil {
return 0, deploymentconfig.BackendSlot{}, "", err
}
active, activeErr := u.engine.InspectContainer(ctx, activeSlot.ContainerName)
if activeErr != nil && !errors.Is(activeErr, containerengine.ErrNotFound) {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect active backend container %s: %w", activeSlot.ContainerName, activeErr)
}
inactive, inactiveErr := u.engine.InspectContainer(ctx, inactiveSlot.ContainerName)
if inactiveErr != nil && !errors.Is(inactiveErr, containerengine.ErrNotFound) {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect inactive backend container %s: %w", inactiveSlot.ContainerName, inactiveErr)
}
activeFound := activeErr == nil
inactiveFound := inactiveErr == nil
if hasDeployment {
if deployment.ActivePort != activePort {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container port %d does not match gateway active port %d", deployment.ActivePort, activePort)
}
if deployment.ContainerName != activeSlot.ContainerName {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container %s does not match gateway slot container %s", deployment.ContainerName, activeSlot.ContainerName)
}
if !activeFound {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed active backend container %s is missing", activeSlot.ContainerName)
}
if active.ID != deployment.ContainerID {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s identity does not match committed deployment", activeSlot.ContainerName)
}
}
if activeFound {
if !active.Running || active.Dead {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is not running", activeSlot.ContainerName)
}
if inactiveFound && inactive.Running && !inactive.Dead {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inactive backend container %s is unexpectedly running", inactiveSlot.ContainerName)
}
return inactivePort, inactiveSlot, activeSlot.ContainerName, nil
}
if hasHistory {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing on a server with committed backend container transaction history", activeSlot.ContainerName)
}
if inactiveFound && inactive.Running && !inactive.Dead {
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing but inactive container %s is running", activeSlot.ContainerName, inactiveSlot.ContainerName)
}
return inactivePort, inactiveSlot, "", nil
}
type resolvedImage struct {
ImmutableReference string
Digest string
Platform containerengine.Platform
}
func (u *Updater) pullAndResolveImage(ctx context.Context, imageReference string) (resolvedImage, error) {
named, err := reference.ParseNormalizedNamed(imageReference)
if err != nil {
return resolvedImage{}, fmt.Errorf("parse container image reference: %w", err)
}
if _, ok := named.(reference.Tagged); !ok {
return resolvedImage{}, errors.New("--container-image requires a tag-qualified image reference")
}
if err := u.engine.Ping(ctx); err != nil {
return resolvedImage{}, err
}
if err := u.engine.PullImage(ctx, imageReference); err != nil {
return resolvedImage{}, err
}
image, err := u.engine.InspectImage(ctx, imageReference)
if err != nil {
return resolvedImage{}, fmt.Errorf("inspect pulled backend image: %w", err)
}
if image.Platform.OS == "" || image.Platform.Architecture == "" {
return resolvedImage{}, errors.New("pulled backend image does not report an exact platform")
}
repository := reference.TrimNamed(named).Name()
matches := make(map[string]string)
for _, value := range image.RepoDigests {
digested, err := reference.ParseNormalizedNamed(value)
if err != nil {
return resolvedImage{}, fmt.Errorf("parse pulled repository digest %q: %w", value, err)
}
withDigest, ok := digested.(reference.Digested)
if !ok || reference.TrimNamed(digested).Name() != repository {
continue
}
matches[withDigest.Digest().String()] = value
}
if len(matches) == 0 {
return resolvedImage{}, fmt.Errorf("pulled backend image has no repository digest for %s", repository)
}
if len(matches) != 1 {
return resolvedImage{}, fmt.Errorf("pulled backend image has multiple repository digests for %s", repository)
}
for digest, immutableReference := range matches {
return resolvedImage{ImmutableReference: immutableReference, Digest: digest, Platform: image.Platform}, nil
}
return resolvedImage{}, errors.New("repository digest resolution produced no result")
}
func (u *Updater) runContainerUpdate(ctx context.Context, record transaction.Transaction, request persistedContainerRequest, created bool, report ProgressReporter) (transaction.Transaction, error) {
message := "Resuming container backend transaction " + record.ID
if created {
message = "Created container backend transaction " + record.ID
}
reportProgress(report, Progress{TransactionID: record.ID, State: record.State, Message: message})
if record.State.Terminal() {
return terminalResult(record)
}
executorRequest := backendexecutor.Request{
ImageAcquisition: backendexecutor.ImageAcquisitionPull,
ImageReference: request.ImmutableReference, ExpectedImageDigest: request.ImageDigest, Platform: request.Platform,
ContainerName: request.TargetContainer, Port: request.TargetPort,
PortEnvironmentKey: deploymentconfig.ContainerPortEnvironment,
ConfigSource: u.containerConfigSource, ConfigTarget: u.containerConfigTarget,
TmpSource: u.containerTmpSource, TmpTarget: u.containerTmpTarget,
ConfigEnvironmentKey: deploymentconfig.ContainerConfigEnvironment,
ConfigLocation: deploymentconfig.ContainerConfigLocation,
RestartPolicy: containerengine.RestartPolicy{Name: "no"},
HealthEndpoint: request.TargetHealthEndpoint,
}
switch record.State {
case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting:
if err := u.containerExecutor.Run(ctx, record.ID, executorRequest); err != nil {
current, readErr := u.store.Transaction(ctx, record.ID)
if readErr != nil || current.State == transaction.StateFailed {
return u.currentWithError(ctx, record.ID, errors.Join(err, readErr))
}
before, beforeErr := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
after, afterErr := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
if snapshotErr := errors.Join(beforeErr, afterErr); snapshotErr != nil {
return u.currentWithError(ctx, record.ID, errors.Join(err, snapshotErr))
}
rollbackErr := u.rollbackContainer(ctx, record.ID, request, before, after, err)
return u.currentWithError(ctx, record.ID, rollbackErr)
}
case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, transaction.StateRollingBack:
default:
return u.currentWithError(ctx, record.ID, fmt.Errorf("container backend update cannot resume transaction %s in state %s", record.ID, record.State))
}
if err := u.switchAndCommitContainer(ctx, record.ID, request, report); err != nil {
return u.currentWithError(ctx, record.ID, err)
}
return u.store.Transaction(ctx, record.ID)
}
func (u *Updater) switchAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error {
before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
if err != nil {
return err
}
after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
if err != nil {
return err
}
for {
record, err := u.store.Transaction(ctx, transactionID)
if err != nil {
return err
}
switch record.State {
case transaction.StateSwitching:
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)})
operation := &gatewayOperation{controller: u.gateway, before: before, after: after, receiptPath: request.GatewayReceiptPath}
if _, err := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewaySwitchIntent(request), operation); err != nil {
return u.rollbackContainer(ctx, transactionID, request, before, after, err)
}
if _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy container slot"); err != nil {
return err
}
case transaction.StateVerifying:
message := "container backend switch verified"
if request.PreviousContainer != "" {
message += "; previous container draining"
}
if _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, message); err != nil {
return err
}
case transaction.StateDraining:
if request.PreviousContainer != "" {
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Draining previous backend container for %s", u.drain)})
if err := waitContext(ctx, u.drain); err != nil {
return err
}
operation := &containerStopOperation{engine: u.engine, name: request.PreviousContainer}
if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousContainerIntent(request), operation); err != nil {
return err
}
}
target, err := u.engine.InspectContainer(ctx, request.TargetContainer)
if err != nil {
return fmt.Errorf("inspect target backend container before commit: %w", err)
}
if !target.Running || target.Dead || target.ID == "" {
return fmt.Errorf("target backend container %s is not running with an exact identity", request.TargetContainer)
}
if target.ImageReference != request.ImmutableReference {
return fmt.Errorf("target backend container %s image reference does not match the transaction", request.TargetContainer)
}
_, err = u.store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{
ActivePort: request.TargetPort,
ContainerName: request.TargetContainer,
ImageDigest: request.ImageDigest,
ContainerID: target.ID,
}, "container backend update committed")
if err == nil {
reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Container backend update committed"})
}
return err
case transaction.StateCommitted:
return nil
case transaction.StateRollingBack:
return u.rollbackContainer(ctx, transactionID, request, before, after, errors.New("resuming container backend rollback"))
default:
return fmt.Errorf("container backend commit cannot continue transaction %s in state %s", transactionID, record.State)
}
}
}
func (u *Updater) rollbackContainer(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error {
record, err := u.store.Transaction(ctx, transactionID)
if err != nil {
return errors.Join(cause, err)
}
if record.State != transaction.StateRollingBack {
if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "container backend compensation started"); err != nil {
return errors.Join(cause, err)
}
}
gateway := &gatewayOperation{controller: u.gateway, before: after, after: before, receiptPath: request.GatewayReceiptPath + ".restore"}
_, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewayRestoreIntent(request), gateway)
stop := &containerStopOperation{engine: u.engine, name: request.TargetContainer}
_, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetContainerIntent(request), stop)
if err := errors.Join(gatewayErr, stopErr); err != nil {
return errors.Join(cause, err)
}
_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "container backend compensation completed")
return errors.Join(cause, transitionErr)
}
type containerStopOperation struct {
engine containerengine.Engine
name string
}
func (o *containerStopOperation) Apply(ctx context.Context) error {
err := o.engine.StopContainer(ctx, o.name)
if errors.Is(err, containerengine.ErrNotFound) {
return nil
}
return err
}
func (o *containerStopOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
record, err := o.engine.InspectContainer(ctx, o.name)
if errors.Is(err, containerengine.ErrNotFound) {
return transaction.Inspection{Status: transaction.InspectionApplied}, nil
}
if err != nil {
return transaction.Inspection{}, err
}
result, _ := json.Marshal(record)
if !record.Running || record.Dead {
return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
}
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
}
func containerGatewaySwitchIntent(request persistedContainerRequest) transaction.StepIntent {
return stepIntent("backend.container.gateway.switch", "switch host Nginx to container backend", struct {
BeforePort int `json:"beforePort"`
AfterPort int `json:"afterPort"`
}{request.PreviousPort, request.TargetPort})
}
func containerGatewayRestoreIntent(request persistedContainerRequest) transaction.StepIntent {
return stepIntent("backend.container.gateway.restore", "restore host Nginx after container backend failure", struct {
Port int `json:"port"`
}{request.PreviousPort})
}
func stopPreviousContainerIntent(request persistedContainerRequest) transaction.StepIntent {
return stepIntent("backend.previous-container.stop", "stop previous backend container after drain", struct {
ContainerName string `json:"containerName"`
}{request.PreviousContainer})
}
func stopTargetContainerIntent(request persistedContainerRequest) transaction.StepIntent {
return stepIntent("backend.target-container.stop", "stop compensated backend container", struct {
ContainerName string `json:"containerName"`
}{request.TargetContainer})
}
func decodeContainerRequest(content json.RawMessage, request *persistedContainerRequest) error {
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields()
if err := decoder.Decode(request); err != nil {
return fmt.Errorf("decode persisted container backend request: %w", err)
}
return nil
}
var _ transaction.Operation = (*containerStopOperation)(nil)
+366
View File
@@ -0,0 +1,366 @@
package backendupdate
import (
"bytes"
"context"
"errors"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"yms-daemon/internal/backendexecutor"
"yms-daemon/internal/containerengine"
"yms-daemon/internal/deploymentconfig"
"yms-daemon/internal/hostnginx"
"yms-daemon/internal/transaction"
)
const containerTestImage = "harbor.ymswell.asia/ymswell/glory-ymswell:20260813-184902-a37bf50d-v1.1.8.1"
const containerTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
func TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot(t *testing.T) {
ctx := context.Background()
updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{
"backend-8080": {
ID: "existing-backend-8080", Name: "backend-8080", Running: true,
},
}, 0)
record, err := updater.UpdateContainerImage(ctx, containerTestImage, nil)
if err != nil {
t.Fatalf("update container backend: %v", err)
}
if record.State != transaction.StateCommitted || gateway.snapshot.ActivePort != 8081 {
t.Fatalf("unexpected committed container update: record=%+v gateway=%+v", record, gateway.snapshot)
}
if engine.containers["backend-8080"].Running {
t.Fatal("previous backend container is still running")
}
if len(engine.stopped) != 1 || engine.stopped[0] != "backend-8080" {
t.Fatalf("unexpected stopped containers: %+v", engine.stopped)
}
assertCreatedContainerSpec(t, engine.lastCreateSpec)
assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
}
func TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, time.Hour)
var progress []Progress
gateway.beforeApply = func(snapshot hostnginx.Snapshot) error {
target, found := engine.containers["backend-8081"]
if !found || !target.Running || engine.healthChecks == 0 {
return errors.New("gateway switch occurred before the target container passed health checking")
}
return nil
}
record, err := updater.UpdateContainerImage(ctx, containerTestImage, func(item Progress) {
progress = append(progress, item)
})
if err != nil {
t.Fatalf("first install container backend: %v", err)
}
if record.State != transaction.StateCommitted {
t.Fatalf("unexpected committed first install: record=%+v", record)
}
if gateway.snapshot.ActivePort != 8081 || gateway.applyCount != 1 {
t.Fatalf("first install did not switch once to the healthy inactive slot: gateway=%+v applies=%d", gateway.snapshot, gateway.applyCount)
}
target, found := engine.containers["backend-8081"]
if !found || !target.Running || target.Dead {
t.Fatalf("first install target container is not running: %+v", target)
}
if len(engine.stopped) != 0 {
t.Fatalf("first install must not stop a previous container: %+v", engine.stopped)
}
if engine.healthChecks != 1 {
t.Fatalf("unexpected first-install health check count: %d", engine.healthChecks)
}
for _, item := range progress {
if strings.HasPrefix(item.Message, "Draining previous backend container") {
t.Fatalf("first install entered previous-container drain: %+v", progress)
}
}
assertCreatedContainerSpec(t, engine.lastCreateSpec)
assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
}
func TestContainerUpdaterRejectsMissingActiveWithPresentInactive(t *testing.T) {
updater, _, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{
"backend-8081": {
ID: "unexpected-backend-8081", Name: "backend-8081", Running: true,
},
}, 0)
_, err := updater.UpdateContainerImage(context.Background(), containerTestImage, nil)
if err == nil || !strings.Contains(err.Error(), "active backend container backend-8080 is missing but inactive container backend-8081 is running") {
t.Fatalf("unexpected missing-active result: %v", err)
}
}
func TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage(t *testing.T) {
ctx := context.Background()
updater, _, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
failGateway := true
gateway.beforeApply = func(hostnginx.Snapshot) error {
if failGateway {
return errors.New("host Nginx is unavailable")
}
return nil
}
rollingBack, err := updater.UpdateContainerImage(ctx, containerTestImage, nil)
if err == nil || rollingBack.State != transaction.StateRollingBack {
t.Fatalf("unexpected failed rollback result: record=%+v err=%v", rollingBack, err)
}
target, found := engine.containers["backend-8081"]
if !found || target.Running {
t.Fatalf("failed update target was not stopped: %+v", target)
}
failGateway = false
rolledBack, err := updater.UpdateContainerImage(ctx, containerTestImage, nil)
if err == nil || rolledBack.State != transaction.StateRolledBack {
t.Fatalf("unexpected resumed rollback result: record=%+v err=%v", rolledBack, err)
}
committed, err := updater.UpdateContainerImage(ctx, containerTestImage, nil)
if err != nil || committed.State != transaction.StateCommitted {
t.Fatalf("retry same image after rollback: record=%+v err=%v", committed, err)
}
if gateway.snapshot.ActivePort != 8081 {
t.Fatalf("retry did not switch to the healthy container: %+v", gateway.snapshot)
}
}
func TestContainerUpdaterRejectsMissingCommittedContainer(t *testing.T) {
ctx := context.Background()
updater, store, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
transactionID := "committed-container-transaction"
_, _, err := store.CreateTransaction(ctx, transaction.CreateRequest{
ID: transactionID, IdempotencyKey: "backend:container:previous", Source: sourceLocalCLI,
Service: serviceBackend, Request: []byte(`{"inputType":"container-image"}`),
})
if err != nil {
t.Fatalf("create previous backend transaction: %v", err)
}
for _, state := range []transaction.State{
transaction.StateValidating,
transaction.StatePrepared,
transaction.StateStarting,
transaction.StateSwitching,
transaction.StateVerifying,
transaction.StateDraining,
} {
if _, err := store.Transition(ctx, transactionID, state, "seed committed deployment"); err != nil {
t.Fatalf("transition previous backend transaction to %s: %v", state, err)
}
}
if _, err := store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{
ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "missing-backend-8080",
}, "seed committed backend container deployment"); err != nil {
t.Fatalf("commit previous backend deployment: %v", err)
}
_, err = updater.UpdateContainerImage(ctx, containerTestImage, nil)
if err == nil || !strings.Contains(err.Error(), "committed active backend container backend-8080 is missing") {
t.Fatalf("unexpected committed-container drift result: %v", err)
}
}
func newContainerUpdaterFixture(
t *testing.T,
containers map[string]containerengine.Container,
drain time.Duration,
) (*Updater, *transaction.Store, *containerUpdateEngine, *memoryGateway) {
t.Helper()
ctx := context.Background()
root := t.TempDir()
store, err := transaction.OpenStore(ctx, filepath.Join(root, "transactions.db"))
if err != nil {
t.Fatalf("open transaction store: %v", err)
}
t.Cleanup(func() {
if err := store.Close(); err != nil {
t.Errorf("close transaction store: %v", err)
}
})
coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("create transaction coordinator: %v", err)
}
engine := &containerUpdateEngine{
image: containerengine.Image{
ID: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
RepoDigests: []string{"harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest},
Platform: containerengine.Platform{OS: "linux", Architecture: "amd64"},
},
containers: containers,
}
configSource := filepath.Join(root, "yms.yaml")
if err := os.WriteFile(configSource, []byte("server: {}\n"), 0o600); err != nil {
t.Fatalf("write backend configuration: %v", err)
}
tmpSource := filepath.Join(root, "tmp")
if err := os.Mkdir(tmpSource, 0o755); err != nil {
t.Fatalf("create backend temporary directory: %v", err)
}
client := &http.Client{Transport: containerUpdateRoundTripFunc(func(request *http.Request) (*http.Response, error) {
engine.healthChecks++
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(bytes.NewBufferString(`{"status":"UP","components":{"ping":{"status":"UP"}}}`)),
Request: request,
}, nil
})}
executor, err := backendexecutor.New(store, coordinator, engine, client)
if err != nil {
t.Fatalf("create backend container executor: %v", err)
}
gateway := &memoryGateway{snapshot: hostnginx.Snapshot{Content: []byte(serverConfiguration8080), ActivePort: 8080}}
updater := &Updater{
config: deploymentconfig.Config{
Daemon: deploymentconfig.Daemon{Environment: deploymentconfig.EnvironmentDev},
Backend: deploymentconfig.Backend{Type: deploymentconfig.BackendTypeContainer, Slot: deploymentconfig.BackendSlots{
Port8080: deploymentconfig.BackendSlot{ContainerName: "backend-8080", HealthEndpoint: "http://127.0.0.1:8080/yms/actuator/health"},
Port8081: deploymentconfig.BackendSlot{ContainerName: "backend-8081", HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health"},
}},
},
workRoot: filepath.Join(root, "work"), store: store, coordinator: coordinator,
gateway: gateway, containerExecutor: executor, engine: engine, drain: drain,
containerConfigSource: configSource, containerConfigTarget: deploymentconfig.ContainerConfigTarget,
containerTmpSource: tmpSource, containerTmpTarget: deploymentconfig.ContainerTmpTarget,
}
return updater, store, engine, gateway
}
func assertCreatedContainerSpec(t *testing.T, request containerengine.ContainerSpec) {
t.Helper()
if request.Name != "backend-8081" || request.ImageReference != "harbor.ymswell.asia/ymswell/glory-ymswell@"+containerTestDigest {
t.Fatalf("unexpected target container identity: %+v", request)
}
if request.NetworkMode != "host" || request.RestartPolicy.Name != "no" || request.User != "0:0" {
t.Fatalf("unexpected target container runtime contract: %+v", request)
}
if len(request.Environment) != 2 || request.Environment[0] != "SERVER_PORT=8081" || request.Environment[1] != "SPRING_CONFIG_LOCATION=file:/app/config/yms.yaml" {
t.Fatalf("unexpected target container environment: %+v", request.Environment)
}
if len(request.Mounts) != 2 || request.Mounts[0].Target != deploymentconfig.ContainerConfigTarget || !request.Mounts[0].ReadOnly || request.Mounts[1].Target != deploymentconfig.ContainerTmpTarget {
t.Fatalf("unexpected target container mounts: %+v", request.Mounts)
}
}
func assertCommittedContainerDeployment(
t *testing.T,
store *transaction.Store,
transactionID string,
port int,
containerName string,
containerID string,
) {
t.Helper()
deployment, err := store.BackendContainerDeployment(context.Background())
if err != nil {
t.Fatalf("read committed backend container deployment: %v", err)
}
if deployment.ActivePort != port || deployment.ContainerName != containerName || deployment.ContainerID != containerID || deployment.ImageDigest != containerTestDigest || deployment.TransactionID != transactionID {
t.Fatalf("unexpected committed backend container deployment: %+v", deployment)
}
}
const serverConfiguration8080 = `http {
upstream yms-server {
# yms-update managed upstream begin
server 10.11.1.117:8080 max_fails=1 fail_timeout=2s;
# server 10.11.1.117:8081 max_fails=1 fail_timeout=2s;
# yms-update managed upstream end
}
}
`
type containerUpdateRoundTripFunc func(*http.Request) (*http.Response, error)
func (function containerUpdateRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return function(request)
}
type containerUpdateEngine struct {
image containerengine.Image
containers map[string]containerengine.Container
stopped []string
lastCreateSpec containerengine.ContainerSpec
healthChecks int
}
func (e *containerUpdateEngine) Ping(context.Context) error { return nil }
func (e *containerUpdateEngine) PullImage(context.Context, string) error { return nil }
func (e *containerUpdateEngine) LoadImage(context.Context, io.Reader) error { return nil }
func (e *containerUpdateEngine) InspectImage(context.Context, string) (containerengine.Image, error) {
return e.image, nil
}
func (e *containerUpdateEngine) CreateContainer(_ context.Context, spec containerengine.ContainerSpec) (containerengine.Container, error) {
e.lastCreateSpec = spec
record := containerengine.Container{
ID: "container-id-" + spec.Name,
Name: spec.Name,
ImageID: e.image.ID,
ImageReference: spec.ImageReference,
Platform: spec.Platform.OS + "/" + spec.Platform.Architecture,
Status: "created",
Environment: append([]string(nil), spec.Environment...),
Labels: spec.Labels,
NetworkMode: spec.NetworkMode,
RestartPolicy: spec.RestartPolicy,
Mounts: append([]containerengine.Mount(nil), spec.Mounts...),
User: spec.User,
StopTimeoutSeconds: spec.StopTimeoutSeconds,
}
e.containers[spec.Name] = record
return record, nil
}
func (e *containerUpdateEngine) StartContainer(_ context.Context, name string) error {
record, found := e.containers[name]
if !found {
return containerengine.ErrNotFound
}
record.Running = true
record.Status = "running"
e.containers[name] = record
return nil
}
func (e *containerUpdateEngine) StopContainer(_ context.Context, name string) error {
e.stopped = append(e.stopped, name)
record, found := e.containers[name]
if !found {
return containerengine.ErrNotFound
}
record.Running = false
record.Status = "exited"
e.containers[name] = record
return nil
}
func (e *containerUpdateEngine) InspectContainer(_ context.Context, name string) (containerengine.Container, error) {
record, found := e.containers[name]
if !found {
return containerengine.Container{}, containerengine.ErrNotFound
}
return record, nil
}
func (e *containerUpdateEngine) RemoveContainer(_ context.Context, name string, _ bool) error {
if _, found := e.containers[name]; !found {
return containerengine.ErrNotFound
}
delete(e.containers, name)
return nil
}
func (e *containerUpdateEngine) Close() error { return nil }
var _ containerengine.Engine = (*containerUpdateEngine)(nil)
+4
View File
@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"yms-daemon/internal/deploymentconfig"
"yms-daemon/internal/transaction"
"yms-daemon/internal/updatepackage"
)
@@ -15,6 +16,9 @@ import (
// Restart performs a zero-downtime rotation with the exact release currently
// exposed by the compatibility JAR path.
func (u *Updater) Restart(ctx context.Context, report ProgressReporter) (transaction.Transaction, error) {
if u.config.Backend.Type == deploymentconfig.BackendTypeContainer {
return transaction.Transaction{}, errors.New("restart is not implemented for container backend")
}
reportProgress(report, Progress{Message: "Resolving the current native backend release"})
active, err := u.store.ActiveTransaction(ctx)
if err == nil {
+74 -10
View File
@@ -14,6 +14,8 @@ import (
"path/filepath"
"time"
"yms-daemon/internal/backendexecutor"
"yms-daemon/internal/containerengine"
"yms-daemon/internal/daemonapi"
"yms-daemon/internal/deploymentconfig"
"yms-daemon/internal/filestore"
@@ -36,16 +38,22 @@ const (
// Updater executes the current native backend contract on one server.
type Updater struct {
config deploymentconfig.Config
workRoot string
store *transaction.Store
coordinator *transaction.Coordinator
releaseStore *filestore.Store
units systemd.Manager
gateway gatewayController
executor nativeExecutor
logger *slog.Logger
drain time.Duration
config deploymentconfig.Config
workRoot string
store *transaction.Store
coordinator *transaction.Coordinator
releaseStore *filestore.Store
units systemd.Manager
gateway gatewayController
executor nativeExecutor
containerExecutor containerExecutor
engine containerengine.Engine
containerConfigSource string
containerConfigTarget string
containerTmpSource string
containerTmpTarget string
logger *slog.Logger
drain time.Duration
}
type gatewayController interface {
@@ -57,6 +65,10 @@ type nativeExecutor interface {
Run(context.Context, string, nativebackendexecutor.Request) error
}
type containerExecutor interface {
Run(context.Context, string, backendexecutor.Request) error
}
// New creates the complete native backend update orchestrator.
func New(
config deploymentconfig.Config,
@@ -72,6 +84,9 @@ func New(
if err := config.Validate(); err != nil {
return nil, err
}
if config.Backend.Type != deploymentconfig.BackendTypeNative {
return nil, errors.New("backend.type must be native")
}
if !filepath.IsAbs(workRoot) || filepath.Clean(workRoot) != workRoot {
return nil, errors.New("backend update work root must be a clean absolute path")
}
@@ -99,8 +114,54 @@ func New(
}, nil
}
// NewContainer creates the Docker standalone backend update orchestrator.
func NewContainer(
config deploymentconfig.Config,
workRoot string,
store *transaction.Store,
coordinator *transaction.Coordinator,
engine containerengine.Engine,
gateway gatewayController,
httpClient *http.Client,
logger *slog.Logger,
) (*Updater, error) {
if err := config.Validate(); err != nil {
return nil, err
}
if config.Backend.Type != deploymentconfig.BackendTypeContainer {
return nil, errors.New("backend.type must be container")
}
if config.Daemon.Environment != deploymentconfig.EnvironmentDev {
return nil, errors.New("--container-image requires daemon.environment = dev")
}
if !filepath.IsAbs(workRoot) || filepath.Clean(workRoot) != workRoot {
return nil, errors.New("container backend update work root must be a clean absolute path")
}
if store == nil || coordinator == nil || engine == nil || gateway == nil {
return nil, errors.New("container backend update dependencies are required")
}
if logger == nil {
logger = slog.Default()
}
executor, err := backendexecutor.New(store, coordinator, engine, httpClient)
if err != nil {
return nil, err
}
return &Updater{
config: config, workRoot: workRoot, store: store, coordinator: coordinator, gateway: gateway,
containerExecutor: executor, engine: engine, logger: logger, drain: drainDuration,
containerConfigSource: deploymentconfig.ContainerConfigSource,
containerConfigTarget: deploymentconfig.ContainerConfigTarget,
containerTmpSource: deploymentconfig.ContainerTmpSource,
containerTmpTarget: deploymentconfig.ContainerTmpTarget,
}, nil
}
// UpdateRepack applies one repack ZIP selected by an absolute local path.
func (u *Updater) UpdateRepack(ctx context.Context, packagePath string, report ProgressReporter) (transaction.Transaction, error) {
if u.config.Backend.Type == deploymentconfig.BackendTypeContainer {
return transaction.Transaction{}, errors.New("repack ZIP update is not implemented for container backend")
}
reportProgress(report, Progress{Message: "Validating repack ZIP"})
updatePackage, err := updatepackage.OpenBackendNative(packagePath)
if err != nil {
@@ -125,6 +186,9 @@ func (u *Updater) UpdateRepack(ctx context.Context, packagePath string, report P
// UpdateNativeJAR applies one JAR copied directly to the server.
func (u *Updater) UpdateNativeJAR(ctx context.Context, jarPath string, report ProgressReporter) (transaction.Transaction, error) {
if u.config.Backend.Type == deploymentconfig.BackendTypeContainer {
return transaction.Transaction{}, errors.New("--native-jar requires backend.type = native")
}
reportProgress(report, Progress{Message: "Validating direct native backend JAR and computing SHA-256"})
jar, err := updatepackage.OpenDirectNativeJAR(jarPath)
if err != nil {
+9 -1
View File
@@ -175,7 +175,9 @@ const serverConfiguration8081 = `http {
`
type memoryGateway struct {
snapshot hostnginx.Snapshot
snapshot hostnginx.Snapshot
applyCount int
beforeApply func(hostnginx.Snapshot) error
}
func (g *memoryGateway) Read() (hostnginx.Snapshot, error) {
@@ -183,7 +185,13 @@ func (g *memoryGateway) Read() (hostnginx.Snapshot, error) {
}
func (g *memoryGateway) Apply(_ context.Context, snapshot hostnginx.Snapshot) error {
if g.beforeApply != nil {
if err := g.beforeApply(snapshot); err != nil {
return err
}
}
g.snapshot = hostnginx.Snapshot{Content: append([]byte(nil), snapshot.Content...), ActivePort: snapshot.ActivePort}
g.applyCount++
return nil
}