feat: backend native executor implement

This commit is contained in:
2026-08-16 01:27:30 +08:00
parent fbce91d797
commit 390e0565d3
44 changed files with 6430 additions and 31 deletions
+404
View File
@@ -0,0 +1,404 @@
// Package nativebackendexecutor prepares and starts one explicitly configured native backend slot.
// Gateway switching is deliberately outside this package.
package nativebackendexecutor
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"yms-daemon/internal/filestore"
"yms-daemon/internal/healthcheck"
"yms-daemon/internal/systemd"
"yms-daemon/internal/transaction"
)
const (
healthPath = "/yms/actuator/health"
healthTimeout = 120 * time.Second
healthInterval = time.Second
stepInstallJar = "backend.native.jar.install"
stepBindSlot = "backend.native.slot.bind"
stepStartUnit = "backend.native.service.start"
stepCheckHealth = "backend.native.health"
stepStopUnit = "backend.native.service.stop"
stepRestoreSlot = "backend.native.slot.restore"
activeState = "active"
inactiveState = "inactive"
failedState = "failed"
)
// Request contains exact values from the immutable update request and local deployment configuration.
// UnitName, SlotJarPath and PreviousSlotTarget are opaque and are never derived from filenames or ports.
type Request struct {
ArtifactPath string
ArtifactIdentity filestore.Identity
ReleasePath string
SlotJarPath string
PreviousSlotTarget string
UnitName string
Port int
HealthEndpoint string
Progress func(transaction.State, string)
}
type actuatorChecker interface {
Check(context.Context, string, healthcheck.RunningProbe) (healthcheck.ActuatorReport, bool, error)
Wait(context.Context, string, time.Duration, healthcheck.RunningProbe) (healthcheck.ActuatorReport, error)
}
// Executor drives the persisted transaction up to SWITCHING after the inactive native backend is healthy.
type Executor struct {
store *transaction.Store
coordinator *transaction.Coordinator
releaseStore *filestore.Store
units systemd.Manager
checker actuatorChecker
}
func New(store *transaction.Store, coordinator *transaction.Coordinator, releaseStore *filestore.Store, units systemd.Manager, httpClient *http.Client) (*Executor, error) {
if store == nil {
return nil, errors.New("transaction store is required")
}
if coordinator == nil {
return nil, errors.New("transaction coordinator is required")
}
if releaseStore == nil {
return nil, errors.New("native backend release store is required")
}
if units == nil {
return nil, errors.New("systemd manager is required")
}
checker, err := healthcheck.NewActuatorChecker(httpClient, healthInterval)
if err != nil {
return nil, err
}
return &Executor{store: store, coordinator: coordinator, releaseStore: releaseStore, units: units, checker: checker}, nil
}
// Run resumes from the transaction's persisted state. It does not switch gateway traffic.
func (e *Executor) Run(ctx context.Context, transactionID string, request Request) error {
if strings.TrimSpace(transactionID) == "" {
return errors.New("transaction ID is required")
}
return e.coordinator.RunExclusive(ctx, func(ctx context.Context) error {
return e.run(ctx, transactionID, request)
})
}
func (e *Executor) run(ctx context.Context, transactionID string, request Request) error {
for {
record, err := e.store.Transaction(ctx, transactionID)
if err != nil {
return err
}
switch record.State {
case transaction.StateCreated:
if _, err := e.store.Transition(ctx, transactionID, transaction.StateValidating, "native backend validation started"); err != nil {
return err
}
case transaction.StateValidating:
reportProgress(request, record.State, "Validating inactive native backend slot")
if err := e.validate(ctx, request); err != nil {
_, transitionErr := e.store.Transition(ctx, transactionID, transaction.StateFailed, err.Error())
return errors.Join(err, transitionErr)
}
if _, err := e.store.Transition(ctx, transactionID, transaction.StatePrepared, "native backend inputs validated"); err != nil {
return err
}
case transaction.StatePrepared:
reportProgress(request, record.State, "Installing backend JAR and binding the inactive slot")
if _, err := e.prepare(ctx, transactionID, request); err != nil {
return e.failUnlessRecoverable(ctx, transactionID, err)
}
if _, err := e.store.Transition(ctx, transactionID, transaction.StateStarting, "native backend slot prepared"); err != nil {
return err
}
case transaction.StateStarting:
reportProgress(request, record.State, fmt.Sprintf("Preparing to start %s on port %d", request.UnitName, request.Port))
installedPath, err := e.prepare(ctx, transactionID, request)
if err != nil {
return e.failUnlessRecoverable(ctx, transactionID, err)
}
if err := e.startAndCheck(ctx, transactionID, request); err != nil {
if recoverable(err) {
return err
}
return e.rollbackBeforeSwitch(ctx, transactionID, request, installedPath, err)
}
if _, err := e.store.Transition(ctx, transactionID, transaction.StateSwitching, "native backend is healthy"); err != nil {
return err
}
case transaction.StateSwitching:
return nil
case transaction.StateRollingBack:
reportProgress(request, record.State, "Resuming native backend preparation compensation")
installed, found, err := e.releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
if err != nil {
return fmt.Errorf("inspect native backend JAR while resuming compensation: %w", err)
}
if !found {
return errors.New("installed native backend JAR is missing while compensation is pending")
}
if err := e.compensateBeforeSwitch(ctx, transactionID, request, installed.Path); err != nil {
return err
}
if _, err := e.store.Transition(ctx, transactionID, transaction.StateRolledBack, "native backend preparation rollback resumed and completed"); err != nil {
return err
}
return nil
default:
return fmt.Errorf("native backend executor cannot run transaction %s in state %s", transactionID, record.State)
}
}
}
func (e *Executor) validate(ctx context.Context, request Request) error {
if err := validateRequest(request); err != nil {
return err
}
info, err := os.Lstat(request.ArtifactPath)
if err != nil {
return fmt.Errorf("inspect native backend JAR %s: %w", request.ArtifactPath, err)
}
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("native backend JAR is not a regular file: %s", request.ArtifactPath)
}
if info.Size() != request.ArtifactIdentity.Size {
return fmt.Errorf("native backend JAR size mismatch: got %d, want %d", info.Size(), request.ArtifactIdentity.Size)
}
if err := inspectInitialSlot(request.SlotJarPath, request.PreviousSlotTarget); err != nil {
return err
}
unit, err := e.units.Inspect(ctx, request.UnitName)
if err != nil {
return fmt.Errorf("inspect inactive native backend unit: %w", err)
}
if unit.ActiveState != inactiveState && unit.ActiveState != failedState {
return fmt.Errorf("native backend unit %s must be inactive or failed before preparation, active state is %q", request.UnitName, unit.ActiveState)
}
return nil
}
func (e *Executor) prepare(ctx context.Context, transactionID string, request Request) (string, error) {
installOperation := &installJarOperation{
store: e.releaseStore,
sourcePath: request.ArtifactPath,
releasePath: request.ReleasePath,
identity: request.ArtifactIdentity,
}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, installIntent(request), installOperation); err != nil {
return "", err
}
installed, found, err := e.releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
if err != nil {
return "", fmt.Errorf("inspect installed native backend JAR: %w", err)
}
if !found {
return "", errors.New("installed native backend JAR is missing after completed install step")
}
bindOperation := &slotLinkOperation{
path: request.SlotJarPath,
desiredTarget: installed.Path,
previousTarget: request.PreviousSlotTarget,
}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, bindIntent(request, installed.Path), bindOperation); err != nil {
return "", err
}
return installed.Path, nil
}
func (e *Executor) startAndCheck(ctx context.Context, transactionID string, request Request) error {
reportProgress(request, transaction.StateStarting, "Starting native backend unit "+request.UnitName)
startOperation := &unitStartOperation{units: e.units, name: request.UnitName}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, startIntent(request), startOperation); err != nil {
return err
}
reportProgress(request, transaction.StateStarting, fmt.Sprintf("Waiting up to %s for Actuator health: %s", healthTimeout, request.HealthEndpoint))
healthOperation := &healthOperation{
units: e.units,
checker: e.checker,
unitName: request.UnitName,
endpoint: request.HealthEndpoint,
timeout: healthTimeout,
}
_, err := e.coordinator.ExecuteStep(ctx, transactionID, healthIntent(request), healthOperation)
if err == nil {
reportProgress(request, transaction.StateStarting, "Actuator health status is UP")
}
return err
}
func reportProgress(request Request, state transaction.State, message string) {
if request.Progress != nil {
request.Progress(state, message)
}
}
func (e *Executor) rollbackBeforeSwitch(ctx context.Context, transactionID string, request Request, installedPath string, cause error) error {
if _, err := e.store.Transition(ctx, transactionID, transaction.StateRollingBack, "native backend preparation failed; compensation started"); err != nil {
return errors.Join(cause, err)
}
if err := e.compensateBeforeSwitch(ctx, transactionID, request, installedPath); err != nil {
return errors.Join(cause, err)
}
if _, err := e.store.Transition(ctx, transactionID, transaction.StateRolledBack, "native backend preparation rolled back"); err != nil {
return errors.Join(cause, err)
}
return cause
}
func (e *Executor) compensateBeforeSwitch(ctx context.Context, transactionID string, request Request, installedPath string) error {
stopOperation := &unitStopOperation{units: e.units, name: request.UnitName}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, stopIntent(request), stopOperation); err != nil {
return err
}
restoreOperation := &slotLinkOperation{
path: request.SlotJarPath,
desiredTarget: request.PreviousSlotTarget,
previousTarget: installedPath,
}
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, restoreIntent(request, installedPath), restoreOperation); err != nil {
return err
}
return nil
}
func (e *Executor) failUnlessRecoverable(ctx context.Context, transactionID string, cause error) error {
if recoverable(cause) {
return cause
}
_, transitionErr := e.store.Transition(ctx, transactionID, transaction.StateFailed, cause.Error())
return errors.Join(cause, transitionErr)
}
func recoverable(cause error) bool {
var uncertain *transaction.UncertainStepError
return errors.As(cause, &uncertain) || errors.Is(cause, transaction.ErrStepConflict)
}
func validateRequest(request Request) error {
if !filepath.IsAbs(request.ArtifactPath) {
return errors.New("native backend JAR path must be absolute")
}
if err := request.ArtifactIdentity.Validate(); err != nil {
return fmt.Errorf("invalid native backend JAR identity: %w", err)
}
if !filepath.IsLocal(request.ReleasePath) || request.ReleasePath == "." {
return fmt.Errorf("native backend release path must be a local relative path: %q", request.ReleasePath)
}
if !filepath.IsAbs(request.SlotJarPath) {
return errors.New("native backend slot JAR path must be absolute")
}
if request.PreviousSlotTarget != "" && !filepath.IsAbs(request.PreviousSlotTarget) {
return errors.New("previous native backend slot target must be empty or absolute")
}
if request.UnitName == "" || strings.TrimSpace(request.UnitName) != request.UnitName {
return errors.New("exact native backend systemd unit name is required")
}
if request.Port != 8080 && request.Port != 8081 {
return fmt.Errorf("native backend port must be 8080 or 8081: %d", request.Port)
}
parsed, err := url.ParseRequestURI(request.HealthEndpoint)
if err != nil || parsed.Scheme != "http" || parsed.Host == "" || parsed.Path != healthPath {
return fmt.Errorf("health endpoint must be an HTTP URL with exact path %s", healthPath)
}
if parsed.Port() != strconv.Itoa(request.Port) {
return fmt.Errorf("health endpoint port must equal native backend port %d", request.Port)
}
return nil
}
func inspectInitialSlot(path, previousTarget string) error {
info, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
if previousTarget == "" {
return nil
}
return fmt.Errorf("native backend slot link %s is missing; expected target %s", path, previousTarget)
}
if err != nil {
return fmt.Errorf("inspect native backend slot link %s: %w", path, err)
}
if info.Mode()&os.ModeSymlink == 0 {
return fmt.Errorf("native backend slot path is not a symbolic link: %s", path)
}
target, err := os.Readlink(path)
if err != nil {
return fmt.Errorf("read native backend slot link %s: %w", path, err)
}
if target != previousTarget {
return fmt.Errorf("native backend slot link %s targets %q, expected %q", path, target, previousTarget)
}
if previousTarget != "" {
targetInfo, err := os.Lstat(previousTarget)
if err != nil {
return fmt.Errorf("inspect previous native backend slot target %s: %w", previousTarget, err)
}
if !targetInfo.Mode().IsRegular() || targetInfo.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("previous native backend slot target is not a regular file: %s", previousTarget)
}
}
return nil
}
func installIntent(request Request) transaction.StepIntent {
return intent(stepInstallJar, "install immutable native backend JAR", struct {
SourcePath string `json:"sourcePath"`
ReleasePath string `json:"releasePath"`
Identity filestore.Identity `json:"identity"`
}{request.ArtifactPath, request.ReleasePath, request.ArtifactIdentity})
}
func bindIntent(request Request, installedPath string) transaction.StepIntent {
return intent(stepBindSlot, "bind inactive native backend slot", struct {
SlotJarPath string `json:"slotJarPath"`
PreviousTarget string `json:"previousTarget"`
InstalledPath string `json:"installedPath"`
}{request.SlotJarPath, request.PreviousSlotTarget, installedPath})
}
func startIntent(request Request) transaction.StepIntent {
return intent(stepStartUnit, "start inactive native backend systemd unit", struct {
UnitName string `json:"unitName"`
}{request.UnitName})
}
func healthIntent(request Request) transaction.StepIntent {
return intent(stepCheckHealth, "wait for native backend Actuator health", struct {
UnitName string `json:"unitName"`
Endpoint string `json:"endpoint"`
Timeout time.Duration `json:"timeout"`
}{request.UnitName, request.HealthEndpoint, healthTimeout})
}
func stopIntent(request Request) transaction.StepIntent {
return intent(stepStopUnit, "stop failed inactive native backend systemd unit", struct {
UnitName string `json:"unitName"`
}{request.UnitName})
}
func restoreIntent(request Request, installedPath string) transaction.StepIntent {
return intent(stepRestoreSlot, "restore inactive native backend slot", struct {
SlotJarPath string `json:"slotJarPath"`
InstalledPath string `json:"installedPath"`
PreviousTarget string `json:"previousTarget"`
}{request.SlotJarPath, installedPath, request.PreviousSlotTarget})
}
func intent(key, name string, value any) transaction.StepIntent {
payload, err := json.Marshal(value)
if err != nil {
panic(fmt.Sprintf("marshal internal step intent: %v", err))
}
return transaction.StepIntent{Key: key, Name: name, Intent: payload}
}
@@ -0,0 +1,430 @@
package nativebackendexecutor
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"sync"
"testing"
"time"
"yms-daemon/internal/filestore"
"yms-daemon/internal/healthcheck"
"yms-daemon/internal/systemd"
"yms-daemon/internal/transaction"
)
func TestExecutorInstallsJarStartsExactUnitAndReachesSwitching(t *testing.T) {
ctx := context.Background()
executor, store, releaseStore, units, request, previousTarget := testNativeExecutor(t)
record := createNativeTransaction(t, store, "success")
if err := executor.Run(ctx, record.ID, request); err != nil {
t.Fatalf("run native backend executor: %v", err)
}
current, err := store.Transaction(ctx, record.ID)
if err != nil || current.State != transaction.StateSwitching {
t.Fatalf("unexpected transaction after native preparation: record=%+v err=%v", current, err)
}
installed, found, err := releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
if err != nil || !found {
t.Fatalf("inspect installed backend JAR: file=%+v found=%t err=%v", installed, found, err)
}
actualTarget, err := os.Readlink(request.SlotJarPath)
if err != nil || actualTarget != installed.Path || actualTarget == previousTarget {
t.Fatalf("unexpected slot link: target=%q installed=%q previous=%q err=%v", actualTarget, installed.Path, previousTarget, err)
}
units.mu.Lock()
startCalls := units.startCalls
stopCalls := units.stopCalls
unitActiveState := units.unit.ActiveState
startedName := units.startedName
units.mu.Unlock()
if startCalls != 1 || stopCalls != 0 || unitActiveState != activeState || startedName != request.UnitName {
t.Fatalf("unexpected systemd calls: start=%d stop=%d state=%s name=%s", startCalls, stopCalls, unitActiveState, startedName)
}
pending, err := store.PendingSteps(ctx, record.ID)
if err != nil || len(pending) != 0 {
t.Fatalf("unexpected pending native steps: steps=%+v err=%v", pending, err)
}
if err := executor.Run(ctx, record.ID, request); err != nil {
t.Fatalf("repeat executor at switching state: %v", err)
}
units.mu.Lock()
repeatedStartCalls := units.startCalls
units.mu.Unlock()
if repeatedStartCalls != startCalls {
t.Fatalf("switching state repeated systemd start: before=%d after=%d", startCalls, repeatedStartCalls)
}
}
func TestExecutorRollsBackSlotAndStopsUnitWhenHealthFails(t *testing.T) {
ctx := context.Background()
executor, store, _, units, request, previousTarget := testNativeExecutor(t)
executor.checker = &fakeActuatorChecker{
waitErr: errors.New("Actuator rejected native backend"),
checkReady: false,
}
record := createNativeTransaction(t, store, "health-failure")
err := executor.Run(ctx, record.ID, request)
if err == nil {
t.Fatal("expected native backend health failure")
}
current, readErr := store.Transaction(ctx, record.ID)
if readErr != nil || current.State != transaction.StateRolledBack {
t.Fatalf("unexpected compensated transaction: record=%+v err=%v", current, readErr)
}
actualTarget, linkErr := os.Readlink(request.SlotJarPath)
if linkErr != nil || actualTarget != previousTarget {
t.Fatalf("native slot was not restored: target=%q previous=%q err=%v", actualTarget, previousTarget, linkErr)
}
units.mu.Lock()
startCalls := units.startCalls
stopCalls := units.stopCalls
unitActiveState := units.unit.ActiveState
units.mu.Unlock()
if startCalls != 1 || stopCalls != 1 || unitActiveState != inactiveState {
t.Fatalf("unexpected compensated systemd state: start=%d stop=%d state=%s", startCalls, stopCalls, unitActiveState)
}
}
func TestExecutorResumesPersistedRollback(t *testing.T) {
ctx := context.Background()
executor, store, _, units, request, previousTarget := testNativeExecutor(t)
record := createNativeTransaction(t, store, "resume-rollback")
transitionNativeToPrepared(t, store, record.ID)
if _, err := executor.prepare(ctx, record.ID, request); err != nil {
t.Fatalf("prepare native backend before rollback interruption: %v", err)
}
if _, err := store.Transition(ctx, record.ID, transaction.StateStarting, "test starting"); err != nil {
t.Fatalf("transition native transaction to starting: %v", err)
}
if err := units.Start(ctx, request.UnitName); err != nil {
t.Fatalf("start native backend before rollback interruption: %v", err)
}
if _, err := store.Transition(ctx, record.ID, transaction.StateRollingBack, "test interrupted rollback"); err != nil {
t.Fatalf("persist interrupted rollback state: %v", err)
}
if err := executor.Run(ctx, record.ID, request); err != nil {
t.Fatalf("resume native backend rollback: %v", err)
}
current, err := store.Transaction(ctx, record.ID)
if err != nil || current.State != transaction.StateRolledBack {
t.Fatalf("unexpected resumed rollback state: record=%+v err=%v", current, err)
}
actualTarget, err := os.Readlink(request.SlotJarPath)
if err != nil || actualTarget != previousTarget {
t.Fatalf("resumed rollback did not restore slot: target=%q previous=%q err=%v", actualTarget, previousTarget, err)
}
units.mu.Lock()
stopCalls := units.stopCalls
unitActiveState := units.unit.ActiveState
units.mu.Unlock()
if stopCalls != 1 || unitActiveState != inactiveState {
t.Fatalf("resumed rollback did not stop unit: stop=%d state=%s", stopCalls, unitActiveState)
}
}
func TestExecutorRollbackRemovesFirstDeploymentSlotLink(t *testing.T) {
ctx := context.Background()
executor, store, _, _, request, _ := testNativeExecutor(t)
if err := os.Remove(request.SlotJarPath); err != nil {
t.Fatalf("remove seeded slot link: %v", err)
}
request.PreviousSlotTarget = ""
executor.checker = &fakeActuatorChecker{
waitErr: errors.New("Actuator rejected first native backend deployment"),
checkReady: false,
}
record := createNativeTransaction(t, store, "first-deployment-rollback")
if err := executor.Run(ctx, record.ID, request); err == nil {
t.Fatal("expected first native backend deployment health failure")
}
current, err := store.Transaction(ctx, record.ID)
if err != nil || current.State != transaction.StateRolledBack {
t.Fatalf("unexpected first deployment rollback state: record=%+v err=%v", current, err)
}
if _, err := os.Lstat(request.SlotJarPath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("first deployment rollback retained slot link: %v", err)
}
}
func TestExecutorRecoversRecordedSlotIntentWithoutChangingRequest(t *testing.T) {
ctx := context.Background()
executor, store, releaseStore, units, request, _ := testNativeExecutor(t)
record := createNativeTransaction(t, store, "recover-slot")
transitionNativeToPrepared(t, store, record.ID)
installOperation := &installJarOperation{
store: releaseStore,
sourcePath: request.ArtifactPath,
releasePath: request.ReleasePath,
identity: request.ArtifactIdentity,
}
if _, err := executor.coordinator.ExecuteStep(ctx, record.ID, installIntent(request), installOperation); err != nil {
t.Fatalf("install backend JAR before simulated crash: %v", err)
}
installed, found, err := releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
if err != nil || !found {
t.Fatalf("inspect backend JAR before simulated crash: file=%+v found=%t err=%v", installed, found, err)
}
if _, _, err := store.RecordStepIntent(ctx, record.ID, bindIntent(request, installed.Path)); err != nil {
t.Fatalf("record slot intent before simulated crash: %v", err)
}
if err := executor.Run(ctx, record.ID, request); err != nil {
t.Fatalf("recover native backend executor: %v", err)
}
actualTarget, err := os.Readlink(request.SlotJarPath)
if err != nil || actualTarget != installed.Path {
t.Fatalf("unexpected recovered slot link: target=%q installed=%q err=%v", actualTarget, installed.Path, err)
}
units.mu.Lock()
startCalls := units.startCalls
units.mu.Unlock()
if startCalls != 1 {
t.Fatalf("unexpected recovered systemd start count: %d", startCalls)
}
}
func TestExecutorRejectsChangedRecoveryIntentAndPreservesStartingState(t *testing.T) {
ctx := context.Background()
executor, store, _, _, request, _ := testNativeExecutor(t)
record := createNativeTransaction(t, store, "changed-request")
transitionNativeToPrepared(t, store, record.ID)
if _, err := executor.prepare(ctx, record.ID, request); err != nil {
t.Fatalf("prepare original native backend request: %v", err)
}
if _, err := store.Transition(ctx, record.ID, transaction.StateStarting, "test starting"); err != nil {
t.Fatalf("transition native backend to starting: %v", err)
}
changed := request
changed.ReleasePath = "different-release.jar"
err := executor.Run(ctx, record.ID, changed)
if !errors.Is(err, transaction.ErrStepConflict) {
t.Fatalf("expected persisted native intent conflict, got %v", err)
}
current, readErr := store.Transaction(ctx, record.ID)
if readErr != nil || current.State != transaction.StateStarting {
t.Fatalf("changed recovery request altered transaction: record=%+v err=%v", current, readErr)
}
}
func TestExecutorRejectsActiveUnitBeforeChangingFiles(t *testing.T) {
ctx := context.Background()
executor, store, releaseStore, units, request, previousTarget := testNativeExecutor(t)
units.mu.Lock()
units.unit.ActiveState = activeState
units.unit.SubState = "running"
units.mu.Unlock()
record := createNativeTransaction(t, store, "active-unit")
if err := executor.Run(ctx, record.ID, request); err == nil {
t.Fatal("expected active unit validation failure")
}
current, err := store.Transaction(ctx, record.ID)
if err != nil || current.State != transaction.StateFailed {
t.Fatalf("unexpected active-unit transaction state: record=%+v err=%v", current, err)
}
if _, found, err := releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity); err != nil || found {
t.Fatalf("validation failure changed release store: found=%t err=%v", found, err)
}
actualTarget, err := os.Readlink(request.SlotJarPath)
if err != nil || actualTarget != previousTarget {
t.Fatalf("validation failure changed slot link: target=%q previous=%q err=%v", actualTarget, previousTarget, err)
}
}
func TestUnitStopOperationAcceptsSystemdFailedAsStopped(t *testing.T) {
units := &fakeUnitManager{unit: systemd.Unit{
Name: "yms-backend@8080.service",
LoadState: "loaded",
ActiveState: failedState,
SubState: "failed",
}}
operation := &unitStopOperation{units: units, name: units.unit.Name}
inspection, err := operation.Inspect(context.Background())
if err != nil {
t.Fatalf("inspect stopped native backend unit: %v", err)
}
if inspection.Status != transaction.InspectionApplied {
t.Fatalf("unexpected stopped native backend unit inspection: %+v", inspection)
}
}
func testNativeExecutor(t *testing.T) (*Executor, *transaction.Store, *filestore.Store, *fakeUnitManager, Request, string) {
t.Helper()
store, err := transaction.OpenStore(context.Background(), filepath.Join(t.TempDir(), "transactions.db"))
if err != nil {
t.Fatalf("open transaction store: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatalf("create transaction coordinator: %v", err)
}
releaseStore, err := filestore.New(t.TempDir())
if err != nil {
t.Fatalf("create native release store: %v", err)
}
artifactContent := []byte("native backend jar content")
artifactPath := filepath.Join(t.TempDir(), "glory-soft-yms.jar")
if err := os.WriteFile(artifactPath, artifactContent, 0o640); err != nil {
t.Fatalf("write native backend artifact: %v", err)
}
previousTarget := filepath.Join(t.TempDir(), "previous-backend.jar")
if err := os.WriteFile(previousTarget, []byte("previous backend jar"), 0o640); err != nil {
t.Fatalf("write previous backend JAR: %v", err)
}
slotDirectory := t.TempDir()
slotJarPath := filepath.Join(slotDirectory, "backend-green.jar")
if err := os.Symlink(previousTarget, slotJarPath); err != nil {
t.Fatalf("create previous native backend slot link: %v", err)
}
request := Request{
ArtifactPath: artifactPath,
ArtifactIdentity: testIdentity(artifactContent),
ReleasePath: "glory-soft-yms-20260815.jar",
SlotJarPath: slotJarPath,
PreviousSlotTarget: previousTarget,
UnitName: "yms-green.service",
Port: 8081,
HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health",
}
units := &fakeUnitManager{unit: systemd.Unit{
Name: request.UnitName,
LoadState: "loaded",
ActiveState: inactiveState,
SubState: "dead",
}}
executor, err := New(store, coordinator, releaseStore, units, &http.Client{})
if err != nil {
t.Fatalf("create native backend executor: %v", err)
}
executor.checker = &fakeActuatorChecker{
waitReport: healthcheck.ActuatorReport{Status: "UP", Components: map[string]string{"db": "UP"}},
checkReport: healthcheck.ActuatorReport{Status: "UP", Components: map[string]string{"db": "UP"}},
checkReady: true,
}
return executor, store, releaseStore, units, request, previousTarget
}
func createNativeTransaction(t *testing.T, store *transaction.Store, suffix string) transaction.Transaction {
t.Helper()
record, _, err := store.CreateTransaction(context.Background(), transaction.CreateRequest{
ID: "native-backend-" + suffix,
IdempotencyKey: "native-backend-request-" + suffix,
Source: "test",
Service: "backend",
})
if err != nil {
t.Fatalf("create native backend transaction: %v", err)
}
return record
}
func transitionNativeToPrepared(t *testing.T, store *transaction.Store, transactionID string) {
t.Helper()
ctx := context.Background()
if _, err := store.Transition(ctx, transactionID, transaction.StateValidating, "test validating"); err != nil {
t.Fatalf("transition native transaction to validating: %v", err)
}
if _, err := store.Transition(ctx, transactionID, transaction.StatePrepared, "test prepared"); err != nil {
t.Fatalf("transition native transaction to prepared: %v", err)
}
}
func testIdentity(content []byte) filestore.Identity {
digest := sha256.Sum256(content)
return filestore.Identity{Size: int64(len(content)), SHA256: hex.EncodeToString(digest[:])}
}
type fakeUnitManager struct {
mu sync.Mutex
unit systemd.Unit
startErr error
stopErr error
startCalls int
stopCalls int
startedName string
}
func (m *fakeUnitManager) Inspect(_ context.Context, name string) (systemd.Unit, error) {
m.mu.Lock()
defer m.mu.Unlock()
if name != m.unit.Name {
return systemd.Unit{}, systemd.ErrUnitNotFound
}
return m.unit, nil
}
func (m *fakeUnitManager) Start(_ context.Context, name string) error {
m.mu.Lock()
defer m.mu.Unlock()
m.startCalls++
m.startedName = name
if m.startErr != nil {
m.unit.ActiveState = failedState
m.unit.SubState = "failed"
return m.startErr
}
m.unit.ActiveState = activeState
m.unit.SubState = "running"
return nil
}
func (m *fakeUnitManager) Stop(context.Context, string) error {
m.mu.Lock()
defer m.mu.Unlock()
m.stopCalls++
if m.stopErr != nil {
return m.stopErr
}
m.unit.ActiveState = inactiveState
m.unit.SubState = "dead"
return nil
}
type fakeActuatorChecker struct {
waitReport healthcheck.ActuatorReport
waitErr error
checkReport healthcheck.ActuatorReport
checkReady bool
checkErr error
}
func (c *fakeActuatorChecker) Wait(ctx context.Context, _ string, _ time.Duration, running healthcheck.RunningProbe) (healthcheck.ActuatorReport, error) {
isRunning, err := running(ctx)
if err != nil {
return healthcheck.ActuatorReport{}, err
}
if !isRunning {
return healthcheck.ActuatorReport{}, healthcheck.ErrWorkloadStopped
}
return c.waitReport, c.waitErr
}
func (c *fakeActuatorChecker) Check(ctx context.Context, _ string, running healthcheck.RunningProbe) (healthcheck.ActuatorReport, bool, error) {
isRunning, err := running(ctx)
if err != nil {
return healthcheck.ActuatorReport{}, false, err
}
if !isRunning {
return healthcheck.ActuatorReport{}, false, healthcheck.ErrWorkloadStopped
}
return c.checkReport, c.checkReady, c.checkErr
}
var _ systemd.Manager = (*fakeUnitManager)(nil)
var _ actuatorChecker = (*fakeActuatorChecker)(nil)
@@ -0,0 +1,307 @@
package nativebackendexecutor
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"yms-daemon/internal/filestore"
"yms-daemon/internal/healthcheck"
"yms-daemon/internal/systemd"
"yms-daemon/internal/transaction"
)
type installJarOperation struct {
store *filestore.Store
sourcePath string
releasePath string
identity filestore.Identity
}
func (o *installJarOperation) Apply(context.Context) error {
source, err := os.Open(o.sourcePath)
if err != nil {
return fmt.Errorf("open native backend JAR: %w", err)
}
defer source.Close()
_, err = o.store.Commit(o.releasePath, source, o.identity)
return err
}
func (o *installJarOperation) Inspect(context.Context) (transaction.Inspection, error) {
file, found, err := o.store.Inspect(o.releasePath, o.identity)
if errors.Is(err, filestore.ErrDestinationConflict) {
return transaction.Inspection{Status: transaction.InspectionUnknown}, nil
}
if err != nil {
return transaction.Inspection{}, err
}
if !found {
return transaction.Inspection{Status: transaction.InspectionNotApplied}, nil
}
return transaction.Inspection{Status: transaction.InspectionApplied, Result: resultJSON(file)}, nil
}
type slotLinkOperation struct {
path string
desiredTarget string
previousTarget string
}
func (o *slotLinkOperation) Apply(ctx context.Context) error {
inspection, err := o.Inspect(ctx)
if err != nil {
return err
}
switch inspection.Status {
case transaction.InspectionApplied:
return nil
case transaction.InspectionNotApplied:
case transaction.InspectionUnknown:
return fmt.Errorf("native backend slot link %s does not match the recorded previous target", o.path)
default:
return fmt.Errorf("native backend slot link %s returned invalid inspection status %q", o.path, inspection.Status)
}
parent := filepath.Dir(o.path)
parentInfo, err := os.Lstat(parent)
if err != nil {
return fmt.Errorf("inspect native backend slot directory %s: %w", parent, err)
}
if !parentInfo.IsDir() || parentInfo.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("native backend slot parent is not a direct directory: %s", parent)
}
if o.desiredTarget == "" {
if err := os.Remove(o.path); err != nil {
return fmt.Errorf("remove native backend slot link %s: %w", o.path, err)
}
return syncDirectory(parent)
}
targetInfo, err := os.Lstat(o.desiredTarget)
if err != nil {
return fmt.Errorf("inspect native backend slot target %s: %w", o.desiredTarget, err)
}
if !targetInfo.Mode().IsRegular() || targetInfo.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("native backend slot target is not a regular file: %s", o.desiredTarget)
}
temporary, err := os.CreateTemp(parent, ".slot-link-*")
if err != nil {
return fmt.Errorf("reserve native backend slot link path: %w", err)
}
temporaryPath := temporary.Name()
if err := temporary.Close(); err != nil {
_ = os.Remove(temporaryPath)
return fmt.Errorf("close native backend slot link reservation: %w", err)
}
if err := os.Remove(temporaryPath); err != nil {
return fmt.Errorf("remove native backend slot link reservation: %w", err)
}
committed := false
defer func() {
if !committed {
_ = os.Remove(temporaryPath)
}
}()
if err := os.Symlink(o.desiredTarget, temporaryPath); err != nil {
return fmt.Errorf("create native backend slot link: %w", err)
}
if err := os.Rename(temporaryPath, o.path); err != nil {
return fmt.Errorf("replace native backend slot link %s: %w", o.path, err)
}
committed = true
return syncDirectory(parent)
}
func (o *slotLinkOperation) Inspect(context.Context) (transaction.Inspection, error) {
info, err := os.Lstat(o.path)
if errors.Is(err, os.ErrNotExist) {
if o.desiredTarget == "" {
if err := syncDirectory(filepath.Dir(o.path)); err != nil {
return transaction.Inspection{}, err
}
return transaction.Inspection{Status: transaction.InspectionApplied, Result: linkResult(o.path, "")}, nil
}
if o.previousTarget == "" {
return transaction.Inspection{Status: transaction.InspectionNotApplied}, nil
}
return transaction.Inspection{Status: transaction.InspectionUnknown}, nil
}
if err != nil {
return transaction.Inspection{}, err
}
if info.Mode()&os.ModeSymlink == 0 {
return transaction.Inspection{Status: transaction.InspectionUnknown}, nil
}
target, err := os.Readlink(o.path)
if err != nil {
return transaction.Inspection{}, err
}
result := linkResult(o.path, target)
if target == o.desiredTarget {
if err := syncDirectory(filepath.Dir(o.path)); err != nil {
return transaction.Inspection{}, err
}
return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
}
if target == o.previousTarget {
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
}
return transaction.Inspection{Status: transaction.InspectionUnknown, Result: result}, nil
}
type unitStartOperation struct {
units systemd.Manager
name string
}
func (o *unitStartOperation) Apply(ctx context.Context) error {
return o.units.Start(ctx, o.name)
}
func (o *unitStartOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
unit, err := o.units.Inspect(ctx, o.name)
if err != nil {
return transaction.Inspection{}, err
}
result := unitResult(unit)
switch unit.ActiveState {
case activeState:
return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
case inactiveState, failedState:
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
default:
return transaction.Inspection{Status: transaction.InspectionUnknown, Result: result}, nil
}
}
type unitStopOperation struct {
units systemd.Manager
name string
}
func (o *unitStopOperation) Apply(ctx context.Context) error {
return o.units.Stop(ctx, o.name)
}
func (o *unitStopOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
unit, err := o.units.Inspect(ctx, o.name)
if err != nil {
return transaction.Inspection{}, err
}
result := unitResult(unit)
switch unit.ActiveState {
case inactiveState, failedState:
return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
case activeState:
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
default:
return transaction.Inspection{Status: transaction.InspectionUnknown, Result: result}, nil
}
}
type healthOperation struct {
units systemd.Manager
checker actuatorChecker
unitName string
endpoint string
timeout time.Duration
mu sync.Mutex
confirmedReport healthcheck.ActuatorReport
confirmed bool
}
func (o *healthOperation) Apply(ctx context.Context) error {
report, err := o.checker.Wait(ctx, o.endpoint, o.timeout, o.running)
if err != nil {
return err
}
o.mu.Lock()
o.confirmedReport = report
o.confirmed = true
o.mu.Unlock()
return nil
}
func (o *healthOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
o.mu.Lock()
if o.confirmed {
report := o.confirmedReport
o.mu.Unlock()
return healthInspection(report, true, nil)
}
o.mu.Unlock()
report, ready, err := o.checker.Check(ctx, o.endpoint, o.running)
return healthInspection(report, ready, err)
}
func (o *healthOperation) running(ctx context.Context) (bool, error) {
unit, err := o.units.Inspect(ctx, o.unitName)
if errors.Is(err, systemd.ErrUnitNotFound) {
return false, nil
}
if err != nil {
return false, err
}
return unit.ActiveState == activeState, nil
}
func healthInspection(report healthcheck.ActuatorReport, ready bool, err error) (transaction.Inspection, error) {
result := resultJSON(report)
if errors.Is(err, healthcheck.ErrWorkloadStopped) {
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
}
if err != nil || !ready {
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
}
return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
}
func linkResult(path, target string) json.RawMessage {
return resultJSON(struct {
Path string `json:"path"`
Target string `json:"target"`
}{path, target})
}
func unitResult(unit systemd.Unit) json.RawMessage {
return resultJSON(struct {
Name string `json:"name"`
LoadState string `json:"loadState"`
ActiveState string `json:"activeState"`
SubState string `json:"subState"`
}{unit.Name, unit.LoadState, unit.ActiveState, unit.SubState})
}
func resultJSON(value any) json.RawMessage {
payload, err := json.Marshal(value)
if err != nil {
panic(fmt.Sprintf("marshal internal step result: %v", err))
}
return payload
}
func syncDirectory(path string) error {
directory, err := os.Open(path)
if err != nil {
return fmt.Errorf("open directory for flush: %w", err)
}
syncErr := directory.Sync()
closeErr := directory.Close()
if err := errors.Join(syncErr, closeErr); err != nil {
return fmt.Errorf("flush directory: %w", err)
}
return nil
}
var _ transaction.Operation = (*installJarOperation)(nil)
var _ transaction.Operation = (*slotLinkOperation)(nil)
var _ transaction.Operation = (*unitStartOperation)(nil)
var _ transaction.Operation = (*unitStopOperation)(nil)
var _ transaction.Operation = (*healthOperation)(nil)