feat: backend native executor implement
This commit is contained in:
@@ -0,0 +1,430 @@
|
||||
package nativebackendexecutor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"yms-daemon/internal/filestore"
|
||||
"yms-daemon/internal/healthcheck"
|
||||
"yms-daemon/internal/systemd"
|
||||
"yms-daemon/internal/transaction"
|
||||
)
|
||||
|
||||
func TestExecutorInstallsJarStartsExactUnitAndReachesSwitching(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
executor, store, releaseStore, units, request, previousTarget := testNativeExecutor(t)
|
||||
record := createNativeTransaction(t, store, "success")
|
||||
|
||||
if err := executor.Run(ctx, record.ID, request); err != nil {
|
||||
t.Fatalf("run native backend executor: %v", err)
|
||||
}
|
||||
current, err := store.Transaction(ctx, record.ID)
|
||||
if err != nil || current.State != transaction.StateSwitching {
|
||||
t.Fatalf("unexpected transaction after native preparation: record=%+v err=%v", current, err)
|
||||
}
|
||||
installed, found, err := releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("inspect installed backend JAR: file=%+v found=%t err=%v", installed, found, err)
|
||||
}
|
||||
actualTarget, err := os.Readlink(request.SlotJarPath)
|
||||
if err != nil || actualTarget != installed.Path || actualTarget == previousTarget {
|
||||
t.Fatalf("unexpected slot link: target=%q installed=%q previous=%q err=%v", actualTarget, installed.Path, previousTarget, err)
|
||||
}
|
||||
units.mu.Lock()
|
||||
startCalls := units.startCalls
|
||||
stopCalls := units.stopCalls
|
||||
unitActiveState := units.unit.ActiveState
|
||||
startedName := units.startedName
|
||||
units.mu.Unlock()
|
||||
if startCalls != 1 || stopCalls != 0 || unitActiveState != activeState || startedName != request.UnitName {
|
||||
t.Fatalf("unexpected systemd calls: start=%d stop=%d state=%s name=%s", startCalls, stopCalls, unitActiveState, startedName)
|
||||
}
|
||||
pending, err := store.PendingSteps(ctx, record.ID)
|
||||
if err != nil || len(pending) != 0 {
|
||||
t.Fatalf("unexpected pending native steps: steps=%+v err=%v", pending, err)
|
||||
}
|
||||
|
||||
if err := executor.Run(ctx, record.ID, request); err != nil {
|
||||
t.Fatalf("repeat executor at switching state: %v", err)
|
||||
}
|
||||
units.mu.Lock()
|
||||
repeatedStartCalls := units.startCalls
|
||||
units.mu.Unlock()
|
||||
if repeatedStartCalls != startCalls {
|
||||
t.Fatalf("switching state repeated systemd start: before=%d after=%d", startCalls, repeatedStartCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorRollsBackSlotAndStopsUnitWhenHealthFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
executor, store, _, units, request, previousTarget := testNativeExecutor(t)
|
||||
executor.checker = &fakeActuatorChecker{
|
||||
waitErr: errors.New("Actuator rejected native backend"),
|
||||
checkReady: false,
|
||||
}
|
||||
record := createNativeTransaction(t, store, "health-failure")
|
||||
|
||||
err := executor.Run(ctx, record.ID, request)
|
||||
if err == nil {
|
||||
t.Fatal("expected native backend health failure")
|
||||
}
|
||||
current, readErr := store.Transaction(ctx, record.ID)
|
||||
if readErr != nil || current.State != transaction.StateRolledBack {
|
||||
t.Fatalf("unexpected compensated transaction: record=%+v err=%v", current, readErr)
|
||||
}
|
||||
actualTarget, linkErr := os.Readlink(request.SlotJarPath)
|
||||
if linkErr != nil || actualTarget != previousTarget {
|
||||
t.Fatalf("native slot was not restored: target=%q previous=%q err=%v", actualTarget, previousTarget, linkErr)
|
||||
}
|
||||
units.mu.Lock()
|
||||
startCalls := units.startCalls
|
||||
stopCalls := units.stopCalls
|
||||
unitActiveState := units.unit.ActiveState
|
||||
units.mu.Unlock()
|
||||
if startCalls != 1 || stopCalls != 1 || unitActiveState != inactiveState {
|
||||
t.Fatalf("unexpected compensated systemd state: start=%d stop=%d state=%s", startCalls, stopCalls, unitActiveState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorResumesPersistedRollback(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
executor, store, _, units, request, previousTarget := testNativeExecutor(t)
|
||||
record := createNativeTransaction(t, store, "resume-rollback")
|
||||
transitionNativeToPrepared(t, store, record.ID)
|
||||
if _, err := executor.prepare(ctx, record.ID, request); err != nil {
|
||||
t.Fatalf("prepare native backend before rollback interruption: %v", err)
|
||||
}
|
||||
if _, err := store.Transition(ctx, record.ID, transaction.StateStarting, "test starting"); err != nil {
|
||||
t.Fatalf("transition native transaction to starting: %v", err)
|
||||
}
|
||||
if err := units.Start(ctx, request.UnitName); err != nil {
|
||||
t.Fatalf("start native backend before rollback interruption: %v", err)
|
||||
}
|
||||
if _, err := store.Transition(ctx, record.ID, transaction.StateRollingBack, "test interrupted rollback"); err != nil {
|
||||
t.Fatalf("persist interrupted rollback state: %v", err)
|
||||
}
|
||||
|
||||
if err := executor.Run(ctx, record.ID, request); err != nil {
|
||||
t.Fatalf("resume native backend rollback: %v", err)
|
||||
}
|
||||
current, err := store.Transaction(ctx, record.ID)
|
||||
if err != nil || current.State != transaction.StateRolledBack {
|
||||
t.Fatalf("unexpected resumed rollback state: record=%+v err=%v", current, err)
|
||||
}
|
||||
actualTarget, err := os.Readlink(request.SlotJarPath)
|
||||
if err != nil || actualTarget != previousTarget {
|
||||
t.Fatalf("resumed rollback did not restore slot: target=%q previous=%q err=%v", actualTarget, previousTarget, err)
|
||||
}
|
||||
units.mu.Lock()
|
||||
stopCalls := units.stopCalls
|
||||
unitActiveState := units.unit.ActiveState
|
||||
units.mu.Unlock()
|
||||
if stopCalls != 1 || unitActiveState != inactiveState {
|
||||
t.Fatalf("resumed rollback did not stop unit: stop=%d state=%s", stopCalls, unitActiveState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorRollbackRemovesFirstDeploymentSlotLink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
executor, store, _, _, request, _ := testNativeExecutor(t)
|
||||
if err := os.Remove(request.SlotJarPath); err != nil {
|
||||
t.Fatalf("remove seeded slot link: %v", err)
|
||||
}
|
||||
request.PreviousSlotTarget = ""
|
||||
executor.checker = &fakeActuatorChecker{
|
||||
waitErr: errors.New("Actuator rejected first native backend deployment"),
|
||||
checkReady: false,
|
||||
}
|
||||
record := createNativeTransaction(t, store, "first-deployment-rollback")
|
||||
|
||||
if err := executor.Run(ctx, record.ID, request); err == nil {
|
||||
t.Fatal("expected first native backend deployment health failure")
|
||||
}
|
||||
current, err := store.Transaction(ctx, record.ID)
|
||||
if err != nil || current.State != transaction.StateRolledBack {
|
||||
t.Fatalf("unexpected first deployment rollback state: record=%+v err=%v", current, err)
|
||||
}
|
||||
if _, err := os.Lstat(request.SlotJarPath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("first deployment rollback retained slot link: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorRecoversRecordedSlotIntentWithoutChangingRequest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
executor, store, releaseStore, units, request, _ := testNativeExecutor(t)
|
||||
record := createNativeTransaction(t, store, "recover-slot")
|
||||
transitionNativeToPrepared(t, store, record.ID)
|
||||
|
||||
installOperation := &installJarOperation{
|
||||
store: releaseStore,
|
||||
sourcePath: request.ArtifactPath,
|
||||
releasePath: request.ReleasePath,
|
||||
identity: request.ArtifactIdentity,
|
||||
}
|
||||
if _, err := executor.coordinator.ExecuteStep(ctx, record.ID, installIntent(request), installOperation); err != nil {
|
||||
t.Fatalf("install backend JAR before simulated crash: %v", err)
|
||||
}
|
||||
installed, found, err := releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("inspect backend JAR before simulated crash: file=%+v found=%t err=%v", installed, found, err)
|
||||
}
|
||||
if _, _, err := store.RecordStepIntent(ctx, record.ID, bindIntent(request, installed.Path)); err != nil {
|
||||
t.Fatalf("record slot intent before simulated crash: %v", err)
|
||||
}
|
||||
|
||||
if err := executor.Run(ctx, record.ID, request); err != nil {
|
||||
t.Fatalf("recover native backend executor: %v", err)
|
||||
}
|
||||
actualTarget, err := os.Readlink(request.SlotJarPath)
|
||||
if err != nil || actualTarget != installed.Path {
|
||||
t.Fatalf("unexpected recovered slot link: target=%q installed=%q err=%v", actualTarget, installed.Path, err)
|
||||
}
|
||||
units.mu.Lock()
|
||||
startCalls := units.startCalls
|
||||
units.mu.Unlock()
|
||||
if startCalls != 1 {
|
||||
t.Fatalf("unexpected recovered systemd start count: %d", startCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorRejectsChangedRecoveryIntentAndPreservesStartingState(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
executor, store, _, _, request, _ := testNativeExecutor(t)
|
||||
record := createNativeTransaction(t, store, "changed-request")
|
||||
transitionNativeToPrepared(t, store, record.ID)
|
||||
if _, err := executor.prepare(ctx, record.ID, request); err != nil {
|
||||
t.Fatalf("prepare original native backend request: %v", err)
|
||||
}
|
||||
if _, err := store.Transition(ctx, record.ID, transaction.StateStarting, "test starting"); err != nil {
|
||||
t.Fatalf("transition native backend to starting: %v", err)
|
||||
}
|
||||
|
||||
changed := request
|
||||
changed.ReleasePath = "different-release.jar"
|
||||
err := executor.Run(ctx, record.ID, changed)
|
||||
if !errors.Is(err, transaction.ErrStepConflict) {
|
||||
t.Fatalf("expected persisted native intent conflict, got %v", err)
|
||||
}
|
||||
current, readErr := store.Transaction(ctx, record.ID)
|
||||
if readErr != nil || current.State != transaction.StateStarting {
|
||||
t.Fatalf("changed recovery request altered transaction: record=%+v err=%v", current, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorRejectsActiveUnitBeforeChangingFiles(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
executor, store, releaseStore, units, request, previousTarget := testNativeExecutor(t)
|
||||
units.mu.Lock()
|
||||
units.unit.ActiveState = activeState
|
||||
units.unit.SubState = "running"
|
||||
units.mu.Unlock()
|
||||
record := createNativeTransaction(t, store, "active-unit")
|
||||
|
||||
if err := executor.Run(ctx, record.ID, request); err == nil {
|
||||
t.Fatal("expected active unit validation failure")
|
||||
}
|
||||
current, err := store.Transaction(ctx, record.ID)
|
||||
if err != nil || current.State != transaction.StateFailed {
|
||||
t.Fatalf("unexpected active-unit transaction state: record=%+v err=%v", current, err)
|
||||
}
|
||||
if _, found, err := releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity); err != nil || found {
|
||||
t.Fatalf("validation failure changed release store: found=%t err=%v", found, err)
|
||||
}
|
||||
actualTarget, err := os.Readlink(request.SlotJarPath)
|
||||
if err != nil || actualTarget != previousTarget {
|
||||
t.Fatalf("validation failure changed slot link: target=%q previous=%q err=%v", actualTarget, previousTarget, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnitStopOperationAcceptsSystemdFailedAsStopped(t *testing.T) {
|
||||
units := &fakeUnitManager{unit: systemd.Unit{
|
||||
Name: "yms-backend@8080.service",
|
||||
LoadState: "loaded",
|
||||
ActiveState: failedState,
|
||||
SubState: "failed",
|
||||
}}
|
||||
operation := &unitStopOperation{units: units, name: units.unit.Name}
|
||||
inspection, err := operation.Inspect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("inspect stopped native backend unit: %v", err)
|
||||
}
|
||||
if inspection.Status != transaction.InspectionApplied {
|
||||
t.Fatalf("unexpected stopped native backend unit inspection: %+v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func testNativeExecutor(t *testing.T) (*Executor, *transaction.Store, *filestore.Store, *fakeUnitManager, Request, string) {
|
||||
t.Helper()
|
||||
store, err := transaction.OpenStore(context.Background(), filepath.Join(t.TempDir(), "transactions.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open transaction store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = store.Close() })
|
||||
coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
if err != nil {
|
||||
t.Fatalf("create transaction coordinator: %v", err)
|
||||
}
|
||||
releaseStore, err := filestore.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("create native release store: %v", err)
|
||||
}
|
||||
artifactContent := []byte("native backend jar content")
|
||||
artifactPath := filepath.Join(t.TempDir(), "glory-soft-yms.jar")
|
||||
if err := os.WriteFile(artifactPath, artifactContent, 0o640); err != nil {
|
||||
t.Fatalf("write native backend artifact: %v", err)
|
||||
}
|
||||
previousTarget := filepath.Join(t.TempDir(), "previous-backend.jar")
|
||||
if err := os.WriteFile(previousTarget, []byte("previous backend jar"), 0o640); err != nil {
|
||||
t.Fatalf("write previous backend JAR: %v", err)
|
||||
}
|
||||
slotDirectory := t.TempDir()
|
||||
slotJarPath := filepath.Join(slotDirectory, "backend-green.jar")
|
||||
if err := os.Symlink(previousTarget, slotJarPath); err != nil {
|
||||
t.Fatalf("create previous native backend slot link: %v", err)
|
||||
}
|
||||
request := Request{
|
||||
ArtifactPath: artifactPath,
|
||||
ArtifactIdentity: testIdentity(artifactContent),
|
||||
ReleasePath: "glory-soft-yms-20260815.jar",
|
||||
SlotJarPath: slotJarPath,
|
||||
PreviousSlotTarget: previousTarget,
|
||||
UnitName: "yms-green.service",
|
||||
Port: 8081,
|
||||
HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health",
|
||||
}
|
||||
units := &fakeUnitManager{unit: systemd.Unit{
|
||||
Name: request.UnitName,
|
||||
LoadState: "loaded",
|
||||
ActiveState: inactiveState,
|
||||
SubState: "dead",
|
||||
}}
|
||||
executor, err := New(store, coordinator, releaseStore, units, &http.Client{})
|
||||
if err != nil {
|
||||
t.Fatalf("create native backend executor: %v", err)
|
||||
}
|
||||
executor.checker = &fakeActuatorChecker{
|
||||
waitReport: healthcheck.ActuatorReport{Status: "UP", Components: map[string]string{"db": "UP"}},
|
||||
checkReport: healthcheck.ActuatorReport{Status: "UP", Components: map[string]string{"db": "UP"}},
|
||||
checkReady: true,
|
||||
}
|
||||
return executor, store, releaseStore, units, request, previousTarget
|
||||
}
|
||||
|
||||
func createNativeTransaction(t *testing.T, store *transaction.Store, suffix string) transaction.Transaction {
|
||||
t.Helper()
|
||||
record, _, err := store.CreateTransaction(context.Background(), transaction.CreateRequest{
|
||||
ID: "native-backend-" + suffix,
|
||||
IdempotencyKey: "native-backend-request-" + suffix,
|
||||
Source: "test",
|
||||
Service: "backend",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create native backend transaction: %v", err)
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func transitionNativeToPrepared(t *testing.T, store *transaction.Store, transactionID string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if _, err := store.Transition(ctx, transactionID, transaction.StateValidating, "test validating"); err != nil {
|
||||
t.Fatalf("transition native transaction to validating: %v", err)
|
||||
}
|
||||
if _, err := store.Transition(ctx, transactionID, transaction.StatePrepared, "test prepared"); err != nil {
|
||||
t.Fatalf("transition native transaction to prepared: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testIdentity(content []byte) filestore.Identity {
|
||||
digest := sha256.Sum256(content)
|
||||
return filestore.Identity{Size: int64(len(content)), SHA256: hex.EncodeToString(digest[:])}
|
||||
}
|
||||
|
||||
type fakeUnitManager struct {
|
||||
mu sync.Mutex
|
||||
unit systemd.Unit
|
||||
startErr error
|
||||
stopErr error
|
||||
startCalls int
|
||||
stopCalls int
|
||||
startedName string
|
||||
}
|
||||
|
||||
func (m *fakeUnitManager) Inspect(_ context.Context, name string) (systemd.Unit, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if name != m.unit.Name {
|
||||
return systemd.Unit{}, systemd.ErrUnitNotFound
|
||||
}
|
||||
return m.unit, nil
|
||||
}
|
||||
|
||||
func (m *fakeUnitManager) Start(_ context.Context, name string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.startCalls++
|
||||
m.startedName = name
|
||||
if m.startErr != nil {
|
||||
m.unit.ActiveState = failedState
|
||||
m.unit.SubState = "failed"
|
||||
return m.startErr
|
||||
}
|
||||
m.unit.ActiveState = activeState
|
||||
m.unit.SubState = "running"
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *fakeUnitManager) Stop(context.Context, string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.stopCalls++
|
||||
if m.stopErr != nil {
|
||||
return m.stopErr
|
||||
}
|
||||
m.unit.ActiveState = inactiveState
|
||||
m.unit.SubState = "dead"
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeActuatorChecker struct {
|
||||
waitReport healthcheck.ActuatorReport
|
||||
waitErr error
|
||||
checkReport healthcheck.ActuatorReport
|
||||
checkReady bool
|
||||
checkErr error
|
||||
}
|
||||
|
||||
func (c *fakeActuatorChecker) Wait(ctx context.Context, _ string, _ time.Duration, running healthcheck.RunningProbe) (healthcheck.ActuatorReport, error) {
|
||||
isRunning, err := running(ctx)
|
||||
if err != nil {
|
||||
return healthcheck.ActuatorReport{}, err
|
||||
}
|
||||
if !isRunning {
|
||||
return healthcheck.ActuatorReport{}, healthcheck.ErrWorkloadStopped
|
||||
}
|
||||
return c.waitReport, c.waitErr
|
||||
}
|
||||
|
||||
func (c *fakeActuatorChecker) Check(ctx context.Context, _ string, running healthcheck.RunningProbe) (healthcheck.ActuatorReport, bool, error) {
|
||||
isRunning, err := running(ctx)
|
||||
if err != nil {
|
||||
return healthcheck.ActuatorReport{}, false, err
|
||||
}
|
||||
if !isRunning {
|
||||
return healthcheck.ActuatorReport{}, false, healthcheck.ErrWorkloadStopped
|
||||
}
|
||||
return c.checkReport, c.checkReady, c.checkErr
|
||||
}
|
||||
|
||||
var _ systemd.Manager = (*fakeUnitManager)(nil)
|
||||
var _ actuatorChecker = (*fakeActuatorChecker)(nil)
|
||||
Reference in New Issue
Block a user