// Package backendupdate orchestrates one native backend update through commit or compensation. package backendupdate import ( "bytes" "context" "crypto/rand" "encoding/json" "errors" "fmt" "log/slog" "net/http" "os" "path/filepath" "time" "yms-daemon/internal/backendexecutor" "yms-daemon/internal/containerengine" "yms-daemon/internal/daemonapi" "yms-daemon/internal/deploymentconfig" "yms-daemon/internal/filestore" "yms-daemon/internal/hostnginx" "yms-daemon/internal/nativebackendexecutor" "yms-daemon/internal/systemd" "yms-daemon/internal/transaction" "yms-daemon/internal/updatepackage" ) const ( serviceBackend = "backend" sourceLocalCLI = "local-cli" drainDuration = 5 * time.Second directReleaseDigestLength = 12 inputTypeCurrentRelease = "current-native-release" legacyUnit8080 = "yms.service" legacyUnit8081 = "ymsback.service" ) // Updater executes the current native backend contract on one server. type Updater struct { config deploymentconfig.Config workRoot string store *transaction.Store coordinator *transaction.Coordinator releaseStore *filestore.Store units systemd.Manager gateway gatewayController executor nativeExecutor containerExecutor containerExecutor engine containerengine.Engine containerConfigSource string containerConfigTarget string containerTmpSource string containerTmpTarget string logger *slog.Logger drain time.Duration } type gatewayController interface { Read() (hostnginx.Snapshot, error) Apply(context.Context, hostnginx.Snapshot) error } type nativeExecutor interface { Run(context.Context, string, nativebackendexecutor.Request) error } type containerExecutor interface { Run(context.Context, string, backendexecutor.Request) error } // New creates the complete native backend update orchestrator. func New( config deploymentconfig.Config, workRoot string, store *transaction.Store, coordinator *transaction.Coordinator, releaseStore *filestore.Store, units systemd.Manager, gateway gatewayController, httpClient *http.Client, logger *slog.Logger, ) (*Updater, error) { if err := config.Validate(); err != nil { return nil, err } if config.Backend.Type != deploymentconfig.BackendTypeNative { return nil, errors.New("backend.type must be native") } if !filepath.IsAbs(workRoot) || filepath.Clean(workRoot) != workRoot { return nil, errors.New("backend update work root must be a clean absolute path") } if store == nil || coordinator == nil || releaseStore == nil || units == nil || gateway == nil { return nil, errors.New("backend update dependencies are required") } if logger == nil { logger = slog.Default() } executor, err := nativebackendexecutor.New(store, coordinator, releaseStore, units, httpClient) if err != nil { return nil, err } return &Updater{ config: config, workRoot: workRoot, store: store, coordinator: coordinator, releaseStore: releaseStore, units: units, gateway: gateway, executor: executor, logger: logger, drain: drainDuration, }, nil } // NewContainer creates the Docker standalone backend update orchestrator. func NewContainer( config deploymentconfig.Config, workRoot string, store *transaction.Store, coordinator *transaction.Coordinator, engine containerengine.Engine, gateway gatewayController, httpClient *http.Client, logger *slog.Logger, ) (*Updater, error) { if err := config.Validate(); err != nil { return nil, err } if config.Backend.Type != deploymentconfig.BackendTypeContainer { return nil, errors.New("backend.type must be container") } if config.Daemon.Environment != deploymentconfig.EnvironmentDev { return nil, errors.New("--container-image requires daemon.environment = dev") } if !filepath.IsAbs(workRoot) || filepath.Clean(workRoot) != workRoot { return nil, errors.New("container backend update work root must be a clean absolute path") } if store == nil || coordinator == nil || engine == nil || gateway == nil { return nil, errors.New("container backend update dependencies are required") } if logger == nil { logger = slog.Default() } executor, err := backendexecutor.New(store, coordinator, engine, httpClient) if err != nil { return nil, err } return &Updater{ config: config, workRoot: workRoot, store: store, coordinator: coordinator, gateway: gateway, containerExecutor: executor, engine: engine, logger: logger, drain: drainDuration, containerConfigSource: deploymentconfig.ContainerConfigSource, containerConfigTarget: deploymentconfig.ContainerConfigTarget, containerTmpSource: deploymentconfig.ContainerTmpSource, containerTmpTarget: deploymentconfig.ContainerTmpTarget, }, nil } // UpdateRepack applies one repack ZIP selected by an absolute local path. func (u *Updater) UpdateRepack(ctx context.Context, packagePath string, report ProgressReporter) (transaction.Transaction, error) { if u.config.Backend.Type == deploymentconfig.BackendTypeContainer { return transaction.Transaction{}, errors.New("repack ZIP update is not implemented for container backend") } reportProgress(report, Progress{Message: "Validating repack ZIP"}) updatePackage, err := updatepackage.OpenBackendNative(packagePath) if err != nil { return transaction.Transaction{}, err } defer updatePackage.Close() reportProgress(report, Progress{Message: "Repack ZIP validated"}) return u.update(ctx, updateInput{ IdempotencyKey: serviceBackend + ":" + updatePackage.PackageSHA256, InputType: daemonapi.InputTypeRepackZIP, SourcePath: updatePackage.PackagePath, SourceSHA256: updatePackage.PackageSHA256, CustomerCode: updatePackage.CustomerCode, VersionID: updatePackage.VersionID, ArtifactID: updatePackage.ArtifactID, ArtifactFileName: updatePackage.FileName, ArtifactIdentity: updatePackage.Identity, ReleasePath: updatePackage.FileName, Materialize: updatePackage.ExtractArtifact, }, report) } // UpdateNativeJAR applies one JAR copied directly to the server. func (u *Updater) UpdateNativeJAR(ctx context.Context, jarPath string, report ProgressReporter) (transaction.Transaction, error) { if u.config.Backend.Type == deploymentconfig.BackendTypeContainer { return transaction.Transaction{}, errors.New("--native-jar requires backend.type = native") } reportProgress(report, Progress{Message: "Validating direct native backend JAR and computing SHA-256"}) jar, err := updatepackage.OpenDirectNativeJAR(jarPath) if err != nil { return transaction.Transaction{}, err } reportProgress(report, Progress{Message: "Direct native backend JAR validated: sha256=" + jar.SHA256}) return u.update(ctx, updateInput{ IdempotencyKey: serviceBackend + ":" + jar.SHA256, InputType: daemonapi.InputTypeNativeJAR, SourcePath: jar.Path, SourceSHA256: jar.SHA256, ArtifactFileName: jar.FileName, ArtifactIdentity: jar.Identity, ReleasePath: filepath.Join("direct", jar.SHA256[:directReleaseDigestLength], jar.FileName), Materialize: jar.CopyArtifact, }, report) } func (u *Updater) update(ctx context.Context, input updateInput, report ProgressReporter) (transaction.Transaction, error) { existing, request, created, err := u.createOrResume(ctx, input) if err != nil { return transaction.Transaction{}, err } operation := operationLabel(input.InputType) transactionMessage := "Resuming backend " + operation + " transaction" if created { transactionMessage = "Created backend " + operation + " transaction" } transactionMessage += " " + existing.ID reportProgress(report, Progress{TransactionID: existing.ID, State: existing.State, Message: transactionMessage}) if !created && existing.State.Terminal() { return terminalResult(existing) } if err := os.MkdirAll(filepath.Dir(request.ArtifactPath), 0o750); err != nil { return u.fail(ctx, existing.ID, fmt.Errorf("create backend transaction work directory: %w", err)) } reportProgress(report, Progress{TransactionID: existing.ID, State: existing.State, Message: "Staging backend artifact in transaction workspace"}) if err := ensureTransactionArtifact(input, request); err != nil { return u.fail(ctx, existing.ID, err) } executorRequest := nativebackendexecutor.Request{ ArtifactPath: request.ArtifactPath, ArtifactIdentity: request.ArtifactIdentity, ReleasePath: request.ReleasePath, SlotJarPath: request.TargetSlotJAR, PreviousSlotTarget: request.PreviousSlotTarget, UnitName: request.TargetUnit, Port: request.TargetPort, HealthEndpoint: request.TargetHealthEndpoint, Progress: func(state transaction.State, message string) { reportProgress(report, Progress{TransactionID: existing.ID, State: state, Message: message}) }, } switch existing.State { case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting: if err := u.executor.Run(ctx, existing.ID, executorRequest); err != nil { return u.currentWithError(ctx, existing.ID, err) } case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining: case transaction.StateRollingBack: if err := u.executor.Run(ctx, existing.ID, executorRequest); err != nil { return u.currentWithError(ctx, existing.ID, err) } default: return u.currentWithError(ctx, existing.ID, fmt.Errorf("backend update cannot resume transaction %s in state %s", existing.ID, existing.State)) } current, err := u.store.Transaction(ctx, existing.ID) if err != nil { return transaction.Transaction{}, err } if current.State.Terminal() { return terminalResult(current) } if err := u.switchAndCommit(ctx, existing.ID, request, report); err != nil { return u.currentWithError(ctx, existing.ID, err) } return u.store.Transaction(ctx, existing.ID) } func terminalResult(record transaction.Transaction) (transaction.Transaction, error) { if record.State == transaction.StateCommitted { return record, nil } return record, fmt.Errorf("backend update transaction %s is terminal in state %s", record.ID, record.State) } func (u *Updater) createOrResume(ctx context.Context, input updateInput) (transaction.Transaction, persistedRequest, bool, error) { gatewayBefore, err := u.gateway.Read() if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } targetPort := otherPort(gatewayBefore.ActivePort) targetSlot, err := u.config.Backend.SlotForPort(targetPort) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } currentSlot, err := u.config.Backend.SlotForPort(gatewayBefore.ActivePort) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } previousUnit, err := u.resolveCurrentUnit(ctx, gatewayBefore.ActivePort, currentSlot.Unit) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } previousSlotTarget, err := readOptionalSymlink(targetSlot.JAR) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } transactionID := rand.Text() transactionRoot := filepath.Join(u.workRoot, transactionID) if err := os.MkdirAll(transactionRoot, 0o750); err != nil { return transaction.Transaction{}, persistedRequest{}, false, fmt.Errorf("create backend transaction directory: %w", err) } gatewayAfterContent, err := hostnginx.RenderBackendPort(gatewayBefore.Content, targetPort) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } gatewayBeforePath := filepath.Join(transactionRoot, "gateway.before.conf") gatewayAfterPath := filepath.Join(transactionRoot, "gateway.after.conf") if err := writeImmutableFile(gatewayBeforePath, gatewayBefore.Content, 0o640); err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } if err := writeImmutableFile(gatewayAfterPath, gatewayAfterContent, 0o640); err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } activeBefore, err := snapshotPath(u.config.Backend.ActiveJAR, filepath.Join(transactionRoot, "active-jar.before")) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } request := persistedRequest{ InputType: input.InputType, SourcePath: input.SourcePath, SourceSHA256: input.SourceSHA256, CustomerCode: input.CustomerCode, VersionID: input.VersionID, ArtifactID: input.ArtifactID, ArtifactFileName: input.ArtifactFileName, ArtifactPath: filepath.Join(transactionRoot, "backend.jar"), ArtifactIdentity: input.ArtifactIdentity, ReleasePath: input.ReleasePath, TargetPort: targetPort, TargetUnit: targetSlot.Unit, TargetSlotJAR: targetSlot.JAR, TargetHealthEndpoint: targetSlot.HealthEndpoint, PreviousSlotTarget: previousSlotTarget, PreviousGatewayPort: gatewayBefore.ActivePort, PreviousUnit: previousUnit, GatewayBeforePath: gatewayBeforePath, GatewayAfterPath: gatewayAfterPath, GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"), ActiveJARPath: u.config.Backend.ActiveJAR, ActiveJARBefore: activeBefore, } requestJSON, err := json.Marshal(request) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, fmt.Errorf("encode backend update request: %w", err) } record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{ ID: transactionID, IdempotencyKey: input.IdempotencyKey, Source: sourceLocalCLI, Service: serviceBackend, Request: requestJSON, }) if err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } if created { return record, request, true, nil } var persisted persistedRequest if err := decodePersistedRequest(record.Request, &persisted); err != nil { return transaction.Transaction{}, persistedRequest{}, false, err } if persisted.InputType != input.InputType || persisted.SourceSHA256 != input.SourceSHA256 { return transaction.Transaction{}, persistedRequest{}, false, errors.New("persisted backend transaction input identity mismatch") } return record, persisted, false, nil } func (u *Updater) switchAndCommit(ctx context.Context, transactionID string, request persistedRequest, report ProgressReporter) error { operation := operationLabel(request.InputType) before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousGatewayPort) if err != nil { return err } after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort) if err != nil { return err } for { record, err := u.store.Transaction(ctx, transactionID) if err != nil { return err } switch record.State { case transaction.StateSwitching: reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)}) gatewayOperation := &gatewayOperation{ controller: u.gateway, before: before, after: after, receiptPath: request.GatewayReceiptPath, } if _, err := u.coordinator.ExecuteStep(ctx, transactionID, gatewaySwitchIntent(request, before, after), gatewayOperation); err != nil { return u.rollbackAfterPreparation(ctx, transactionID, request, before, after, err) } if _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy native slot"); err != nil { return err } case transaction.StateVerifying: reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Updating the active compatibility JAR link"}) installedPath := filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath) activeOperation := &pathOperation{ path: request.ActiveJARPath, before: request.ActiveJARBefore, desired: pathState{Kind: pathKindSymlink, Target: installedPath}, } if _, err := u.coordinator.ExecuteStep(ctx, transactionID, activeLinkIntent(request, installedPath), activeOperation); err != nil { return u.rollbackAfterPreparation(ctx, transactionID, request, before, after, err) } if _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, "backend compatibility link committed; previous unit draining"); err != nil { return err } case transaction.StateDraining: reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Draining previous backend unit for %s", u.drain)}) if err := waitContext(ctx, u.drain); err != nil { return err } reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Stopping previous backend unit " + request.PreviousUnit}) stopOperation := &unitStopOperation{units: u.units, name: request.PreviousUnit} if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousUnitIntent(request), stopOperation); err != nil { return err } _, err = u.store.Transition(ctx, transactionID, transaction.StateCommitted, "native backend "+operation+" committed") if err == nil { reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Native backend " + operation + " committed"}) } return err case transaction.StateCommitted: reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Native backend " + operation + " already committed"}) return nil default: return fmt.Errorf("backend commit cannot continue transaction %s in state %s", transactionID, record.State) } } } func reportProgress(report ProgressReporter, progress Progress) { if report != nil { report(progress) } } func (u *Updater) rollbackAfterPreparation(ctx context.Context, transactionID string, request persistedRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error { record, readErr := u.store.Transaction(ctx, transactionID) if readErr != nil { return errors.Join(cause, readErr) } if record.State != transaction.StateRollingBack { if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "native backend post-start compensation started"); err != nil { return errors.Join(cause, err) } } activeRestore := &pathOperation{ path: request.ActiveJARPath, before: pathState{Kind: pathKindSymlink, Target: filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath)}, desired: request.ActiveJARBefore, } _, activeErr := u.coordinator.ExecuteStep(ctx, transactionID, activeLinkRestoreIntent(request), activeRestore) gatewayRestore := &gatewayOperation{ controller: u.gateway, before: after, after: before, receiptPath: request.GatewayReceiptPath + ".restore", } _, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, gatewayRestoreIntent(request), gatewayRestore) stopTarget := &unitStopOperation{units: u.units, name: request.TargetUnit} _, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetUnitIntent(request), stopTarget) installedPath := filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath) previousSlotState := pathState{Kind: pathKindAbsent} if request.PreviousSlotTarget != "" { previousSlotState = pathState{Kind: pathKindSymlink, Target: request.PreviousSlotTarget} } slotRestore := &pathOperation{ path: request.TargetSlotJAR, before: pathState{Kind: pathKindSymlink, Target: installedPath}, desired: previousSlotState, } _, slotErr := u.coordinator.ExecuteStep(ctx, transactionID, restoreTargetSlotIntent(request, installedPath), slotRestore) if err := errors.Join(activeErr, gatewayErr, stopErr, slotErr); err != nil { return errors.Join(cause, err) } _, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "native backend post-start compensation completed") return errors.Join(cause, transitionErr) } func (u *Updater) resolveCurrentUnit(ctx context.Context, port int, configuredUnit string) (string, error) { legacyUnit, err := legacyUnitForPort(port) if err != nil { return "", err } configured, err := u.units.Inspect(ctx, configuredUnit) if err != nil { return "", fmt.Errorf("inspect configured active-port unit %s: %w", configuredUnit, err) } legacy, err := u.units.Inspect(ctx, legacyUnit) if err != nil { return "", fmt.Errorf("inspect legacy active-port unit %s: %w", legacyUnit, err) } configuredRunning := unitRunning(configured) legacyRunning := unitRunning(legacy) if configuredRunning == legacyRunning { return "", fmt.Errorf("backend port %d requires exactly one running unit, configured=%s(%s), legacy=%s(%s)", port, configuredUnit, configured.ActiveState, legacyUnit, legacy.ActiveState) } if configuredRunning { return configuredUnit, nil } return legacyUnit, nil } func (u *Updater) fail(ctx context.Context, transactionID string, cause error) (transaction.Transaction, error) { _, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateFailed, cause.Error()) return u.currentWithError(ctx, transactionID, errors.Join(cause, transitionErr)) } func (u *Updater) currentWithError(ctx context.Context, transactionID string, cause error) (transaction.Transaction, error) { record, err := u.store.Transaction(ctx, transactionID) return record, errors.Join(cause, err) } func otherPort(port int) int { if port == deploymentconfig.BackendPort8080 { return deploymentconfig.BackendPort8081 } return deploymentconfig.BackendPort8080 } func legacyUnitForPort(port int) (string, error) { switch port { case deploymentconfig.BackendPort8080: return legacyUnit8080, nil case deploymentconfig.BackendPort8081: return legacyUnit8081, nil default: return "", fmt.Errorf("unsupported legacy backend port: %d", port) } } func unitRunning(unit systemd.Unit) bool { return unit.ActiveState != "inactive" && unit.ActiveState != "failed" } func readOptionalSymlink(path string) (string, error) { info, err := os.Lstat(path) if errors.Is(err, os.ErrNotExist) { return "", nil } if err != nil { return "", fmt.Errorf("inspect native backend target slot %s: %w", path, err) } if info.Mode()&os.ModeSymlink == 0 { return "", fmt.Errorf("native backend target slot is not a symbolic link: %s", path) } target, err := os.Readlink(path) if err != nil { return "", fmt.Errorf("read native backend target slot %s: %w", path, err) } return target, nil } func ensureTransactionArtifact(input updateInput, request persistedRequest) error { info, err := os.Lstat(request.ArtifactPath) if errors.Is(err, os.ErrNotExist) { return input.Materialize(request.ArtifactPath) } if err != nil { return fmt.Errorf("inspect extracted native backend artifact: %w", err) } if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { return errors.New("extracted native backend artifact is not a direct regular file") } return verifyFileIdentity(request.ArtifactPath, request.ArtifactIdentity) } func decodePersistedRequest(content json.RawMessage, request *persistedRequest) error { decoder := json.NewDecoder(bytes.NewReader(content)) decoder.DisallowUnknownFields() if err := decoder.Decode(request); err != nil { return fmt.Errorf("decode persisted backend update request: %w", err) } return nil } func waitContext(ctx context.Context, duration time.Duration) error { timer := time.NewTimer(duration) defer timer.Stop() select { case <-ctx.Done(): return ctx.Err() case <-timer.C: return nil } }