Files
yms-daemon/internal/backendupdate/updater.go
T

600 lines
23 KiB
Go

// Package backendupdate orchestrates one native backend update through commit or compensation.
package backendupdate
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"path/filepath"
"time"
"yms-daemon/internal/backendexecutor"
"yms-daemon/internal/containerengine"
"yms-daemon/internal/daemonapi"
"yms-daemon/internal/deploymentconfig"
"yms-daemon/internal/filestore"
"yms-daemon/internal/hostnginx"
"yms-daemon/internal/nativebackendexecutor"
"yms-daemon/internal/systemd"
"yms-daemon/internal/transaction"
"yms-daemon/internal/updatepackage"
)
const (
serviceBackend = "backend"
sourceLocalCLI = "local-cli"
drainDuration = 5 * time.Second
directReleaseDigestLength = 12
inputTypeCurrentRelease = "current-native-release"
legacyUnit8080 = "yms.service"
legacyUnit8081 = "ymsback.service"
)
// Updater executes the current native backend contract on one server.
type Updater struct {
config deploymentconfig.Config
workRoot string
store *transaction.Store
coordinator *transaction.Coordinator
releaseStore *filestore.Store
units systemd.Manager
gateway gatewayController
executor nativeExecutor
containerExecutor containerExecutor
engine containerengine.Engine
containerConfigSource string
containerConfigTarget string
containerTmpSource string
containerTmpTarget string
logger *slog.Logger
drain time.Duration
}
type gatewayController interface {
Read() (hostnginx.Snapshot, error)
Apply(context.Context, hostnginx.Snapshot) error
}
type nativeExecutor interface {
Run(context.Context, string, nativebackendexecutor.Request) error
}
type containerExecutor interface {
Run(context.Context, string, backendexecutor.Request) error
}
// New creates the complete native backend update orchestrator.
func New(
config deploymentconfig.Config,
workRoot string,
store *transaction.Store,
coordinator *transaction.Coordinator,
releaseStore *filestore.Store,
units systemd.Manager,
gateway gatewayController,
httpClient *http.Client,
logger *slog.Logger,
) (*Updater, error) {
if err := config.Validate(); err != nil {
return nil, err
}
if config.Backend.Type != deploymentconfig.BackendTypeNative {
return nil, errors.New("backend.type must be native")
}
if !filepath.IsAbs(workRoot) || filepath.Clean(workRoot) != workRoot {
return nil, errors.New("backend update work root must be a clean absolute path")
}
if store == nil || coordinator == nil || releaseStore == nil || units == nil || gateway == nil {
return nil, errors.New("backend update dependencies are required")
}
if logger == nil {
logger = slog.Default()
}
executor, err := nativebackendexecutor.New(store, coordinator, releaseStore, units, httpClient)
if err != nil {
return nil, err
}
return &Updater{
config: config,
workRoot: workRoot,
store: store,
coordinator: coordinator,
releaseStore: releaseStore,
units: units,
gateway: gateway,
executor: executor,
logger: logger,
drain: drainDuration,
}, nil
}
// NewContainer creates the Docker standalone backend update orchestrator.
func NewContainer(
config deploymentconfig.Config,
workRoot string,
store *transaction.Store,
coordinator *transaction.Coordinator,
engine containerengine.Engine,
gateway gatewayController,
httpClient *http.Client,
logger *slog.Logger,
) (*Updater, error) {
if err := config.Validate(); err != nil {
return nil, err
}
if config.Backend.Type != deploymentconfig.BackendTypeContainer {
return nil, errors.New("backend.type must be container")
}
if config.Daemon.Environment != deploymentconfig.EnvironmentDev {
return nil, errors.New("--container-image requires daemon.environment = dev")
}
if !filepath.IsAbs(workRoot) || filepath.Clean(workRoot) != workRoot {
return nil, errors.New("container backend update work root must be a clean absolute path")
}
if store == nil || coordinator == nil || engine == nil || gateway == nil {
return nil, errors.New("container backend update dependencies are required")
}
if logger == nil {
logger = slog.Default()
}
executor, err := backendexecutor.New(store, coordinator, engine, httpClient)
if err != nil {
return nil, err
}
return &Updater{
config: config, workRoot: workRoot, store: store, coordinator: coordinator, gateway: gateway,
containerExecutor: executor, engine: engine, logger: logger, drain: drainDuration,
containerConfigSource: deploymentconfig.ContainerConfigSource,
containerConfigTarget: deploymentconfig.ContainerConfigTarget,
containerTmpSource: deploymentconfig.ContainerTmpSource,
containerTmpTarget: deploymentconfig.ContainerTmpTarget,
}, nil
}
// UpdateRepack applies one repack ZIP selected by an absolute local path.
func (u *Updater) UpdateRepack(ctx context.Context, packagePath string, report ProgressReporter) (transaction.Transaction, error) {
if u.config.Backend.Type == deploymentconfig.BackendTypeContainer {
return transaction.Transaction{}, errors.New("repack ZIP update is not implemented for container backend")
}
reportProgress(report, Progress{Message: "Validating repack ZIP"})
updatePackage, err := updatepackage.OpenBackendNative(packagePath)
if err != nil {
return transaction.Transaction{}, err
}
defer updatePackage.Close()
reportProgress(report, Progress{Message: "Repack ZIP validated"})
return u.update(ctx, updateInput{
IdempotencyKey: serviceBackend + ":" + updatePackage.PackageSHA256,
InputType: daemonapi.InputTypeRepackZIP,
SourcePath: updatePackage.PackagePath,
SourceSHA256: updatePackage.PackageSHA256,
CustomerCode: updatePackage.CustomerCode,
VersionID: updatePackage.VersionID,
ArtifactID: updatePackage.ArtifactID,
ArtifactFileName: updatePackage.FileName,
ArtifactIdentity: updatePackage.Identity,
ReleasePath: updatePackage.FileName,
Materialize: updatePackage.ExtractArtifact,
}, report)
}
// UpdateNativeJAR applies one JAR copied directly to the server.
func (u *Updater) UpdateNativeJAR(ctx context.Context, jarPath string, report ProgressReporter) (transaction.Transaction, error) {
if u.config.Backend.Type == deploymentconfig.BackendTypeContainer {
return transaction.Transaction{}, errors.New("--native-jar requires backend.type = native")
}
reportProgress(report, Progress{Message: "Validating direct native backend JAR and computing SHA-256"})
jar, err := updatepackage.OpenDirectNativeJAR(jarPath)
if err != nil {
return transaction.Transaction{}, err
}
reportProgress(report, Progress{Message: "Direct native backend JAR validated: sha256=" + jar.SHA256})
return u.update(ctx, updateInput{
IdempotencyKey: serviceBackend + ":" + jar.SHA256,
InputType: daemonapi.InputTypeNativeJAR,
SourcePath: jar.Path,
SourceSHA256: jar.SHA256,
ArtifactFileName: jar.FileName,
ArtifactIdentity: jar.Identity,
ReleasePath: filepath.Join("direct", jar.SHA256[:directReleaseDigestLength], jar.FileName),
Materialize: jar.CopyArtifact,
}, report)
}
func (u *Updater) update(ctx context.Context, input updateInput, report ProgressReporter) (transaction.Transaction, error) {
existing, request, created, err := u.createOrResume(ctx, input)
if err != nil {
return transaction.Transaction{}, err
}
operation := operationLabel(input.InputType)
transactionMessage := "Resuming backend " + operation + " transaction"
if created {
transactionMessage = "Created backend " + operation + " transaction"
}
transactionMessage += " " + existing.ID
reportProgress(report, Progress{TransactionID: existing.ID, State: existing.State, Message: transactionMessage})
if !created && existing.State.Terminal() {
return terminalResult(existing)
}
if err := os.MkdirAll(filepath.Dir(request.ArtifactPath), 0o750); err != nil {
return u.fail(ctx, existing.ID, fmt.Errorf("create backend transaction work directory: %w", err))
}
reportProgress(report, Progress{TransactionID: existing.ID, State: existing.State, Message: "Staging backend artifact in transaction workspace"})
if err := ensureTransactionArtifact(input, request); err != nil {
return u.fail(ctx, existing.ID, err)
}
executorRequest := nativebackendexecutor.Request{
ArtifactPath: request.ArtifactPath,
ArtifactIdentity: request.ArtifactIdentity,
ReleasePath: request.ReleasePath,
SlotJarPath: request.TargetSlotJAR,
PreviousSlotTarget: request.PreviousSlotTarget,
UnitName: request.TargetUnit,
Port: request.TargetPort,
HealthEndpoint: request.TargetHealthEndpoint,
Progress: func(state transaction.State, message string) {
reportProgress(report, Progress{TransactionID: existing.ID, State: state, Message: message})
},
}
switch existing.State {
case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting:
if err := u.executor.Run(ctx, existing.ID, executorRequest); err != nil {
return u.currentWithError(ctx, existing.ID, err)
}
case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining:
case transaction.StateRollingBack:
if err := u.executor.Run(ctx, existing.ID, executorRequest); err != nil {
return u.currentWithError(ctx, existing.ID, err)
}
default:
return u.currentWithError(ctx, existing.ID, fmt.Errorf("backend update cannot resume transaction %s in state %s", existing.ID, existing.State))
}
current, err := u.store.Transaction(ctx, existing.ID)
if err != nil {
return transaction.Transaction{}, err
}
if current.State.Terminal() {
return terminalResult(current)
}
if err := u.switchAndCommit(ctx, existing.ID, request, report); err != nil {
return u.currentWithError(ctx, existing.ID, err)
}
return u.store.Transaction(ctx, existing.ID)
}
func terminalResult(record transaction.Transaction) (transaction.Transaction, error) {
if record.State == transaction.StateCommitted {
return record, nil
}
return record, fmt.Errorf("backend update transaction %s is terminal in state %s", record.ID, record.State)
}
func (u *Updater) createOrResume(ctx context.Context, input updateInput) (transaction.Transaction, persistedRequest, bool, error) {
gatewayBefore, err := u.gateway.Read()
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
targetPort := otherPort(gatewayBefore.ActivePort)
targetSlot, err := u.config.Backend.SlotForPort(targetPort)
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
currentSlot, err := u.config.Backend.SlotForPort(gatewayBefore.ActivePort)
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
previousUnit, err := u.resolveCurrentUnit(ctx, gatewayBefore.ActivePort, currentSlot.Unit)
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
previousSlotTarget, err := readOptionalSymlink(targetSlot.JAR)
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
transactionID := rand.Text()
transactionRoot := filepath.Join(u.workRoot, transactionID)
if err := os.MkdirAll(transactionRoot, 0o750); err != nil {
return transaction.Transaction{}, persistedRequest{}, false, fmt.Errorf("create backend transaction directory: %w", err)
}
gatewayAfterContent, err := hostnginx.RenderBackendPort(gatewayBefore.Content, targetPort)
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
gatewayBeforePath := filepath.Join(transactionRoot, "gateway.before.conf")
gatewayAfterPath := filepath.Join(transactionRoot, "gateway.after.conf")
if err := writeImmutableFile(gatewayBeforePath, gatewayBefore.Content, 0o640); err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
if err := writeImmutableFile(gatewayAfterPath, gatewayAfterContent, 0o640); err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
activeBefore, err := snapshotPath(u.config.Backend.ActiveJAR, filepath.Join(transactionRoot, "active-jar.before"))
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
request := persistedRequest{
InputType: input.InputType,
SourcePath: input.SourcePath,
SourceSHA256: input.SourceSHA256,
CustomerCode: input.CustomerCode,
VersionID: input.VersionID,
ArtifactID: input.ArtifactID,
ArtifactFileName: input.ArtifactFileName,
ArtifactPath: filepath.Join(transactionRoot, "backend.jar"),
ArtifactIdentity: input.ArtifactIdentity,
ReleasePath: input.ReleasePath,
TargetPort: targetPort,
TargetUnit: targetSlot.Unit,
TargetSlotJAR: targetSlot.JAR,
TargetHealthEndpoint: targetSlot.HealthEndpoint,
PreviousSlotTarget: previousSlotTarget,
PreviousGatewayPort: gatewayBefore.ActivePort,
PreviousUnit: previousUnit,
GatewayBeforePath: gatewayBeforePath,
GatewayAfterPath: gatewayAfterPath,
GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"),
ActiveJARPath: u.config.Backend.ActiveJAR,
ActiveJARBefore: activeBefore,
}
requestJSON, err := json.Marshal(request)
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, fmt.Errorf("encode backend update request: %w", err)
}
record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{
ID: transactionID,
IdempotencyKey: input.IdempotencyKey,
Source: sourceLocalCLI,
Service: serviceBackend,
Request: requestJSON,
})
if err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
if created {
return record, request, true, nil
}
var persisted persistedRequest
if err := decodePersistedRequest(record.Request, &persisted); err != nil {
return transaction.Transaction{}, persistedRequest{}, false, err
}
if persisted.InputType != input.InputType || persisted.SourceSHA256 != input.SourceSHA256 {
return transaction.Transaction{}, persistedRequest{}, false, errors.New("persisted backend transaction input identity mismatch")
}
return record, persisted, false, nil
}
func (u *Updater) switchAndCommit(ctx context.Context, transactionID string, request persistedRequest, report ProgressReporter) error {
operation := operationLabel(request.InputType)
before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousGatewayPort)
if err != nil {
return err
}
after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
if err != nil {
return err
}
for {
record, err := u.store.Transaction(ctx, transactionID)
if err != nil {
return err
}
switch record.State {
case transaction.StateSwitching:
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)})
gatewayOperation := &gatewayOperation{
controller: u.gateway,
before: before,
after: after,
receiptPath: request.GatewayReceiptPath,
}
if _, err := u.coordinator.ExecuteStep(ctx, transactionID, gatewaySwitchIntent(request, before, after), gatewayOperation); err != nil {
return u.rollbackAfterPreparation(ctx, transactionID, request, before, after, err)
}
if _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy native slot"); err != nil {
return err
}
case transaction.StateVerifying:
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Updating the active compatibility JAR link"})
installedPath := filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath)
activeOperation := &pathOperation{
path: request.ActiveJARPath,
before: request.ActiveJARBefore,
desired: pathState{Kind: pathKindSymlink, Target: installedPath},
}
if _, err := u.coordinator.ExecuteStep(ctx, transactionID, activeLinkIntent(request, installedPath), activeOperation); err != nil {
return u.rollbackAfterPreparation(ctx, transactionID, request, before, after, err)
}
if _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, "backend compatibility link committed; previous unit draining"); err != nil {
return err
}
case transaction.StateDraining:
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Draining previous backend unit for %s", u.drain)})
if err := waitContext(ctx, u.drain); err != nil {
return err
}
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Stopping previous backend unit " + request.PreviousUnit})
stopOperation := &unitStopOperation{units: u.units, name: request.PreviousUnit}
if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousUnitIntent(request), stopOperation); err != nil {
return err
}
_, err = u.store.Transition(ctx, transactionID, transaction.StateCommitted, "native backend "+operation+" committed")
if err == nil {
reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Native backend " + operation + " committed"})
}
return err
case transaction.StateCommitted:
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Native backend " + operation + " already committed"})
return nil
default:
return fmt.Errorf("backend commit cannot continue transaction %s in state %s", transactionID, record.State)
}
}
}
func reportProgress(report ProgressReporter, progress Progress) {
if report != nil {
report(progress)
}
}
func (u *Updater) rollbackAfterPreparation(ctx context.Context, transactionID string, request persistedRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error {
record, readErr := u.store.Transaction(ctx, transactionID)
if readErr != nil {
return errors.Join(cause, readErr)
}
if record.State != transaction.StateRollingBack {
if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "native backend post-start compensation started"); err != nil {
return errors.Join(cause, err)
}
}
activeRestore := &pathOperation{
path: request.ActiveJARPath,
before: pathState{Kind: pathKindSymlink, Target: filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath)},
desired: request.ActiveJARBefore,
}
_, activeErr := u.coordinator.ExecuteStep(ctx, transactionID, activeLinkRestoreIntent(request), activeRestore)
gatewayRestore := &gatewayOperation{
controller: u.gateway,
before: after,
after: before,
receiptPath: request.GatewayReceiptPath + ".restore",
}
_, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, gatewayRestoreIntent(request), gatewayRestore)
stopTarget := &unitStopOperation{units: u.units, name: request.TargetUnit}
_, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetUnitIntent(request), stopTarget)
installedPath := filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath)
previousSlotState := pathState{Kind: pathKindAbsent}
if request.PreviousSlotTarget != "" {
previousSlotState = pathState{Kind: pathKindSymlink, Target: request.PreviousSlotTarget}
}
slotRestore := &pathOperation{
path: request.TargetSlotJAR,
before: pathState{Kind: pathKindSymlink, Target: installedPath},
desired: previousSlotState,
}
_, slotErr := u.coordinator.ExecuteStep(ctx, transactionID, restoreTargetSlotIntent(request, installedPath), slotRestore)
if err := errors.Join(activeErr, gatewayErr, stopErr, slotErr); err != nil {
return errors.Join(cause, err)
}
_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "native backend post-start compensation completed")
return errors.Join(cause, transitionErr)
}
func (u *Updater) resolveCurrentUnit(ctx context.Context, port int, configuredUnit string) (string, error) {
legacyUnit, err := legacyUnitForPort(port)
if err != nil {
return "", err
}
configured, err := u.units.Inspect(ctx, configuredUnit)
if err != nil {
return "", fmt.Errorf("inspect configured active-port unit %s: %w", configuredUnit, err)
}
legacy, err := u.units.Inspect(ctx, legacyUnit)
if err != nil {
return "", fmt.Errorf("inspect legacy active-port unit %s: %w", legacyUnit, err)
}
configuredRunning := unitRunning(configured)
legacyRunning := unitRunning(legacy)
if configuredRunning == legacyRunning {
return "", fmt.Errorf("backend port %d requires exactly one running unit, configured=%s(%s), legacy=%s(%s)", port, configuredUnit, configured.ActiveState, legacyUnit, legacy.ActiveState)
}
if configuredRunning {
return configuredUnit, nil
}
return legacyUnit, nil
}
func (u *Updater) fail(ctx context.Context, transactionID string, cause error) (transaction.Transaction, error) {
_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateFailed, cause.Error())
return u.currentWithError(ctx, transactionID, errors.Join(cause, transitionErr))
}
func (u *Updater) currentWithError(ctx context.Context, transactionID string, cause error) (transaction.Transaction, error) {
record, err := u.store.Transaction(ctx, transactionID)
return record, errors.Join(cause, err)
}
func otherPort(port int) int {
if port == deploymentconfig.BackendPort8080 {
return deploymentconfig.BackendPort8081
}
return deploymentconfig.BackendPort8080
}
func legacyUnitForPort(port int) (string, error) {
switch port {
case deploymentconfig.BackendPort8080:
return legacyUnit8080, nil
case deploymentconfig.BackendPort8081:
return legacyUnit8081, nil
default:
return "", fmt.Errorf("unsupported legacy backend port: %d", port)
}
}
func unitRunning(unit systemd.Unit) bool {
return unit.ActiveState != "inactive" && unit.ActiveState != "failed"
}
func readOptionalSymlink(path string) (string, error) {
info, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
return "", nil
}
if err != nil {
return "", fmt.Errorf("inspect native backend target slot %s: %w", path, err)
}
if info.Mode()&os.ModeSymlink == 0 {
return "", fmt.Errorf("native backend target slot is not a symbolic link: %s", path)
}
target, err := os.Readlink(path)
if err != nil {
return "", fmt.Errorf("read native backend target slot %s: %w", path, err)
}
return target, nil
}
func ensureTransactionArtifact(input updateInput, request persistedRequest) error {
info, err := os.Lstat(request.ArtifactPath)
if errors.Is(err, os.ErrNotExist) {
return input.Materialize(request.ArtifactPath)
}
if err != nil {
return fmt.Errorf("inspect extracted native backend artifact: %w", err)
}
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return errors.New("extracted native backend artifact is not a direct regular file")
}
return verifyFileIdentity(request.ArtifactPath, request.ArtifactIdentity)
}
func decodePersistedRequest(content json.RawMessage, request *persistedRequest) error {
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields()
if err := decoder.Decode(request); err != nil {
return fmt.Errorf("decode persisted backend update request: %w", err)
}
return nil
}
func waitContext(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}