feat: backend native executor implement

2026-08-16 01:27:30 +08:00
parent fbce91d797
commit 390e0565d3
44 changed files with 6430 additions and 31 deletions
@@ -0,0 +1,535 @@
+// Package backendupdate orchestrates one native backend update through commit or compensation.
+package backendupdate
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"yms-daemon/internal/daemonapi"
+	"yms-daemon/internal/deploymentconfig"
+	"yms-daemon/internal/filestore"
+	"yms-daemon/internal/hostnginx"
+	"yms-daemon/internal/nativebackendexecutor"
+	"yms-daemon/internal/systemd"
+	"yms-daemon/internal/transaction"
+	"yms-daemon/internal/updatepackage"
+)
+
+const (
+	serviceBackend            = "backend"
+	sourceLocalCLI            = "local-cli"
+	drainDuration             = 5 * time.Second
+	directReleaseDigestLength = 12
+	inputTypeCurrentRelease   = "current-native-release"
+	legacyUnit8080            = "yms.service"
+	legacyUnit8081            = "ymsback.service"
+)
+
+// Updater executes the current native backend contract on one server.
+type Updater struct {
+	config       deploymentconfig.Config
+	workRoot     string
+	store        *transaction.Store
+	coordinator  *transaction.Coordinator
+	releaseStore *filestore.Store
+	units        systemd.Manager
+	gateway      gatewayController
+	executor     nativeExecutor
+	logger       *slog.Logger
+	drain        time.Duration
+}
+
+type gatewayController interface {
+	Read() (hostnginx.Snapshot, error)
+	Apply(context.Context, hostnginx.Snapshot) error
+}
+
+type nativeExecutor interface {
+	Run(context.Context, string, nativebackendexecutor.Request) error
+}
+
+// New creates the complete native backend update orchestrator.
+func New(
+	config deploymentconfig.Config,
+	workRoot string,
+	store *transaction.Store,
+	coordinator *transaction.Coordinator,
+	releaseStore *filestore.Store,
+	units systemd.Manager,
+	gateway gatewayController,
+	httpClient *http.Client,
+	logger *slog.Logger,
+) (*Updater, error) {
+	if err := config.Validate(); err != nil {
+		return nil, err
+	}
+	if !filepath.IsAbs(workRoot) || filepath.Clean(workRoot) != workRoot {
+		return nil, errors.New("backend update work root must be a clean absolute path")
+	}
+	if store == nil || coordinator == nil || releaseStore == nil || units == nil || gateway == nil {
+		return nil, errors.New("backend update dependencies are required")
+	}
+	if logger == nil {
+		logger = slog.Default()
+	}
+	executor, err := nativebackendexecutor.New(store, coordinator, releaseStore, units, httpClient)
+	if err != nil {
+		return nil, err
+	}
+	return &Updater{
+		config:       config,
+		workRoot:     workRoot,
+		store:        store,
+		coordinator:  coordinator,
+		releaseStore: releaseStore,
+		units:        units,
+		gateway:      gateway,
+		executor:     executor,
+		logger:       logger,
+		drain:        drainDuration,
+	}, nil
+}
+
+// UpdateRepack applies one repack ZIP selected by an absolute local path.
+func (u *Updater) UpdateRepack(ctx context.Context, packagePath string, report ProgressReporter) (transaction.Transaction, error) {
+	reportProgress(report, Progress{Message: "Validating repack ZIP"})
+	updatePackage, err := updatepackage.OpenBackendNative(packagePath)
+	if err != nil {
+		return transaction.Transaction{}, err
+	}
+	defer updatePackage.Close()
+	reportProgress(report, Progress{Message: "Repack ZIP validated"})
+	return u.update(ctx, updateInput{
+		IdempotencyKey:   serviceBackend + ":" + updatePackage.PackageSHA256,
+		InputType:        daemonapi.InputTypeRepackZIP,
+		SourcePath:       updatePackage.PackagePath,
+		SourceSHA256:     updatePackage.PackageSHA256,
+		CustomerCode:     updatePackage.CustomerCode,
+		VersionID:        updatePackage.VersionID,
+		ArtifactID:       updatePackage.ArtifactID,
+		ArtifactFileName: updatePackage.FileName,
+		ArtifactIdentity: updatePackage.Identity,
+		ReleasePath:      updatePackage.FileName,
+		Materialize:      updatePackage.ExtractArtifact,
+	}, report)
+}
+
+// UpdateNativeJAR applies one JAR copied directly to the server.
+func (u *Updater) UpdateNativeJAR(ctx context.Context, jarPath string, report ProgressReporter) (transaction.Transaction, error) {
+	reportProgress(report, Progress{Message: "Validating direct native backend JAR and computing SHA-256"})
+	jar, err := updatepackage.OpenDirectNativeJAR(jarPath)
+	if err != nil {
+		return transaction.Transaction{}, err
+	}
+	reportProgress(report, Progress{Message: "Direct native backend JAR validated: sha256=" + jar.SHA256})
+	return u.update(ctx, updateInput{
+		IdempotencyKey:   serviceBackend + ":" + jar.SHA256,
+		InputType:        daemonapi.InputTypeNativeJAR,
+		SourcePath:       jar.Path,
+		SourceSHA256:     jar.SHA256,
+		ArtifactFileName: jar.FileName,
+		ArtifactIdentity: jar.Identity,
+		ReleasePath:      filepath.Join("direct", jar.SHA256[:directReleaseDigestLength], jar.FileName),
+		Materialize:      jar.CopyArtifact,
+	}, report)
+}
+
+func (u *Updater) update(ctx context.Context, input updateInput, report ProgressReporter) (transaction.Transaction, error) {
+	existing, request, created, err := u.createOrResume(ctx, input)
+	if err != nil {
+		return transaction.Transaction{}, err
+	}
+	operation := operationLabel(input.InputType)
+	transactionMessage := "Resuming backend " + operation + " transaction"
+	if created {
+		transactionMessage = "Created backend " + operation + " transaction"
+	}
+	transactionMessage += " " + existing.ID
+	reportProgress(report, Progress{TransactionID: existing.ID, State: existing.State, Message: transactionMessage})
+	if !created && existing.State.Terminal() {
+		return terminalResult(existing)
+	}
+	if err := os.MkdirAll(filepath.Dir(request.ArtifactPath), 0o750); err != nil {
+		return u.fail(ctx, existing.ID, fmt.Errorf("create backend transaction work directory: %w", err))
+	}
+	reportProgress(report, Progress{TransactionID: existing.ID, State: existing.State, Message: "Staging backend artifact in transaction workspace"})
+	if err := ensureTransactionArtifact(input, request); err != nil {
+		return u.fail(ctx, existing.ID, err)
+	}
+
+	executorRequest := nativebackendexecutor.Request{
+		ArtifactPath:       request.ArtifactPath,
+		ArtifactIdentity:   request.ArtifactIdentity,
+		ReleasePath:        request.ReleasePath,
+		SlotJarPath:        request.TargetSlotJAR,
+		PreviousSlotTarget: request.PreviousSlotTarget,
+		UnitName:           request.TargetUnit,
+		Port:               request.TargetPort,
+		HealthEndpoint:     request.TargetHealthEndpoint,
+		Progress: func(state transaction.State, message string) {
+			reportProgress(report, Progress{TransactionID: existing.ID, State: state, Message: message})
+		},
+	}
+	switch existing.State {
+	case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting:
+		if err := u.executor.Run(ctx, existing.ID, executorRequest); err != nil {
+			return u.currentWithError(ctx, existing.ID, err)
+		}
+	case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining:
+	case transaction.StateRollingBack:
+		if err := u.executor.Run(ctx, existing.ID, executorRequest); err != nil {
+			return u.currentWithError(ctx, existing.ID, err)
+		}
+	default:
+		return u.currentWithError(ctx, existing.ID, fmt.Errorf("backend update cannot resume transaction %s in state %s", existing.ID, existing.State))
+	}
+	current, err := u.store.Transaction(ctx, existing.ID)
+	if err != nil {
+		return transaction.Transaction{}, err
+	}
+	if current.State.Terminal() {
+		return terminalResult(current)
+	}
+	if err := u.switchAndCommit(ctx, existing.ID, request, report); err != nil {
+		return u.currentWithError(ctx, existing.ID, err)
+	}
+	return u.store.Transaction(ctx, existing.ID)
+}
+
+func terminalResult(record transaction.Transaction) (transaction.Transaction, error) {
+	if record.State == transaction.StateCommitted {
+		return record, nil
+	}
+	return record, fmt.Errorf("backend update transaction %s is terminal in state %s", record.ID, record.State)
+}
+
+func (u *Updater) createOrResume(ctx context.Context, input updateInput) (transaction.Transaction, persistedRequest, bool, error) {
+	gatewayBefore, err := u.gateway.Read()
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	targetPort := otherPort(gatewayBefore.ActivePort)
+	targetSlot, err := u.config.Backend.SlotForPort(targetPort)
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	currentSlot, err := u.config.Backend.SlotForPort(gatewayBefore.ActivePort)
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	previousUnit, err := u.resolveCurrentUnit(ctx, gatewayBefore.ActivePort, currentSlot.Unit)
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	previousSlotTarget, err := readOptionalSymlink(targetSlot.JAR)
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+
+	transactionID := rand.Text()
+	transactionRoot := filepath.Join(u.workRoot, transactionID)
+	if err := os.MkdirAll(transactionRoot, 0o750); err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, fmt.Errorf("create backend transaction directory: %w", err)
+	}
+	gatewayAfterContent, err := hostnginx.RenderBackendPort(gatewayBefore.Content, targetPort)
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	gatewayBeforePath := filepath.Join(transactionRoot, "gateway.before.conf")
+	gatewayAfterPath := filepath.Join(transactionRoot, "gateway.after.conf")
+	if err := writeImmutableFile(gatewayBeforePath, gatewayBefore.Content, 0o640); err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	if err := writeImmutableFile(gatewayAfterPath, gatewayAfterContent, 0o640); err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+
+	activeBefore, err := snapshotPath(u.config.Backend.ActiveJAR, filepath.Join(transactionRoot, "active-jar.before"))
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	request := persistedRequest{
+		InputType:            input.InputType,
+		SourcePath:           input.SourcePath,
+		SourceSHA256:         input.SourceSHA256,
+		CustomerCode:         input.CustomerCode,
+		VersionID:            input.VersionID,
+		ArtifactID:           input.ArtifactID,
+		ArtifactFileName:     input.ArtifactFileName,
+		ArtifactPath:         filepath.Join(transactionRoot, "backend.jar"),
+		ArtifactIdentity:     input.ArtifactIdentity,
+		ReleasePath:          input.ReleasePath,
+		TargetPort:           targetPort,
+		TargetUnit:           targetSlot.Unit,
+		TargetSlotJAR:        targetSlot.JAR,
+		TargetHealthEndpoint: targetSlot.HealthEndpoint,
+		PreviousSlotTarget:   previousSlotTarget,
+		PreviousGatewayPort:  gatewayBefore.ActivePort,
+		PreviousUnit:         previousUnit,
+		GatewayBeforePath:    gatewayBeforePath,
+		GatewayAfterPath:     gatewayAfterPath,
+		GatewayReceiptPath:   filepath.Join(transactionRoot, "gateway.applied"),
+		ActiveJARPath:        u.config.Backend.ActiveJAR,
+		ActiveJARBefore:      activeBefore,
+	}
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, fmt.Errorf("encode backend update request: %w", err)
+	}
+	record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{
+		ID:             transactionID,
+		IdempotencyKey: input.IdempotencyKey,
+		Source:         sourceLocalCLI,
+		Service:        serviceBackend,
+		Request:        requestJSON,
+	})
+	if err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	if created {
+		return record, request, true, nil
+	}
+	var persisted persistedRequest
+	if err := decodePersistedRequest(record.Request, &persisted); err != nil {
+		return transaction.Transaction{}, persistedRequest{}, false, err
+	}
+	if persisted.InputType != input.InputType || persisted.SourceSHA256 != input.SourceSHA256 {
+		return transaction.Transaction{}, persistedRequest{}, false, errors.New("persisted backend transaction input identity mismatch")
+	}
+	return record, persisted, false, nil
+}
+
+func (u *Updater) switchAndCommit(ctx context.Context, transactionID string, request persistedRequest, report ProgressReporter) error {
+	operation := operationLabel(request.InputType)
+	before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousGatewayPort)
+	if err != nil {
+		return err
+	}
+	after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
+	if err != nil {
+		return err
+	}
+	for {
+		record, err := u.store.Transaction(ctx, transactionID)
+		if err != nil {
+			return err
+		}
+		switch record.State {
+		case transaction.StateSwitching:
+			reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)})
+			gatewayOperation := &gatewayOperation{
+				controller:  u.gateway,
+				before:      before,
+				after:       after,
+				receiptPath: request.GatewayReceiptPath,
+			}
+			if _, err := u.coordinator.ExecuteStep(ctx, transactionID, gatewaySwitchIntent(request, before, after), gatewayOperation); err != nil {
+				return u.rollbackAfterPreparation(ctx, transactionID, request, before, after, err)
+			}
+			if _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy native slot"); err != nil {
+				return err
+			}
+		case transaction.StateVerifying:
+			reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Updating the active compatibility JAR link"})
+			installedPath := filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath)
+			activeOperation := &pathOperation{
+				path:    request.ActiveJARPath,
+				before:  request.ActiveJARBefore,
+				desired: pathState{Kind: pathKindSymlink, Target: installedPath},
+			}
+			if _, err := u.coordinator.ExecuteStep(ctx, transactionID, activeLinkIntent(request, installedPath), activeOperation); err != nil {
+				return u.rollbackAfterPreparation(ctx, transactionID, request, before, after, err)
+			}
+			if _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, "backend compatibility link committed; previous unit draining"); err != nil {
+				return err
+			}
+		case transaction.StateDraining:
+			reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Draining previous backend unit for %s", u.drain)})
+			if err := waitContext(ctx, u.drain); err != nil {
+				return err
+			}
+			reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Stopping previous backend unit " + request.PreviousUnit})
+			stopOperation := &unitStopOperation{units: u.units, name: request.PreviousUnit}
+			if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousUnitIntent(request), stopOperation); err != nil {
+				return err
+			}
+			_, err = u.store.Transition(ctx, transactionID, transaction.StateCommitted, "native backend "+operation+" committed")
+			if err == nil {
+				reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Native backend " + operation + " committed"})
+			}
+			return err
+		case transaction.StateCommitted:
+			reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: "Native backend " + operation + " already committed"})
+			return nil
+		default:
+			return fmt.Errorf("backend commit cannot continue transaction %s in state %s", transactionID, record.State)
+		}
+	}
+}
+
+func reportProgress(report ProgressReporter, progress Progress) {
+	if report != nil {
+		report(progress)
+	}
+}
+
+func (u *Updater) rollbackAfterPreparation(ctx context.Context, transactionID string, request persistedRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error {
+	record, readErr := u.store.Transaction(ctx, transactionID)
+	if readErr != nil {
+		return errors.Join(cause, readErr)
+	}
+	if record.State != transaction.StateRollingBack {
+		if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "native backend post-start compensation started"); err != nil {
+			return errors.Join(cause, err)
+		}
+	}
+
+	activeRestore := &pathOperation{
+		path:    request.ActiveJARPath,
+		before:  pathState{Kind: pathKindSymlink, Target: filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath)},
+		desired: request.ActiveJARBefore,
+	}
+	_, activeErr := u.coordinator.ExecuteStep(ctx, transactionID, activeLinkRestoreIntent(request), activeRestore)
+	gatewayRestore := &gatewayOperation{
+		controller:  u.gateway,
+		before:      after,
+		after:       before,
+		receiptPath: request.GatewayReceiptPath + ".restore",
+	}
+	_, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, gatewayRestoreIntent(request), gatewayRestore)
+	stopTarget := &unitStopOperation{units: u.units, name: request.TargetUnit}
+	_, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetUnitIntent(request), stopTarget)
+	installedPath := filepath.Join(u.config.Backend.ReleaseDir, request.ReleasePath)
+	previousSlotState := pathState{Kind: pathKindAbsent}
+	if request.PreviousSlotTarget != "" {
+		previousSlotState = pathState{Kind: pathKindSymlink, Target: request.PreviousSlotTarget}
+	}
+	slotRestore := &pathOperation{
+		path:    request.TargetSlotJAR,
+		before:  pathState{Kind: pathKindSymlink, Target: installedPath},
+		desired: previousSlotState,
+	}
+	_, slotErr := u.coordinator.ExecuteStep(ctx, transactionID, restoreTargetSlotIntent(request, installedPath), slotRestore)
+	if err := errors.Join(activeErr, gatewayErr, stopErr, slotErr); err != nil {
+		return errors.Join(cause, err)
+	}
+	_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "native backend post-start compensation completed")
+	return errors.Join(cause, transitionErr)
+}
+
+func (u *Updater) resolveCurrentUnit(ctx context.Context, port int, configuredUnit string) (string, error) {
+	legacyUnit, err := legacyUnitForPort(port)
+	if err != nil {
+		return "", err
+	}
+	configured, err := u.units.Inspect(ctx, configuredUnit)
+	if err != nil {
+		return "", fmt.Errorf("inspect configured active-port unit %s: %w", configuredUnit, err)
+	}
+	legacy, err := u.units.Inspect(ctx, legacyUnit)
+	if err != nil {
+		return "", fmt.Errorf("inspect legacy active-port unit %s: %w", legacyUnit, err)
+	}
+	configuredRunning := unitRunning(configured)
+	legacyRunning := unitRunning(legacy)
+	if configuredRunning == legacyRunning {
+		return "", fmt.Errorf("backend port %d requires exactly one running unit, configured=%s(%s), legacy=%s(%s)", port, configuredUnit, configured.ActiveState, legacyUnit, legacy.ActiveState)
+	}
+	if configuredRunning {
+		return configuredUnit, nil
+	}
+	return legacyUnit, nil
+}
+
+func (u *Updater) fail(ctx context.Context, transactionID string, cause error) (transaction.Transaction, error) {
+	_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateFailed, cause.Error())
+	return u.currentWithError(ctx, transactionID, errors.Join(cause, transitionErr))
+}
+
+func (u *Updater) currentWithError(ctx context.Context, transactionID string, cause error) (transaction.Transaction, error) {
+	record, err := u.store.Transaction(ctx, transactionID)
+	return record, errors.Join(cause, err)
+}
+
+func otherPort(port int) int {
+	if port == deploymentconfig.BackendPort8080 {
+		return deploymentconfig.BackendPort8081
+	}
+	return deploymentconfig.BackendPort8080
+}
+
+func legacyUnitForPort(port int) (string, error) {
+	switch port {
+	case deploymentconfig.BackendPort8080:
+		return legacyUnit8080, nil
+	case deploymentconfig.BackendPort8081:
+		return legacyUnit8081, nil
+	default:
+		return "", fmt.Errorf("unsupported legacy backend port: %d", port)
+	}
+}
+
+func unitRunning(unit systemd.Unit) bool {
+	return unit.ActiveState != "inactive" && unit.ActiveState != "failed"
+}
+
+func readOptionalSymlink(path string) (string, error) {
+	info, err := os.Lstat(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("inspect native backend target slot %s: %w", path, err)
+	}
+	if info.Mode()&os.ModeSymlink == 0 {
+		return "", fmt.Errorf("native backend target slot is not a symbolic link: %s", path)
+	}
+	target, err := os.Readlink(path)
+	if err != nil {
+		return "", fmt.Errorf("read native backend target slot %s: %w", path, err)
+	}
+	return target, nil
+}
+
+func ensureTransactionArtifact(input updateInput, request persistedRequest) error {
+	info, err := os.Lstat(request.ArtifactPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return input.Materialize(request.ArtifactPath)
+	}
+	if err != nil {
+		return fmt.Errorf("inspect extracted native backend artifact: %w", err)
+	}
+	if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
+		return errors.New("extracted native backend artifact is not a direct regular file")
+	}
+	return verifyFileIdentity(request.ArtifactPath, request.ArtifactIdentity)
+}
+
+func decodePersistedRequest(content json.RawMessage, request *persistedRequest) error {
+	decoder := json.NewDecoder(bytes.NewReader(content))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(request); err != nil {
+		return fmt.Errorf("decode persisted backend update request: %w", err)
+	}
+	return nil
+}
+
+func waitContext(ctx context.Context, duration time.Duration) error {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}