yms-daemon/internal/nativebackendexecutor/executor.go

// Package nativebackendexecutor prepares and starts one explicitly configured native backend slot.
// Gateway switching is deliberately outside this package.
package nativebackendexecutor

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"yms-daemon/internal/filestore"
	"yms-daemon/internal/healthcheck"
	"yms-daemon/internal/systemd"
	"yms-daemon/internal/transaction"
)

const (
	healthPath      = "/yms/actuator/health"
	healthTimeout   = 120 * time.Second
	healthInterval  = time.Second
	stepInstallJar  = "backend.native.jar.install"
	stepBindSlot    = "backend.native.slot.bind"
	stepStartUnit   = "backend.native.service.start"
	stepCheckHealth = "backend.native.health"
	stepStopUnit    = "backend.native.service.stop"
	stepRestoreSlot = "backend.native.slot.restore"
	activeState     = "active"
	inactiveState   = "inactive"
	failedState     = "failed"
)

// Request contains exact values from the immutable update request and local deployment configuration.
// UnitName, SlotJarPath and PreviousSlotTarget are opaque and are never derived from filenames or ports.
type Request struct {
	ArtifactPath       string
	ArtifactIdentity   filestore.Identity
	ReleasePath        string
	SlotJarPath        string
	PreviousSlotTarget string
	UnitName           string
	Port               int
	HealthEndpoint     string
	Progress           func(transaction.State, string)
}

type actuatorChecker interface {
	Check(context.Context, string, healthcheck.RunningProbe) (healthcheck.ActuatorReport, bool, error)
	Wait(context.Context, string, time.Duration, healthcheck.RunningProbe) (healthcheck.ActuatorReport, error)
}

// Executor drives the persisted transaction up to SWITCHING after the inactive native backend is healthy.
type Executor struct {
	store        *transaction.Store
	coordinator  *transaction.Coordinator
	releaseStore *filestore.Store
	units        systemd.Manager
	checker      actuatorChecker
}

func New(store *transaction.Store, coordinator *transaction.Coordinator, releaseStore *filestore.Store, units systemd.Manager, httpClient *http.Client) (*Executor, error) {
	if store == nil {
		return nil, errors.New("transaction store is required")
	}
	if coordinator == nil {
		return nil, errors.New("transaction coordinator is required")
	}
	if releaseStore == nil {
		return nil, errors.New("native backend release store is required")
	}
	if units == nil {
		return nil, errors.New("systemd manager is required")
	}
	checker, err := healthcheck.NewActuatorChecker(httpClient, healthInterval)
	if err != nil {
		return nil, err
	}
	return &Executor{store: store, coordinator: coordinator, releaseStore: releaseStore, units: units, checker: checker}, nil
}

// Run resumes from the transaction's persisted state. It does not switch gateway traffic.
func (e *Executor) Run(ctx context.Context, transactionID string, request Request) error {
	if strings.TrimSpace(transactionID) == "" {
		return errors.New("transaction ID is required")
	}
	return e.coordinator.RunExclusive(ctx, func(ctx context.Context) error {
		return e.run(ctx, transactionID, request)
	})
}

func (e *Executor) run(ctx context.Context, transactionID string, request Request) error {
	for {
		record, err := e.store.Transaction(ctx, transactionID)
		if err != nil {
			return err
		}
		switch record.State {
		case transaction.StateCreated:
			if _, err := e.store.Transition(ctx, transactionID, transaction.StateValidating, "native backend validation started"); err != nil {
				return err
			}
		case transaction.StateValidating:
			reportProgress(request, record.State, "Validating inactive native backend slot")
			if err := e.validate(ctx, request); err != nil {
				_, transitionErr := e.store.Transition(ctx, transactionID, transaction.StateFailed, err.Error())
				return errors.Join(err, transitionErr)
			}
			if _, err := e.store.Transition(ctx, transactionID, transaction.StatePrepared, "native backend inputs validated"); err != nil {
				return err
			}
		case transaction.StatePrepared:
			reportProgress(request, record.State, "Installing backend JAR and binding the inactive slot")
			if _, err := e.prepare(ctx, transactionID, request); err != nil {
				return e.failUnlessRecoverable(ctx, transactionID, err)
			}
			if _, err := e.store.Transition(ctx, transactionID, transaction.StateStarting, "native backend slot prepared"); err != nil {
				return err
			}
		case transaction.StateStarting:
			reportProgress(request, record.State, fmt.Sprintf("Preparing to start %s on port %d", request.UnitName, request.Port))
			installedPath, err := e.prepare(ctx, transactionID, request)
			if err != nil {
				return e.failUnlessRecoverable(ctx, transactionID, err)
			}
			if err := e.startAndCheck(ctx, transactionID, request); err != nil {
				if recoverable(err) {
					return err
				}
				return e.rollbackBeforeSwitch(ctx, transactionID, request, installedPath, err)
			}
			if _, err := e.store.Transition(ctx, transactionID, transaction.StateSwitching, "native backend is healthy"); err != nil {
				return err
			}
		case transaction.StateSwitching:
			return nil
		case transaction.StateRollingBack:
			reportProgress(request, record.State, "Resuming native backend preparation compensation")
			installed, found, err := e.releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
			if err != nil {
				return fmt.Errorf("inspect native backend JAR while resuming compensation: %w", err)
			}
			if !found {
				return errors.New("installed native backend JAR is missing while compensation is pending")
			}
			if err := e.compensateBeforeSwitch(ctx, transactionID, request, installed.Path); err != nil {
				return err
			}
			if _, err := e.store.Transition(ctx, transactionID, transaction.StateRolledBack, "native backend preparation rollback resumed and completed"); err != nil {
				return err
			}
			return nil
		default:
			return fmt.Errorf("native backend executor cannot run transaction %s in state %s", transactionID, record.State)
		}
	}
}

func (e *Executor) validate(ctx context.Context, request Request) error {
	if err := validateRequest(request); err != nil {
		return err
	}
	info, err := os.Lstat(request.ArtifactPath)
	if err != nil {
		return fmt.Errorf("inspect native backend JAR %s: %w", request.ArtifactPath, err)
	}
	if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
		return fmt.Errorf("native backend JAR is not a regular file: %s", request.ArtifactPath)
	}
	if info.Size() != request.ArtifactIdentity.Size {
		return fmt.Errorf("native backend JAR size mismatch: got %d, want %d", info.Size(), request.ArtifactIdentity.Size)
	}
	if err := inspectInitialSlot(request.SlotJarPath, request.PreviousSlotTarget); err != nil {
		return err
	}
	unit, err := e.units.Inspect(ctx, request.UnitName)
	if err != nil {
		return fmt.Errorf("inspect inactive native backend unit: %w", err)
	}
	if unit.ActiveState != inactiveState && unit.ActiveState != failedState {
		return fmt.Errorf("native backend unit %s must be inactive or failed before preparation, active state is %q", request.UnitName, unit.ActiveState)
	}
	return nil
}

func (e *Executor) prepare(ctx context.Context, transactionID string, request Request) (string, error) {
	installOperation := &installJarOperation{
		store:       e.releaseStore,
		sourcePath:  request.ArtifactPath,
		releasePath: request.ReleasePath,
		identity:    request.ArtifactIdentity,
	}
	if _, err := e.coordinator.ExecuteStep(ctx, transactionID, installIntent(request), installOperation); err != nil {
		return "", err
	}
	installed, found, err := e.releaseStore.Inspect(request.ReleasePath, request.ArtifactIdentity)
	if err != nil {
		return "", fmt.Errorf("inspect installed native backend JAR: %w", err)
	}
	if !found {
		return "", errors.New("installed native backend JAR is missing after completed install step")
	}

	bindOperation := &slotLinkOperation{
		path:           request.SlotJarPath,
		desiredTarget:  installed.Path,
		previousTarget: request.PreviousSlotTarget,
	}
	if _, err := e.coordinator.ExecuteStep(ctx, transactionID, bindIntent(request, installed.Path), bindOperation); err != nil {
		return "", err
	}
	return installed.Path, nil
}

func (e *Executor) startAndCheck(ctx context.Context, transactionID string, request Request) error {
	reportProgress(request, transaction.StateStarting, "Starting native backend unit "+request.UnitName)
	startOperation := &unitStartOperation{units: e.units, name: request.UnitName}
	if _, err := e.coordinator.ExecuteStep(ctx, transactionID, startIntent(request), startOperation); err != nil {
		return err
	}
	reportProgress(request, transaction.StateStarting, fmt.Sprintf("Waiting up to %s for Actuator health: %s", healthTimeout, request.HealthEndpoint))
	healthOperation := &healthOperation{
		units:    e.units,
		checker:  e.checker,
		unitName: request.UnitName,
		endpoint: request.HealthEndpoint,
		timeout:  healthTimeout,
	}
	_, err := e.coordinator.ExecuteStep(ctx, transactionID, healthIntent(request), healthOperation)
	if err == nil {
		reportProgress(request, transaction.StateStarting, "Actuator health status is UP")
	}
	return err
}

func reportProgress(request Request, state transaction.State, message string) {
	if request.Progress != nil {
		request.Progress(state, message)
	}
}

func (e *Executor) rollbackBeforeSwitch(ctx context.Context, transactionID string, request Request, installedPath string, cause error) error {
	if _, err := e.store.Transition(ctx, transactionID, transaction.StateRollingBack, "native backend preparation failed; compensation started"); err != nil {
		return errors.Join(cause, err)
	}
	if err := e.compensateBeforeSwitch(ctx, transactionID, request, installedPath); err != nil {
		return errors.Join(cause, err)
	}
	if _, err := e.store.Transition(ctx, transactionID, transaction.StateRolledBack, "native backend preparation rolled back"); err != nil {
		return errors.Join(cause, err)
	}
	return cause
}

func (e *Executor) compensateBeforeSwitch(ctx context.Context, transactionID string, request Request, installedPath string) error {
	stopOperation := &unitStopOperation{units: e.units, name: request.UnitName}
	if _, err := e.coordinator.ExecuteStep(ctx, transactionID, stopIntent(request), stopOperation); err != nil {
		return err
	}
	restoreOperation := &slotLinkOperation{
		path:           request.SlotJarPath,
		desiredTarget:  request.PreviousSlotTarget,
		previousTarget: installedPath,
	}
	if _, err := e.coordinator.ExecuteStep(ctx, transactionID, restoreIntent(request, installedPath), restoreOperation); err != nil {
		return err
	}
	return nil
}

func (e *Executor) failUnlessRecoverable(ctx context.Context, transactionID string, cause error) error {
	if recoverable(cause) {
		return cause
	}
	_, transitionErr := e.store.Transition(ctx, transactionID, transaction.StateFailed, cause.Error())
	return errors.Join(cause, transitionErr)
}

func recoverable(cause error) bool {
	var uncertain *transaction.UncertainStepError
	return errors.As(cause, &uncertain) || errors.Is(cause, transaction.ErrStepConflict)
}

func validateRequest(request Request) error {
	if !filepath.IsAbs(request.ArtifactPath) {
		return errors.New("native backend JAR path must be absolute")
	}
	if err := request.ArtifactIdentity.Validate(); err != nil {
		return fmt.Errorf("invalid native backend JAR identity: %w", err)
	}
	if !filepath.IsLocal(request.ReleasePath) || request.ReleasePath == "." {
		return fmt.Errorf("native backend release path must be a local relative path: %q", request.ReleasePath)
	}
	if !filepath.IsAbs(request.SlotJarPath) {
		return errors.New("native backend slot JAR path must be absolute")
	}
	if request.PreviousSlotTarget != "" && !filepath.IsAbs(request.PreviousSlotTarget) {
		return errors.New("previous native backend slot target must be empty or absolute")
	}
	if request.UnitName == "" || strings.TrimSpace(request.UnitName) != request.UnitName {
		return errors.New("exact native backend systemd unit name is required")
	}
	if request.Port != 8080 && request.Port != 8081 {
		return fmt.Errorf("native backend port must be 8080 or 8081: %d", request.Port)
	}
	parsed, err := url.ParseRequestURI(request.HealthEndpoint)
	if err != nil || parsed.Scheme != "http" || parsed.Host == "" || parsed.Path != healthPath {
		return fmt.Errorf("health endpoint must be an HTTP URL with exact path %s", healthPath)
	}
	if parsed.Port() != strconv.Itoa(request.Port) {
		return fmt.Errorf("health endpoint port must equal native backend port %d", request.Port)
	}
	return nil
}

func inspectInitialSlot(path, previousTarget string) error {
	info, err := os.Lstat(path)
	if errors.Is(err, os.ErrNotExist) {
		if previousTarget == "" {
			return nil
		}
		return fmt.Errorf("native backend slot link %s is missing; expected target %s", path, previousTarget)
	}
	if err != nil {
		return fmt.Errorf("inspect native backend slot link %s: %w", path, err)
	}
	if info.Mode()&os.ModeSymlink == 0 {
		return fmt.Errorf("native backend slot path is not a symbolic link: %s", path)
	}
	target, err := os.Readlink(path)
	if err != nil {
		return fmt.Errorf("read native backend slot link %s: %w", path, err)
	}
	if target != previousTarget {
		return fmt.Errorf("native backend slot link %s targets %q, expected %q", path, target, previousTarget)
	}
	if previousTarget != "" {
		targetInfo, err := os.Lstat(previousTarget)
		if err != nil {
			return fmt.Errorf("inspect previous native backend slot target %s: %w", previousTarget, err)
		}
		if !targetInfo.Mode().IsRegular() || targetInfo.Mode()&os.ModeSymlink != 0 {
			return fmt.Errorf("previous native backend slot target is not a regular file: %s", previousTarget)
		}
	}
	return nil
}

func installIntent(request Request) transaction.StepIntent {
	return intent(stepInstallJar, "install immutable native backend JAR", struct {
		SourcePath  string             `json:"sourcePath"`
		ReleasePath string             `json:"releasePath"`
		Identity    filestore.Identity `json:"identity"`
	}{request.ArtifactPath, request.ReleasePath, request.ArtifactIdentity})
}

func bindIntent(request Request, installedPath string) transaction.StepIntent {
	return intent(stepBindSlot, "bind inactive native backend slot", struct {
		SlotJarPath    string `json:"slotJarPath"`
		PreviousTarget string `json:"previousTarget"`
		InstalledPath  string `json:"installedPath"`
	}{request.SlotJarPath, request.PreviousSlotTarget, installedPath})
}

func startIntent(request Request) transaction.StepIntent {
	return intent(stepStartUnit, "start inactive native backend systemd unit", struct {
		UnitName string `json:"unitName"`
	}{request.UnitName})
}

func healthIntent(request Request) transaction.StepIntent {
	return intent(stepCheckHealth, "wait for native backend Actuator health", struct {
		UnitName string        `json:"unitName"`
		Endpoint string        `json:"endpoint"`
		Timeout  time.Duration `json:"timeout"`
	}{request.UnitName, request.HealthEndpoint, healthTimeout})
}

func stopIntent(request Request) transaction.StepIntent {
	return intent(stepStopUnit, "stop failed inactive native backend systemd unit", struct {
		UnitName string `json:"unitName"`
	}{request.UnitName})
}

func restoreIntent(request Request, installedPath string) transaction.StepIntent {
	return intent(stepRestoreSlot, "restore inactive native backend slot", struct {
		SlotJarPath    string `json:"slotJarPath"`
		InstalledPath  string `json:"installedPath"`
		PreviousTarget string `json:"previousTarget"`
	}{request.SlotJarPath, installedPath, request.PreviousSlotTarget})
}

func intent(key, name string, value any) transaction.StepIntent {
	payload, err := json.Marshal(value)
	if err != nil {
		panic(fmt.Sprintf("marshal internal step intent: %v", err))
	}
	return transaction.StepIntent{Key: key, Name: name, Intent: payload}
}