448 lines
16 KiB
Go
448 lines
16 KiB
Go
// Package backendexecutor prepares and starts one explicitly named backend container.
|
|
// Gateway switching is deliberately outside this package.
|
|
package backendexecutor
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/distribution/reference"
|
|
opencontainersdigest "github.com/opencontainers/go-digest"
|
|
|
|
"yms-daemon/internal/containerengine"
|
|
"yms-daemon/internal/healthcheck"
|
|
"yms-daemon/internal/transaction"
|
|
)
|
|
|
|
const (
|
|
healthPath = "/yms/actuator/health"
|
|
healthTimeout = 120 * time.Second
|
|
healthInterval = time.Second
|
|
containerStopTimeoutSeconds = 150 * 60
|
|
hostNetworkMode = "host"
|
|
bindMountType = "bind"
|
|
stepLoadImage = "backend.image.load"
|
|
stepPullImage = "backend.image.pull"
|
|
stepRemoveContainer = "backend.container.remove-inactive"
|
|
stepCreateContainer = "backend.container.create"
|
|
stepStartContainer = "backend.container.start"
|
|
stepCheckHealth = "backend.container.health"
|
|
)
|
|
|
|
const (
|
|
ImageAcquisitionLoad = "load"
|
|
ImageAcquisitionPull = "pull"
|
|
)
|
|
|
|
// Request contains exact values supplied by the update package and local deployment configuration.
|
|
// ImageReference is opaque: the executor never extracts meaning from its tag.
|
|
type Request struct {
|
|
ImageAcquisition string
|
|
ArchivePath string
|
|
ImageReference string
|
|
ExpectedImageDigest string
|
|
Platform containerengine.Platform
|
|
ContainerName string
|
|
Port int
|
|
PortEnvironmentKey string
|
|
ConfigSource string
|
|
ConfigTarget string
|
|
TmpSource string
|
|
TmpTarget string
|
|
ConfigEnvironmentKey string
|
|
ConfigLocation string
|
|
RestartPolicy containerengine.RestartPolicy
|
|
HealthEndpoint string
|
|
StartLog bool
|
|
LogReporter func(string)
|
|
}
|
|
|
|
// Executor drives the persisted transaction up to SWITCHING after the new container is healthy.
|
|
type Executor struct {
|
|
store *transaction.Store
|
|
coordinator *transaction.Coordinator
|
|
engine containerengine.Engine
|
|
checker *healthcheck.ActuatorChecker
|
|
}
|
|
|
|
func New(store *transaction.Store, coordinator *transaction.Coordinator, engine containerengine.Engine, httpClient *http.Client) (*Executor, error) {
|
|
if store == nil {
|
|
return nil, errors.New("transaction store is required")
|
|
}
|
|
if coordinator == nil {
|
|
return nil, errors.New("transaction coordinator is required")
|
|
}
|
|
if engine == nil {
|
|
return nil, errors.New("container engine is required")
|
|
}
|
|
checker, err := healthcheck.NewActuatorChecker(httpClient, healthInterval)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Executor{store: store, coordinator: coordinator, engine: engine, checker: checker}, nil
|
|
}
|
|
|
|
// Run resumes from the transaction's persisted state. It does not switch gateway traffic.
|
|
func (e *Executor) Run(ctx context.Context, transactionID string, request Request) error {
|
|
if strings.TrimSpace(transactionID) == "" {
|
|
return errors.New("transaction ID is required")
|
|
}
|
|
return e.coordinator.RunExclusive(ctx, func(ctx context.Context) error {
|
|
return e.run(ctx, transactionID, request)
|
|
})
|
|
}
|
|
|
|
func (e *Executor) run(ctx context.Context, transactionID string, request Request) error {
|
|
for {
|
|
record, err := e.store.Transaction(ctx, transactionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch record.State {
|
|
case transaction.StateCreated:
|
|
if _, err := e.store.Transition(ctx, transactionID, transaction.StateValidating, "backend container validation started"); err != nil {
|
|
return err
|
|
}
|
|
case transaction.StateValidating:
|
|
if err := e.validate(ctx, request); err != nil {
|
|
_, transitionErr := e.store.Transition(ctx, transactionID, transaction.StateFailed, err.Error())
|
|
return errors.Join(err, transitionErr)
|
|
}
|
|
if _, err := e.store.Transition(ctx, transactionID, transaction.StatePrepared, "backend container inputs validated"); err != nil {
|
|
return err
|
|
}
|
|
case transaction.StatePrepared:
|
|
if err := e.prepare(ctx, transactionID, request); err != nil {
|
|
return e.failUnlessRecoverable(ctx, transactionID, err)
|
|
}
|
|
if _, err := e.store.Transition(ctx, transactionID, transaction.StateStarting, "backend container prepared"); err != nil {
|
|
return err
|
|
}
|
|
case transaction.StateStarting:
|
|
// 重放 PREPARED 步骤会核对持久化意图，阻止恢复时换入另一组请求参数。
|
|
if err := e.prepare(ctx, transactionID, request); err != nil {
|
|
return err
|
|
}
|
|
if err := e.startAndCheck(ctx, transactionID, request); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.store.Transition(ctx, transactionID, transaction.StateSwitching, "backend container is healthy"); err != nil {
|
|
return err
|
|
}
|
|
case transaction.StateSwitching:
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("backend container executor cannot run transaction %s in state %s", transactionID, record.State)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Executor) failUnlessRecoverable(ctx context.Context, transactionID string, cause error) error {
|
|
var uncertain *transaction.UncertainStepError
|
|
if errors.As(cause, &uncertain) || errors.Is(cause, transaction.ErrStepConflict) {
|
|
return cause
|
|
}
|
|
_, transitionErr := e.store.Transition(ctx, transactionID, transaction.StateFailed, cause.Error())
|
|
return errors.Join(cause, transitionErr)
|
|
}
|
|
|
|
func (e *Executor) validate(ctx context.Context, request Request) error {
|
|
if err := validateRequest(request); err != nil {
|
|
return err
|
|
}
|
|
if request.ImageAcquisition == ImageAcquisitionLoad {
|
|
if err := regularFile(request.ArchivePath, "image archive"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := regularFile(request.ConfigSource, "backend configuration"); err != nil {
|
|
return err
|
|
}
|
|
if err := directDirectory(request.TmpSource, "backend temporary directory"); err != nil {
|
|
return err
|
|
}
|
|
if err := e.engine.Ping(ctx); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (e *Executor) prepare(ctx context.Context, transactionID string, request Request) error {
|
|
switch request.ImageAcquisition {
|
|
case ImageAcquisitionLoad:
|
|
operation := &loadImageOperation{engine: e.engine, archivePath: request.ArchivePath, imageReference: request.ImageReference, expectedDigest: request.ExpectedImageDigest, platform: request.Platform}
|
|
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, loadIntent(request), operation); err != nil {
|
|
return err
|
|
}
|
|
case ImageAcquisitionPull:
|
|
operation := &pullImageOperation{engine: e.engine, imageReference: request.ImageReference, expectedDigest: request.ExpectedImageDigest, platform: request.Platform}
|
|
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, pullIntent(request), operation); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
image, err := e.engine.InspectImage(ctx, request.ImageReference)
|
|
if err != nil {
|
|
return fmt.Errorf("inspect prepared backend image: %w", err)
|
|
}
|
|
removeOperation := &removeContainerOperation{engine: e.engine, name: request.ContainerName}
|
|
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, removeIntent(request), removeOperation); err != nil {
|
|
return err
|
|
}
|
|
createOperation := &createContainerOperation{
|
|
engine: e.engine,
|
|
expectedImage: image,
|
|
spec: containerSpec(request),
|
|
}
|
|
_, err = e.coordinator.ExecuteStep(ctx, transactionID, createIntent(request, image.ID), createOperation)
|
|
return err
|
|
}
|
|
|
|
func (e *Executor) startAndCheck(ctx context.Context, transactionID string, request Request) error {
|
|
startOperation := &startContainerOperation{engine: e.engine, name: request.ContainerName}
|
|
if _, err := e.coordinator.ExecuteStep(ctx, transactionID, startIntent(request), startOperation); err != nil {
|
|
return err
|
|
}
|
|
if request.StartLog && request.LogReporter != nil {
|
|
logs, err := e.engine.ContainerLogs(ctx, request.ContainerName)
|
|
if err == nil {
|
|
scanner := bufio.NewScanner(logs)
|
|
for scanner.Scan() {
|
|
request.LogReporter(scanner.Text())
|
|
}
|
|
_ = logs.Close()
|
|
if err := scanner.Err(); err != nil {
|
|
return fmt.Errorf("read container startup logs: %w", err)
|
|
}
|
|
} else {
|
|
request.LogReporter("unable to read container startup logs: " + err.Error())
|
|
}
|
|
}
|
|
healthOperation := &healthOperation{
|
|
engine: e.engine,
|
|
checker: e.checker,
|
|
name: request.ContainerName,
|
|
endpoint: request.HealthEndpoint,
|
|
timeout: healthTimeout,
|
|
}
|
|
_, err := e.coordinator.ExecuteStep(ctx, transactionID, healthIntent(request), healthOperation)
|
|
return err
|
|
}
|
|
|
|
func validateRequest(request Request) error {
|
|
switch request.ImageAcquisition {
|
|
case ImageAcquisitionLoad:
|
|
if !filepath.IsAbs(request.ArchivePath) {
|
|
return errors.New("image archive path must be absolute")
|
|
}
|
|
case ImageAcquisitionPull:
|
|
if request.ArchivePath != "" {
|
|
return errors.New("pull image acquisition does not accept an archive path")
|
|
}
|
|
default:
|
|
return fmt.Errorf("unsupported image acquisition: %q", request.ImageAcquisition)
|
|
}
|
|
if request.ImageReference == "" || strings.TrimSpace(request.ImageReference) != request.ImageReference {
|
|
return errors.New("exact image reference is required")
|
|
}
|
|
if _, err := opencontainersdigest.Parse(request.ExpectedImageDigest); err != nil {
|
|
return fmt.Errorf("invalid expected image digest: %w", err)
|
|
}
|
|
if request.Platform.OS == "" || request.Platform.Architecture == "" {
|
|
return errors.New("explicit image operating system and architecture are required")
|
|
}
|
|
if request.ContainerName == "" || strings.TrimSpace(request.ContainerName) != request.ContainerName {
|
|
return errors.New("exact container name is required")
|
|
}
|
|
if request.Port != 8080 && request.Port != 8081 {
|
|
return fmt.Errorf("backend container port must be 8080 or 8081: %d", request.Port)
|
|
}
|
|
if request.PortEnvironmentKey == "" || strings.Contains(request.PortEnvironmentKey, "=") || strings.TrimSpace(request.PortEnvironmentKey) != request.PortEnvironmentKey {
|
|
return errors.New("exact port environment key is required")
|
|
}
|
|
if !filepath.IsAbs(request.ConfigSource) || !filepath.IsAbs(request.ConfigTarget) {
|
|
return errors.New("backend configuration source and target must be absolute paths")
|
|
}
|
|
if !filepath.IsAbs(request.TmpSource) || !filepath.IsAbs(request.TmpTarget) {
|
|
return errors.New("backend temporary source and target must be absolute paths")
|
|
}
|
|
if request.ConfigEnvironmentKey == "" || strings.Contains(request.ConfigEnvironmentKey, "=") || strings.TrimSpace(request.ConfigEnvironmentKey) != request.ConfigEnvironmentKey {
|
|
return errors.New("exact backend configuration environment key is required")
|
|
}
|
|
if request.ConfigLocation == "" || strings.TrimSpace(request.ConfigLocation) != request.ConfigLocation {
|
|
return errors.New("exact backend configuration location is required")
|
|
}
|
|
if err := validateRestartPolicy(request.RestartPolicy); err != nil {
|
|
return err
|
|
}
|
|
parsed, err := url.ParseRequestURI(request.HealthEndpoint)
|
|
if err != nil || parsed.Scheme != "http" || parsed.Host == "" || parsed.Path != healthPath {
|
|
return fmt.Errorf("health endpoint must be an HTTP URL with exact path %s", healthPath)
|
|
}
|
|
if parsed.Port() != strconv.Itoa(request.Port) {
|
|
return fmt.Errorf("health endpoint port must equal backend container port %d", request.Port)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRestartPolicy(policy containerengine.RestartPolicy) error {
|
|
switch policy.Name {
|
|
case "no", "always", "unless-stopped":
|
|
if policy.MaximumRetryCount != 0 {
|
|
return fmt.Errorf("restart policy %s does not accept a maximum retry count", policy.Name)
|
|
}
|
|
case "on-failure":
|
|
if policy.MaximumRetryCount < 0 {
|
|
return errors.New("on-failure maximum retry count cannot be negative")
|
|
}
|
|
default:
|
|
return fmt.Errorf("unsupported explicit restart policy: %q", policy.Name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func regularFile(path, description string) error {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return fmt.Errorf("inspect %s %s: %w", description, path, err)
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return fmt.Errorf("%s is not a regular file: %s", description, path)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func directDirectory(path, description string) error {
|
|
info, err := os.Lstat(path)
|
|
if err != nil {
|
|
return fmt.Errorf("inspect %s %s: %w", description, path, err)
|
|
}
|
|
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
|
return fmt.Errorf("%s is not a direct directory: %s", description, path)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func containerSpec(request Request) containerengine.ContainerSpec {
|
|
return containerengine.ContainerSpec{
|
|
Name: request.ContainerName,
|
|
ImageReference: request.ImageReference,
|
|
Platform: request.Platform,
|
|
Environment: []string{
|
|
request.PortEnvironmentKey + "=" + strconv.Itoa(request.Port),
|
|
request.ConfigEnvironmentKey + "=" + request.ConfigLocation,
|
|
},
|
|
NetworkMode: hostNetworkMode,
|
|
RestartPolicy: request.RestartPolicy,
|
|
Mounts: []containerengine.Mount{
|
|
{Type: bindMountType, Source: request.ConfigSource, Target: request.ConfigTarget, ReadOnly: true},
|
|
{Type: bindMountType, Source: request.TmpSource, Target: request.TmpTarget},
|
|
},
|
|
User: "0:0",
|
|
StopTimeoutSeconds: containerStopTimeoutSeconds,
|
|
}
|
|
}
|
|
|
|
func pullIntent(request Request) transaction.StepIntent {
|
|
return intent(stepPullImage, "pull and verify backend image", struct {
|
|
ImageReference string `json:"imageReference"`
|
|
ImageDigest string `json:"imageDigest"`
|
|
Platform containerengine.Platform `json:"platform"`
|
|
}{request.ImageReference, request.ExpectedImageDigest, request.Platform})
|
|
}
|
|
|
|
func loadIntent(request Request) transaction.StepIntent {
|
|
return intent(stepLoadImage, "load and verify backend image", struct {
|
|
ArchivePath string `json:"archivePath"`
|
|
ImageReference string `json:"imageReference"`
|
|
ImageDigest string `json:"imageDigest"`
|
|
Platform containerengine.Platform `json:"platform"`
|
|
}{request.ArchivePath, request.ImageReference, request.ExpectedImageDigest, request.Platform})
|
|
}
|
|
|
|
func createIntent(request Request, imageID string) transaction.StepIntent {
|
|
return intent(stepCreateContainer, "create inactive backend container", struct {
|
|
Spec containerengine.ContainerSpec `json:"spec"`
|
|
ImageID string `json:"imageId"`
|
|
}{containerSpec(request), imageID})
|
|
}
|
|
|
|
func removeIntent(request Request) transaction.StepIntent {
|
|
return intent(stepRemoveContainer, "remove inactive backend container", struct {
|
|
ContainerName string `json:"containerName"`
|
|
}{request.ContainerName})
|
|
}
|
|
|
|
func startIntent(request Request) transaction.StepIntent {
|
|
return intent(stepStartContainer, "start inactive backend container", struct {
|
|
ContainerName string `json:"containerName"`
|
|
}{request.ContainerName})
|
|
}
|
|
|
|
func healthIntent(request Request) transaction.StepIntent {
|
|
return intent(stepCheckHealth, "wait for backend Actuator health", struct {
|
|
ContainerName string `json:"containerName"`
|
|
Endpoint string `json:"endpoint"`
|
|
Timeout time.Duration `json:"timeout"`
|
|
}{request.ContainerName, request.HealthEndpoint, healthTimeout})
|
|
}
|
|
|
|
func intent(key, name string, value any) transaction.StepIntent {
|
|
payload, err := json.Marshal(value)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("marshal internal step intent: %v", err))
|
|
}
|
|
return transaction.StepIntent{Key: key, Name: name, Intent: payload}
|
|
}
|
|
|
|
func imageMatches(image containerengine.Image, expectedDigest string, expectedPlatform containerengine.Platform) (bool, error) {
|
|
expected, err := opencontainersdigest.Parse(expectedDigest)
|
|
if err != nil {
|
|
return false, fmt.Errorf("parse expected image digest: %w", err)
|
|
}
|
|
if image.Platform != expectedPlatform {
|
|
return false, nil
|
|
}
|
|
if image.DescriptorDigest != "" {
|
|
actual, err := opencontainersdigest.Parse(image.DescriptorDigest)
|
|
if err != nil {
|
|
return false, fmt.Errorf("parse inspected image descriptor digest: %w", err)
|
|
}
|
|
if actual == expected {
|
|
return true, nil
|
|
}
|
|
}
|
|
for _, repoDigest := range image.RepoDigests {
|
|
parsed, err := reference.ParseAnyReference(repoDigest)
|
|
if err != nil {
|
|
return false, fmt.Errorf("parse inspected repository digest %q: %w", repoDigest, err)
|
|
}
|
|
digested, ok := parsed.(reference.Digested)
|
|
if !ok {
|
|
return false, fmt.Errorf("inspected repository digest is not digest-qualified: %q", repoDigest)
|
|
}
|
|
if digested.Digest() == expected {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func resultJSON(value any) json.RawMessage {
|
|
payload, err := json.Marshal(value)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("marshal internal step result: %v", err))
|
|
}
|
|
return payload
|
|
}
|