yms-daemon/internal/backendupdate/container.go

package backendupdate

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/distribution/reference"

	"yms-daemon/internal/backendexecutor"
	"yms-daemon/internal/containerengine"
	"yms-daemon/internal/deploymentconfig"
	"yms-daemon/internal/hostnginx"
	"yms-daemon/internal/transaction"
)

const inputTypeContainerImage = "container-image"

type persistedContainerRequest struct {
	InputType            string                   `json:"inputType"`
	ImageReference       string                   `json:"imageReference"`
	ImmutableReference   string                   `json:"immutableReference"`
	ImageDigest          string                   `json:"imageDigest"`
	Platform             containerengine.Platform `json:"platform"`
	TargetPort           int                      `json:"targetPort"`
	TargetContainer      string                   `json:"targetContainer"`
	PreviousPort         int                      `json:"previousPort"`
	PreviousContainer    string                   `json:"previousContainer"`
	TargetHealthEndpoint string                   `json:"targetHealthEndpoint"`
	StartLog             bool                     `json:"startLog"`
	GatewayBeforePath    string                   `json:"gatewayBeforePath"`
	GatewayAfterPath     string                   `json:"gatewayAfterPath"`
	GatewayReceiptPath   string                   `json:"gatewayReceiptPath"`
}

// UpdateContainerImage pulls one development image, freezes its repository
// digest, and updates the inactive Docker backend slot.
func (u *Updater) UpdateContainerImage(ctx context.Context, imageReference string, startLog bool, report ProgressReporter) (transaction.Transaction, error) {
	if u.containerExecutor == nil || u.engine == nil {
		return transaction.Transaction{}, errors.New("container backend updater is not configured")
	}
	if strings.TrimSpace(imageReference) != imageReference || imageReference == "" {
		return transaction.Transaction{}, errors.New("exact container image reference is required")
	}
	configInfo, err := os.Lstat(u.containerConfigSource)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("inspect backend configuration %s: %w", u.containerConfigSource, err)
	}
	if !configInfo.Mode().IsRegular() || configInfo.Mode()&os.ModeSymlink != 0 {
		return transaction.Transaction{}, fmt.Errorf("backend configuration is not a direct regular file: %s", u.containerConfigSource)
	}
	tmpInfo, err := os.Lstat(u.containerTmpSource)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("inspect backend temporary directory %s: %w", u.containerTmpSource, err)
	}
	if !tmpInfo.IsDir() || tmpInfo.Mode()&os.ModeSymlink != 0 {
		return transaction.Transaction{}, fmt.Errorf("backend temporary path is not a direct directory: %s", u.containerTmpSource)
	}

	active, err := u.store.ActiveTransaction(ctx)
	if err == nil {
		var request persistedContainerRequest
		if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil {
			return active, decodeErr
		}
		if active.Service != serviceBackend || request.InputType != inputTypeContainerImage {
			return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
		}
		if request.ImageReference != imageReference {
			return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
		}
		return u.runContainerUpdate(ctx, active, request, false, report)
	}
	if !errors.Is(err, transaction.ErrNotFound) {
		return transaction.Transaction{}, err
	}

	reportProgress(report, Progress{Message: "Pulling backend image " + imageReference})
	resolved, err := u.pullAndResolveImage(ctx, imageReference)
	if err != nil {
		return transaction.Transaction{}, err
	}
	reportProgress(report, Progress{Message: "Backend image resolved: " + resolved.ImmutableReference})

	before, err := u.gateway.Read()
	if err != nil {
		return transaction.Transaction{}, err
	}
	activeSlot, err := u.config.Backend.SlotForPort(before.ActivePort)
	if err != nil {
		return transaction.Transaction{}, err
	}
	deployment, deploymentErr := u.store.BackendContainerDeployment(ctx)
	hasDeployment := deploymentErr == nil
	if deploymentErr != nil && !errors.Is(deploymentErr, transaction.ErrNotFound) {
		return transaction.Transaction{}, deploymentErr
	}
	hasHistory, err := u.store.HasCommittedBackendContainerTransactionHistory(ctx)
	if err != nil {
		return transaction.Transaction{}, err
	}
	targetPort, targetSlot, previousContainer, err := u.resolveContainerSlots(
		ctx,
		before.ActivePort,
		activeSlot,
		deployment,
		hasDeployment,
		hasHistory,
	)
	if err != nil {
		return transaction.Transaction{}, err
	}
	afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort)
	if err != nil {
		return transaction.Transaction{}, err
	}
	transactionID := rand.Text()
	transactionRoot := filepath.Join(u.workRoot, transactionID)
	if err := os.MkdirAll(transactionRoot, 0o750); err != nil {
		return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err)
	}
	beforePath := filepath.Join(transactionRoot, "gateway.before.conf")
	afterPath := filepath.Join(transactionRoot, "gateway.after.conf")
	if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil {
		return transaction.Transaction{}, err
	}
	if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil {
		return transaction.Transaction{}, err
	}
	request := persistedContainerRequest{
		InputType: inputTypeContainerImage, ImageReference: imageReference,
		ImmutableReference: resolved.ImmutableReference, ImageDigest: resolved.Digest, Platform: resolved.Platform,
		TargetPort: targetPort, TargetContainer: targetSlot.ContainerName,
		PreviousPort: before.ActivePort, PreviousContainer: previousContainer,
		TargetHealthEndpoint: targetSlot.HealthEndpoint,
		StartLog:             startLog,
		GatewayBeforePath:    beforePath, GatewayAfterPath: afterPath,
		GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"),
	}
	content, err := json.Marshal(request)
	if err != nil {
		return transaction.Transaction{}, fmt.Errorf("encode container backend update request: %w", err)
	}
	record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{
		ID: transactionID, IdempotencyKey: serviceBackend + ":container:" + resolved.Digest,
		Source: sourceLocalCLI, Service: serviceBackend, Request: content,
	})
	if err != nil {
		return transaction.Transaction{}, err
	}
	if !created {
		if err := decodeContainerRequest(record.Request, &request); err != nil {
			return record, err
		}
	}
	return u.runContainerUpdate(ctx, record, request, created, report)
}

// resolveContainerSlots reconciles the committed deployment, gateway and both
// exact container names before selecting a target. A fresh installation has no
// deployment row, no committed container transaction history and no slot
// containers. It starts on the non-routed slot so traffic is exposed only after
// health passes.
func (u *Updater) resolveContainerSlots(
	ctx context.Context,
	activePort int,
	activeSlot deploymentconfig.BackendSlot,
	deployment transaction.BackendContainerDeployment,
	hasDeployment bool,
	hasHistory bool,
) (int, deploymentconfig.BackendSlot, string, error) {
	inactivePort := otherPort(activePort)
	inactiveSlot, err := u.config.Backend.SlotForPort(inactivePort)
	if err != nil {
		return 0, deploymentconfig.BackendSlot{}, "", err
	}
	active, activeErr := u.engine.InspectContainer(ctx, activeSlot.ContainerName)
	if activeErr != nil && !errors.Is(activeErr, containerengine.ErrNotFound) {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect active backend container %s: %w", activeSlot.ContainerName, activeErr)
	}
	inactive, inactiveErr := u.engine.InspectContainer(ctx, inactiveSlot.ContainerName)
	if inactiveErr != nil && !errors.Is(inactiveErr, containerengine.ErrNotFound) {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect inactive backend container %s: %w", inactiveSlot.ContainerName, inactiveErr)
	}
	activeFound := activeErr == nil
	inactiveFound := inactiveErr == nil

	if hasDeployment {
		if deployment.ActivePort != activePort {
			return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container port %d does not match gateway active port %d", deployment.ActivePort, activePort)
		}
		if deployment.ContainerName != activeSlot.ContainerName {
			return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container %s does not match gateway slot container %s", deployment.ContainerName, activeSlot.ContainerName)
		}
		if !activeFound {
			return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed active backend container %s is missing", activeSlot.ContainerName)
		}
		if active.ID != deployment.ContainerID {
			return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s identity does not match committed deployment", activeSlot.ContainerName)
		}
	}

	if activeFound {
		if !active.Running || active.Dead {
			return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is not running", activeSlot.ContainerName)
		}
		if inactiveFound && inactive.Running && !inactive.Dead {
			return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inactive backend container %s is unexpectedly running", inactiveSlot.ContainerName)
		}
		return inactivePort, inactiveSlot, activeSlot.ContainerName, nil
	}
	if hasHistory {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing on a server with committed backend container transaction history", activeSlot.ContainerName)
	}
	if inactiveFound && inactive.Running && !inactive.Dead {
		return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing but inactive container %s is running", activeSlot.ContainerName, inactiveSlot.ContainerName)
	}
	return inactivePort, inactiveSlot, "", nil
}

type resolvedImage struct {
	ImmutableReference string
	Digest             string
	Platform           containerengine.Platform
}

func (u *Updater) pullAndResolveImage(ctx context.Context, imageReference string) (resolvedImage, error) {
	named, err := reference.ParseNormalizedNamed(imageReference)
	if err != nil {
		return resolvedImage{}, fmt.Errorf("parse container image reference: %w", err)
	}
	if _, ok := named.(reference.Tagged); !ok {
		return resolvedImage{}, errors.New("--container-image requires a tag-qualified image reference")
	}
	if err := u.engine.Ping(ctx); err != nil {
		return resolvedImage{}, err
	}
	if err := u.engine.PullImage(ctx, imageReference); err != nil {
		return resolvedImage{}, err
	}
	image, err := u.engine.InspectImage(ctx, imageReference)
	if err != nil {
		return resolvedImage{}, fmt.Errorf("inspect pulled backend image: %w", err)
	}
	if image.Platform.OS == "" || image.Platform.Architecture == "" {
		return resolvedImage{}, errors.New("pulled backend image does not report an exact platform")
	}
	repository := reference.TrimNamed(named).Name()
	matches := make(map[string]string)
	for _, value := range image.RepoDigests {
		digested, err := reference.ParseNormalizedNamed(value)
		if err != nil {
			return resolvedImage{}, fmt.Errorf("parse pulled repository digest %q: %w", value, err)
		}
		withDigest, ok := digested.(reference.Digested)
		if !ok || reference.TrimNamed(digested).Name() != repository {
			continue
		}
		matches[withDigest.Digest().String()] = value
	}
	if len(matches) == 0 {
		return resolvedImage{}, fmt.Errorf("pulled backend image has no repository digest for %s", repository)
	}
	if len(matches) != 1 {
		return resolvedImage{}, fmt.Errorf("pulled backend image has multiple repository digests for %s", repository)
	}
	for digest, immutableReference := range matches {
		return resolvedImage{ImmutableReference: immutableReference, Digest: digest, Platform: image.Platform}, nil
	}
	return resolvedImage{}, errors.New("repository digest resolution produced no result")
}

func (u *Updater) runContainerUpdate(ctx context.Context, record transaction.Transaction, request persistedContainerRequest, created bool, report ProgressReporter) (transaction.Transaction, error) {
	message := "Resuming container backend transaction " + record.ID
	if created {
		message = "Created container backend transaction " + record.ID
	}
	reportProgress(report, Progress{TransactionID: record.ID, State: record.State, Message: message})
	if record.State.Terminal() {
		return terminalResult(record)
	}
	executorRequest := backendexecutor.Request{
		ImageAcquisition: backendexecutor.ImageAcquisitionPull,
		ImageReference:   request.ImmutableReference, ExpectedImageDigest: request.ImageDigest, Platform: request.Platform,
		ContainerName: request.TargetContainer, Port: request.TargetPort,
		PortEnvironmentKey: deploymentconfig.ContainerPortEnvironment,
		ConfigSource:       u.containerConfigSource, ConfigTarget: u.containerConfigTarget,
		TmpSource: u.containerTmpSource, TmpTarget: u.containerTmpTarget,
		ConfigEnvironmentKey: deploymentconfig.ContainerConfigEnvironment,
		ConfigLocation:       deploymentconfig.ContainerConfigLocation,
		RestartPolicy:        containerengine.RestartPolicy{Name: "no"},
		HealthEndpoint:       request.TargetHealthEndpoint,
		StartLog:             request.StartLog,
		LogReporter: func(line string) {
			reportProgress(report, Progress{TransactionID: record.ID, State: transaction.StateStarting, Message: "CONTAINER LOG  " + line})
		},
	}
	switch record.State {
	case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting:
		if err := u.containerExecutor.Run(ctx, record.ID, executorRequest); err != nil {
			current, readErr := u.store.Transaction(ctx, record.ID)
			if readErr != nil || current.State == transaction.StateFailed {
				return u.currentWithError(ctx, record.ID, errors.Join(err, readErr))
			}
			before, beforeErr := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
			after, afterErr := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
			if snapshotErr := errors.Join(beforeErr, afterErr); snapshotErr != nil {
				return u.currentWithError(ctx, record.ID, errors.Join(err, snapshotErr))
			}
			rollbackErr := u.rollbackContainer(ctx, record.ID, request, before, after, err)
			return u.currentWithError(ctx, record.ID, rollbackErr)
		}
	case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, transaction.StateRollingBack:
	default:
		return u.currentWithError(ctx, record.ID, fmt.Errorf("container backend update cannot resume transaction %s in state %s", record.ID, record.State))
	}
	if err := u.switchAndCommitContainer(ctx, record.ID, request, report); err != nil {
		return u.currentWithError(ctx, record.ID, err)
	}
	return u.store.Transaction(ctx, record.ID)
}

func (u *Updater) switchAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error {
	before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
	if err != nil {
		return err
	}
	after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
	if err != nil {
		return err
	}
	for {
		record, err := u.store.Transaction(ctx, transactionID)
		if err != nil {
			return err
		}
		switch record.State {
		case transaction.StateSwitching:
			reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)})
			operation := &gatewayOperation{controller: u.gateway, before: before, after: after, receiptPath: request.GatewayReceiptPath}
			if _, err := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewaySwitchIntent(request), operation); err != nil {
				return u.rollbackContainer(ctx, transactionID, request, before, after, err)
			}
			if _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy container slot"); err != nil {
				return err
			}
		case transaction.StateVerifying:
			message := "container backend switch verified"
			if request.PreviousContainer != "" {
				message += "; previous container draining"
			}
			if _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, message); err != nil {
				return err
			}
		case transaction.StateDraining:
			if request.PreviousContainer != "" {
				reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Draining previous backend container for %s", u.drain)})
				if err := waitContext(ctx, u.drain); err != nil {
					return err
				}
				operation := &containerStopOperation{engine: u.engine, name: request.PreviousContainer}
				if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousContainerIntent(request), operation); err != nil {
					return err
				}
			}
			target, err := u.engine.InspectContainer(ctx, request.TargetContainer)
			if err != nil {
				return fmt.Errorf("inspect target backend container before commit: %w", err)
			}
			if !target.Running || target.Dead || target.ID == "" {
				return fmt.Errorf("target backend container %s is not running with an exact identity", request.TargetContainer)
			}
			if target.ImageReference != request.ImmutableReference {
				return fmt.Errorf("target backend container %s image reference does not match the transaction", request.TargetContainer)
			}
			_, err = u.store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{
				ActivePort:    request.TargetPort,
				ContainerName: request.TargetContainer,
				ImageDigest:   request.ImageDigest,
				ContainerID:   target.ID,
			}, "container backend update committed")
			if err == nil {
				reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Container backend update committed"})
			}
			return err
		case transaction.StateCommitted:
			return nil
		case transaction.StateRollingBack:
			return u.rollbackContainer(ctx, transactionID, request, before, after, errors.New("resuming container backend rollback"))
		default:
			return fmt.Errorf("container backend commit cannot continue transaction %s in state %s", transactionID, record.State)
		}
	}
}

func (u *Updater) rollbackContainer(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error {
	record, err := u.store.Transaction(ctx, transactionID)
	if err != nil {
		return errors.Join(cause, err)
	}
	if record.State != transaction.StateRollingBack {
		if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "container backend compensation started"); err != nil {
			return errors.Join(cause, err)
		}
	}
	gateway := &gatewayOperation{controller: u.gateway, before: after, after: before, receiptPath: request.GatewayReceiptPath + ".restore"}
	_, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewayRestoreIntent(request), gateway)
	stop := &containerStopOperation{engine: u.engine, name: request.TargetContainer}
	_, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetContainerIntent(request), stop)
	if err := errors.Join(gatewayErr, stopErr); err != nil {
		return errors.Join(cause, err)
	}
	_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "container backend compensation completed")
	return errors.Join(cause, transitionErr)
}

type containerStopOperation struct {
	engine containerengine.Engine
	name   string
}

func (o *containerStopOperation) Apply(ctx context.Context) error {
	err := o.engine.StopContainer(ctx, o.name)
	if errors.Is(err, containerengine.ErrNotFound) {
		return nil
	}
	return err
}

func (o *containerStopOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
	record, err := o.engine.InspectContainer(ctx, o.name)
	if errors.Is(err, containerengine.ErrNotFound) {
		return transaction.Inspection{Status: transaction.InspectionApplied}, nil
	}
	if err != nil {
		return transaction.Inspection{}, err
	}
	result, _ := json.Marshal(record)
	if !record.Running || record.Dead {
		return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
	}
	return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
}

func containerGatewaySwitchIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.container.gateway.switch", "switch host Nginx to container backend", struct {
		BeforePort int `json:"beforePort"`
		AfterPort  int `json:"afterPort"`
	}{request.PreviousPort, request.TargetPort})
}

func containerGatewayRestoreIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.container.gateway.restore", "restore host Nginx after container backend failure", struct {
		Port int `json:"port"`
	}{request.PreviousPort})
}

func stopPreviousContainerIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.previous-container.stop", "stop previous backend container after drain", struct {
		ContainerName string `json:"containerName"`
	}{request.PreviousContainer})
}

func stopTargetContainerIntent(request persistedContainerRequest) transaction.StepIntent {
	return stepIntent("backend.target-container.stop", "stop compensated backend container", struct {
		ContainerName string `json:"containerName"`
	}{request.TargetContainer})
}

func decodeContainerRequest(content json.RawMessage, request *persistedContainerRequest) error {
	decoder := json.NewDecoder(bytes.NewReader(content))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(request); err != nil {
		return fmt.Errorf("decode persisted container backend request: %w", err)
	}
	return nil
}

var _ transaction.Operation = (*containerStopOperation)(nil)