480 lines
21 KiB
Go
480 lines
21 KiB
Go
|
package backendupdate
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"crypto/rand"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
|
||
|
|
"github.com/distribution/reference"
|
||
|
|
|
||
|
|
"yms-daemon/internal/backendexecutor"
|
||
|
|
"yms-daemon/internal/containerengine"
|
||
|
|
"yms-daemon/internal/deploymentconfig"
|
||
|
|
"yms-daemon/internal/hostnginx"
|
||
|
|
"yms-daemon/internal/transaction"
|
||
|
|
)
|
||
|
|
|
||
|
|
const inputTypeContainerImage = "container-image"
|
||
|
|
|
||
|
|
type persistedContainerRequest struct {
|
||
|
|
InputType string `json:"inputType"`
|
||
|
|
ImageReference string `json:"imageReference"`
|
||
|
|
ImmutableReference string `json:"immutableReference"`
|
||
|
|
ImageDigest string `json:"imageDigest"`
|
||
|
|
Platform containerengine.Platform `json:"platform"`
|
||
|
|
TargetPort int `json:"targetPort"`
|
||
|
|
TargetContainer string `json:"targetContainer"`
|
||
|
|
PreviousPort int `json:"previousPort"`
|
||
|
|
PreviousContainer string `json:"previousContainer"`
|
||
|
|
TargetHealthEndpoint string `json:"targetHealthEndpoint"`
|
||
|
|
GatewayBeforePath string `json:"gatewayBeforePath"`
|
||
|
|
GatewayAfterPath string `json:"gatewayAfterPath"`
|
||
|
|
GatewayReceiptPath string `json:"gatewayReceiptPath"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// UpdateContainerImage pulls one development image, freezes its repository
|
||
|
|
// digest, and updates the inactive Docker backend slot.
|
||
|
|
func (u *Updater) UpdateContainerImage(ctx context.Context, imageReference string, report ProgressReporter) (transaction.Transaction, error) {
|
||
|
|
if u.containerExecutor == nil || u.engine == nil {
|
||
|
|
return transaction.Transaction{}, errors.New("container backend updater is not configured")
|
||
|
|
}
|
||
|
|
if strings.TrimSpace(imageReference) != imageReference || imageReference == "" {
|
||
|
|
return transaction.Transaction{}, errors.New("exact container image reference is required")
|
||
|
|
}
|
||
|
|
configInfo, err := os.Lstat(u.containerConfigSource)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, fmt.Errorf("inspect backend configuration %s: %w", u.containerConfigSource, err)
|
||
|
|
}
|
||
|
|
if !configInfo.Mode().IsRegular() || configInfo.Mode()&os.ModeSymlink != 0 {
|
||
|
|
return transaction.Transaction{}, fmt.Errorf("backend configuration is not a direct regular file: %s", u.containerConfigSource)
|
||
|
|
}
|
||
|
|
tmpInfo, err := os.Lstat(u.containerTmpSource)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, fmt.Errorf("inspect backend temporary directory %s: %w", u.containerTmpSource, err)
|
||
|
|
}
|
||
|
|
if !tmpInfo.IsDir() || tmpInfo.Mode()&os.ModeSymlink != 0 {
|
||
|
|
return transaction.Transaction{}, fmt.Errorf("backend temporary path is not a direct directory: %s", u.containerTmpSource)
|
||
|
|
}
|
||
|
|
|
||
|
|
active, err := u.store.ActiveTransaction(ctx)
|
||
|
|
if err == nil {
|
||
|
|
var request persistedContainerRequest
|
||
|
|
if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil {
|
||
|
|
return active, decodeErr
|
||
|
|
}
|
||
|
|
if active.Service != serviceBackend || request.InputType != inputTypeContainerImage {
|
||
|
|
return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
|
||
|
|
}
|
||
|
|
if request.ImageReference != imageReference {
|
||
|
|
return active, &transaction.ActiveTransactionError{TransactionID: active.ID}
|
||
|
|
}
|
||
|
|
return u.runContainerUpdate(ctx, active, request, false, report)
|
||
|
|
}
|
||
|
|
if !errors.Is(err, transaction.ErrNotFound) {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
|
||
|
|
reportProgress(report, Progress{Message: "Pulling backend image " + imageReference})
|
||
|
|
resolved, err := u.pullAndResolveImage(ctx, imageReference)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
reportProgress(report, Progress{Message: "Backend image resolved: " + resolved.ImmutableReference})
|
||
|
|
|
||
|
|
before, err := u.gateway.Read()
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
activeSlot, err := u.config.Backend.SlotForPort(before.ActivePort)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
deployment, deploymentErr := u.store.BackendContainerDeployment(ctx)
|
||
|
|
hasDeployment := deploymentErr == nil
|
||
|
|
if deploymentErr != nil && !errors.Is(deploymentErr, transaction.ErrNotFound) {
|
||
|
|
return transaction.Transaction{}, deploymentErr
|
||
|
|
}
|
||
|
|
hasHistory, err := u.store.HasCommittedBackendContainerTransactionHistory(ctx)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
targetPort, targetSlot, previousContainer, err := u.resolveContainerSlots(
|
||
|
|
ctx,
|
||
|
|
before.ActivePort,
|
||
|
|
activeSlot,
|
||
|
|
deployment,
|
||
|
|
hasDeployment,
|
||
|
|
hasHistory,
|
||
|
|
)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
transactionID := rand.Text()
|
||
|
|
transactionRoot := filepath.Join(u.workRoot, transactionID)
|
||
|
|
if err := os.MkdirAll(transactionRoot, 0o750); err != nil {
|
||
|
|
return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err)
|
||
|
|
}
|
||
|
|
beforePath := filepath.Join(transactionRoot, "gateway.before.conf")
|
||
|
|
afterPath := filepath.Join(transactionRoot, "gateway.after.conf")
|
||
|
|
if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
request := persistedContainerRequest{
|
||
|
|
InputType: inputTypeContainerImage, ImageReference: imageReference,
|
||
|
|
ImmutableReference: resolved.ImmutableReference, ImageDigest: resolved.Digest, Platform: resolved.Platform,
|
||
|
|
TargetPort: targetPort, TargetContainer: targetSlot.ContainerName,
|
||
|
|
PreviousPort: before.ActivePort, PreviousContainer: previousContainer,
|
||
|
|
TargetHealthEndpoint: targetSlot.HealthEndpoint,
|
||
|
|
GatewayBeforePath: beforePath, GatewayAfterPath: afterPath,
|
||
|
|
GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"),
|
||
|
|
}
|
||
|
|
content, err := json.Marshal(request)
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, fmt.Errorf("encode container backend update request: %w", err)
|
||
|
|
}
|
||
|
|
record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{
|
||
|
|
ID: transactionID, IdempotencyKey: serviceBackend + ":container:" + resolved.Digest,
|
||
|
|
Source: sourceLocalCLI, Service: serviceBackend, Request: content,
|
||
|
|
})
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Transaction{}, err
|
||
|
|
}
|
||
|
|
if !created {
|
||
|
|
if err := decodeContainerRequest(record.Request, &request); err != nil {
|
||
|
|
return record, err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return u.runContainerUpdate(ctx, record, request, created, report)
|
||
|
|
}
|
||
|
|
|
||
|
|
// resolveContainerSlots reconciles the committed deployment, gateway and both
|
||
|
|
// exact container names before selecting a target. A fresh installation has no
|
||
|
|
// deployment row, no committed container transaction history and no slot
|
||
|
|
// containers. It starts on the non-routed slot so traffic is exposed only after
|
||
|
|
// health passes.
|
||
|
|
func (u *Updater) resolveContainerSlots(
|
||
|
|
ctx context.Context,
|
||
|
|
activePort int,
|
||
|
|
activeSlot deploymentconfig.BackendSlot,
|
||
|
|
deployment transaction.BackendContainerDeployment,
|
||
|
|
hasDeployment bool,
|
||
|
|
hasHistory bool,
|
||
|
|
) (int, deploymentconfig.BackendSlot, string, error) {
|
||
|
|
inactivePort := otherPort(activePort)
|
||
|
|
inactiveSlot, err := u.config.Backend.SlotForPort(inactivePort)
|
||
|
|
if err != nil {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", err
|
||
|
|
}
|
||
|
|
active, activeErr := u.engine.InspectContainer(ctx, activeSlot.ContainerName)
|
||
|
|
if activeErr != nil && !errors.Is(activeErr, containerengine.ErrNotFound) {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect active backend container %s: %w", activeSlot.ContainerName, activeErr)
|
||
|
|
}
|
||
|
|
inactive, inactiveErr := u.engine.InspectContainer(ctx, inactiveSlot.ContainerName)
|
||
|
|
if inactiveErr != nil && !errors.Is(inactiveErr, containerengine.ErrNotFound) {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect inactive backend container %s: %w", inactiveSlot.ContainerName, inactiveErr)
|
||
|
|
}
|
||
|
|
activeFound := activeErr == nil
|
||
|
|
inactiveFound := inactiveErr == nil
|
||
|
|
|
||
|
|
if hasDeployment {
|
||
|
|
if deployment.ActivePort != activePort {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container port %d does not match gateway active port %d", deployment.ActivePort, activePort)
|
||
|
|
}
|
||
|
|
if deployment.ContainerName != activeSlot.ContainerName {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container %s does not match gateway slot container %s", deployment.ContainerName, activeSlot.ContainerName)
|
||
|
|
}
|
||
|
|
if !activeFound {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed active backend container %s is missing", activeSlot.ContainerName)
|
||
|
|
}
|
||
|
|
if active.ID != deployment.ContainerID {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s identity does not match committed deployment", activeSlot.ContainerName)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if activeFound {
|
||
|
|
if !active.Running || active.Dead {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is not running", activeSlot.ContainerName)
|
||
|
|
}
|
||
|
|
if inactiveFound && inactive.Running && !inactive.Dead {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inactive backend container %s is unexpectedly running", inactiveSlot.ContainerName)
|
||
|
|
}
|
||
|
|
return inactivePort, inactiveSlot, activeSlot.ContainerName, nil
|
||
|
|
}
|
||
|
|
if hasHistory {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing on a server with committed backend container transaction history", activeSlot.ContainerName)
|
||
|
|
}
|
||
|
|
if inactiveFound && inactive.Running && !inactive.Dead {
|
||
|
|
return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing but inactive container %s is running", activeSlot.ContainerName, inactiveSlot.ContainerName)
|
||
|
|
}
|
||
|
|
return inactivePort, inactiveSlot, "", nil
|
||
|
|
}
|
||
|
|
|
||
|
|
type resolvedImage struct {
|
||
|
|
ImmutableReference string
|
||
|
|
Digest string
|
||
|
|
Platform containerengine.Platform
|
||
|
|
}
|
||
|
|
|
||
|
|
func (u *Updater) pullAndResolveImage(ctx context.Context, imageReference string) (resolvedImage, error) {
|
||
|
|
named, err := reference.ParseNormalizedNamed(imageReference)
|
||
|
|
if err != nil {
|
||
|
|
return resolvedImage{}, fmt.Errorf("parse container image reference: %w", err)
|
||
|
|
}
|
||
|
|
if _, ok := named.(reference.Tagged); !ok {
|
||
|
|
return resolvedImage{}, errors.New("--container-image requires a tag-qualified image reference")
|
||
|
|
}
|
||
|
|
if err := u.engine.Ping(ctx); err != nil {
|
||
|
|
return resolvedImage{}, err
|
||
|
|
}
|
||
|
|
if err := u.engine.PullImage(ctx, imageReference); err != nil {
|
||
|
|
return resolvedImage{}, err
|
||
|
|
}
|
||
|
|
image, err := u.engine.InspectImage(ctx, imageReference)
|
||
|
|
if err != nil {
|
||
|
|
return resolvedImage{}, fmt.Errorf("inspect pulled backend image: %w", err)
|
||
|
|
}
|
||
|
|
if image.Platform.OS == "" || image.Platform.Architecture == "" {
|
||
|
|
return resolvedImage{}, errors.New("pulled backend image does not report an exact platform")
|
||
|
|
}
|
||
|
|
repository := reference.TrimNamed(named).Name()
|
||
|
|
matches := make(map[string]string)
|
||
|
|
for _, value := range image.RepoDigests {
|
||
|
|
digested, err := reference.ParseNormalizedNamed(value)
|
||
|
|
if err != nil {
|
||
|
|
return resolvedImage{}, fmt.Errorf("parse pulled repository digest %q: %w", value, err)
|
||
|
|
}
|
||
|
|
withDigest, ok := digested.(reference.Digested)
|
||
|
|
if !ok || reference.TrimNamed(digested).Name() != repository {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
matches[withDigest.Digest().String()] = value
|
||
|
|
}
|
||
|
|
if len(matches) == 0 {
|
||
|
|
return resolvedImage{}, fmt.Errorf("pulled backend image has no repository digest for %s", repository)
|
||
|
|
}
|
||
|
|
if len(matches) != 1 {
|
||
|
|
return resolvedImage{}, fmt.Errorf("pulled backend image has multiple repository digests for %s", repository)
|
||
|
|
}
|
||
|
|
for digest, immutableReference := range matches {
|
||
|
|
return resolvedImage{ImmutableReference: immutableReference, Digest: digest, Platform: image.Platform}, nil
|
||
|
|
}
|
||
|
|
return resolvedImage{}, errors.New("repository digest resolution produced no result")
|
||
|
|
}
|
||
|
|
|
||
|
|
func (u *Updater) runContainerUpdate(ctx context.Context, record transaction.Transaction, request persistedContainerRequest, created bool, report ProgressReporter) (transaction.Transaction, error) {
|
||
|
|
message := "Resuming container backend transaction " + record.ID
|
||
|
|
if created {
|
||
|
|
message = "Created container backend transaction " + record.ID
|
||
|
|
}
|
||
|
|
reportProgress(report, Progress{TransactionID: record.ID, State: record.State, Message: message})
|
||
|
|
if record.State.Terminal() {
|
||
|
|
return terminalResult(record)
|
||
|
|
}
|
||
|
|
executorRequest := backendexecutor.Request{
|
||
|
|
ImageAcquisition: backendexecutor.ImageAcquisitionPull,
|
||
|
|
ImageReference: request.ImmutableReference, ExpectedImageDigest: request.ImageDigest, Platform: request.Platform,
|
||
|
|
ContainerName: request.TargetContainer, Port: request.TargetPort,
|
||
|
|
PortEnvironmentKey: deploymentconfig.ContainerPortEnvironment,
|
||
|
|
ConfigSource: u.containerConfigSource, ConfigTarget: u.containerConfigTarget,
|
||
|
|
TmpSource: u.containerTmpSource, TmpTarget: u.containerTmpTarget,
|
||
|
|
ConfigEnvironmentKey: deploymentconfig.ContainerConfigEnvironment,
|
||
|
|
ConfigLocation: deploymentconfig.ContainerConfigLocation,
|
||
|
|
RestartPolicy: containerengine.RestartPolicy{Name: "no"},
|
||
|
|
HealthEndpoint: request.TargetHealthEndpoint,
|
||
|
|
}
|
||
|
|
switch record.State {
|
||
|
|
case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting:
|
||
|
|
if err := u.containerExecutor.Run(ctx, record.ID, executorRequest); err != nil {
|
||
|
|
current, readErr := u.store.Transaction(ctx, record.ID)
|
||
|
|
if readErr != nil || current.State == transaction.StateFailed {
|
||
|
|
return u.currentWithError(ctx, record.ID, errors.Join(err, readErr))
|
||
|
|
}
|
||
|
|
before, beforeErr := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
|
||
|
|
after, afterErr := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
|
||
|
|
if snapshotErr := errors.Join(beforeErr, afterErr); snapshotErr != nil {
|
||
|
|
return u.currentWithError(ctx, record.ID, errors.Join(err, snapshotErr))
|
||
|
|
}
|
||
|
|
rollbackErr := u.rollbackContainer(ctx, record.ID, request, before, after, err)
|
||
|
|
return u.currentWithError(ctx, record.ID, rollbackErr)
|
||
|
|
}
|
||
|
|
case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, transaction.StateRollingBack:
|
||
|
|
default:
|
||
|
|
return u.currentWithError(ctx, record.ID, fmt.Errorf("container backend update cannot resume transaction %s in state %s", record.ID, record.State))
|
||
|
|
}
|
||
|
|
if err := u.switchAndCommitContainer(ctx, record.ID, request, report); err != nil {
|
||
|
|
return u.currentWithError(ctx, record.ID, err)
|
||
|
|
}
|
||
|
|
return u.store.Transaction(ctx, record.ID)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (u *Updater) switchAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error {
|
||
|
|
before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for {
|
||
|
|
record, err := u.store.Transaction(ctx, transactionID)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
switch record.State {
|
||
|
|
case transaction.StateSwitching:
|
||
|
|
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)})
|
||
|
|
operation := &gatewayOperation{controller: u.gateway, before: before, after: after, receiptPath: request.GatewayReceiptPath}
|
||
|
|
if _, err := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewaySwitchIntent(request), operation); err != nil {
|
||
|
|
return u.rollbackContainer(ctx, transactionID, request, before, after, err)
|
||
|
|
}
|
||
|
|
if _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy container slot"); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
case transaction.StateVerifying:
|
||
|
|
message := "container backend switch verified"
|
||
|
|
if request.PreviousContainer != "" {
|
||
|
|
message += "; previous container draining"
|
||
|
|
}
|
||
|
|
if _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, message); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
case transaction.StateDraining:
|
||
|
|
if request.PreviousContainer != "" {
|
||
|
|
reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Draining previous backend container for %s", u.drain)})
|
||
|
|
if err := waitContext(ctx, u.drain); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
operation := &containerStopOperation{engine: u.engine, name: request.PreviousContainer}
|
||
|
|
if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousContainerIntent(request), operation); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
target, err := u.engine.InspectContainer(ctx, request.TargetContainer)
|
||
|
|
if err != nil {
|
||
|
|
return fmt.Errorf("inspect target backend container before commit: %w", err)
|
||
|
|
}
|
||
|
|
if !target.Running || target.Dead || target.ID == "" {
|
||
|
|
return fmt.Errorf("target backend container %s is not running with an exact identity", request.TargetContainer)
|
||
|
|
}
|
||
|
|
if target.ImageReference != request.ImmutableReference {
|
||
|
|
return fmt.Errorf("target backend container %s image reference does not match the transaction", request.TargetContainer)
|
||
|
|
}
|
||
|
|
_, err = u.store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{
|
||
|
|
ActivePort: request.TargetPort,
|
||
|
|
ContainerName: request.TargetContainer,
|
||
|
|
ImageDigest: request.ImageDigest,
|
||
|
|
ContainerID: target.ID,
|
||
|
|
}, "container backend update committed")
|
||
|
|
if err == nil {
|
||
|
|
reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Container backend update committed"})
|
||
|
|
}
|
||
|
|
return err
|
||
|
|
case transaction.StateCommitted:
|
||
|
|
return nil
|
||
|
|
case transaction.StateRollingBack:
|
||
|
|
return u.rollbackContainer(ctx, transactionID, request, before, after, errors.New("resuming container backend rollback"))
|
||
|
|
default:
|
||
|
|
return fmt.Errorf("container backend commit cannot continue transaction %s in state %s", transactionID, record.State)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (u *Updater) rollbackContainer(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error {
|
||
|
|
record, err := u.store.Transaction(ctx, transactionID)
|
||
|
|
if err != nil {
|
||
|
|
return errors.Join(cause, err)
|
||
|
|
}
|
||
|
|
if record.State != transaction.StateRollingBack {
|
||
|
|
if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "container backend compensation started"); err != nil {
|
||
|
|
return errors.Join(cause, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
gateway := &gatewayOperation{controller: u.gateway, before: after, after: before, receiptPath: request.GatewayReceiptPath + ".restore"}
|
||
|
|
_, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewayRestoreIntent(request), gateway)
|
||
|
|
stop := &containerStopOperation{engine: u.engine, name: request.TargetContainer}
|
||
|
|
_, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetContainerIntent(request), stop)
|
||
|
|
if err := errors.Join(gatewayErr, stopErr); err != nil {
|
||
|
|
return errors.Join(cause, err)
|
||
|
|
}
|
||
|
|
_, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "container backend compensation completed")
|
||
|
|
return errors.Join(cause, transitionErr)
|
||
|
|
}
|
||
|
|
|
||
|
|
type containerStopOperation struct {
|
||
|
|
engine containerengine.Engine
|
||
|
|
name string
|
||
|
|
}
|
||
|
|
|
||
|
|
func (o *containerStopOperation) Apply(ctx context.Context) error {
|
||
|
|
err := o.engine.StopContainer(ctx, o.name)
|
||
|
|
if errors.Is(err, containerengine.ErrNotFound) {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (o *containerStopOperation) Inspect(ctx context.Context) (transaction.Inspection, error) {
|
||
|
|
record, err := o.engine.InspectContainer(ctx, o.name)
|
||
|
|
if errors.Is(err, containerengine.ErrNotFound) {
|
||
|
|
return transaction.Inspection{Status: transaction.InspectionApplied}, nil
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
return transaction.Inspection{}, err
|
||
|
|
}
|
||
|
|
result, _ := json.Marshal(record)
|
||
|
|
if !record.Running || record.Dead {
|
||
|
|
return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil
|
||
|
|
}
|
||
|
|
return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func containerGatewaySwitchIntent(request persistedContainerRequest) transaction.StepIntent {
|
||
|
|
return stepIntent("backend.container.gateway.switch", "switch host Nginx to container backend", struct {
|
||
|
|
BeforePort int `json:"beforePort"`
|
||
|
|
AfterPort int `json:"afterPort"`
|
||
|
|
}{request.PreviousPort, request.TargetPort})
|
||
|
|
}
|
||
|
|
|
||
|
|
func containerGatewayRestoreIntent(request persistedContainerRequest) transaction.StepIntent {
|
||
|
|
return stepIntent("backend.container.gateway.restore", "restore host Nginx after container backend failure", struct {
|
||
|
|
Port int `json:"port"`
|
||
|
|
}{request.PreviousPort})
|
||
|
|
}
|
||
|
|
|
||
|
|
func stopPreviousContainerIntent(request persistedContainerRequest) transaction.StepIntent {
|
||
|
|
return stepIntent("backend.previous-container.stop", "stop previous backend container after drain", struct {
|
||
|
|
ContainerName string `json:"containerName"`
|
||
|
|
}{request.PreviousContainer})
|
||
|
|
}
|
||
|
|
|
||
|
|
func stopTargetContainerIntent(request persistedContainerRequest) transaction.StepIntent {
|
||
|
|
return stepIntent("backend.target-container.stop", "stop compensated backend container", struct {
|
||
|
|
ContainerName string `json:"containerName"`
|
||
|
|
}{request.TargetContainer})
|
||
|
|
}
|
||
|
|
|
||
|
|
func decodeContainerRequest(content json.RawMessage, request *persistedContainerRequest) error {
|
||
|
|
decoder := json.NewDecoder(bytes.NewReader(content))
|
||
|
|
decoder.DisallowUnknownFields()
|
||
|
|
if err := decoder.Decode(request); err != nil {
|
||
|
|
return fmt.Errorf("decode persisted container backend request: %w", err)
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
var _ transaction.Operation = (*containerStopOperation)(nil)
|