package backendupdate import ( "bytes" "context" "crypto/rand" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "github.com/distribution/reference" "yms-daemon/internal/backendexecutor" "yms-daemon/internal/containerengine" "yms-daemon/internal/deploymentconfig" "yms-daemon/internal/hostnginx" "yms-daemon/internal/transaction" ) const inputTypeContainerImage = "container-image" type persistedContainerRequest struct { InputType string `json:"inputType"` ImageReference string `json:"imageReference"` ImmutableReference string `json:"immutableReference"` ImageDigest string `json:"imageDigest"` Platform containerengine.Platform `json:"platform"` TargetPort int `json:"targetPort"` TargetContainer string `json:"targetContainer"` PreviousPort int `json:"previousPort"` PreviousContainer string `json:"previousContainer"` TargetHealthEndpoint string `json:"targetHealthEndpoint"` StartLog bool `json:"startLog"` GatewayBeforePath string `json:"gatewayBeforePath"` GatewayAfterPath string `json:"gatewayAfterPath"` GatewayReceiptPath string `json:"gatewayReceiptPath"` } // UpdateContainerImage pulls one development image, freezes its repository // digest, and updates the inactive Docker backend slot. func (u *Updater) UpdateContainerImage(ctx context.Context, imageReference string, startLog bool, report ProgressReporter) (transaction.Transaction, error) { if u.containerExecutor == nil || u.engine == nil { return transaction.Transaction{}, errors.New("container backend updater is not configured") } if strings.TrimSpace(imageReference) != imageReference || imageReference == "" { return transaction.Transaction{}, errors.New("exact container image reference is required") } configInfo, err := os.Lstat(u.containerConfigSource) if err != nil { return transaction.Transaction{}, fmt.Errorf("inspect backend configuration %s: %w", u.containerConfigSource, err) } if !configInfo.Mode().IsRegular() || configInfo.Mode()&os.ModeSymlink != 0 { return transaction.Transaction{}, fmt.Errorf("backend configuration is not a direct regular file: %s", u.containerConfigSource) } tmpInfo, err := os.Lstat(u.containerTmpSource) if err != nil { return transaction.Transaction{}, fmt.Errorf("inspect backend temporary directory %s: %w", u.containerTmpSource, err) } if !tmpInfo.IsDir() || tmpInfo.Mode()&os.ModeSymlink != 0 { return transaction.Transaction{}, fmt.Errorf("backend temporary path is not a direct directory: %s", u.containerTmpSource) } active, err := u.store.ActiveTransaction(ctx) if err == nil { var request persistedContainerRequest if decodeErr := decodeContainerRequest(active.Request, &request); decodeErr != nil { return active, decodeErr } if active.Service != serviceBackend || request.InputType != inputTypeContainerImage { return active, &transaction.ActiveTransactionError{TransactionID: active.ID} } if request.ImageReference != imageReference { return active, &transaction.ActiveTransactionError{TransactionID: active.ID} } return u.runContainerUpdate(ctx, active, request, false, report) } if !errors.Is(err, transaction.ErrNotFound) { return transaction.Transaction{}, err } reportProgress(report, Progress{Message: "Pulling backend image " + imageReference}) resolved, err := u.pullAndResolveImage(ctx, imageReference) if err != nil { return transaction.Transaction{}, err } reportProgress(report, Progress{Message: "Backend image resolved: " + resolved.ImmutableReference}) before, err := u.gateway.Read() if err != nil { return transaction.Transaction{}, err } activeSlot, err := u.config.Backend.SlotForPort(before.ActivePort) if err != nil { return transaction.Transaction{}, err } deployment, deploymentErr := u.store.BackendContainerDeployment(ctx) hasDeployment := deploymentErr == nil if deploymentErr != nil && !errors.Is(deploymentErr, transaction.ErrNotFound) { return transaction.Transaction{}, deploymentErr } hasHistory, err := u.store.HasCommittedBackendContainerTransactionHistory(ctx) if err != nil { return transaction.Transaction{}, err } targetPort, targetSlot, previousContainer, err := u.resolveContainerSlots( ctx, before.ActivePort, activeSlot, deployment, hasDeployment, hasHistory, ) if err != nil { return transaction.Transaction{}, err } afterContent, err := hostnginx.RenderBackendPort(before.Content, targetPort) if err != nil { return transaction.Transaction{}, err } transactionID := rand.Text() transactionRoot := filepath.Join(u.workRoot, transactionID) if err := os.MkdirAll(transactionRoot, 0o750); err != nil { return transaction.Transaction{}, fmt.Errorf("create container backend transaction directory: %w", err) } beforePath := filepath.Join(transactionRoot, "gateway.before.conf") afterPath := filepath.Join(transactionRoot, "gateway.after.conf") if err := writeImmutableFile(beforePath, before.Content, 0o640); err != nil { return transaction.Transaction{}, err } if err := writeImmutableFile(afterPath, afterContent, 0o640); err != nil { return transaction.Transaction{}, err } request := persistedContainerRequest{ InputType: inputTypeContainerImage, ImageReference: imageReference, ImmutableReference: resolved.ImmutableReference, ImageDigest: resolved.Digest, Platform: resolved.Platform, TargetPort: targetPort, TargetContainer: targetSlot.ContainerName, PreviousPort: before.ActivePort, PreviousContainer: previousContainer, TargetHealthEndpoint: targetSlot.HealthEndpoint, StartLog: startLog, GatewayBeforePath: beforePath, GatewayAfterPath: afterPath, GatewayReceiptPath: filepath.Join(transactionRoot, "gateway.applied"), } content, err := json.Marshal(request) if err != nil { return transaction.Transaction{}, fmt.Errorf("encode container backend update request: %w", err) } record, created, err := u.store.CreateTransaction(ctx, transaction.CreateRequest{ ID: transactionID, IdempotencyKey: serviceBackend + ":container:" + resolved.Digest, Source: sourceLocalCLI, Service: serviceBackend, Request: content, }) if err != nil { return transaction.Transaction{}, err } if !created { if err := decodeContainerRequest(record.Request, &request); err != nil { return record, err } } return u.runContainerUpdate(ctx, record, request, created, report) } // resolveContainerSlots reconciles the committed deployment, gateway and both // exact container names before selecting a target. A fresh installation has no // deployment row, no committed container transaction history and no slot // containers. It starts on the non-routed slot so traffic is exposed only after // health passes. func (u *Updater) resolveContainerSlots( ctx context.Context, activePort int, activeSlot deploymentconfig.BackendSlot, deployment transaction.BackendContainerDeployment, hasDeployment bool, hasHistory bool, ) (int, deploymentconfig.BackendSlot, string, error) { inactivePort := otherPort(activePort) inactiveSlot, err := u.config.Backend.SlotForPort(inactivePort) if err != nil { return 0, deploymentconfig.BackendSlot{}, "", err } active, activeErr := u.engine.InspectContainer(ctx, activeSlot.ContainerName) if activeErr != nil && !errors.Is(activeErr, containerengine.ErrNotFound) { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect active backend container %s: %w", activeSlot.ContainerName, activeErr) } inactive, inactiveErr := u.engine.InspectContainer(ctx, inactiveSlot.ContainerName) if inactiveErr != nil && !errors.Is(inactiveErr, containerengine.ErrNotFound) { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inspect inactive backend container %s: %w", inactiveSlot.ContainerName, inactiveErr) } activeFound := activeErr == nil inactiveFound := inactiveErr == nil if hasDeployment { if deployment.ActivePort != activePort { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container port %d does not match gateway active port %d", deployment.ActivePort, activePort) } if deployment.ContainerName != activeSlot.ContainerName { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed backend container %s does not match gateway slot container %s", deployment.ContainerName, activeSlot.ContainerName) } if !activeFound { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("committed active backend container %s is missing", activeSlot.ContainerName) } if active.ID != deployment.ContainerID { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s identity does not match committed deployment", activeSlot.ContainerName) } } if activeFound { if !active.Running || active.Dead { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is not running", activeSlot.ContainerName) } if inactiveFound && inactive.Running && !inactive.Dead { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("inactive backend container %s is unexpectedly running", inactiveSlot.ContainerName) } return inactivePort, inactiveSlot, activeSlot.ContainerName, nil } if hasHistory { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing on a server with committed backend container transaction history", activeSlot.ContainerName) } if inactiveFound && inactive.Running && !inactive.Dead { return 0, deploymentconfig.BackendSlot{}, "", fmt.Errorf("active backend container %s is missing but inactive container %s is running", activeSlot.ContainerName, inactiveSlot.ContainerName) } return inactivePort, inactiveSlot, "", nil } type resolvedImage struct { ImmutableReference string Digest string Platform containerengine.Platform } func (u *Updater) pullAndResolveImage(ctx context.Context, imageReference string) (resolvedImage, error) { named, err := reference.ParseNormalizedNamed(imageReference) if err != nil { return resolvedImage{}, fmt.Errorf("parse container image reference: %w", err) } if _, ok := named.(reference.Tagged); !ok { return resolvedImage{}, errors.New("--container-image requires a tag-qualified image reference") } if err := u.engine.Ping(ctx); err != nil { return resolvedImage{}, err } if err := u.engine.PullImage(ctx, imageReference); err != nil { return resolvedImage{}, err } image, err := u.engine.InspectImage(ctx, imageReference) if err != nil { return resolvedImage{}, fmt.Errorf("inspect pulled backend image: %w", err) } if image.Platform.OS == "" || image.Platform.Architecture == "" { return resolvedImage{}, errors.New("pulled backend image does not report an exact platform") } repository := reference.TrimNamed(named).Name() matches := make(map[string]string) for _, value := range image.RepoDigests { digested, err := reference.ParseNormalizedNamed(value) if err != nil { return resolvedImage{}, fmt.Errorf("parse pulled repository digest %q: %w", value, err) } withDigest, ok := digested.(reference.Digested) if !ok || reference.TrimNamed(digested).Name() != repository { continue } matches[withDigest.Digest().String()] = value } if len(matches) == 0 { return resolvedImage{}, fmt.Errorf("pulled backend image has no repository digest for %s", repository) } if len(matches) != 1 { return resolvedImage{}, fmt.Errorf("pulled backend image has multiple repository digests for %s", repository) } for digest, immutableReference := range matches { return resolvedImage{ImmutableReference: immutableReference, Digest: digest, Platform: image.Platform}, nil } return resolvedImage{}, errors.New("repository digest resolution produced no result") } func (u *Updater) runContainerUpdate(ctx context.Context, record transaction.Transaction, request persistedContainerRequest, created bool, report ProgressReporter) (transaction.Transaction, error) { message := "Resuming container backend transaction " + record.ID if created { message = "Created container backend transaction " + record.ID } reportProgress(report, Progress{TransactionID: record.ID, State: record.State, Message: message}) if record.State.Terminal() { return terminalResult(record) } executorRequest := backendexecutor.Request{ ImageAcquisition: backendexecutor.ImageAcquisitionPull, ImageReference: request.ImmutableReference, ExpectedImageDigest: request.ImageDigest, Platform: request.Platform, ContainerName: request.TargetContainer, Port: request.TargetPort, PortEnvironmentKey: deploymentconfig.ContainerPortEnvironment, ConfigSource: u.containerConfigSource, ConfigTarget: u.containerConfigTarget, TmpSource: u.containerTmpSource, TmpTarget: u.containerTmpTarget, ConfigEnvironmentKey: deploymentconfig.ContainerConfigEnvironment, ConfigLocation: deploymentconfig.ContainerConfigLocation, RestartPolicy: containerengine.RestartPolicy{Name: "no"}, HealthEndpoint: request.TargetHealthEndpoint, StartLog: request.StartLog, LogReporter: func(line string) { reportProgress(report, Progress{TransactionID: record.ID, State: transaction.StateStarting, Message: "CONTAINER LOG " + line}) }, } switch record.State { case transaction.StateCreated, transaction.StateValidating, transaction.StatePrepared, transaction.StateStarting: if err := u.containerExecutor.Run(ctx, record.ID, executorRequest); err != nil { current, readErr := u.store.Transaction(ctx, record.ID) if readErr != nil || current.State == transaction.StateFailed { return u.currentWithError(ctx, record.ID, errors.Join(err, readErr)) } before, beforeErr := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort) after, afterErr := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort) if snapshotErr := errors.Join(beforeErr, afterErr); snapshotErr != nil { return u.currentWithError(ctx, record.ID, errors.Join(err, snapshotErr)) } rollbackErr := u.rollbackContainer(ctx, record.ID, request, before, after, err) return u.currentWithError(ctx, record.ID, rollbackErr) } case transaction.StateSwitching, transaction.StateVerifying, transaction.StateDraining, transaction.StateRollingBack: default: return u.currentWithError(ctx, record.ID, fmt.Errorf("container backend update cannot resume transaction %s in state %s", record.ID, record.State)) } if err := u.switchAndCommitContainer(ctx, record.ID, request, report); err != nil { return u.currentWithError(ctx, record.ID, err) } return u.store.Transaction(ctx, record.ID) } func (u *Updater) switchAndCommitContainer(ctx context.Context, transactionID string, request persistedContainerRequest, report ProgressReporter) error { before, err := readGatewaySnapshot(request.GatewayBeforePath, request.PreviousPort) if err != nil { return err } after, err := readGatewaySnapshot(request.GatewayAfterPath, request.TargetPort) if err != nil { return err } for { record, err := u.store.Transaction(ctx, transactionID) if err != nil { return err } switch record.State { case transaction.StateSwitching: reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Switching host Nginx backend traffic to port %d", request.TargetPort)}) operation := &gatewayOperation{controller: u.gateway, before: before, after: after, receiptPath: request.GatewayReceiptPath} if _, err := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewaySwitchIntent(request), operation); err != nil { return u.rollbackContainer(ctx, transactionID, request, before, after, err) } if _, err := u.store.Transition(ctx, transactionID, transaction.StateVerifying, "host Nginx now routes backend traffic to the healthy container slot"); err != nil { return err } case transaction.StateVerifying: message := "container backend switch verified" if request.PreviousContainer != "" { message += "; previous container draining" } if _, err := u.store.Transition(ctx, transactionID, transaction.StateDraining, message); err != nil { return err } case transaction.StateDraining: if request.PreviousContainer != "" { reportProgress(report, Progress{TransactionID: transactionID, State: record.State, Message: fmt.Sprintf("Draining previous backend container for %s", u.drain)}) if err := waitContext(ctx, u.drain); err != nil { return err } operation := &containerStopOperation{engine: u.engine, name: request.PreviousContainer} if _, err := u.coordinator.ExecuteStep(ctx, transactionID, stopPreviousContainerIntent(request), operation); err != nil { return err } } target, err := u.engine.InspectContainer(ctx, request.TargetContainer) if err != nil { return fmt.Errorf("inspect target backend container before commit: %w", err) } if !target.Running || target.Dead || target.ID == "" { return fmt.Errorf("target backend container %s is not running with an exact identity", request.TargetContainer) } if target.ImageReference != request.ImmutableReference { return fmt.Errorf("target backend container %s image reference does not match the transaction", request.TargetContainer) } _, err = u.store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{ ActivePort: request.TargetPort, ContainerName: request.TargetContainer, ImageDigest: request.ImageDigest, ContainerID: target.ID, }, "container backend update committed") if err == nil { reportProgress(report, Progress{TransactionID: transactionID, State: transaction.StateCommitted, Message: "Container backend update committed"}) } return err case transaction.StateCommitted: return nil case transaction.StateRollingBack: return u.rollbackContainer(ctx, transactionID, request, before, after, errors.New("resuming container backend rollback")) default: return fmt.Errorf("container backend commit cannot continue transaction %s in state %s", transactionID, record.State) } } } func (u *Updater) rollbackContainer(ctx context.Context, transactionID string, request persistedContainerRequest, before hostnginx.Snapshot, after hostnginx.Snapshot, cause error) error { record, err := u.store.Transaction(ctx, transactionID) if err != nil { return errors.Join(cause, err) } if record.State != transaction.StateRollingBack { if _, err := u.store.Transition(ctx, transactionID, transaction.StateRollingBack, "container backend compensation started"); err != nil { return errors.Join(cause, err) } } gateway := &gatewayOperation{controller: u.gateway, before: after, after: before, receiptPath: request.GatewayReceiptPath + ".restore"} _, gatewayErr := u.coordinator.ExecuteStep(ctx, transactionID, containerGatewayRestoreIntent(request), gateway) stop := &containerStopOperation{engine: u.engine, name: request.TargetContainer} _, stopErr := u.coordinator.ExecuteStep(ctx, transactionID, stopTargetContainerIntent(request), stop) if err := errors.Join(gatewayErr, stopErr); err != nil { return errors.Join(cause, err) } _, transitionErr := u.store.Transition(ctx, transactionID, transaction.StateRolledBack, "container backend compensation completed") return errors.Join(cause, transitionErr) } type containerStopOperation struct { engine containerengine.Engine name string } func (o *containerStopOperation) Apply(ctx context.Context) error { err := o.engine.StopContainer(ctx, o.name) if errors.Is(err, containerengine.ErrNotFound) { return nil } return err } func (o *containerStopOperation) Inspect(ctx context.Context) (transaction.Inspection, error) { record, err := o.engine.InspectContainer(ctx, o.name) if errors.Is(err, containerengine.ErrNotFound) { return transaction.Inspection{Status: transaction.InspectionApplied}, nil } if err != nil { return transaction.Inspection{}, err } result, _ := json.Marshal(record) if !record.Running || record.Dead { return transaction.Inspection{Status: transaction.InspectionApplied, Result: result}, nil } return transaction.Inspection{Status: transaction.InspectionNotApplied, Result: result}, nil } func containerGatewaySwitchIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.container.gateway.switch", "switch host Nginx to container backend", struct { BeforePort int `json:"beforePort"` AfterPort int `json:"afterPort"` }{request.PreviousPort, request.TargetPort}) } func containerGatewayRestoreIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.container.gateway.restore", "restore host Nginx after container backend failure", struct { Port int `json:"port"` }{request.PreviousPort}) } func stopPreviousContainerIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.previous-container.stop", "stop previous backend container after drain", struct { ContainerName string `json:"containerName"` }{request.PreviousContainer}) } func stopTargetContainerIntent(request persistedContainerRequest) transaction.StepIntent { return stepIntent("backend.target-container.stop", "stop compensated backend container", struct { ContainerName string `json:"containerName"` }{request.TargetContainer}) } func decodeContainerRequest(content json.RawMessage, request *persistedContainerRequest) error { decoder := json.NewDecoder(bytes.NewReader(content)) decoder.DisallowUnknownFields() if err := decoder.Decode(request); err != nil { return fmt.Errorf("decode persisted container backend request: %w", err) } return nil } var _ transaction.Operation = (*containerStopOperation)(nil)