371 lines
15 KiB
Go
371 lines
15 KiB
Go
package backendupdate
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"yms-daemon/internal/backendexecutor"
|
|
"yms-daemon/internal/containerengine"
|
|
"yms-daemon/internal/deploymentconfig"
|
|
"yms-daemon/internal/hostnginx"
|
|
"yms-daemon/internal/transaction"
|
|
)
|
|
|
|
const containerTestImage = "harbor.ymswell.asia/ymswell/glory-ymswell:20260813-184902-a37bf50d-v1.1.8.1"
|
|
const containerTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
|
|
func TestContainerUpdaterPullsSwitchesAndStopsPreviousSlot(t *testing.T) {
|
|
ctx := context.Background()
|
|
updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{
|
|
"backend-8080": {
|
|
ID: "existing-backend-8080", Name: "backend-8080", Running: true,
|
|
},
|
|
}, 0)
|
|
|
|
record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
|
|
if err != nil {
|
|
t.Fatalf("update container backend: %v", err)
|
|
}
|
|
if record.State != transaction.StateCommitted || gateway.snapshot.ActivePort != 8081 {
|
|
t.Fatalf("unexpected committed container update: record=%+v gateway=%+v", record, gateway.snapshot)
|
|
}
|
|
if engine.containers["backend-8080"].Running {
|
|
t.Fatal("previous backend container is still running")
|
|
}
|
|
if len(engine.stopped) != 1 || engine.stopped[0] != "backend-8080" {
|
|
t.Fatalf("unexpected stopped containers: %+v", engine.stopped)
|
|
}
|
|
assertCreatedContainerSpec(t, engine.lastCreateSpec)
|
|
assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
|
|
}
|
|
|
|
func TestContainerUpdaterFirstInstallCreatesInactiveSlotBeforeSwitch(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
updater, store, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, time.Hour)
|
|
var progress []Progress
|
|
gateway.beforeApply = func(snapshot hostnginx.Snapshot) error {
|
|
target, found := engine.containers["backend-8081"]
|
|
if !found || !target.Running || engine.healthChecks == 0 {
|
|
return errors.New("gateway switch occurred before the target container passed health checking")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
record, err := updater.UpdateContainerImage(ctx, containerTestImage, true, func(item Progress) {
|
|
progress = append(progress, item)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("first install container backend: %v", err)
|
|
}
|
|
if record.State != transaction.StateCommitted {
|
|
t.Fatalf("unexpected committed first install: record=%+v", record)
|
|
}
|
|
if gateway.snapshot.ActivePort != 8081 || gateway.applyCount != 1 {
|
|
t.Fatalf("first install did not switch once to the healthy inactive slot: gateway=%+v applies=%d", gateway.snapshot, gateway.applyCount)
|
|
}
|
|
target, found := engine.containers["backend-8081"]
|
|
if !found || !target.Running || target.Dead {
|
|
t.Fatalf("first install target container is not running: %+v", target)
|
|
}
|
|
if len(engine.stopped) != 0 {
|
|
t.Fatalf("first install must not stop a previous container: %+v", engine.stopped)
|
|
}
|
|
if engine.healthChecks != 1 {
|
|
t.Fatalf("unexpected first-install health check count: %d", engine.healthChecks)
|
|
}
|
|
for _, item := range progress {
|
|
if strings.HasPrefix(item.Message, "Draining previous backend container") {
|
|
t.Fatalf("first install entered previous-container drain: %+v", progress)
|
|
}
|
|
}
|
|
assertCreatedContainerSpec(t, engine.lastCreateSpec)
|
|
assertCommittedContainerDeployment(t, store, record.ID, 8081, "backend-8081", "container-id-backend-8081")
|
|
}
|
|
|
|
func TestContainerUpdaterRejectsMissingActiveWithPresentInactive(t *testing.T) {
|
|
updater, _, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{
|
|
"backend-8081": {
|
|
ID: "unexpected-backend-8081", Name: "backend-8081", Running: true,
|
|
},
|
|
}, 0)
|
|
|
|
_, err := updater.UpdateContainerImage(context.Background(), containerTestImage, true, nil)
|
|
if err == nil || !strings.Contains(err.Error(), "active backend container backend-8080 is missing but inactive container backend-8081 is running") {
|
|
t.Fatalf("unexpected missing-active result: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestContainerUpdaterRecoversFailedRollbackThenRetriesSameImage(t *testing.T) {
|
|
ctx := context.Background()
|
|
updater, _, engine, gateway := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
|
|
failGateway := true
|
|
gateway.beforeApply = func(hostnginx.Snapshot) error {
|
|
if failGateway {
|
|
return errors.New("host Nginx is unavailable")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
rollingBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
|
|
if err == nil || rollingBack.State != transaction.StateRollingBack {
|
|
t.Fatalf("unexpected failed rollback result: record=%+v err=%v", rollingBack, err)
|
|
}
|
|
target, found := engine.containers["backend-8081"]
|
|
if !found || target.Running {
|
|
t.Fatalf("failed update target was not stopped: %+v", target)
|
|
}
|
|
|
|
failGateway = false
|
|
rolledBack, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
|
|
if err == nil || rolledBack.State != transaction.StateRolledBack {
|
|
t.Fatalf("unexpected resumed rollback result: record=%+v err=%v", rolledBack, err)
|
|
}
|
|
|
|
committed, err := updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
|
|
if err != nil || committed.State != transaction.StateCommitted {
|
|
t.Fatalf("retry same image after rollback: record=%+v err=%v", committed, err)
|
|
}
|
|
if gateway.snapshot.ActivePort != 8081 {
|
|
t.Fatalf("retry did not switch to the healthy container: %+v", gateway.snapshot)
|
|
}
|
|
}
|
|
|
|
func TestContainerUpdaterRejectsMissingCommittedContainer(t *testing.T) {
|
|
ctx := context.Background()
|
|
updater, store, _, _ := newContainerUpdaterFixture(t, map[string]containerengine.Container{}, 0)
|
|
transactionID := "committed-container-transaction"
|
|
_, _, err := store.CreateTransaction(ctx, transaction.CreateRequest{
|
|
ID: transactionID, IdempotencyKey: "backend:container:previous", Source: sourceLocalCLI,
|
|
Service: serviceBackend, Request: []byte(`{"inputType":"container-image"}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create previous backend transaction: %v", err)
|
|
}
|
|
for _, state := range []transaction.State{
|
|
transaction.StateValidating,
|
|
transaction.StatePrepared,
|
|
transaction.StateStarting,
|
|
transaction.StateSwitching,
|
|
transaction.StateVerifying,
|
|
transaction.StateDraining,
|
|
} {
|
|
if _, err := store.Transition(ctx, transactionID, state, "seed committed deployment"); err != nil {
|
|
t.Fatalf("transition previous backend transaction to %s: %v", state, err)
|
|
}
|
|
}
|
|
if _, err := store.CommitBackendContainerDeployment(ctx, transactionID, transaction.BackendContainerDeployment{
|
|
ActivePort: 8080, ContainerName: "backend-8080", ImageDigest: containerTestDigest, ContainerID: "missing-backend-8080",
|
|
}, "seed committed backend container deployment"); err != nil {
|
|
t.Fatalf("commit previous backend deployment: %v", err)
|
|
}
|
|
|
|
_, err = updater.UpdateContainerImage(ctx, containerTestImage, true, nil)
|
|
if err == nil || !strings.Contains(err.Error(), "committed active backend container backend-8080 is missing") {
|
|
t.Fatalf("unexpected committed-container drift result: %v", err)
|
|
}
|
|
}
|
|
|
|
func newContainerUpdaterFixture(
|
|
t *testing.T,
|
|
containers map[string]containerengine.Container,
|
|
drain time.Duration,
|
|
) (*Updater, *transaction.Store, *containerUpdateEngine, *memoryGateway) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
root := t.TempDir()
|
|
store, err := transaction.OpenStore(ctx, filepath.Join(root, "transactions.db"))
|
|
if err != nil {
|
|
t.Fatalf("open transaction store: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
if err := store.Close(); err != nil {
|
|
t.Errorf("close transaction store: %v", err)
|
|
}
|
|
})
|
|
coordinator, err := transaction.NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
|
if err != nil {
|
|
t.Fatalf("create transaction coordinator: %v", err)
|
|
}
|
|
engine := &containerUpdateEngine{
|
|
image: containerengine.Image{
|
|
ID: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
RepoDigests: []string{"harbor.ymswell.asia/ymswell/glory-ymswell@" + containerTestDigest},
|
|
Platform: containerengine.Platform{OS: "linux", Architecture: "amd64"},
|
|
},
|
|
containers: containers,
|
|
}
|
|
configSource := filepath.Join(root, "yms.yaml")
|
|
if err := os.WriteFile(configSource, []byte("server: {}\n"), 0o600); err != nil {
|
|
t.Fatalf("write backend configuration: %v", err)
|
|
}
|
|
tmpSource := filepath.Join(root, "tmp")
|
|
if err := os.Mkdir(tmpSource, 0o755); err != nil {
|
|
t.Fatalf("create backend temporary directory: %v", err)
|
|
}
|
|
client := &http.Client{Transport: containerUpdateRoundTripFunc(func(request *http.Request) (*http.Response, error) {
|
|
engine.healthChecks++
|
|
return &http.Response{
|
|
StatusCode: http.StatusOK,
|
|
Header: make(http.Header),
|
|
Body: io.NopCloser(bytes.NewBufferString(`{"status":"UP","components":{"ping":{"status":"UP"}}}`)),
|
|
Request: request,
|
|
}, nil
|
|
})}
|
|
executor, err := backendexecutor.New(store, coordinator, engine, client)
|
|
if err != nil {
|
|
t.Fatalf("create backend container executor: %v", err)
|
|
}
|
|
gateway := &memoryGateway{snapshot: hostnginx.Snapshot{Content: []byte(serverConfiguration8080), ActivePort: 8080}}
|
|
updater := &Updater{
|
|
config: deploymentconfig.Config{
|
|
Daemon: deploymentconfig.Daemon{Environment: deploymentconfig.EnvironmentDev},
|
|
Backend: deploymentconfig.Backend{Type: deploymentconfig.BackendTypeContainer, Slot: deploymentconfig.BackendSlots{
|
|
Port8080: deploymentconfig.BackendSlot{ContainerName: "backend-8080", HealthEndpoint: "http://127.0.0.1:8080/yms/actuator/health"},
|
|
Port8081: deploymentconfig.BackendSlot{ContainerName: "backend-8081", HealthEndpoint: "http://127.0.0.1:8081/yms/actuator/health"},
|
|
}},
|
|
},
|
|
workRoot: filepath.Join(root, "work"), store: store, coordinator: coordinator,
|
|
gateway: gateway, containerExecutor: executor, engine: engine, drain: drain,
|
|
containerConfigSource: configSource, containerConfigTarget: deploymentconfig.ContainerConfigTarget,
|
|
containerTmpSource: tmpSource, containerTmpTarget: deploymentconfig.ContainerTmpTarget,
|
|
}
|
|
return updater, store, engine, gateway
|
|
}
|
|
|
|
func assertCreatedContainerSpec(t *testing.T, request containerengine.ContainerSpec) {
|
|
t.Helper()
|
|
if request.Name != "backend-8081" || request.ImageReference != "harbor.ymswell.asia/ymswell/glory-ymswell@"+containerTestDigest {
|
|
t.Fatalf("unexpected target container identity: %+v", request)
|
|
}
|
|
if request.NetworkMode != "host" || request.RestartPolicy.Name != "no" || request.User != "0:0" {
|
|
t.Fatalf("unexpected target container runtime contract: %+v", request)
|
|
}
|
|
if len(request.Environment) != 2 || request.Environment[0] != "SERVER_PORT=8081" || request.Environment[1] != "SPRING_CONFIG_LOCATION=file:/app/config/yms.yaml" {
|
|
t.Fatalf("unexpected target container environment: %+v", request.Environment)
|
|
}
|
|
if len(request.Mounts) != 2 || request.Mounts[0].Target != deploymentconfig.ContainerConfigTarget || !request.Mounts[0].ReadOnly || request.Mounts[1].Target != deploymentconfig.ContainerTmpTarget {
|
|
t.Fatalf("unexpected target container mounts: %+v", request.Mounts)
|
|
}
|
|
}
|
|
|
|
func assertCommittedContainerDeployment(
|
|
t *testing.T,
|
|
store *transaction.Store,
|
|
transactionID string,
|
|
port int,
|
|
containerName string,
|
|
containerID string,
|
|
) {
|
|
t.Helper()
|
|
deployment, err := store.BackendContainerDeployment(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("read committed backend container deployment: %v", err)
|
|
}
|
|
if deployment.ActivePort != port || deployment.ContainerName != containerName || deployment.ContainerID != containerID || deployment.ImageDigest != containerTestDigest || deployment.TransactionID != transactionID {
|
|
t.Fatalf("unexpected committed backend container deployment: %+v", deployment)
|
|
}
|
|
}
|
|
|
|
const serverConfiguration8080 = `http {
|
|
upstream yms-server {
|
|
# yms-update managed upstream begin
|
|
server 10.11.1.117:8080 max_fails=1 fail_timeout=2s;
|
|
# server 10.11.1.117:8081 max_fails=1 fail_timeout=2s;
|
|
# yms-update managed upstream end
|
|
}
|
|
}
|
|
`
|
|
|
|
type containerUpdateRoundTripFunc func(*http.Request) (*http.Response, error)
|
|
|
|
func (function containerUpdateRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
|
return function(request)
|
|
}
|
|
|
|
type containerUpdateEngine struct {
|
|
image containerengine.Image
|
|
containers map[string]containerengine.Container
|
|
stopped []string
|
|
lastCreateSpec containerengine.ContainerSpec
|
|
healthChecks int
|
|
}
|
|
|
|
func (e *containerUpdateEngine) Ping(context.Context) error { return nil }
|
|
func (e *containerUpdateEngine) PullImage(context.Context, string) error { return nil }
|
|
func (e *containerUpdateEngine) LoadImage(context.Context, io.Reader) error { return nil }
|
|
func (e *containerUpdateEngine) InspectImage(context.Context, string) (containerengine.Image, error) {
|
|
return e.image, nil
|
|
}
|
|
func (e *containerUpdateEngine) CreateContainer(_ context.Context, spec containerengine.ContainerSpec) (containerengine.Container, error) {
|
|
e.lastCreateSpec = spec
|
|
record := containerengine.Container{
|
|
ID: "container-id-" + spec.Name,
|
|
Name: spec.Name,
|
|
ImageID: e.image.ID,
|
|
ImageReference: spec.ImageReference,
|
|
Platform: spec.Platform.OS + "/" + spec.Platform.Architecture,
|
|
Status: "created",
|
|
Environment: append([]string(nil), spec.Environment...),
|
|
Labels: spec.Labels,
|
|
NetworkMode: spec.NetworkMode,
|
|
RestartPolicy: spec.RestartPolicy,
|
|
Mounts: append([]containerengine.Mount(nil), spec.Mounts...),
|
|
User: spec.User,
|
|
StopTimeoutSeconds: spec.StopTimeoutSeconds,
|
|
}
|
|
e.containers[spec.Name] = record
|
|
return record, nil
|
|
}
|
|
func (e *containerUpdateEngine) StartContainer(_ context.Context, name string) error {
|
|
record, found := e.containers[name]
|
|
if !found {
|
|
return containerengine.ErrNotFound
|
|
}
|
|
record.Running = true
|
|
record.Status = "running"
|
|
e.containers[name] = record
|
|
return nil
|
|
}
|
|
|
|
func (e *containerUpdateEngine) ContainerLogs(context.Context, string) (io.ReadCloser, error) {
|
|
return io.NopCloser(strings.NewReader("")), nil
|
|
}
|
|
func (e *containerUpdateEngine) StopContainer(_ context.Context, name string) error {
|
|
e.stopped = append(e.stopped, name)
|
|
record, found := e.containers[name]
|
|
if !found {
|
|
return containerengine.ErrNotFound
|
|
}
|
|
record.Running = false
|
|
record.Status = "exited"
|
|
e.containers[name] = record
|
|
return nil
|
|
}
|
|
func (e *containerUpdateEngine) InspectContainer(_ context.Context, name string) (containerengine.Container, error) {
|
|
record, found := e.containers[name]
|
|
if !found {
|
|
return containerengine.Container{}, containerengine.ErrNotFound
|
|
}
|
|
return record, nil
|
|
}
|
|
func (e *containerUpdateEngine) RemoveContainer(_ context.Context, name string, _ bool) error {
|
|
if _, found := e.containers[name]; !found {
|
|
return containerengine.ErrNotFound
|
|
}
|
|
delete(e.containers, name)
|
|
return nil
|
|
}
|
|
func (e *containerUpdateEngine) Close() error { return nil }
|
|
|
|
var _ containerengine.Engine = (*containerUpdateEngine)(nil)
|