yms-daemon/internal/transaction/coordinator_test.go

package transaction

import (
	"context"
	"encoding/json"
	"errors"
	"io"
	"log/slog"
	"sync/atomic"
	"testing"
)

func TestCoordinatorRecoversIntentWithoutRepeatingAppliedOperation(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	store := openTestStore(t)
	record := createTestTransaction(t, store, "recover-applied")
	intent := StepIntent{Key: "start-green", Name: "start green", Intent: json.RawMessage(`{"slot":"green"}`)}
	if _, _, err := store.RecordStepIntent(ctx, record.ID, intent); err != nil {
		t.Fatalf("record crash-window intent: %v", err)
	}
	operation := &fakeOperation{applied: true, result: json.RawMessage(`{"running":true}`)}
	coordinator := newTestCoordinator(t, store)
	step, err := coordinator.ExecuteStep(ctx, record.ID, intent, operation)
	if err != nil {
		t.Fatalf("recover applied operation: %v", err)
	}
	if step.Status != StepSucceeded || operation.applyCalls.Load() != 0 || operation.inspectCalls.Load() != 1 {
		t.Fatalf("unexpected recovery result: step=%+v apply=%d inspect=%d", step, operation.applyCalls.Load(), operation.inspectCalls.Load())
	}
}

func TestCoordinatorRecoversNotAppliedIntentThenExecutesOnce(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	store := openTestStore(t)
	record := createTestTransaction(t, store, "recover-not-applied")
	intent := StepIntent{Key: "write-files", Name: "write files", Intent: json.RawMessage(`{}`)}
	if _, _, err := store.RecordStepIntent(ctx, record.ID, intent); err != nil {
		t.Fatalf("record crash-window intent: %v", err)
	}
	operation := &fakeOperation{result: json.RawMessage(`{"written":true}`)}
	coordinator := newTestCoordinator(t, store)
	step, err := coordinator.ExecuteStep(ctx, record.ID, intent, operation)
	if err != nil {
		t.Fatalf("recover not-applied operation: %v", err)
	}
	if step.Status != StepSucceeded || operation.applyCalls.Load() != 1 || operation.inspectCalls.Load() != 2 {
		t.Fatalf("unexpected recovery result: step=%+v apply=%d inspect=%d", step, operation.applyCalls.Load(), operation.inspectCalls.Load())
	}
}

func TestCoordinatorLeavesIntentPendingWhenInspectionIsUnknown(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	store := openTestStore(t)
	record := createTestTransaction(t, store, "recover-unknown")
	intent := StepIntent{Key: "reload-gateway", Name: "reload gateway", Intent: json.RawMessage(`{}`)}
	operation := &fakeOperation{inspectErr: errors.New("gateway unavailable")}
	coordinator := newTestCoordinator(t, store)
	step, err := coordinator.ExecuteStep(ctx, record.ID, intent, operation)
	var uncertain *UncertainStepError
	if !errors.As(err, &uncertain) {
		t.Fatalf("expected uncertain step error, got step=%+v err=%v", step, err)
	}
	pending, err := store.PendingSteps(ctx, record.ID)
	if err != nil {
		t.Fatalf("read pending steps: %v", err)
	}
	if len(pending) != 1 || pending[0].Status != StepIntentRecorded {
		t.Fatalf("uncertain step did not remain pending: %+v", pending)
	}
}

func TestCoordinatorRetriesFailedStepOnlyAfterManualResumeInspection(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	store := openTestStore(t)
	record := createTestTransaction(t, store, "retry-failed")
	intent := StepIntent{Key: "restore-gateway", Name: "restore gateway", Intent: json.RawMessage(`{}`)}
	operation := &fakeOperation{applyErr: errors.New("gateway executable unavailable"), result: json.RawMessage(`{"restored":true}`)}
	coordinator := newTestCoordinator(t, store)

	failed, err := coordinator.ExecuteStep(ctx, record.ID, intent, operation)
	if err == nil || failed.Status != StepFailed {
		t.Fatalf("unexpected initial failed step: step=%+v err=%v", failed, err)
	}
	if operation.applyCalls.Load() != 1 || operation.inspectCalls.Load() != 1 {
		t.Fatalf("initial call retried unexpectedly: apply=%d inspect=%d", operation.applyCalls.Load(), operation.inspectCalls.Load())
	}

	operation.applyErr = nil
	recovered, err := coordinator.ExecuteStep(ctx, record.ID, intent, operation)
	if err != nil || recovered.Status != StepSucceeded {
		t.Fatalf("recover failed step: step=%+v err=%v", recovered, err)
	}
	if operation.applyCalls.Load() != 2 || operation.inspectCalls.Load() != 3 {
		t.Fatalf("unexpected manual recovery calls: apply=%d inspect=%d", operation.applyCalls.Load(), operation.inspectCalls.Load())
	}
}

func TestCoordinatorCompletesFailedStepAlreadyAppliedBeforeManualResume(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	store := openTestStore(t)
	record := createTestTransaction(t, store, "recover-failed-applied")
	intent := StepIntent{Key: "restore-gateway", Name: "restore gateway", Intent: json.RawMessage(`{}`)}
	operation := &fakeOperation{applyErr: errors.New("gateway reload result unavailable"), result: json.RawMessage(`{"restored":true}`)}
	coordinator := newTestCoordinator(t, store)

	if _, err := coordinator.ExecuteStep(ctx, record.ID, intent, operation); err == nil {
		t.Fatal("expected initial external step failure")
	}
	operation.applyErr = nil
	operation.applied = true
	recovered, err := coordinator.ExecuteStep(ctx, record.ID, intent, operation)
	if err != nil || recovered.Status != StepSucceeded {
		t.Fatalf("complete externally applied failed step: step=%+v err=%v", recovered, err)
	}
	if operation.applyCalls.Load() != 1 || operation.inspectCalls.Load() != 2 {
		t.Fatalf("externally applied step was repeated: apply=%d inspect=%d", operation.applyCalls.Load(), operation.inspectCalls.Load())
	}
}

func TestCoordinatorExclusiveExecutionHonorsContext(t *testing.T) {
	t.Parallel()
	store := openTestStore(t)
	coordinator := newTestCoordinator(t, store)
	firstEntered := make(chan struct{})
	releaseFirst := make(chan struct{})
	firstDone := make(chan error, 1)
	go func() {
		firstDone <- coordinator.RunExclusive(context.Background(), func(context.Context) error {
			close(firstEntered)
			<-releaseFirst
			return nil
		})
	}()
	<-firstEntered
	cancelled, cancel := context.WithCancel(context.Background())
	cancel()
	err := coordinator.RunExclusive(cancelled, func(context.Context) error {
		t.Fatal("cancelled update entered exclusive section")
		return nil
	})
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("expected context cancellation, got %v", err)
	}
	close(releaseFirst)
	if err := <-firstDone; err != nil {
		t.Fatalf("first exclusive update failed: %v", err)
	}
}

type fakeOperation struct {
	applied      bool
	result       json.RawMessage
	applyErr     error
	inspectErr   error
	applyCalls   atomic.Int32
	inspectCalls atomic.Int32
}

func (o *fakeOperation) Apply(context.Context) error {
	o.applyCalls.Add(1)
	if o.applyErr == nil {
		o.applied = true
	}
	return o.applyErr
}

func (o *fakeOperation) Inspect(context.Context) (Inspection, error) {
	o.inspectCalls.Add(1)
	if o.inspectErr != nil {
		return Inspection{}, o.inspectErr
	}
	if o.applied {
		return Inspection{Status: InspectionApplied, Result: o.result}, nil
	}
	return Inspection{Status: InspectionNotApplied}, nil
}

func createTestTransaction(t *testing.T, store *Store, suffix string) Transaction {
	t.Helper()
	record, _, err := store.CreateTransaction(context.Background(), CreateRequest{
		ID:             "transaction-" + suffix,
		IdempotencyKey: "request-" + suffix,
		Source:         "test",
		Service:        "backend",
	})
	if err != nil {
		t.Fatalf("create test transaction: %v", err)
	}
	return record
}

func newTestCoordinator(t *testing.T, store *Store) *Coordinator {
	t.Helper()
	coordinator, err := NewCoordinator(store, slog.New(slog.NewTextHandler(io.Discard, nil)))
	if err != nil {
		t.Fatalf("create coordinator: %v", err)
	}
	return coordinator
}