soft_quay/core/installer/durability_test.go

package installer

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

type durabilityEvent struct {
	kind string
	path string
}

type recordingDurabilityFence struct {
	events            []durabilityEvent
	fail              func(durabilityEvent) error
	failNextDirectory bool
}

func (fence *recordingDurabilityFence) syncFile(file *os.File) error {
	return fence.record(durabilityEvent{kind: "file", path: file.Name()})
}

func (fence *recordingDurabilityFence) syncDirectory(path string) error {
	event := durabilityEvent{kind: "directory", path: path}
	if fence.failNextDirectory {
		fence.failNextDirectory = false
		fence.events = append(fence.events, event)
		return errors.New("injected directory fence failure")
	}
	return fence.record(event)
}

func (fence *recordingDurabilityFence) record(event durabilityEvent) error {
	fence.events = append(fence.events, event)
	if fence.fail != nil {
		return fence.fail(event)
	}
	return nil
}

func TestExtractorDurabilityFencesPayloadThenStagingTree(t *testing.T) {
	archivePath := writeTestZIP(t, []testZIPEntry{
		{name: "app.json", body: []byte(`{"entrypoint":"bin/nested/App.exe"}`)},
		{name: "payload/bin/", mode: os.ModeDir | 0o755},
		{name: "payload/bin/nested/", mode: os.ModeDir | 0o755},
		{name: "payload/bin/nested/App.exe", body: []byte("executable"), mode: 0o755},
		{name: "payload/readme.txt", body: []byte("readme")},
	})
	destination := filepath.Join(t.TempDir(), "staging")
	fence := &recordingDurabilityFence{}
	extractor := mustExtractor(t, testLimits())
	extractor.durability = fence

	if _, err := extractor.ExtractFile(
		archivePath,
		destination,
		"bin/nested/App.exe",
		archiveSize(t, archivePath),
	); err != nil {
		t.Fatalf("ExtractFile() error = %v", err)
	}

	want := []durabilityEvent{
		{kind: "file", path: filepath.Join(destination, "bin", "nested", "App.exe")},
		{kind: "file", path: filepath.Join(destination, "readme.txt")},
		{kind: "directory", path: filepath.Join(destination, "bin", "nested")},
		{kind: "directory", path: filepath.Join(destination, "bin")},
		{kind: "directory", path: destination},
		{kind: "directory", path: filepath.Dir(destination)},
	}
	assertDurabilityEvents(t, fence.events, want)
}

func TestExtractorDurabilityFailuresRemoveStaging(t *testing.T) {
	archivePath := writeTestZIP(t, []testZIPEntry{
		{name: "app.json", body: []byte(`{"entrypoint":"App.exe"}`)},
		{name: "payload/App.exe", body: []byte("executable"), mode: 0o755},
	})

	for _, test := range []struct {
		name string
		fail func(durabilityEvent) error
	}{
		{
			name: "payload sync",
			fail: func(event durabilityEvent) error {
				if event.kind == "file" {
					return errors.New("injected payload sync failure")
				}
				return nil
			},
		},
		{
			name: "staging tree sync",
			fail: func(event durabilityEvent) error {
				if event.kind == "directory" {
					return errors.New("injected staging tree sync failure")
				}
				return nil
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			destination := filepath.Join(t.TempDir(), "staging")
			fence := &recordingDurabilityFence{fail: test.fail}
			extractor := mustExtractor(t, testLimits())
			extractor.durability = fence

			_, err := extractor.ExtractFile(
				archivePath,
				destination,
				"App.exe",
				archiveSize(t, archivePath),
			)
			if !errors.Is(err, ErrDurability) {
				t.Fatalf("ExtractFile() error = %v, want %v", err, ErrDurability)
			}
			assertMissing(t, destination)
		})
	}
}

func TestWriteTransactionFailsWhenJournalRootFenceFails(t *testing.T) {
	root := t.TempDir()
	layout, err := inspectAppLayout(root)
	if err != nil {
		t.Fatalf("inspectAppLayout() error = %v", err)
	}
	fence := &recordingDurabilityFence{fail: func(event durabilityEvent) error {
		if event.kind == "directory" {
			return errors.New("injected journal root fence failure")
		}
		return nil
	}}

	err = writeTransactionWithFence(layout, newTransaction(phasePrepared, true), fence)
	if !errors.Is(err, ErrDurability) {
		t.Fatalf("writeTransactionWithFence() error = %v, want %v", err, ErrDurability)
	}
	if len(fence.events) != 2 || fence.events[0].kind != "file" ||
		fence.events[1] != (durabilityEvent{kind: "directory", path: root}) {
		t.Fatalf("journal fences = %#v, want temporary file then app-root directory", fence.events)
	}
	if _, exists, err := loadTransaction(layout); err != nil || !exists {
		t.Fatalf("loadTransaction() exists=%t error=%v, want prepared journal retained for recovery", exists, err)
	}
}

func TestSwitcherFencesEachPhaseBeforeAfterStep(t *testing.T) {
	root := makeInstallRoot(t, "old", "new")
	fence := &recordingDurabilityFence{}
	switcher := NewSwitcher(func(currentPath string) error {
		assertVersion(t, currentPath, "new")
		return nil
	})
	switcher.durability = fence
	switcher.afterStep = func(step switchStep) error {
		if len(fence.events) == 0 {
			return fmt.Errorf("%s ran without a durability fence", step)
		}
		last := fence.events[len(fence.events)-1]
		if last != (durabilityEvent{kind: "directory", path: root}) {
			return fmt.Errorf("%s ran after %#v, want app-root directory fence", step, last)
		}
		return nil
	}

	if err := switcher.Switch(root); err != nil {
		t.Fatalf("Switch() error = %v", err)
	}
	if !hasJournalFileFence(fence.events) {
		t.Fatalf("fences = %#v, want temporary journal file sync", fence.events)
	}
	if countDirectoryFences(fence.events, root) < 10 {
		t.Fatalf("app-root directory fences = %d, want at least 10", countDirectoryFences(fence.events, root))
	}
}

func TestSwitcherRenameFenceFailureLeavesRecoverablePreparedJournal(t *testing.T) {
	root := makeInstallRoot(t, "old", "new")
	fence := &recordingDurabilityFence{}
	switcher := NewSwitcher(func(string) error { return nil })
	switcher.durability = fence
	switcher.afterStep = func(step switchStep) error {
		if step == stepPrepared {
			fence.failNextDirectory = true
		}
		return nil
	}

	err := switcher.Switch(root)
	if !errors.Is(err, ErrDurability) {
		t.Fatalf("Switch() error = %v, want %v", err, ErrDurability)
	}
	layout, layoutErr := inspectAppLayout(root)
	if layoutErr != nil {
		t.Fatalf("inspectAppLayout() error = %v", layoutErr)
	}
	record, exists, loadErr := loadTransaction(layout)
	if loadErr != nil || !exists || record.Phase != phasePrepared {
		t.Fatalf("transaction = %#v exists=%t error=%v, want prepared journal", record, exists, loadErr)
	}

	result, err := Recover(root)
	if err != nil {
		t.Fatalf("Recover() error = %v", err)
	}
	if result.Action != RecoveryRolledBack {
		t.Fatalf("Recovery action = %q, want %q", result.Action, RecoveryRolledBack)
	}
	assertVersion(t, filepath.Join(root, "current"), "old")
	assertMissing(t, filepath.Join(root, "staging"))
	assertMissing(t, filepath.Join(root, "backup"))
	assertMissing(t, filepath.Join(root, transactionFileName))
}

func TestRollbackAndRecoveryFenceFailuresRemainRecoverable(t *testing.T) {
	t.Run("rollback", func(t *testing.T) {
		root := makeInstallRoot(t, "old", "new")
		fence := &recordingDurabilityFence{}
		switcher := NewSwitcher(func(string) error { return errors.New("health failed") })
		switcher.durability = fence
		switcher.afterStep = func(step switchStep) error {
			if step == stepRollbackRequired {
				fence.failNextDirectory = true
			}
			return nil
		}

		err := switcher.Switch(root)
		var rollbackErr *RollbackError
		if !errors.As(err, &rollbackErr) || !errors.Is(rollbackErr.Rollback, ErrDurability) {
			t.Fatalf("Switch() error = %v, want rollback durability failure", err)
		}
		result, err := Recover(root)
		if err != nil {
			t.Fatalf("Recover() error = %v", err)
		}
		if result.Action != RecoveryRolledBack {
			t.Fatalf("Recovery action = %q, want %q", result.Action, RecoveryRolledBack)
		}
		assertVersion(t, filepath.Join(root, "current"), "old")
	})

	t.Run("recovery", func(t *testing.T) {
		root := makeInstallRoot(t, "old", "new")
		switcher := NewSwitcher(func(string) error { return nil })
		switcher.afterStep = func(step switchStep) error {
			if step == stepStagingRenamed {
				return errSimulatedCrash
			}
			return nil
		}
		if err := switcher.Switch(root); !errors.Is(err, errSimulatedCrash) {
			t.Fatalf("Switch() error = %v, want %v", err, errSimulatedCrash)
		}

		fence := &recordingDurabilityFence{failNextDirectory: true}
		if _, err := recoverWithFence(root, fence); !errors.Is(err, ErrDurability) {
			t.Fatalf("recoverWithFence() error = %v, want %v", err, ErrDurability)
		}
		result, err := Recover(root)
		if err != nil {
			t.Fatalf("Recover() error = %v", err)
		}
		if result.Action != RecoveryRolledBack {
			t.Fatalf("Recovery action = %q, want %q", result.Action, RecoveryRolledBack)
		}
		assertVersion(t, filepath.Join(root, "current"), "old")
	})
}

func TestCommittedCleanupFenceFailureRemainsRecoverable(t *testing.T) {
	root := makeInstallRoot(t, "old", "new")
	fence := &recordingDurabilityFence{}
	switcher := NewSwitcher(func(string) error { return nil })
	switcher.durability = fence
	switcher.afterStep = func(step switchStep) error {
		if step == stepCommitted {
			fence.failNextDirectory = true
		}
		return nil
	}

	err := switcher.Switch(root)
	if !errors.Is(err, ErrRecoveryRequired) || !errors.Is(err, ErrDurability) {
		t.Fatalf("Switch() error = %v, want recovery-required durability failure", err)
	}
	result, err := Recover(root)
	if err != nil {
		t.Fatalf("Recover() error = %v", err)
	}
	if result.Action != RecoveryCommitted {
		t.Fatalf("Recovery action = %q, want %q", result.Action, RecoveryCommitted)
	}
	assertVersion(t, filepath.Join(root, "current"), "new")
	assertMissing(t, filepath.Join(root, "backup"))
	assertMissing(t, filepath.Join(root, transactionFileName))
}

func assertDurabilityEvents(t *testing.T, got, want []durabilityEvent) {
	t.Helper()
	if len(got) != len(want) {
		t.Fatalf("durability events = %#v, want %#v", got, want)
	}
	for index := range want {
		if got[index] != want[index] {
			t.Fatalf("durability event %d = %#v, want %#v", index, got[index], want[index])
		}
	}
}

func hasJournalFileFence(events []durabilityEvent) bool {
	for _, event := range events {
		if event.kind == "file" && strings.HasPrefix(filepath.Base(event.path), ".install-transaction-") {
			return true
		}
	}
	return false
}

func countDirectoryFences(events []durabilityEvent, path string) int {
	count := 0
	for _, event := range events {
		if event == (durabilityEvent{kind: "directory", path: path}) {
			count++
		}
	}
	return count
}