feat(sense): add reconciliation safety controls [T-012]

2026-08-07 23:00:03 +08:00
parent 677ed732f7
commit 12857fdf32
35 changed files with 2817 additions and 124 deletions
@@ -0,0 +1,400 @@
+// Package orphan reports MediaMTX configuration paths that do not match the
+// current Sense ledger. Only paths with durable Sense ownership evidence can
+// ever enter the controlled cleanup set.
+package orphan
+
+import (
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"math/big"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"yovision/sense/internal/metrics"
+	"yovision/sense/internal/store"
+)
+
+const (
+	defaultLeaseDuration    = 30 * time.Second
+	defaultOperationTimeout = 20 * time.Second
+	reportTTL               = 15 * time.Minute
+	maxCleanupItems         = 128
+	crockford               = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+)
+
+var (
+	ErrLeaseHeld       = errors.New("orphan operation lease held")
+	ErrConfirmation    = errors.New("orphan cleanup confirmation invalid")
+	ErrSnapshotExpired = errors.New("orphan scan snapshot expired")
+	ErrSafetyBlocked   = errors.New("orphan cleanup safety gate blocked")
+	operatorIDPattern  = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
+)
+
+type Runtime interface {
+	ListPathNames(context.Context) ([]string, error)
+	DeletePath(context.Context, string) error
+}
+
+type Manager struct {
+	repository       store.OrphanRepository
+	runtime          Runtime
+	instanceID       string
+	metrics          *metrics.Registry
+	now              func() time.Time
+	leaseDuration    time.Duration
+	operationTimeout time.Duration
+}
+
+type CleanupResult struct {
+	ScanID  string `json:"scan_id"`
+	Deleted int    `json:"deleted"`
+	Failed  int    `json:"failed"`
+}
+
+func New(
+	repository store.OrphanRepository,
+	runtime Runtime,
+	instanceID string,
+	registry *metrics.Registry,
+) *Manager {
+	return &Manager{
+		repository: repository, runtime: runtime, instanceID: instanceID, metrics: registry,
+		now: time.Now, leaseDuration: defaultLeaseDuration, operationTimeout: defaultOperationTimeout,
+	}
+}
+
+func (m *Manager) Report(ctx context.Context) (store.OrphanScan, error) {
+	now := m.now().UTC()
+	token, err := randomToken()
+	if err != nil {
+		return store.OrphanScan{}, err
+	}
+	acquired, err := m.repository.AcquireOperationalLease(
+		ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, now, m.leaseDuration,
+	)
+	if err != nil {
+		m.observeScan(0, 0, 0, err)
+		return store.OrphanScan{}, err
+	}
+	if !acquired {
+		return store.OrphanScan{}, ErrLeaseHeld
+	}
+	saved := false
+	defer func() {
+		if saved {
+			return
+		}
+		releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		_ = m.repository.ReleaseOperationalLease(
+			releaseCtx, store.OperationalLeaseOrphanScan, m.instanceID, token, m.now().UTC(),
+		)
+	}()
+
+	operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout)
+	paths, err := m.runtime.ListPathNames(operationCtx)
+	cancel()
+	if err != nil {
+		m.observeScan(0, 0, 0, err)
+		return store.OrphanScan{}, fmt.Errorf("list MediaMTX paths: %w", err)
+	}
+	ownership, err := m.repository.ListMediaPathOwnership(ctx)
+	if err != nil {
+		m.observeScan(0, 0, 0, err)
+		return store.OrphanScan{}, err
+	}
+	findings, stale, unowned := classify(paths, ownership)
+	allowed, reason := safetyGate(stale, len(paths))
+	completedAt := m.now().UTC()
+	acquired, err = m.repository.AcquireOperationalLease(
+		ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, completedAt, m.leaseDuration,
+	)
+	if err != nil || !acquired {
+		if err == nil {
+			err = store.ErrOperationalLeaseLost
+		}
+		m.observeScan(len(paths), stale, unowned, err)
+		return store.OrphanScan{}, err
+	}
+	id, err := newScanID(completedAt)
+	if err != nil {
+		m.observeScan(0, 0, 0, err)
+		return store.OrphanScan{}, err
+	}
+	scan := store.OrphanScan{
+		ID: id, InstanceID: m.instanceID, ObservedCount: len(paths),
+		OwnedStaleCount: stale, UnownedCount: unowned,
+		SafetyAllowed: allowed, SafetyReason: reason,
+		CompletedAt: completedAt, ExpiresAt: completedAt.Add(reportTTL), Findings: findings,
+	}
+	if err := m.repository.SaveOrphanScan(ctx, scan, m.instanceID, token); err != nil {
+		m.observeScan(len(paths), stale, unowned, err)
+		return store.OrphanScan{}, err
+	}
+	saved = true
+	m.observeScan(len(paths), stale, unowned, nil)
+	return scan, nil
+}
+
+func classify(
+	paths []string,
+	ownership []store.MediaPathOwnership,
+) ([]store.OrphanFinding, int, int) {
+	history := make(map[string]store.MediaPathOwnership, len(ownership))
+	for _, value := range ownership {
+		history[value.PathName] = value
+	}
+	unique := make(map[string]struct{}, len(paths))
+	for _, path := range paths {
+		unique[path] = struct{}{}
+	}
+	ordered := make([]string, 0, len(unique))
+	for path := range unique {
+		ordered = append(ordered, path)
+	}
+	sort.Strings(ordered)
+	findings := make([]store.OrphanFinding, 0)
+	stale, unowned := 0, 0
+	for _, path := range ordered {
+		value, known := history[path]
+		switch {
+		case known && value.CurrentClaim:
+			continue
+		case known:
+			stale++
+			findings = append(findings, store.OrphanFinding{
+				PathName: path, Classification: store.OrphanOwnedStale, DeviceID: value.DeviceID,
+			})
+		default:
+			unowned++
+			findings = append(findings, store.OrphanFinding{
+				PathName: path, Classification: store.OrphanUnowned,
+			})
+		}
+	}
+	return findings, stale, unowned
+}
+
+func safetyGate(candidates, observed int) (bool, string) {
+	switch {
+	case candidates <= 0:
+		return false, "no_candidates"
+	case observed <= 0:
+		return false, "empty_inventory"
+	case candidates > maxCleanupItems:
+		return false, "scope_exceeded"
+	case candidates*100 > observed*10:
+		return false, "ratio_exceeded"
+	default:
+		return true, "allowed"
+	}
+}
+
+func (m *Manager) Apply(
+	ctx context.Context,
+	scanID, actorID, confirmation string,
+) (CleanupResult, error) {
+	result := CleanupResult{ScanID: scanID}
+	if !operatorIDPattern.MatchString(actorID) || confirmation != "DELETE "+scanID {
+		m.blocked("scope_changed")
+		return result, ErrConfirmation
+	}
+	now := m.now().UTC()
+	token, err := randomToken()
+	if err != nil {
+		return result, err
+	}
+	acquired, err := m.repository.AcquireOperationalLease(
+		ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration,
+	)
+	if err != nil {
+		return result, err
+	}
+	if !acquired {
+		return result, ErrLeaseHeld
+	}
+	defer func() {
+		releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		_ = m.repository.ReleaseOperationalLease(
+			releaseCtx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, m.now().UTC(),
+		)
+	}()
+
+	scan, err := m.repository.GetOrphanScan(ctx, scanID)
+	if err != nil {
+		m.blocked("scope_changed")
+		return result, err
+	}
+	if !scan.ExpiresAt.After(now) {
+		m.blocked("snapshot_expired")
+		return result, ErrSnapshotExpired
+	}
+	if !scan.SafetyAllowed {
+		m.blocked(scan.SafetyReason)
+		return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, scan.SafetyReason)
+	}
+	operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout)
+	paths, err := m.runtime.ListPathNames(operationCtx)
+	cancel()
+	if err != nil {
+		return result, fmt.Errorf("refresh MediaMTX path inventory: %w", err)
+	}
+	ownership, err := m.repository.ListMediaPathOwnership(ctx)
+	if err != nil {
+		return result, err
+	}
+	candidates := cleanupCandidates(scan, paths, ownership)
+	if len(candidates) == 0 {
+		return result, nil
+	}
+	allowed, reason := safetyGate(len(candidates), len(uniqueNames(paths)))
+	if !allowed {
+		m.blocked(reason)
+		return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, reason)
+	}
+	var failures []error
+	for _, path := range candidates {
+		now = m.now().UTC()
+		acquired, err = m.repository.AcquireOperationalLease(
+			ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration,
+		)
+		if err != nil || !acquired {
+			if err == nil {
+				err = ErrLeaseHeld
+			}
+			failures = append(failures, err)
+			break
+		}
+		itemCtx, itemCancel := context.WithTimeout(ctx, m.operationTimeout)
+		deleteErr := m.runtime.DeletePath(itemCtx, path)
+		itemCancel()
+		status, code := "deleted", ""
+		if deleteErr != nil {
+			status, code = "failed", "media_error"
+			result.Failed++
+			failures = append(failures, fmt.Errorf("delete owned stale path: %w", deleteErr))
+		} else {
+			result.Deleted++
+		}
+		if err := m.repository.RecordOrphanCleanup(
+			ctx, scanID, path, actorID, status, code, m.now().UTC(),
+		); err != nil {
+			failures = append(failures, err)
+			break
+		}
+	}
+	if m.metrics != nil {
+		m.metrics.ObserveOrphanCleanup(result.Deleted, result.Failed)
+	}
+	return result, errors.Join(failures...)
+}
+
+func cleanupCandidates(
+	scan store.OrphanScan,
+	paths []string,
+	ownership []store.MediaPathOwnership,
+) []string {
+	runtime := uniqueNames(paths)
+	history := make(map[string]store.MediaPathOwnership, len(ownership))
+	for _, value := range ownership {
+		history[value.PathName] = value
+	}
+	values := make([]string, 0)
+	for _, finding := range scan.Findings {
+		if finding.Classification != store.OrphanOwnedStale || finding.Deleted {
+			continue
+		}
+		_, present := runtime[finding.PathName]
+		owner, known := history[finding.PathName]
+		if present && known && !owner.CurrentClaim && owner.DeviceID == finding.DeviceID {
+			values = append(values, finding.PathName)
+		}
+	}
+	sort.Strings(values)
+	return values
+}
+
+func uniqueNames(values []string) map[string]struct{} {
+	result := make(map[string]struct{}, len(values))
+	for _, value := range values {
+		result[value] = struct{}{}
+	}
+	return result
+}
+
+func (m *Manager) Run(ctx context.Context, interval time.Duration, report func(error)) {
+	run := func() {
+		_, err := m.Report(ctx)
+		if err != nil && !errors.Is(err, ErrLeaseHeld) && ctx.Err() == nil && report != nil {
+			report(err)
+		}
+	}
+	run()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			run()
+		}
+	}
+}
+
+func (m *Manager) observeScan(observed, stale, unowned int, err error) {
+	if m.metrics != nil {
+		m.metrics.ObserveOrphanScan(observed, stale, unowned, err)
+	}
+}
+
+func (m *Manager) blocked(reason string) {
+	if m.metrics != nil {
+		m.metrics.ObserveOrphanCleanupBlocked(reason)
+	}
+}
+
+func randomToken() (string, error) {
+	value := make([]byte, 16)
+	if _, err := rand.Read(value); err != nil {
+		return "", errors.New("generate fencing token")
+	}
+	const alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	result := make([]byte, 22)
+	number := new(big.Int).SetBytes(value)
+	base := big.NewInt(int64(len(alphabet)))
+	remainder := new(big.Int)
+	for index := len(result) - 1; index >= 0; index-- {
+		number.QuoRem(number, base, remainder)
+		result[index] = alphabet[remainder.Int64()]
+	}
+	return string(result), nil
+}
+
+func newScanID(now time.Time) (string, error) {
+	value := make([]byte, 16)
+	milliseconds := uint64(now.UTC().UnixMilli())
+	value[0], value[1], value[2] = byte(milliseconds>>40), byte(milliseconds>>32), byte(milliseconds>>24)
+	value[3], value[4], value[5] = byte(milliseconds>>16), byte(milliseconds>>8), byte(milliseconds)
+	if _, err := rand.Read(value[6:]); err != nil {
+		return "", errors.New("generate orphan scan identifier")
+	}
+	number := new(big.Int).SetBytes(value)
+	base := big.NewInt(32)
+	remainder := new(big.Int)
+	encoded := make([]byte, 26)
+	for index := len(encoded) - 1; index >= 0; index-- {
+		number.QuoRem(number, base, remainder)
+		encoded[index] = crockford[remainder.Int64()]
+	}
+	return "scan_" + string(encoded), nil
+}
+
+func Confirmation(scanID string) string {
+	return strings.Join([]string{"DELETE", scanID}, " ")
+}
@@ -0,0 +1,272 @@
+package orphan
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"yovision/sense/internal/store"
+)
+
+type fakeLease struct {
+	owner, token string
+	expires      time.Time
+}
+
+type fakeRepository struct {
+	mu        sync.Mutex
+	leases    map[string]fakeLease
+	ownership []store.MediaPathOwnership
+	scans     map[string]store.OrphanScan
+	deleted   map[string]map[string]bool
+}
+
+func newFakeRepository() *fakeRepository {
+	return &fakeRepository{
+		leases: make(map[string]fakeLease), scans: make(map[string]store.OrphanScan),
+		deleted: make(map[string]map[string]bool),
+	}
+}
+
+func (f *fakeRepository) AcquireOperationalLease(
+	_ context.Context,
+	name, owner, token string,
+	now time.Time,
+	duration time.Duration,
+) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	current, exists := f.leases[name]
+	if exists && current.expires.After(now) && (current.owner != owner || current.token != token) {
+		return false, nil
+	}
+	f.leases[name] = fakeLease{owner: owner, token: token, expires: now.Add(duration)}
+	return true, nil
+}
+
+func (f *fakeRepository) ReleaseOperationalLease(
+	_ context.Context,
+	name, owner, token string,
+	now time.Time,
+) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	current := f.leases[name]
+	if current.owner == owner && current.token == token {
+		current.expires = now
+		f.leases[name] = current
+	}
+	return nil
+}
+
+func (f *fakeRepository) ListMediaPathOwnership(context.Context) ([]store.MediaPathOwnership, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]store.MediaPathOwnership(nil), f.ownership...), nil
+}
+
+func (f *fakeRepository) SaveOrphanScan(
+	_ context.Context,
+	scan store.OrphanScan,
+	owner, token string,
+) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	lease := f.leases[store.OperationalLeaseOrphanScan]
+	if lease.owner != owner || lease.token != token || !lease.expires.After(scan.CompletedAt) {
+		return store.ErrOperationalLeaseLost
+	}
+	f.scans[scan.ID] = cloneScan(scan)
+	lease.expires = scan.CompletedAt
+	f.leases[store.OperationalLeaseOrphanScan] = lease
+	return nil
+}
+
+func (f *fakeRepository) GetOrphanScan(_ context.Context, id string) (store.OrphanScan, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	scan, exists := f.scans[id]
+	if !exists {
+		return store.OrphanScan{}, store.ErrOrphanScanNotFound
+	}
+	result := cloneScan(scan)
+	for index := range result.Findings {
+		result.Findings[index].Deleted = f.deleted[id][result.Findings[index].PathName]
+	}
+	return result, nil
+}
+
+func (f *fakeRepository) RecordOrphanCleanup(
+	_ context.Context,
+	scanID, pathName, _, status, _ string,
+	_ time.Time,
+) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.deleted[scanID] == nil {
+		f.deleted[scanID] = make(map[string]bool)
+	}
+	if status == "deleted" {
+		f.deleted[scanID][pathName] = true
+	}
+	return nil
+}
+
+func cloneScan(value store.OrphanScan) store.OrphanScan {
+	value.Findings = append([]store.OrphanFinding(nil), value.Findings...)
+	return value
+}
+
+type fakeRuntime struct {
+	paths   map[string]bool
+	fail    map[string]bool
+	deleted []string
+}
+
+func (f *fakeRuntime) ListPathNames(context.Context) ([]string, error) {
+	values := make([]string, 0, len(f.paths))
+	for path, present := range f.paths {
+		if present {
+			values = append(values, path)
+		}
+	}
+	sort.Strings(values)
+	return values, nil
+}
+
+func (f *fakeRuntime) DeletePath(_ context.Context, path string) error {
+	if f.fail[path] {
+		return errors.New("redacted media failure")
+	}
+	delete(f.paths, path)
+	f.deleted = append(f.deleted, path)
+	return nil
+}
+
+func TestSafetyGateUsesExactTenPercentBoundary(t *testing.T) {
+	t.Parallel()
+	if allowed, reason := safetyGate(1, 10); !allowed || reason != "allowed" {
+		t.Fatalf("exact 10%% boundary was rejected: %v %s", allowed, reason)
+	}
+	if allowed, reason := safetyGate(1, 9); allowed || reason != "ratio_exceeded" {
+		t.Fatalf("more than 10%% was accepted: %v %s", allowed, reason)
+	}
+	if allowed, reason := safetyGate(129, 2000); allowed || reason != "scope_exceeded" {
+		t.Fatalf("129-item scope was accepted: %v %s", allowed, reason)
+	}
+}
+
+func TestReportClassifiesOwnershipAndApplyNeverDeletesUnowned(t *testing.T) {
+	repository := newFakeRepository()
+	repository.ownership = []store.MediaPathOwnership{
+		{PathName: "owned-stale", DeviceID: "old-device"},
+		{PathName: "owned-current", DeviceID: "live-device", CurrentClaim: true},
+	}
+	runtime := &fakeRuntime{paths: map[string]bool{
+		"owned-stale": true, "owned-current": true,
+		"unowned-1": true, "unowned-2": true, "unowned-3": true, "unowned-4": true,
+		"unowned-5": true, "unowned-6": true, "unowned-7": true, "unowned-8": true,
+	}, fail: make(map[string]bool)}
+	now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
+	manager := New(repository, runtime, "ins_test", nil)
+	manager.now = func() time.Time { return now }
+	scan, err := manager.Report(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if scan.ObservedCount != 10 || scan.OwnedStaleCount != 1 || scan.UnownedCount != 8 || !scan.SafetyAllowed {
+		t.Fatalf("unexpected report: %+v", scan)
+	}
+	result, err := manager.Apply(
+		context.Background(), scan.ID, "operator-1", Confirmation(scan.ID),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Deleted != 1 || len(runtime.deleted) != 1 || runtime.deleted[0] != "owned-stale" {
+		t.Fatalf("cleanup escaped owned stale set: result=%+v deleted=%v", result, runtime.deleted)
+	}
+	for _, path := range []string{"owned-current", "unowned-1", "unowned-8"} {
+		if !runtime.paths[path] {
+			t.Fatalf("cleanup deleted protected path %q", path)
+		}
+	}
+}
+
+func TestApplyBlocksExpiredOrOversizedSnapshotBeforeDelete(t *testing.T) {
+	repository := newFakeRepository()
+	repository.ownership = []store.MediaPathOwnership{{PathName: "stale", DeviceID: "old"}}
+	runtime := &fakeRuntime{paths: map[string]bool{"stale": true}, fail: make(map[string]bool)}
+	now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
+	manager := New(repository, runtime, "ins_test", nil)
+	manager.now = func() time.Time { return now }
+	scan, err := manager.Report(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if scan.SafetyAllowed || scan.SafetyReason != "ratio_exceeded" {
+		t.Fatalf("single-path 100%% cleanup was not blocked: %+v", scan)
+	}
+	if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSafetyBlocked) {
+		t.Fatalf("expected ratio gate, got %v", err)
+	}
+	if len(runtime.deleted) != 0 {
+		t.Fatal("ratio-blocked cleanup mutated MediaMTX")
+	}
+
+	for index := 0; index < 9; index++ {
+		runtime.paths[string(rune('a'+index))] = true
+	}
+	scan, err = manager.Report(context.Background())
+	if err != nil || !scan.SafetyAllowed {
+		t.Fatalf("expected a fresh 1/10 executable report: %+v %v", scan, err)
+	}
+	now = now.Add(reportTTL)
+	if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSnapshotExpired) {
+		t.Fatalf("expected expired snapshot, got %v", err)
+	}
+	if len(runtime.deleted) != 0 {
+		t.Fatal("expired cleanup mutated MediaMTX")
+	}
+}
+
+func TestApplyIsRetryableAndCannotExpandBeyondSnapshot(t *testing.T) {
+	repository := newFakeRepository()
+	repository.ownership = []store.MediaPathOwnership{
+		{PathName: "stale-a", DeviceID: "old-a"},
+		{PathName: "stale-b", DeviceID: "old-b"},
+	}
+	runtime := &fakeRuntime{paths: make(map[string]bool), fail: map[string]bool{"stale-b": true}}
+	runtime.paths["stale-a"], runtime.paths["stale-b"] = true, true
+	for index := 0; index < 18; index++ {
+		runtime.paths[string(rune(0x100+index))] = true
+	}
+	now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
+	manager := New(repository, runtime, "ins_test", nil)
+	manager.now = func() time.Time { return now }
+	scan, err := manager.Report(context.Background())
+	if err != nil || !scan.SafetyAllowed {
+		t.Fatalf("expected 2/20 report: %+v %v", scan, err)
+	}
+	repository.ownership = append(repository.ownership,
+		store.MediaPathOwnership{PathName: "new-stale", DeviceID: "new-old"})
+	runtime.paths["new-stale"] = true
+	result, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID))
+	if err == nil || result.Deleted != 1 || result.Failed != 1 {
+		t.Fatalf("expected one partial failure: %+v %v", result, err)
+	}
+	if !runtime.paths["new-stale"] {
+		t.Fatal("cleanup expanded beyond the approved snapshot")
+	}
+	runtime.fail["stale-b"] = false
+	result, err = manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID))
+	if err != nil || result.Deleted != 1 || result.Failed != 0 {
+		t.Fatalf("failed item was not retryable: %+v %v", result, err)
+	}
+	if !runtime.paths["new-stale"] {
+		t.Fatal("retry expanded beyond the approved snapshot")
+	}
+}