Prototype signed catalog fallback (T-101)

This commit is contained in:
ila
2026-07-16 16:13:18 +08:00
parent 45c242ecec
commit 0d6ed05d7e
15 changed files with 959 additions and 10 deletions
+166
View File
@@ -0,0 +1,166 @@
package catalog
import (
"bytes"
"encoding/json"
"fmt"
"io"
"regexp"
"sort"
"unicode/utf8"
)
var integerJSONNumber = regexp.MustCompile(`^-?(0|[1-9][0-9]*)$`)
func parseRestrictedJSON(data []byte) (any, error) {
if !utf8.Valid(data) {
return nil, fmt.Errorf("%w: input is not valid UTF-8", ErrInvalidDocument)
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
value, err := decodeJSONValue(decoder)
if err != nil {
return nil, err
}
if _, err := decoder.Token(); err != io.EOF {
if err == nil {
return nil, fmt.Errorf("%w: trailing JSON value", ErrInvalidDocument)
}
return nil, fmt.Errorf("%w: trailing data: %v", ErrInvalidDocument, err)
}
return value, nil
}
func decodeJSONValue(decoder *json.Decoder) (any, error) {
token, err := decoder.Token()
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidDocument, err)
}
switch value := token.(type) {
case json.Delim:
switch value {
case '{':
object := make(map[string]any)
for decoder.More() {
keyToken, err := decoder.Token()
if err != nil {
return nil, fmt.Errorf("%w: object key: %v", ErrInvalidDocument, err)
}
key, ok := keyToken.(string)
if !ok {
return nil, fmt.Errorf("%w: object key is not a string", ErrInvalidDocument)
}
if _, exists := object[key]; exists {
return nil, fmt.Errorf("%w: %q", ErrDuplicateField, key)
}
child, err := decodeJSONValue(decoder)
if err != nil {
return nil, err
}
object[key] = child
}
end, err := decoder.Token()
if err != nil || end != json.Delim('}') {
return nil, fmt.Errorf("%w: unterminated object", ErrInvalidDocument)
}
return object, nil
case '[':
var array []any
for decoder.More() {
child, err := decodeJSONValue(decoder)
if err != nil {
return nil, err
}
array = append(array, child)
}
end, err := decoder.Token()
if err != nil || end != json.Delim(']') {
return nil, fmt.Errorf("%w: unterminated array", ErrInvalidDocument)
}
return array, nil
default:
return nil, fmt.Errorf("%w: unexpected delimiter %q", ErrInvalidDocument, value)
}
case json.Number:
if !integerJSONNumber.MatchString(string(value)) {
return nil, fmt.Errorf("%w: %q", ErrUnsupportedNumber, value)
}
return value, nil
case string, bool, nil:
return value, nil
default:
return nil, fmt.Errorf("%w: unsupported token %T", ErrInvalidDocument, token)
}
}
func canonicalJSON(value any) ([]byte, error) {
var buffer bytes.Buffer
if err := appendCanonicalJSON(&buffer, value); err != nil {
return nil, err
}
return buffer.Bytes(), nil
}
func appendCanonicalJSON(buffer *bytes.Buffer, value any) error {
switch value := value.(type) {
case nil:
buffer.WriteString("null")
case bool:
if value {
buffer.WriteString("true")
} else {
buffer.WriteString("false")
}
case string:
encoded, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("%w: encode string: %v", ErrInvalidDocument, err)
}
buffer.Write(encoded)
case json.Number:
if !integerJSONNumber.MatchString(string(value)) {
return fmt.Errorf("%w: %q", ErrUnsupportedNumber, value)
}
buffer.WriteString(string(value))
case []any:
buffer.WriteByte('[')
for index, child := range value {
if index > 0 {
buffer.WriteByte(',')
}
if err := appendCanonicalJSON(buffer, child); err != nil {
return err
}
}
buffer.WriteByte(']')
case map[string]any:
keys := make([]string, 0, len(value))
for key := range value {
keys = append(keys, key)
}
sort.Strings(keys)
buffer.WriteByte('{')
for index, key := range keys {
if index > 0 {
buffer.WriteByte(',')
}
encodedKey, err := json.Marshal(key)
if err != nil {
return fmt.Errorf("%w: encode key: %v", ErrInvalidDocument, err)
}
buffer.Write(encodedKey)
buffer.WriteByte(':')
if err := appendCanonicalJSON(buffer, value[key]); err != nil {
return err
}
}
buffer.WriteByte('}')
default:
return fmt.Errorf("%w: unsupported value %T", ErrInvalidDocument, value)
}
return nil
}
+105
View File
@@ -0,0 +1,105 @@
package catalog
import (
"errors"
"fmt"
"os"
"path/filepath"
"sync"
)
var ErrCachePathEmpty = errors.New("catalog cache path is empty")
// FileCache stores a signed Catalog document and a crash-recovery backup.
type FileCache struct {
path string
mu sync.Mutex
}
func NewFileCache(path string) *FileCache {
return &FileCache{path: path}
}
func (cache *FileCache) Load() ([]byte, error) {
cache.mu.Lock()
defer cache.mu.Unlock()
if cache.path == "" {
return nil, ErrCachePathEmpty
}
document, err := os.ReadFile(cache.path)
if err == nil {
return document, nil
}
if !os.IsNotExist(err) {
return nil, err
}
return os.ReadFile(cache.backupPath())
}
func (cache *FileCache) Store(document []byte) error {
cache.mu.Lock()
defer cache.mu.Unlock()
if cache.path == "" {
return ErrCachePathEmpty
}
directory := filepath.Dir(cache.path)
if err := os.MkdirAll(directory, 0o700); err != nil {
return fmt.Errorf("create catalog cache directory: %w", err)
}
temporary, err := os.CreateTemp(directory, ".catalog-*.tmp")
if err != nil {
return fmt.Errorf("create catalog cache temp file: %w", err)
}
temporaryPath := temporary.Name()
defer os.Remove(temporaryPath)
if err := temporary.Chmod(0o600); err != nil {
temporary.Close()
return fmt.Errorf("protect catalog cache temp file: %w", err)
}
if _, err := temporary.Write(document); err != nil {
temporary.Close()
return fmt.Errorf("write catalog cache temp file: %w", err)
}
if err := temporary.Sync(); err != nil {
temporary.Close()
return fmt.Errorf("sync catalog cache temp file: %w", err)
}
if err := temporary.Close(); err != nil {
return fmt.Errorf("close catalog cache temp file: %w", err)
}
backupPath := cache.backupPath()
movedCurrent := false
if _, err := os.Stat(cache.path); err == nil {
if err := os.Remove(backupPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove stale catalog cache backup: %w", err)
}
if err := os.Rename(cache.path, backupPath); err != nil {
return fmt.Errorf("backup current catalog cache: %w", err)
}
movedCurrent = true
} else if !os.IsNotExist(err) {
return fmt.Errorf("inspect current catalog cache: %w", err)
}
if err := os.Rename(temporaryPath, cache.path); err != nil {
if movedCurrent {
_ = os.Rename(backupPath, cache.path)
}
return fmt.Errorf("activate catalog cache: %w", err)
}
if movedCurrent {
if err := os.Remove(backupPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove catalog cache backup: %w", err)
}
}
return nil
}
func (cache *FileCache) backupPath() string {
return cache.path + ".backup"
}
+49
View File
@@ -0,0 +1,49 @@
package catalog
import (
"os"
"path/filepath"
"testing"
)
func TestFileCacheStoreAndUpdate(t *testing.T) {
path := filepath.Join(t.TempDir(), "cache", "manifest.json")
cache := NewFileCache(path)
if err := cache.Store([]byte("first")); err != nil {
t.Fatalf("Store(first) error = %v", err)
}
if err := cache.Store([]byte("second")); err != nil {
t.Fatalf("Store(second) error = %v", err)
}
got, err := cache.Load()
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if string(got) != "second" {
t.Fatalf("Load() = %q, want %q", got, "second")
}
if _, err := os.Stat(path + ".backup"); !os.IsNotExist(err) {
t.Fatalf("backup should be removed after successful update, stat error = %v", err)
}
}
func TestFileCacheLoadsBackupAfterInterruptedSwitch(t *testing.T) {
path := filepath.Join(t.TempDir(), "manifest.json")
cache := NewFileCache(path)
if err := cache.Store([]byte("verified")); err != nil {
t.Fatalf("Store() error = %v", err)
}
if err := os.Rename(path, path+".backup"); err != nil {
t.Fatalf("simulate interrupted switch: %v", err)
}
got, err := cache.Load()
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if string(got) != "verified" {
t.Fatalf("Load() = %q, want %q", got, "verified")
}
}
+106
View File
@@ -0,0 +1,106 @@
package catalog
import (
"context"
"errors"
"fmt"
)
var ErrNoValidCatalog = errors.New("no valid catalog available")
// Fetcher obtains a signed Catalog document from a remote or test source.
type Fetcher interface {
Fetch(context.Context) ([]byte, error)
}
// FetchFunc adapts a function to Fetcher.
type FetchFunc func(context.Context) ([]byte, error)
func (function FetchFunc) Fetch(ctx context.Context) ([]byte, error) {
return function(ctx)
}
// Cache stores the last verified signed document.
type Cache interface {
Load() ([]byte, error)
Store([]byte) error
}
// LoadSource describes where a verified result came from.
type LoadSource string
const (
SourceRemote LoadSource = "remote"
SourceCache LoadSource = "cache"
)
// LoadResult returns verified bytes and a non-fatal refresh/cache warning.
type LoadResult struct {
Document VerifiedDocument
Source LoadSource
Warning error
}
// LoadError preserves both the refresh and cache failure.
type LoadError struct {
Refresh error
Cache error
}
func (err *LoadError) Error() string {
return fmt.Sprintf("%s: refresh=%v; cache=%v", ErrNoValidCatalog, err.Refresh, err.Cache)
}
func (err *LoadError) Unwrap() error {
return ErrNoValidCatalog
}
// Loader verifies remote data before storing it and re-verifies cache fallback.
type Loader struct {
verifier Verifier
fetcher Fetcher
cache Cache
}
func NewLoader(verifier Verifier, fetcher Fetcher, cache Cache) *Loader {
return &Loader{
verifier: verifier,
fetcher: fetcher,
cache: cache,
}
}
// Load prefers a verified remote document and falls back to verified cache.
func (loader *Loader) Load(ctx context.Context) (LoadResult, error) {
remoteBytes, refreshErr := loader.fetcher.Fetch(ctx)
if refreshErr == nil {
verified, verifyErr := loader.verifier.Verify(remoteBytes)
if verifyErr == nil {
storeErr := loader.cache.Store(verified.Bytes)
return LoadResult{
Document: verified,
Source: SourceRemote,
Warning: storeErr,
}, nil
}
refreshErr = verifyErr
}
cachedBytes, cacheErr := loader.cache.Load()
if cacheErr == nil {
var verified VerifiedDocument
verified, cacheErr = loader.verifier.Verify(cachedBytes)
if cacheErr == nil {
return LoadResult{
Document: verified,
Source: SourceCache,
Warning: refreshErr,
}, nil
}
}
return LoadResult{}, &LoadError{
Refresh: refreshErr,
Cache: cacheErr,
}
}
+148
View File
@@ -0,0 +1,148 @@
package catalog
import (
"context"
"errors"
"testing"
)
func TestLoaderUsesVerifiedRemoteAndStoresCache(t *testing.T) {
verifier, validDocument, _ := loaderTestDocuments(t)
cache := &memoryCache{}
loader := NewLoader(verifier, FetchFunc(func(context.Context) ([]byte, error) {
return validDocument, nil
}), cache)
result, err := loader.Load(context.Background())
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if result.Source != SourceRemote {
t.Fatalf("Source = %q, want %q", result.Source, SourceRemote)
}
if string(cache.document) != string(validDocument) {
t.Fatal("verified remote document was not stored")
}
}
func TestLoaderFallsBackToVerifiedCache(t *testing.T) {
verifier, validDocument, _ := loaderTestDocuments(t)
offline := errors.New("offline")
cache := &memoryCache{document: append([]byte(nil), validDocument...)}
loader := NewLoader(verifier, FetchFunc(func(context.Context) ([]byte, error) {
return nil, offline
}), cache)
result, err := loader.Load(context.Background())
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if result.Source != SourceCache {
t.Fatalf("Source = %q, want %q", result.Source, SourceCache)
}
if !errors.Is(result.Warning, offline) {
t.Fatalf("Warning = %v, want %v", result.Warning, offline)
}
}
func TestLoaderRejectsRemoteWithoutOverwritingCache(t *testing.T) {
verifier, validDocument, tamperedDocument := loaderTestDocuments(t)
cache := &memoryCache{document: append([]byte(nil), validDocument...)}
loader := NewLoader(verifier, FetchFunc(func(context.Context) ([]byte, error) {
return tamperedDocument, nil
}), cache)
result, err := loader.Load(context.Background())
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if result.Source != SourceCache {
t.Fatalf("Source = %q, want %q", result.Source, SourceCache)
}
if !errors.Is(result.Warning, ErrSignatureInvalid) {
t.Fatalf("Warning = %v, want %v", result.Warning, ErrSignatureInvalid)
}
if cache.storeCalls != 0 {
t.Fatalf("Store() called %d times for invalid remote", cache.storeCalls)
}
if string(cache.document) != string(validDocument) {
t.Fatal("invalid remote changed cached document")
}
}
func TestLoaderRejectsInvalidCache(t *testing.T) {
verifier, _, tamperedDocument := loaderTestDocuments(t)
offline := errors.New("offline")
loader := NewLoader(verifier, FetchFunc(func(context.Context) ([]byte, error) {
return nil, offline
}), &memoryCache{document: tamperedDocument})
_, err := loader.Load(context.Background())
if !errors.Is(err, ErrNoValidCatalog) {
t.Fatalf("Load() error = %v, want %v", err, ErrNoValidCatalog)
}
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("Load() error type = %T, want *LoadError", err)
}
if !errors.Is(loadErr.Cache, ErrSignatureInvalid) {
t.Fatalf("cache error = %v, want %v", loadErr.Cache, ErrSignatureInvalid)
}
}
func TestLoaderReturnsCacheStoreWarning(t *testing.T) {
verifier, validDocument, _ := loaderTestDocuments(t)
storeFailure := errors.New("disk full")
cache := &memoryCache{storeErr: storeFailure}
loader := NewLoader(verifier, FetchFunc(func(context.Context) ([]byte, error) {
return validDocument, nil
}), cache)
result, err := loader.Load(context.Background())
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if result.Source != SourceRemote {
t.Fatalf("Source = %q, want %q", result.Source, SourceRemote)
}
if !errors.Is(result.Warning, storeFailure) {
t.Fatalf("Warning = %v, want %v", result.Warning, storeFailure)
}
}
type memoryCache struct {
document []byte
loadErr error
storeErr error
storeCalls int
}
func (cache *memoryCache) Load() ([]byte, error) {
if cache.loadErr != nil {
return nil, cache.loadErr
}
return append([]byte(nil), cache.document...), nil
}
func (cache *memoryCache) Store(document []byte) error {
cache.storeCalls++
if cache.storeErr != nil {
return cache.storeErr
}
cache.document = append([]byte(nil), document...)
return nil
}
func loaderTestDocuments(t *testing.T) (Verifier, []byte, []byte) {
t.Helper()
publicKey, privateKey := catalogTestKey()
verifier, err := NewVerifier(publicKey)
if err != nil {
t.Fatalf("NewVerifier() error = %v", err)
}
validPayload := readCatalogFixture(t, "manifest-valid-payload.json")
validDocument, signature := signCatalogPayload(t, validPayload, privateKey)
tamperedPayload := readCatalogFixture(t, "manifest-tampered-payload.json")
tamperedDocument := attachCatalogSignature(t, tamperedPayload, signature)
return verifier, validDocument, tamperedDocument
}
+89
View File
@@ -0,0 +1,89 @@
package catalog
import (
"crypto/ed25519"
"encoding/base64"
"errors"
"fmt"
)
var (
ErrInvalidDocument = errors.New("invalid catalog document")
ErrDuplicateField = errors.New("duplicate catalog field")
ErrUnsupportedNumber = errors.New("unsupported catalog number")
ErrSignatureMissing = errors.New("catalog signature missing")
ErrSignatureInvalid = errors.New("catalog signature invalid")
ErrPublicKeyInvalid = errors.New("catalog public key invalid")
)
// VerifiedDocument contains the original signed document and its signing bytes.
type VerifiedDocument struct {
Bytes []byte
SignedPayload []byte
}
// Verifier validates signed Catalog JSON documents with one Ed25519 public key.
type Verifier struct {
publicKey ed25519.PublicKey
}
// NewVerifier copies and validates the public key.
func NewVerifier(publicKey []byte) (Verifier, error) {
if len(publicKey) != ed25519.PublicKeySize {
return Verifier{}, fmt.Errorf(
"%w: got %d bytes, want %d",
ErrPublicKeyInvalid,
len(publicKey),
ed25519.PublicKeySize,
)
}
keyCopy := append(ed25519.PublicKey(nil), publicKey...)
return Verifier{publicKey: keyCopy}, nil
}
// Verify rejects ambiguous JSON and validates the top-level signature.
func (verifier Verifier) Verify(document []byte) (VerifiedDocument, error) {
rootValue, err := parseRestrictedJSON(document)
if err != nil {
return VerifiedDocument{}, err
}
root, ok := rootValue.(map[string]any)
if !ok {
return VerifiedDocument{}, fmt.Errorf("%w: root must be an object", ErrInvalidDocument)
}
signatureValue, exists := root["signature"]
if !exists {
return VerifiedDocument{}, ErrSignatureMissing
}
signatureText, ok := signatureValue.(string)
if !ok {
return VerifiedDocument{}, fmt.Errorf("%w: signature must be a string", ErrSignatureInvalid)
}
delete(root, "signature")
signedPayload, err := canonicalJSON(root)
if err != nil {
return VerifiedDocument{}, err
}
signature, err := base64.StdEncoding.Strict().DecodeString(signatureText)
if err != nil {
return VerifiedDocument{}, fmt.Errorf("%w: base64: %v", ErrSignatureInvalid, err)
}
if len(signature) != ed25519.SignatureSize {
return VerifiedDocument{}, fmt.Errorf(
"%w: got %d signature bytes, want %d",
ErrSignatureInvalid,
len(signature),
ed25519.SignatureSize,
)
}
if !ed25519.Verify(verifier.publicKey, signedPayload, signature) {
return VerifiedDocument{}, ErrSignatureInvalid
}
return VerifiedDocument{
Bytes: append([]byte(nil), document...),
SignedPayload: append([]byte(nil), signedPayload...),
}, nil
}
+136
View File
@@ -0,0 +1,136 @@
package catalog
import (
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
)
func TestVerifierFixtures(t *testing.T) {
publicKey, privateKey := catalogTestKey()
verifier, err := NewVerifier(publicKey)
if err != nil {
t.Fatalf("NewVerifier() error = %v", err)
}
validPayload := readCatalogFixture(t, "manifest-valid-payload.json")
validDocument, validSignature := signCatalogPayload(t, validPayload, privateKey)
tamperedPayload := readCatalogFixture(t, "manifest-tampered-payload.json")
tamperedDocument := attachCatalogSignature(t, tamperedPayload, validSignature)
forgedDocument := readCatalogFixture(t, "manifest-forged.json")
tests := []struct {
name string
document []byte
wantErr error
}{
{name: "valid", document: validDocument},
{name: "tampered", document: tamperedDocument, wantErr: ErrSignatureInvalid},
{name: "forged", document: forgedDocument, wantErr: ErrSignatureInvalid},
{
name: "duplicate field",
document: []byte(`{"channel":"modern","channel":"win7","signature":"x"}`),
wantErr: ErrDuplicateField,
},
{
name: "fractional number",
document: []byte(`{"schema_version":1.5,"signature":"x"}`),
wantErr: ErrUnsupportedNumber,
},
{
name: "exponent number",
document: []byte(`{"schema_version":1e2,"signature":"x"}`),
wantErr: ErrUnsupportedNumber,
},
{
name: "missing signature",
document: validPayload,
wantErr: ErrSignatureMissing,
},
{
name: "trailing value",
document: append(append([]byte(nil), validDocument...), []byte(` {}`)...),
wantErr: ErrInvalidDocument,
},
{
name: "invalid UTF-8",
document: []byte{0xff, 0xfe},
wantErr: ErrInvalidDocument,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
verified, err := verifier.Verify(test.document)
if test.wantErr == nil {
if err != nil {
t.Fatalf("Verify() error = %v", err)
}
if len(verified.SignedPayload) == 0 {
t.Fatal("Verify() returned empty signed payload")
}
return
}
if !errors.Is(err, test.wantErr) {
t.Fatalf("Verify() error = %v, want %v", err, test.wantErr)
}
})
}
}
func catalogTestKey() (ed25519.PublicKey, ed25519.PrivateKey) {
seed := sha256.Sum256([]byte("SoftBox catalog verifier test key - never use in production"))
privateKey := ed25519.NewKeyFromSeed(seed[:])
return privateKey.Public().(ed25519.PublicKey), privateKey
}
func readCatalogFixture(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("..", "..", "testdata", "catalog", name)
document, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", name, err)
}
return document
}
func signCatalogPayload(t *testing.T, payload []byte, privateKey ed25519.PrivateKey) ([]byte, string) {
t.Helper()
value, err := parseRestrictedJSON(payload)
if err != nil {
t.Fatalf("parse payload: %v", err)
}
root, ok := value.(map[string]any)
if !ok {
t.Fatal("payload root is not an object")
}
canonical, err := canonicalJSON(root)
if err != nil {
t.Fatalf("canonicalize payload: %v", err)
}
signature := base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical))
return attachCatalogSignature(t, payload, signature), signature
}
func attachCatalogSignature(t *testing.T, payload []byte, signature string) []byte {
t.Helper()
value, err := parseRestrictedJSON(payload)
if err != nil {
t.Fatalf("parse payload: %v", err)
}
root, ok := value.(map[string]any)
if !ok {
t.Fatal("payload root is not an object")
}
root["signature"] = signature
document, err := json.MarshalIndent(root, "", " ")
if err != nil {
t.Fatalf("encode signed document: %v", err)
}
return document
}