Implement resumable download queue (T-301)

This commit is contained in:
ila
2026-07-16 19:49:06 +08:00
parent 2d302b731a
commit 8dad40f934
27 changed files with 5498 additions and 18 deletions
+45
View File
@@ -0,0 +1,45 @@
package application
// DownloadStartedPayload begins one transfer attempt. A later attempt may
// reset Done when a remote entity cannot be safely resumed.
type DownloadStartedPayload struct {
Attempt uint64
Done int64
TotalKnown bool
Total int64
}
// DownloadProgressPayload reports monotonic progress within one attempt.
type DownloadProgressPayload struct {
Attempt uint64
Done int64
TotalKnown bool
Total int64
SpeedBytesSec int64
}
// DownloadPausedPayload maps to domain queued while preserving pause intent.
type DownloadPausedPayload struct {
Attempt uint64
Done int64
}
// DownloadCompletedPayload means bytes are durably downloaded but still
// untrusted. T-302 must verify the signed Catalog identity, size and hashes.
type DownloadCompletedPayload struct {
Attempt uint64
Done int64
Path string
}
// DownloadFailedPayload carries a stable low-level transfer/storage code.
type DownloadFailedPayload struct {
Attempt uint64
Done int64
ErrorCode string
}
// DownloadCanceledPayload confirms cleanup and suppresses the old attempt.
type DownloadCanceledPayload struct {
Attempt uint64
}
+2
View File
@@ -11,6 +11,7 @@ const (
EventDownloadPaused EventType = "DownloadPaused"
EventDownloadCompleted EventType = "DownloadCompleted"
EventDownloadFailed EventType = "DownloadFailed"
EventDownloadCanceled EventType = "DownloadCanceled"
EventInstallCompleted EventType = "InstallCompleted"
EventInstallRolledBack EventType = "InstallRolledBack"
EventAppStarted EventType = "AppStarted"
@@ -26,6 +27,7 @@ var validEventTypes = map[EventType]struct{}{
EventDownloadPaused: {},
EventDownloadCompleted: {},
EventDownloadFailed: {},
EventDownloadCanceled: {},
EventInstallCompleted: {},
EventInstallRolledBack: {},
EventAppStarted: {},
+1
View File
@@ -11,6 +11,7 @@ func TestEventTypeValid(t *testing.T) {
EventDownloadPaused,
EventDownloadCompleted,
EventDownloadFailed,
EventDownloadCanceled,
EventInstallCompleted,
EventInstallRolledBack,
EventAppStarted,
+79
View File
@@ -0,0 +1,79 @@
package downloader
import (
"context"
"softbox.local/core/application"
)
// ApplicationPublisher is implemented by application.Runtime.
type ApplicationPublisher interface {
Publish(context.Context, application.Event) error
}
// ApplicationObserver maps downloader events to the documented application
// event envelope and concrete payloads.
type ApplicationObserver struct {
Publisher ApplicationPublisher
}
// PublishDownload implements Observer.
func (observer ApplicationObserver) PublishDownload(
ctx context.Context,
event Event,
) error {
if observer.Publisher == nil {
return nil
}
envelope := application.Event{
RequestID: event.RequestID,
AppID: event.AppID,
}
switch event.Type {
case EventStarted:
envelope.Type = application.EventDownloadStarted
envelope.Payload = application.DownloadStartedPayload{
Attempt: event.Attempt,
Done: event.Done,
TotalKnown: event.TotalKnown,
Total: event.Total,
}
case EventProgress:
envelope.Type = application.EventDownloadProgress
envelope.Payload = application.DownloadProgressPayload{
Attempt: event.Attempt,
Done: event.Done,
TotalKnown: event.TotalKnown,
Total: event.Total,
SpeedBytesSec: event.SpeedBytesSec,
}
case EventPaused:
envelope.Type = application.EventDownloadPaused
envelope.Payload = application.DownloadPausedPayload{
Attempt: event.Attempt,
Done: event.Done,
}
case EventCompleted:
envelope.Type = application.EventDownloadCompleted
envelope.Payload = application.DownloadCompletedPayload{
Attempt: event.Attempt,
Done: event.Done,
Path: event.CompletedPath,
}
case EventFailed:
envelope.Type = application.EventDownloadFailed
envelope.Payload = application.DownloadFailedPayload{
Attempt: event.Attempt,
Done: event.Done,
ErrorCode: event.ErrorCode,
}
case EventCanceled:
envelope.Type = application.EventDownloadCanceled
envelope.Payload = application.DownloadCanceledPayload{
Attempt: event.Attempt,
}
default:
return nil
}
return observer.Publisher.Publish(ctx, envelope)
}
@@ -0,0 +1,102 @@
package downloader
import (
"context"
"reflect"
"testing"
"softbox.local/core/application"
)
func TestApplicationObserverUsesConcretePayloads(t *testing.T) {
publisher := &recordingApplicationPublisher{}
observer := ApplicationObserver{Publisher: publisher}
tests := []struct {
event Event
wantType application.EventType
wantPayload any
}{
{
event: Event{
Type: EventStarted, RequestID: "request-1", AppID: "app-1",
Attempt: 1, Done: 2, TotalKnown: true, Total: 10,
},
wantType: application.EventDownloadStarted,
wantPayload: application.DownloadStartedPayload{
Attempt: 1, Done: 2, TotalKnown: true, Total: 10,
},
},
{
event: Event{
Type: EventProgress, RequestID: "request-1", AppID: "app-1",
Attempt: 1, Done: 4, TotalKnown: true, Total: 10, SpeedBytesSec: 2,
},
wantType: application.EventDownloadProgress,
wantPayload: application.DownloadProgressPayload{
Attempt: 1, Done: 4, TotalKnown: true, Total: 10, SpeedBytesSec: 2,
},
},
{
event: Event{
Type: EventPaused, RequestID: "request-1", AppID: "app-1",
Attempt: 1, Done: 4,
},
wantType: application.EventDownloadPaused,
wantPayload: application.DownloadPausedPayload{Attempt: 1, Done: 4},
},
{
event: Event{
Type: EventCompleted, RequestID: "request-1", AppID: "app-1",
Attempt: 1, Done: 10, CompletedPath: "download",
},
wantType: application.EventDownloadCompleted,
wantPayload: application.DownloadCompletedPayload{
Attempt: 1, Done: 10, Path: "download",
},
},
{
event: Event{
Type: EventFailed, RequestID: "request-1", AppID: "app-1",
Attempt: 1, Done: 4, ErrorCode: "http_status",
},
wantType: application.EventDownloadFailed,
wantPayload: application.DownloadFailedPayload{
Attempt: 1, Done: 4, ErrorCode: "http_status",
},
},
{
event: Event{
Type: EventCanceled, RequestID: "request-1", AppID: "app-1",
Attempt: 1,
},
wantType: application.EventDownloadCanceled,
wantPayload: application.DownloadCanceledPayload{Attempt: 1},
},
}
for _, test := range tests {
if err := observer.PublishDownload(context.Background(), test.event); err != nil {
t.Fatalf("PublishDownload(%s) error = %v", test.event.Type, err)
}
got := publisher.events[len(publisher.events)-1]
if got.Type != test.wantType ||
got.RequestID != test.event.RequestID ||
got.AppID != test.event.AppID {
t.Fatalf("event = %#v", got)
}
if !reflect.DeepEqual(got.Payload, test.wantPayload) {
t.Fatalf("payload = %#v, want %#v", got.Payload, test.wantPayload)
}
}
}
type recordingApplicationPublisher struct {
events []application.Event
}
func (publisher *recordingApplicationPublisher) Publish(
_ context.Context,
event application.Event,
) error {
publisher.events = append(publisher.events, event)
return nil
}
+6
View File
@@ -0,0 +1,6 @@
// Package downloader provides resumable byte-transfer queues.
//
// A completed download is still untrusted input. Callers must verify the
// signed Catalog identity, exact size, SHA-256 and package signature before
// handing the file to the installer.
package downloader
+43
View File
@@ -0,0 +1,43 @@
package downloader
import "context"
// EventType identifies an observer event emitted by Queue.
type EventType string
const (
EventStarted EventType = "started"
EventProgress EventType = "progress"
EventPaused EventType = "paused"
EventCompleted EventType = "completed"
EventFailed EventType = "failed"
EventCanceled EventType = "canceled"
)
// Event reports one generation of a download task. Done is monotonic within a
// RequestID+Attempt pair. A new Attempt permits a reset to zero when a server
// ignores Range or an entity cannot be safely resumed.
type Event struct {
Type EventType
RequestID string
AppID string
Attempt uint64
Done int64
TotalKnown bool
Total int64
SpeedBytesSec int64
CompletedPath string
ErrorCode string
}
// Observer receives queue events. Implementations must honor context
// cancellation so pause/cancel can wait for the old generation to exit.
type Observer interface {
PublishDownload(context.Context, Event) error
}
type discardObserver struct{}
func (discardObserver) PublishDownload(context.Context, Event) error {
return nil
}
+234
View File
@@ -0,0 +1,234 @@
package downloader
import (
"fmt"
"io"
"os"
)
func ensureTransferLayout(downloadsRoot, requestID string) (TaskPaths, error) {
paths, err := DeriveTaskPaths(downloadsRoot, requestID)
if err != nil {
return TaskPaths{}, err
}
if err := os.MkdirAll(paths.Root, 0o700); err != nil {
return TaskPaths{}, fmt.Errorf("create downloads root: %w", err)
}
for _, directory := range []string{paths.Root, paths.TasksDir, paths.FilesDir} {
if err := os.Mkdir(directory, 0o700); err != nil && !os.IsExist(err) {
return TaskPaths{}, fmt.Errorf("create download directory: %w", err)
}
info, err := os.Lstat(directory)
if err != nil {
return TaskPaths{}, fmt.Errorf("inspect download directory: %w", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return TaskPaths{}, fmt.Errorf("%w: managed directory is unsafe", ErrTaskCorrupt)
}
}
return paths, nil
}
func regularFileSize(filePath string) (size int64, exists bool, err error) {
info, err := os.Lstat(filePath)
if os.IsNotExist(err) {
return 0, false, nil
}
if err != nil {
return 0, false, fmt.Errorf("inspect download file: %w", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return 0, false, fmt.Errorf("%w: managed file is not regular", ErrTaskCorrupt)
}
return info.Size(), true, nil
}
func regularFileExists(filePath string) (bool, error) {
_, exists, err := regularFileSize(filePath)
return exists, err
}
func truncatePart(filePath string) error {
file, _, err := openVerifiedRegular(filePath, os.O_RDWR)
if err != nil {
return err
}
if err := file.Truncate(0); err != nil {
file.Close()
return fmt.Errorf("truncate part: %w", err)
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
file.Close()
return fmt.Errorf("seek truncated part: %w", err)
}
if err := file.Sync(); err != nil {
file.Close()
return fmt.Errorf("sync truncated part: %w", err)
}
return file.Close()
}
func openPart(paths TaskPaths, offset int64, restart bool) (*os.File, error) {
if restart {
if _, exists, err := regularFileSize(paths.Part); err != nil {
return nil, err
} else if exists {
file, _, err := openVerifiedRegular(paths.Part, os.O_RDWR)
if err != nil {
return nil, err
}
if err := file.Truncate(0); err != nil {
file.Close()
return nil, fmt.Errorf("truncate restarted part: %w", err)
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
file.Close()
return nil, fmt.Errorf("seek restarted part: %w", err)
}
return file, nil
}
return os.OpenFile(paths.Part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
}
size, exists, err := regularFileSize(paths.Part)
if err != nil {
return nil, err
}
if !exists {
if offset != 0 {
return nil, fmt.Errorf("%w: missing part for non-zero offset", ErrTaskCorrupt)
}
return os.OpenFile(paths.Part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
}
if size != offset {
return nil, fmt.Errorf("%w: part size changed", ErrTaskCorrupt)
}
file, _, err := openVerifiedRegular(paths.Part, os.O_WRONLY)
if err != nil {
return nil, err
}
if _, err := file.Seek(offset, io.SeekStart); err != nil {
file.Close()
return nil, fmt.Errorf("seek part: %w", err)
}
return file, nil
}
func syncRegularFile(filePath string) error {
file, _, err := openVerifiedRegular(filePath, os.O_RDWR)
if err != nil {
return err
}
if err := file.Sync(); err != nil {
file.Close()
return err
}
return file.Close()
}
func activateCompleted(paths TaskPaths, expected os.FileInfo) error {
if exists, err := regularFileExists(paths.Completed); err != nil {
return err
} else if exists {
return fmt.Errorf("%w: completed path already exists", ErrTaskCorrupt)
}
if expected == nil {
part, partInfo, err := openVerifiedRegular(paths.Part, os.O_RDONLY)
if err != nil {
return err
}
if err := part.Close(); err != nil {
return fmt.Errorf("close part before activation: %w", err)
}
expected = partInfo
}
if err := verifyRegularIdentity(paths.Part, expected); err != nil {
return err
}
if err := os.Rename(paths.Part, paths.Completed); err != nil {
return fmt.Errorf("activate completed download: %w", err)
}
completedInfo, err := os.Lstat(paths.Completed)
if err != nil {
return fmt.Errorf("inspect activated download: %w", err)
}
if completedInfo.Mode()&os.ModeSymlink != 0 ||
!completedInfo.Mode().IsRegular() ||
!os.SameFile(expected, completedInfo) {
_ = removeExactRegularFile(paths.Completed)
return fmt.Errorf("%w: activated file identity changed", ErrTaskCorrupt)
}
return nil
}
func verifyRegularIdentity(filePath string, expected os.FileInfo) error {
current, err := os.Lstat(filePath)
if err != nil {
return fmt.Errorf("inspect managed file identity: %w", err)
}
if current.Mode()&os.ModeSymlink != 0 ||
!current.Mode().IsRegular() ||
!os.SameFile(expected, current) {
return fmt.Errorf("%w: managed file identity changed", ErrTaskCorrupt)
}
return nil
}
func removeExactRegularFile(filePath string) error {
info, err := os.Lstat(filePath)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf("%w: refuse removal of non-regular file", ErrTaskCorrupt)
}
return os.Remove(filePath)
}
func syncAndClose(file *os.File) error {
if file == nil {
return nil
}
if err := file.Sync(); err != nil {
file.Close()
return err
}
return file.Close()
}
func openVerifiedRegular(
filePath string,
flags int,
) (*os.File, os.FileInfo, error) {
before, err := os.Lstat(filePath)
if err != nil {
return nil, nil, err
}
if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() {
return nil, nil, fmt.Errorf("%w: managed file is not regular", ErrTaskCorrupt)
}
file, err := os.OpenFile(filePath, flags, 0o600)
if err != nil {
return nil, nil, fmt.Errorf("open managed file: %w", err)
}
opened, err := file.Stat()
if err != nil {
file.Close()
return nil, nil, fmt.Errorf("stat managed file: %w", err)
}
after, err := os.Lstat(filePath)
if err != nil {
file.Close()
return nil, nil, fmt.Errorf("restat managed file: %w", err)
}
if after.Mode()&os.ModeSymlink != 0 ||
!after.Mode().IsRegular() ||
!os.SameFile(before, opened) ||
!os.SameFile(opened, after) {
file.Close()
return nil, nil, fmt.Errorf("%w: managed file changed while opening", ErrTaskCorrupt)
}
return file, opened, nil
}
+45
View File
@@ -0,0 +1,45 @@
package downloader
import (
"errors"
"os"
"testing"
)
func TestActivateCompletedRejectsReplacementOfWrittenPart(t *testing.T) {
paths, err := ensureTransferLayout(t.TempDir(), "request-identity")
if err != nil {
t.Fatalf("ensureTransferLayout() error = %v", err)
}
if err := os.WriteFile(paths.Part, []byte("trusted"), 0o600); err != nil {
t.Fatalf("WriteFile(original) error = %v", err)
}
original, expected, err := openVerifiedRegular(paths.Part, os.O_RDONLY)
if err != nil {
t.Fatalf("openVerifiedRegular() error = %v", err)
}
if err := original.Close(); err != nil {
t.Fatalf("Close(original) error = %v", err)
}
if err := os.Remove(paths.Part); err != nil {
t.Fatalf("Remove(original) error = %v", err)
}
if err := os.WriteFile(paths.Part, []byte("replacement"), 0o600); err != nil {
t.Fatalf("WriteFile(replacement) error = %v", err)
}
err = activateCompleted(paths, expected)
if !errors.Is(err, ErrTaskCorrupt) {
t.Fatalf("activateCompleted() error = %v, want %v", err, ErrTaskCorrupt)
}
if _, err := os.Stat(paths.Completed); !os.IsNotExist(err) {
t.Fatalf("completed path exists after identity mismatch: %v", err)
}
document, err := os.ReadFile(paths.Part)
if err != nil {
t.Fatalf("ReadFile(replacement) error = %v", err)
}
if string(document) != "replacement" {
t.Fatalf("replacement bytes = %q", document)
}
}
+280
View File
@@ -0,0 +1,280 @@
package downloader
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
)
var (
ErrHTTPStatus = errors.New("download HTTP status is not successful")
ErrRangeMismatch = errors.New("download Content-Range does not match request")
ErrRangeEntityChanged = errors.New("download range entity validator changed")
ErrResponseEncoding = errors.New("download response encoding is not identity")
ErrInsecureRedirect = errors.New("download redirect target is not secure")
ErrTransferTooLarge = errors.New("download response exceeds byte limit")
ErrTransferIncomplete = errors.New("download response ended before expected length")
)
// OpenRequest describes one HTTP attempt.
type OpenRequest struct {
URL string
Offset int64
Validator EntityValidator
}
// OpenResponse owns Body until the caller closes it.
type OpenResponse struct {
Body io.ReadCloser
Restart bool
TotalKnown bool
Total int64
ResponseLengthKnown bool
ResponseLength int64
Validator EntityValidator
}
// Transport opens a remote byte stream at a requested offset.
type Transport interface {
Open(context.Context, OpenRequest) (OpenResponse, error)
}
// HTTPTransport implements strict HTTPS and Range semantics with net/http.
type HTTPTransport struct {
client *http.Client
}
// NewHTTPTransport creates a standard-library transport.
func NewHTTPTransport(client *http.Client) *HTTPTransport {
if client == nil {
client = http.DefaultClient
}
clientCopy := *client
previousRedirectCheck := client.CheckRedirect
clientCopy.CheckRedirect = func(request *http.Request, via []*http.Request) error {
if request.URL == nil || ValidateHTTPSURL(request.URL.String()) != nil {
return ErrInsecureRedirect
}
if previousRedirectCheck != nil {
return previousRedirectCheck(request, via)
}
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
return nil
}
return &HTTPTransport{client: &clientCopy}
}
// Open starts a transfer. A 200 response to a Range request is returned as
// Restart=true so the queue truncates the old part before consuming bytes.
func (transport *HTTPTransport) Open(
ctx context.Context,
request OpenRequest,
) (OpenResponse, error) {
if err := ValidateHTTPSURL(request.URL); err != nil {
return OpenResponse{}, err
}
if request.Offset < 0 {
return OpenResponse{}, fmt.Errorf("%w: negative offset", ErrRangeMismatch)
}
if err := request.Validator.Validate(); err != nil {
return OpenResponse{}, fmt.Errorf("%w: invalid validator", ErrRangeMismatch)
}
if request.Offset > 0 && request.Validator.Empty() {
return OpenResponse{}, fmt.Errorf("%w: resume needs an entity validator", ErrRangeMismatch)
}
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, request.URL, nil)
if err != nil {
return OpenResponse{}, fmt.Errorf("create download request: %w", err)
}
httpRequest.Header.Set("Accept-Encoding", "identity")
if request.Offset > 0 {
httpRequest.Header.Set("Range", fmt.Sprintf("bytes=%d-", request.Offset))
httpRequest.Header.Set("If-Range", request.Validator.HeaderValue())
}
response, err := transport.client.Do(httpRequest)
if err != nil {
return OpenResponse{}, err
}
closeWithError := func(openErr error) (OpenResponse, error) {
_ = response.Body.Close()
return OpenResponse{}, openErr
}
if response.Request == nil || response.Request.URL == nil ||
ValidateHTTPSURL(response.Request.URL.String()) != nil {
return closeWithError(ErrInsecureRedirect)
}
encoding := strings.TrimSpace(response.Header.Get("Content-Encoding"))
if encoding != "" && !strings.EqualFold(encoding, "identity") {
return closeWithError(ErrResponseEncoding)
}
validator := responseValidator(response.Header)
switch response.StatusCode {
case http.StatusOK:
if response.Header.Get("Content-Range") != "" {
return closeWithError(fmt.Errorf("%w: 200 includes Content-Range", ErrRangeMismatch))
}
totalKnown := response.ContentLength >= 0
total := response.ContentLength
if !totalKnown {
total = 0
}
return OpenResponse{
Body: response.Body,
Restart: request.Offset > 0,
TotalKnown: totalKnown,
Total: total,
ResponseLengthKnown: totalKnown,
ResponseLength: total,
Validator: validator,
}, nil
case http.StatusPartialContent:
if request.Offset == 0 {
return closeWithError(fmt.Errorf("%w: unsolicited partial response", ErrRangeMismatch))
}
start, end, totalKnown, total, parseErr := parseContentRange(
response.Header.Get("Content-Range"),
)
if parseErr != nil || start != request.Offset {
return closeWithError(ErrRangeMismatch)
}
rangeLength := end - start + 1
if rangeLength <= 0 ||
(response.ContentLength >= 0 && response.ContentLength != rangeLength) {
return closeWithError(ErrRangeMismatch)
}
if !validatorsMatch(request.Validator, validator) {
return closeWithError(ErrRangeEntityChanged)
}
return OpenResponse{
Body: response.Body,
TotalKnown: totalKnown,
Total: total,
ResponseLengthKnown: true,
ResponseLength: rangeLength,
Validator: request.Validator,
}, nil
default:
return closeWithError(fmt.Errorf("%w: %d", ErrHTTPStatus, response.StatusCode))
}
}
func parseContentRange(value string) (
start int64,
end int64,
totalKnown bool,
total int64,
err error,
) {
if !strings.HasPrefix(value, "bytes ") {
return 0, 0, false, 0, ErrRangeMismatch
}
rangeAndTotal := strings.Split(strings.TrimPrefix(value, "bytes "), "/")
if len(rangeAndTotal) != 2 {
return 0, 0, false, 0, ErrRangeMismatch
}
bounds := strings.Split(rangeAndTotal[0], "-")
if len(bounds) != 2 {
return 0, 0, false, 0, ErrRangeMismatch
}
start, err = strconv.ParseInt(bounds[0], 10, 64)
if err != nil || start < 0 {
return 0, 0, false, 0, ErrRangeMismatch
}
end, err = strconv.ParseInt(bounds[1], 10, 64)
if err != nil || end < start || end == int64(^uint64(0)>>1) {
return 0, 0, false, 0, ErrRangeMismatch
}
if rangeAndTotal[1] == "*" {
return start, end, false, 0, nil
}
total, err = strconv.ParseInt(rangeAndTotal[1], 10, 64)
if err != nil || total <= end {
return 0, 0, false, 0, ErrRangeMismatch
}
return start, end, true, total, nil
}
// CopyOptions bounds one response copy and tags progress with the generation
// that owns it.
type CopyOptions struct {
MaxBytes int64
ExpectedBytesKnown bool
ExpectedBytes int64
Generation uint64
Progress func(CopyProgress) error
}
// CopyProgress reports bytes written during this response only.
type CopyProgress struct {
Generation uint64
Written int64
}
// CopyResponse copies a response with a hard byte cap. It reads at most
// MaxBytes+1 bytes so an oversized body is detected without unbounded IO.
func CopyResponse(
ctx context.Context,
dst io.Writer,
src io.Reader,
options CopyOptions,
) (int64, error) {
if options.MaxBytes < 0 || options.MaxBytes == int64(^uint64(0)>>1) {
return 0, fmt.Errorf("%w: invalid byte limit", ErrTransferTooLarge)
}
if options.ExpectedBytesKnown &&
(options.ExpectedBytes < 0 || options.ExpectedBytes > options.MaxBytes) {
return 0, fmt.Errorf("%w: invalid expected response length", ErrTransferIncomplete)
}
limited := &io.LimitedReader{R: src, N: options.MaxBytes + 1}
buffer := make([]byte, 32*1024)
var written int64
for {
if err := ctx.Err(); err != nil {
return written, err
}
readCount, readErr := limited.Read(buffer)
if readCount > 0 {
if int64(readCount) > options.MaxBytes-written {
return written, ErrTransferTooLarge
}
writeCount, writeErr := dst.Write(buffer[:readCount])
written += int64(writeCount)
if writeErr != nil {
return written, writeErr
}
if writeCount != readCount {
return written, io.ErrShortWrite
}
if options.Progress != nil {
if err := options.Progress(CopyProgress{
Generation: options.Generation,
Written: written,
}); err != nil {
return written, err
}
}
}
if readErr != nil {
if readErr != io.EOF {
return written, readErr
}
break
}
}
if options.ExpectedBytesKnown && written != options.ExpectedBytes {
return written, ErrTransferIncomplete
}
return written, nil
}
+295
View File
@@ -0,0 +1,295 @@
package downloader
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"testing"
)
func TestHTTPTransportFreshAndResume(t *testing.T) {
content := []byte("abcdefghij")
server := httptest.NewTLSServer(http.HandlerFunc(func(
writer http.ResponseWriter,
request *http.Request,
) {
if request.Header.Get("Accept-Encoding") != "identity" {
t.Errorf("Accept-Encoding = %q", request.Header.Get("Accept-Encoding"))
}
switch request.Header.Get("Range") {
case "":
writer.Header().Set("ETag", `"v1"`)
writer.Header().Set("Content-Length", "10")
_, _ = writer.Write(content)
case "bytes=4-":
if request.Header.Get("If-Range") != `"v1"` {
t.Errorf("If-Range = %q", request.Header.Get("If-Range"))
}
writer.Header().Set("ETag", `"v1"`)
writer.Header().Set("Content-Range", "bytes 4-9/10")
writer.Header().Set("Content-Length", "6")
writer.WriteHeader(http.StatusPartialContent)
_, _ = writer.Write(content[4:])
default:
t.Errorf("unexpected Range %q", request.Header.Get("Range"))
}
}))
defer server.Close()
transport := NewHTTPTransport(server.Client())
fresh, err := transport.Open(context.Background(), OpenRequest{URL: server.URL})
if err != nil {
t.Fatalf("Open(fresh) error = %v", err)
}
freshBytes, err := io.ReadAll(fresh.Body)
fresh.Body.Close()
if err != nil || !bytes.Equal(freshBytes, content) {
t.Fatalf("fresh bytes = %q, error=%v", freshBytes, err)
}
if fresh.Restart || !fresh.TotalKnown || fresh.Total != 10 ||
fresh.Validator.ETag != `"v1"` {
t.Fatalf("fresh response = %#v", fresh)
}
resumed, err := transport.Open(context.Background(), OpenRequest{
URL: server.URL,
Offset: 4,
Validator: EntityValidator{ETag: `"v1"`},
})
if err != nil {
t.Fatalf("Open(resume) error = %v", err)
}
resumedBytes, err := io.ReadAll(resumed.Body)
resumed.Body.Close()
if err != nil || !bytes.Equal(resumedBytes, content[4:]) {
t.Fatalf("resume bytes = %q, error=%v", resumedBytes, err)
}
if resumed.Restart || resumed.ResponseLength != 6 || resumed.Total != 10 {
t.Fatalf("resume response = %#v", resumed)
}
}
func TestHTTPTransportRestartsWhenRangeIgnored(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(
writer http.ResponseWriter,
_ *http.Request,
) {
writer.Header().Set("ETag", `"v2"`)
writer.Header().Set("Content-Length", "4")
_, _ = writer.Write([]byte("new!"))
}))
defer server.Close()
response, err := NewHTTPTransport(server.Client()).Open(
context.Background(),
OpenRequest{
URL: server.URL,
Offset: 2,
Validator: EntityValidator{ETag: `"v1"`},
},
)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer response.Body.Close()
if !response.Restart || response.Validator.ETag != `"v2"` {
t.Fatalf("response = %#v", response)
}
}
func TestHTTPTransportRejectsInvalidRangeResponses(t *testing.T) {
tests := []struct {
name string
contentRange string
etag string
want error
}{
{name: "wrong start", contentRange: "bytes 3-9/10", etag: `"v1"`, want: ErrRangeMismatch},
{name: "malformed", contentRange: "invalid", etag: `"v1"`, want: ErrRangeMismatch},
{name: "entity changed", contentRange: "bytes 4-9/10", etag: `"v2"`, want: ErrRangeEntityChanged},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(
writer http.ResponseWriter,
_ *http.Request,
) {
writer.Header().Set("Content-Range", test.contentRange)
writer.Header().Set("Content-Length", "6")
writer.Header().Set("ETag", test.etag)
writer.WriteHeader(http.StatusPartialContent)
_, _ = writer.Write([]byte("efghij"))
}))
defer server.Close()
_, err := NewHTTPTransport(server.Client()).Open(
context.Background(),
OpenRequest{
URL: server.URL,
Offset: 4,
Validator: EntityValidator{ETag: `"v1"`},
},
)
if !errors.Is(err, test.want) {
t.Fatalf("Open() error = %v, want %v", err, test.want)
}
})
}
}
func TestHTTPTransportRejectsNonSuccessStatuses(t *testing.T) {
for _, status := range []int{
http.StatusNotFound,
http.StatusRequestedRangeNotSatisfiable,
http.StatusInternalServerError,
} {
t.Run(http.StatusText(status), func(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(
writer http.ResponseWriter,
_ *http.Request,
) {
writer.WriteHeader(status)
}))
defer server.Close()
_, err := NewHTTPTransport(server.Client()).Open(
context.Background(),
OpenRequest{URL: server.URL},
)
if !errors.Is(err, ErrHTTPStatus) {
t.Fatalf("Open() error = %v, want %v", err, ErrHTTPStatus)
}
})
}
}
func TestHTTPTransportRejectsUnsafeSuccessMetadata(t *testing.T) {
tests := []struct {
name string
offset int64
headers map[string]string
status int
want error
}{
{
name: "encoded body",
status: http.StatusOK,
headers: map[string]string{
"Content-Encoding": "gzip",
},
want: ErrResponseEncoding,
},
{
name: "200 with content range",
status: http.StatusOK,
headers: map[string]string{
"Content-Range": "bytes 0-3/4",
},
want: ErrRangeMismatch,
},
{
name: "206 missing content range",
offset: 2,
status: http.StatusPartialContent,
headers: map[string]string{
"ETag": `"v1"`,
},
want: ErrRangeMismatch,
},
{
name: "206 content length mismatch",
offset: 2,
status: http.StatusPartialContent,
headers: map[string]string{
"Content-Range": "bytes 2-3/4",
"Content-Length": "1",
"ETag": `"v1"`,
},
want: ErrRangeMismatch,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(
writer http.ResponseWriter,
_ *http.Request,
) {
for name, value := range test.headers {
writer.Header().Set(name, value)
}
writer.WriteHeader(test.status)
_, _ = writer.Write([]byte("data"))
}))
defer server.Close()
request := OpenRequest{URL: server.URL, Offset: test.offset}
if test.offset > 0 {
request.Validator = EntityValidator{ETag: `"v1"`}
}
_, err := NewHTTPTransport(server.Client()).Open(
context.Background(),
request,
)
if !errors.Is(err, test.want) {
t.Fatalf("Open() error = %v, want %v", err, test.want)
}
})
}
}
func TestHTTPTransportRejectsDowngradeRedirect(t *testing.T) {
insecure := httptest.NewServer(http.HandlerFunc(func(
writer http.ResponseWriter,
_ *http.Request,
) {
_, _ = writer.Write([]byte("unsafe"))
}))
defer insecure.Close()
secure := httptest.NewTLSServer(http.HandlerFunc(func(
writer http.ResponseWriter,
request *http.Request,
) {
http.Redirect(writer, request, insecure.URL, http.StatusFound)
}))
defer secure.Close()
_, err := NewHTTPTransport(secure.Client()).Open(
context.Background(),
OpenRequest{URL: secure.URL},
)
if !errors.Is(err, ErrInsecureRedirect) {
t.Fatalf("Open() error = %v, want %v", err, ErrInsecureRedirect)
}
}
func TestCopyResponseBoundsAndCompleteness(t *testing.T) {
var destination bytes.Buffer
written, err := CopyResponse(
context.Background(),
&destination,
bytes.NewReader([]byte("1234")),
CopyOptions{MaxBytes: 4, ExpectedBytesKnown: true, ExpectedBytes: 4},
)
if err != nil || written != 4 || destination.String() != "1234" {
t.Fatalf("CopyResponse() = %d, %q, %v", written, destination.String(), err)
}
_, err = CopyResponse(
context.Background(),
io.Discard,
bytes.NewReader([]byte("12345")),
CopyOptions{MaxBytes: 4},
)
if !errors.Is(err, ErrTransferTooLarge) {
t.Fatalf("oversize error = %v, want %v", err, ErrTransferTooLarge)
}
_, err = CopyResponse(
context.Background(),
io.Discard,
bytes.NewReader([]byte("123")),
CopyOptions{MaxBytes: 4, ExpectedBytesKnown: true, ExpectedBytes: 4},
)
if !errors.Is(err, ErrTransferIncomplete) {
t.Fatalf("short error = %v, want %v", err, ErrTransferIncomplete)
}
}
+60
View File
@@ -0,0 +1,60 @@
package downloader
import (
"fmt"
"path/filepath"
"strings"
)
// TaskPaths are fixed local names derived only from RequestID.
type TaskPaths struct {
Root string
TasksDir string
FilesDir string
Metadata string
Backup string
Part string
Completed string
}
// DeriveTaskPaths derives all local paths without accepting remote filenames
// or Content-Disposition values.
func DeriveTaskPaths(downloadsRoot, requestID string) (TaskPaths, error) {
if downloadsRoot == "" {
return TaskPaths{}, fmt.Errorf("%w: empty downloads root", ErrInvalidTask)
}
if !ValidRequestID(requestID) {
return TaskPaths{}, fmt.Errorf("%w: invalid request_id", ErrInvalidTask)
}
absoluteRoot, err := filepath.Abs(downloadsRoot)
if err != nil {
return TaskPaths{}, fmt.Errorf("%w: resolve downloads root", ErrInvalidTask)
}
tasksDir := filepath.Join(absoluteRoot, "tasks")
filesDir := filepath.Join(absoluteRoot, "files")
paths := TaskPaths{
Root: absoluteRoot,
TasksDir: tasksDir,
FilesDir: filesDir,
Metadata: filepath.Join(tasksDir, requestID+".json"),
Backup: filepath.Join(tasksDir, requestID+".json.backup"),
Part: filepath.Join(filesDir, requestID+".part"),
Completed: filepath.Join(filesDir, requestID+".download"),
}
for _, candidate := range []string{
paths.TasksDir,
paths.FilesDir,
paths.Metadata,
paths.Backup,
paths.Part,
paths.Completed,
} {
relative, relativeErr := filepath.Rel(absoluteRoot, candidate)
if relativeErr != nil ||
relative == ".." ||
strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return TaskPaths{}, fmt.Errorf("%w: derived path escapes root", ErrInvalidTask)
}
}
return paths, nil
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+239
View File
@@ -0,0 +1,239 @@
package downloader
import (
"errors"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"time"
)
const (
TaskSchemaVersion = 1
MaxRequestIDBytes = 128
MaxAppIDBytes = 128
MaxURLBytes = 4096
MaxErrorCodeBytes = 64
MaxValidatorBytes = 512
)
var (
ErrInvalidTask = errors.New("download task is invalid")
ErrTaskConflict = errors.New("download task conflicts with an existing task")
ErrTaskNotFound = errors.New("download task not found")
ErrInvalidCommand = errors.New("download command is invalid for task state")
ErrTaskBusy = errors.New("download task is busy")
ErrQueueClosed = errors.New("download queue is closed")
ErrTaskCorrupt = errors.New("download task state is corrupt")
requestIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,127}$`)
appIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,127}$`)
errorCodePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_]{0,63}$`)
strongETagPattern = regexp.MustCompile(`^"[\x21\x23-\x7E\x80-\xFF]*"$`)
)
// TaskStatus is the durable lifecycle of one byte transfer.
type TaskStatus string
const (
StatusQueued TaskStatus = "queued"
StatusDownloading TaskStatus = "downloading"
StatusPaused TaskStatus = "paused"
StatusFailed TaskStatus = "failed"
StatusCompleted TaskStatus = "completed"
)
// EntityValidator binds a resumed range to the same remote representation.
// ETag must be strong. LastModified must be an HTTP-date.
type EntityValidator struct {
ETag string `json:"etag,omitempty"`
LastModified string `json:"last_modified,omitempty"`
}
// Task is the persisted download-task.json v1 protocol.
//
// Local paths are deliberately absent. They are derived from RequestID below
// the configured downloads root so remote metadata cannot choose filesystem
// destinations.
type Task struct {
SchemaVersion int `json:"schema_version"`
RequestID string `json:"request_id"`
AppID string `json:"app_id"`
URL string `json:"url"`
Status TaskStatus `json:"status"`
Attempt uint64 `json:"attempt"`
Done int64 `json:"done"`
TotalKnown bool `json:"total_known"`
Total int64 `json:"total"`
Validator EntityValidator `json:"validator"`
ErrorCode string `json:"error_code"`
CreatedAt string `json:"created_at"`
}
// Terminal reports whether the task no longer occupies the per-app active
// task slot. Completed transfers remain queryable until a later workflow
// consumes or removes them.
func (task Task) Terminal() bool {
return task.Status == StatusCompleted
}
// Validate enforces the durable protocol independently of JSON Schema.
func (task Task) Validate() error {
if task.SchemaVersion != TaskSchemaVersion {
return fmt.Errorf("%w: schema_version=%d", ErrInvalidTask, task.SchemaVersion)
}
if !ValidRequestID(task.RequestID) {
return fmt.Errorf("%w: invalid request_id", ErrInvalidTask)
}
if !appIDPattern.MatchString(task.AppID) || len(task.AppID) > MaxAppIDBytes {
return fmt.Errorf("%w: invalid app_id", ErrInvalidTask)
}
if err := ValidateHTTPSURL(task.URL); err != nil {
return fmt.Errorf("%w: url: %v", ErrInvalidTask, err)
}
switch task.Status {
case StatusQueued, StatusDownloading, StatusPaused, StatusFailed, StatusCompleted:
default:
return fmt.Errorf("%w: invalid status %q", ErrInvalidTask, task.Status)
}
if task.Attempt > uint64(^uint64(0)>>1) {
return fmt.Errorf("%w: attempt is too large", ErrInvalidTask)
}
if task.Done < 0 {
return fmt.Errorf("%w: done=%d", ErrInvalidTask, task.Done)
}
if task.TotalKnown {
if task.Total <= 0 || task.Done > task.Total {
return fmt.Errorf(
"%w: invalid known total done=%d total=%d",
ErrInvalidTask,
task.Done,
task.Total,
)
}
if task.Status == StatusCompleted && task.Done != task.Total {
return fmt.Errorf(
"%w: completed done=%d total=%d",
ErrInvalidTask,
task.Done,
task.Total,
)
}
} else if task.Total != 0 {
return fmt.Errorf("%w: unknown total must be zero", ErrInvalidTask)
}
if err := task.Validator.Validate(); err != nil {
return fmt.Errorf("%w: validator: %v", ErrInvalidTask, err)
}
if task.Status == StatusFailed {
if !errorCodePattern.MatchString(task.ErrorCode) ||
len(task.ErrorCode) > MaxErrorCodeBytes {
return fmt.Errorf("%w: failed task needs a stable error_code", ErrInvalidTask)
}
} else if task.ErrorCode != "" {
return fmt.Errorf("%w: error_code is only valid for failed tasks", ErrInvalidTask)
}
createdAt, err := time.Parse(time.RFC3339Nano, task.CreatedAt)
if err != nil {
return fmt.Errorf("%w: created_at: %v", ErrInvalidTask, err)
}
_, offset := createdAt.Zone()
if offset != 0 {
return fmt.Errorf("%w: created_at must be UTC", ErrInvalidTask)
}
return nil
}
// Validate validates an entity validator.
func (validator EntityValidator) Validate() error {
if validator.ETag != "" && validator.LastModified != "" {
return errors.New("etag and last_modified are mutually exclusive")
}
if validator.ETag != "" {
if len(validator.ETag) > MaxValidatorBytes ||
strings.HasPrefix(strings.ToUpper(validator.ETag), "W/") ||
!strongETagPattern.MatchString(validator.ETag) {
return errors.New("etag is not a strong ETag")
}
}
if validator.LastModified != "" {
if len(validator.LastModified) > MaxValidatorBytes {
return errors.New("last_modified is too long")
}
if _, err := http.ParseTime(validator.LastModified); err != nil {
return errors.New("last_modified is not an HTTP-date")
}
}
return nil
}
// Empty reports whether no reliable resume validator is available.
func (validator EntityValidator) Empty() bool {
return validator.ETag == "" && validator.LastModified == ""
}
// HeaderValue returns the If-Range value.
func (validator EntityValidator) HeaderValue() string {
if validator.ETag != "" {
return validator.ETag
}
return validator.LastModified
}
// ValidRequestID reports whether a request ID can safely derive local names.
func ValidRequestID(requestID string) bool {
return len(requestID) <= MaxRequestIDBytes && requestIDPattern.MatchString(requestID)
}
// ValidateHTTPSURL applies the same transport restrictions as the Catalog.
func ValidateHTTPSURL(value string) error {
if value == "" || len(value) > MaxURLBytes {
return errors.New("URL length is invalid")
}
parsed, err := url.Parse(value)
if err != nil {
return errors.New("URL is malformed")
}
if parsed.Scheme != "https" ||
parsed.Host == "" ||
parsed.User != nil ||
parsed.Fragment != "" ||
!parsed.IsAbs() {
return errors.New("URL must be absolute HTTPS without user info or fragment")
}
return nil
}
func validatorsMatch(expected, actual EntityValidator) bool {
if expected.ETag != "" {
if actual.Empty() {
return true
}
return expected.ETag == actual.ETag
}
if expected.LastModified != "" {
if actual.Empty() {
return true
}
return expected.LastModified == actual.LastModified
}
return true
}
func responseValidator(header http.Header) EntityValidator {
etag := strings.TrimSpace(header.Get("ETag"))
if etag != "" &&
!strings.HasPrefix(strings.ToUpper(etag), "W/") &&
strongETagPattern.MatchString(etag) &&
len(etag) <= MaxValidatorBytes {
return EntityValidator{ETag: etag}
}
lastModified := strings.TrimSpace(header.Get("Last-Modified"))
if lastModified != "" && len(lastModified) <= MaxValidatorBytes {
if _, err := http.ParseTime(lastModified); err == nil {
return EntityValidator{LastModified: lastModified}
}
}
return EntityValidator{}
}
+89
View File
@@ -0,0 +1,89 @@
package downloader
import (
"errors"
"path/filepath"
"testing"
)
func TestTaskValidateAndDerivedPaths(t *testing.T) {
task := validTaskForTest()
if err := task.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
paths, err := DeriveTaskPaths(filepath.Join(t.TempDir(), "downloads"), task.RequestID)
if err != nil {
t.Fatalf("DeriveTaskPaths() error = %v", err)
}
for _, path := range []string{
paths.Metadata,
paths.Backup,
paths.Part,
paths.Completed,
} {
relative, err := filepath.Rel(paths.Root, path)
if err != nil || relative == ".." {
t.Fatalf("derived path escapes root: %s", path)
}
}
}
func TestTaskRejectsInvalidFields(t *testing.T) {
tests := []struct {
name string
mutate func(*Task)
}{
{name: "request id", mutate: func(task *Task) { task.RequestID = "../escape" }},
{name: "app id", mutate: func(task *Task) { task.AppID = "Bad App" }},
{name: "URL", mutate: func(task *Task) { task.URL = "http://example.invalid/app.zip" }},
{name: "state", mutate: func(task *Task) { task.Status = "unknown" }},
{name: "negative done", mutate: func(task *Task) { task.Done = -1 }},
{name: "done exceeds total", mutate: func(task *Task) { task.Done = task.Total + 1 }},
{name: "unknown total value", mutate: func(task *Task) {
task.TotalKnown = false
task.Total = 10
}},
{name: "weak etag", mutate: func(task *Task) {
task.Validator = EntityValidator{ETag: `W/"weak"`}
}},
{name: "failed without code", mutate: func(task *Task) {
task.Status = StatusFailed
task.ErrorCode = ""
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
task := validTaskForTest()
test.mutate(&task)
if err := task.Validate(); !errors.Is(err, ErrInvalidTask) {
t.Fatalf("Validate() error = %v, want %v", err, ErrInvalidTask)
}
})
}
}
func TestValidatorsMatchRejectsDifferentKinds(t *testing.T) {
expected := EntityValidator{ETag: `"v1"`}
if validatorsMatch(expected, EntityValidator{LastModified: "Wed, 16 Jul 2026 10:00:00 GMT"}) {
t.Fatal("ETag unexpectedly matched Last-Modified")
}
if validatorsMatch(expected, EntityValidator{ETag: `"v2"`}) {
t.Fatal("different ETags matched")
}
if !validatorsMatch(expected, EntityValidator{}) {
t.Fatal("absent response validator should not contradict If-Range")
}
}
func validTaskForTest() Task {
return Task{
SchemaVersion: TaskSchemaVersion,
RequestID: "request-001",
AppID: "json-parser",
URL: "https://download.invalid/json-parser.zip",
Status: StatusQueued,
TotalKnown: true,
Total: 42,
CreatedAt: "2026-07-16T00:00:00Z",
}
}
+335
View File
@@ -0,0 +1,335 @@
package downloader
import (
"context"
"fmt"
"os"
"time"
)
func (queue *Queue) transfer(
runtime *runtimeTask,
ctx context.Context,
attempt uint64,
) (completedPath string, currentAttempt uint64, transferErr error) {
currentAttempt = attempt
task, valid := queue.taskForAttempt(runtime, currentAttempt)
if !valid {
return "", currentAttempt, context.Canceled
}
paths, err := ensureTransferLayout(queue.root, task.RequestID)
if err != nil {
return "", currentAttempt, err
}
offset, _, err := regularFileSize(paths.Part)
if err != nil {
return "", currentAttempt, err
}
response, err := queue.transport.Open(ctx, OpenRequest{
URL: task.URL,
Offset: offset,
Validator: task.Validator,
})
if err != nil {
return "", currentAttempt, err
}
body := response.Body
defer func() {
if body != nil {
_ = body.Close()
}
}()
if task.TotalKnown && response.TotalKnown && response.Total != task.Total {
return "", currentAttempt, fmt.Errorf(
"%w: response total %d, expected %d",
ErrTransferIncomplete,
response.Total,
task.Total,
)
}
if !task.TotalKnown && response.TotalKnown && response.Total > queue.maxUnknown {
return "", currentAttempt, ErrTransferTooLarge
}
if !task.TotalKnown && offset > 0 && !response.Restart && !response.TotalKnown {
return "", currentAttempt, fmt.Errorf(
"%w: resumed response has unknown complete length",
ErrTransferIncomplete,
)
}
var part *os.File
if response.Restart {
part, err = openPart(paths, offset, true)
if err != nil {
return "", currentAttempt, err
}
if err := part.Sync(); err != nil {
_ = part.Close()
return "", currentAttempt, fmt.Errorf("sync restarted part: %w", err)
}
currentAttempt, task, err = queue.restartAttempt(
runtime,
currentAttempt,
response,
)
if err != nil {
_ = part.Close()
return "", currentAttempt, err
}
offset = 0
} else {
task, err = queue.applyResponseFacts(runtime, currentAttempt, response)
if err != nil {
return "", currentAttempt, err
}
}
var maximum int64
var expectedKnown bool
var expected int64
if task.TotalKnown {
if offset > task.Total {
return "", currentAttempt, fmt.Errorf(
"%w: offset exceeds total",
ErrTaskCorrupt,
)
}
maximum = task.Total - offset
expectedKnown = true
expected = maximum
} else {
if offset >= queue.maxUnknown {
return "", currentAttempt, ErrTransferTooLarge
}
maximum = queue.maxUnknown - offset
if response.ResponseLengthKnown {
expectedKnown = true
expected = response.ResponseLength
}
}
if !task.TotalKnown && response.TotalKnown {
if response.Total > queue.maxUnknown ||
offset > response.Total ||
!response.ResponseLengthKnown ||
response.ResponseLength != response.Total-offset {
if part != nil {
_ = part.Close()
}
return "", currentAttempt, ErrTransferIncomplete
}
}
if response.ResponseLengthKnown && response.ResponseLength > maximum {
if part != nil {
_ = part.Close()
}
return "", currentAttempt, ErrTransferTooLarge
}
if expectedKnown &&
response.ResponseLengthKnown &&
response.ResponseLength != expected {
if part != nil {
_ = part.Close()
}
return "", currentAttempt, ErrTransferIncomplete
}
if part == nil {
part, err = openPart(paths, offset, false)
if err != nil {
return "", currentAttempt, err
}
}
lastEventAt := queue.clock.Now()
lastEventDone := offset
lastPublishedDone := offset
copyOptions := CopyOptions{
MaxBytes: maximum,
ExpectedBytesKnown: expectedKnown,
ExpectedBytes: expected,
Generation: currentAttempt,
Progress: func(progress CopyProgress) error {
absoluteDone := offset + progress.Written
now := queue.clock.Now()
emit := queue.progressEvery == 0 ||
now.Sub(lastEventAt) >= queue.progressEvery
if err := queue.updateAttemptDone(
runtime,
currentAttempt,
absoluteDone,
); err != nil {
return err
}
if !emit {
return nil
}
speed := progressSpeed(lastEventAt, now, lastEventDone, absoluteDone)
if err := queue.persistAndPublishProgress(
runtime,
currentAttempt,
speed,
); err != nil {
return err
}
lastEventAt = now
lastEventDone = absoluteDone
lastPublishedDone = absoluteDone
return nil
},
}
written, copyErr := CopyResponse(ctx, part, body, copyOptions)
writtenInfo, statErr := part.Stat()
bodyCloseErr := body.Close()
body = nil
syncErr := syncAndClose(part)
if syncErr != nil {
return "", currentAttempt, syncErr
}
if bodyCloseErr != nil {
return "", currentAttempt, bodyCloseErr
}
if statErr != nil {
return "", currentAttempt, fmt.Errorf("stat written part: %w", statErr)
}
if err := verifyRegularIdentity(paths.Part, writtenInfo); err != nil {
return "", currentAttempt, err
}
if copyErr != nil {
return "", currentAttempt, copyErr
}
finalDone := offset + written
if err := queue.updateAttemptDone(runtime, currentAttempt, finalDone); err != nil {
return "", currentAttempt, err
}
if finalDone != lastPublishedDone {
now := queue.clock.Now()
speed := progressSpeed(lastEventAt, now, lastEventDone, finalDone)
if err := queue.persistAndPublishProgress(
runtime,
currentAttempt,
speed,
); err != nil {
return "", currentAttempt, err
}
}
if task.TotalKnown && finalDone != task.Total {
return "", currentAttempt, ErrTransferIncomplete
}
if err := activateCompleted(paths, writtenInfo); err != nil {
return "", currentAttempt, err
}
return paths.Completed, currentAttempt, nil
}
func (queue *Queue) restartAttempt(
runtime *runtimeTask,
attempt uint64,
response OpenResponse,
) (uint64, Task, error) {
queue.mu.Lock()
if !queue.attemptCurrent(runtime, attempt) || runtime.stop != stopNone {
queue.mu.Unlock()
return attempt, Task{}, context.Canceled
}
runtime.task.Attempt++
runtime.task.Done = 0
runtime.task.Validator = response.Validator
task := runtime.task
newAttempt := task.Attempt
queue.mu.Unlock()
if err := queue.store.Write(task); err != nil {
return newAttempt, task, err
}
queue.publishBestEffort(Event{
Type: EventStarted,
RequestID: task.RequestID,
AppID: task.AppID,
Attempt: task.Attempt,
Done: 0,
TotalKnown: task.TotalKnown,
Total: task.Total,
})
return newAttempt, task, nil
}
func (queue *Queue) applyResponseFacts(
runtime *runtimeTask,
attempt uint64,
response OpenResponse,
) (Task, error) {
queue.mu.Lock()
if !queue.attemptCurrent(runtime, attempt) || runtime.stop != stopNone {
queue.mu.Unlock()
return Task{}, context.Canceled
}
if runtime.task.Validator.Empty() {
runtime.task.Validator = response.Validator
}
task := runtime.task
queue.mu.Unlock()
if err := queue.store.Write(task); err != nil {
return Task{}, err
}
return task, nil
}
func (queue *Queue) updateAttemptDone(
runtime *runtimeTask,
attempt uint64,
done int64,
) error {
queue.mu.Lock()
defer queue.mu.Unlock()
if !queue.attemptCurrent(runtime, attempt) ||
runtime.stop != stopNone ||
done < runtime.task.Done {
return context.Canceled
}
if runtime.task.TotalKnown && done > runtime.task.Total {
return ErrTransferTooLarge
}
runtime.task.Done = done
return nil
}
func (queue *Queue) persistAndPublishProgress(
runtime *runtimeTask,
attempt uint64,
speed int64,
) error {
queue.mu.Lock()
if !queue.attemptCurrent(runtime, attempt) || runtime.stop != stopNone {
queue.mu.Unlock()
return context.Canceled
}
task := runtime.task
queue.mu.Unlock()
if err := queue.store.Write(task); err != nil {
return err
}
queue.publishBestEffort(Event{
Type: EventProgress,
RequestID: task.RequestID,
AppID: task.AppID,
Attempt: task.Attempt,
Done: task.Done,
TotalKnown: task.TotalKnown,
Total: task.Total,
SpeedBytesSec: speed,
})
return nil
}
func progressSpeed(start, end time.Time, startDone, endDone int64) int64 {
elapsed := end.Sub(start)
if elapsed <= 0 || endDone <= startDone {
return 0
}
bytesPerSecond := float64(endDone-startDone) / elapsed.Seconds()
if bytesPerSecond <= 0 {
return 0
}
return int64(bytesPerSecond)
}
+354
View File
@@ -0,0 +1,354 @@
package storage
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sort"
"strings"
"sync"
"softbox.local/core/downloader"
)
const (
maxDownloadTaskDocumentBytes = 64 * 1024
maxDownloadTaskCount = 4096
)
var ErrDownloadTaskLayoutUnsafe = errors.New("download task storage layout is unsafe")
// DownloadTaskStore atomically persists download metadata below
// downloads/tasks. Transfer bytes live separately below downloads/files.
type DownloadTaskStore struct {
root string
mu sync.Mutex
}
// NewDownloadTaskStore creates a strict JSON task store.
func NewDownloadTaskStore(downloadsRoot string) *DownloadTaskStore {
return &DownloadTaskStore{root: downloadsRoot}
}
// Write validates and atomically replaces one task document.
func (store *DownloadTaskStore) Write(task downloader.Task) error {
store.mu.Lock()
defer store.mu.Unlock()
if err := task.Validate(); err != nil {
return err
}
paths, err := store.ensureLayout(task.RequestID)
if err != nil {
return err
}
document, err := json.Marshal(task)
if err != nil {
return fmt.Errorf("encode download task: %w", err)
}
document = append(document, '\n')
if len(document) > maxDownloadTaskDocumentBytes {
return fmt.Errorf("%w: metadata is too large", downloader.ErrInvalidTask)
}
return replaceDownloadTaskFile(paths.TasksDir, paths.Metadata, paths.Backup, document)
}
// LoadAll reads every task, falling back to a backup only when its main file
// is absent.
func (store *DownloadTaskStore) LoadAll() ([]downloader.Task, error) {
store.mu.Lock()
defer store.mu.Unlock()
paths, err := store.ensureLayout("layout-probe")
if err != nil {
return nil, err
}
entries, err := os.ReadDir(paths.TasksDir)
if err != nil {
return nil, fmt.Errorf("read download task directory: %w", err)
}
requestIDs := make(map[string]struct{})
for _, entry := range entries {
name := entry.Name()
if strings.HasPrefix(name, ".download-task-") && strings.HasSuffix(name, ".tmp") {
continue
}
requestID, recognized := metadataRequestID(name)
if !recognized || !downloader.ValidRequestID(requestID) {
return nil, fmt.Errorf(
"%w: unexpected metadata entry %q",
ErrDownloadTaskLayoutUnsafe,
name,
)
}
info, infoErr := entry.Info()
if infoErr != nil {
return nil, fmt.Errorf("inspect download task entry: %w", infoErr)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return nil, fmt.Errorf(
"%w: metadata entry is not regular",
ErrDownloadTaskLayoutUnsafe,
)
}
requestIDs[requestID] = struct{}{}
}
if len(requestIDs) > maxDownloadTaskCount {
return nil, fmt.Errorf("%w: too many tasks", downloader.ErrInvalidTask)
}
sortedIDs := make([]string, 0, len(requestIDs))
for requestID := range requestIDs {
sortedIDs = append(sortedIDs, requestID)
}
sort.Strings(sortedIDs)
tasks := make([]downloader.Task, 0, len(sortedIDs))
for _, requestID := range sortedIDs {
taskPaths, deriveErr := downloader.DeriveTaskPaths(store.root, requestID)
if deriveErr != nil {
return nil, deriveErr
}
document, readErr := readBoundedRegularFile(
taskPaths.Metadata,
maxDownloadTaskDocumentBytes,
)
if os.IsNotExist(readErr) {
document, readErr = readBoundedRegularFile(
taskPaths.Backup,
maxDownloadTaskDocumentBytes,
)
}
if readErr != nil {
return nil, readErr
}
task, decodeErr := decodeDownloadTask(document)
if decodeErr != nil {
return nil, decodeErr
}
if task.RequestID != requestID {
return nil, fmt.Errorf(
"%w: request_id does not match metadata filename",
downloader.ErrInvalidTask,
)
}
tasks = append(tasks, task)
}
return tasks, nil
}
// Delete removes the exact metadata and backup files for a request.
func (store *DownloadTaskStore) Delete(requestID string) error {
store.mu.Lock()
defer store.mu.Unlock()
paths, err := downloader.DeriveTaskPaths(store.root, requestID)
if err != nil {
return err
}
for _, candidate := range []string{paths.Metadata, paths.Backup} {
info, inspectErr := os.Lstat(candidate)
if os.IsNotExist(inspectErr) {
continue
}
if inspectErr != nil {
return fmt.Errorf("inspect download task metadata: %w", inspectErr)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf(
"%w: metadata is not a regular file",
ErrDownloadTaskLayoutUnsafe,
)
}
if removeErr := os.Remove(candidate); removeErr != nil {
return fmt.Errorf("remove download task metadata: %w", removeErr)
}
}
return nil
}
func (store *DownloadTaskStore) ensureLayout(requestID string) (downloader.TaskPaths, error) {
paths, err := downloader.DeriveTaskPaths(store.root, requestID)
if err != nil {
return downloader.TaskPaths{}, err
}
if err := os.MkdirAll(paths.Root, 0o700); err != nil {
return downloader.TaskPaths{}, fmt.Errorf("create downloads root: %w", err)
}
for _, directory := range []string{paths.Root, paths.TasksDir, paths.FilesDir} {
if err := os.Mkdir(directory, 0o700); err != nil && !os.IsExist(err) {
return downloader.TaskPaths{}, fmt.Errorf("create download directory: %w", err)
}
info, inspectErr := os.Lstat(directory)
if inspectErr != nil {
return downloader.TaskPaths{}, fmt.Errorf("inspect download directory: %w", inspectErr)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return downloader.TaskPaths{}, fmt.Errorf(
"%w: managed path is not a real directory",
ErrDownloadTaskLayoutUnsafe,
)
}
}
return paths, nil
}
func metadataRequestID(name string) (string, bool) {
switch {
case strings.HasSuffix(name, ".json.backup"):
return strings.TrimSuffix(name, ".json.backup"), true
case strings.HasSuffix(name, ".json"):
return strings.TrimSuffix(name, ".json"), true
default:
return "", false
}
}
func decodeDownloadTask(document []byte) (downloader.Task, error) {
var task downloader.Task
decoder := json.NewDecoder(bytes.NewReader(document))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&task); err != nil {
return downloader.Task{}, fmt.Errorf("%w: decode: %v", downloader.ErrInvalidTask, err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
if err == nil {
return downloader.Task{}, fmt.Errorf(
"%w: trailing JSON value",
downloader.ErrInvalidTask,
)
}
return downloader.Task{}, fmt.Errorf(
"%w: trailing data: %v",
downloader.ErrInvalidTask,
err,
)
}
if err := task.Validate(); err != nil {
return downloader.Task{}, err
}
return task, nil
}
func readBoundedRegularFile(filePath string, maximum int64) ([]byte, error) {
info, err := os.Lstat(filePath)
if err != nil {
return nil, err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return nil, fmt.Errorf(
"%w: metadata is not a regular file",
ErrDownloadTaskLayoutUnsafe,
)
}
if info.Size() > maximum {
return nil, fmt.Errorf("%w: metadata is too large", downloader.ErrInvalidTask)
}
file, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("open download task metadata: %w", err)
}
defer file.Close()
document, err := io.ReadAll(io.LimitReader(file, maximum+1))
if err != nil {
return nil, fmt.Errorf("read download task metadata: %w", err)
}
if int64(len(document)) > maximum {
return nil, fmt.Errorf("%w: metadata is too large", downloader.ErrInvalidTask)
}
return document, nil
}
func replaceDownloadTaskFile(directory, target, backup string, document []byte) error {
temporary, err := os.CreateTemp(directory, ".download-task-*.tmp")
if err != nil {
return fmt.Errorf("create download task temp file: %w", err)
}
temporaryPath := temporary.Name()
defer os.Remove(temporaryPath)
if err := temporary.Chmod(0o600); err != nil {
temporary.Close()
return fmt.Errorf("protect download task temp file: %w", err)
}
if _, err := temporary.Write(document); err != nil {
temporary.Close()
return fmt.Errorf("write download task temp file: %w", err)
}
if err := temporary.Sync(); err != nil {
temporary.Close()
return fmt.Errorf("sync download task temp file: %w", err)
}
if err := temporary.Close(); err != nil {
return fmt.Errorf("close download task temp file: %w", err)
}
movedTarget := false
hadBackup := false
if info, inspectErr := os.Lstat(target); inspectErr == nil {
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf(
"%w: metadata target is not regular",
ErrDownloadTaskLayoutUnsafe,
)
}
if err := removeExactRegularFile(backup); err != nil {
return err
}
if err := os.Rename(target, backup); err != nil {
return fmt.Errorf("backup download task metadata: %w", err)
}
movedTarget = true
} else if !os.IsNotExist(inspectErr) {
return fmt.Errorf("inspect download task metadata: %w", inspectErr)
} else {
backupInfo, backupErr := os.Lstat(backup)
switch {
case backupErr == nil:
if backupInfo.Mode()&os.ModeSymlink != 0 || !backupInfo.Mode().IsRegular() {
return fmt.Errorf(
"%w: metadata backup is not regular",
ErrDownloadTaskLayoutUnsafe,
)
}
hadBackup = true
case !os.IsNotExist(backupErr):
return fmt.Errorf("inspect download task backup: %w", backupErr)
}
}
if err := os.Rename(temporaryPath, target); err != nil {
if movedTarget {
_ = os.Rename(backup, target)
}
return fmt.Errorf("activate download task metadata: %w", err)
}
if movedTarget || hadBackup {
if err := removeExactRegularFile(backup); err != nil {
return err
}
}
return nil
}
func removeExactRegularFile(filePath string) error {
info, err := os.Lstat(filePath)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return fmt.Errorf("inspect download task backup: %w", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf(
"%w: metadata backup is not regular",
ErrDownloadTaskLayoutUnsafe,
)
}
if err := os.Remove(filePath); err != nil {
return fmt.Errorf("remove download task backup: %w", err)
}
return nil
}
+30
View File
@@ -0,0 +1,30 @@
package storage
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
func TestDownloadTaskSchemaIsValidJSONObject(t *testing.T) {
document, err := os.ReadFile(filepath.Join(
"..",
"..",
"schemas",
"download-task.schema.json",
))
if err != nil {
t.Fatalf("read schema: %v", err)
}
var schema map[string]any
if err := json.Unmarshal(document, &schema); err != nil {
t.Fatalf("decode schema: %v", err)
}
if schema["$schema"] != "https://json-schema.org/draft/2020-12/schema" {
t.Fatalf("$schema = %v", schema["$schema"])
}
if schema["type"] != "object" {
t.Fatalf("type = %v", schema["type"])
}
}
+146
View File
@@ -0,0 +1,146 @@
package storage
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"softbox.local/core/downloader"
)
func TestDownloadTaskStoreWriteLoadAndBackupFallback(t *testing.T) {
root := filepath.Join(t.TempDir(), "nested", "downloads")
store := NewDownloadTaskStore(root)
task := validDownloadTask()
if err := store.Write(task); err != nil {
t.Fatalf("Write() error = %v", err)
}
loaded, err := store.LoadAll()
if err != nil {
t.Fatalf("LoadAll() error = %v", err)
}
if len(loaded) != 1 || loaded[0].RequestID != task.RequestID {
t.Fatalf("loaded = %#v", loaded)
}
task.Status = downloader.StatusPaused
task.Done = 10
task.Validator = downloader.EntityValidator{ETag: `"v1"`}
if err := store.Write(task); err != nil {
t.Fatalf("Write(update) error = %v", err)
}
paths, err := downloader.DeriveTaskPaths(root, task.RequestID)
if err != nil {
t.Fatalf("DeriveTaskPaths() error = %v", err)
}
if err := os.Rename(paths.Metadata, paths.Backup); err != nil {
t.Fatalf("simulate interrupted replacement: %v", err)
}
loaded, err = store.LoadAll()
if err != nil {
t.Fatalf("LoadAll(backup) error = %v", err)
}
if len(loaded) != 1 || loaded[0].Status != downloader.StatusPaused ||
loaded[0].Done != 10 {
t.Fatalf("backup loaded = %#v", loaded)
}
}
func TestDownloadTaskStoreRejectsUnknownAndInvalidData(t *testing.T) {
root := t.TempDir()
store := NewDownloadTaskStore(root)
task := validDownloadTask()
if err := store.Write(task); err != nil {
t.Fatalf("Write() error = %v", err)
}
paths, _ := downloader.DeriveTaskPaths(root, task.RequestID)
encoded, err := json.Marshal(task)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
var object map[string]any
if err := json.Unmarshal(encoded, &object); err != nil {
t.Fatalf("Unmarshal() error = %v", err)
}
object["unknown"] = true
encoded, err = json.Marshal(object)
if err != nil {
t.Fatalf("Marshal(object) error = %v", err)
}
if err := os.WriteFile(paths.Metadata, encoded, 0o600); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := store.LoadAll(); !errors.Is(err, downloader.ErrInvalidTask) {
t.Fatalf("LoadAll(unknown) error = %v, want %v", err, downloader.ErrInvalidTask)
}
task = validDownloadTask()
task.Done = task.Total + 1
if err := store.Write(task); !errors.Is(err, downloader.ErrInvalidTask) {
t.Fatalf("Write(invalid) error = %v, want %v", err, downloader.ErrInvalidTask)
}
}
func TestDownloadTaskStoreRejectsUnexpectedAndUnsafeEntries(t *testing.T) {
root := t.TempDir()
store := NewDownloadTaskStore(root)
task := validDownloadTask()
if err := store.Write(task); err != nil {
t.Fatalf("Write() error = %v", err)
}
paths, _ := downloader.DeriveTaskPaths(root, task.RequestID)
if err := os.WriteFile(filepath.Join(paths.TasksDir, "unexpected.txt"), []byte("x"), 0o600); err != nil {
t.Fatalf("WriteFile(unexpected) error = %v", err)
}
if _, err := store.LoadAll(); !errors.Is(err, ErrDownloadTaskLayoutUnsafe) {
t.Fatalf("LoadAll(unexpected) error = %v, want %v", err, ErrDownloadTaskLayoutUnsafe)
}
if err := os.Remove(filepath.Join(paths.TasksDir, "unexpected.txt")); err != nil {
t.Fatalf("Remove(unexpected) error = %v", err)
}
if err := os.Remove(paths.Metadata); err != nil {
t.Fatalf("Remove(metadata) error = %v", err)
}
if err := os.Symlink(filepath.Join(root, "outside.json"), paths.Metadata); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
if _, err := store.LoadAll(); !errors.Is(err, ErrDownloadTaskLayoutUnsafe) {
t.Fatalf("LoadAll(symlink) error = %v, want %v", err, ErrDownloadTaskLayoutUnsafe)
}
}
func TestDownloadTaskStoreDeleteIsExact(t *testing.T) {
root := t.TempDir()
store := NewDownloadTaskStore(root)
task := validDownloadTask()
if err := store.Write(task); err != nil {
t.Fatalf("Write() error = %v", err)
}
paths, _ := downloader.DeriveTaskPaths(root, task.RequestID)
if err := store.Delete(task.RequestID); err != nil {
t.Fatalf("Delete() error = %v", err)
}
if _, err := os.Stat(paths.Metadata); !os.IsNotExist(err) {
t.Fatalf("metadata still exists: %v", err)
}
if info, err := os.Stat(paths.FilesDir); err != nil || !info.IsDir() {
t.Fatalf("files directory was removed: %v", err)
}
}
func validDownloadTask() downloader.Task {
return downloader.Task{
SchemaVersion: downloader.TaskSchemaVersion,
RequestID: "request-001",
AppID: "json-parser",
URL: "https://download.invalid/json-parser.zip",
Status: downloader.StatusQueued,
TotalKnown: true,
Total: 42,
CreatedAt: "2026-07-16T00:00:00Z",
}
}
+11
View File
@@ -142,6 +142,17 @@ T-202 将本地识别拆为两层:
磁盘扫描结果必须在进入 Gio Layout 前准备好;UI 不直接读取 installed-app.json。完整字段见 [api.md](api.md),Schema 为 `schemas/installed-app.schema.json`。
T-301 下载队列:
- `core/downloader.Queue` 默认并发 2,用 request_id 主键和 app 非终态唯一约束串行化 enqueue/pause/resume/cancel/retry;网络/磁盘 IO 与事件发布都在全局锁外。
- `.part` / `.download` / metadata 路径只由受限 request_id 派生。元数据通过 `core/storage.DownloadTaskStore` 临时文件 + backup 原子替换。
- Range 必须绑定 strong ETag 或 Last-Modified 并发送 If-Range;无 validator 从 0 重下,`200` 忽略 Range 时创建新 attempt。进度只在 request_id+attempt 内单调。
- pause/cancel/close 先取消 generation context,等待 body 关闭、part sync/close 和旧 worker 完全退出,再发布终态/释放并发槽,阻断迟到 progress。
- cancel 一旦先于 completion 取得线性化点,即使 body close/sync 报错也记录后继续精确清理;若 completion 先完成,后续 cancel 明确拒绝。known-total 完整 part 在同进程 resume/retry 与启动恢复中都直接 finalize,不发送 offset==total 的 Range。
- 写入句柄的文件身份贯穿 sync/close 与 rename 前后核对,防止活跃 `.part` 路径被替换后发布错误文件。崩溃恢复对账 metadata、part、final 三份事实:完整 part 可 finalize,已 rename 的 final 可补 completed metadata;缺 final 的 completed、part+final、超出 expected/unknown 上限均 fail closed。
- 事件投递失败通过 `OnObserverError` 显式报告,不改变 durable transfer 结果;application/UI 启动或重连后用 `Queue.Tasks()` 对账终态,避免 DownloadCompleted 等一次性通知丢失后永久停链。
- DownloadCompleted 仅证明传输字节完整落盘。下载元数据和 `.download` 可被本地篡改,T-302 不得把它们当信任根,仍需从已验签 Catalog 重新取得并核对身份/size/hash/signature。
### 4.3 事件模型
后台任务只发布事件(`DownloadStarted / DownloadProgress / DownloadPaused / DownloadCompleted / DownloadFailed` 等),UI 按 RequestID 和软件 ID 回填,见 [api.md](api.md) 事件合约。
+42 -5
View File
@@ -200,6 +200,42 @@ T-102 Phase 1 原型进一步固定:
phase 只允许:`prepared`、`current_backed_up`、`staging_activated`、`rollback_required`、`committed`。日志使用临时文件 + 同目录 backup 原子替换;恢复时同时检查日志与 current/staging/backup 实际状态。未完成健康检查的 current 不视为可信:有旧版时恢复 backup,首次安装则撤销 current。
### 2.6 下载任务元数据 download-task.json(本地)
每个任务以稳定 `request_id` 为主键,元数据位于 `downloads/tasks/<request_id>.json`,字节文件位于 `downloads/files/<request_id>.part|.download`。本地路径只由客户端从 request_id 派生,不接受 URL 或 Content-Disposition 提供的文件名。
```json
{
"schema_version": 1,
"request_id": "download-json-parser-001",
"app_id": "json-parser",
"url": "https://download.example.com/json-parser.zip",
"status": "paused",
"attempt": 2,
"done": 1048576,
"total_known": true,
"total": 12345678,
"validator": {
"etag": "\"release-1.4.2\""
},
"error_code": "",
"created_at": "2026-07-16T00:00:00Z"
}
```
规则:
- Schema 为 `schemas/download-task.schema.json`;内部状态只允许 queued/downloading/paused/failed/completed。paused 对 UI 映射为 queued + DownloadPaused 事件,不增加第 13 个 AppStatus。
- 默认最多 2 个 downloading;同一 request_id 幂等,身份/URL/size 不同则冲突;同一 app 同时只允许一个非 completed 任务。
- 暂停保留 `.part` 和元数据;失败 retry 保留 `.part`;取消固定删除该 request_id 精确派生的 part/final/metadata,不使用递归删除。取消已线性化后,body close/sync 等非致命错误只记录日志,不能把取消反转成 failed 或跳过清理。
- 续传只在存在 strong ETag 或合法 Last-Modified 时发送 Range + If-Range;无可靠 validator 时从 0 重下。请求固定 `Accept-Encoding: identity`,所有 redirect 重新校验 HTTPS。
- `206` 必须严格匹配 Content-Range 起点、长度、total 与期望 size;已有 part 但服务端返回 `200` 时截断并创建新 attempt,不能 old+new 拼接。
- known total 使用已验签 Catalog package.size 作为精确传输长度;unknown total 使用显式 `total_known: false` 并受 4 GiB 默认硬上限保护。
- known total 的 `.part` 已达到精确 total 时,同进程 resume/retry 与重启恢复都直接 sync、核对身份并 finalize,不得再请求 `Range: bytes=<total>-`。
- 完成顺序为 part sync/close → 核对实际写入文件身份 → rename `.download` → metadata completed → DownloadCompleted。rename 前后必须仍是同一普通文件;恢复时以普通文件实际长度对账,不信任 metadata.done。
- 事件投递是 best-effort,失败不得反向把已完成/已取消的持久任务改成失败。队列必须把投递错误交给 `OnObserverError` 记录;application/UI 在启动或事件通道重连后必须用 `Queue.Tasks()` 对账持久状态,不能只依赖某一次终态事件。
- `.download` 仍是**不可信隔离字节**。T-302 必须重新从已验签 Catalog 取得 app/version/arch/size/SHA-256/signature 并验证,通过后才可读取 app.json 或解压。
## 3. 许可证(服务端签发 → 本地离线验证)
```json
@@ -229,11 +265,12 @@ phase 只允许:`prepared`、`current_backed_up`、`staging_activated`、`rollba
| --- | --- | --- | --- |
| CatalogRefreshed | 清单验签并缓存成功 | catalog 摘要、generated_at | 列表刷新 |
| CatalogRejected | 清单验签失败 | 原因码 | 提示 + 继续用缓存 |
| DownloadStarted | 下载任务开始 | request_id, app_id | 状态 → downloading |
| DownloadProgress | 进度更新 | request_id, app_id, done, total, speed | 进度条刷新 |
| DownloadPaused | 用户暂停 | request_id, app_id | 状态 → queued(暂停态) |
| DownloadCompleted | 下载并校验通过 | request_id, app_id | 状态 → verifying/extracting |
| DownloadFailed | 失败(网络/哈希/磁盘) | request_id, app_id, error_code | 状态 → failed + 可重试 |
| DownloadStarted | 一个下载 attempt 开始/因安全重下而重置 | request_id, app_id, attempt, done, total_known, total | 状态 → downloading |
| DownloadProgress | 当前 attempt 进度更新 | request_id, app_id, attempt, done, total_known, total, speed | 进度条刷新;done 在同 attempt 内单调 |
| DownloadPaused | 用户暂停且旧 worker 已完全退出 | request_id, app_id, attempt, done | 状态 → queued(暂停态) |
| DownloadCompleted | 字节完整落盘,尚未通过 SHA/签名 | request_id, app_id, attempt, done, path | 状态 → verifying;T-302 接管信任校验 |
| DownloadFailed | 传输/Range/存储失败 | request_id, app_id, attempt, done, error_code | 状态 → failed + 可重试 |
| DownloadCanceled | 取消清理完成且旧 attempt 不再发事件 | request_id, app_id, attempt | 从任务视图移除并回到本地基础状态 |
| InstallCompleted | 原子切换成功 + 健康检查通过 | app_id, version | 状态 → installed |
| InstallRolledBack | 切换失败恢复 backup | app_id, error_code | 状态 → rollback 完成提示 |
| AppStarted / AppExited | 进程启动/退出检测 | app_id, pid | 状态 → running / installed |
+12 -12
View File
@@ -13,36 +13,36 @@
## 当前快照
- 日期:2026-07-16
- 阶段:Phase 2 已完成(T-201~T-204);T-301 下载队列已落成并领取,准备按单写入者 + 多只读审查者模式执行
- 阶段:Phase 2 已完成(T-201~T-204);Phase 3 的 T-301 可恢复下载队列已完成,T-302 尚待正式落成任务文件
- 技术栈:根 Go workspace 纳入 core/app-modern/app-win7 三模块;`app-win7/go.work` 隔离 Go 1.20.14 构建;modern Gio v0.10.1 与 win7 Gio v0.6.0 已实际接入
- 生产代码:core 已有 Catalog/本地状态/存储、无 IO 软件列表模型与按 digest+DPI 的可信图标缓存;modern/win7 AppShell 已实现搜索/分类/视图、惰性列表、详情右栏与内存图标
- 测试:core 覆盖 Catalog、SemVer/12 状态、本地安装记录、列表筛选和图标内存/磁盘/离线/损坏恢复;两个 app 覆盖 500 项虚拟列表、ID 控件稳定性、详情/ApplyIcon 与平台 stub;ZIP/安装恢复矩阵保持通过
- 数据:`schemas/` 已有 manifest/app.json/installed-app.json v1 Schema;`testdata/catalog/` 有公开虚构清单样例;`testdata/zip/` 记录运行时生成的 ZIP 攻击矩阵
- 生产代码:core 已有 Catalog/本地状态/存储、无 IO 软件列表模型、可信图标缓存和默认并发 2 的持久可恢复下载队列;modern/win7 AppShell 已实现搜索/分类/视图、惰性列表、详情右栏与内存图标
- 测试:core 覆盖 Catalog、SemVer/12 状态、本地安装记录、列表/图标、下载并发/暂停/取消/重试/Range/断连/恢复/事件失败与文件身份替换;两个 app 覆盖 500 项虚拟列表、ID 控件稳定性、详情/ApplyIcon 与平台 stub;ZIP/安装恢复矩阵保持通过
- 数据:`schemas/` 已有 manifest/app.json/installed-app.json/download-task.json v1 Schema;`testdata/catalog/` 有公开虚构清单样例;`testdata/zip/` 与 `testdata/download/` 记录运行时生成的攻击/传输矩阵
- 标准启动路径:`./init.sh` / `./init.ps1`(同步依赖、执行完整 Phase 0 闸门、打印双目标构建命令)
- 标准验证路径:`bash scripts/verify_phase0.sh` / `./scripts/verify_phase0.ps1`
- 版本管理:git 已初始化,main 分支,远端 origin 为 Gitea `opc/soft_quay`;harness 文档已提交
- 当前 blocker:无;当前任务 T-301,后置 T-302/T-303 仍受依赖门槛约束
- 当前 blocker:无;下一个任务是 T-302 安装流程整合,必须先正式落成并提交任务文件,再启动一个写入 Agent + 测试/安全只读 Agents
## 当前目录要点
| 路径 | 状态 | 说明 |
| --- | --- | --- |
| `docs/` | 已有 | harness coding 文档集(本次初始化完成) |
| `docs/tasks/` | 已有 | Phase 0~2 已完成;T-301 已落成并处于 DOING |
| `docs/tasks/` | 已有 | Phase 0~2 与 T-301 已完成;T-302 尚未落成 |
| `scripts/` | 已有 | harness 治理、core 边界、Go 版本检查与 Phase 0 双平台验证入口 |
| `core/` | 已建 | Go 1.20 兼容;已有正式 Catalog、本地状态/存储、列表模型、图标缓存与 Phase 1 安装安全原型 |
| `core/` | 已建 | Go 1.20 兼容;已有正式 Catalog、本地状态/存储、列表模型、图标缓存、可恢复下载队列与 Phase 1 安装安全原型 |
| `app-modern/` | 已建 | Go 1.25.0 + Gio v0.10.1;Modern AppShell 已接入虚拟列表、详情和内存图标 |
| `app-win7/` | 已建 | Go 1.20 + Gio v0.6.0;Legacy AppShell 已接入低成本列表、详情和内存图标 |
| `schemas/` | 已建 | `manifest.schema.json`、`app.schema.json` 与 `installed-app.schema.json` |
| `testdata/` | 已建 | 当前包含 Catalog 假数据与恶意样例;后续任务继续扩展 |
| `schemas/` | 已建 | `manifest.schema.json`、`app.schema.json`、`installed-app.schema.json` 与 `download-task.schema.json` |
| `testdata/` | 已建 | 包含 Catalog 假数据、ZIP 恶意矩阵与下载协议测试说明;后续任务继续扩展 |
## 任务状态
任务状态以 `docs/tasks/` 各任务文件 frontmatter 的 `status` 为准。本节只写项目级摘要:
- 已完成:Phase 0 的 `T-001`~`T-004`;Phase 1 的 `T-101`、`T-102`、`T-103`;Phase 2 的 `T-201`~`T-204`。
- 正在进行:`T-301 下载队列`(一个写入 Agent + 测试/安全只读 Agents)。
- 下一个可领取任务:无;T-302 必须等待 T-301 DONE。
- 已完成:Phase 0 的 `T-001`~`T-004`;Phase 1 的 `T-101`、`T-102`、`T-103`;Phase 2 的 `T-201`~`T-204`;Phase 3 的 `T-301`。
- 正在进行:无。
- 下一个可领取任务:先按路线图正式落成 `T-302 安装流程整合` 并提交,然后才能领取和启动多 Agent。
## 当前可运行内容
+5 -1
View File
@@ -3,7 +3,7 @@ id: T-301
title: 可恢复下载队列
phase: 3
deps: [T-202]
status: DOING
status: DONE
created: 2026-07-16
issue: null
context_ref: a52acbf926041550ebc30049ff87f08faaa5ba0c
@@ -75,3 +75,7 @@ T-301 是 T-302/T-303 的前置安全边界。它只负责把远端 ZIP 稳定
## 执行记录
- 2026-07-16:正式落成多 Agent 协作方式;采用一个写入 Agent + 测试设计/安全架构两个只读 Agent,保持 T-301 → T-302 → T-303 依赖串行。
- 2026-07-16:实现 `core/downloader.Queue`、严格 HTTPS/Range transport、任务状态/路径协议、pause/resume/cancel/retry、默认并发 2、进度节流与 application 具体事件负载;新增 `core/storage.DownloadTaskStore`、`download-task.schema.json` 和测试数据说明。
- 2026-07-16:根据多轮只读审查修复安全边界:服务器忽略 Range 时先截断并 sync `.part` 再持久化新 attempt/validator;unknown total 全程受硬上限约束且续传必须证明完整 total;写入句柄身份贯穿 rename 前后;pause 不吞 body close/sync 错误,cancel 则记录非致命关闭错误后仍完成固定清理;Close 不覆盖用户 cancel;known-total 完整 part 在同进程 resume/retry 直接 finalize;observer 失败不反向改变 durable 结果并通过 `OnObserverError` 报告。
- 2026-07-16:补齐并发补位、断连续传、restart 写失败窗口、未知 total、HTTP 404/416/500 与畸形响应、进度单调/节流/速度、迟到 generation、事件字段、observer 失败对账、崩溃恢复和文件身份替换回归测试;最终只读测试/安全审查均无 blocker/high。
- 2026-07-16:验证通过:`go -C core vet ./...`;`go -C core test -count=1 ./...`;`go -C core test -count=10 ./downloader`;`./scripts/verify_phase0.ps1`。`go test -race` 已尝试,当前环境 `CGO_ENABLED=0` 且未安装 gcc,因此 race detector 无法启动;非 race 闸门全部通过。
+144
View File
@@ -0,0 +1,144 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://softbox.invalid/schemas/download-task.schema.json",
"title": "SoftBox download-task.json v1",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"request_id",
"app_id",
"url",
"status",
"attempt",
"done",
"total_known",
"total",
"validator",
"error_code",
"created_at"
],
"properties": {
"schema_version": {
"const": 1
},
"request_id": {
"type": "string",
"maxLength": 128,
"pattern": "^[a-z0-9][a-z0-9-]{0,127}$"
},
"app_id": {
"type": "string",
"maxLength": 128,
"pattern": "^[a-z0-9][a-z0-9-]{0,127}$"
},
"url": {
"type": "string",
"maxLength": 4096,
"format": "uri",
"pattern": "^https://"
},
"status": {
"enum": ["queued", "downloading", "paused", "failed", "completed"]
},
"attempt": {
"type": "integer",
"minimum": 0
},
"done": {
"type": "integer",
"minimum": 0
},
"total_known": {
"type": "boolean"
},
"total": {
"type": "integer",
"minimum": 0
},
"validator": {
"oneOf": [
{
"type": "object",
"additionalProperties": false,
"maxProperties": 0
},
{
"type": "object",
"additionalProperties": false,
"required": ["etag"],
"properties": {
"etag": {
"type": "string",
"maxLength": 512,
"pattern": "^\".*\"$"
}
}
},
{
"type": "object",
"additionalProperties": false,
"required": ["last_modified"],
"properties": {
"last_modified": {
"type": "string",
"maxLength": 512
}
}
}
]
},
"error_code": {
"type": "string",
"maxLength": 64,
"pattern": "^$|^[a-z0-9][a-z0-9_]{0,63}$"
},
"created_at": {
"type": "string",
"format": "date-time"
}
},
"allOf": [
{
"if": {
"properties": {
"total_known": {
"const": false
}
},
"required": ["total_known"]
},
"then": {
"properties": {
"total": {
"const": 0
}
}
}
},
{
"if": {
"properties": {
"status": {
"const": "failed"
}
},
"required": ["status"]
},
"then": {
"properties": {
"error_code": {
"minLength": 1
}
}
},
"else": {
"properties": {
"error_code": {
"const": ""
}
}
}
}
]
}
+1
View File
@@ -6,3 +6,4 @@
- 测试若需要签名,使用测试代码中明确标注的专用测试密钥。
- 恶意样例用于证明解析器和安全边界会拒绝输入,不得被发布流程消费。
- `catalog/manifest-valid-payload.json` 同时作为 manifest v1 强类型解析与目标过滤的公开虚构样例;包哈希与签名只保证格式合法,不对应真实下载物。
- `download/`:T-301 在运行时生成 HTTPS Range、断连、并发和恢复样例,不保存真实下载包。
+13
View File
@@ -0,0 +1,13 @@
# 下载测试数据
T-301 的 HTTP Range、断连、错误响应和并发阻塞样例由 Go 测试使用
`httptest` 与内存 Transport 在运行时生成,避免提交真实 URL 或大型二进制包。
覆盖场景包括:
- 新下载 200 与续传 206。
- Content-Range 起点/实体 validator 不匹配。
- 服务端忽略 Range 后从 0 安全重下。
- 连接中断后保留 `.part` 并按实际长度续传。
- 默认并发 2、暂停、取消、重试、完整 part 同进程 finalize 和崩溃恢复。
- observer 投递失败、body close 错误和活跃 `.part` 文件身份替换。