Implement resumable download queue (T-301)
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
package application
|
||||
|
||||
// DownloadStartedPayload begins one transfer attempt. A later attempt may
|
||||
// reset Done when a remote entity cannot be safely resumed.
|
||||
type DownloadStartedPayload struct {
|
||||
Attempt uint64
|
||||
Done int64
|
||||
TotalKnown bool
|
||||
Total int64
|
||||
}
|
||||
|
||||
// DownloadProgressPayload reports monotonic progress within one attempt.
|
||||
type DownloadProgressPayload struct {
|
||||
Attempt uint64
|
||||
Done int64
|
||||
TotalKnown bool
|
||||
Total int64
|
||||
SpeedBytesSec int64
|
||||
}
|
||||
|
||||
// DownloadPausedPayload maps to domain queued while preserving pause intent.
|
||||
type DownloadPausedPayload struct {
|
||||
Attempt uint64
|
||||
Done int64
|
||||
}
|
||||
|
||||
// DownloadCompletedPayload means bytes are durably downloaded but still
|
||||
// untrusted. T-302 must verify the signed Catalog identity, size and hashes.
|
||||
type DownloadCompletedPayload struct {
|
||||
Attempt uint64
|
||||
Done int64
|
||||
Path string
|
||||
}
|
||||
|
||||
// DownloadFailedPayload carries a stable low-level transfer/storage code.
|
||||
type DownloadFailedPayload struct {
|
||||
Attempt uint64
|
||||
Done int64
|
||||
ErrorCode string
|
||||
}
|
||||
|
||||
// DownloadCanceledPayload confirms cleanup and suppresses the old attempt.
|
||||
type DownloadCanceledPayload struct {
|
||||
Attempt uint64
|
||||
}
|
||||
@@ -11,6 +11,7 @@ const (
|
||||
EventDownloadPaused EventType = "DownloadPaused"
|
||||
EventDownloadCompleted EventType = "DownloadCompleted"
|
||||
EventDownloadFailed EventType = "DownloadFailed"
|
||||
EventDownloadCanceled EventType = "DownloadCanceled"
|
||||
EventInstallCompleted EventType = "InstallCompleted"
|
||||
EventInstallRolledBack EventType = "InstallRolledBack"
|
||||
EventAppStarted EventType = "AppStarted"
|
||||
@@ -26,6 +27,7 @@ var validEventTypes = map[EventType]struct{}{
|
||||
EventDownloadPaused: {},
|
||||
EventDownloadCompleted: {},
|
||||
EventDownloadFailed: {},
|
||||
EventDownloadCanceled: {},
|
||||
EventInstallCompleted: {},
|
||||
EventInstallRolledBack: {},
|
||||
EventAppStarted: {},
|
||||
|
||||
@@ -11,6 +11,7 @@ func TestEventTypeValid(t *testing.T) {
|
||||
EventDownloadPaused,
|
||||
EventDownloadCompleted,
|
||||
EventDownloadFailed,
|
||||
EventDownloadCanceled,
|
||||
EventInstallCompleted,
|
||||
EventInstallRolledBack,
|
||||
EventAppStarted,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"softbox.local/core/application"
|
||||
)
|
||||
|
||||
// ApplicationPublisher is implemented by application.Runtime.
|
||||
type ApplicationPublisher interface {
|
||||
Publish(context.Context, application.Event) error
|
||||
}
|
||||
|
||||
// ApplicationObserver maps downloader events to the documented application
|
||||
// event envelope and concrete payloads.
|
||||
type ApplicationObserver struct {
|
||||
Publisher ApplicationPublisher
|
||||
}
|
||||
|
||||
// PublishDownload implements Observer.
|
||||
func (observer ApplicationObserver) PublishDownload(
|
||||
ctx context.Context,
|
||||
event Event,
|
||||
) error {
|
||||
if observer.Publisher == nil {
|
||||
return nil
|
||||
}
|
||||
envelope := application.Event{
|
||||
RequestID: event.RequestID,
|
||||
AppID: event.AppID,
|
||||
}
|
||||
switch event.Type {
|
||||
case EventStarted:
|
||||
envelope.Type = application.EventDownloadStarted
|
||||
envelope.Payload = application.DownloadStartedPayload{
|
||||
Attempt: event.Attempt,
|
||||
Done: event.Done,
|
||||
TotalKnown: event.TotalKnown,
|
||||
Total: event.Total,
|
||||
}
|
||||
case EventProgress:
|
||||
envelope.Type = application.EventDownloadProgress
|
||||
envelope.Payload = application.DownloadProgressPayload{
|
||||
Attempt: event.Attempt,
|
||||
Done: event.Done,
|
||||
TotalKnown: event.TotalKnown,
|
||||
Total: event.Total,
|
||||
SpeedBytesSec: event.SpeedBytesSec,
|
||||
}
|
||||
case EventPaused:
|
||||
envelope.Type = application.EventDownloadPaused
|
||||
envelope.Payload = application.DownloadPausedPayload{
|
||||
Attempt: event.Attempt,
|
||||
Done: event.Done,
|
||||
}
|
||||
case EventCompleted:
|
||||
envelope.Type = application.EventDownloadCompleted
|
||||
envelope.Payload = application.DownloadCompletedPayload{
|
||||
Attempt: event.Attempt,
|
||||
Done: event.Done,
|
||||
Path: event.CompletedPath,
|
||||
}
|
||||
case EventFailed:
|
||||
envelope.Type = application.EventDownloadFailed
|
||||
envelope.Payload = application.DownloadFailedPayload{
|
||||
Attempt: event.Attempt,
|
||||
Done: event.Done,
|
||||
ErrorCode: event.ErrorCode,
|
||||
}
|
||||
case EventCanceled:
|
||||
envelope.Type = application.EventDownloadCanceled
|
||||
envelope.Payload = application.DownloadCanceledPayload{
|
||||
Attempt: event.Attempt,
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return observer.Publisher.Publish(ctx, envelope)
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"softbox.local/core/application"
|
||||
)
|
||||
|
||||
func TestApplicationObserverUsesConcretePayloads(t *testing.T) {
|
||||
publisher := &recordingApplicationPublisher{}
|
||||
observer := ApplicationObserver{Publisher: publisher}
|
||||
tests := []struct {
|
||||
event Event
|
||||
wantType application.EventType
|
||||
wantPayload any
|
||||
}{
|
||||
{
|
||||
event: Event{
|
||||
Type: EventStarted, RequestID: "request-1", AppID: "app-1",
|
||||
Attempt: 1, Done: 2, TotalKnown: true, Total: 10,
|
||||
},
|
||||
wantType: application.EventDownloadStarted,
|
||||
wantPayload: application.DownloadStartedPayload{
|
||||
Attempt: 1, Done: 2, TotalKnown: true, Total: 10,
|
||||
},
|
||||
},
|
||||
{
|
||||
event: Event{
|
||||
Type: EventProgress, RequestID: "request-1", AppID: "app-1",
|
||||
Attempt: 1, Done: 4, TotalKnown: true, Total: 10, SpeedBytesSec: 2,
|
||||
},
|
||||
wantType: application.EventDownloadProgress,
|
||||
wantPayload: application.DownloadProgressPayload{
|
||||
Attempt: 1, Done: 4, TotalKnown: true, Total: 10, SpeedBytesSec: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
event: Event{
|
||||
Type: EventPaused, RequestID: "request-1", AppID: "app-1",
|
||||
Attempt: 1, Done: 4,
|
||||
},
|
||||
wantType: application.EventDownloadPaused,
|
||||
wantPayload: application.DownloadPausedPayload{Attempt: 1, Done: 4},
|
||||
},
|
||||
{
|
||||
event: Event{
|
||||
Type: EventCompleted, RequestID: "request-1", AppID: "app-1",
|
||||
Attempt: 1, Done: 10, CompletedPath: "download",
|
||||
},
|
||||
wantType: application.EventDownloadCompleted,
|
||||
wantPayload: application.DownloadCompletedPayload{
|
||||
Attempt: 1, Done: 10, Path: "download",
|
||||
},
|
||||
},
|
||||
{
|
||||
event: Event{
|
||||
Type: EventFailed, RequestID: "request-1", AppID: "app-1",
|
||||
Attempt: 1, Done: 4, ErrorCode: "http_status",
|
||||
},
|
||||
wantType: application.EventDownloadFailed,
|
||||
wantPayload: application.DownloadFailedPayload{
|
||||
Attempt: 1, Done: 4, ErrorCode: "http_status",
|
||||
},
|
||||
},
|
||||
{
|
||||
event: Event{
|
||||
Type: EventCanceled, RequestID: "request-1", AppID: "app-1",
|
||||
Attempt: 1,
|
||||
},
|
||||
wantType: application.EventDownloadCanceled,
|
||||
wantPayload: application.DownloadCanceledPayload{Attempt: 1},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
if err := observer.PublishDownload(context.Background(), test.event); err != nil {
|
||||
t.Fatalf("PublishDownload(%s) error = %v", test.event.Type, err)
|
||||
}
|
||||
got := publisher.events[len(publisher.events)-1]
|
||||
if got.Type != test.wantType ||
|
||||
got.RequestID != test.event.RequestID ||
|
||||
got.AppID != test.event.AppID {
|
||||
t.Fatalf("event = %#v", got)
|
||||
}
|
||||
if !reflect.DeepEqual(got.Payload, test.wantPayload) {
|
||||
t.Fatalf("payload = %#v, want %#v", got.Payload, test.wantPayload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type recordingApplicationPublisher struct {
|
||||
events []application.Event
|
||||
}
|
||||
|
||||
func (publisher *recordingApplicationPublisher) Publish(
|
||||
_ context.Context,
|
||||
event application.Event,
|
||||
) error {
|
||||
publisher.events = append(publisher.events, event)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Package downloader provides resumable byte-transfer queues.
|
||||
//
|
||||
// A completed download is still untrusted input. Callers must verify the
|
||||
// signed Catalog identity, exact size, SHA-256 and package signature before
|
||||
// handing the file to the installer.
|
||||
package downloader
|
||||
@@ -0,0 +1,43 @@
|
||||
package downloader
|
||||
|
||||
import "context"
|
||||
|
||||
// EventType identifies an observer event emitted by Queue.
|
||||
type EventType string
|
||||
|
||||
const (
|
||||
EventStarted EventType = "started"
|
||||
EventProgress EventType = "progress"
|
||||
EventPaused EventType = "paused"
|
||||
EventCompleted EventType = "completed"
|
||||
EventFailed EventType = "failed"
|
||||
EventCanceled EventType = "canceled"
|
||||
)
|
||||
|
||||
// Event reports one generation of a download task. Done is monotonic within a
|
||||
// RequestID+Attempt pair. A new Attempt permits a reset to zero when a server
|
||||
// ignores Range or an entity cannot be safely resumed.
|
||||
type Event struct {
|
||||
Type EventType
|
||||
RequestID string
|
||||
AppID string
|
||||
Attempt uint64
|
||||
Done int64
|
||||
TotalKnown bool
|
||||
Total int64
|
||||
SpeedBytesSec int64
|
||||
CompletedPath string
|
||||
ErrorCode string
|
||||
}
|
||||
|
||||
// Observer receives queue events. Implementations must honor context
|
||||
// cancellation so pause/cancel can wait for the old generation to exit.
|
||||
type Observer interface {
|
||||
PublishDownload(context.Context, Event) error
|
||||
}
|
||||
|
||||
type discardObserver struct{}
|
||||
|
||||
func (discardObserver) PublishDownload(context.Context, Event) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
func ensureTransferLayout(downloadsRoot, requestID string) (TaskPaths, error) {
|
||||
paths, err := DeriveTaskPaths(downloadsRoot, requestID)
|
||||
if err != nil {
|
||||
return TaskPaths{}, err
|
||||
}
|
||||
if err := os.MkdirAll(paths.Root, 0o700); err != nil {
|
||||
return TaskPaths{}, fmt.Errorf("create downloads root: %w", err)
|
||||
}
|
||||
for _, directory := range []string{paths.Root, paths.TasksDir, paths.FilesDir} {
|
||||
if err := os.Mkdir(directory, 0o700); err != nil && !os.IsExist(err) {
|
||||
return TaskPaths{}, fmt.Errorf("create download directory: %w", err)
|
||||
}
|
||||
info, err := os.Lstat(directory)
|
||||
if err != nil {
|
||||
return TaskPaths{}, fmt.Errorf("inspect download directory: %w", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return TaskPaths{}, fmt.Errorf("%w: managed directory is unsafe", ErrTaskCorrupt)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func regularFileSize(filePath string) (size int64, exists bool, err error) {
|
||||
info, err := os.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
return 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("inspect download file: %w", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return 0, false, fmt.Errorf("%w: managed file is not regular", ErrTaskCorrupt)
|
||||
}
|
||||
return info.Size(), true, nil
|
||||
}
|
||||
|
||||
func regularFileExists(filePath string) (bool, error) {
|
||||
_, exists, err := regularFileSize(filePath)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func truncatePart(filePath string) error {
|
||||
file, _, err := openVerifiedRegular(filePath, os.O_RDWR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Truncate(0); err != nil {
|
||||
file.Close()
|
||||
return fmt.Errorf("truncate part: %w", err)
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
file.Close()
|
||||
return fmt.Errorf("seek truncated part: %w", err)
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
file.Close()
|
||||
return fmt.Errorf("sync truncated part: %w", err)
|
||||
}
|
||||
return file.Close()
|
||||
}
|
||||
|
||||
func openPart(paths TaskPaths, offset int64, restart bool) (*os.File, error) {
|
||||
if restart {
|
||||
if _, exists, err := regularFileSize(paths.Part); err != nil {
|
||||
return nil, err
|
||||
} else if exists {
|
||||
file, _, err := openVerifiedRegular(paths.Part, os.O_RDWR)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := file.Truncate(0); err != nil {
|
||||
file.Close()
|
||||
return nil, fmt.Errorf("truncate restarted part: %w", err)
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
file.Close()
|
||||
return nil, fmt.Errorf("seek restarted part: %w", err)
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
return os.OpenFile(paths.Part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
}
|
||||
size, exists, err := regularFileSize(paths.Part)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exists {
|
||||
if offset != 0 {
|
||||
return nil, fmt.Errorf("%w: missing part for non-zero offset", ErrTaskCorrupt)
|
||||
}
|
||||
return os.OpenFile(paths.Part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
}
|
||||
if size != offset {
|
||||
return nil, fmt.Errorf("%w: part size changed", ErrTaskCorrupt)
|
||||
}
|
||||
file, _, err := openVerifiedRegular(paths.Part, os.O_WRONLY)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := file.Seek(offset, io.SeekStart); err != nil {
|
||||
file.Close()
|
||||
return nil, fmt.Errorf("seek part: %w", err)
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func syncRegularFile(filePath string) error {
|
||||
file, _, err := openVerifiedRegular(filePath, os.O_RDWR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
return file.Close()
|
||||
}
|
||||
|
||||
func activateCompleted(paths TaskPaths, expected os.FileInfo) error {
|
||||
if exists, err := regularFileExists(paths.Completed); err != nil {
|
||||
return err
|
||||
} else if exists {
|
||||
return fmt.Errorf("%w: completed path already exists", ErrTaskCorrupt)
|
||||
}
|
||||
if expected == nil {
|
||||
part, partInfo, err := openVerifiedRegular(paths.Part, os.O_RDONLY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := part.Close(); err != nil {
|
||||
return fmt.Errorf("close part before activation: %w", err)
|
||||
}
|
||||
expected = partInfo
|
||||
}
|
||||
if err := verifyRegularIdentity(paths.Part, expected); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(paths.Part, paths.Completed); err != nil {
|
||||
return fmt.Errorf("activate completed download: %w", err)
|
||||
}
|
||||
completedInfo, err := os.Lstat(paths.Completed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect activated download: %w", err)
|
||||
}
|
||||
if completedInfo.Mode()&os.ModeSymlink != 0 ||
|
||||
!completedInfo.Mode().IsRegular() ||
|
||||
!os.SameFile(expected, completedInfo) {
|
||||
_ = removeExactRegularFile(paths.Completed)
|
||||
return fmt.Errorf("%w: activated file identity changed", ErrTaskCorrupt)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyRegularIdentity(filePath string, expected os.FileInfo) error {
|
||||
current, err := os.Lstat(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect managed file identity: %w", err)
|
||||
}
|
||||
if current.Mode()&os.ModeSymlink != 0 ||
|
||||
!current.Mode().IsRegular() ||
|
||||
!os.SameFile(expected, current) {
|
||||
return fmt.Errorf("%w: managed file identity changed", ErrTaskCorrupt)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeExactRegularFile(filePath string) error {
|
||||
info, err := os.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("%w: refuse removal of non-regular file", ErrTaskCorrupt)
|
||||
}
|
||||
return os.Remove(filePath)
|
||||
}
|
||||
|
||||
func syncAndClose(file *os.File) error {
|
||||
if file == nil {
|
||||
return nil
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
return file.Close()
|
||||
}
|
||||
|
||||
func openVerifiedRegular(
|
||||
filePath string,
|
||||
flags int,
|
||||
) (*os.File, os.FileInfo, error) {
|
||||
before, err := os.Lstat(filePath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() {
|
||||
return nil, nil, fmt.Errorf("%w: managed file is not regular", ErrTaskCorrupt)
|
||||
}
|
||||
file, err := os.OpenFile(filePath, flags, 0o600)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("open managed file: %w", err)
|
||||
}
|
||||
opened, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, nil, fmt.Errorf("stat managed file: %w", err)
|
||||
}
|
||||
after, err := os.Lstat(filePath)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, nil, fmt.Errorf("restat managed file: %w", err)
|
||||
}
|
||||
if after.Mode()&os.ModeSymlink != 0 ||
|
||||
!after.Mode().IsRegular() ||
|
||||
!os.SameFile(before, opened) ||
|
||||
!os.SameFile(opened, after) {
|
||||
file.Close()
|
||||
return nil, nil, fmt.Errorf("%w: managed file changed while opening", ErrTaskCorrupt)
|
||||
}
|
||||
return file, opened, nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestActivateCompletedRejectsReplacementOfWrittenPart(t *testing.T) {
|
||||
paths, err := ensureTransferLayout(t.TempDir(), "request-identity")
|
||||
if err != nil {
|
||||
t.Fatalf("ensureTransferLayout() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(paths.Part, []byte("trusted"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(original) error = %v", err)
|
||||
}
|
||||
original, expected, err := openVerifiedRegular(paths.Part, os.O_RDONLY)
|
||||
if err != nil {
|
||||
t.Fatalf("openVerifiedRegular() error = %v", err)
|
||||
}
|
||||
if err := original.Close(); err != nil {
|
||||
t.Fatalf("Close(original) error = %v", err)
|
||||
}
|
||||
if err := os.Remove(paths.Part); err != nil {
|
||||
t.Fatalf("Remove(original) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(paths.Part, []byte("replacement"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(replacement) error = %v", err)
|
||||
}
|
||||
|
||||
err = activateCompleted(paths, expected)
|
||||
if !errors.Is(err, ErrTaskCorrupt) {
|
||||
t.Fatalf("activateCompleted() error = %v, want %v", err, ErrTaskCorrupt)
|
||||
}
|
||||
if _, err := os.Stat(paths.Completed); !os.IsNotExist(err) {
|
||||
t.Fatalf("completed path exists after identity mismatch: %v", err)
|
||||
}
|
||||
document, err := os.ReadFile(paths.Part)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(replacement) error = %v", err)
|
||||
}
|
||||
if string(document) != "replacement" {
|
||||
t.Fatalf("replacement bytes = %q", document)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrHTTPStatus = errors.New("download HTTP status is not successful")
|
||||
ErrRangeMismatch = errors.New("download Content-Range does not match request")
|
||||
ErrRangeEntityChanged = errors.New("download range entity validator changed")
|
||||
ErrResponseEncoding = errors.New("download response encoding is not identity")
|
||||
ErrInsecureRedirect = errors.New("download redirect target is not secure")
|
||||
ErrTransferTooLarge = errors.New("download response exceeds byte limit")
|
||||
ErrTransferIncomplete = errors.New("download response ended before expected length")
|
||||
)
|
||||
|
||||
// OpenRequest describes one HTTP attempt.
|
||||
type OpenRequest struct {
|
||||
URL string
|
||||
Offset int64
|
||||
Validator EntityValidator
|
||||
}
|
||||
|
||||
// OpenResponse owns Body until the caller closes it.
|
||||
type OpenResponse struct {
|
||||
Body io.ReadCloser
|
||||
Restart bool
|
||||
TotalKnown bool
|
||||
Total int64
|
||||
ResponseLengthKnown bool
|
||||
ResponseLength int64
|
||||
Validator EntityValidator
|
||||
}
|
||||
|
||||
// Transport opens a remote byte stream at a requested offset.
|
||||
type Transport interface {
|
||||
Open(context.Context, OpenRequest) (OpenResponse, error)
|
||||
}
|
||||
|
||||
// HTTPTransport implements strict HTTPS and Range semantics with net/http.
|
||||
type HTTPTransport struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewHTTPTransport creates a standard-library transport.
|
||||
func NewHTTPTransport(client *http.Client) *HTTPTransport {
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
clientCopy := *client
|
||||
previousRedirectCheck := client.CheckRedirect
|
||||
clientCopy.CheckRedirect = func(request *http.Request, via []*http.Request) error {
|
||||
if request.URL == nil || ValidateHTTPSURL(request.URL.String()) != nil {
|
||||
return ErrInsecureRedirect
|
||||
}
|
||||
if previousRedirectCheck != nil {
|
||||
return previousRedirectCheck(request, via)
|
||||
}
|
||||
if len(via) >= 10 {
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return &HTTPTransport{client: &clientCopy}
|
||||
}
|
||||
|
||||
// Open starts a transfer. A 200 response to a Range request is returned as
|
||||
// Restart=true so the queue truncates the old part before consuming bytes.
|
||||
func (transport *HTTPTransport) Open(
|
||||
ctx context.Context,
|
||||
request OpenRequest,
|
||||
) (OpenResponse, error) {
|
||||
if err := ValidateHTTPSURL(request.URL); err != nil {
|
||||
return OpenResponse{}, err
|
||||
}
|
||||
if request.Offset < 0 {
|
||||
return OpenResponse{}, fmt.Errorf("%w: negative offset", ErrRangeMismatch)
|
||||
}
|
||||
if err := request.Validator.Validate(); err != nil {
|
||||
return OpenResponse{}, fmt.Errorf("%w: invalid validator", ErrRangeMismatch)
|
||||
}
|
||||
if request.Offset > 0 && request.Validator.Empty() {
|
||||
return OpenResponse{}, fmt.Errorf("%w: resume needs an entity validator", ErrRangeMismatch)
|
||||
}
|
||||
|
||||
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, request.URL, nil)
|
||||
if err != nil {
|
||||
return OpenResponse{}, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
httpRequest.Header.Set("Accept-Encoding", "identity")
|
||||
if request.Offset > 0 {
|
||||
httpRequest.Header.Set("Range", fmt.Sprintf("bytes=%d-", request.Offset))
|
||||
httpRequest.Header.Set("If-Range", request.Validator.HeaderValue())
|
||||
}
|
||||
|
||||
response, err := transport.client.Do(httpRequest)
|
||||
if err != nil {
|
||||
return OpenResponse{}, err
|
||||
}
|
||||
closeWithError := func(openErr error) (OpenResponse, error) {
|
||||
_ = response.Body.Close()
|
||||
return OpenResponse{}, openErr
|
||||
}
|
||||
if response.Request == nil || response.Request.URL == nil ||
|
||||
ValidateHTTPSURL(response.Request.URL.String()) != nil {
|
||||
return closeWithError(ErrInsecureRedirect)
|
||||
}
|
||||
encoding := strings.TrimSpace(response.Header.Get("Content-Encoding"))
|
||||
if encoding != "" && !strings.EqualFold(encoding, "identity") {
|
||||
return closeWithError(ErrResponseEncoding)
|
||||
}
|
||||
|
||||
validator := responseValidator(response.Header)
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
if response.Header.Get("Content-Range") != "" {
|
||||
return closeWithError(fmt.Errorf("%w: 200 includes Content-Range", ErrRangeMismatch))
|
||||
}
|
||||
totalKnown := response.ContentLength >= 0
|
||||
total := response.ContentLength
|
||||
if !totalKnown {
|
||||
total = 0
|
||||
}
|
||||
return OpenResponse{
|
||||
Body: response.Body,
|
||||
Restart: request.Offset > 0,
|
||||
TotalKnown: totalKnown,
|
||||
Total: total,
|
||||
ResponseLengthKnown: totalKnown,
|
||||
ResponseLength: total,
|
||||
Validator: validator,
|
||||
}, nil
|
||||
|
||||
case http.StatusPartialContent:
|
||||
if request.Offset == 0 {
|
||||
return closeWithError(fmt.Errorf("%w: unsolicited partial response", ErrRangeMismatch))
|
||||
}
|
||||
start, end, totalKnown, total, parseErr := parseContentRange(
|
||||
response.Header.Get("Content-Range"),
|
||||
)
|
||||
if parseErr != nil || start != request.Offset {
|
||||
return closeWithError(ErrRangeMismatch)
|
||||
}
|
||||
rangeLength := end - start + 1
|
||||
if rangeLength <= 0 ||
|
||||
(response.ContentLength >= 0 && response.ContentLength != rangeLength) {
|
||||
return closeWithError(ErrRangeMismatch)
|
||||
}
|
||||
if !validatorsMatch(request.Validator, validator) {
|
||||
return closeWithError(ErrRangeEntityChanged)
|
||||
}
|
||||
return OpenResponse{
|
||||
Body: response.Body,
|
||||
TotalKnown: totalKnown,
|
||||
Total: total,
|
||||
ResponseLengthKnown: true,
|
||||
ResponseLength: rangeLength,
|
||||
Validator: request.Validator,
|
||||
}, nil
|
||||
|
||||
default:
|
||||
return closeWithError(fmt.Errorf("%w: %d", ErrHTTPStatus, response.StatusCode))
|
||||
}
|
||||
}
|
||||
|
||||
func parseContentRange(value string) (
|
||||
start int64,
|
||||
end int64,
|
||||
totalKnown bool,
|
||||
total int64,
|
||||
err error,
|
||||
) {
|
||||
if !strings.HasPrefix(value, "bytes ") {
|
||||
return 0, 0, false, 0, ErrRangeMismatch
|
||||
}
|
||||
rangeAndTotal := strings.Split(strings.TrimPrefix(value, "bytes "), "/")
|
||||
if len(rangeAndTotal) != 2 {
|
||||
return 0, 0, false, 0, ErrRangeMismatch
|
||||
}
|
||||
bounds := strings.Split(rangeAndTotal[0], "-")
|
||||
if len(bounds) != 2 {
|
||||
return 0, 0, false, 0, ErrRangeMismatch
|
||||
}
|
||||
start, err = strconv.ParseInt(bounds[0], 10, 64)
|
||||
if err != nil || start < 0 {
|
||||
return 0, 0, false, 0, ErrRangeMismatch
|
||||
}
|
||||
end, err = strconv.ParseInt(bounds[1], 10, 64)
|
||||
if err != nil || end < start || end == int64(^uint64(0)>>1) {
|
||||
return 0, 0, false, 0, ErrRangeMismatch
|
||||
}
|
||||
if rangeAndTotal[1] == "*" {
|
||||
return start, end, false, 0, nil
|
||||
}
|
||||
total, err = strconv.ParseInt(rangeAndTotal[1], 10, 64)
|
||||
if err != nil || total <= end {
|
||||
return 0, 0, false, 0, ErrRangeMismatch
|
||||
}
|
||||
return start, end, true, total, nil
|
||||
}
|
||||
|
||||
// CopyOptions bounds one response copy and tags progress with the generation
|
||||
// that owns it.
|
||||
type CopyOptions struct {
|
||||
MaxBytes int64
|
||||
ExpectedBytesKnown bool
|
||||
ExpectedBytes int64
|
||||
Generation uint64
|
||||
Progress func(CopyProgress) error
|
||||
}
|
||||
|
||||
// CopyProgress reports bytes written during this response only.
|
||||
type CopyProgress struct {
|
||||
Generation uint64
|
||||
Written int64
|
||||
}
|
||||
|
||||
// CopyResponse copies a response with a hard byte cap. It reads at most
|
||||
// MaxBytes+1 bytes so an oversized body is detected without unbounded IO.
|
||||
func CopyResponse(
|
||||
ctx context.Context,
|
||||
dst io.Writer,
|
||||
src io.Reader,
|
||||
options CopyOptions,
|
||||
) (int64, error) {
|
||||
if options.MaxBytes < 0 || options.MaxBytes == int64(^uint64(0)>>1) {
|
||||
return 0, fmt.Errorf("%w: invalid byte limit", ErrTransferTooLarge)
|
||||
}
|
||||
if options.ExpectedBytesKnown &&
|
||||
(options.ExpectedBytes < 0 || options.ExpectedBytes > options.MaxBytes) {
|
||||
return 0, fmt.Errorf("%w: invalid expected response length", ErrTransferIncomplete)
|
||||
}
|
||||
|
||||
limited := &io.LimitedReader{R: src, N: options.MaxBytes + 1}
|
||||
buffer := make([]byte, 32*1024)
|
||||
var written int64
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return written, err
|
||||
}
|
||||
readCount, readErr := limited.Read(buffer)
|
||||
if readCount > 0 {
|
||||
if int64(readCount) > options.MaxBytes-written {
|
||||
return written, ErrTransferTooLarge
|
||||
}
|
||||
writeCount, writeErr := dst.Write(buffer[:readCount])
|
||||
written += int64(writeCount)
|
||||
if writeErr != nil {
|
||||
return written, writeErr
|
||||
}
|
||||
if writeCount != readCount {
|
||||
return written, io.ErrShortWrite
|
||||
}
|
||||
if options.Progress != nil {
|
||||
if err := options.Progress(CopyProgress{
|
||||
Generation: options.Generation,
|
||||
Written: written,
|
||||
}); err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if readErr != nil {
|
||||
if readErr != io.EOF {
|
||||
return written, readErr
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if options.ExpectedBytesKnown && written != options.ExpectedBytes {
|
||||
return written, ErrTransferIncomplete
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPTransportFreshAndResume(t *testing.T) {
|
||||
content := []byte("abcdefghij")
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(
|
||||
writer http.ResponseWriter,
|
||||
request *http.Request,
|
||||
) {
|
||||
if request.Header.Get("Accept-Encoding") != "identity" {
|
||||
t.Errorf("Accept-Encoding = %q", request.Header.Get("Accept-Encoding"))
|
||||
}
|
||||
switch request.Header.Get("Range") {
|
||||
case "":
|
||||
writer.Header().Set("ETag", `"v1"`)
|
||||
writer.Header().Set("Content-Length", "10")
|
||||
_, _ = writer.Write(content)
|
||||
case "bytes=4-":
|
||||
if request.Header.Get("If-Range") != `"v1"` {
|
||||
t.Errorf("If-Range = %q", request.Header.Get("If-Range"))
|
||||
}
|
||||
writer.Header().Set("ETag", `"v1"`)
|
||||
writer.Header().Set("Content-Range", "bytes 4-9/10")
|
||||
writer.Header().Set("Content-Length", "6")
|
||||
writer.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = writer.Write(content[4:])
|
||||
default:
|
||||
t.Errorf("unexpected Range %q", request.Header.Get("Range"))
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
transport := NewHTTPTransport(server.Client())
|
||||
|
||||
fresh, err := transport.Open(context.Background(), OpenRequest{URL: server.URL})
|
||||
if err != nil {
|
||||
t.Fatalf("Open(fresh) error = %v", err)
|
||||
}
|
||||
freshBytes, err := io.ReadAll(fresh.Body)
|
||||
fresh.Body.Close()
|
||||
if err != nil || !bytes.Equal(freshBytes, content) {
|
||||
t.Fatalf("fresh bytes = %q, error=%v", freshBytes, err)
|
||||
}
|
||||
if fresh.Restart || !fresh.TotalKnown || fresh.Total != 10 ||
|
||||
fresh.Validator.ETag != `"v1"` {
|
||||
t.Fatalf("fresh response = %#v", fresh)
|
||||
}
|
||||
|
||||
resumed, err := transport.Open(context.Background(), OpenRequest{
|
||||
URL: server.URL,
|
||||
Offset: 4,
|
||||
Validator: EntityValidator{ETag: `"v1"`},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Open(resume) error = %v", err)
|
||||
}
|
||||
resumedBytes, err := io.ReadAll(resumed.Body)
|
||||
resumed.Body.Close()
|
||||
if err != nil || !bytes.Equal(resumedBytes, content[4:]) {
|
||||
t.Fatalf("resume bytes = %q, error=%v", resumedBytes, err)
|
||||
}
|
||||
if resumed.Restart || resumed.ResponseLength != 6 || resumed.Total != 10 {
|
||||
t.Fatalf("resume response = %#v", resumed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPTransportRestartsWhenRangeIgnored(t *testing.T) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(
|
||||
writer http.ResponseWriter,
|
||||
_ *http.Request,
|
||||
) {
|
||||
writer.Header().Set("ETag", `"v2"`)
|
||||
writer.Header().Set("Content-Length", "4")
|
||||
_, _ = writer.Write([]byte("new!"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
response, err := NewHTTPTransport(server.Client()).Open(
|
||||
context.Background(),
|
||||
OpenRequest{
|
||||
URL: server.URL,
|
||||
Offset: 2,
|
||||
Validator: EntityValidator{ETag: `"v1"`},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Open() error = %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if !response.Restart || response.Validator.ETag != `"v2"` {
|
||||
t.Fatalf("response = %#v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPTransportRejectsInvalidRangeResponses(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
contentRange string
|
||||
etag string
|
||||
want error
|
||||
}{
|
||||
{name: "wrong start", contentRange: "bytes 3-9/10", etag: `"v1"`, want: ErrRangeMismatch},
|
||||
{name: "malformed", contentRange: "invalid", etag: `"v1"`, want: ErrRangeMismatch},
|
||||
{name: "entity changed", contentRange: "bytes 4-9/10", etag: `"v2"`, want: ErrRangeEntityChanged},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(
|
||||
writer http.ResponseWriter,
|
||||
_ *http.Request,
|
||||
) {
|
||||
writer.Header().Set("Content-Range", test.contentRange)
|
||||
writer.Header().Set("Content-Length", "6")
|
||||
writer.Header().Set("ETag", test.etag)
|
||||
writer.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = writer.Write([]byte("efghij"))
|
||||
}))
|
||||
defer server.Close()
|
||||
_, err := NewHTTPTransport(server.Client()).Open(
|
||||
context.Background(),
|
||||
OpenRequest{
|
||||
URL: server.URL,
|
||||
Offset: 4,
|
||||
Validator: EntityValidator{ETag: `"v1"`},
|
||||
},
|
||||
)
|
||||
if !errors.Is(err, test.want) {
|
||||
t.Fatalf("Open() error = %v, want %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPTransportRejectsNonSuccessStatuses(t *testing.T) {
|
||||
for _, status := range []int{
|
||||
http.StatusNotFound,
|
||||
http.StatusRequestedRangeNotSatisfiable,
|
||||
http.StatusInternalServerError,
|
||||
} {
|
||||
t.Run(http.StatusText(status), func(t *testing.T) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(
|
||||
writer http.ResponseWriter,
|
||||
_ *http.Request,
|
||||
) {
|
||||
writer.WriteHeader(status)
|
||||
}))
|
||||
defer server.Close()
|
||||
_, err := NewHTTPTransport(server.Client()).Open(
|
||||
context.Background(),
|
||||
OpenRequest{URL: server.URL},
|
||||
)
|
||||
if !errors.Is(err, ErrHTTPStatus) {
|
||||
t.Fatalf("Open() error = %v, want %v", err, ErrHTTPStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPTransportRejectsUnsafeSuccessMetadata(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
offset int64
|
||||
headers map[string]string
|
||||
status int
|
||||
want error
|
||||
}{
|
||||
{
|
||||
name: "encoded body",
|
||||
status: http.StatusOK,
|
||||
headers: map[string]string{
|
||||
"Content-Encoding": "gzip",
|
||||
},
|
||||
want: ErrResponseEncoding,
|
||||
},
|
||||
{
|
||||
name: "200 with content range",
|
||||
status: http.StatusOK,
|
||||
headers: map[string]string{
|
||||
"Content-Range": "bytes 0-3/4",
|
||||
},
|
||||
want: ErrRangeMismatch,
|
||||
},
|
||||
{
|
||||
name: "206 missing content range",
|
||||
offset: 2,
|
||||
status: http.StatusPartialContent,
|
||||
headers: map[string]string{
|
||||
"ETag": `"v1"`,
|
||||
},
|
||||
want: ErrRangeMismatch,
|
||||
},
|
||||
{
|
||||
name: "206 content length mismatch",
|
||||
offset: 2,
|
||||
status: http.StatusPartialContent,
|
||||
headers: map[string]string{
|
||||
"Content-Range": "bytes 2-3/4",
|
||||
"Content-Length": "1",
|
||||
"ETag": `"v1"`,
|
||||
},
|
||||
want: ErrRangeMismatch,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(
|
||||
writer http.ResponseWriter,
|
||||
_ *http.Request,
|
||||
) {
|
||||
for name, value := range test.headers {
|
||||
writer.Header().Set(name, value)
|
||||
}
|
||||
writer.WriteHeader(test.status)
|
||||
_, _ = writer.Write([]byte("data"))
|
||||
}))
|
||||
defer server.Close()
|
||||
request := OpenRequest{URL: server.URL, Offset: test.offset}
|
||||
if test.offset > 0 {
|
||||
request.Validator = EntityValidator{ETag: `"v1"`}
|
||||
}
|
||||
_, err := NewHTTPTransport(server.Client()).Open(
|
||||
context.Background(),
|
||||
request,
|
||||
)
|
||||
if !errors.Is(err, test.want) {
|
||||
t.Fatalf("Open() error = %v, want %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPTransportRejectsDowngradeRedirect(t *testing.T) {
|
||||
insecure := httptest.NewServer(http.HandlerFunc(func(
|
||||
writer http.ResponseWriter,
|
||||
_ *http.Request,
|
||||
) {
|
||||
_, _ = writer.Write([]byte("unsafe"))
|
||||
}))
|
||||
defer insecure.Close()
|
||||
secure := httptest.NewTLSServer(http.HandlerFunc(func(
|
||||
writer http.ResponseWriter,
|
||||
request *http.Request,
|
||||
) {
|
||||
http.Redirect(writer, request, insecure.URL, http.StatusFound)
|
||||
}))
|
||||
defer secure.Close()
|
||||
|
||||
_, err := NewHTTPTransport(secure.Client()).Open(
|
||||
context.Background(),
|
||||
OpenRequest{URL: secure.URL},
|
||||
)
|
||||
if !errors.Is(err, ErrInsecureRedirect) {
|
||||
t.Fatalf("Open() error = %v, want %v", err, ErrInsecureRedirect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyResponseBoundsAndCompleteness(t *testing.T) {
|
||||
var destination bytes.Buffer
|
||||
written, err := CopyResponse(
|
||||
context.Background(),
|
||||
&destination,
|
||||
bytes.NewReader([]byte("1234")),
|
||||
CopyOptions{MaxBytes: 4, ExpectedBytesKnown: true, ExpectedBytes: 4},
|
||||
)
|
||||
if err != nil || written != 4 || destination.String() != "1234" {
|
||||
t.Fatalf("CopyResponse() = %d, %q, %v", written, destination.String(), err)
|
||||
}
|
||||
|
||||
_, err = CopyResponse(
|
||||
context.Background(),
|
||||
io.Discard,
|
||||
bytes.NewReader([]byte("12345")),
|
||||
CopyOptions{MaxBytes: 4},
|
||||
)
|
||||
if !errors.Is(err, ErrTransferTooLarge) {
|
||||
t.Fatalf("oversize error = %v, want %v", err, ErrTransferTooLarge)
|
||||
}
|
||||
_, err = CopyResponse(
|
||||
context.Background(),
|
||||
io.Discard,
|
||||
bytes.NewReader([]byte("123")),
|
||||
CopyOptions{MaxBytes: 4, ExpectedBytesKnown: true, ExpectedBytes: 4},
|
||||
)
|
||||
if !errors.Is(err, ErrTransferIncomplete) {
|
||||
t.Fatalf("short error = %v, want %v", err, ErrTransferIncomplete)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TaskPaths are fixed local names derived only from RequestID.
|
||||
type TaskPaths struct {
|
||||
Root string
|
||||
TasksDir string
|
||||
FilesDir string
|
||||
Metadata string
|
||||
Backup string
|
||||
Part string
|
||||
Completed string
|
||||
}
|
||||
|
||||
// DeriveTaskPaths derives all local paths without accepting remote filenames
|
||||
// or Content-Disposition values.
|
||||
func DeriveTaskPaths(downloadsRoot, requestID string) (TaskPaths, error) {
|
||||
if downloadsRoot == "" {
|
||||
return TaskPaths{}, fmt.Errorf("%w: empty downloads root", ErrInvalidTask)
|
||||
}
|
||||
if !ValidRequestID(requestID) {
|
||||
return TaskPaths{}, fmt.Errorf("%w: invalid request_id", ErrInvalidTask)
|
||||
}
|
||||
absoluteRoot, err := filepath.Abs(downloadsRoot)
|
||||
if err != nil {
|
||||
return TaskPaths{}, fmt.Errorf("%w: resolve downloads root", ErrInvalidTask)
|
||||
}
|
||||
tasksDir := filepath.Join(absoluteRoot, "tasks")
|
||||
filesDir := filepath.Join(absoluteRoot, "files")
|
||||
paths := TaskPaths{
|
||||
Root: absoluteRoot,
|
||||
TasksDir: tasksDir,
|
||||
FilesDir: filesDir,
|
||||
Metadata: filepath.Join(tasksDir, requestID+".json"),
|
||||
Backup: filepath.Join(tasksDir, requestID+".json.backup"),
|
||||
Part: filepath.Join(filesDir, requestID+".part"),
|
||||
Completed: filepath.Join(filesDir, requestID+".download"),
|
||||
}
|
||||
for _, candidate := range []string{
|
||||
paths.TasksDir,
|
||||
paths.FilesDir,
|
||||
paths.Metadata,
|
||||
paths.Backup,
|
||||
paths.Part,
|
||||
paths.Completed,
|
||||
} {
|
||||
relative, relativeErr := filepath.Rel(absoluteRoot, candidate)
|
||||
if relativeErr != nil ||
|
||||
relative == ".." ||
|
||||
strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return TaskPaths{}, fmt.Errorf("%w: derived path escapes root", ErrInvalidTask)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
TaskSchemaVersion = 1
|
||||
MaxRequestIDBytes = 128
|
||||
MaxAppIDBytes = 128
|
||||
MaxURLBytes = 4096
|
||||
MaxErrorCodeBytes = 64
|
||||
MaxValidatorBytes = 512
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidTask = errors.New("download task is invalid")
|
||||
ErrTaskConflict = errors.New("download task conflicts with an existing task")
|
||||
ErrTaskNotFound = errors.New("download task not found")
|
||||
ErrInvalidCommand = errors.New("download command is invalid for task state")
|
||||
ErrTaskBusy = errors.New("download task is busy")
|
||||
ErrQueueClosed = errors.New("download queue is closed")
|
||||
ErrTaskCorrupt = errors.New("download task state is corrupt")
|
||||
requestIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,127}$`)
|
||||
appIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,127}$`)
|
||||
errorCodePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_]{0,63}$`)
|
||||
strongETagPattern = regexp.MustCompile(`^"[\x21\x23-\x7E\x80-\xFF]*"$`)
|
||||
)
|
||||
|
||||
// TaskStatus is the durable lifecycle of one byte transfer.
|
||||
type TaskStatus string
|
||||
|
||||
const (
|
||||
StatusQueued TaskStatus = "queued"
|
||||
StatusDownloading TaskStatus = "downloading"
|
||||
StatusPaused TaskStatus = "paused"
|
||||
StatusFailed TaskStatus = "failed"
|
||||
StatusCompleted TaskStatus = "completed"
|
||||
)
|
||||
|
||||
// EntityValidator binds a resumed range to the same remote representation.
|
||||
// ETag must be strong. LastModified must be an HTTP-date.
|
||||
type EntityValidator struct {
|
||||
ETag string `json:"etag,omitempty"`
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
}
|
||||
|
||||
// Task is the persisted download-task.json v1 protocol.
|
||||
//
|
||||
// Local paths are deliberately absent. They are derived from RequestID below
|
||||
// the configured downloads root so remote metadata cannot choose filesystem
|
||||
// destinations.
|
||||
type Task struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
RequestID string `json:"request_id"`
|
||||
AppID string `json:"app_id"`
|
||||
URL string `json:"url"`
|
||||
Status TaskStatus `json:"status"`
|
||||
Attempt uint64 `json:"attempt"`
|
||||
Done int64 `json:"done"`
|
||||
TotalKnown bool `json:"total_known"`
|
||||
Total int64 `json:"total"`
|
||||
Validator EntityValidator `json:"validator"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
// Terminal reports whether the task no longer occupies the per-app active
|
||||
// task slot. Completed transfers remain queryable until a later workflow
|
||||
// consumes or removes them.
|
||||
func (task Task) Terminal() bool {
|
||||
return task.Status == StatusCompleted
|
||||
}
|
||||
|
||||
// Validate enforces the durable protocol independently of JSON Schema.
|
||||
func (task Task) Validate() error {
|
||||
if task.SchemaVersion != TaskSchemaVersion {
|
||||
return fmt.Errorf("%w: schema_version=%d", ErrInvalidTask, task.SchemaVersion)
|
||||
}
|
||||
if !ValidRequestID(task.RequestID) {
|
||||
return fmt.Errorf("%w: invalid request_id", ErrInvalidTask)
|
||||
}
|
||||
if !appIDPattern.MatchString(task.AppID) || len(task.AppID) > MaxAppIDBytes {
|
||||
return fmt.Errorf("%w: invalid app_id", ErrInvalidTask)
|
||||
}
|
||||
if err := ValidateHTTPSURL(task.URL); err != nil {
|
||||
return fmt.Errorf("%w: url: %v", ErrInvalidTask, err)
|
||||
}
|
||||
switch task.Status {
|
||||
case StatusQueued, StatusDownloading, StatusPaused, StatusFailed, StatusCompleted:
|
||||
default:
|
||||
return fmt.Errorf("%w: invalid status %q", ErrInvalidTask, task.Status)
|
||||
}
|
||||
if task.Attempt > uint64(^uint64(0)>>1) {
|
||||
return fmt.Errorf("%w: attempt is too large", ErrInvalidTask)
|
||||
}
|
||||
if task.Done < 0 {
|
||||
return fmt.Errorf("%w: done=%d", ErrInvalidTask, task.Done)
|
||||
}
|
||||
if task.TotalKnown {
|
||||
if task.Total <= 0 || task.Done > task.Total {
|
||||
return fmt.Errorf(
|
||||
"%w: invalid known total done=%d total=%d",
|
||||
ErrInvalidTask,
|
||||
task.Done,
|
||||
task.Total,
|
||||
)
|
||||
}
|
||||
if task.Status == StatusCompleted && task.Done != task.Total {
|
||||
return fmt.Errorf(
|
||||
"%w: completed done=%d total=%d",
|
||||
ErrInvalidTask,
|
||||
task.Done,
|
||||
task.Total,
|
||||
)
|
||||
}
|
||||
} else if task.Total != 0 {
|
||||
return fmt.Errorf("%w: unknown total must be zero", ErrInvalidTask)
|
||||
}
|
||||
if err := task.Validator.Validate(); err != nil {
|
||||
return fmt.Errorf("%w: validator: %v", ErrInvalidTask, err)
|
||||
}
|
||||
if task.Status == StatusFailed {
|
||||
if !errorCodePattern.MatchString(task.ErrorCode) ||
|
||||
len(task.ErrorCode) > MaxErrorCodeBytes {
|
||||
return fmt.Errorf("%w: failed task needs a stable error_code", ErrInvalidTask)
|
||||
}
|
||||
} else if task.ErrorCode != "" {
|
||||
return fmt.Errorf("%w: error_code is only valid for failed tasks", ErrInvalidTask)
|
||||
}
|
||||
createdAt, err := time.Parse(time.RFC3339Nano, task.CreatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: created_at: %v", ErrInvalidTask, err)
|
||||
}
|
||||
_, offset := createdAt.Zone()
|
||||
if offset != 0 {
|
||||
return fmt.Errorf("%w: created_at must be UTC", ErrInvalidTask)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate validates an entity validator.
|
||||
func (validator EntityValidator) Validate() error {
|
||||
if validator.ETag != "" && validator.LastModified != "" {
|
||||
return errors.New("etag and last_modified are mutually exclusive")
|
||||
}
|
||||
if validator.ETag != "" {
|
||||
if len(validator.ETag) > MaxValidatorBytes ||
|
||||
strings.HasPrefix(strings.ToUpper(validator.ETag), "W/") ||
|
||||
!strongETagPattern.MatchString(validator.ETag) {
|
||||
return errors.New("etag is not a strong ETag")
|
||||
}
|
||||
}
|
||||
if validator.LastModified != "" {
|
||||
if len(validator.LastModified) > MaxValidatorBytes {
|
||||
return errors.New("last_modified is too long")
|
||||
}
|
||||
if _, err := http.ParseTime(validator.LastModified); err != nil {
|
||||
return errors.New("last_modified is not an HTTP-date")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Empty reports whether no reliable resume validator is available.
|
||||
func (validator EntityValidator) Empty() bool {
|
||||
return validator.ETag == "" && validator.LastModified == ""
|
||||
}
|
||||
|
||||
// HeaderValue returns the If-Range value.
|
||||
func (validator EntityValidator) HeaderValue() string {
|
||||
if validator.ETag != "" {
|
||||
return validator.ETag
|
||||
}
|
||||
return validator.LastModified
|
||||
}
|
||||
|
||||
// ValidRequestID reports whether a request ID can safely derive local names.
|
||||
func ValidRequestID(requestID string) bool {
|
||||
return len(requestID) <= MaxRequestIDBytes && requestIDPattern.MatchString(requestID)
|
||||
}
|
||||
|
||||
// ValidateHTTPSURL applies the same transport restrictions as the Catalog.
|
||||
func ValidateHTTPSURL(value string) error {
|
||||
if value == "" || len(value) > MaxURLBytes {
|
||||
return errors.New("URL length is invalid")
|
||||
}
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil {
|
||||
return errors.New("URL is malformed")
|
||||
}
|
||||
if parsed.Scheme != "https" ||
|
||||
parsed.Host == "" ||
|
||||
parsed.User != nil ||
|
||||
parsed.Fragment != "" ||
|
||||
!parsed.IsAbs() {
|
||||
return errors.New("URL must be absolute HTTPS without user info or fragment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatorsMatch(expected, actual EntityValidator) bool {
|
||||
if expected.ETag != "" {
|
||||
if actual.Empty() {
|
||||
return true
|
||||
}
|
||||
return expected.ETag == actual.ETag
|
||||
}
|
||||
if expected.LastModified != "" {
|
||||
if actual.Empty() {
|
||||
return true
|
||||
}
|
||||
return expected.LastModified == actual.LastModified
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func responseValidator(header http.Header) EntityValidator {
|
||||
etag := strings.TrimSpace(header.Get("ETag"))
|
||||
if etag != "" &&
|
||||
!strings.HasPrefix(strings.ToUpper(etag), "W/") &&
|
||||
strongETagPattern.MatchString(etag) &&
|
||||
len(etag) <= MaxValidatorBytes {
|
||||
return EntityValidator{ETag: etag}
|
||||
}
|
||||
lastModified := strings.TrimSpace(header.Get("Last-Modified"))
|
||||
if lastModified != "" && len(lastModified) <= MaxValidatorBytes {
|
||||
if _, err := http.ParseTime(lastModified); err == nil {
|
||||
return EntityValidator{LastModified: lastModified}
|
||||
}
|
||||
}
|
||||
return EntityValidator{}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTaskValidateAndDerivedPaths(t *testing.T) {
|
||||
task := validTaskForTest()
|
||||
if err := task.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
paths, err := DeriveTaskPaths(filepath.Join(t.TempDir(), "downloads"), task.RequestID)
|
||||
if err != nil {
|
||||
t.Fatalf("DeriveTaskPaths() error = %v", err)
|
||||
}
|
||||
for _, path := range []string{
|
||||
paths.Metadata,
|
||||
paths.Backup,
|
||||
paths.Part,
|
||||
paths.Completed,
|
||||
} {
|
||||
relative, err := filepath.Rel(paths.Root, path)
|
||||
if err != nil || relative == ".." {
|
||||
t.Fatalf("derived path escapes root: %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskRejectsInvalidFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Task)
|
||||
}{
|
||||
{name: "request id", mutate: func(task *Task) { task.RequestID = "../escape" }},
|
||||
{name: "app id", mutate: func(task *Task) { task.AppID = "Bad App" }},
|
||||
{name: "URL", mutate: func(task *Task) { task.URL = "http://example.invalid/app.zip" }},
|
||||
{name: "state", mutate: func(task *Task) { task.Status = "unknown" }},
|
||||
{name: "negative done", mutate: func(task *Task) { task.Done = -1 }},
|
||||
{name: "done exceeds total", mutate: func(task *Task) { task.Done = task.Total + 1 }},
|
||||
{name: "unknown total value", mutate: func(task *Task) {
|
||||
task.TotalKnown = false
|
||||
task.Total = 10
|
||||
}},
|
||||
{name: "weak etag", mutate: func(task *Task) {
|
||||
task.Validator = EntityValidator{ETag: `W/"weak"`}
|
||||
}},
|
||||
{name: "failed without code", mutate: func(task *Task) {
|
||||
task.Status = StatusFailed
|
||||
task.ErrorCode = ""
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
task := validTaskForTest()
|
||||
test.mutate(&task)
|
||||
if err := task.Validate(); !errors.Is(err, ErrInvalidTask) {
|
||||
t.Fatalf("Validate() error = %v, want %v", err, ErrInvalidTask)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorsMatchRejectsDifferentKinds(t *testing.T) {
|
||||
expected := EntityValidator{ETag: `"v1"`}
|
||||
if validatorsMatch(expected, EntityValidator{LastModified: "Wed, 16 Jul 2026 10:00:00 GMT"}) {
|
||||
t.Fatal("ETag unexpectedly matched Last-Modified")
|
||||
}
|
||||
if validatorsMatch(expected, EntityValidator{ETag: `"v2"`}) {
|
||||
t.Fatal("different ETags matched")
|
||||
}
|
||||
if !validatorsMatch(expected, EntityValidator{}) {
|
||||
t.Fatal("absent response validator should not contradict If-Range")
|
||||
}
|
||||
}
|
||||
|
||||
func validTaskForTest() Task {
|
||||
return Task{
|
||||
SchemaVersion: TaskSchemaVersion,
|
||||
RequestID: "request-001",
|
||||
AppID: "json-parser",
|
||||
URL: "https://download.invalid/json-parser.zip",
|
||||
Status: StatusQueued,
|
||||
TotalKnown: true,
|
||||
Total: 42,
|
||||
CreatedAt: "2026-07-16T00:00:00Z",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (queue *Queue) transfer(
|
||||
runtime *runtimeTask,
|
||||
ctx context.Context,
|
||||
attempt uint64,
|
||||
) (completedPath string, currentAttempt uint64, transferErr error) {
|
||||
currentAttempt = attempt
|
||||
task, valid := queue.taskForAttempt(runtime, currentAttempt)
|
||||
if !valid {
|
||||
return "", currentAttempt, context.Canceled
|
||||
}
|
||||
paths, err := ensureTransferLayout(queue.root, task.RequestID)
|
||||
if err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
offset, _, err := regularFileSize(paths.Part)
|
||||
if err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
|
||||
response, err := queue.transport.Open(ctx, OpenRequest{
|
||||
URL: task.URL,
|
||||
Offset: offset,
|
||||
Validator: task.Validator,
|
||||
})
|
||||
if err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
body := response.Body
|
||||
defer func() {
|
||||
if body != nil {
|
||||
_ = body.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
if task.TotalKnown && response.TotalKnown && response.Total != task.Total {
|
||||
return "", currentAttempt, fmt.Errorf(
|
||||
"%w: response total %d, expected %d",
|
||||
ErrTransferIncomplete,
|
||||
response.Total,
|
||||
task.Total,
|
||||
)
|
||||
}
|
||||
if !task.TotalKnown && response.TotalKnown && response.Total > queue.maxUnknown {
|
||||
return "", currentAttempt, ErrTransferTooLarge
|
||||
}
|
||||
if !task.TotalKnown && offset > 0 && !response.Restart && !response.TotalKnown {
|
||||
return "", currentAttempt, fmt.Errorf(
|
||||
"%w: resumed response has unknown complete length",
|
||||
ErrTransferIncomplete,
|
||||
)
|
||||
}
|
||||
|
||||
var part *os.File
|
||||
if response.Restart {
|
||||
part, err = openPart(paths, offset, true)
|
||||
if err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
if err := part.Sync(); err != nil {
|
||||
_ = part.Close()
|
||||
return "", currentAttempt, fmt.Errorf("sync restarted part: %w", err)
|
||||
}
|
||||
currentAttempt, task, err = queue.restartAttempt(
|
||||
runtime,
|
||||
currentAttempt,
|
||||
response,
|
||||
)
|
||||
if err != nil {
|
||||
_ = part.Close()
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
offset = 0
|
||||
} else {
|
||||
task, err = queue.applyResponseFacts(runtime, currentAttempt, response)
|
||||
if err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
}
|
||||
|
||||
var maximum int64
|
||||
var expectedKnown bool
|
||||
var expected int64
|
||||
if task.TotalKnown {
|
||||
if offset > task.Total {
|
||||
return "", currentAttempt, fmt.Errorf(
|
||||
"%w: offset exceeds total",
|
||||
ErrTaskCorrupt,
|
||||
)
|
||||
}
|
||||
maximum = task.Total - offset
|
||||
expectedKnown = true
|
||||
expected = maximum
|
||||
} else {
|
||||
if offset >= queue.maxUnknown {
|
||||
return "", currentAttempt, ErrTransferTooLarge
|
||||
}
|
||||
maximum = queue.maxUnknown - offset
|
||||
if response.ResponseLengthKnown {
|
||||
expectedKnown = true
|
||||
expected = response.ResponseLength
|
||||
}
|
||||
}
|
||||
if !task.TotalKnown && response.TotalKnown {
|
||||
if response.Total > queue.maxUnknown ||
|
||||
offset > response.Total ||
|
||||
!response.ResponseLengthKnown ||
|
||||
response.ResponseLength != response.Total-offset {
|
||||
if part != nil {
|
||||
_ = part.Close()
|
||||
}
|
||||
return "", currentAttempt, ErrTransferIncomplete
|
||||
}
|
||||
}
|
||||
if response.ResponseLengthKnown && response.ResponseLength > maximum {
|
||||
if part != nil {
|
||||
_ = part.Close()
|
||||
}
|
||||
return "", currentAttempt, ErrTransferTooLarge
|
||||
}
|
||||
if expectedKnown &&
|
||||
response.ResponseLengthKnown &&
|
||||
response.ResponseLength != expected {
|
||||
if part != nil {
|
||||
_ = part.Close()
|
||||
}
|
||||
return "", currentAttempt, ErrTransferIncomplete
|
||||
}
|
||||
|
||||
if part == nil {
|
||||
part, err = openPart(paths, offset, false)
|
||||
if err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
}
|
||||
lastEventAt := queue.clock.Now()
|
||||
lastEventDone := offset
|
||||
lastPublishedDone := offset
|
||||
copyOptions := CopyOptions{
|
||||
MaxBytes: maximum,
|
||||
ExpectedBytesKnown: expectedKnown,
|
||||
ExpectedBytes: expected,
|
||||
Generation: currentAttempt,
|
||||
Progress: func(progress CopyProgress) error {
|
||||
absoluteDone := offset + progress.Written
|
||||
now := queue.clock.Now()
|
||||
emit := queue.progressEvery == 0 ||
|
||||
now.Sub(lastEventAt) >= queue.progressEvery
|
||||
if err := queue.updateAttemptDone(
|
||||
runtime,
|
||||
currentAttempt,
|
||||
absoluteDone,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
if !emit {
|
||||
return nil
|
||||
}
|
||||
speed := progressSpeed(lastEventAt, now, lastEventDone, absoluteDone)
|
||||
if err := queue.persistAndPublishProgress(
|
||||
runtime,
|
||||
currentAttempt,
|
||||
speed,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
lastEventAt = now
|
||||
lastEventDone = absoluteDone
|
||||
lastPublishedDone = absoluteDone
|
||||
return nil
|
||||
},
|
||||
}
|
||||
written, copyErr := CopyResponse(ctx, part, body, copyOptions)
|
||||
writtenInfo, statErr := part.Stat()
|
||||
bodyCloseErr := body.Close()
|
||||
body = nil
|
||||
syncErr := syncAndClose(part)
|
||||
if syncErr != nil {
|
||||
return "", currentAttempt, syncErr
|
||||
}
|
||||
if bodyCloseErr != nil {
|
||||
return "", currentAttempt, bodyCloseErr
|
||||
}
|
||||
if statErr != nil {
|
||||
return "", currentAttempt, fmt.Errorf("stat written part: %w", statErr)
|
||||
}
|
||||
if err := verifyRegularIdentity(paths.Part, writtenInfo); err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
if copyErr != nil {
|
||||
return "", currentAttempt, copyErr
|
||||
}
|
||||
|
||||
finalDone := offset + written
|
||||
if err := queue.updateAttemptDone(runtime, currentAttempt, finalDone); err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
if finalDone != lastPublishedDone {
|
||||
now := queue.clock.Now()
|
||||
speed := progressSpeed(lastEventAt, now, lastEventDone, finalDone)
|
||||
if err := queue.persistAndPublishProgress(
|
||||
runtime,
|
||||
currentAttempt,
|
||||
speed,
|
||||
); err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
}
|
||||
if task.TotalKnown && finalDone != task.Total {
|
||||
return "", currentAttempt, ErrTransferIncomplete
|
||||
}
|
||||
if err := activateCompleted(paths, writtenInfo); err != nil {
|
||||
return "", currentAttempt, err
|
||||
}
|
||||
return paths.Completed, currentAttempt, nil
|
||||
}
|
||||
|
||||
func (queue *Queue) restartAttempt(
|
||||
runtime *runtimeTask,
|
||||
attempt uint64,
|
||||
response OpenResponse,
|
||||
) (uint64, Task, error) {
|
||||
queue.mu.Lock()
|
||||
if !queue.attemptCurrent(runtime, attempt) || runtime.stop != stopNone {
|
||||
queue.mu.Unlock()
|
||||
return attempt, Task{}, context.Canceled
|
||||
}
|
||||
runtime.task.Attempt++
|
||||
runtime.task.Done = 0
|
||||
runtime.task.Validator = response.Validator
|
||||
task := runtime.task
|
||||
newAttempt := task.Attempt
|
||||
queue.mu.Unlock()
|
||||
if err := queue.store.Write(task); err != nil {
|
||||
return newAttempt, task, err
|
||||
}
|
||||
queue.publishBestEffort(Event{
|
||||
Type: EventStarted,
|
||||
RequestID: task.RequestID,
|
||||
AppID: task.AppID,
|
||||
Attempt: task.Attempt,
|
||||
Done: 0,
|
||||
TotalKnown: task.TotalKnown,
|
||||
Total: task.Total,
|
||||
})
|
||||
return newAttempt, task, nil
|
||||
}
|
||||
|
||||
func (queue *Queue) applyResponseFacts(
|
||||
runtime *runtimeTask,
|
||||
attempt uint64,
|
||||
response OpenResponse,
|
||||
) (Task, error) {
|
||||
queue.mu.Lock()
|
||||
if !queue.attemptCurrent(runtime, attempt) || runtime.stop != stopNone {
|
||||
queue.mu.Unlock()
|
||||
return Task{}, context.Canceled
|
||||
}
|
||||
if runtime.task.Validator.Empty() {
|
||||
runtime.task.Validator = response.Validator
|
||||
}
|
||||
task := runtime.task
|
||||
queue.mu.Unlock()
|
||||
if err := queue.store.Write(task); err != nil {
|
||||
return Task{}, err
|
||||
}
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func (queue *Queue) updateAttemptDone(
|
||||
runtime *runtimeTask,
|
||||
attempt uint64,
|
||||
done int64,
|
||||
) error {
|
||||
queue.mu.Lock()
|
||||
defer queue.mu.Unlock()
|
||||
if !queue.attemptCurrent(runtime, attempt) ||
|
||||
runtime.stop != stopNone ||
|
||||
done < runtime.task.Done {
|
||||
return context.Canceled
|
||||
}
|
||||
if runtime.task.TotalKnown && done > runtime.task.Total {
|
||||
return ErrTransferTooLarge
|
||||
}
|
||||
runtime.task.Done = done
|
||||
return nil
|
||||
}
|
||||
|
||||
func (queue *Queue) persistAndPublishProgress(
|
||||
runtime *runtimeTask,
|
||||
attempt uint64,
|
||||
speed int64,
|
||||
) error {
|
||||
queue.mu.Lock()
|
||||
if !queue.attemptCurrent(runtime, attempt) || runtime.stop != stopNone {
|
||||
queue.mu.Unlock()
|
||||
return context.Canceled
|
||||
}
|
||||
task := runtime.task
|
||||
queue.mu.Unlock()
|
||||
if err := queue.store.Write(task); err != nil {
|
||||
return err
|
||||
}
|
||||
queue.publishBestEffort(Event{
|
||||
Type: EventProgress,
|
||||
RequestID: task.RequestID,
|
||||
AppID: task.AppID,
|
||||
Attempt: task.Attempt,
|
||||
Done: task.Done,
|
||||
TotalKnown: task.TotalKnown,
|
||||
Total: task.Total,
|
||||
SpeedBytesSec: speed,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func progressSpeed(start, end time.Time, startDone, endDone int64) int64 {
|
||||
elapsed := end.Sub(start)
|
||||
if elapsed <= 0 || endDone <= startDone {
|
||||
return 0
|
||||
}
|
||||
bytesPerSecond := float64(endDone-startDone) / elapsed.Seconds()
|
||||
if bytesPerSecond <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int64(bytesPerSecond)
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"softbox.local/core/downloader"
|
||||
)
|
||||
|
||||
const (
|
||||
maxDownloadTaskDocumentBytes = 64 * 1024
|
||||
maxDownloadTaskCount = 4096
|
||||
)
|
||||
|
||||
var ErrDownloadTaskLayoutUnsafe = errors.New("download task storage layout is unsafe")
|
||||
|
||||
// DownloadTaskStore atomically persists download metadata below
|
||||
// downloads/tasks. Transfer bytes live separately below downloads/files.
|
||||
type DownloadTaskStore struct {
|
||||
root string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewDownloadTaskStore creates a strict JSON task store.
|
||||
func NewDownloadTaskStore(downloadsRoot string) *DownloadTaskStore {
|
||||
return &DownloadTaskStore{root: downloadsRoot}
|
||||
}
|
||||
|
||||
// Write validates and atomically replaces one task document.
|
||||
func (store *DownloadTaskStore) Write(task downloader.Task) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
if err := task.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
paths, err := store.ensureLayout(task.RequestID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
document, err := json.Marshal(task)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode download task: %w", err)
|
||||
}
|
||||
document = append(document, '\n')
|
||||
if len(document) > maxDownloadTaskDocumentBytes {
|
||||
return fmt.Errorf("%w: metadata is too large", downloader.ErrInvalidTask)
|
||||
}
|
||||
return replaceDownloadTaskFile(paths.TasksDir, paths.Metadata, paths.Backup, document)
|
||||
}
|
||||
|
||||
// LoadAll reads every task, falling back to a backup only when its main file
|
||||
// is absent.
|
||||
func (store *DownloadTaskStore) LoadAll() ([]downloader.Task, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
paths, err := store.ensureLayout("layout-probe")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(paths.TasksDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read download task directory: %w", err)
|
||||
}
|
||||
requestIDs := make(map[string]struct{})
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if strings.HasPrefix(name, ".download-task-") && strings.HasSuffix(name, ".tmp") {
|
||||
continue
|
||||
}
|
||||
requestID, recognized := metadataRequestID(name)
|
||||
if !recognized || !downloader.ValidRequestID(requestID) {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: unexpected metadata entry %q",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
name,
|
||||
)
|
||||
}
|
||||
info, infoErr := entry.Info()
|
||||
if infoErr != nil {
|
||||
return nil, fmt.Errorf("inspect download task entry: %w", infoErr)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: metadata entry is not regular",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
)
|
||||
}
|
||||
requestIDs[requestID] = struct{}{}
|
||||
}
|
||||
if len(requestIDs) > maxDownloadTaskCount {
|
||||
return nil, fmt.Errorf("%w: too many tasks", downloader.ErrInvalidTask)
|
||||
}
|
||||
sortedIDs := make([]string, 0, len(requestIDs))
|
||||
for requestID := range requestIDs {
|
||||
sortedIDs = append(sortedIDs, requestID)
|
||||
}
|
||||
sort.Strings(sortedIDs)
|
||||
|
||||
tasks := make([]downloader.Task, 0, len(sortedIDs))
|
||||
for _, requestID := range sortedIDs {
|
||||
taskPaths, deriveErr := downloader.DeriveTaskPaths(store.root, requestID)
|
||||
if deriveErr != nil {
|
||||
return nil, deriveErr
|
||||
}
|
||||
document, readErr := readBoundedRegularFile(
|
||||
taskPaths.Metadata,
|
||||
maxDownloadTaskDocumentBytes,
|
||||
)
|
||||
if os.IsNotExist(readErr) {
|
||||
document, readErr = readBoundedRegularFile(
|
||||
taskPaths.Backup,
|
||||
maxDownloadTaskDocumentBytes,
|
||||
)
|
||||
}
|
||||
if readErr != nil {
|
||||
return nil, readErr
|
||||
}
|
||||
task, decodeErr := decodeDownloadTask(document)
|
||||
if decodeErr != nil {
|
||||
return nil, decodeErr
|
||||
}
|
||||
if task.RequestID != requestID {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: request_id does not match metadata filename",
|
||||
downloader.ErrInvalidTask,
|
||||
)
|
||||
}
|
||||
tasks = append(tasks, task)
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// Delete removes the exact metadata and backup files for a request.
|
||||
func (store *DownloadTaskStore) Delete(requestID string) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
paths, err := downloader.DeriveTaskPaths(store.root, requestID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, candidate := range []string{paths.Metadata, paths.Backup} {
|
||||
info, inspectErr := os.Lstat(candidate)
|
||||
if os.IsNotExist(inspectErr) {
|
||||
continue
|
||||
}
|
||||
if inspectErr != nil {
|
||||
return fmt.Errorf("inspect download task metadata: %w", inspectErr)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf(
|
||||
"%w: metadata is not a regular file",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
)
|
||||
}
|
||||
if removeErr := os.Remove(candidate); removeErr != nil {
|
||||
return fmt.Errorf("remove download task metadata: %w", removeErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *DownloadTaskStore) ensureLayout(requestID string) (downloader.TaskPaths, error) {
|
||||
paths, err := downloader.DeriveTaskPaths(store.root, requestID)
|
||||
if err != nil {
|
||||
return downloader.TaskPaths{}, err
|
||||
}
|
||||
if err := os.MkdirAll(paths.Root, 0o700); err != nil {
|
||||
return downloader.TaskPaths{}, fmt.Errorf("create downloads root: %w", err)
|
||||
}
|
||||
for _, directory := range []string{paths.Root, paths.TasksDir, paths.FilesDir} {
|
||||
if err := os.Mkdir(directory, 0o700); err != nil && !os.IsExist(err) {
|
||||
return downloader.TaskPaths{}, fmt.Errorf("create download directory: %w", err)
|
||||
}
|
||||
info, inspectErr := os.Lstat(directory)
|
||||
if inspectErr != nil {
|
||||
return downloader.TaskPaths{}, fmt.Errorf("inspect download directory: %w", inspectErr)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return downloader.TaskPaths{}, fmt.Errorf(
|
||||
"%w: managed path is not a real directory",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func metadataRequestID(name string) (string, bool) {
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".json.backup"):
|
||||
return strings.TrimSuffix(name, ".json.backup"), true
|
||||
case strings.HasSuffix(name, ".json"):
|
||||
return strings.TrimSuffix(name, ".json"), true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func decodeDownloadTask(document []byte) (downloader.Task, error) {
|
||||
var task downloader.Task
|
||||
decoder := json.NewDecoder(bytes.NewReader(document))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&task); err != nil {
|
||||
return downloader.Task{}, fmt.Errorf("%w: decode: %v", downloader.ErrInvalidTask, err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
if err == nil {
|
||||
return downloader.Task{}, fmt.Errorf(
|
||||
"%w: trailing JSON value",
|
||||
downloader.ErrInvalidTask,
|
||||
)
|
||||
}
|
||||
return downloader.Task{}, fmt.Errorf(
|
||||
"%w: trailing data: %v",
|
||||
downloader.ErrInvalidTask,
|
||||
err,
|
||||
)
|
||||
}
|
||||
if err := task.Validate(); err != nil {
|
||||
return downloader.Task{}, err
|
||||
}
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func readBoundedRegularFile(filePath string, maximum int64) ([]byte, error) {
|
||||
info, err := os.Lstat(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: metadata is not a regular file",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
)
|
||||
}
|
||||
if info.Size() > maximum {
|
||||
return nil, fmt.Errorf("%w: metadata is too large", downloader.ErrInvalidTask)
|
||||
}
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open download task metadata: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
document, err := io.ReadAll(io.LimitReader(file, maximum+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read download task metadata: %w", err)
|
||||
}
|
||||
if int64(len(document)) > maximum {
|
||||
return nil, fmt.Errorf("%w: metadata is too large", downloader.ErrInvalidTask)
|
||||
}
|
||||
return document, nil
|
||||
}
|
||||
|
||||
func replaceDownloadTaskFile(directory, target, backup string, document []byte) error {
|
||||
temporary, err := os.CreateTemp(directory, ".download-task-*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create download task temp file: %w", err)
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer os.Remove(temporaryPath)
|
||||
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
temporary.Close()
|
||||
return fmt.Errorf("protect download task temp file: %w", err)
|
||||
}
|
||||
if _, err := temporary.Write(document); err != nil {
|
||||
temporary.Close()
|
||||
return fmt.Errorf("write download task temp file: %w", err)
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
temporary.Close()
|
||||
return fmt.Errorf("sync download task temp file: %w", err)
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return fmt.Errorf("close download task temp file: %w", err)
|
||||
}
|
||||
|
||||
movedTarget := false
|
||||
hadBackup := false
|
||||
if info, inspectErr := os.Lstat(target); inspectErr == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf(
|
||||
"%w: metadata target is not regular",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
)
|
||||
}
|
||||
if err := removeExactRegularFile(backup); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(target, backup); err != nil {
|
||||
return fmt.Errorf("backup download task metadata: %w", err)
|
||||
}
|
||||
movedTarget = true
|
||||
} else if !os.IsNotExist(inspectErr) {
|
||||
return fmt.Errorf("inspect download task metadata: %w", inspectErr)
|
||||
} else {
|
||||
backupInfo, backupErr := os.Lstat(backup)
|
||||
switch {
|
||||
case backupErr == nil:
|
||||
if backupInfo.Mode()&os.ModeSymlink != 0 || !backupInfo.Mode().IsRegular() {
|
||||
return fmt.Errorf(
|
||||
"%w: metadata backup is not regular",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
)
|
||||
}
|
||||
hadBackup = true
|
||||
case !os.IsNotExist(backupErr):
|
||||
return fmt.Errorf("inspect download task backup: %w", backupErr)
|
||||
}
|
||||
}
|
||||
if err := os.Rename(temporaryPath, target); err != nil {
|
||||
if movedTarget {
|
||||
_ = os.Rename(backup, target)
|
||||
}
|
||||
return fmt.Errorf("activate download task metadata: %w", err)
|
||||
}
|
||||
if movedTarget || hadBackup {
|
||||
if err := removeExactRegularFile(backup); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeExactRegularFile(filePath string) error {
|
||||
info, err := os.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect download task backup: %w", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf(
|
||||
"%w: metadata backup is not regular",
|
||||
ErrDownloadTaskLayoutUnsafe,
|
||||
)
|
||||
}
|
||||
if err := os.Remove(filePath); err != nil {
|
||||
return fmt.Errorf("remove download task backup: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDownloadTaskSchemaIsValidJSONObject(t *testing.T) {
|
||||
document, err := os.ReadFile(filepath.Join(
|
||||
"..",
|
||||
"..",
|
||||
"schemas",
|
||||
"download-task.schema.json",
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("read schema: %v", err)
|
||||
}
|
||||
var schema map[string]any
|
||||
if err := json.Unmarshal(document, &schema); err != nil {
|
||||
t.Fatalf("decode schema: %v", err)
|
||||
}
|
||||
if schema["$schema"] != "https://json-schema.org/draft/2020-12/schema" {
|
||||
t.Fatalf("$schema = %v", schema["$schema"])
|
||||
}
|
||||
if schema["type"] != "object" {
|
||||
t.Fatalf("type = %v", schema["type"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"softbox.local/core/downloader"
|
||||
)
|
||||
|
||||
func TestDownloadTaskStoreWriteLoadAndBackupFallback(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "nested", "downloads")
|
||||
store := NewDownloadTaskStore(root)
|
||||
task := validDownloadTask()
|
||||
if err := store.Write(task); err != nil {
|
||||
t.Fatalf("Write() error = %v", err)
|
||||
}
|
||||
loaded, err := store.LoadAll()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAll() error = %v", err)
|
||||
}
|
||||
if len(loaded) != 1 || loaded[0].RequestID != task.RequestID {
|
||||
t.Fatalf("loaded = %#v", loaded)
|
||||
}
|
||||
|
||||
task.Status = downloader.StatusPaused
|
||||
task.Done = 10
|
||||
task.Validator = downloader.EntityValidator{ETag: `"v1"`}
|
||||
if err := store.Write(task); err != nil {
|
||||
t.Fatalf("Write(update) error = %v", err)
|
||||
}
|
||||
paths, err := downloader.DeriveTaskPaths(root, task.RequestID)
|
||||
if err != nil {
|
||||
t.Fatalf("DeriveTaskPaths() error = %v", err)
|
||||
}
|
||||
if err := os.Rename(paths.Metadata, paths.Backup); err != nil {
|
||||
t.Fatalf("simulate interrupted replacement: %v", err)
|
||||
}
|
||||
loaded, err = store.LoadAll()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAll(backup) error = %v", err)
|
||||
}
|
||||
if len(loaded) != 1 || loaded[0].Status != downloader.StatusPaused ||
|
||||
loaded[0].Done != 10 {
|
||||
t.Fatalf("backup loaded = %#v", loaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadTaskStoreRejectsUnknownAndInvalidData(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
store := NewDownloadTaskStore(root)
|
||||
task := validDownloadTask()
|
||||
if err := store.Write(task); err != nil {
|
||||
t.Fatalf("Write() error = %v", err)
|
||||
}
|
||||
paths, _ := downloader.DeriveTaskPaths(root, task.RequestID)
|
||||
|
||||
encoded, err := json.Marshal(task)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
var object map[string]any
|
||||
if err := json.Unmarshal(encoded, &object); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v", err)
|
||||
}
|
||||
object["unknown"] = true
|
||||
encoded, err = json.Marshal(object)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(object) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(paths.Metadata, encoded, 0o600); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if _, err := store.LoadAll(); !errors.Is(err, downloader.ErrInvalidTask) {
|
||||
t.Fatalf("LoadAll(unknown) error = %v, want %v", err, downloader.ErrInvalidTask)
|
||||
}
|
||||
|
||||
task = validDownloadTask()
|
||||
task.Done = task.Total + 1
|
||||
if err := store.Write(task); !errors.Is(err, downloader.ErrInvalidTask) {
|
||||
t.Fatalf("Write(invalid) error = %v, want %v", err, downloader.ErrInvalidTask)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadTaskStoreRejectsUnexpectedAndUnsafeEntries(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
store := NewDownloadTaskStore(root)
|
||||
task := validDownloadTask()
|
||||
if err := store.Write(task); err != nil {
|
||||
t.Fatalf("Write() error = %v", err)
|
||||
}
|
||||
paths, _ := downloader.DeriveTaskPaths(root, task.RequestID)
|
||||
if err := os.WriteFile(filepath.Join(paths.TasksDir, "unexpected.txt"), []byte("x"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(unexpected) error = %v", err)
|
||||
}
|
||||
if _, err := store.LoadAll(); !errors.Is(err, ErrDownloadTaskLayoutUnsafe) {
|
||||
t.Fatalf("LoadAll(unexpected) error = %v, want %v", err, ErrDownloadTaskLayoutUnsafe)
|
||||
}
|
||||
|
||||
if err := os.Remove(filepath.Join(paths.TasksDir, "unexpected.txt")); err != nil {
|
||||
t.Fatalf("Remove(unexpected) error = %v", err)
|
||||
}
|
||||
if err := os.Remove(paths.Metadata); err != nil {
|
||||
t.Fatalf("Remove(metadata) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join(root, "outside.json"), paths.Metadata); err != nil {
|
||||
t.Skipf("symlink unavailable: %v", err)
|
||||
}
|
||||
if _, err := store.LoadAll(); !errors.Is(err, ErrDownloadTaskLayoutUnsafe) {
|
||||
t.Fatalf("LoadAll(symlink) error = %v, want %v", err, ErrDownloadTaskLayoutUnsafe)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadTaskStoreDeleteIsExact(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
store := NewDownloadTaskStore(root)
|
||||
task := validDownloadTask()
|
||||
if err := store.Write(task); err != nil {
|
||||
t.Fatalf("Write() error = %v", err)
|
||||
}
|
||||
paths, _ := downloader.DeriveTaskPaths(root, task.RequestID)
|
||||
if err := store.Delete(task.RequestID); err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(paths.Metadata); !os.IsNotExist(err) {
|
||||
t.Fatalf("metadata still exists: %v", err)
|
||||
}
|
||||
if info, err := os.Stat(paths.FilesDir); err != nil || !info.IsDir() {
|
||||
t.Fatalf("files directory was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func validDownloadTask() downloader.Task {
|
||||
return downloader.Task{
|
||||
SchemaVersion: downloader.TaskSchemaVersion,
|
||||
RequestID: "request-001",
|
||||
AppID: "json-parser",
|
||||
URL: "https://download.invalid/json-parser.zip",
|
||||
Status: downloader.StatusQueued,
|
||||
TotalKnown: true,
|
||||
Total: 42,
|
||||
CreatedAt: "2026-07-16T00:00:00Z",
|
||||
}
|
||||
}
|
||||
@@ -142,6 +142,17 @@ T-202 将本地识别拆为两层:
|
||||
|
||||
磁盘扫描结果必须在进入 Gio Layout 前准备好;UI 不直接读取 installed-app.json。完整字段见 [api.md](api.md),Schema 为 `schemas/installed-app.schema.json`。
|
||||
|
||||
T-301 下载队列:
|
||||
|
||||
- `core/downloader.Queue` 默认并发 2,用 request_id 主键和 app 非终态唯一约束串行化 enqueue/pause/resume/cancel/retry;网络/磁盘 IO 与事件发布都在全局锁外。
|
||||
- `.part` / `.download` / metadata 路径只由受限 request_id 派生。元数据通过 `core/storage.DownloadTaskStore` 临时文件 + backup 原子替换。
|
||||
- Range 必须绑定 strong ETag 或 Last-Modified 并发送 If-Range;无 validator 从 0 重下,`200` 忽略 Range 时创建新 attempt。进度只在 request_id+attempt 内单调。
|
||||
- pause/cancel/close 先取消 generation context,等待 body 关闭、part sync/close 和旧 worker 完全退出,再发布终态/释放并发槽,阻断迟到 progress。
|
||||
- cancel 一旦先于 completion 取得线性化点,即使 body close/sync 报错也记录后继续精确清理;若 completion 先完成,后续 cancel 明确拒绝。known-total 完整 part 在同进程 resume/retry 与启动恢复中都直接 finalize,不发送 offset==total 的 Range。
|
||||
- 写入句柄的文件身份贯穿 sync/close 与 rename 前后核对,防止活跃 `.part` 路径被替换后发布错误文件。崩溃恢复对账 metadata、part、final 三份事实:完整 part 可 finalize,已 rename 的 final 可补 completed metadata;缺 final 的 completed、part+final、超出 expected/unknown 上限均 fail closed。
|
||||
- 事件投递失败通过 `OnObserverError` 显式报告,不改变 durable transfer 结果;application/UI 启动或重连后用 `Queue.Tasks()` 对账终态,避免 DownloadCompleted 等一次性通知丢失后永久停链。
|
||||
- DownloadCompleted 仅证明传输字节完整落盘。下载元数据和 `.download` 可被本地篡改,T-302 不得把它们当信任根,仍需从已验签 Catalog 重新取得并核对身份/size/hash/signature。
|
||||
|
||||
### 4.3 事件模型
|
||||
|
||||
后台任务只发布事件(`DownloadStarted / DownloadProgress / DownloadPaused / DownloadCompleted / DownloadFailed` 等),UI 按 RequestID 和软件 ID 回填,见 [api.md](api.md) 事件合约。
|
||||
|
||||
+42
-5
@@ -200,6 +200,42 @@ T-102 Phase 1 原型进一步固定:
|
||||
|
||||
phase 只允许:`prepared`、`current_backed_up`、`staging_activated`、`rollback_required`、`committed`。日志使用临时文件 + 同目录 backup 原子替换;恢复时同时检查日志与 current/staging/backup 实际状态。未完成健康检查的 current 不视为可信:有旧版时恢复 backup,首次安装则撤销 current。
|
||||
|
||||
### 2.6 下载任务元数据 download-task.json(本地)
|
||||
|
||||
每个任务以稳定 `request_id` 为主键,元数据位于 `downloads/tasks/<request_id>.json`,字节文件位于 `downloads/files/<request_id>.part|.download`。本地路径只由客户端从 request_id 派生,不接受 URL 或 Content-Disposition 提供的文件名。
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"request_id": "download-json-parser-001",
|
||||
"app_id": "json-parser",
|
||||
"url": "https://download.example.com/json-parser.zip",
|
||||
"status": "paused",
|
||||
"attempt": 2,
|
||||
"done": 1048576,
|
||||
"total_known": true,
|
||||
"total": 12345678,
|
||||
"validator": {
|
||||
"etag": "\"release-1.4.2\""
|
||||
},
|
||||
"error_code": "",
|
||||
"created_at": "2026-07-16T00:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
- Schema 为 `schemas/download-task.schema.json`;内部状态只允许 queued/downloading/paused/failed/completed。paused 对 UI 映射为 queued + DownloadPaused 事件,不增加第 13 个 AppStatus。
|
||||
- 默认最多 2 个 downloading;同一 request_id 幂等,身份/URL/size 不同则冲突;同一 app 同时只允许一个非 completed 任务。
|
||||
- 暂停保留 `.part` 和元数据;失败 retry 保留 `.part`;取消固定删除该 request_id 精确派生的 part/final/metadata,不使用递归删除。取消已线性化后,body close/sync 等非致命错误只记录日志,不能把取消反转成 failed 或跳过清理。
|
||||
- 续传只在存在 strong ETag 或合法 Last-Modified 时发送 Range + If-Range;无可靠 validator 时从 0 重下。请求固定 `Accept-Encoding: identity`,所有 redirect 重新校验 HTTPS。
|
||||
- `206` 必须严格匹配 Content-Range 起点、长度、total 与期望 size;已有 part 但服务端返回 `200` 时截断并创建新 attempt,不能 old+new 拼接。
|
||||
- known total 使用已验签 Catalog package.size 作为精确传输长度;unknown total 使用显式 `total_known: false` 并受 4 GiB 默认硬上限保护。
|
||||
- known total 的 `.part` 已达到精确 total 时,同进程 resume/retry 与重启恢复都直接 sync、核对身份并 finalize,不得再请求 `Range: bytes=<total>-`。
|
||||
- 完成顺序为 part sync/close → 核对实际写入文件身份 → rename `.download` → metadata completed → DownloadCompleted。rename 前后必须仍是同一普通文件;恢复时以普通文件实际长度对账,不信任 metadata.done。
|
||||
- 事件投递是 best-effort,失败不得反向把已完成/已取消的持久任务改成失败。队列必须把投递错误交给 `OnObserverError` 记录;application/UI 在启动或事件通道重连后必须用 `Queue.Tasks()` 对账持久状态,不能只依赖某一次终态事件。
|
||||
- `.download` 仍是**不可信隔离字节**。T-302 必须重新从已验签 Catalog 取得 app/version/arch/size/SHA-256/signature 并验证,通过后才可读取 app.json 或解压。
|
||||
|
||||
## 3. 许可证(服务端签发 → 本地离线验证)
|
||||
|
||||
```json
|
||||
@@ -229,11 +265,12 @@ phase 只允许:`prepared`、`current_backed_up`、`staging_activated`、`rollba
|
||||
| --- | --- | --- | --- |
|
||||
| CatalogRefreshed | 清单验签并缓存成功 | catalog 摘要、generated_at | 列表刷新 |
|
||||
| CatalogRejected | 清单验签失败 | 原因码 | 提示 + 继续用缓存 |
|
||||
| DownloadStarted | 下载任务开始 | request_id, app_id | 状态 → downloading |
|
||||
| DownloadProgress | 进度更新 | request_id, app_id, done, total, speed | 进度条刷新 |
|
||||
| DownloadPaused | 用户暂停 | request_id, app_id | 状态 → queued(暂停态) |
|
||||
| DownloadCompleted | 下载并校验通过 | request_id, app_id | 状态 → verifying/extracting |
|
||||
| DownloadFailed | 失败(网络/哈希/磁盘) | request_id, app_id, error_code | 状态 → failed + 可重试 |
|
||||
| DownloadStarted | 一个下载 attempt 开始/因安全重下而重置 | request_id, app_id, attempt, done, total_known, total | 状态 → downloading |
|
||||
| DownloadProgress | 当前 attempt 进度更新 | request_id, app_id, attempt, done, total_known, total, speed | 进度条刷新;done 在同 attempt 内单调 |
|
||||
| DownloadPaused | 用户暂停且旧 worker 已完全退出 | request_id, app_id, attempt, done | 状态 → queued(暂停态) |
|
||||
| DownloadCompleted | 字节完整落盘,尚未通过 SHA/签名 | request_id, app_id, attempt, done, path | 状态 → verifying;T-302 接管信任校验 |
|
||||
| DownloadFailed | 传输/Range/存储失败 | request_id, app_id, attempt, done, error_code | 状态 → failed + 可重试 |
|
||||
| DownloadCanceled | 取消清理完成且旧 attempt 不再发事件 | request_id, app_id, attempt | 从任务视图移除并回到本地基础状态 |
|
||||
| InstallCompleted | 原子切换成功 + 健康检查通过 | app_id, version | 状态 → installed |
|
||||
| InstallRolledBack | 切换失败恢复 backup | app_id, error_code | 状态 → rollback 完成提示 |
|
||||
| AppStarted / AppExited | 进程启动/退出检测 | app_id, pid | 状态 → running / installed |
|
||||
|
||||
+12
-12
@@ -13,36 +13,36 @@
|
||||
## 当前快照
|
||||
|
||||
- 日期:2026-07-16
|
||||
- 阶段:Phase 2 已完成(T-201~T-204);T-301 下载队列已落成并领取,准备按单写入者 + 多只读审查者模式执行
|
||||
- 阶段:Phase 2 已完成(T-201~T-204);Phase 3 的 T-301 可恢复下载队列已完成,T-302 尚待正式落成任务文件
|
||||
- 技术栈:根 Go workspace 纳入 core/app-modern/app-win7 三模块;`app-win7/go.work` 隔离 Go 1.20.14 构建;modern Gio v0.10.1 与 win7 Gio v0.6.0 已实际接入
|
||||
- 生产代码:core 已有 Catalog/本地状态/存储、无 IO 软件列表模型与按 digest+DPI 的可信图标缓存;modern/win7 AppShell 已实现搜索/分类/视图、惰性列表、详情右栏与内存图标
|
||||
- 测试:core 覆盖 Catalog、SemVer/12 状态、本地安装记录、列表筛选和图标内存/磁盘/离线/损坏恢复;两个 app 覆盖 500 项虚拟列表、ID 控件稳定性、详情/ApplyIcon 与平台 stub;ZIP/安装恢复矩阵保持通过
|
||||
- 数据:`schemas/` 已有 manifest/app.json/installed-app.json v1 Schema;`testdata/catalog/` 有公开虚构清单样例;`testdata/zip/` 记录运行时生成的 ZIP 攻击矩阵
|
||||
- 生产代码:core 已有 Catalog/本地状态/存储、无 IO 软件列表模型、可信图标缓存和默认并发 2 的持久可恢复下载队列;modern/win7 AppShell 已实现搜索/分类/视图、惰性列表、详情右栏与内存图标
|
||||
- 测试:core 覆盖 Catalog、SemVer/12 状态、本地安装记录、列表/图标、下载并发/暂停/取消/重试/Range/断连/恢复/事件失败与文件身份替换;两个 app 覆盖 500 项虚拟列表、ID 控件稳定性、详情/ApplyIcon 与平台 stub;ZIP/安装恢复矩阵保持通过
|
||||
- 数据:`schemas/` 已有 manifest/app.json/installed-app.json/download-task.json v1 Schema;`testdata/catalog/` 有公开虚构清单样例;`testdata/zip/` 与 `testdata/download/` 记录运行时生成的攻击/传输矩阵
|
||||
- 标准启动路径:`./init.sh` / `./init.ps1`(同步依赖、执行完整 Phase 0 闸门、打印双目标构建命令)
|
||||
- 标准验证路径:`bash scripts/verify_phase0.sh` / `./scripts/verify_phase0.ps1`
|
||||
- 版本管理:git 已初始化,main 分支,远端 origin 为 Gitea `opc/soft_quay`;harness 文档已提交
|
||||
- 当前 blocker:无;当前任务 T-301,后置 T-302/T-303 仍受依赖门槛约束
|
||||
- 当前 blocker:无;下一个任务是 T-302 安装流程整合,必须先正式落成并提交任务文件,再启动一个写入 Agent + 测试/安全只读 Agents
|
||||
|
||||
## 当前目录要点
|
||||
|
||||
| 路径 | 状态 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `docs/` | 已有 | harness coding 文档集(本次初始化完成) |
|
||||
| `docs/tasks/` | 已有 | Phase 0~2 已完成;T-301 已落成并处于 DOING |
|
||||
| `docs/tasks/` | 已有 | Phase 0~2 与 T-301 已完成;T-302 尚未落成 |
|
||||
| `scripts/` | 已有 | harness 治理、core 边界、Go 版本检查与 Phase 0 双平台验证入口 |
|
||||
| `core/` | 已建 | Go 1.20 兼容;已有正式 Catalog、本地状态/存储、列表模型、图标缓存与 Phase 1 安装安全原型 |
|
||||
| `core/` | 已建 | Go 1.20 兼容;已有正式 Catalog、本地状态/存储、列表模型、图标缓存、可恢复下载队列与 Phase 1 安装安全原型 |
|
||||
| `app-modern/` | 已建 | Go 1.25.0 + Gio v0.10.1;Modern AppShell 已接入虚拟列表、详情和内存图标 |
|
||||
| `app-win7/` | 已建 | Go 1.20 + Gio v0.6.0;Legacy AppShell 已接入低成本列表、详情和内存图标 |
|
||||
| `schemas/` | 已建 | `manifest.schema.json`、`app.schema.json` 与 `installed-app.schema.json` |
|
||||
| `testdata/` | 已建 | 当前包含 Catalog 假数据与恶意样例;后续任务继续扩展 |
|
||||
| `schemas/` | 已建 | `manifest.schema.json`、`app.schema.json`、`installed-app.schema.json` 与 `download-task.schema.json` |
|
||||
| `testdata/` | 已建 | 包含 Catalog 假数据、ZIP 恶意矩阵与下载协议测试说明;后续任务继续扩展 |
|
||||
|
||||
## 任务状态
|
||||
|
||||
任务状态以 `docs/tasks/` 各任务文件 frontmatter 的 `status` 为准。本节只写项目级摘要:
|
||||
|
||||
- 已完成:Phase 0 的 `T-001`~`T-004`;Phase 1 的 `T-101`、`T-102`、`T-103`;Phase 2 的 `T-201`~`T-204`。
|
||||
- 正在进行:`T-301 下载队列`(一个写入 Agent + 测试/安全只读 Agents)。
|
||||
- 下一个可领取任务:无;T-302 必须等待 T-301 DONE。
|
||||
- 已完成:Phase 0 的 `T-001`~`T-004`;Phase 1 的 `T-101`、`T-102`、`T-103`;Phase 2 的 `T-201`~`T-204`;Phase 3 的 `T-301`。
|
||||
- 正在进行:无。
|
||||
- 下一个可领取任务:先按路线图正式落成 `T-302 安装流程整合` 并提交,然后才能领取和启动多 Agent。
|
||||
|
||||
## 当前可运行内容
|
||||
|
||||
|
||||
+5
-1
@@ -3,7 +3,7 @@ id: T-301
|
||||
title: 可恢复下载队列
|
||||
phase: 3
|
||||
deps: [T-202]
|
||||
status: DOING
|
||||
status: DONE
|
||||
created: 2026-07-16
|
||||
issue: null
|
||||
context_ref: a52acbf926041550ebc30049ff87f08faaa5ba0c
|
||||
@@ -75,3 +75,7 @@ T-301 是 T-302/T-303 的前置安全边界。它只负责把远端 ZIP 稳定
|
||||
## 执行记录
|
||||
|
||||
- 2026-07-16:正式落成多 Agent 协作方式;采用一个写入 Agent + 测试设计/安全架构两个只读 Agent,保持 T-301 → T-302 → T-303 依赖串行。
|
||||
- 2026-07-16:实现 `core/downloader.Queue`、严格 HTTPS/Range transport、任务状态/路径协议、pause/resume/cancel/retry、默认并发 2、进度节流与 application 具体事件负载;新增 `core/storage.DownloadTaskStore`、`download-task.schema.json` 和测试数据说明。
|
||||
- 2026-07-16:根据多轮只读审查修复安全边界:服务器忽略 Range 时先截断并 sync `.part` 再持久化新 attempt/validator;unknown total 全程受硬上限约束且续传必须证明完整 total;写入句柄身份贯穿 rename 前后;pause 不吞 body close/sync 错误,cancel 则记录非致命关闭错误后仍完成固定清理;Close 不覆盖用户 cancel;known-total 完整 part 在同进程 resume/retry 直接 finalize;observer 失败不反向改变 durable 结果并通过 `OnObserverError` 报告。
|
||||
- 2026-07-16:补齐并发补位、断连续传、restart 写失败窗口、未知 total、HTTP 404/416/500 与畸形响应、进度单调/节流/速度、迟到 generation、事件字段、observer 失败对账、崩溃恢复和文件身份替换回归测试;最终只读测试/安全审查均无 blocker/high。
|
||||
- 2026-07-16:验证通过:`go -C core vet ./...`;`go -C core test -count=1 ./...`;`go -C core test -count=10 ./downloader`;`./scripts/verify_phase0.ps1`。`go test -race` 已尝试,当前环境 `CGO_ENABLED=0` 且未安装 gcc,因此 race detector 无法启动;非 race 闸门全部通过。
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://softbox.invalid/schemas/download-task.schema.json",
|
||||
"title": "SoftBox download-task.json v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_version",
|
||||
"request_id",
|
||||
"app_id",
|
||||
"url",
|
||||
"status",
|
||||
"attempt",
|
||||
"done",
|
||||
"total_known",
|
||||
"total",
|
||||
"validator",
|
||||
"error_code",
|
||||
"created_at"
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"const": 1
|
||||
},
|
||||
"request_id": {
|
||||
"type": "string",
|
||||
"maxLength": 128,
|
||||
"pattern": "^[a-z0-9][a-z0-9-]{0,127}$"
|
||||
},
|
||||
"app_id": {
|
||||
"type": "string",
|
||||
"maxLength": 128,
|
||||
"pattern": "^[a-z0-9][a-z0-9-]{0,127}$"
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"maxLength": 4096,
|
||||
"format": "uri",
|
||||
"pattern": "^https://"
|
||||
},
|
||||
"status": {
|
||||
"enum": ["queued", "downloading", "paused", "failed", "completed"]
|
||||
},
|
||||
"attempt": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"done": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"total_known": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"total": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"validator": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"maxProperties": 0
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["etag"],
|
||||
"properties": {
|
||||
"etag": {
|
||||
"type": "string",
|
||||
"maxLength": 512,
|
||||
"pattern": "^\".*\"$"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["last_modified"],
|
||||
"properties": {
|
||||
"last_modified": {
|
||||
"type": "string",
|
||||
"maxLength": 512
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"error_code": {
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"pattern": "^$|^[a-z0-9][a-z0-9_]{0,63}$"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
}
|
||||
},
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"total_known": {
|
||||
"const": false
|
||||
}
|
||||
},
|
||||
"required": ["total_known"]
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"total": {
|
||||
"const": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"status": {
|
||||
"const": "failed"
|
||||
}
|
||||
},
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"error_code": {
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"else": {
|
||||
"properties": {
|
||||
"error_code": {
|
||||
"const": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+1
@@ -6,3 +6,4 @@
|
||||
- 测试若需要签名,使用测试代码中明确标注的专用测试密钥。
|
||||
- 恶意样例用于证明解析器和安全边界会拒绝输入,不得被发布流程消费。
|
||||
- `catalog/manifest-valid-payload.json` 同时作为 manifest v1 强类型解析与目标过滤的公开虚构样例;包哈希与签名只保证格式合法,不对应真实下载物。
|
||||
- `download/`:T-301 在运行时生成 HTTPS Range、断连、并发和恢复样例,不保存真实下载包。
|
||||
|
||||
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
# 下载测试数据
|
||||
|
||||
T-301 的 HTTP Range、断连、错误响应和并发阻塞样例由 Go 测试使用
|
||||
`httptest` 与内存 Transport 在运行时生成,避免提交真实 URL 或大型二进制包。
|
||||
|
||||
覆盖场景包括:
|
||||
|
||||
- 新下载 200 与续传 206。
|
||||
- Content-Range 起点/实体 validator 不匹配。
|
||||
- 服务端忽略 Range 后从 0 安全重下。
|
||||
- 连接中断后保留 `.part` 并按实际长度续传。
|
||||
- 默认并发 2、暂停、取消、重试、完整 part 同进程 finalize 和崩溃恢复。
|
||||
- observer 投递失败、body close 错误和活跃 `.part` 文件身份替换。
|
||||
Reference in New Issue
Block a user