Implement resumable download queue (T-301)

2026-07-16 19:49:06 +08:00
parent 2d302b731a
commit 8dad40f934
27 changed files with 5498 additions and 18 deletions
@@ -0,0 +1,280 @@
+package downloader
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+var (
+	ErrHTTPStatus         = errors.New("download HTTP status is not successful")
+	ErrRangeMismatch      = errors.New("download Content-Range does not match request")
+	ErrRangeEntityChanged = errors.New("download range entity validator changed")
+	ErrResponseEncoding   = errors.New("download response encoding is not identity")
+	ErrInsecureRedirect   = errors.New("download redirect target is not secure")
+	ErrTransferTooLarge   = errors.New("download response exceeds byte limit")
+	ErrTransferIncomplete = errors.New("download response ended before expected length")
+)
+
+// OpenRequest describes one HTTP attempt.
+type OpenRequest struct {
+	URL       string
+	Offset    int64
+	Validator EntityValidator
+}
+
+// OpenResponse owns Body until the caller closes it.
+type OpenResponse struct {
+	Body                io.ReadCloser
+	Restart             bool
+	TotalKnown          bool
+	Total               int64
+	ResponseLengthKnown bool
+	ResponseLength      int64
+	Validator           EntityValidator
+}
+
+// Transport opens a remote byte stream at a requested offset.
+type Transport interface {
+	Open(context.Context, OpenRequest) (OpenResponse, error)
+}
+
+// HTTPTransport implements strict HTTPS and Range semantics with net/http.
+type HTTPTransport struct {
+	client *http.Client
+}
+
+// NewHTTPTransport creates a standard-library transport.
+func NewHTTPTransport(client *http.Client) *HTTPTransport {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	clientCopy := *client
+	previousRedirectCheck := client.CheckRedirect
+	clientCopy.CheckRedirect = func(request *http.Request, via []*http.Request) error {
+		if request.URL == nil || ValidateHTTPSURL(request.URL.String()) != nil {
+			return ErrInsecureRedirect
+		}
+		if previousRedirectCheck != nil {
+			return previousRedirectCheck(request, via)
+		}
+		if len(via) >= 10 {
+			return errors.New("stopped after 10 redirects")
+		}
+		return nil
+	}
+	return &HTTPTransport{client: &clientCopy}
+}
+
+// Open starts a transfer. A 200 response to a Range request is returned as
+// Restart=true so the queue truncates the old part before consuming bytes.
+func (transport *HTTPTransport) Open(
+	ctx context.Context,
+	request OpenRequest,
+) (OpenResponse, error) {
+	if err := ValidateHTTPSURL(request.URL); err != nil {
+		return OpenResponse{}, err
+	}
+	if request.Offset < 0 {
+		return OpenResponse{}, fmt.Errorf("%w: negative offset", ErrRangeMismatch)
+	}
+	if err := request.Validator.Validate(); err != nil {
+		return OpenResponse{}, fmt.Errorf("%w: invalid validator", ErrRangeMismatch)
+	}
+	if request.Offset > 0 && request.Validator.Empty() {
+		return OpenResponse{}, fmt.Errorf("%w: resume needs an entity validator", ErrRangeMismatch)
+	}
+
+	httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, request.URL, nil)
+	if err != nil {
+		return OpenResponse{}, fmt.Errorf("create download request: %w", err)
+	}
+	httpRequest.Header.Set("Accept-Encoding", "identity")
+	if request.Offset > 0 {
+		httpRequest.Header.Set("Range", fmt.Sprintf("bytes=%d-", request.Offset))
+		httpRequest.Header.Set("If-Range", request.Validator.HeaderValue())
+	}
+
+	response, err := transport.client.Do(httpRequest)
+	if err != nil {
+		return OpenResponse{}, err
+	}
+	closeWithError := func(openErr error) (OpenResponse, error) {
+		_ = response.Body.Close()
+		return OpenResponse{}, openErr
+	}
+	if response.Request == nil || response.Request.URL == nil ||
+		ValidateHTTPSURL(response.Request.URL.String()) != nil {
+		return closeWithError(ErrInsecureRedirect)
+	}
+	encoding := strings.TrimSpace(response.Header.Get("Content-Encoding"))
+	if encoding != "" && !strings.EqualFold(encoding, "identity") {
+		return closeWithError(ErrResponseEncoding)
+	}
+
+	validator := responseValidator(response.Header)
+	switch response.StatusCode {
+	case http.StatusOK:
+		if response.Header.Get("Content-Range") != "" {
+			return closeWithError(fmt.Errorf("%w: 200 includes Content-Range", ErrRangeMismatch))
+		}
+		totalKnown := response.ContentLength >= 0
+		total := response.ContentLength
+		if !totalKnown {
+			total = 0
+		}
+		return OpenResponse{
+			Body:                response.Body,
+			Restart:             request.Offset > 0,
+			TotalKnown:          totalKnown,
+			Total:               total,
+			ResponseLengthKnown: totalKnown,
+			ResponseLength:      total,
+			Validator:           validator,
+		}, nil
+
+	case http.StatusPartialContent:
+		if request.Offset == 0 {
+			return closeWithError(fmt.Errorf("%w: unsolicited partial response", ErrRangeMismatch))
+		}
+		start, end, totalKnown, total, parseErr := parseContentRange(
+			response.Header.Get("Content-Range"),
+		)
+		if parseErr != nil || start != request.Offset {
+			return closeWithError(ErrRangeMismatch)
+		}
+		rangeLength := end - start + 1
+		if rangeLength <= 0 ||
+			(response.ContentLength >= 0 && response.ContentLength != rangeLength) {
+			return closeWithError(ErrRangeMismatch)
+		}
+		if !validatorsMatch(request.Validator, validator) {
+			return closeWithError(ErrRangeEntityChanged)
+		}
+		return OpenResponse{
+			Body:                response.Body,
+			TotalKnown:          totalKnown,
+			Total:               total,
+			ResponseLengthKnown: true,
+			ResponseLength:      rangeLength,
+			Validator:           request.Validator,
+		}, nil
+
+	default:
+		return closeWithError(fmt.Errorf("%w: %d", ErrHTTPStatus, response.StatusCode))
+	}
+}
+
+func parseContentRange(value string) (
+	start int64,
+	end int64,
+	totalKnown bool,
+	total int64,
+	err error,
+) {
+	if !strings.HasPrefix(value, "bytes ") {
+		return 0, 0, false, 0, ErrRangeMismatch
+	}
+	rangeAndTotal := strings.Split(strings.TrimPrefix(value, "bytes "), "/")
+	if len(rangeAndTotal) != 2 {
+		return 0, 0, false, 0, ErrRangeMismatch
+	}
+	bounds := strings.Split(rangeAndTotal[0], "-")
+	if len(bounds) != 2 {
+		return 0, 0, false, 0, ErrRangeMismatch
+	}
+	start, err = strconv.ParseInt(bounds[0], 10, 64)
+	if err != nil || start < 0 {
+		return 0, 0, false, 0, ErrRangeMismatch
+	}
+	end, err = strconv.ParseInt(bounds[1], 10, 64)
+	if err != nil || end < start || end == int64(^uint64(0)>>1) {
+		return 0, 0, false, 0, ErrRangeMismatch
+	}
+	if rangeAndTotal[1] == "*" {
+		return start, end, false, 0, nil
+	}
+	total, err = strconv.ParseInt(rangeAndTotal[1], 10, 64)
+	if err != nil || total <= end {
+		return 0, 0, false, 0, ErrRangeMismatch
+	}
+	return start, end, true, total, nil
+}
+
+// CopyOptions bounds one response copy and tags progress with the generation
+// that owns it.
+type CopyOptions struct {
+	MaxBytes           int64
+	ExpectedBytesKnown bool
+	ExpectedBytes      int64
+	Generation         uint64
+	Progress           func(CopyProgress) error
+}
+
+// CopyProgress reports bytes written during this response only.
+type CopyProgress struct {
+	Generation uint64
+	Written    int64
+}
+
+// CopyResponse copies a response with a hard byte cap. It reads at most
+// MaxBytes+1 bytes so an oversized body is detected without unbounded IO.
+func CopyResponse(
+	ctx context.Context,
+	dst io.Writer,
+	src io.Reader,
+	options CopyOptions,
+) (int64, error) {
+	if options.MaxBytes < 0 || options.MaxBytes == int64(^uint64(0)>>1) {
+		return 0, fmt.Errorf("%w: invalid byte limit", ErrTransferTooLarge)
+	}
+	if options.ExpectedBytesKnown &&
+		(options.ExpectedBytes < 0 || options.ExpectedBytes > options.MaxBytes) {
+		return 0, fmt.Errorf("%w: invalid expected response length", ErrTransferIncomplete)
+	}
+
+	limited := &io.LimitedReader{R: src, N: options.MaxBytes + 1}
+	buffer := make([]byte, 32*1024)
+	var written int64
+	for {
+		if err := ctx.Err(); err != nil {
+			return written, err
+		}
+		readCount, readErr := limited.Read(buffer)
+		if readCount > 0 {
+			if int64(readCount) > options.MaxBytes-written {
+				return written, ErrTransferTooLarge
+			}
+			writeCount, writeErr := dst.Write(buffer[:readCount])
+			written += int64(writeCount)
+			if writeErr != nil {
+				return written, writeErr
+			}
+			if writeCount != readCount {
+				return written, io.ErrShortWrite
+			}
+			if options.Progress != nil {
+				if err := options.Progress(CopyProgress{
+					Generation: options.Generation,
+					Written:    written,
+				}); err != nil {
+					return written, err
+				}
+			}
+		}
+		if readErr != nil {
+			if readErr != io.EOF {
+				return written, readErr
+			}
+			break
+		}
+	}
+	if options.ExpectedBytesKnown && written != options.ExpectedBytes {
+		return written, ErrTransferIncomplete
+	}
+	return written, nil
+}