281 lines
8.2 KiB
Go
281 lines
8.2 KiB
Go
|
package downloader
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"net/http"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
)
|
||
|
|
|
||
|
|
var (
|
||
|
|
ErrHTTPStatus = errors.New("download HTTP status is not successful")
|
||
|
|
ErrRangeMismatch = errors.New("download Content-Range does not match request")
|
||
|
|
ErrRangeEntityChanged = errors.New("download range entity validator changed")
|
||
|
|
ErrResponseEncoding = errors.New("download response encoding is not identity")
|
||
|
|
ErrInsecureRedirect = errors.New("download redirect target is not secure")
|
||
|
|
ErrTransferTooLarge = errors.New("download response exceeds byte limit")
|
||
|
|
ErrTransferIncomplete = errors.New("download response ended before expected length")
|
||
|
|
)
|
||
|
|
|
||
|
|
// OpenRequest describes one HTTP attempt.
|
||
|
|
type OpenRequest struct {
|
||
|
|
URL string
|
||
|
|
Offset int64
|
||
|
|
Validator EntityValidator
|
||
|
|
}
|
||
|
|
|
||
|
|
// OpenResponse owns Body until the caller closes it.
|
||
|
|
type OpenResponse struct {
|
||
|
|
Body io.ReadCloser
|
||
|
|
Restart bool
|
||
|
|
TotalKnown bool
|
||
|
|
Total int64
|
||
|
|
ResponseLengthKnown bool
|
||
|
|
ResponseLength int64
|
||
|
|
Validator EntityValidator
|
||
|
|
}
|
||
|
|
|
||
|
|
// Transport opens a remote byte stream at a requested offset.
|
||
|
|
type Transport interface {
|
||
|
|
Open(context.Context, OpenRequest) (OpenResponse, error)
|
||
|
|
}
|
||
|
|
|
||
|
|
// HTTPTransport implements strict HTTPS and Range semantics with net/http.
|
||
|
|
type HTTPTransport struct {
|
||
|
|
client *http.Client
|
||
|
|
}
|
||
|
|
|
||
|
|
// NewHTTPTransport creates a standard-library transport.
|
||
|
|
func NewHTTPTransport(client *http.Client) *HTTPTransport {
|
||
|
|
if client == nil {
|
||
|
|
client = http.DefaultClient
|
||
|
|
}
|
||
|
|
clientCopy := *client
|
||
|
|
previousRedirectCheck := client.CheckRedirect
|
||
|
|
clientCopy.CheckRedirect = func(request *http.Request, via []*http.Request) error {
|
||
|
|
if request.URL == nil || ValidateHTTPSURL(request.URL.String()) != nil {
|
||
|
|
return ErrInsecureRedirect
|
||
|
|
}
|
||
|
|
if previousRedirectCheck != nil {
|
||
|
|
return previousRedirectCheck(request, via)
|
||
|
|
}
|
||
|
|
if len(via) >= 10 {
|
||
|
|
return errors.New("stopped after 10 redirects")
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
return &HTTPTransport{client: &clientCopy}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Open starts a transfer. A 200 response to a Range request is returned as
|
||
|
|
// Restart=true so the queue truncates the old part before consuming bytes.
|
||
|
|
func (transport *HTTPTransport) Open(
|
||
|
|
ctx context.Context,
|
||
|
|
request OpenRequest,
|
||
|
|
) (OpenResponse, error) {
|
||
|
|
if err := ValidateHTTPSURL(request.URL); err != nil {
|
||
|
|
return OpenResponse{}, err
|
||
|
|
}
|
||
|
|
if request.Offset < 0 {
|
||
|
|
return OpenResponse{}, fmt.Errorf("%w: negative offset", ErrRangeMismatch)
|
||
|
|
}
|
||
|
|
if err := request.Validator.Validate(); err != nil {
|
||
|
|
return OpenResponse{}, fmt.Errorf("%w: invalid validator", ErrRangeMismatch)
|
||
|
|
}
|
||
|
|
if request.Offset > 0 && request.Validator.Empty() {
|
||
|
|
return OpenResponse{}, fmt.Errorf("%w: resume needs an entity validator", ErrRangeMismatch)
|
||
|
|
}
|
||
|
|
|
||
|
|
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, request.URL, nil)
|
||
|
|
if err != nil {
|
||
|
|
return OpenResponse{}, fmt.Errorf("create download request: %w", err)
|
||
|
|
}
|
||
|
|
httpRequest.Header.Set("Accept-Encoding", "identity")
|
||
|
|
if request.Offset > 0 {
|
||
|
|
httpRequest.Header.Set("Range", fmt.Sprintf("bytes=%d-", request.Offset))
|
||
|
|
httpRequest.Header.Set("If-Range", request.Validator.HeaderValue())
|
||
|
|
}
|
||
|
|
|
||
|
|
response, err := transport.client.Do(httpRequest)
|
||
|
|
if err != nil {
|
||
|
|
return OpenResponse{}, err
|
||
|
|
}
|
||
|
|
closeWithError := func(openErr error) (OpenResponse, error) {
|
||
|
|
_ = response.Body.Close()
|
||
|
|
return OpenResponse{}, openErr
|
||
|
|
}
|
||
|
|
if response.Request == nil || response.Request.URL == nil ||
|
||
|
|
ValidateHTTPSURL(response.Request.URL.String()) != nil {
|
||
|
|
return closeWithError(ErrInsecureRedirect)
|
||
|
|
}
|
||
|
|
encoding := strings.TrimSpace(response.Header.Get("Content-Encoding"))
|
||
|
|
if encoding != "" && !strings.EqualFold(encoding, "identity") {
|
||
|
|
return closeWithError(ErrResponseEncoding)
|
||
|
|
}
|
||
|
|
|
||
|
|
validator := responseValidator(response.Header)
|
||
|
|
switch response.StatusCode {
|
||
|
|
case http.StatusOK:
|
||
|
|
if response.Header.Get("Content-Range") != "" {
|
||
|
|
return closeWithError(fmt.Errorf("%w: 200 includes Content-Range", ErrRangeMismatch))
|
||
|
|
}
|
||
|
|
totalKnown := response.ContentLength >= 0
|
||
|
|
total := response.ContentLength
|
||
|
|
if !totalKnown {
|
||
|
|
total = 0
|
||
|
|
}
|
||
|
|
return OpenResponse{
|
||
|
|
Body: response.Body,
|
||
|
|
Restart: request.Offset > 0,
|
||
|
|
TotalKnown: totalKnown,
|
||
|
|
Total: total,
|
||
|
|
ResponseLengthKnown: totalKnown,
|
||
|
|
ResponseLength: total,
|
||
|
|
Validator: validator,
|
||
|
|
}, nil
|
||
|
|
|
||
|
|
case http.StatusPartialContent:
|
||
|
|
if request.Offset == 0 {
|
||
|
|
return closeWithError(fmt.Errorf("%w: unsolicited partial response", ErrRangeMismatch))
|
||
|
|
}
|
||
|
|
start, end, totalKnown, total, parseErr := parseContentRange(
|
||
|
|
response.Header.Get("Content-Range"),
|
||
|
|
)
|
||
|
|
if parseErr != nil || start != request.Offset {
|
||
|
|
return closeWithError(ErrRangeMismatch)
|
||
|
|
}
|
||
|
|
rangeLength := end - start + 1
|
||
|
|
if rangeLength <= 0 ||
|
||
|
|
(response.ContentLength >= 0 && response.ContentLength != rangeLength) {
|
||
|
|
return closeWithError(ErrRangeMismatch)
|
||
|
|
}
|
||
|
|
if !validatorsMatch(request.Validator, validator) {
|
||
|
|
return closeWithError(ErrRangeEntityChanged)
|
||
|
|
}
|
||
|
|
return OpenResponse{
|
||
|
|
Body: response.Body,
|
||
|
|
TotalKnown: totalKnown,
|
||
|
|
Total: total,
|
||
|
|
ResponseLengthKnown: true,
|
||
|
|
ResponseLength: rangeLength,
|
||
|
|
Validator: request.Validator,
|
||
|
|
}, nil
|
||
|
|
|
||
|
|
default:
|
||
|
|
return closeWithError(fmt.Errorf("%w: %d", ErrHTTPStatus, response.StatusCode))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func parseContentRange(value string) (
|
||
|
|
start int64,
|
||
|
|
end int64,
|
||
|
|
totalKnown bool,
|
||
|
|
total int64,
|
||
|
|
err error,
|
||
|
|
) {
|
||
|
|
if !strings.HasPrefix(value, "bytes ") {
|
||
|
|
return 0, 0, false, 0, ErrRangeMismatch
|
||
|
|
}
|
||
|
|
rangeAndTotal := strings.Split(strings.TrimPrefix(value, "bytes "), "/")
|
||
|
|
if len(rangeAndTotal) != 2 {
|
||
|
|
return 0, 0, false, 0, ErrRangeMismatch
|
||
|
|
}
|
||
|
|
bounds := strings.Split(rangeAndTotal[0], "-")
|
||
|
|
if len(bounds) != 2 {
|
||
|
|
return 0, 0, false, 0, ErrRangeMismatch
|
||
|
|
}
|
||
|
|
start, err = strconv.ParseInt(bounds[0], 10, 64)
|
||
|
|
if err != nil || start < 0 {
|
||
|
|
return 0, 0, false, 0, ErrRangeMismatch
|
||
|
|
}
|
||
|
|
end, err = strconv.ParseInt(bounds[1], 10, 64)
|
||
|
|
if err != nil || end < start || end == int64(^uint64(0)>>1) {
|
||
|
|
return 0, 0, false, 0, ErrRangeMismatch
|
||
|
|
}
|
||
|
|
if rangeAndTotal[1] == "*" {
|
||
|
|
return start, end, false, 0, nil
|
||
|
|
}
|
||
|
|
total, err = strconv.ParseInt(rangeAndTotal[1], 10, 64)
|
||
|
|
if err != nil || total <= end {
|
||
|
|
return 0, 0, false, 0, ErrRangeMismatch
|
||
|
|
}
|
||
|
|
return start, end, true, total, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// CopyOptions bounds one response copy and tags progress with the generation
|
||
|
|
// that owns it.
|
||
|
|
type CopyOptions struct {
|
||
|
|
MaxBytes int64
|
||
|
|
ExpectedBytesKnown bool
|
||
|
|
ExpectedBytes int64
|
||
|
|
Generation uint64
|
||
|
|
Progress func(CopyProgress) error
|
||
|
|
}
|
||
|
|
|
||
|
|
// CopyProgress reports bytes written during this response only.
|
||
|
|
type CopyProgress struct {
|
||
|
|
Generation uint64
|
||
|
|
Written int64
|
||
|
|
}
|
||
|
|
|
||
|
|
// CopyResponse copies a response with a hard byte cap. It reads at most
|
||
|
|
// MaxBytes+1 bytes so an oversized body is detected without unbounded IO.
|
||
|
|
func CopyResponse(
|
||
|
|
ctx context.Context,
|
||
|
|
dst io.Writer,
|
||
|
|
src io.Reader,
|
||
|
|
options CopyOptions,
|
||
|
|
) (int64, error) {
|
||
|
|
if options.MaxBytes < 0 || options.MaxBytes == int64(^uint64(0)>>1) {
|
||
|
|
return 0, fmt.Errorf("%w: invalid byte limit", ErrTransferTooLarge)
|
||
|
|
}
|
||
|
|
if options.ExpectedBytesKnown &&
|
||
|
|
(options.ExpectedBytes < 0 || options.ExpectedBytes > options.MaxBytes) {
|
||
|
|
return 0, fmt.Errorf("%w: invalid expected response length", ErrTransferIncomplete)
|
||
|
|
}
|
||
|
|
|
||
|
|
limited := &io.LimitedReader{R: src, N: options.MaxBytes + 1}
|
||
|
|
buffer := make([]byte, 32*1024)
|
||
|
|
var written int64
|
||
|
|
for {
|
||
|
|
if err := ctx.Err(); err != nil {
|
||
|
|
return written, err
|
||
|
|
}
|
||
|
|
readCount, readErr := limited.Read(buffer)
|
||
|
|
if readCount > 0 {
|
||
|
|
if int64(readCount) > options.MaxBytes-written {
|
||
|
|
return written, ErrTransferTooLarge
|
||
|
|
}
|
||
|
|
writeCount, writeErr := dst.Write(buffer[:readCount])
|
||
|
|
written += int64(writeCount)
|
||
|
|
if writeErr != nil {
|
||
|
|
return written, writeErr
|
||
|
|
}
|
||
|
|
if writeCount != readCount {
|
||
|
|
return written, io.ErrShortWrite
|
||
|
|
}
|
||
|
|
if options.Progress != nil {
|
||
|
|
if err := options.Progress(CopyProgress{
|
||
|
|
Generation: options.Generation,
|
||
|
|
Written: written,
|
||
|
|
}); err != nil {
|
||
|
|
return written, err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if readErr != nil {
|
||
|
|
if readErr != io.EOF {
|
||
|
|
return written, readErr
|
||
|
|
}
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if options.ExpectedBytesKnown && written != options.ExpectedBytes {
|
||
|
|
return written, ErrTransferIncomplete
|
||
|
|
}
|
||
|
|
return written, nil
|
||
|
|
}
|