package downloader import ( "context" "errors" "fmt" "io" "net/http" "strconv" "strings" ) var ( ErrHTTPStatus = errors.New("download HTTP status is not successful") ErrRangeMismatch = errors.New("download Content-Range does not match request") ErrRangeEntityChanged = errors.New("download range entity validator changed") ErrResponseEncoding = errors.New("download response encoding is not identity") ErrInsecureRedirect = errors.New("download redirect target is not secure") ErrTransferTooLarge = errors.New("download response exceeds byte limit") ErrTransferIncomplete = errors.New("download response ended before expected length") ) // OpenRequest describes one HTTP attempt. type OpenRequest struct { URL string Offset int64 Validator EntityValidator } // OpenResponse owns Body until the caller closes it. type OpenResponse struct { Body io.ReadCloser Restart bool TotalKnown bool Total int64 ResponseLengthKnown bool ResponseLength int64 Validator EntityValidator } // Transport opens a remote byte stream at a requested offset. type Transport interface { Open(context.Context, OpenRequest) (OpenResponse, error) } // HTTPTransport implements strict HTTPS and Range semantics with net/http. type HTTPTransport struct { client *http.Client } // NewHTTPTransport creates a standard-library transport. func NewHTTPTransport(client *http.Client) *HTTPTransport { if client == nil { client = http.DefaultClient } clientCopy := *client previousRedirectCheck := client.CheckRedirect clientCopy.CheckRedirect = func(request *http.Request, via []*http.Request) error { if request.URL == nil || ValidateHTTPSURL(request.URL.String()) != nil { return ErrInsecureRedirect } if previousRedirectCheck != nil { return previousRedirectCheck(request, via) } if len(via) >= 10 { return errors.New("stopped after 10 redirects") } return nil } return &HTTPTransport{client: &clientCopy} } // Open starts a transfer. A 200 response to a Range request is returned as // Restart=true so the queue truncates the old part before consuming bytes. func (transport *HTTPTransport) Open( ctx context.Context, request OpenRequest, ) (OpenResponse, error) { if err := ValidateHTTPSURL(request.URL); err != nil { return OpenResponse{}, err } if request.Offset < 0 { return OpenResponse{}, fmt.Errorf("%w: negative offset", ErrRangeMismatch) } if err := request.Validator.Validate(); err != nil { return OpenResponse{}, fmt.Errorf("%w: invalid validator", ErrRangeMismatch) } if request.Offset > 0 && request.Validator.Empty() { return OpenResponse{}, fmt.Errorf("%w: resume needs an entity validator", ErrRangeMismatch) } httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, request.URL, nil) if err != nil { return OpenResponse{}, fmt.Errorf("create download request: %w", err) } httpRequest.Header.Set("Accept-Encoding", "identity") if request.Offset > 0 { httpRequest.Header.Set("Range", fmt.Sprintf("bytes=%d-", request.Offset)) httpRequest.Header.Set("If-Range", request.Validator.HeaderValue()) } response, err := transport.client.Do(httpRequest) if err != nil { return OpenResponse{}, err } closeWithError := func(openErr error) (OpenResponse, error) { _ = response.Body.Close() return OpenResponse{}, openErr } if response.Request == nil || response.Request.URL == nil || ValidateHTTPSURL(response.Request.URL.String()) != nil { return closeWithError(ErrInsecureRedirect) } encoding := strings.TrimSpace(response.Header.Get("Content-Encoding")) if encoding != "" && !strings.EqualFold(encoding, "identity") { return closeWithError(ErrResponseEncoding) } validator := responseValidator(response.Header) switch response.StatusCode { case http.StatusOK: if response.Header.Get("Content-Range") != "" { return closeWithError(fmt.Errorf("%w: 200 includes Content-Range", ErrRangeMismatch)) } totalKnown := response.ContentLength >= 0 total := response.ContentLength if !totalKnown { total = 0 } return OpenResponse{ Body: response.Body, Restart: request.Offset > 0, TotalKnown: totalKnown, Total: total, ResponseLengthKnown: totalKnown, ResponseLength: total, Validator: validator, }, nil case http.StatusPartialContent: if request.Offset == 0 { return closeWithError(fmt.Errorf("%w: unsolicited partial response", ErrRangeMismatch)) } start, end, totalKnown, total, parseErr := parseContentRange( response.Header.Get("Content-Range"), ) if parseErr != nil || start != request.Offset { return closeWithError(ErrRangeMismatch) } rangeLength := end - start + 1 if rangeLength <= 0 || (response.ContentLength >= 0 && response.ContentLength != rangeLength) { return closeWithError(ErrRangeMismatch) } if !validatorsMatch(request.Validator, validator) { return closeWithError(ErrRangeEntityChanged) } return OpenResponse{ Body: response.Body, TotalKnown: totalKnown, Total: total, ResponseLengthKnown: true, ResponseLength: rangeLength, Validator: request.Validator, }, nil default: return closeWithError(fmt.Errorf("%w: %d", ErrHTTPStatus, response.StatusCode)) } } func parseContentRange(value string) ( start int64, end int64, totalKnown bool, total int64, err error, ) { if !strings.HasPrefix(value, "bytes ") { return 0, 0, false, 0, ErrRangeMismatch } rangeAndTotal := strings.Split(strings.TrimPrefix(value, "bytes "), "/") if len(rangeAndTotal) != 2 { return 0, 0, false, 0, ErrRangeMismatch } bounds := strings.Split(rangeAndTotal[0], "-") if len(bounds) != 2 { return 0, 0, false, 0, ErrRangeMismatch } start, err = strconv.ParseInt(bounds[0], 10, 64) if err != nil || start < 0 { return 0, 0, false, 0, ErrRangeMismatch } end, err = strconv.ParseInt(bounds[1], 10, 64) if err != nil || end < start || end == int64(^uint64(0)>>1) { return 0, 0, false, 0, ErrRangeMismatch } if rangeAndTotal[1] == "*" { return start, end, false, 0, nil } total, err = strconv.ParseInt(rangeAndTotal[1], 10, 64) if err != nil || total <= end { return 0, 0, false, 0, ErrRangeMismatch } return start, end, true, total, nil } // CopyOptions bounds one response copy and tags progress with the generation // that owns it. type CopyOptions struct { MaxBytes int64 ExpectedBytesKnown bool ExpectedBytes int64 Generation uint64 Progress func(CopyProgress) error } // CopyProgress reports bytes written during this response only. type CopyProgress struct { Generation uint64 Written int64 } // CopyResponse copies a response with a hard byte cap. It reads at most // MaxBytes+1 bytes so an oversized body is detected without unbounded IO. func CopyResponse( ctx context.Context, dst io.Writer, src io.Reader, options CopyOptions, ) (int64, error) { if options.MaxBytes < 0 || options.MaxBytes == int64(^uint64(0)>>1) { return 0, fmt.Errorf("%w: invalid byte limit", ErrTransferTooLarge) } if options.ExpectedBytesKnown && (options.ExpectedBytes < 0 || options.ExpectedBytes > options.MaxBytes) { return 0, fmt.Errorf("%w: invalid expected response length", ErrTransferIncomplete) } limited := &io.LimitedReader{R: src, N: options.MaxBytes + 1} buffer := make([]byte, 32*1024) var written int64 for { if err := ctx.Err(); err != nil { return written, err } readCount, readErr := limited.Read(buffer) if readCount > 0 { if int64(readCount) > options.MaxBytes-written { return written, ErrTransferTooLarge } writeCount, writeErr := dst.Write(buffer[:readCount]) written += int64(writeCount) if writeErr != nil { return written, writeErr } if writeCount != readCount { return written, io.ErrShortWrite } if options.Progress != nil { if err := options.Progress(CopyProgress{ Generation: options.Generation, Written: written, }); err != nil { return written, err } } } if readErr != nil { if readErr != io.EOF { return written, readErr } break } } if options.ExpectedBytesKnown && written != options.ExpectedBytes { return written, ErrTransferIncomplete } return written, nil }