Implement resumable download queue (T-301)

2026-07-16 19:49:06 +08:00
parent 2d302b731a
commit 8dad40f934
27 changed files with 5498 additions and 18 deletions
@@ -0,0 +1,234 @@
+package downloader
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+func ensureTransferLayout(downloadsRoot, requestID string) (TaskPaths, error) {
+	paths, err := DeriveTaskPaths(downloadsRoot, requestID)
+	if err != nil {
+		return TaskPaths{}, err
+	}
+	if err := os.MkdirAll(paths.Root, 0o700); err != nil {
+		return TaskPaths{}, fmt.Errorf("create downloads root: %w", err)
+	}
+	for _, directory := range []string{paths.Root, paths.TasksDir, paths.FilesDir} {
+		if err := os.Mkdir(directory, 0o700); err != nil && !os.IsExist(err) {
+			return TaskPaths{}, fmt.Errorf("create download directory: %w", err)
+		}
+		info, err := os.Lstat(directory)
+		if err != nil {
+			return TaskPaths{}, fmt.Errorf("inspect download directory: %w", err)
+		}
+		if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+			return TaskPaths{}, fmt.Errorf("%w: managed directory is unsafe", ErrTaskCorrupt)
+		}
+	}
+	return paths, nil
+}
+
+func regularFileSize(filePath string) (size int64, exists bool, err error) {
+	info, err := os.Lstat(filePath)
+	if os.IsNotExist(err) {
+		return 0, false, nil
+	}
+	if err != nil {
+		return 0, false, fmt.Errorf("inspect download file: %w", err)
+	}
+	if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
+		return 0, false, fmt.Errorf("%w: managed file is not regular", ErrTaskCorrupt)
+	}
+	return info.Size(), true, nil
+}
+
+func regularFileExists(filePath string) (bool, error) {
+	_, exists, err := regularFileSize(filePath)
+	return exists, err
+}
+
+func truncatePart(filePath string) error {
+	file, _, err := openVerifiedRegular(filePath, os.O_RDWR)
+	if err != nil {
+		return err
+	}
+	if err := file.Truncate(0); err != nil {
+		file.Close()
+		return fmt.Errorf("truncate part: %w", err)
+	}
+	if _, err := file.Seek(0, io.SeekStart); err != nil {
+		file.Close()
+		return fmt.Errorf("seek truncated part: %w", err)
+	}
+	if err := file.Sync(); err != nil {
+		file.Close()
+		return fmt.Errorf("sync truncated part: %w", err)
+	}
+	return file.Close()
+}
+
+func openPart(paths TaskPaths, offset int64, restart bool) (*os.File, error) {
+	if restart {
+		if _, exists, err := regularFileSize(paths.Part); err != nil {
+			return nil, err
+		} else if exists {
+			file, _, err := openVerifiedRegular(paths.Part, os.O_RDWR)
+			if err != nil {
+				return nil, err
+			}
+			if err := file.Truncate(0); err != nil {
+				file.Close()
+				return nil, fmt.Errorf("truncate restarted part: %w", err)
+			}
+			if _, err := file.Seek(0, io.SeekStart); err != nil {
+				file.Close()
+				return nil, fmt.Errorf("seek restarted part: %w", err)
+			}
+			return file, nil
+		}
+		return os.OpenFile(paths.Part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	}
+	size, exists, err := regularFileSize(paths.Part)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		if offset != 0 {
+			return nil, fmt.Errorf("%w: missing part for non-zero offset", ErrTaskCorrupt)
+		}
+		return os.OpenFile(paths.Part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	}
+	if size != offset {
+		return nil, fmt.Errorf("%w: part size changed", ErrTaskCorrupt)
+	}
+	file, _, err := openVerifiedRegular(paths.Part, os.O_WRONLY)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := file.Seek(offset, io.SeekStart); err != nil {
+		file.Close()
+		return nil, fmt.Errorf("seek part: %w", err)
+	}
+	return file, nil
+}
+
+func syncRegularFile(filePath string) error {
+	file, _, err := openVerifiedRegular(filePath, os.O_RDWR)
+	if err != nil {
+		return err
+	}
+	if err := file.Sync(); err != nil {
+		file.Close()
+		return err
+	}
+	return file.Close()
+}
+
+func activateCompleted(paths TaskPaths, expected os.FileInfo) error {
+	if exists, err := regularFileExists(paths.Completed); err != nil {
+		return err
+	} else if exists {
+		return fmt.Errorf("%w: completed path already exists", ErrTaskCorrupt)
+	}
+	if expected == nil {
+		part, partInfo, err := openVerifiedRegular(paths.Part, os.O_RDONLY)
+		if err != nil {
+			return err
+		}
+		if err := part.Close(); err != nil {
+			return fmt.Errorf("close part before activation: %w", err)
+		}
+		expected = partInfo
+	}
+	if err := verifyRegularIdentity(paths.Part, expected); err != nil {
+		return err
+	}
+	if err := os.Rename(paths.Part, paths.Completed); err != nil {
+		return fmt.Errorf("activate completed download: %w", err)
+	}
+	completedInfo, err := os.Lstat(paths.Completed)
+	if err != nil {
+		return fmt.Errorf("inspect activated download: %w", err)
+	}
+	if completedInfo.Mode()&os.ModeSymlink != 0 ||
+		!completedInfo.Mode().IsRegular() ||
+		!os.SameFile(expected, completedInfo) {
+		_ = removeExactRegularFile(paths.Completed)
+		return fmt.Errorf("%w: activated file identity changed", ErrTaskCorrupt)
+	}
+	return nil
+}
+
+func verifyRegularIdentity(filePath string, expected os.FileInfo) error {
+	current, err := os.Lstat(filePath)
+	if err != nil {
+		return fmt.Errorf("inspect managed file identity: %w", err)
+	}
+	if current.Mode()&os.ModeSymlink != 0 ||
+		!current.Mode().IsRegular() ||
+		!os.SameFile(expected, current) {
+		return fmt.Errorf("%w: managed file identity changed", ErrTaskCorrupt)
+	}
+	return nil
+}
+
+func removeExactRegularFile(filePath string) error {
+	info, err := os.Lstat(filePath)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
+		return fmt.Errorf("%w: refuse removal of non-regular file", ErrTaskCorrupt)
+	}
+	return os.Remove(filePath)
+}
+
+func syncAndClose(file *os.File) error {
+	if file == nil {
+		return nil
+	}
+	if err := file.Sync(); err != nil {
+		file.Close()
+		return err
+	}
+	return file.Close()
+}
+
+func openVerifiedRegular(
+	filePath string,
+	flags int,
+) (*os.File, os.FileInfo, error) {
+	before, err := os.Lstat(filePath)
+	if err != nil {
+		return nil, nil, err
+	}
+	if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() {
+		return nil, nil, fmt.Errorf("%w: managed file is not regular", ErrTaskCorrupt)
+	}
+	file, err := os.OpenFile(filePath, flags, 0o600)
+	if err != nil {
+		return nil, nil, fmt.Errorf("open managed file: %w", err)
+	}
+	opened, err := file.Stat()
+	if err != nil {
+		file.Close()
+		return nil, nil, fmt.Errorf("stat managed file: %w", err)
+	}
+	after, err := os.Lstat(filePath)
+	if err != nil {
+		file.Close()
+		return nil, nil, fmt.Errorf("restat managed file: %w", err)
+	}
+	if after.Mode()&os.ModeSymlink != 0 ||
+		!after.Mode().IsRegular() ||
+		!os.SameFile(before, opened) ||
+		!os.SameFile(opened, after) {
+		file.Close()
+		return nil, nil, fmt.Errorf("%w: managed file changed while opening", ErrTaskCorrupt)
+	}
+	return file, opened, nil
+}