add apt gestion

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>

internal/clone/apt/cloner.go

@@ -0,0 +1,184 @@
+package apt
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+const (
+	upstreamInRelease  = ".upstream-InRelease"
+	upstreamPackagesGz = ".upstream-Packages.gz"
+)
+
+// Cloner mirrors an APT repository into a local directory.
+type Cloner struct {
+	SourceURL  string
+	DestDir    string
+	Config     Config
+	HTTPClient *http.Client
+	OnProgress func(file string, bytes int64)
+}
+
+// New creates a Cloner with a 30-minute HTTP timeout.
+func New(sourceURL, destDir string, cfg Config) *Cloner {
+	return &Cloner{
+		SourceURL:  strings.TrimRight(sourceURL, "/"),
+		DestDir:    destDir,
+		Config:     cfg,
+		HTTPClient: &http.Client{Timeout: 30 * time.Minute},
+	}
+}
+
+// Clone performs a full mirror of the configured APT repository.
+func (c *Cloner) Clone(ctx context.Context) error {
+	suite := c.Config.Suite
+
+	// 1. Create base directories.
+	if err := os.MkdirAll(filepath.Join(c.DestDir, "dists", suite), 0o755); err != nil {
+		return fmt.Errorf("create dists dir: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Join(c.DestDir, "pool"), 0o755); err != nil {
+		return fmt.Errorf("create pool dir: %w", err)
+	}
+
+	// 2. Fetch InRelease and save as .upstream-InRelease.
+	inReleaseURL := c.SourceURL + "/dists/" + suite + "/InRelease"
+	inReleaseData, err := fetchBytes(ctx, c.HTTPClient, inReleaseURL)
+	if err != nil {
+		return fmt.Errorf("fetch InRelease: %w", err)
+	}
+	upstreamIRPath := filepath.Join(c.DestDir, "dists", suite, upstreamInRelease)
+	if err := atomicWrite(upstreamIRPath, inReleaseData); err != nil {
+		return fmt.Errorf("save upstream InRelease: %w", err)
+	}
+
+	// 3. Parse InRelease.
+	rf, err := ParseRelease(inReleaseData)
+	if err != nil {
+		return fmt.Errorf("parse InRelease: %w", err)
+	}
+
+	// 4. For each component/arch: find Packages.gz in SHA256 section, download, parse debs.
+	var allDebs []DebPackage
+	for _, component := range c.Config.Components {
+		for _, arch := range c.Config.Architectures {
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+			pkgs, err := c.fetchComponentPackages(ctx, suite, component, arch, rf)
+			if err != nil {
+				return fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
+			}
+			allDebs = append(allDebs, pkgs...)
+		}
+	}
+
+	// 5. Download all .deb files.
+	for _, pkg := range allDebs {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if err := c.downloadDeb(ctx, pkg); err != nil {
+			return fmt.Errorf("download %s: %w", pkg.Filename, err)
+		}
+	}
+
+	// 6. Regenerate local metadata.
+	if err := RegenerateMetadata(c.DestDir, c.Config); err != nil {
+		return fmt.Errorf("regenerate metadata: %w", err)
+	}
+
+	return nil
+}
+
+// fetchComponentPackages downloads Packages.gz for one component/arch pair,
+// saves it as .upstream-Packages.gz, and returns the parsed package list.
+func (c *Cloner) fetchComponentPackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
+	wantPath := component + "/binary-" + arch + "/Packages.gz"
+
+	// Verify this path is listed in the upstream SHA256 section.
+	var found bool
+	for _, e := range rf.SHA256 {
+		if e.Path == wantPath {
+			found = true
+			break
+		}
+	}
+	if !found {
+		// Not an error — this component/arch may just not exist upstream.
+		return nil, nil
+	}
+
+	pkgGzURL := c.SourceURL + "/dists/" + suite + "/" + wantPath
+	data, err := fetchBytes(ctx, c.HTTPClient, pkgGzURL)
+	if err != nil {
+		return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
+	}
+
+	// Save as .upstream-Packages.gz so RegenerateMetadata can find it later.
+	destDir := filepath.Join(c.DestDir, "dists", suite, component, "binary-"+arch)
+	if err := os.MkdirAll(destDir, 0o755); err != nil {
+		return nil, err
+	}
+	upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
+	if err := atomicWrite(upstreamPath, data); err != nil {
+		return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
+	}
+
+	if c.OnProgress != nil {
+		c.OnProgress(wantPath, int64(len(data)))
+	}
+
+	// Decompress and parse.
+	gz, err := gzip.NewReader(strings.NewReader(string(data)))
+	if err != nil {
+		return nil, fmt.Errorf("open gzip for %s: %w", wantPath, err)
+	}
+	defer gz.Close()
+
+	raw, err := io.ReadAll(gz)
+	if err != nil {
+		return nil, fmt.Errorf("decompress %s: %w", wantPath, err)
+	}
+
+	pkgs, err := ParsePackages(raw)
+	if err != nil {
+		return nil, fmt.Errorf("parse Packages for %s: %w", wantPath, err)
+	}
+	return pkgs, nil
+}
+
+// downloadDeb downloads a single .deb file into the local pool tree.
+func (c *Cloner) downloadDeb(ctx context.Context, pkg DebPackage) error {
+	url := c.SourceURL + "/" + pkg.Filename
+	dest := filepath.Join(c.DestDir, filepath.FromSlash(pkg.Filename))
+
+	// Skip if already on disk.
+	if _, err := os.Stat(dest); err == nil {
+		return nil
+	}
+
+	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
+		return err
+	}
+
+	data, err := fetchBytes(ctx, c.HTTPClient, url)
+	if err != nil {
+		return err
+	}
+	if err := atomicWrite(dest, data); err != nil {
+		return err
+	}
+
+	if c.OnProgress != nil {
+		c.OnProgress(pkg.Filename, int64(len(data)))
+	}
+	return nil
+}

internal/clone/apt/config.go

@@ -0,0 +1,8 @@
+package apt
+
+// Config describes which parts of an APT repository to mirror.
+type Config struct {
+	Suite         string   `json:"suite"`
+	Components    []string `json:"components"`
+	Architectures []string `json:"architectures"`
+}

internal/clone/apt/helpers.go

@@ -0,0 +1,44 @@
+package apt
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+// fetchBytes performs a GET request and returns the response body.
+func fetchBytes(ctx context.Context, httpClient *http.Client, url string) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
+	}
+	return io.ReadAll(resp.Body)
+}
+
+// atomicWrite writes data to path via a temporary file and an atomic rename.
+func atomicWrite(path string, data []byte) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return err
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, data, 0o644); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return nil
+}

internal/clone/apt/metadata_gen.go

@@ -0,0 +1,148 @@
+package apt
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// InitEmptyRepo creates the directory structure for an APT mirror and writes an
+// empty Release file so APT clients don't get a 404 before the first clone.
+func InitEmptyRepo(localDir string, cfg Config) error {
+	if err := os.MkdirAll(filepath.Join(localDir, "pool"), 0o755); err != nil {
+		return err
+	}
+	for _, component := range cfg.Components {
+		for _, arch := range cfg.Architectures {
+			dir := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch)
+			if err := os.MkdirAll(dir, 0o755); err != nil {
+				return err
+			}
+		}
+	}
+	releasePath := filepath.Join(localDir, "dists", cfg.Suite, "Release")
+	if _, err := os.Stat(releasePath); os.IsNotExist(err) {
+		releaseData := GenerateRelease(cfg.Suite, cfg.Suite, cfg.Components, cfg.Architectures, nil)
+		if err := atomicWrite(releasePath, releaseData); err != nil {
+			return fmt.Errorf("write empty Release: %w", err)
+		}
+	}
+	return nil
+}
+
+// RegenerateMetadata rebuilds local Packages, Packages.gz, and Release files
+// for all configured component/arch pairs, keeping only packages present on disk.
+func RegenerateMetadata(localDir string, cfg Config) error {
+	suite := cfg.Suite
+	var releaseEntries []ReleaseEntry
+
+	for _, component := range cfg.Components {
+		for _, arch := range cfg.Architectures {
+			entries, err := regenerateComponentArch(localDir, suite, component, arch)
+			if err != nil {
+				return fmt.Errorf("regenerate %s/%s: %w", component, arch, err)
+			}
+			releaseEntries = append(releaseEntries, entries...)
+		}
+	}
+
+	// Write dists/{suite}/Release.
+	releaseData := GenerateRelease(suite, suite, cfg.Components, cfg.Architectures, releaseEntries)
+	releasePath := filepath.Join(localDir, "dists", suite, "Release")
+	if err := os.MkdirAll(filepath.Dir(releasePath), 0o755); err != nil {
+		return fmt.Errorf("create dists dir: %w", err)
+	}
+	if err := atomicWrite(releasePath, releaseData); err != nil {
+		return fmt.Errorf("write Release: %w", err)
+	}
+
+	return nil
+}
+
+// regenerateComponentArch regenerates Packages and Packages.gz for one component/arch
+// and returns the two ReleaseEntry values (plain then gz) for the Release file.
+func regenerateComponentArch(localDir, suite, component, arch string) ([]ReleaseEntry, error) {
+	binDir := filepath.Join(localDir, "dists", suite, component, "binary-"+arch)
+	upstreamGzPath := filepath.Join(binDir, upstreamPackagesGz)
+
+	// 1. Read upstream Packages.gz.
+	gzData, err := os.ReadFile(upstreamGzPath)
+	if err != nil {
+		// Not cloned yet — skip silently.
+		return nil, nil
+	}
+
+	// 2. Decompress.
+	gz, err := gzip.NewReader(bytes.NewReader(gzData))
+	if err != nil {
+		return nil, fmt.Errorf("open gzip %s: %w", upstreamGzPath, err)
+	}
+	rawPackages, err := io.ReadAll(gz)
+	gz.Close()
+	if err != nil {
+		return nil, fmt.Errorf("decompress %s: %w", upstreamGzPath, err)
+	}
+
+	// 3. Parse all stanzas.
+	allPkgs, err := ParsePackages(rawPackages)
+	if err != nil {
+		return nil, fmt.Errorf("parse upstream Packages: %w", err)
+	}
+
+	// 4. Filter: keep only packages present on disk.
+	checker := diskPresenceChecker(localDir)
+	localPkgs := FilterPackages(allPkgs, checker)
+
+	// 5. Re-emit filtered stanzas.
+	plainBytes := emitPackages(localPkgs)
+
+	// 6. Gzip the filtered Packages.
+	var gzBuf bytes.Buffer
+	gzw := gzip.NewWriter(&gzBuf)
+	if _, err := gzw.Write(plainBytes); err != nil {
+		return nil, fmt.Errorf("gzip write: %w", err)
+	}
+	if err := gzw.Close(); err != nil {
+		return nil, fmt.Errorf("gzip close: %w", err)
+	}
+	gzBytes := gzBuf.Bytes()
+
+	// 7. Write Packages and Packages.gz.
+	if err := os.MkdirAll(binDir, 0o755); err != nil {
+		return nil, err
+	}
+	plainPath := filepath.Join(binDir, "Packages")
+	if err := atomicWrite(plainPath, plainBytes); err != nil {
+		return nil, fmt.Errorf("write Packages: %w", err)
+	}
+	gzPath := filepath.Join(binDir, "Packages.gz")
+	if err := atomicWrite(gzPath, gzBytes); err != nil {
+		return nil, fmt.Errorf("write Packages.gz: %w", err)
+	}
+
+	// 8. Compute SHA256 hashes.
+	plainSum := sha256.Sum256(plainBytes)
+	gzSum := sha256.Sum256(gzBytes)
+
+	// Release paths are relative to dists/{suite}/.
+	relBase := component + "/binary-" + arch + "/"
+
+	return []ReleaseEntry{
+		{
+			Hash: hex.EncodeToString(plainSum[:]),
+			Size: int64(len(plainBytes)),
+			Path: relBase + "Packages",
+		},
+		{
+			Hash: hex.EncodeToString(gzSum[:]),
+			Size: int64(len(gzBytes)),
+			Path: relBase + "Packages.gz",
+		},
+	}, nil
+}
+

internal/clone/apt/packages.go

@@ -0,0 +1,128 @@
+package apt
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+// DebPackage represents a single stanza from a Debian Packages file.
+type DebPackage struct {
+	Package      string
+	Version      string
+	Architecture string
+	Filename     string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb
+	SHA256       string
+	Size         int64
+	Raw          []byte // original stanza bytes for faithful re-emission
+}
+
+// ParsePackages parses all stanzas from a Debian Packages control file.
+// Stanzas are separated by blank lines.
+func ParsePackages(data []byte) ([]DebPackage, error) {
+	// Normalise line endings.
+	data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n"))
+
+	var pkgs []DebPackage
+	stanzas := bytes.Split(data, []byte("\n\n"))
+
+	for _, stanza := range stanzas {
+		stanza = bytes.TrimSpace(stanza)
+		if len(stanza) == 0 {
+			continue
+		}
+		pkg, err := parseStanza(stanza)
+		if err != nil {
+			return nil, fmt.Errorf("parse stanza: %w", err)
+		}
+		pkgs = append(pkgs, pkg)
+	}
+	return pkgs, nil
+}
+
+func parseStanza(stanza []byte) (DebPackage, error) {
+	var pkg DebPackage
+	pkg.Raw = stanza
+
+	lines := bytes.Split(stanza, []byte("\n"))
+	for _, rawLine := range lines {
+		line := bytes.TrimRight(rawLine, " \t")
+		// Continuation lines (description body etc.) start with a space — skip.
+		if len(line) == 0 || line[0] == ' ' || line[0] == '\t' {
+			continue
+		}
+
+		colonIdx := bytes.IndexByte(line, ':')
+		if colonIdx < 0 {
+			continue
+		}
+		key := string(bytes.TrimSpace(line[:colonIdx]))
+		value := string(bytes.TrimSpace(line[colonIdx+1:]))
+
+		switch key {
+		case "Package":
+			pkg.Package = value
+		case "Version":
+			pkg.Version = value
+		case "Architecture":
+			pkg.Architecture = value
+		case "Filename":
+			pkg.Filename = value
+		case "SHA256":
+			pkg.SHA256 = value
+		case "Size":
+			n, err := strconv.ParseInt(value, 10, 64)
+			if err != nil {
+				return DebPackage{}, fmt.Errorf("bad Size value %q: %w", value, err)
+			}
+			pkg.Size = n
+		}
+	}
+
+	if pkg.Package == "" {
+		return DebPackage{}, fmt.Errorf("stanza missing Package field")
+	}
+	return pkg, nil
+}
+
+// FilterPackages returns only the packages for which presentOnDisk returns true.
+func FilterPackages(pkgs []DebPackage, presentOnDisk func(filename string) bool) []DebPackage {
+	var out []DebPackage
+	for _, p := range pkgs {
+		if presentOnDisk(p.Filename) {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+// emitPackages serialises a slice of DebPackage back to Packages-file format,
+// using the original Raw bytes for each stanza.
+func emitPackages(pkgs []DebPackage) []byte {
+	var buf bytes.Buffer
+	for i, pkg := range pkgs {
+		// Strip trailing whitespace from each line within the stanza.
+		lines := bytes.Split(pkg.Raw, []byte("\n"))
+		for j, l := range lines {
+			lines[j] = bytes.TrimRight(l, " \t")
+		}
+		buf.Write(bytes.Join(lines, []byte("\n")))
+		if i < len(pkgs)-1 {
+			buf.WriteString("\n\n")
+		} else {
+			buf.WriteString("\n")
+		}
+	}
+	return buf.Bytes()
+}
+
+// diskPresenceChecker returns a presentOnDisk func rooted at localDir.
+func diskPresenceChecker(localDir string) func(string) bool {
+	return func(filename string) bool {
+		p := filepath.Join(localDir, filepath.FromSlash(filename))
+		_, err := os.Stat(p)
+		return err == nil
+	}
+}

internal/clone/apt/release.go

@@ -0,0 +1,154 @@
+package apt
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ReleaseFile represents a parsed Debian Release / InRelease file.
+type ReleaseFile struct {
+	Origin        string
+	Suite         string
+	Codename      string
+	Components    string
+	Architectures string
+	SHA256        []ReleaseEntry
+}
+
+// ReleaseEntry is a single line from the SHA256 section of a Release file.
+type ReleaseEntry struct {
+	Hash string
+	Size int64
+	Path string
+}
+
+// ParseRelease strips PGP armor (if present) and parses an InRelease or Release file.
+func ParseRelease(data []byte) (*ReleaseFile, error) {
+	text := stripPGPArmor(string(data))
+
+	rf := &ReleaseFile{}
+	inSHA256 := false
+
+	for _, rawLine := range strings.Split(text, "\n") {
+		line := strings.TrimRight(rawLine, " \t\r")
+
+		if inSHA256 {
+			if line == "" || (len(line) > 0 && line[0] != ' ' && line[0] != '\t') {
+				// A non-indented non-empty line ends the SHA256 block.
+				if line != "" {
+					inSHA256 = false
+					// Fall through to parse this line as a key-value pair.
+				} else {
+					inSHA256 = false
+					continue
+				}
+			} else {
+				// Indented line: " <hash> <size> <path>"
+				entry, err := parseReleaseEntry(line)
+				if err == nil {
+					rf.SHA256 = append(rf.SHA256, entry)
+				}
+				continue
+			}
+		}
+
+		if strings.HasPrefix(line, "SHA256:") {
+			inSHA256 = true
+			continue
+		}
+
+		key, value, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+
+		switch key {
+		case "Origin":
+			rf.Origin = value
+		case "Suite":
+			rf.Suite = value
+		case "Codename":
+			rf.Codename = value
+		case "Components":
+			rf.Components = value
+		case "Architectures":
+			rf.Architectures = value
+		}
+	}
+
+	return rf, nil
+}
+
+// parseReleaseEntry parses a single indented line from the SHA256 section.
+// Format: " <hash> <size> <path>"
+func parseReleaseEntry(line string) (ReleaseEntry, error) {
+	fields := strings.Fields(line)
+	if len(fields) != 3 {
+		return ReleaseEntry{}, fmt.Errorf("malformed SHA256 entry: %q", line)
+	}
+	size, err := strconv.ParseInt(fields[1], 10, 64)
+	if err != nil {
+		return ReleaseEntry{}, fmt.Errorf("malformed size in SHA256 entry: %w", err)
+	}
+	return ReleaseEntry{Hash: fields[0], Size: size, Path: fields[2]}, nil
+}
+
+// stripPGPArmor removes the PGP signed-message wrapper if present.
+// It returns only the signed body (between the header and the signature).
+func stripPGPArmor(text string) string {
+	const beginSigned = "-----BEGIN PGP SIGNED MESSAGE-----"
+	const beginSig = "-----BEGIN PGP SIGNATURE-----"
+
+	if !strings.Contains(text, beginSigned) {
+		return text
+	}
+
+	// Drop the header lines (Hash: etc.) up to the first blank line.
+	after, found := strings.CutPrefix(text, beginSigned)
+	if !found {
+		// beginSigned not at start — find it
+		idx := strings.Index(text, beginSigned)
+		if idx < 0 {
+			return text
+		}
+		after = text[idx+len(beginSigned):]
+	}
+
+	// Skip the armor header lines (e.g. "Hash: SHA512") until the blank line.
+	blankIdx := strings.Index(after, "\n\n")
+	if blankIdx < 0 {
+		return text
+	}
+	body := after[blankIdx+2:]
+
+	// Cut off at the PGP signature block.
+	if sigIdx := strings.Index(body, beginSig); sigIdx >= 0 {
+		body = body[:sigIdx]
+	}
+
+	return strings.TrimRight(body, "\n\r ") + "\n"
+}
+
+// GenerateRelease generates a plain unsigned Release file.
+func GenerateRelease(suite, codename string, components, architectures []string, entries []ReleaseEntry) []byte {
+	var buf bytes.Buffer
+
+	fmt.Fprintf(&buf, "Origin: ClonePack Mirror\n")
+	fmt.Fprintf(&buf, "Suite: %s\n", suite)
+	fmt.Fprintf(&buf, "Codename: %s\n", codename)
+	fmt.Fprintf(&buf, "Components: %s\n", strings.Join(components, " "))
+	fmt.Fprintf(&buf, "Architectures: %s\n", strings.Join(architectures, " "))
+	fmt.Fprintf(&buf, "Date: %s\n", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 UTC"))
+	fmt.Fprintf(&buf, "SHA256:\n")
+
+	for _, e := range entries {
+		fmt.Fprintf(&buf, " %s %d %s\n", e.Hash, e.Size, e.Path)
+	}
+
+	return buf.Bytes()
+}

internal/clone/apt/scanner.go

@@ -0,0 +1,145 @@
+package apt
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// NewPackage is a package present upstream but absent from the local mirror.
+type NewPackage struct {
+	Package      string
+	Version      string
+	Architecture string
+	Filename     string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb
+	SHA256       string
+	Size         int64
+	Component    string
+}
+
+// Scanner compares an upstream APT repository against a local mirror and
+// returns packages that are present upstream but missing on disk.
+type Scanner struct {
+	SourceURL  string
+	LocalDir   string
+	Config     Config
+	HTTPClient *http.Client
+}
+
+// NewScanner creates a Scanner with a 5-minute HTTP timeout.
+func NewScanner(sourceURL, localDir string, cfg Config) *Scanner {
+	return &Scanner{
+		SourceURL:  strings.TrimRight(sourceURL, "/"),
+		LocalDir:   localDir,
+		Config:     cfg,
+		HTTPClient: &http.Client{Timeout: 5 * time.Minute},
+	}
+}
+
+// Scan fetches the upstream package lists and returns packages not on disk.
+func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) {
+	suite := sc.Config.Suite
+
+	// 1. Fetch InRelease.
+	inReleaseURL := sc.SourceURL + "/dists/" + suite + "/InRelease"
+	inReleaseData, err := fetchBytes(ctx, sc.HTTPClient, inReleaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("fetch InRelease: %w", err)
+	}
+
+	rf, err := ParseRelease(inReleaseData)
+	if err != nil {
+		return nil, fmt.Errorf("parse InRelease: %w", err)
+	}
+
+	// 2 & 3. Fetch Packages.gz for each component/arch, save it, then filter.
+	var missing []NewPackage
+
+	for _, component := range sc.Config.Components {
+		for _, arch := range sc.Config.Architectures {
+			if err := ctx.Err(); err != nil {
+				return nil, err
+			}
+			pkgs, err := sc.fetchAndSavePackages(ctx, suite, component, arch, rf)
+			if err != nil {
+				return nil, fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
+			}
+
+			// 4. Filter: only return packages whose Filename is NOT on disk.
+			checker := diskPresenceChecker(sc.LocalDir)
+			for _, pkg := range pkgs {
+				if !checker(pkg.Filename) {
+					missing = append(missing, NewPackage{
+						Package:      pkg.Package,
+						Version:      pkg.Version,
+						Architecture: pkg.Architecture,
+						Filename:     pkg.Filename,
+						SHA256:       pkg.SHA256,
+						Size:         pkg.Size,
+						Component:    component,
+					})
+				}
+			}
+		}
+	}
+
+	return missing, nil
+}
+
+// fetchAndSavePackages fetches Packages.gz for a component/arch, saves it as
+// .upstream-Packages.gz (so RegenerateMetadata can use it), and returns parsed packages.
+func (sc *Scanner) fetchAndSavePackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
+	wantPath := component + "/binary-" + arch + "/Packages.gz"
+
+	// Check that this path is listed in the upstream SHA256 section.
+	var found bool
+	for _, e := range rf.SHA256 {
+		if e.Path == wantPath {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return nil, nil
+	}
+
+	pkgGzURL := sc.SourceURL + "/dists/" + suite + "/" + wantPath
+	data, err := fetchBytes(ctx, sc.HTTPClient, pkgGzURL)
+	if err != nil {
+		return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
+	}
+
+	// Save as .upstream-Packages.gz.
+	destDir := filepath.Join(sc.LocalDir, "dists", suite, component, "binary-"+arch)
+	if err := os.MkdirAll(destDir, 0o755); err != nil {
+		return nil, err
+	}
+	upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
+	if err := atomicWrite(upstreamPath, data); err != nil {
+		return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
+	}
+
+	// Decompress and parse.
+	gz, err := gzip.NewReader(strings.NewReader(string(data)))
+	if err != nil {
+		return nil, fmt.Errorf("open gzip: %w", err)
+	}
+	defer gz.Close()
+
+	raw, err := io.ReadAll(gz)
+	if err != nil {
+		return nil, fmt.Errorf("decompress: %w", err)
+	}
+
+	pkgs, err := ParsePackages(raw)
+	if err != nil {
+		return nil, fmt.Errorf("parse Packages: %w", err)
+	}
+	return pkgs, nil
+}