package apt import ( "compress/gzip" "context" "fmt" "io" "net/http" "os" "path/filepath" "strings" "time" ) // NewPackage is a package present upstream but absent from the local mirror. type NewPackage struct { Package string Version string Architecture string Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb SHA256 string Size int64 Component string } // Scanner compares an upstream APT repository against a local mirror and // returns packages that are present upstream but missing on disk. type Scanner struct { SourceURL string LocalDir string Config Config HTTPClient *http.Client } // NewScanner creates a Scanner with a 5-minute HTTP timeout. func NewScanner(sourceURL, localDir string, cfg Config) *Scanner { return &Scanner{ SourceURL: strings.TrimRight(sourceURL, "/"), LocalDir: localDir, Config: cfg, HTTPClient: &http.Client{Timeout: 5 * time.Minute}, } } // Scan fetches the upstream package lists and returns packages not on disk. func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) { suite := sc.Config.Suite // 1. Fetch InRelease. inReleaseURL := sc.SourceURL + "/dists/" + suite + "/InRelease" inReleaseData, err := fetchBytes(ctx, sc.HTTPClient, inReleaseURL) if err != nil { return nil, fmt.Errorf("fetch InRelease: %w", err) } rf, err := ParseRelease(inReleaseData) if err != nil { return nil, fmt.Errorf("parse InRelease: %w", err) } // 2 & 3. Fetch Packages.gz for each component/arch, save it, then filter. var missing []NewPackage for _, component := range sc.Config.Components { for _, arch := range sc.Config.Architectures { if err := ctx.Err(); err != nil { return nil, err } pkgs, err := sc.fetchAndSavePackages(ctx, suite, component, arch, rf) if err != nil { return nil, fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err) } // 4. Filter: only return packages whose Filename is NOT on disk. checker := diskPresenceChecker(sc.LocalDir) for _, pkg := range pkgs { if !checker(pkg.Filename) { missing = append(missing, NewPackage{ Package: pkg.Package, Version: pkg.Version, Architecture: pkg.Architecture, Filename: pkg.Filename, SHA256: pkg.SHA256, Size: pkg.Size, Component: component, }) } } } } return missing, nil } // fetchAndSavePackages fetches Packages.gz for a component/arch, saves it as // .upstream-Packages.gz (so RegenerateMetadata can use it), and returns parsed packages. func (sc *Scanner) fetchAndSavePackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) { wantPath := component + "/binary-" + arch + "/Packages.gz" // Check that this path is listed in the upstream SHA256 section. var found bool for _, e := range rf.SHA256 { if e.Path == wantPath { found = true break } } if !found { return nil, nil } pkgGzURL := sc.SourceURL + "/dists/" + suite + "/" + wantPath data, err := fetchBytes(ctx, sc.HTTPClient, pkgGzURL) if err != nil { return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err) } // Save as .upstream-Packages.gz. destDir := filepath.Join(sc.LocalDir, "dists", suite, component, "binary-"+arch) if err := os.MkdirAll(destDir, 0o755); err != nil { return nil, err } upstreamPath := filepath.Join(destDir, upstreamPackagesGz) if err := atomicWrite(upstreamPath, data); err != nil { return nil, fmt.Errorf("save %s: %w", upstreamPath, err) } // Decompress and parse. gz, err := gzip.NewReader(strings.NewReader(string(data))) if err != nil { return nil, fmt.Errorf("open gzip: %w", err) } defer gz.Close() raw, err := io.ReadAll(gz) if err != nil { return nil, fmt.Errorf("decompress: %w", err) } pkgs, err := ParsePackages(raw) if err != nil { return nil, fmt.Errorf("parse Packages: %w", err) } return pkgs, nil }