145 lines
3.9 KiB
Go
145 lines
3.9 KiB
Go
package apt
|
|
|
|
import (
|
|
"compress/gzip"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// NewPackage is a package present upstream but absent from the local mirror.
|
|
type NewPackage struct {
|
|
Package string
|
|
Version string
|
|
Architecture string
|
|
Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb
|
|
SHA256 string
|
|
Size int64
|
|
Component string
|
|
}
|
|
|
|
// Scanner compares an upstream APT repository against a local mirror and
|
|
// returns packages that are present upstream but missing on disk.
|
|
type Scanner struct {
|
|
SourceURL string
|
|
LocalDir string
|
|
Config Config
|
|
HTTPClient *http.Client
|
|
}
|
|
|
|
// NewScanner creates a Scanner with a 5-minute HTTP timeout.
|
|
func NewScanner(sourceURL, localDir string, cfg Config) *Scanner {
|
|
return &Scanner{
|
|
SourceURL: strings.TrimRight(sourceURL, "/"),
|
|
LocalDir: localDir,
|
|
Config: cfg,
|
|
HTTPClient: &http.Client{Timeout: 5 * time.Minute},
|
|
}
|
|
}
|
|
|
|
// Scan fetches the upstream package lists and returns packages not on disk.
|
|
func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) {
|
|
suite := sc.Config.Suite
|
|
|
|
// 1. Fetch InRelease.
|
|
inReleaseURL := sc.SourceURL + "/dists/" + suite + "/InRelease"
|
|
inReleaseData, err := fetchBytes(ctx, sc.HTTPClient, inReleaseURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetch InRelease: %w", err)
|
|
}
|
|
|
|
rf, err := ParseRelease(inReleaseData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse InRelease: %w", err)
|
|
}
|
|
|
|
// 2 & 3. Fetch Packages.gz for each component/arch, save it, then filter.
|
|
var missing []NewPackage
|
|
|
|
for _, component := range sc.Config.Components {
|
|
for _, arch := range sc.Config.Architectures {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
pkgs, err := sc.fetchAndSavePackages(ctx, suite, component, arch, rf)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
|
|
}
|
|
|
|
// 4. Filter: only return packages whose Filename is NOT on disk.
|
|
checker := diskPresenceChecker(sc.LocalDir)
|
|
for _, pkg := range pkgs {
|
|
if !checker(pkg.Filename) {
|
|
missing = append(missing, NewPackage{
|
|
Package: pkg.Package,
|
|
Version: pkg.Version,
|
|
Architecture: pkg.Architecture,
|
|
Filename: pkg.Filename,
|
|
SHA256: pkg.SHA256,
|
|
Size: pkg.Size,
|
|
Component: component,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return missing, nil
|
|
}
|
|
|
|
// fetchAndSavePackages fetches Packages.gz for a component/arch, saves it as
|
|
// .upstream-Packages.gz (so RegenerateMetadata can use it), and returns parsed packages.
|
|
func (sc *Scanner) fetchAndSavePackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
|
|
wantPath := component + "/binary-" + arch + "/Packages.gz"
|
|
|
|
// Check that this path is listed in the upstream SHA256 section.
|
|
var found bool
|
|
for _, e := range rf.SHA256 {
|
|
if e.Path == wantPath {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return nil, nil
|
|
}
|
|
|
|
pkgGzURL := sc.SourceURL + "/dists/" + suite + "/" + wantPath
|
|
data, err := fetchBytes(ctx, sc.HTTPClient, pkgGzURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
|
|
}
|
|
|
|
// Save as .upstream-Packages.gz.
|
|
destDir := filepath.Join(sc.LocalDir, "dists", suite, component, "binary-"+arch)
|
|
if err := os.MkdirAll(destDir, 0o755); err != nil {
|
|
return nil, err
|
|
}
|
|
upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
|
|
if err := atomicWrite(upstreamPath, data); err != nil {
|
|
return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
|
|
}
|
|
|
|
// Decompress and parse.
|
|
gz, err := gzip.NewReader(strings.NewReader(string(data)))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open gzip: %w", err)
|
|
}
|
|
defer gz.Close()
|
|
|
|
raw, err := io.ReadAll(gz)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decompress: %w", err)
|
|
}
|
|
|
|
pkgs, err := ParsePackages(raw)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse Packages: %w", err)
|
|
}
|
|
return pkgs, nil
|
|
}
|