package apt import ( "compress/gzip" "context" "fmt" "io" "net/http" "os" "path/filepath" "strings" "time" ) const ( upstreamInRelease = ".upstream-InRelease" upstreamPackagesGz = ".upstream-Packages.gz" ) // Cloner mirrors an APT repository into a local directory. type Cloner struct { SourceURL string DestDir string Config Config HTTPClient *http.Client OnProgress func(file string, bytes int64) } // New creates a Cloner with a 30-minute HTTP timeout. func New(sourceURL, destDir string, cfg Config) *Cloner { return &Cloner{ SourceURL: strings.TrimRight(sourceURL, "/"), DestDir: destDir, Config: cfg, HTTPClient: &http.Client{Timeout: 30 * time.Minute}, } } // Clone performs a full mirror of the configured APT repository. func (c *Cloner) Clone(ctx context.Context) error { suite := c.Config.Suite // 1. Create base directories. if err := os.MkdirAll(filepath.Join(c.DestDir, "dists", suite), 0o755); err != nil { return fmt.Errorf("create dists dir: %w", err) } if err := os.MkdirAll(filepath.Join(c.DestDir, "pool"), 0o755); err != nil { return fmt.Errorf("create pool dir: %w", err) } // 2. Fetch InRelease and save as .upstream-InRelease. inReleaseURL := c.SourceURL + "/dists/" + suite + "/InRelease" inReleaseData, err := fetchBytes(ctx, c.HTTPClient, inReleaseURL) if err != nil { return fmt.Errorf("fetch InRelease: %w", err) } upstreamIRPath := filepath.Join(c.DestDir, "dists", suite, upstreamInRelease) if err := atomicWrite(upstreamIRPath, inReleaseData); err != nil { return fmt.Errorf("save upstream InRelease: %w", err) } // 3. Parse InRelease. rf, err := ParseRelease(inReleaseData) if err != nil { return fmt.Errorf("parse InRelease: %w", err) } // 4. For each component/arch: find Packages.gz in SHA256 section, download, parse debs. var allDebs []DebPackage for _, component := range c.Config.Components { for _, arch := range c.Config.Architectures { if err := ctx.Err(); err != nil { return err } pkgs, err := c.fetchComponentPackages(ctx, suite, component, arch, rf) if err != nil { return fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err) } allDebs = append(allDebs, pkgs...) } } // 5. Download all .deb files. for _, pkg := range allDebs { if err := ctx.Err(); err != nil { return err } if err := c.downloadDeb(ctx, pkg); err != nil { return fmt.Errorf("download %s: %w", pkg.Filename, err) } } // 6. Regenerate local metadata. if err := RegenerateMetadata(c.DestDir, c.Config); err != nil { return fmt.Errorf("regenerate metadata: %w", err) } return nil } // fetchComponentPackages downloads Packages.gz for one component/arch pair, // saves it as .upstream-Packages.gz, and returns the parsed package list. func (c *Cloner) fetchComponentPackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) { wantPath := component + "/binary-" + arch + "/Packages.gz" // Verify this path is listed in the upstream SHA256 section. var found bool for _, e := range rf.SHA256 { if e.Path == wantPath { found = true break } } if !found { // Not an error — this component/arch may just not exist upstream. return nil, nil } pkgGzURL := c.SourceURL + "/dists/" + suite + "/" + wantPath data, err := fetchBytes(ctx, c.HTTPClient, pkgGzURL) if err != nil { return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err) } // Save as .upstream-Packages.gz so RegenerateMetadata can find it later. destDir := filepath.Join(c.DestDir, "dists", suite, component, "binary-"+arch) if err := os.MkdirAll(destDir, 0o755); err != nil { return nil, err } upstreamPath := filepath.Join(destDir, upstreamPackagesGz) if err := atomicWrite(upstreamPath, data); err != nil { return nil, fmt.Errorf("save %s: %w", upstreamPath, err) } if c.OnProgress != nil { c.OnProgress(wantPath, int64(len(data))) } // Decompress and parse. gz, err := gzip.NewReader(strings.NewReader(string(data))) if err != nil { return nil, fmt.Errorf("open gzip for %s: %w", wantPath, err) } defer gz.Close() raw, err := io.ReadAll(gz) if err != nil { return nil, fmt.Errorf("decompress %s: %w", wantPath, err) } pkgs, err := ParsePackages(raw) if err != nil { return nil, fmt.Errorf("parse Packages for %s: %w", wantPath, err) } return pkgs, nil } // downloadDeb downloads a single .deb file into the local pool tree. func (c *Cloner) downloadDeb(ctx context.Context, pkg DebPackage) error { url := c.SourceURL + "/" + pkg.Filename dest := filepath.Join(c.DestDir, filepath.FromSlash(pkg.Filename)) // Skip if already on disk. if _, err := os.Stat(dest); err == nil { return nil } if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { return err } data, err := fetchBytes(ctx, c.HTTPClient, url) if err != nil { return err } if err := atomicWrite(dest, data); err != nil { return err } if c.OnProgress != nil { c.OnProgress(pkg.Filename, int64(len(data))) } return nil }