clonepack/internal/clone/apt/cloner.go

package apt

import (
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	upstreamInRelease  = ".upstream-InRelease"
	upstreamPackagesGz = ".upstream-Packages.gz"
)

// Cloner mirrors an APT repository into a local directory.
type Cloner struct {
	SourceURL  string
	DestDir    string
	Config     Config
	HTTPClient *http.Client
	OnProgress func(file string, bytes int64)
}

// New creates a Cloner with a 30-minute HTTP timeout.
func New(sourceURL, destDir string, cfg Config) *Cloner {
	return &Cloner{
		SourceURL:  strings.TrimRight(sourceURL, "/"),
		DestDir:    destDir,
		Config:     cfg,
		HTTPClient: &http.Client{Timeout: 30 * time.Minute},
	}
}

// Clone performs a full mirror of the configured APT repository.
func (c *Cloner) Clone(ctx context.Context) error {
	suite := c.Config.Suite

	// 1. Create base directories.
	if err := os.MkdirAll(filepath.Join(c.DestDir, "dists", suite), 0o755); err != nil {
		return fmt.Errorf("create dists dir: %w", err)
	}
	if err := os.MkdirAll(filepath.Join(c.DestDir, "pool"), 0o755); err != nil {
		return fmt.Errorf("create pool dir: %w", err)
	}

	// 2. Fetch InRelease and save as .upstream-InRelease.
	inReleaseURL := c.SourceURL + "/dists/" + suite + "/InRelease"
	inReleaseData, err := fetchBytes(ctx, c.HTTPClient, inReleaseURL)
	if err != nil {
		return fmt.Errorf("fetch InRelease: %w", err)
	}
	upstreamIRPath := filepath.Join(c.DestDir, "dists", suite, upstreamInRelease)
	if err := atomicWrite(upstreamIRPath, inReleaseData); err != nil {
		return fmt.Errorf("save upstream InRelease: %w", err)
	}

	// 3. Parse InRelease.
	rf, err := ParseRelease(inReleaseData)
	if err != nil {
		return fmt.Errorf("parse InRelease: %w", err)
	}

	// 4. For each component/arch: find Packages.gz in SHA256 section, download, parse debs.
	var allDebs []DebPackage
	for _, component := range c.Config.Components {
		for _, arch := range c.Config.Architectures {
			if err := ctx.Err(); err != nil {
				return err
			}
			pkgs, err := c.fetchComponentPackages(ctx, suite, component, arch, rf)
			if err != nil {
				return fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
			}
			allDebs = append(allDebs, pkgs...)
		}
	}

	// 5. Download all .deb files.
	for _, pkg := range allDebs {
		if err := ctx.Err(); err != nil {
			return err
		}
		if err := c.downloadDeb(ctx, pkg); err != nil {
			return fmt.Errorf("download %s: %w", pkg.Filename, err)
		}
	}

	// 6. Regenerate local metadata.
	if err := RegenerateMetadata(c.DestDir, c.Config); err != nil {
		return fmt.Errorf("regenerate metadata: %w", err)
	}

	return nil
}

// fetchComponentPackages downloads Packages.gz for one component/arch pair,
// saves it as .upstream-Packages.gz, and returns the parsed package list.
func (c *Cloner) fetchComponentPackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
	wantPath := component + "/binary-" + arch + "/Packages.gz"

	// Verify this path is listed in the upstream SHA256 section.
	var found bool
	for _, e := range rf.SHA256 {
		if e.Path == wantPath {
			found = true
			break
		}
	}
	if !found {
		// Not an error — this component/arch may just not exist upstream.
		return nil, nil
	}

	pkgGzURL := c.SourceURL + "/dists/" + suite + "/" + wantPath
	data, err := fetchBytes(ctx, c.HTTPClient, pkgGzURL)
	if err != nil {
		return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
	}

	// Save as .upstream-Packages.gz so RegenerateMetadata can find it later.
	destDir := filepath.Join(c.DestDir, "dists", suite, component, "binary-"+arch)
	if err := os.MkdirAll(destDir, 0o755); err != nil {
		return nil, err
	}
	upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
	if err := atomicWrite(upstreamPath, data); err != nil {
		return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
	}

	if c.OnProgress != nil {
		c.OnProgress(wantPath, int64(len(data)))
	}

	// Decompress and parse.
	gz, err := gzip.NewReader(strings.NewReader(string(data)))
	if err != nil {
		return nil, fmt.Errorf("open gzip for %s: %w", wantPath, err)
	}
	defer gz.Close()

	raw, err := io.ReadAll(gz)
	if err != nil {
		return nil, fmt.Errorf("decompress %s: %w", wantPath, err)
	}

	pkgs, err := ParsePackages(raw)
	if err != nil {
		return nil, fmt.Errorf("parse Packages for %s: %w", wantPath, err)
	}
	return pkgs, nil
}

// downloadDeb downloads a single .deb file into the local pool tree.
func (c *Cloner) downloadDeb(ctx context.Context, pkg DebPackage) error {
	url := c.SourceURL + "/" + pkg.Filename
	dest := filepath.Join(c.DestDir, filepath.FromSlash(pkg.Filename))

	// Skip if already on disk.
	if _, err := os.Stat(dest); err == nil {
		return nil
	}

	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
		return err
	}

	data, err := fetchBytes(ctx, c.HTTPClient, url)
	if err != nil {
		return err
	}
	if err := atomicWrite(dest, data); err != nil {
		return err
	}

	if c.OnProgress != nil {
		c.OnProgress(pkg.Filename, int64(len(data)))
	}
	return nil
}