syonad/clonepack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add apt gestion

Browse source 
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-04-25 19:03:26 +02:00
commit c4776d81dd
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
22 changed files with 1465 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

18
internal/api/proxy_handler.go
View file

@ -21,13 +21,15 @@ func NewProxyHandler(repoSvc *core.RepoService, dataDir string) *ProxyHandler {
}
func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "repo_id"), 10, 64)
if err != nil {
http.Error(w, "invalid repo_id", http.StatusBadRequest)
return
}
ref := chi.URLParam(r, "repo_ref")
repo, err := h.repoSvc.Get(r.Context(), repoID)
var repo *store.Repo
var err error
if id, parseErr := strconv.ParseInt(ref, 10, 64); parseErr == nil {
repo, err = h.repoSvc.Get(r.Context(), id)
} else {
repo, err = h.repoSvc.GetByName(r.Context(), ref)
}
if errors.Is(err, store.ErrNotFound) {
http.NotFound(w, r)
return
@ -37,7 +39,7 @@ func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) {
return
}
localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repoID, repo.Type)
prefix := fmt.Sprintf("/mirror/%d", repoID)
localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repo.ID, repo.Type)
prefix := fmt.Sprintf("/mirror/%s", ref)
http.StripPrefix(prefix, http.FileServer(http.Dir(localDir))).ServeHTTP(w, r)
}

54
internal/api/repo_handler.go
View file

@ -7,6 +7,7 @@ import (
"strconv"
"github.com/go-chi/chi/v5"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
"github.com/syonad/clonepack/internal/core"
"github.com/syonad/clonepack/internal/store"
)
@ -27,10 +28,13 @@ func (h *RepoHandler) Create(w http.ResponseWriter, r *http.Request) {
}
repo, err := h.svc.Create(r.Context(), core.CreateRepoInput{
Name: req.Name,
Type: req.Type,
SourceURL: req.SourceURL,
SyncMode: req.SyncMode,
Name: req.Name,
Type: req.Type,
SourceURL: req.SourceURL,
SyncMode: req.SyncMode,
AptSuite: req.AptSuite,
AptComponents: req.AptComponents,
AptArchitectures: req.AptArchitectures,
})
if err != nil {
Error(w, http.StatusUnprocessableEntity, err.Error())
@ -72,6 +76,37 @@ func (h *RepoHandler) Get(w http.ResponseWriter, r *http.Request) {
JSON(w, http.StatusOK, repoToResponse(repo))
}
func (h *RepoHandler) Update(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var req UpdateRepoRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
Error(w, http.StatusBadRequest, "invalid JSON")
return
}
repo, err := h.svc.Update(r.Context(), id, core.UpdateRepoInput{
SourceURL: req.SourceURL,
SyncMode: req.SyncMode,
AptSuite: req.AptSuite,
AptComponents: req.AptComponents,
AptArchitectures: req.AptArchitectures,
})
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
}
if err != nil {
Error(w, http.StatusUnprocessableEntity, err.Error())
return
}
JSON(w, http.StatusOK, repoToResponse(repo))
}
func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
@ -90,7 +125,7 @@ func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) {
}
func repoToResponse(r *store.Repo) RepoResponse {
return RepoResponse{
resp := RepoResponse{
ID: r.ID,
Name: r.Name,
Type: r.Type,
@ -99,4 +134,13 @@ func repoToResponse(r *store.Repo) RepoResponse {
SyncMode: r.SyncMode,
CreatedAt: r.CreatedAt,
}
if r.Type == "apt" && r.Config != "" {
var cfg aptclone.Config
if err := json.Unmarshal([]byte(r.Config), &cfg); err == nil {
resp.AptSuite = cfg.Suite
resp.AptComponents = cfg.Components
resp.AptArchitectures = cfg.Architectures
}
}
return resp
}

3
internal/api/router.go
View file

@ -13,7 +13,7 @@ func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler
r.Use(middleware.Recoverer)
r.Use(middleware.RequestID)
r.Get("/mirror/{repo_id}/*", proxyHandler.ServeFile)
r.Get("/mirror/{repo_ref}/*", proxyHandler.ServeFile)
r.Get("/health", func(w http.ResponseWriter, r *http.Request) {
JSON(w, http.StatusOK, map[string]string{"status": "ok"})
@ -24,6 +24,7 @@ func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler
r.Post("/", repoHandler.Create)
r.Get("/", repoHandler.List)
r.Get("/{id}", repoHandler.Get)
r.Patch("/{id}", repoHandler.Update)
r.Delete("/{id}", repoHandler.Delete)
r.Post("/{id}/clone", cloneHandler.StartClone)

36
internal/api/types.go
View file

@ -3,20 +3,26 @@ package api
import "time"
type CreateRepoRequest struct {
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode,omitempty"`
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode,omitempty"`
AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
}
type RepoResponse struct {
ID int64 `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
Frozen bool `json:"frozen"`
SyncMode string `json:"sync_mode"`
CreatedAt time.Time `json:"created_at"`
ID int64 `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
Frozen bool `json:"frozen"`
SyncMode string `json:"sync_mode"`
AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
type ListReposResponse struct {
@ -24,6 +30,14 @@ type ListReposResponse struct {
Total int `json:"total"`
}
type UpdateRepoRequest struct {
SourceURL *string `json:"source_url,omitempty"`
SyncMode *string `json:"sync_mode,omitempty"`
AptSuite *string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
}
type ErrorResponse struct {
Error string `json:"error"`
}

184
internal/clone/apt/cloner.go Normal file
View file

@ -0,0 +1,184 @@
package apt
import (
"compress/gzip"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
const (
upstreamInRelease = ".upstream-InRelease"
upstreamPackagesGz = ".upstream-Packages.gz"
)
// Cloner mirrors an APT repository into a local directory.
type Cloner struct {
SourceURL string
DestDir string
Config Config
HTTPClient *http.Client
OnProgress func(file string, bytes int64)
}
// New creates a Cloner with a 30-minute HTTP timeout.
func New(sourceURL, destDir string, cfg Config) *Cloner {
return &Cloner{
SourceURL: strings.TrimRight(sourceURL, "/"),
DestDir: destDir,
Config: cfg,
HTTPClient: &http.Client{Timeout: 30 * time.Minute},
}
}
// Clone performs a full mirror of the configured APT repository.
func (c *Cloner) Clone(ctx context.Context) error {
suite := c.Config.Suite
// 1. Create base directories.
if err := os.MkdirAll(filepath.Join(c.DestDir, "dists", suite), 0o755); err != nil {
return fmt.Errorf("create dists dir: %w", err)
}
if err := os.MkdirAll(filepath.Join(c.DestDir, "pool"), 0o755); err != nil {
return fmt.Errorf("create pool dir: %w", err)
}
// 2. Fetch InRelease and save as .upstream-InRelease.
inReleaseURL := c.SourceURL + "/dists/" + suite + "/InRelease"
inReleaseData, err := fetchBytes(ctx, c.HTTPClient, inReleaseURL)
if err != nil {
return fmt.Errorf("fetch InRelease: %w", err)
}
upstreamIRPath := filepath.Join(c.DestDir, "dists", suite, upstreamInRelease)
if err := atomicWrite(upstreamIRPath, inReleaseData); err != nil {
return fmt.Errorf("save upstream InRelease: %w", err)
}
// 3. Parse InRelease.
rf, err := ParseRelease(inReleaseData)
if err != nil {
return fmt.Errorf("parse InRelease: %w", err)
}
// 4. For each component/arch: find Packages.gz in SHA256 section, download, parse debs.
var allDebs []DebPackage
for _, component := range c.Config.Components {
for _, arch := range c.Config.Architectures {
if err := ctx.Err(); err != nil {
return err
}
pkgs, err := c.fetchComponentPackages(ctx, suite, component, arch, rf)
if err != nil {
return fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
}
allDebs = append(allDebs, pkgs...)
}
}
// 5. Download all .deb files.
for _, pkg := range allDebs {
if err := ctx.Err(); err != nil {
return err
}
if err := c.downloadDeb(ctx, pkg); err != nil {
return fmt.Errorf("download %s: %w", pkg.Filename, err)
}
}
// 6. Regenerate local metadata.
if err := RegenerateMetadata(c.DestDir, c.Config); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
return nil
}
// fetchComponentPackages downloads Packages.gz for one component/arch pair,
// saves it as .upstream-Packages.gz, and returns the parsed package list.
func (c *Cloner) fetchComponentPackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
wantPath := component + "/binary-" + arch + "/Packages.gz"
// Verify this path is listed in the upstream SHA256 section.
var found bool
for _, e := range rf.SHA256 {
if e.Path == wantPath {
found = true
break
}
}
if !found {
// Not an error — this component/arch may just not exist upstream.
return nil, nil
}
pkgGzURL := c.SourceURL + "/dists/" + suite + "/" + wantPath
data, err := fetchBytes(ctx, c.HTTPClient, pkgGzURL)
if err != nil {
return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
}
// Save as .upstream-Packages.gz so RegenerateMetadata can find it later.
destDir := filepath.Join(c.DestDir, "dists", suite, component, "binary-"+arch)
if err := os.MkdirAll(destDir, 0o755); err != nil {
return nil, err
}
upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
if err := atomicWrite(upstreamPath, data); err != nil {
return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
}
if c.OnProgress != nil {
c.OnProgress(wantPath, int64(len(data)))
}
// Decompress and parse.
gz, err := gzip.NewReader(strings.NewReader(string(data)))
if err != nil {
return nil, fmt.Errorf("open gzip for %s: %w", wantPath, err)
}
defer gz.Close()
raw, err := io.ReadAll(gz)
if err != nil {
return nil, fmt.Errorf("decompress %s: %w", wantPath, err)
}
pkgs, err := ParsePackages(raw)
if err != nil {
return nil, fmt.Errorf("parse Packages for %s: %w", wantPath, err)
}
return pkgs, nil
}
// downloadDeb downloads a single .deb file into the local pool tree.
func (c *Cloner) downloadDeb(ctx context.Context, pkg DebPackage) error {
url := c.SourceURL + "/" + pkg.Filename
dest := filepath.Join(c.DestDir, filepath.FromSlash(pkg.Filename))
// Skip if already on disk.
if _, err := os.Stat(dest); err == nil {
return nil
}
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return err
}
data, err := fetchBytes(ctx, c.HTTPClient, url)
if err != nil {
return err
}
if err := atomicWrite(dest, data); err != nil {
return err
}
if c.OnProgress != nil {
c.OnProgress(pkg.Filename, int64(len(data)))
}
return nil
}

8
internal/clone/apt/config.go Normal file
View file

@ -0,0 +1,8 @@
package apt
// Config describes which parts of an APT repository to mirror.
type Config struct {
Suite string `json:"suite"`
Components []string `json:"components"`
Architectures []string `json:"architectures"`
}

44
internal/clone/apt/helpers.go Normal file
View file

@ -0,0 +1,44 @@
package apt
import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
)
// fetchBytes performs a GET request and returns the response body.
func fetchBytes(ctx context.Context, httpClient *http.Client, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
return io.ReadAll(resp.Body)
}
// atomicWrite writes data to path via a temporary file and an atomic rename.
func atomicWrite(path string, data []byte) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return err
}
if err := os.Rename(tmp, path); err != nil {
os.Remove(tmp)
return err
}
return nil
}

148
internal/clone/apt/metadata_gen.go Normal file
View file

@ -0,0 +1,148 @@
package apt
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
)
// InitEmptyRepo creates the directory structure for an APT mirror and writes an
// empty Release file so APT clients don't get a 404 before the first clone.
func InitEmptyRepo(localDir string, cfg Config) error {
if err := os.MkdirAll(filepath.Join(localDir, "pool"), 0o755); err != nil {
return err
}
for _, component := range cfg.Components {
for _, arch := range cfg.Architectures {
dir := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch)
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
}
}
releasePath := filepath.Join(localDir, "dists", cfg.Suite, "Release")
if _, err := os.Stat(releasePath); os.IsNotExist(err) {
releaseData := GenerateRelease(cfg.Suite, cfg.Suite, cfg.Components, cfg.Architectures, nil)
if err := atomicWrite(releasePath, releaseData); err != nil {
return fmt.Errorf("write empty Release: %w", err)
}
}
return nil
}
// RegenerateMetadata rebuilds local Packages, Packages.gz, and Release files
// for all configured component/arch pairs, keeping only packages present on disk.
func RegenerateMetadata(localDir string, cfg Config) error {
suite := cfg.Suite
var releaseEntries []ReleaseEntry
for _, component := range cfg.Components {
for _, arch := range cfg.Architectures {
entries, err := regenerateComponentArch(localDir, suite, component, arch)
if err != nil {
return fmt.Errorf("regenerate %s/%s: %w", component, arch, err)
}
releaseEntries = append(releaseEntries, entries...)
}
}
// Write dists/{suite}/Release.
releaseData := GenerateRelease(suite, suite, cfg.Components, cfg.Architectures, releaseEntries)
releasePath := filepath.Join(localDir, "dists", suite, "Release")
if err := os.MkdirAll(filepath.Dir(releasePath), 0o755); err != nil {
return fmt.Errorf("create dists dir: %w", err)
}
if err := atomicWrite(releasePath, releaseData); err != nil {
return fmt.Errorf("write Release: %w", err)
}
return nil
}
// regenerateComponentArch regenerates Packages and Packages.gz for one component/arch
// and returns the two ReleaseEntry values (plain then gz) for the Release file.
func regenerateComponentArch(localDir, suite, component, arch string) ([]ReleaseEntry, error) {
binDir := filepath.Join(localDir, "dists", suite, component, "binary-"+arch)
upstreamGzPath := filepath.Join(binDir, upstreamPackagesGz)
// 1. Read upstream Packages.gz.
gzData, err := os.ReadFile(upstreamGzPath)
if err != nil {
// Not cloned yet — skip silently.
return nil, nil
}
// 2. Decompress.
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return nil, fmt.Errorf("open gzip %s: %w", upstreamGzPath, err)
}
rawPackages, err := io.ReadAll(gz)
gz.Close()
if err != nil {
return nil, fmt.Errorf("decompress %s: %w", upstreamGzPath, err)
}
// 3. Parse all stanzas.
allPkgs, err := ParsePackages(rawPackages)
if err != nil {
return nil, fmt.Errorf("parse upstream Packages: %w", err)
}
// 4. Filter: keep only packages present on disk.
checker := diskPresenceChecker(localDir)
localPkgs := FilterPackages(allPkgs, checker)
// 5. Re-emit filtered stanzas.
plainBytes := emitPackages(localPkgs)
// 6. Gzip the filtered Packages.
var gzBuf bytes.Buffer
gzw := gzip.NewWriter(&gzBuf)
if _, err := gzw.Write(plainBytes); err != nil {
return nil, fmt.Errorf("gzip write: %w", err)
}
if err := gzw.Close(); err != nil {
return nil, fmt.Errorf("gzip close: %w", err)
}
gzBytes := gzBuf.Bytes()
// 7. Write Packages and Packages.gz.
if err := os.MkdirAll(binDir, 0o755); err != nil {
return nil, err
}
plainPath := filepath.Join(binDir, "Packages")
if err := atomicWrite(plainPath, plainBytes); err != nil {
return nil, fmt.Errorf("write Packages: %w", err)
}
gzPath := filepath.Join(binDir, "Packages.gz")
if err := atomicWrite(gzPath, gzBytes); err != nil {
return nil, fmt.Errorf("write Packages.gz: %w", err)
}
// 8. Compute SHA256 hashes.
plainSum := sha256.Sum256(plainBytes)
gzSum := sha256.Sum256(gzBytes)
// Release paths are relative to dists/{suite}/.
relBase := component + "/binary-" + arch + "/"
return []ReleaseEntry{
{
Hash: hex.EncodeToString(plainSum[:]),
Size: int64(len(plainBytes)),
Path: relBase + "Packages",
},
{
Hash: hex.EncodeToString(gzSum[:]),
Size: int64(len(gzBytes)),
Path: relBase + "Packages.gz",
},
}, nil
}

128
internal/clone/apt/packages.go Normal file
View file

@ -0,0 +1,128 @@
package apt
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strconv"
)
// DebPackage represents a single stanza from a Debian Packages file.
type DebPackage struct {
Package string
Version string
Architecture string
Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb
SHA256 string
Size int64
Raw []byte // original stanza bytes for faithful re-emission
}
// ParsePackages parses all stanzas from a Debian Packages control file.
// Stanzas are separated by blank lines.
func ParsePackages(data []byte) ([]DebPackage, error) {
// Normalise line endings.
data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n"))
var pkgs []DebPackage
stanzas := bytes.Split(data, []byte("\n\n"))
for _, stanza := range stanzas {
stanza = bytes.TrimSpace(stanza)
if len(stanza) == 0 {
continue
}
pkg, err := parseStanza(stanza)
if err != nil {
return nil, fmt.Errorf("parse stanza: %w", err)
}
pkgs = append(pkgs, pkg)
}
return pkgs, nil
}
func parseStanza(stanza []byte) (DebPackage, error) {
var pkg DebPackage
pkg.Raw = stanza
lines := bytes.Split(stanza, []byte("\n"))
for _, rawLine := range lines {
line := bytes.TrimRight(rawLine, " \t")
// Continuation lines (description body etc.) start with a space — skip.
if len(line) == 0 || line[0] == ' ' || line[0] == '\t' {
continue
}
colonIdx := bytes.IndexByte(line, ':')
if colonIdx < 0 {
continue
}
key := string(bytes.TrimSpace(line[:colonIdx]))
value := string(bytes.TrimSpace(line[colonIdx+1:]))
switch key {
case "Package":
pkg.Package = value
case "Version":
pkg.Version = value
case "Architecture":
pkg.Architecture = value
case "Filename":
pkg.Filename = value
case "SHA256":
pkg.SHA256 = value
case "Size":
n, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return DebPackage{}, fmt.Errorf("bad Size value %q: %w", value, err)
}
pkg.Size = n
}
}
if pkg.Package == "" {
return DebPackage{}, fmt.Errorf("stanza missing Package field")
}
return pkg, nil
}
// FilterPackages returns only the packages for which presentOnDisk returns true.
func FilterPackages(pkgs []DebPackage, presentOnDisk func(filename string) bool) []DebPackage {
var out []DebPackage
for _, p := range pkgs {
if presentOnDisk(p.Filename) {
out = append(out, p)
}
}
return out
}
// emitPackages serialises a slice of DebPackage back to Packages-file format,
// using the original Raw bytes for each stanza.
func emitPackages(pkgs []DebPackage) []byte {
var buf bytes.Buffer
for i, pkg := range pkgs {
// Strip trailing whitespace from each line within the stanza.
lines := bytes.Split(pkg.Raw, []byte("\n"))
for j, l := range lines {
lines[j] = bytes.TrimRight(l, " \t")
}
buf.Write(bytes.Join(lines, []byte("\n")))
if i < len(pkgs)-1 {
buf.WriteString("\n\n")
} else {
buf.WriteString("\n")
}
}
return buf.Bytes()
}
// diskPresenceChecker returns a presentOnDisk func rooted at localDir.
func diskPresenceChecker(localDir string) func(string) bool {
return func(filename string) bool {
p := filepath.Join(localDir, filepath.FromSlash(filename))
_, err := os.Stat(p)
return err == nil
}
}

154
internal/clone/apt/release.go Normal file
View file

@ -0,0 +1,154 @@
package apt
import (
"bytes"
"fmt"
"strconv"
"strings"
"time"
)
// ReleaseFile represents a parsed Debian Release / InRelease file.
type ReleaseFile struct {
Origin string
Suite string
Codename string
Components string
Architectures string
SHA256 []ReleaseEntry
}
// ReleaseEntry is a single line from the SHA256 section of a Release file.
type ReleaseEntry struct {
Hash string
Size int64
Path string
}
// ParseRelease strips PGP armor (if present) and parses an InRelease or Release file.
func ParseRelease(data []byte) (*ReleaseFile, error) {
text := stripPGPArmor(string(data))
rf := &ReleaseFile{}
inSHA256 := false
for _, rawLine := range strings.Split(text, "\n") {
line := strings.TrimRight(rawLine, " \t\r")
if inSHA256 {
if line == "" || (len(line) > 0 && line[0] != ' ' && line[0] != '\t') {
// A non-indented non-empty line ends the SHA256 block.
if line != "" {
inSHA256 = false
// Fall through to parse this line as a key-value pair.
} else {
inSHA256 = false
continue
}
} else {
// Indented line: " <hash> <size> <path>"
entry, err := parseReleaseEntry(line)
if err == nil {
rf.SHA256 = append(rf.SHA256, entry)
}
continue
}
}
if strings.HasPrefix(line, "SHA256:") {
inSHA256 = true
continue
}
key, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
switch key {
case "Origin":
rf.Origin = value
case "Suite":
rf.Suite = value
case "Codename":
rf.Codename = value
case "Components":
rf.Components = value
case "Architectures":
rf.Architectures = value
}
}
return rf, nil
}
// parseReleaseEntry parses a single indented line from the SHA256 section.
// Format: " <hash> <size> <path>"
func parseReleaseEntry(line string) (ReleaseEntry, error) {
fields := strings.Fields(line)
if len(fields) != 3 {
return ReleaseEntry{}, fmt.Errorf("malformed SHA256 entry: %q", line)
}
size, err := strconv.ParseInt(fields[1], 10, 64)
if err != nil {
return ReleaseEntry{}, fmt.Errorf("malformed size in SHA256 entry: %w", err)
}
return ReleaseEntry{Hash: fields[0], Size: size, Path: fields[2]}, nil
}
// stripPGPArmor removes the PGP signed-message wrapper if present.
// It returns only the signed body (between the header and the signature).
func stripPGPArmor(text string) string {
const beginSigned = "-----BEGIN PGP SIGNED MESSAGE-----"
const beginSig = "-----BEGIN PGP SIGNATURE-----"
if !strings.Contains(text, beginSigned) {
return text
}
// Drop the header lines (Hash: etc.) up to the first blank line.
after, found := strings.CutPrefix(text, beginSigned)
if !found {
// beginSigned not at start — find it
idx := strings.Index(text, beginSigned)
if idx < 0 {
return text
}
after = text[idx+len(beginSigned):]
}
// Skip the armor header lines (e.g. "Hash: SHA512") until the blank line.
blankIdx := strings.Index(after, "\n\n")
if blankIdx < 0 {
return text
}
body := after[blankIdx+2:]
// Cut off at the PGP signature block.
if sigIdx := strings.Index(body, beginSig); sigIdx >= 0 {
body = body[:sigIdx]
}
return strings.TrimRight(body, "\n\r ") + "\n"
}
// GenerateRelease generates a plain unsigned Release file.
func GenerateRelease(suite, codename string, components, architectures []string, entries []ReleaseEntry) []byte {
var buf bytes.Buffer
fmt.Fprintf(&buf, "Origin: ClonePack Mirror\n")
fmt.Fprintf(&buf, "Suite: %s\n", suite)
fmt.Fprintf(&buf, "Codename: %s\n", codename)
fmt.Fprintf(&buf, "Components: %s\n", strings.Join(components, " "))
fmt.Fprintf(&buf, "Architectures: %s\n", strings.Join(architectures, " "))
fmt.Fprintf(&buf, "Date: %s\n", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 UTC"))
fmt.Fprintf(&buf, "SHA256:\n")
for _, e := range entries {
fmt.Fprintf(&buf, " %s %d %s\n", e.Hash, e.Size, e.Path)
}
return buf.Bytes()
}

145
internal/clone/apt/scanner.go Normal file
View file

@ -0,0 +1,145 @@
package apt
import (
"compress/gzip"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
// NewPackage is a package present upstream but absent from the local mirror.
type NewPackage struct {
Package string
Version string
Architecture string
Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb
SHA256 string
Size int64
Component string
}
// Scanner compares an upstream APT repository against a local mirror and
// returns packages that are present upstream but missing on disk.
type Scanner struct {
SourceURL string
LocalDir string
Config Config
HTTPClient *http.Client
}
// NewScanner creates a Scanner with a 5-minute HTTP timeout.
func NewScanner(sourceURL, localDir string, cfg Config) *Scanner {
return &Scanner{
SourceURL: strings.TrimRight(sourceURL, "/"),
LocalDir: localDir,
Config: cfg,
HTTPClient: &http.Client{Timeout: 5 * time.Minute},
}
}
// Scan fetches the upstream package lists and returns packages not on disk.
func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) {
suite := sc.Config.Suite
// 1. Fetch InRelease.
inReleaseURL := sc.SourceURL + "/dists/" + suite + "/InRelease"
inReleaseData, err := fetchBytes(ctx, sc.HTTPClient, inReleaseURL)
if err != nil {
return nil, fmt.Errorf("fetch InRelease: %w", err)
}
rf, err := ParseRelease(inReleaseData)
if err != nil {
return nil, fmt.Errorf("parse InRelease: %w", err)
}
// 2 & 3. Fetch Packages.gz for each component/arch, save it, then filter.
var missing []NewPackage
for _, component := range sc.Config.Components {
for _, arch := range sc.Config.Architectures {
if err := ctx.Err(); err != nil {
return nil, err
}
pkgs, err := sc.fetchAndSavePackages(ctx, suite, component, arch, rf)
if err != nil {
return nil, fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
}
// 4. Filter: only return packages whose Filename is NOT on disk.
checker := diskPresenceChecker(sc.LocalDir)
for _, pkg := range pkgs {
if !checker(pkg.Filename) {
missing = append(missing, NewPackage{
Package: pkg.Package,
Version: pkg.Version,
Architecture: pkg.Architecture,
Filename: pkg.Filename,
SHA256: pkg.SHA256,
Size: pkg.Size,
Component: component,
})
}
}
}
}
return missing, nil
}
// fetchAndSavePackages fetches Packages.gz for a component/arch, saves it as
// .upstream-Packages.gz (so RegenerateMetadata can use it), and returns parsed packages.
func (sc *Scanner) fetchAndSavePackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
wantPath := component + "/binary-" + arch + "/Packages.gz"
// Check that this path is listed in the upstream SHA256 section.
var found bool
for _, e := range rf.SHA256 {
if e.Path == wantPath {
found = true
break
}
}
if !found {
return nil, nil
}
pkgGzURL := sc.SourceURL + "/dists/" + suite + "/" + wantPath
data, err := fetchBytes(ctx, sc.HTTPClient, pkgGzURL)
if err != nil {
return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
}
// Save as .upstream-Packages.gz.
destDir := filepath.Join(sc.LocalDir, "dists", suite, component, "binary-"+arch)
if err := os.MkdirAll(destDir, 0o755); err != nil {
return nil, err
}
upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
if err := atomicWrite(upstreamPath, data); err != nil {
return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
}
// Decompress and parse.
gz, err := gzip.NewReader(strings.NewReader(string(data)))
if err != nil {
return nil, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
raw, err := io.ReadAll(gz)
if err != nil {
return nil, fmt.Errorf("decompress: %w", err)
}
pkgs, err := ParsePackages(raw)
if err != nil {
return nil, fmt.Errorf("parse Packages: %w", err)
}
return pkgs, nil
}

50
internal/core/clone.go
View file

@ -2,11 +2,13 @@ package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"path/filepath"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
"github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -28,8 +30,8 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err
if err != nil {
return 0, err
}
if repo.Type != "rpm" {
return 0, fmt.Errorf("clone is only supported for rpm repositories")
if repo.Type != "rpm" && repo.Type != "apt" {
return 0, fmt.Errorf("clone is only supported for rpm and apt repositories")
}
running, err := s.jobStore.HasRunningCloneJob(ctx, repoID)
@ -45,7 +47,7 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err
return 0, err
}
go s.runClone(repoID, jobID, repo.SourceURL, repo.Type)
go s.runClone(repoID, jobID, repo)
return jobID, nil
}
@ -53,7 +55,7 @@ func (s *CloneService) GetLatestCloneJob(ctx context.Context, repoID int64) (*st
return s.jobStore.GetLatestCloneJob(ctx, repoID)
}
func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string) {
func (s *CloneService) runClone(repoID, jobID int64, repo *store.Repo) {
ctx := context.Background()
if err := s.jobStore.MarkCloneJobStarted(ctx, jobID); err != nil {
@ -61,23 +63,41 @@ func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string)
return
}
destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repoType)
cloner := rpm.New(sourceURL, destDir)
cloner.OnProgress = func(p rpm.Progress) {
log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone)
destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
var cloneErr error
switch repo.Type {
case "rpm":
cloner := rpm.New(repo.SourceURL, destDir)
cloner.OnProgress = func(p rpm.Progress) {
log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone)
}
if err := cloner.Clone(ctx); err != nil {
cloneErr = err
} else if err := rpm.RegenerateMetadata(destDir); err != nil {
log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
cloneErr = fmt.Errorf("parse apt config: %w", err)
break
}
cloner := aptclone.New(repo.SourceURL, destDir, cfg)
cloner.OnProgress = func(file string, bytes int64) {
log.Printf("clone job %d: %s (%d bytes)", jobID, file, bytes)
}
cloneErr = cloner.Clone(ctx)
}
if err := cloner.Clone(ctx); err != nil {
errMsg := err.Error()
if cloneErr != nil {
errMsg := cloneErr.Error()
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobFailed, &errMsg)
log.Printf("clone job %d: failed: %v", jobID, err)
log.Printf("clone job %d: failed: %v", jobID, cloneErr)
return
}
if err := rpm.RegenerateMetadata(destDir); err != nil {
log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err)
}
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobCompleted, nil)
log.Printf("clone job %d: completed", jobID)
}

114
internal/core/repo.go
View file

@ -2,10 +2,12 @@ package core
import (
"context"
"encoding/json"
"fmt"
"log"
"path/filepath"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -24,10 +26,13 @@ func NewRepoService(s store.RepoStore, dataDir string) *RepoService {
var validSyncModes = map[string]bool{"auto": true, "manual": true}
type CreateRepoInput struct {
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode"`
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode"`
AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
}
func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Repo, error) {
@ -46,7 +51,30 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re
return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", in.SyncMode)
}
r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode}
var configJSON string
if in.Type == "apt" {
if in.AptSuite == "" {
return nil, fmt.Errorf("apt_suite is required for apt repositories")
}
if len(in.AptComponents) == 0 {
in.AptComponents = []string{"main"}
}
if len(in.AptArchitectures) == 0 {
in.AptArchitectures = []string{"amd64"}
}
cfg := aptclone.Config{
Suite: in.AptSuite,
Components: in.AptComponents,
Architectures: in.AptArchitectures,
}
b, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("marshal apt config: %w", err)
}
configJSON = string(b)
}
r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode, Config: configJSON}
id, err := s.store.CreateRepo(ctx, r)
if err != nil {
return nil, err
@ -60,15 +88,85 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re
}
func (s *RepoService) initStorage(repo *store.Repo) {
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), repo.Type)
switch repo.Type {
case "rpm":
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), "rpm")
if err := rpmclone.InitEmptyRepo(localDir); err != nil {
log.Printf("init storage for repo %d: %v", repo.ID, err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
log.Printf("init storage for repo %d: parse config: %v", repo.ID, err)
return
}
if err := aptclone.InitEmptyRepo(localDir, cfg); err != nil {
log.Printf("init storage for repo %d: %v", repo.ID, err)
}
}
}
type UpdateRepoInput struct {
SourceURL *string
SyncMode *string
AptSuite *string
AptComponents []string
AptArchitectures []string
}
func (s *RepoService) Update(ctx context.Context, id int64, in UpdateRepoInput) (*store.Repo, error) {
repo, err := s.store.GetRepo(ctx, id)
if err != nil {
return nil, err
}
if in.SourceURL != nil {
if *in.SourceURL == "" {
return nil, fmt.Errorf("source_url cannot be empty")
}
repo.SourceURL = *in.SourceURL
}
if in.SyncMode != nil {
if !validSyncModes[*in.SyncMode] {
return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", *in.SyncMode)
}
repo.SyncMode = *in.SyncMode
}
if repo.Type == "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) {
var cfg aptclone.Config
if repo.Config != "" {
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return nil, fmt.Errorf("parse existing apt config: %w", err)
}
}
if in.AptSuite != nil {
if *in.AptSuite == "" {
return nil, fmt.Errorf("apt_suite cannot be empty")
}
cfg.Suite = *in.AptSuite
}
if in.AptComponents != nil {
cfg.Components = in.AptComponents
}
if in.AptArchitectures != nil {
cfg.Architectures = in.AptArchitectures
}
b, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("marshal apt config: %w", err)
}
repo.Config = string(b)
} else if repo.Type != "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) {
return nil, fmt.Errorf("apt fields only apply to apt repositories")
}
if err := s.store.UpdateRepo(ctx, id, repo.SourceURL, repo.SyncMode, repo.Config); err != nil {
return nil, err
}
return s.store.GetRepo(ctx, id)
}
func (s *RepoService) List(ctx context.Context) ([]store.Repo, error) {
return s.store.ListRepos(ctx)
}
@ -77,6 +175,10 @@ func (s *RepoService) Get(ctx context.Context, id int64) (*store.Repo, error) {
return s.store.GetRepo(ctx, id)
}
func (s *RepoService) GetByName(ctx context.Context, name string) (*store.Repo, error) {
return s.store.GetRepoByName(ctx, name)
}
func (s *RepoService) Delete(ctx context.Context, id int64) error {
return s.store.DeleteRepo(ctx, id)
}

178
internal/core/snapshot.go
View file

@ -4,6 +4,7 @@ import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
@ -13,6 +14,7 @@ import (
"strings"
"time"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -46,60 +48,24 @@ type SnapshotDiff struct {
}
func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label string) (int64, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
repo, err := s.repoStore.GetRepo(ctx, repoID)
if err != nil {
return 0, err
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml"))
var pkgs []store.SnapshotPackage
switch repo.Type {
case "rpm":
pkgs, err = s.snapshotPackagesRPM(localDir)
case "apt":
pkgs, err = s.snapshotPackagesAPT(localDir, repo.Config)
default:
return 0, fmt.Errorf("snapshots not supported for repo type %q", repo.Type)
}
if err != nil {
return 0, fmt.Errorf("read repomd.xml: %w", err)
}
var repomd rpmclone.RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return 0, fmt.Errorf("parse repomd.xml: %w", err)
}
var primaryHref string
for _, entry := range repomd.Data {
if entry.Type == "primary" {
primaryHref = entry.Location.Href
break
}
}
if primaryHref == "" {
return 0, fmt.Errorf("no primary entry in repomd.xml")
}
gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref)))
if err != nil {
return 0, fmt.Errorf("read primary.xml.gz: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return 0, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary rpmclone.PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return 0, fmt.Errorf("parse primary.xml: %w", err)
}
pkgs := make([]store.SnapshotPackage, len(primary.Packages))
for i, p := range primary.Packages {
pkgs[i] = store.SnapshotPackage{
Name: p.Name,
Version: p.Version.Ver + "-" + p.Version.Rel,
Arch: p.Arch,
Location: p.Location.Href,
Checksum: p.Checksum.Value,
ChecksumType: p.Checksum.Type,
Size: p.Size.Package,
}
return 0, err
}
snapID, err := s.snapshotStore.CreateSnapshot(ctx, repoID, label)
@ -114,6 +80,94 @@ func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label
return snapID, nil
}
func (s *SnapshotService) snapshotPackagesRPM(localDir string) ([]store.SnapshotPackage, error) {
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml"))
if err != nil {
return nil, fmt.Errorf("read repomd.xml: %w", err)
}
var repomd rpmclone.RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return nil, fmt.Errorf("parse repomd.xml: %w", err)
}
var primaryHref string
for _, entry := range repomd.Data {
if entry.Type == "primary" {
primaryHref = entry.Location.Href
break
}
}
if primaryHref == "" {
return nil, fmt.Errorf("no primary entry in repomd.xml")
}
gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref)))
if err != nil {
return nil, fmt.Errorf("read primary.xml.gz: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return nil, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary rpmclone.PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return nil, fmt.Errorf("parse primary.xml: %w", err)
}
pkgs := make([]store.SnapshotPackage, len(primary.Packages))
for i, p := range primary.Packages {
pkgs[i] = store.SnapshotPackage{
Name: p.Name,
Version: p.Version.Ver + "-" + p.Version.Rel,
Arch: p.Arch,
Location: p.Location.Href,
Checksum: p.Checksum.Value,
ChecksumType: p.Checksum.Type,
Size: p.Size.Package,
}
}
return pkgs, nil
}
func (s *SnapshotService) snapshotPackagesAPT(localDir, configJSON string) ([]store.SnapshotPackage, error) {
var cfg aptclone.Config
if err := json.Unmarshal([]byte(configJSON), &cfg); err != nil {
return nil, fmt.Errorf("parse apt config: %w", err)
}
seen := make(map[string]struct{})
var pkgs []store.SnapshotPackage
for _, component := range cfg.Components {
for _, arch := range cfg.Architectures {
pkgsPath := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch, "Packages")
data, err := os.ReadFile(pkgsPath)
if err != nil {
// Not yet populated — skip silently.
continue
}
debs, err := aptclone.ParsePackages(data)
if err != nil {
return nil, fmt.Errorf("parse Packages for %s/%s: %w", component, arch, err)
}
for _, d := range debs {
if _, dup := seen[d.Filename]; dup {
continue
}
seen[d.Filename] = struct{}{}
pkgs = append(pkgs, store.SnapshotPackage{
Name: d.Package,
Version: d.Version,
Arch: d.Architecture,
Location: d.Filename,
Checksum: d.SHA256,
ChecksumType: "sha256",
Size: d.Size,
})
}
}
}
return pkgs, nil
}
func (s *SnapshotService) List(ctx context.Context, repoID int64) ([]store.Snapshot, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return nil, err
@ -212,16 +266,18 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64
return err
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
snapSet := make(map[string]store.SnapshotPackage, len(pkgs))
for _, p := range pkgs {
snapSet[p.Location] = p
}
ext := packageExt(repo.Type)
var errs []error
if err := filepath.WalkDir(localDir, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() || filepath.Ext(path) != ".rpm" {
if err != nil || d.IsDir() || filepath.Ext(path) != ext {
return err
}
rel, _ := filepath.Rel(localDir, path)
@ -251,10 +307,28 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64
return err
}
if repo.Type == "rpm" {
switch repo.Type {
case "rpm":
if err := rpmclone.RegenerateMetadata(localDir); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return fmt.Errorf("parse apt config: %w", err)
}
if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
}
return nil
}
func packageExt(repoType string) string {
switch repoType {
case "apt":
return ".deb"
default:
return ".rpm"
}
}

92
internal/core/sync.go
View file

@ -2,6 +2,7 @@ package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
@ -11,6 +12,7 @@ import (
"strings"
"time"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -54,18 +56,54 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
if err != nil {
return err
}
if repo.Type != "rpm" {
return fmt.Errorf("sync only supported for rpm repos")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
type candidate struct {
Name string
Version string
Arch string
Location string
Checksum string
ChecksumType string
Size int64
}
var candidates []candidate
switch repo.Type {
case "rpm":
scanner := rpmclone.NewScanner(repo.SourceURL, localDir)
pkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
for _, p := range pkgs {
candidates = append(candidates, candidate{
Name: p.Name, Version: p.Version, Arch: p.Arch,
Location: p.Location, Checksum: p.Checksum, ChecksumType: p.ChecksumType, Size: p.Size,
})
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return fmt.Errorf("parse apt config for repo %d: %w", repoID, err)
}
scanner := aptclone.NewScanner(repo.SourceURL, localDir, cfg)
pkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
for _, p := range pkgs {
candidates = append(candidates, candidate{
Name: p.Package, Version: p.Version, Arch: p.Architecture,
Location: p.Filename, Checksum: p.SHA256, ChecksumType: "sha256", Size: p.Size,
})
}
default:
return fmt.Errorf("sync only supported for rpm and apt repos")
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
scanner := rpmclone.NewScanner(repo.SourceURL, localDir)
newPkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
if len(newPkgs) == 0 {
if len(candidates) == 0 {
log.Printf("sync: repo %d is up to date", repoID)
return nil
}
@ -82,19 +120,19 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
blockedNames[b.Name] = true
}
}
filtered := newPkgs[:0]
for _, p := range newPkgs {
filtered := candidates[:0]
for _, p := range candidates {
if !blockedLocations[p.Location] && !blockedNames[p.Name] {
filtered = append(filtered, p)
}
}
newPkgs = filtered
candidates = filtered
if len(newPkgs) == 0 {
if len(candidates) == 0 {
log.Printf("sync: repo %d is up to date (all new packages are blocked)", repoID)
return nil
}
log.Printf("sync: repo %d has %d new package(s)", repoID, len(newPkgs))
log.Printf("sync: repo %d has %d new package(s)", repoID, len(candidates))
switch repo.SyncMode {
case "auto":
@ -103,8 +141,8 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
return fmt.Errorf("start clone for repo %d: %w", repoID, err)
}
case "manual":
pending := make([]store.PendingPackage, len(newPkgs))
for i, p := range newPkgs {
pending := make([]store.PendingPackage, len(candidates))
for i, p := range candidates {
pending[i] = store.PendingPackage{
RepoID: repoID,
Name: p.Name,
@ -148,7 +186,7 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in
wanted[id] = true
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
var downloadErrors []error
var anyApproved bool
for _, pkg := range all {
@ -173,14 +211,22 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in
}
if anyApproved {
if repo.Type == "rpm" {
switch repo.Type {
case "rpm":
if err := rpmclone.RegenerateMetadata(localDir); err != nil {
log.Printf("metadata regeneration for repo %d failed: %v", repoID, err)
}
}
label := "auto-" + time.Now().UTC().Format(time.RFC3339)
if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil {
log.Printf("auto-snapshot for repo %d failed: %v", repoID, err)
label := "auto-" + time.Now().UTC().Format(time.RFC3339)
if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil {
log.Printf("auto-snapshot for repo %d failed: %v", repoID, err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
log.Printf("parse apt config for repo %d: %v", repoID, err)
} else if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil {
log.Printf("metadata regeneration for repo %d failed: %v", repoID, err)
}
}
}

4
internal/store/db.go
View file

@ -27,6 +27,9 @@ var migration005 string
//go:embed migrations/000006_blocked_packages_name.up.sql
var migration006 string
//go:embed migrations/000007_repo_config.up.sql
var migration007 string
func Open(cfg config.DBConfig) (*sql.DB, error) {
db, err := sql.Open("sqlite", cfg.Path)
if err != nil {
@ -66,6 +69,7 @@ func runMigrations(db *sql.DB) error {
{4, migration004},
{5, migration005},
{6, migration006},
{7, migration007},
}
for _, m := range migrations {

4
internal/store/migrations/000007_repo_config.down.sql Normal file
View file

@ -0,0 +1,4 @@
-- SQLite ne supporte pas DROP COLUMN avant 3.35
CREATE TABLE repos_backup AS SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos;
DROP TABLE repos;
ALTER TABLE repos_backup RENAME TO repos;

1
internal/store/migrations/000007_repo_config.up.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE repos ADD COLUMN config TEXT NOT NULL DEFAULT '';

44
internal/store/repo.go
View file

@ -16,6 +16,7 @@ type Repo struct {
SourceURL string `db:"source_url"`
Frozen bool `db:"frozen"`
SyncMode string `db:"sync_mode"`
Config string `db:"config"`
CreatedAt time.Time `db:"created_at"`
}
@ -23,8 +24,10 @@ type RepoStore interface {
CreateRepo(ctx context.Context, r *Repo) (int64, error)
ListRepos(ctx context.Context) ([]Repo, error)
GetRepo(ctx context.Context, id int64) (*Repo, error)
GetRepoByName(ctx context.Context, name string) (*Repo, error)
DeleteRepo(ctx context.Context, id int64) error
UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error
UpdateRepo(ctx context.Context, id int64, sourceURL, syncMode, config string) error
}
type SQLiteRepoStore struct {
@ -37,8 +40,8 @@ func NewRepoStore(db *sql.DB) *SQLiteRepoStore {
func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error) {
res, err := s.db.ExecContext(ctx,
`INSERT INTO repos (name, type, source_url, frozen, sync_mode) VALUES (?, ?, ?, ?, ?)`,
r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode,
`INSERT INTO repos (name, type, source_url, frozen, sync_mode, config) VALUES (?, ?, ?, ?, ?, ?)`,
r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, r.Config,
)
if err != nil {
return 0, err
@ -48,7 +51,7 @@ func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error
func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos ORDER BY created_at DESC`)
`SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos ORDER BY created_at DESC`)
if err != nil {
return nil, err
}
@ -57,7 +60,7 @@ func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) {
var repos []Repo
for rows.Next() {
var r Repo
if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt); err != nil {
if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt); err != nil {
return nil, err
}
repos = append(repos, r)
@ -68,8 +71,22 @@ func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) {
func (s *SQLiteRepoStore) GetRepo(ctx context.Context, id int64) (*Repo, error) {
var r Repo
err := s.db.QueryRowContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos WHERE id = ?`, id,
).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt)
`SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos WHERE id = ?`, id,
).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
return &r, nil
}
func (s *SQLiteRepoStore) GetRepoByName(ctx context.Context, name string) (*Repo, error) {
var r Repo
err := s.db.QueryRowContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos WHERE name = ?`, name,
).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
@ -94,6 +111,21 @@ func (s *SQLiteRepoStore) DeleteRepo(ctx context.Context, id int64) error {
return nil
}
func (s *SQLiteRepoStore) UpdateRepo(ctx context.Context, id int64, sourceURL, syncMode, config string) error {
res, err := s.db.ExecContext(ctx,
`UPDATE repos SET source_url = ?, sync_mode = ?, config = ? WHERE id = ?`,
sourceURL, syncMode, config, id,
)
if err != nil {
return err
}
n, _ := res.RowsAffected()
if n == 0 {
return ErrNotFound
}
return nil
}
func (s *SQLiteRepoStore) UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error {
res, err := s.db.ExecContext(ctx, `UPDATE repos SET sync_mode = ? WHERE id = ?`, mode, id)
if err != nil {