syonad/clonepack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add apt gestion

Browse source 
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-04-25 19:03:26 +02:00
commit c4776d81dd
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
22 changed files with 1465 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

50
internal/core/clone.go
View file

@ -2,11 +2,13 @@ package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"path/filepath"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
"github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -28,8 +30,8 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err
if err != nil {
return 0, err
}
if repo.Type != "rpm" {
return 0, fmt.Errorf("clone is only supported for rpm repositories")
if repo.Type != "rpm" && repo.Type != "apt" {
return 0, fmt.Errorf("clone is only supported for rpm and apt repositories")
}
running, err := s.jobStore.HasRunningCloneJob(ctx, repoID)
@ -45,7 +47,7 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err
return 0, err
}
go s.runClone(repoID, jobID, repo.SourceURL, repo.Type)
go s.runClone(repoID, jobID, repo)
return jobID, nil
}
@ -53,7 +55,7 @@ func (s *CloneService) GetLatestCloneJob(ctx context.Context, repoID int64) (*st
return s.jobStore.GetLatestCloneJob(ctx, repoID)
}
func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string) {
func (s *CloneService) runClone(repoID, jobID int64, repo *store.Repo) {
ctx := context.Background()
if err := s.jobStore.MarkCloneJobStarted(ctx, jobID); err != nil {
@ -61,23 +63,41 @@ func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string)
return
}
destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repoType)
cloner := rpm.New(sourceURL, destDir)
cloner.OnProgress = func(p rpm.Progress) {
log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone)
destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
var cloneErr error
switch repo.Type {
case "rpm":
cloner := rpm.New(repo.SourceURL, destDir)
cloner.OnProgress = func(p rpm.Progress) {
log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone)
}
if err := cloner.Clone(ctx); err != nil {
cloneErr = err
} else if err := rpm.RegenerateMetadata(destDir); err != nil {
log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
cloneErr = fmt.Errorf("parse apt config: %w", err)
break
}
cloner := aptclone.New(repo.SourceURL, destDir, cfg)
cloner.OnProgress = func(file string, bytes int64) {
log.Printf("clone job %d: %s (%d bytes)", jobID, file, bytes)
}
cloneErr = cloner.Clone(ctx)
}
if err := cloner.Clone(ctx); err != nil {
errMsg := err.Error()
if cloneErr != nil {
errMsg := cloneErr.Error()
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobFailed, &errMsg)
log.Printf("clone job %d: failed: %v", jobID, err)
log.Printf("clone job %d: failed: %v", jobID, cloneErr)
return
}
if err := rpm.RegenerateMetadata(destDir); err != nil {
log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err)
}
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobCompleted, nil)
log.Printf("clone job %d: completed", jobID)
}

114
internal/core/repo.go
View file

@ -2,10 +2,12 @@ package core
import (
"context"
"encoding/json"
"fmt"
"log"
"path/filepath"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -24,10 +26,13 @@ func NewRepoService(s store.RepoStore, dataDir string) *RepoService {
var validSyncModes = map[string]bool{"auto": true, "manual": true}
type CreateRepoInput struct {
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode"`
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode"`
AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
}
func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Repo, error) {
@ -46,7 +51,30 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re
return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", in.SyncMode)
}
r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode}
var configJSON string
if in.Type == "apt" {
if in.AptSuite == "" {
return nil, fmt.Errorf("apt_suite is required for apt repositories")
}
if len(in.AptComponents) == 0 {
in.AptComponents = []string{"main"}
}
if len(in.AptArchitectures) == 0 {
in.AptArchitectures = []string{"amd64"}
}
cfg := aptclone.Config{
Suite: in.AptSuite,
Components: in.AptComponents,
Architectures: in.AptArchitectures,
}
b, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("marshal apt config: %w", err)
}
configJSON = string(b)
}
r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode, Config: configJSON}
id, err := s.store.CreateRepo(ctx, r)
if err != nil {
return nil, err
@ -60,15 +88,85 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re
}
func (s *RepoService) initStorage(repo *store.Repo) {
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), repo.Type)
switch repo.Type {
case "rpm":
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), "rpm")
if err := rpmclone.InitEmptyRepo(localDir); err != nil {
log.Printf("init storage for repo %d: %v", repo.ID, err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
log.Printf("init storage for repo %d: parse config: %v", repo.ID, err)
return
}
if err := aptclone.InitEmptyRepo(localDir, cfg); err != nil {
log.Printf("init storage for repo %d: %v", repo.ID, err)
}
}
}
type UpdateRepoInput struct {
SourceURL *string
SyncMode *string
AptSuite *string
AptComponents []string
AptArchitectures []string
}
func (s *RepoService) Update(ctx context.Context, id int64, in UpdateRepoInput) (*store.Repo, error) {
repo, err := s.store.GetRepo(ctx, id)
if err != nil {
return nil, err
}
if in.SourceURL != nil {
if *in.SourceURL == "" {
return nil, fmt.Errorf("source_url cannot be empty")
}
repo.SourceURL = *in.SourceURL
}
if in.SyncMode != nil {
if !validSyncModes[*in.SyncMode] {
return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", *in.SyncMode)
}
repo.SyncMode = *in.SyncMode
}
if repo.Type == "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) {
var cfg aptclone.Config
if repo.Config != "" {
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return nil, fmt.Errorf("parse existing apt config: %w", err)
}
}
if in.AptSuite != nil {
if *in.AptSuite == "" {
return nil, fmt.Errorf("apt_suite cannot be empty")
}
cfg.Suite = *in.AptSuite
}
if in.AptComponents != nil {
cfg.Components = in.AptComponents
}
if in.AptArchitectures != nil {
cfg.Architectures = in.AptArchitectures
}
b, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("marshal apt config: %w", err)
}
repo.Config = string(b)
} else if repo.Type != "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) {
return nil, fmt.Errorf("apt fields only apply to apt repositories")
}
if err := s.store.UpdateRepo(ctx, id, repo.SourceURL, repo.SyncMode, repo.Config); err != nil {
return nil, err
}
return s.store.GetRepo(ctx, id)
}
func (s *RepoService) List(ctx context.Context) ([]store.Repo, error) {
return s.store.ListRepos(ctx)
}
@ -77,6 +175,10 @@ func (s *RepoService) Get(ctx context.Context, id int64) (*store.Repo, error) {
return s.store.GetRepo(ctx, id)
}
func (s *RepoService) GetByName(ctx context.Context, name string) (*store.Repo, error) {
return s.store.GetRepoByName(ctx, name)
}
func (s *RepoService) Delete(ctx context.Context, id int64) error {
return s.store.DeleteRepo(ctx, id)
}

178
internal/core/snapshot.go
View file

@ -4,6 +4,7 @@ import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
@ -13,6 +14,7 @@ import (
"strings"
"time"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -46,60 +48,24 @@ type SnapshotDiff struct {
}
func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label string) (int64, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
repo, err := s.repoStore.GetRepo(ctx, repoID)
if err != nil {
return 0, err
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml"))
var pkgs []store.SnapshotPackage
switch repo.Type {
case "rpm":
pkgs, err = s.snapshotPackagesRPM(localDir)
case "apt":
pkgs, err = s.snapshotPackagesAPT(localDir, repo.Config)
default:
return 0, fmt.Errorf("snapshots not supported for repo type %q", repo.Type)
}
if err != nil {
return 0, fmt.Errorf("read repomd.xml: %w", err)
}
var repomd rpmclone.RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return 0, fmt.Errorf("parse repomd.xml: %w", err)
}
var primaryHref string
for _, entry := range repomd.Data {
if entry.Type == "primary" {
primaryHref = entry.Location.Href
break
}
}
if primaryHref == "" {
return 0, fmt.Errorf("no primary entry in repomd.xml")
}
gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref)))
if err != nil {
return 0, fmt.Errorf("read primary.xml.gz: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return 0, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary rpmclone.PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return 0, fmt.Errorf("parse primary.xml: %w", err)
}
pkgs := make([]store.SnapshotPackage, len(primary.Packages))
for i, p := range primary.Packages {
pkgs[i] = store.SnapshotPackage{
Name: p.Name,
Version: p.Version.Ver + "-" + p.Version.Rel,
Arch: p.Arch,
Location: p.Location.Href,
Checksum: p.Checksum.Value,
ChecksumType: p.Checksum.Type,
Size: p.Size.Package,
}
return 0, err
}
snapID, err := s.snapshotStore.CreateSnapshot(ctx, repoID, label)
@ -114,6 +80,94 @@ func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label
return snapID, nil
}
func (s *SnapshotService) snapshotPackagesRPM(localDir string) ([]store.SnapshotPackage, error) {
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml"))
if err != nil {
return nil, fmt.Errorf("read repomd.xml: %w", err)
}
var repomd rpmclone.RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return nil, fmt.Errorf("parse repomd.xml: %w", err)
}
var primaryHref string
for _, entry := range repomd.Data {
if entry.Type == "primary" {
primaryHref = entry.Location.Href
break
}
}
if primaryHref == "" {
return nil, fmt.Errorf("no primary entry in repomd.xml")
}
gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref)))
if err != nil {
return nil, fmt.Errorf("read primary.xml.gz: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return nil, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary rpmclone.PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return nil, fmt.Errorf("parse primary.xml: %w", err)
}
pkgs := make([]store.SnapshotPackage, len(primary.Packages))
for i, p := range primary.Packages {
pkgs[i] = store.SnapshotPackage{
Name: p.Name,
Version: p.Version.Ver + "-" + p.Version.Rel,
Arch: p.Arch,
Location: p.Location.Href,
Checksum: p.Checksum.Value,
ChecksumType: p.Checksum.Type,
Size: p.Size.Package,
}
}
return pkgs, nil
}
func (s *SnapshotService) snapshotPackagesAPT(localDir, configJSON string) ([]store.SnapshotPackage, error) {
var cfg aptclone.Config
if err := json.Unmarshal([]byte(configJSON), &cfg); err != nil {
return nil, fmt.Errorf("parse apt config: %w", err)
}
seen := make(map[string]struct{})
var pkgs []store.SnapshotPackage
for _, component := range cfg.Components {
for _, arch := range cfg.Architectures {
pkgsPath := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch, "Packages")
data, err := os.ReadFile(pkgsPath)
if err != nil {
// Not yet populated — skip silently.
continue
}
debs, err := aptclone.ParsePackages(data)
if err != nil {
return nil, fmt.Errorf("parse Packages for %s/%s: %w", component, arch, err)
}
for _, d := range debs {
if _, dup := seen[d.Filename]; dup {
continue
}
seen[d.Filename] = struct{}{}
pkgs = append(pkgs, store.SnapshotPackage{
Name: d.Package,
Version: d.Version,
Arch: d.Architecture,
Location: d.Filename,
Checksum: d.SHA256,
ChecksumType: "sha256",
Size: d.Size,
})
}
}
}
return pkgs, nil
}
func (s *SnapshotService) List(ctx context.Context, repoID int64) ([]store.Snapshot, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return nil, err
@ -212,16 +266,18 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64
return err
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
snapSet := make(map[string]store.SnapshotPackage, len(pkgs))
for _, p := range pkgs {
snapSet[p.Location] = p
}
ext := packageExt(repo.Type)
var errs []error
if err := filepath.WalkDir(localDir, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() || filepath.Ext(path) != ".rpm" {
if err != nil || d.IsDir() || filepath.Ext(path) != ext {
return err
}
rel, _ := filepath.Rel(localDir, path)
@ -251,10 +307,28 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64
return err
}
if repo.Type == "rpm" {
switch repo.Type {
case "rpm":
if err := rpmclone.RegenerateMetadata(localDir); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return fmt.Errorf("parse apt config: %w", err)
}
if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
}
return nil
}
func packageExt(repoType string) string {
switch repoType {
case "apt":
return ".deb"
default:
return ".rpm"
}
}

92
internal/core/sync.go
View file

@ -2,6 +2,7 @@ package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
@ -11,6 +12,7 @@ import (
"strings"
"time"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
@ -54,18 +56,54 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
if err != nil {
return err
}
if repo.Type != "rpm" {
return fmt.Errorf("sync only supported for rpm repos")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
type candidate struct {
Name string
Version string
Arch string
Location string
Checksum string
ChecksumType string
Size int64
}
var candidates []candidate
switch repo.Type {
case "rpm":
scanner := rpmclone.NewScanner(repo.SourceURL, localDir)
pkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
for _, p := range pkgs {
candidates = append(candidates, candidate{
Name: p.Name, Version: p.Version, Arch: p.Arch,
Location: p.Location, Checksum: p.Checksum, ChecksumType: p.ChecksumType, Size: p.Size,
})
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return fmt.Errorf("parse apt config for repo %d: %w", repoID, err)
}
scanner := aptclone.NewScanner(repo.SourceURL, localDir, cfg)
pkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
for _, p := range pkgs {
candidates = append(candidates, candidate{
Name: p.Package, Version: p.Version, Arch: p.Architecture,
Location: p.Filename, Checksum: p.SHA256, ChecksumType: "sha256", Size: p.Size,
})
}
default:
return fmt.Errorf("sync only supported for rpm and apt repos")
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
scanner := rpmclone.NewScanner(repo.SourceURL, localDir)
newPkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
if len(newPkgs) == 0 {
if len(candidates) == 0 {
log.Printf("sync: repo %d is up to date", repoID)
return nil
}
@ -82,19 +120,19 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
blockedNames[b.Name] = true
}
}
filtered := newPkgs[:0]
for _, p := range newPkgs {
filtered := candidates[:0]
for _, p := range candidates {
if !blockedLocations[p.Location] && !blockedNames[p.Name] {
filtered = append(filtered, p)
}
}
newPkgs = filtered
candidates = filtered
if len(newPkgs) == 0 {
if len(candidates) == 0 {
log.Printf("sync: repo %d is up to date (all new packages are blocked)", repoID)
return nil
}
log.Printf("sync: repo %d has %d new package(s)", repoID, len(newPkgs))
log.Printf("sync: repo %d has %d new package(s)", repoID, len(candidates))
switch repo.SyncMode {
case "auto":
@ -103,8 +141,8 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
return fmt.Errorf("start clone for repo %d: %w", repoID, err)
}
case "manual":
pending := make([]store.PendingPackage, len(newPkgs))
for i, p := range newPkgs {
pending := make([]store.PendingPackage, len(candidates))
for i, p := range candidates {
pending[i] = store.PendingPackage{
RepoID: repoID,
Name: p.Name,
@ -148,7 +186,7 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in
wanted[id] = true
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
var downloadErrors []error
var anyApproved bool
for _, pkg := range all {
@ -173,14 +211,22 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in
}
if anyApproved {
if repo.Type == "rpm" {
switch repo.Type {
case "rpm":
if err := rpmclone.RegenerateMetadata(localDir); err != nil {
log.Printf("metadata regeneration for repo %d failed: %v", repoID, err)
}
}
label := "auto-" + time.Now().UTC().Format(time.RFC3339)
if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil {
log.Printf("auto-snapshot for repo %d failed: %v", repoID, err)
label := "auto-" + time.Now().UTC().Format(time.RFC3339)
if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil {
log.Printf("auto-snapshot for repo %d failed: %v", repoID, err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
log.Printf("parse apt config for repo %d: %v", repoID, err)
} else if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil {
log.Printf("metadata regeneration for repo %d failed: %v", repoID, err)
}
}
}