add apt gestion

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-04-25 19:03:26 +02:00
commit c4776d81dd
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
22 changed files with 1465 additions and 163 deletions

View file

@ -5,17 +5,18 @@ ClonePack est un système de clonage et de gel d'artefacts. Il permet de synchro
## Types d'artefacts supportés ## Types d'artefacts supportés
- Enterprise Linux (RPM/YUM/DNF) - Enterprise Linux (RPM/YUM/DNF)
- Debian (APT) — *à venir* - Debian/Ubuntu (APT)
- Docker images — *à venir* - Docker images — *à venir*
- Binaires — *à venir* - Binaires — *à venir*
## Fonctionnalités ## Fonctionnalités
- Clonage de dépôts externes avec vérification SHA256 - Clonage de dépôts RPM et APT avec vérification SHA256
- Scan régulier des sources pour détecter les nouveaux paquets - Scan régulier des sources pour détecter les nouveaux paquets
- Validation manuelle ou automatique des mises à jour - Validation manuelle ou automatique des mises à jour
- Snapshots métadonnées avec diff et rollback - Blocklist permanente par paquet (toutes versions)
- Proxy miroir HTTP (consommable directement par `yum`/`apt`) - Snapshots avec diff et rollback (RPM)
- Proxy miroir HTTP accessible par ID ou par nom de dépôt
--- ---
@ -85,11 +86,21 @@ Toutes les commandes CLI appellent l'API REST. L'URL du serveur se configure ave
--type rpm \ --type rpm \
--source-url https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os --source-url https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os
# Créer un dépôt APT
./bin/clonepack repo create \
--name debian-bookworm \
--type apt \
--source-url https://deb.debian.org/debian \
--suite bookworm \
--components main,contrib \
--arch amd64
# Créer avec validation manuelle # Créer avec validation manuelle
./bin/clonepack repo create \ ./bin/clonepack repo create \
--name rocky9-baseos \ --name debian-bookworm \
--type rpm \ --type apt \
--source-url https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os \ --source-url https://deb.debian.org/debian \
--suite bookworm \
--sync-mode manual --sync-mode manual
# Lister les dépôts # Lister les dépôts
@ -98,10 +109,19 @@ Toutes les commandes CLI appellent l'API REST. L'URL du serveur se configure ave
# Détail d'un dépôt # Détail d'un dépôt
./bin/clonepack repo get 1 ./bin/clonepack repo get 1
# Modifier un dépôt (source URL, sync mode, config APT)
./bin/clonepack repo update 1 --source-url https://archive.debian.org/debian
./bin/clonepack repo update 1 --suite bookworm-backports
./bin/clonepack repo update 1 --components main,contrib,non-free
./bin/clonepack repo update 1 --sync-mode manual
# Supprimer un dépôt # Supprimer un dépôt
./bin/clonepack repo delete 1 ./bin/clonepack repo delete 1
``` ```
Les flags `--suite`, `--components` et `--arch` ne s'appliquent qu'aux dépôts de type `apt`.
Seuls les flags fournis sont modifiés — les autres champs restent inchangés.
### Clonage ### Clonage
```bash ```bash
@ -171,34 +191,35 @@ Les paquets bloqués n'apparaissent plus jamais dans la liste pending, quelle qu
./bin/clonepack repo snapshot delete 1 3 ./bin/clonepack repo snapshot delete 1 3
``` ```
Un snapshot automatique est créé après chaque `sync-approve`. Un snapshot automatique est créé après chaque `sync-approve` sur un dépôt RPM.
--- ---
## Proxy miroir ## Proxy miroir
ClonePack expose chaque dépôt cloné comme un miroir HTTP à l'adresse : ClonePack expose chaque dépôt cloné comme un miroir HTTP. L'accès est possible par **ID numérique** ou par **nom du dépôt** :
``` ```
http://<host>:<port>/mirror/<repo_id>/ http://<host>:<port>/mirror/<repo_id>/
http://<host>:<port>/mirror/<nom_du_depot>/
``` ```
Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si un fichier est absent, le serveur retourne 404.
**Configuration YUM/DNF** (`/etc/yum.repos.d/clonepack.repo`) : **Configuration YUM/DNF** (`/etc/yum.repos.d/clonepack.repo`) :
```ini ```ini
[rocky9-baseos] [rocky9-baseos]
name=Rocky Linux 9 BaseOS via ClonePack name=Rocky Linux 9 BaseOS via ClonePack
baseurl=http://clonepack:8080/mirror/1 baseurl=http://clonepack:8080/mirror/rocky9-baseos
enabled=1 enabled=1
gpgcheck=0 gpgcheck=0
``` ```
**Configuration APT** (`/etc/apt/sources.list.d/clonepack.list`) : **Configuration APT** (`/etc/apt/sources.list.d/clonepack.list`) :
``` ```
deb [trusted=yes] http://clonepack:8080/mirror/2 bookworm main deb [trusted=yes] http://clonepack:8080/mirror/debian-bookworm bookworm main contrib
``` ```
Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si un fichier est absent, le serveur retourne 404.
--- ---
## API REST ## API REST
@ -209,6 +230,7 @@ Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si
| `POST` | `/api/v1/repos` | Créer un dépôt | | `POST` | `/api/v1/repos` | Créer un dépôt |
| `GET` | `/api/v1/repos` | Lister les dépôts | | `GET` | `/api/v1/repos` | Lister les dépôts |
| `GET` | `/api/v1/repos/{id}` | Détail d'un dépôt | | `GET` | `/api/v1/repos/{id}` | Détail d'un dépôt |
| `PATCH` | `/api/v1/repos/{id}` | Modifier un dépôt |
| `DELETE` | `/api/v1/repos/{id}` | Supprimer un dépôt | | `DELETE` | `/api/v1/repos/{id}` | Supprimer un dépôt |
| `POST` | `/api/v1/repos/{id}/clone` | Démarrer le clonage | | `POST` | `/api/v1/repos/{id}/clone` | Démarrer le clonage |
| `GET` | `/api/v1/repos/{id}/clone/status` | Statut du clonage | | `GET` | `/api/v1/repos/{id}/clone/status` | Statut du clonage |
@ -225,4 +247,26 @@ Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si
| `DELETE` | `/api/v1/repos/{id}/snapshots/{snap_id}` | Supprimer un snapshot | | `DELETE` | `/api/v1/repos/{id}/snapshots/{snap_id}` | Supprimer un snapshot |
| `GET` | `/api/v1/repos/{id}/snapshots/diff?from=X&to=Y` | Diff deux snapshots | | `GET` | `/api/v1/repos/{id}/snapshots/diff?from=X&to=Y` | Diff deux snapshots |
| `POST` | `/api/v1/repos/{id}/snapshots/{snap_id}/rollback` | Rollback | | `POST` | `/api/v1/repos/{id}/snapshots/{snap_id}/rollback` | Rollback |
| `GET` | `/mirror/{id}/*` | Proxy miroir HTTP | | `GET` | `/mirror/{id_ou_nom}/*` | Proxy miroir HTTP |
**Exemple de création d'un dépôt APT via REST :**
```bash
curl -X POST http://localhost:8080/api/v1/repos \
-H "Content-Type: application/json" \
-d '{
"name": "debian-bookworm",
"type": "apt",
"source_url": "https://deb.debian.org/debian",
"apt_suite": "bookworm",
"apt_components": ["main", "contrib"],
"apt_architectures": ["amd64"],
"sync_mode": "manual"
}'
```
**Exemple de modification via REST :**
```bash
curl -X PATCH http://localhost:8080/api/v1/repos/1 \
-H "Content-Type: application/json" \
-d '{"source_url": "https://archive.debian.org/debian", "apt_suite": "buster"}'
```

View file

@ -25,20 +25,26 @@ func New(baseURL string) *Client {
} }
type CreateRepoInput struct { type CreateRepoInput struct {
Name string `json:"name"` Name string `json:"name"`
Type string `json:"type"` Type string `json:"type"`
SourceURL string `json:"source_url"` SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode,omitempty"` SyncMode string `json:"sync_mode,omitempty"`
AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
} }
type Repo struct { type Repo struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Name string `json:"name"` Name string `json:"name"`
Type string `json:"type"` Type string `json:"type"`
SourceURL string `json:"source_url"` SourceURL string `json:"source_url"`
Frozen bool `json:"frozen"` Frozen bool `json:"frozen"`
SyncMode string `json:"sync_mode"` SyncMode string `json:"sync_mode"`
CreatedAt time.Time `json:"created_at"` AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
CreatedAt time.Time `json:"created_at"`
} }
type listReposResponse struct { type listReposResponse struct {
@ -84,6 +90,29 @@ func (c *Client) GetRepo(ctx context.Context, id int64) (*Repo, error) {
return &repo, nil return &repo, nil
} }
type UpdateRepoInput struct {
SourceURL *string `json:"source_url,omitempty"`
SyncMode *string `json:"sync_mode,omitempty"`
AptSuite *string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
}
func (c *Client) UpdateRepo(ctx context.Context, id int64, in UpdateRepoInput) (*Repo, error) {
body, _ := json.Marshal(in)
req, err := http.NewRequestWithContext(ctx, http.MethodPatch,
fmt.Sprintf("%s/api/v1/repos/%d", c.baseURL, id), bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
var repo Repo
if err := c.do(req, &repo); err != nil {
return nil, err
}
return &repo, nil
}
func (c *Client) DeleteRepo(ctx context.Context, id int64) error { func (c *Client) DeleteRepo(ctx context.Context, id int64) error {
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fmt.Sprintf("%s/api/v1/repos/%d", c.baseURL, id), nil) req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fmt.Sprintf("%s/api/v1/repos/%d", c.baseURL, id), nil)
if err != nil { if err != nil {

View file

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"os" "os"
"strconv" "strconv"
"strings"
"text/tabwriter" "text/tabwriter"
"time" "time"
@ -63,9 +64,12 @@ var repoCmd = &cobra.Command{
} }
var ( var (
repoName string repoName string
repoType string repoType string
repoSourceURL string repoSourceURL string
repoAptSuite string
repoAptComponents []string
repoAptArchs []string
) )
var repoCreateCmd = &cobra.Command{ var repoCreateCmd = &cobra.Command{
@ -74,10 +78,13 @@ var repoCreateCmd = &cobra.Command{
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
c := client.New(apiURL) c := client.New(apiURL)
repo, err := c.CreateRepo(cmd.Context(), client.CreateRepoInput{ repo, err := c.CreateRepo(cmd.Context(), client.CreateRepoInput{
Name: repoName, Name: repoName,
Type: repoType, Type: repoType,
SourceURL: repoSourceURL, SourceURL: repoSourceURL,
SyncMode: repoSyncMode, SyncMode: repoSyncMode,
AptSuite: repoAptSuite,
AptComponents: repoAptComponents,
AptArchitectures: repoAptArchs,
}) })
if err != nil { if err != nil {
return err return err
@ -87,6 +94,54 @@ var repoCreateCmd = &cobra.Command{
}, },
} }
var (
updateSourceURL string
updateSyncMode string
updateAptSuite string
updateComponents []string
updateArchs []string
)
var repoUpdateCmd = &cobra.Command{
Use: "update <id>",
Short: "Update a repository's source URL, sync mode, or APT config",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
id, err := strconv.ParseInt(args[0], 10, 64)
if err != nil {
return fmt.Errorf("invalid id: %s", args[0])
}
in := client.UpdateRepoInput{}
if cmd.Flags().Changed("source-url") {
in.SourceURL = &updateSourceURL
}
if cmd.Flags().Changed("sync-mode") {
in.SyncMode = &updateSyncMode
}
if cmd.Flags().Changed("suite") {
in.AptSuite = &updateAptSuite
}
if cmd.Flags().Changed("components") {
in.AptComponents = updateComponents
}
if cmd.Flags().Changed("arch") {
in.AptArchitectures = updateArchs
}
c := client.New(apiURL)
repo, err := c.UpdateRepo(cmd.Context(), id, in)
if errors.Is(err, client.ErrNotFound) {
return fmt.Errorf("repo %d not found", id)
}
if err != nil {
return err
}
printRepos([]client.Repo{*repo})
return nil
},
}
var repoListCmd = &cobra.Command{ var repoListCmd = &cobra.Command{
Use: "list", Use: "list",
Short: "List all repositories", Short: "List all repositories",
@ -149,11 +204,21 @@ var repoDeleteCmd = &cobra.Command{
func printRepos(repos []client.Repo) { func printRepos(repos []client.Repo) {
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "ID\tNAME\tTYPE\tSOURCE URL\tFROZEN\tSYNC MODE\tCREATED AT") fmt.Fprintln(w, "ID\tNAME\tTYPE\tSOURCE URL\tFROZEN\tSYNC MODE\tAPT CONFIG\tCREATED AT")
for _, r := range repos { for _, r := range repos {
fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%v\t%s\t%s\n", aptInfo := ""
if r.Type == "apt" {
aptInfo = r.AptSuite
if len(r.AptComponents) > 0 {
aptInfo += " [" + strings.Join(r.AptComponents, ",") + "]"
}
if len(r.AptArchitectures) > 0 {
aptInfo += " (" + strings.Join(r.AptArchitectures, ",") + ")"
}
}
fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%v\t%s\t%s\t%s\n",
r.ID, r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, r.ID, r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode,
r.CreatedAt.Format(time.RFC3339), aptInfo, r.CreatedAt.Format(time.RFC3339),
) )
} }
w.Flush() w.Flush()
@ -481,17 +546,26 @@ func init() {
repoCreateCmd.Flags().StringVar(&repoType, "type", "", "Type: apt|rpm|docker|binary (required)") repoCreateCmd.Flags().StringVar(&repoType, "type", "", "Type: apt|rpm|docker|binary (required)")
repoCreateCmd.Flags().StringVar(&repoSourceURL, "source-url", "", "Upstream source URL (required)") repoCreateCmd.Flags().StringVar(&repoSourceURL, "source-url", "", "Upstream source URL (required)")
repoCreateCmd.Flags().StringVar(&repoSyncMode, "sync-mode", "auto", "Sync mode: auto|manual") repoCreateCmd.Flags().StringVar(&repoSyncMode, "sync-mode", "auto", "Sync mode: auto|manual")
repoCreateCmd.Flags().StringVar(&repoAptSuite, "suite", "", "APT suite/codename (required for apt, e.g. bookworm)")
repoCreateCmd.Flags().StringSliceVar(&repoAptComponents, "components", nil, "APT components (default: main)")
repoCreateCmd.Flags().StringSliceVar(&repoAptArchs, "arch", nil, "APT architectures (default: amd64)")
repoCreateCmd.MarkFlagRequired("name") repoCreateCmd.MarkFlagRequired("name")
repoCreateCmd.MarkFlagRequired("type") repoCreateCmd.MarkFlagRequired("type")
repoCreateCmd.MarkFlagRequired("source-url") repoCreateCmd.MarkFlagRequired("source-url")
repoUpdateCmd.Flags().StringVar(&updateSourceURL, "source-url", "", "New upstream source URL")
repoUpdateCmd.Flags().StringVar(&updateSyncMode, "sync-mode", "", "New sync mode: auto|manual")
repoUpdateCmd.Flags().StringVar(&updateAptSuite, "suite", "", "New APT suite/codename")
repoUpdateCmd.Flags().StringSliceVar(&updateComponents, "components", nil, "New APT components")
repoUpdateCmd.Flags().StringSliceVar(&updateArchs, "arch", nil, "New APT architectures")
repoSyncApproveCmd.Flags().BoolVar(&syncApproveAll, "all", false, "Approuver tous les paquets en attente") repoSyncApproveCmd.Flags().BoolVar(&syncApproveAll, "all", false, "Approuver tous les paquets en attente")
repoSnapshotCreateCmd.Flags().StringVar(&snapshotLabel, "label", "", "Snapshot label") repoSnapshotCreateCmd.Flags().StringVar(&snapshotLabel, "label", "", "Snapshot label")
repoSnapshotCmd.AddCommand(repoSnapshotCreateCmd, repoSnapshotListCmd, repoSnapshotShowCmd, repoSnapshotCmd.AddCommand(repoSnapshotCreateCmd, repoSnapshotListCmd, repoSnapshotShowCmd,
repoSnapshotDeleteCmd, repoSnapshotDiffCmd, repoSnapshotRollbackCmd) repoSnapshotDeleteCmd, repoSnapshotDiffCmd, repoSnapshotRollbackCmd)
repoCmd.AddCommand(repoCreateCmd, repoListCmd, repoGetCmd, repoDeleteCmd, repoCloneCmd, repoCmd.AddCommand(repoCreateCmd, repoUpdateCmd, repoListCmd, repoGetCmd, repoDeleteCmd, repoCloneCmd,
repoSyncCmd, repoSyncListCmd, repoSyncApproveCmd, repoSyncRejectCmd, repoSyncCmd, repoSyncListCmd, repoSyncApproveCmd, repoSyncRejectCmd,
repoSyncBlockCmd, repoSyncUnblockCmd, repoSyncBlockListCmd, repoSyncBlockCmd, repoSyncUnblockCmd, repoSyncBlockListCmd,
repoSnapshotCmd) repoSnapshotCmd)

View file

@ -21,13 +21,15 @@ func NewProxyHandler(repoSvc *core.RepoService, dataDir string) *ProxyHandler {
} }
func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) { func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "repo_id"), 10, 64) ref := chi.URLParam(r, "repo_ref")
if err != nil {
http.Error(w, "invalid repo_id", http.StatusBadRequest)
return
}
repo, err := h.repoSvc.Get(r.Context(), repoID) var repo *store.Repo
var err error
if id, parseErr := strconv.ParseInt(ref, 10, 64); parseErr == nil {
repo, err = h.repoSvc.Get(r.Context(), id)
} else {
repo, err = h.repoSvc.GetByName(r.Context(), ref)
}
if errors.Is(err, store.ErrNotFound) { if errors.Is(err, store.ErrNotFound) {
http.NotFound(w, r) http.NotFound(w, r)
return return
@ -37,7 +39,7 @@ func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) {
return return
} }
localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repoID, repo.Type) localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repo.ID, repo.Type)
prefix := fmt.Sprintf("/mirror/%d", repoID) prefix := fmt.Sprintf("/mirror/%s", ref)
http.StripPrefix(prefix, http.FileServer(http.Dir(localDir))).ServeHTTP(w, r) http.StripPrefix(prefix, http.FileServer(http.Dir(localDir))).ServeHTTP(w, r)
} }

View file

@ -7,6 +7,7 @@ import (
"strconv" "strconv"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
"github.com/syonad/clonepack/internal/core" "github.com/syonad/clonepack/internal/core"
"github.com/syonad/clonepack/internal/store" "github.com/syonad/clonepack/internal/store"
) )
@ -27,10 +28,13 @@ func (h *RepoHandler) Create(w http.ResponseWriter, r *http.Request) {
} }
repo, err := h.svc.Create(r.Context(), core.CreateRepoInput{ repo, err := h.svc.Create(r.Context(), core.CreateRepoInput{
Name: req.Name, Name: req.Name,
Type: req.Type, Type: req.Type,
SourceURL: req.SourceURL, SourceURL: req.SourceURL,
SyncMode: req.SyncMode, SyncMode: req.SyncMode,
AptSuite: req.AptSuite,
AptComponents: req.AptComponents,
AptArchitectures: req.AptArchitectures,
}) })
if err != nil { if err != nil {
Error(w, http.StatusUnprocessableEntity, err.Error()) Error(w, http.StatusUnprocessableEntity, err.Error())
@ -72,6 +76,37 @@ func (h *RepoHandler) Get(w http.ResponseWriter, r *http.Request) {
JSON(w, http.StatusOK, repoToResponse(repo)) JSON(w, http.StatusOK, repoToResponse(repo))
} }
func (h *RepoHandler) Update(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var req UpdateRepoRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
Error(w, http.StatusBadRequest, "invalid JSON")
return
}
repo, err := h.svc.Update(r.Context(), id, core.UpdateRepoInput{
SourceURL: req.SourceURL,
SyncMode: req.SyncMode,
AptSuite: req.AptSuite,
AptComponents: req.AptComponents,
AptArchitectures: req.AptArchitectures,
})
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
}
if err != nil {
Error(w, http.StatusUnprocessableEntity, err.Error())
return
}
JSON(w, http.StatusOK, repoToResponse(repo))
}
func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) { func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil { if err != nil {
@ -90,7 +125,7 @@ func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) {
} }
func repoToResponse(r *store.Repo) RepoResponse { func repoToResponse(r *store.Repo) RepoResponse {
return RepoResponse{ resp := RepoResponse{
ID: r.ID, ID: r.ID,
Name: r.Name, Name: r.Name,
Type: r.Type, Type: r.Type,
@ -99,4 +134,13 @@ func repoToResponse(r *store.Repo) RepoResponse {
SyncMode: r.SyncMode, SyncMode: r.SyncMode,
CreatedAt: r.CreatedAt, CreatedAt: r.CreatedAt,
} }
if r.Type == "apt" && r.Config != "" {
var cfg aptclone.Config
if err := json.Unmarshal([]byte(r.Config), &cfg); err == nil {
resp.AptSuite = cfg.Suite
resp.AptComponents = cfg.Components
resp.AptArchitectures = cfg.Architectures
}
}
return resp
} }

View file

@ -13,7 +13,7 @@ func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler
r.Use(middleware.Recoverer) r.Use(middleware.Recoverer)
r.Use(middleware.RequestID) r.Use(middleware.RequestID)
r.Get("/mirror/{repo_id}/*", proxyHandler.ServeFile) r.Get("/mirror/{repo_ref}/*", proxyHandler.ServeFile)
r.Get("/health", func(w http.ResponseWriter, r *http.Request) { r.Get("/health", func(w http.ResponseWriter, r *http.Request) {
JSON(w, http.StatusOK, map[string]string{"status": "ok"}) JSON(w, http.StatusOK, map[string]string{"status": "ok"})
@ -24,6 +24,7 @@ func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler
r.Post("/", repoHandler.Create) r.Post("/", repoHandler.Create)
r.Get("/", repoHandler.List) r.Get("/", repoHandler.List)
r.Get("/{id}", repoHandler.Get) r.Get("/{id}", repoHandler.Get)
r.Patch("/{id}", repoHandler.Update)
r.Delete("/{id}", repoHandler.Delete) r.Delete("/{id}", repoHandler.Delete)
r.Post("/{id}/clone", cloneHandler.StartClone) r.Post("/{id}/clone", cloneHandler.StartClone)

View file

@ -3,20 +3,26 @@ package api
import "time" import "time"
type CreateRepoRequest struct { type CreateRepoRequest struct {
Name string `json:"name"` Name string `json:"name"`
Type string `json:"type"` Type string `json:"type"`
SourceURL string `json:"source_url"` SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode,omitempty"` SyncMode string `json:"sync_mode,omitempty"`
AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
} }
type RepoResponse struct { type RepoResponse struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Name string `json:"name"` Name string `json:"name"`
Type string `json:"type"` Type string `json:"type"`
SourceURL string `json:"source_url"` SourceURL string `json:"source_url"`
Frozen bool `json:"frozen"` Frozen bool `json:"frozen"`
SyncMode string `json:"sync_mode"` SyncMode string `json:"sync_mode"`
CreatedAt time.Time `json:"created_at"` AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
CreatedAt time.Time `json:"created_at"`
} }
type ListReposResponse struct { type ListReposResponse struct {
@ -24,6 +30,14 @@ type ListReposResponse struct {
Total int `json:"total"` Total int `json:"total"`
} }
type UpdateRepoRequest struct {
SourceURL *string `json:"source_url,omitempty"`
SyncMode *string `json:"sync_mode,omitempty"`
AptSuite *string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
}
type ErrorResponse struct { type ErrorResponse struct {
Error string `json:"error"` Error string `json:"error"`
} }

View file

@ -0,0 +1,184 @@
package apt
import (
"compress/gzip"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
const (
upstreamInRelease = ".upstream-InRelease"
upstreamPackagesGz = ".upstream-Packages.gz"
)
// Cloner mirrors an APT repository into a local directory.
type Cloner struct {
SourceURL string
DestDir string
Config Config
HTTPClient *http.Client
OnProgress func(file string, bytes int64)
}
// New creates a Cloner with a 30-minute HTTP timeout.
func New(sourceURL, destDir string, cfg Config) *Cloner {
return &Cloner{
SourceURL: strings.TrimRight(sourceURL, "/"),
DestDir: destDir,
Config: cfg,
HTTPClient: &http.Client{Timeout: 30 * time.Minute},
}
}
// Clone performs a full mirror of the configured APT repository.
func (c *Cloner) Clone(ctx context.Context) error {
suite := c.Config.Suite
// 1. Create base directories.
if err := os.MkdirAll(filepath.Join(c.DestDir, "dists", suite), 0o755); err != nil {
return fmt.Errorf("create dists dir: %w", err)
}
if err := os.MkdirAll(filepath.Join(c.DestDir, "pool"), 0o755); err != nil {
return fmt.Errorf("create pool dir: %w", err)
}
// 2. Fetch InRelease and save as .upstream-InRelease.
inReleaseURL := c.SourceURL + "/dists/" + suite + "/InRelease"
inReleaseData, err := fetchBytes(ctx, c.HTTPClient, inReleaseURL)
if err != nil {
return fmt.Errorf("fetch InRelease: %w", err)
}
upstreamIRPath := filepath.Join(c.DestDir, "dists", suite, upstreamInRelease)
if err := atomicWrite(upstreamIRPath, inReleaseData); err != nil {
return fmt.Errorf("save upstream InRelease: %w", err)
}
// 3. Parse InRelease.
rf, err := ParseRelease(inReleaseData)
if err != nil {
return fmt.Errorf("parse InRelease: %w", err)
}
// 4. For each component/arch: find Packages.gz in SHA256 section, download, parse debs.
var allDebs []DebPackage
for _, component := range c.Config.Components {
for _, arch := range c.Config.Architectures {
if err := ctx.Err(); err != nil {
return err
}
pkgs, err := c.fetchComponentPackages(ctx, suite, component, arch, rf)
if err != nil {
return fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
}
allDebs = append(allDebs, pkgs...)
}
}
// 5. Download all .deb files.
for _, pkg := range allDebs {
if err := ctx.Err(); err != nil {
return err
}
if err := c.downloadDeb(ctx, pkg); err != nil {
return fmt.Errorf("download %s: %w", pkg.Filename, err)
}
}
// 6. Regenerate local metadata.
if err := RegenerateMetadata(c.DestDir, c.Config); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
return nil
}
// fetchComponentPackages downloads Packages.gz for one component/arch pair,
// saves it as .upstream-Packages.gz, and returns the parsed package list.
func (c *Cloner) fetchComponentPackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
wantPath := component + "/binary-" + arch + "/Packages.gz"
// Verify this path is listed in the upstream SHA256 section.
var found bool
for _, e := range rf.SHA256 {
if e.Path == wantPath {
found = true
break
}
}
if !found {
// Not an error — this component/arch may just not exist upstream.
return nil, nil
}
pkgGzURL := c.SourceURL + "/dists/" + suite + "/" + wantPath
data, err := fetchBytes(ctx, c.HTTPClient, pkgGzURL)
if err != nil {
return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
}
// Save as .upstream-Packages.gz so RegenerateMetadata can find it later.
destDir := filepath.Join(c.DestDir, "dists", suite, component, "binary-"+arch)
if err := os.MkdirAll(destDir, 0o755); err != nil {
return nil, err
}
upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
if err := atomicWrite(upstreamPath, data); err != nil {
return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
}
if c.OnProgress != nil {
c.OnProgress(wantPath, int64(len(data)))
}
// Decompress and parse.
gz, err := gzip.NewReader(strings.NewReader(string(data)))
if err != nil {
return nil, fmt.Errorf("open gzip for %s: %w", wantPath, err)
}
defer gz.Close()
raw, err := io.ReadAll(gz)
if err != nil {
return nil, fmt.Errorf("decompress %s: %w", wantPath, err)
}
pkgs, err := ParsePackages(raw)
if err != nil {
return nil, fmt.Errorf("parse Packages for %s: %w", wantPath, err)
}
return pkgs, nil
}
// downloadDeb downloads a single .deb file into the local pool tree.
func (c *Cloner) downloadDeb(ctx context.Context, pkg DebPackage) error {
url := c.SourceURL + "/" + pkg.Filename
dest := filepath.Join(c.DestDir, filepath.FromSlash(pkg.Filename))
// Skip if already on disk.
if _, err := os.Stat(dest); err == nil {
return nil
}
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return err
}
data, err := fetchBytes(ctx, c.HTTPClient, url)
if err != nil {
return err
}
if err := atomicWrite(dest, data); err != nil {
return err
}
if c.OnProgress != nil {
c.OnProgress(pkg.Filename, int64(len(data)))
}
return nil
}

View file

@ -0,0 +1,8 @@
package apt
// Config describes which parts of an APT repository to mirror.
type Config struct {
Suite string `json:"suite"`
Components []string `json:"components"`
Architectures []string `json:"architectures"`
}

View file

@ -0,0 +1,44 @@
package apt
import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
)
// fetchBytes performs a GET request and returns the response body.
func fetchBytes(ctx context.Context, httpClient *http.Client, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
return io.ReadAll(resp.Body)
}
// atomicWrite writes data to path via a temporary file and an atomic rename.
func atomicWrite(path string, data []byte) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return err
}
if err := os.Rename(tmp, path); err != nil {
os.Remove(tmp)
return err
}
return nil
}

View file

@ -0,0 +1,148 @@
package apt
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
)
// InitEmptyRepo creates the directory structure for an APT mirror and writes an
// empty Release file so APT clients don't get a 404 before the first clone.
func InitEmptyRepo(localDir string, cfg Config) error {
if err := os.MkdirAll(filepath.Join(localDir, "pool"), 0o755); err != nil {
return err
}
for _, component := range cfg.Components {
for _, arch := range cfg.Architectures {
dir := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch)
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
}
}
releasePath := filepath.Join(localDir, "dists", cfg.Suite, "Release")
if _, err := os.Stat(releasePath); os.IsNotExist(err) {
releaseData := GenerateRelease(cfg.Suite, cfg.Suite, cfg.Components, cfg.Architectures, nil)
if err := atomicWrite(releasePath, releaseData); err != nil {
return fmt.Errorf("write empty Release: %w", err)
}
}
return nil
}
// RegenerateMetadata rebuilds local Packages, Packages.gz, and Release files
// for all configured component/arch pairs, keeping only packages present on disk.
func RegenerateMetadata(localDir string, cfg Config) error {
suite := cfg.Suite
var releaseEntries []ReleaseEntry
for _, component := range cfg.Components {
for _, arch := range cfg.Architectures {
entries, err := regenerateComponentArch(localDir, suite, component, arch)
if err != nil {
return fmt.Errorf("regenerate %s/%s: %w", component, arch, err)
}
releaseEntries = append(releaseEntries, entries...)
}
}
// Write dists/{suite}/Release.
releaseData := GenerateRelease(suite, suite, cfg.Components, cfg.Architectures, releaseEntries)
releasePath := filepath.Join(localDir, "dists", suite, "Release")
if err := os.MkdirAll(filepath.Dir(releasePath), 0o755); err != nil {
return fmt.Errorf("create dists dir: %w", err)
}
if err := atomicWrite(releasePath, releaseData); err != nil {
return fmt.Errorf("write Release: %w", err)
}
return nil
}
// regenerateComponentArch regenerates Packages and Packages.gz for one component/arch
// and returns the two ReleaseEntry values (plain then gz) for the Release file.
func regenerateComponentArch(localDir, suite, component, arch string) ([]ReleaseEntry, error) {
binDir := filepath.Join(localDir, "dists", suite, component, "binary-"+arch)
upstreamGzPath := filepath.Join(binDir, upstreamPackagesGz)
// 1. Read upstream Packages.gz.
gzData, err := os.ReadFile(upstreamGzPath)
if err != nil {
// Not cloned yet — skip silently.
return nil, nil
}
// 2. Decompress.
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return nil, fmt.Errorf("open gzip %s: %w", upstreamGzPath, err)
}
rawPackages, err := io.ReadAll(gz)
gz.Close()
if err != nil {
return nil, fmt.Errorf("decompress %s: %w", upstreamGzPath, err)
}
// 3. Parse all stanzas.
allPkgs, err := ParsePackages(rawPackages)
if err != nil {
return nil, fmt.Errorf("parse upstream Packages: %w", err)
}
// 4. Filter: keep only packages present on disk.
checker := diskPresenceChecker(localDir)
localPkgs := FilterPackages(allPkgs, checker)
// 5. Re-emit filtered stanzas.
plainBytes := emitPackages(localPkgs)
// 6. Gzip the filtered Packages.
var gzBuf bytes.Buffer
gzw := gzip.NewWriter(&gzBuf)
if _, err := gzw.Write(plainBytes); err != nil {
return nil, fmt.Errorf("gzip write: %w", err)
}
if err := gzw.Close(); err != nil {
return nil, fmt.Errorf("gzip close: %w", err)
}
gzBytes := gzBuf.Bytes()
// 7. Write Packages and Packages.gz.
if err := os.MkdirAll(binDir, 0o755); err != nil {
return nil, err
}
plainPath := filepath.Join(binDir, "Packages")
if err := atomicWrite(plainPath, plainBytes); err != nil {
return nil, fmt.Errorf("write Packages: %w", err)
}
gzPath := filepath.Join(binDir, "Packages.gz")
if err := atomicWrite(gzPath, gzBytes); err != nil {
return nil, fmt.Errorf("write Packages.gz: %w", err)
}
// 8. Compute SHA256 hashes.
plainSum := sha256.Sum256(plainBytes)
gzSum := sha256.Sum256(gzBytes)
// Release paths are relative to dists/{suite}/.
relBase := component + "/binary-" + arch + "/"
return []ReleaseEntry{
{
Hash: hex.EncodeToString(plainSum[:]),
Size: int64(len(plainBytes)),
Path: relBase + "Packages",
},
{
Hash: hex.EncodeToString(gzSum[:]),
Size: int64(len(gzBytes)),
Path: relBase + "Packages.gz",
},
}, nil
}

View file

@ -0,0 +1,128 @@
package apt
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strconv"
)
// DebPackage represents a single stanza from a Debian Packages file.
type DebPackage struct {
Package string
Version string
Architecture string
Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb
SHA256 string
Size int64
Raw []byte // original stanza bytes for faithful re-emission
}
// ParsePackages parses all stanzas from a Debian Packages control file.
// Stanzas are separated by blank lines.
func ParsePackages(data []byte) ([]DebPackage, error) {
// Normalise line endings.
data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n"))
var pkgs []DebPackage
stanzas := bytes.Split(data, []byte("\n\n"))
for _, stanza := range stanzas {
stanza = bytes.TrimSpace(stanza)
if len(stanza) == 0 {
continue
}
pkg, err := parseStanza(stanza)
if err != nil {
return nil, fmt.Errorf("parse stanza: %w", err)
}
pkgs = append(pkgs, pkg)
}
return pkgs, nil
}
func parseStanza(stanza []byte) (DebPackage, error) {
var pkg DebPackage
pkg.Raw = stanza
lines := bytes.Split(stanza, []byte("\n"))
for _, rawLine := range lines {
line := bytes.TrimRight(rawLine, " \t")
// Continuation lines (description body etc.) start with a space — skip.
if len(line) == 0 || line[0] == ' ' || line[0] == '\t' {
continue
}
colonIdx := bytes.IndexByte(line, ':')
if colonIdx < 0 {
continue
}
key := string(bytes.TrimSpace(line[:colonIdx]))
value := string(bytes.TrimSpace(line[colonIdx+1:]))
switch key {
case "Package":
pkg.Package = value
case "Version":
pkg.Version = value
case "Architecture":
pkg.Architecture = value
case "Filename":
pkg.Filename = value
case "SHA256":
pkg.SHA256 = value
case "Size":
n, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return DebPackage{}, fmt.Errorf("bad Size value %q: %w", value, err)
}
pkg.Size = n
}
}
if pkg.Package == "" {
return DebPackage{}, fmt.Errorf("stanza missing Package field")
}
return pkg, nil
}
// FilterPackages returns only the packages for which presentOnDisk returns true.
func FilterPackages(pkgs []DebPackage, presentOnDisk func(filename string) bool) []DebPackage {
var out []DebPackage
for _, p := range pkgs {
if presentOnDisk(p.Filename) {
out = append(out, p)
}
}
return out
}
// emitPackages serialises a slice of DebPackage back to Packages-file format,
// using the original Raw bytes for each stanza.
func emitPackages(pkgs []DebPackage) []byte {
var buf bytes.Buffer
for i, pkg := range pkgs {
// Strip trailing whitespace from each line within the stanza.
lines := bytes.Split(pkg.Raw, []byte("\n"))
for j, l := range lines {
lines[j] = bytes.TrimRight(l, " \t")
}
buf.Write(bytes.Join(lines, []byte("\n")))
if i < len(pkgs)-1 {
buf.WriteString("\n\n")
} else {
buf.WriteString("\n")
}
}
return buf.Bytes()
}
// diskPresenceChecker returns a presentOnDisk func rooted at localDir.
func diskPresenceChecker(localDir string) func(string) bool {
return func(filename string) bool {
p := filepath.Join(localDir, filepath.FromSlash(filename))
_, err := os.Stat(p)
return err == nil
}
}

View file

@ -0,0 +1,154 @@
package apt
import (
"bytes"
"fmt"
"strconv"
"strings"
"time"
)
// ReleaseFile represents a parsed Debian Release / InRelease file.
type ReleaseFile struct {
Origin string
Suite string
Codename string
Components string
Architectures string
SHA256 []ReleaseEntry
}
// ReleaseEntry is a single line from the SHA256 section of a Release file.
type ReleaseEntry struct {
Hash string
Size int64
Path string
}
// ParseRelease strips PGP armor (if present) and parses an InRelease or Release file.
func ParseRelease(data []byte) (*ReleaseFile, error) {
text := stripPGPArmor(string(data))
rf := &ReleaseFile{}
inSHA256 := false
for _, rawLine := range strings.Split(text, "\n") {
line := strings.TrimRight(rawLine, " \t\r")
if inSHA256 {
if line == "" || (len(line) > 0 && line[0] != ' ' && line[0] != '\t') {
// A non-indented non-empty line ends the SHA256 block.
if line != "" {
inSHA256 = false
// Fall through to parse this line as a key-value pair.
} else {
inSHA256 = false
continue
}
} else {
// Indented line: " <hash> <size> <path>"
entry, err := parseReleaseEntry(line)
if err == nil {
rf.SHA256 = append(rf.SHA256, entry)
}
continue
}
}
if strings.HasPrefix(line, "SHA256:") {
inSHA256 = true
continue
}
key, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
switch key {
case "Origin":
rf.Origin = value
case "Suite":
rf.Suite = value
case "Codename":
rf.Codename = value
case "Components":
rf.Components = value
case "Architectures":
rf.Architectures = value
}
}
return rf, nil
}
// parseReleaseEntry parses a single indented line from the SHA256 section.
// Format: " <hash> <size> <path>"
func parseReleaseEntry(line string) (ReleaseEntry, error) {
fields := strings.Fields(line)
if len(fields) != 3 {
return ReleaseEntry{}, fmt.Errorf("malformed SHA256 entry: %q", line)
}
size, err := strconv.ParseInt(fields[1], 10, 64)
if err != nil {
return ReleaseEntry{}, fmt.Errorf("malformed size in SHA256 entry: %w", err)
}
return ReleaseEntry{Hash: fields[0], Size: size, Path: fields[2]}, nil
}
// stripPGPArmor removes the PGP signed-message wrapper if present.
// It returns only the signed body (between the header and the signature).
func stripPGPArmor(text string) string {
const beginSigned = "-----BEGIN PGP SIGNED MESSAGE-----"
const beginSig = "-----BEGIN PGP SIGNATURE-----"
if !strings.Contains(text, beginSigned) {
return text
}
// Drop the header lines (Hash: etc.) up to the first blank line.
after, found := strings.CutPrefix(text, beginSigned)
if !found {
// beginSigned not at start — find it
idx := strings.Index(text, beginSigned)
if idx < 0 {
return text
}
after = text[idx+len(beginSigned):]
}
// Skip the armor header lines (e.g. "Hash: SHA512") until the blank line.
blankIdx := strings.Index(after, "\n\n")
if blankIdx < 0 {
return text
}
body := after[blankIdx+2:]
// Cut off at the PGP signature block.
if sigIdx := strings.Index(body, beginSig); sigIdx >= 0 {
body = body[:sigIdx]
}
return strings.TrimRight(body, "\n\r ") + "\n"
}
// GenerateRelease generates a plain unsigned Release file.
func GenerateRelease(suite, codename string, components, architectures []string, entries []ReleaseEntry) []byte {
var buf bytes.Buffer
fmt.Fprintf(&buf, "Origin: ClonePack Mirror\n")
fmt.Fprintf(&buf, "Suite: %s\n", suite)
fmt.Fprintf(&buf, "Codename: %s\n", codename)
fmt.Fprintf(&buf, "Components: %s\n", strings.Join(components, " "))
fmt.Fprintf(&buf, "Architectures: %s\n", strings.Join(architectures, " "))
fmt.Fprintf(&buf, "Date: %s\n", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 UTC"))
fmt.Fprintf(&buf, "SHA256:\n")
for _, e := range entries {
fmt.Fprintf(&buf, " %s %d %s\n", e.Hash, e.Size, e.Path)
}
return buf.Bytes()
}

View file

@ -0,0 +1,145 @@
package apt
import (
"compress/gzip"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
// NewPackage is a package present upstream but absent from the local mirror.
type NewPackage struct {
Package string
Version string
Architecture string
Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb
SHA256 string
Size int64
Component string
}
// Scanner compares an upstream APT repository against a local mirror and
// returns packages that are present upstream but missing on disk.
type Scanner struct {
SourceURL string
LocalDir string
Config Config
HTTPClient *http.Client
}
// NewScanner creates a Scanner with a 5-minute HTTP timeout.
func NewScanner(sourceURL, localDir string, cfg Config) *Scanner {
return &Scanner{
SourceURL: strings.TrimRight(sourceURL, "/"),
LocalDir: localDir,
Config: cfg,
HTTPClient: &http.Client{Timeout: 5 * time.Minute},
}
}
// Scan fetches the upstream package lists and returns packages not on disk.
func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) {
suite := sc.Config.Suite
// 1. Fetch InRelease.
inReleaseURL := sc.SourceURL + "/dists/" + suite + "/InRelease"
inReleaseData, err := fetchBytes(ctx, sc.HTTPClient, inReleaseURL)
if err != nil {
return nil, fmt.Errorf("fetch InRelease: %w", err)
}
rf, err := ParseRelease(inReleaseData)
if err != nil {
return nil, fmt.Errorf("parse InRelease: %w", err)
}
// 2 & 3. Fetch Packages.gz for each component/arch, save it, then filter.
var missing []NewPackage
for _, component := range sc.Config.Components {
for _, arch := range sc.Config.Architectures {
if err := ctx.Err(); err != nil {
return nil, err
}
pkgs, err := sc.fetchAndSavePackages(ctx, suite, component, arch, rf)
if err != nil {
return nil, fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err)
}
// 4. Filter: only return packages whose Filename is NOT on disk.
checker := diskPresenceChecker(sc.LocalDir)
for _, pkg := range pkgs {
if !checker(pkg.Filename) {
missing = append(missing, NewPackage{
Package: pkg.Package,
Version: pkg.Version,
Architecture: pkg.Architecture,
Filename: pkg.Filename,
SHA256: pkg.SHA256,
Size: pkg.Size,
Component: component,
})
}
}
}
}
return missing, nil
}
// fetchAndSavePackages fetches Packages.gz for a component/arch, saves it as
// .upstream-Packages.gz (so RegenerateMetadata can use it), and returns parsed packages.
func (sc *Scanner) fetchAndSavePackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) {
wantPath := component + "/binary-" + arch + "/Packages.gz"
// Check that this path is listed in the upstream SHA256 section.
var found bool
for _, e := range rf.SHA256 {
if e.Path == wantPath {
found = true
break
}
}
if !found {
return nil, nil
}
pkgGzURL := sc.SourceURL + "/dists/" + suite + "/" + wantPath
data, err := fetchBytes(ctx, sc.HTTPClient, pkgGzURL)
if err != nil {
return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err)
}
// Save as .upstream-Packages.gz.
destDir := filepath.Join(sc.LocalDir, "dists", suite, component, "binary-"+arch)
if err := os.MkdirAll(destDir, 0o755); err != nil {
return nil, err
}
upstreamPath := filepath.Join(destDir, upstreamPackagesGz)
if err := atomicWrite(upstreamPath, data); err != nil {
return nil, fmt.Errorf("save %s: %w", upstreamPath, err)
}
// Decompress and parse.
gz, err := gzip.NewReader(strings.NewReader(string(data)))
if err != nil {
return nil, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
raw, err := io.ReadAll(gz)
if err != nil {
return nil, fmt.Errorf("decompress: %w", err)
}
pkgs, err := ParsePackages(raw)
if err != nil {
return nil, fmt.Errorf("parse Packages: %w", err)
}
return pkgs, nil
}

View file

@ -2,11 +2,13 @@ package core
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"log" "log"
"path/filepath" "path/filepath"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
"github.com/syonad/clonepack/internal/clone/rpm" "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store" "github.com/syonad/clonepack/internal/store"
) )
@ -28,8 +30,8 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err
if err != nil { if err != nil {
return 0, err return 0, err
} }
if repo.Type != "rpm" { if repo.Type != "rpm" && repo.Type != "apt" {
return 0, fmt.Errorf("clone is only supported for rpm repositories") return 0, fmt.Errorf("clone is only supported for rpm and apt repositories")
} }
running, err := s.jobStore.HasRunningCloneJob(ctx, repoID) running, err := s.jobStore.HasRunningCloneJob(ctx, repoID)
@ -45,7 +47,7 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err
return 0, err return 0, err
} }
go s.runClone(repoID, jobID, repo.SourceURL, repo.Type) go s.runClone(repoID, jobID, repo)
return jobID, nil return jobID, nil
} }
@ -53,7 +55,7 @@ func (s *CloneService) GetLatestCloneJob(ctx context.Context, repoID int64) (*st
return s.jobStore.GetLatestCloneJob(ctx, repoID) return s.jobStore.GetLatestCloneJob(ctx, repoID)
} }
func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string) { func (s *CloneService) runClone(repoID, jobID int64, repo *store.Repo) {
ctx := context.Background() ctx := context.Background()
if err := s.jobStore.MarkCloneJobStarted(ctx, jobID); err != nil { if err := s.jobStore.MarkCloneJobStarted(ctx, jobID); err != nil {
@ -61,23 +63,41 @@ func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string)
return return
} }
destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repoType) destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
cloner := rpm.New(sourceURL, destDir) var cloneErr error
cloner.OnProgress = func(p rpm.Progress) {
log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone) switch repo.Type {
case "rpm":
cloner := rpm.New(repo.SourceURL, destDir)
cloner.OnProgress = func(p rpm.Progress) {
log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone)
}
if err := cloner.Clone(ctx); err != nil {
cloneErr = err
} else if err := rpm.RegenerateMetadata(destDir); err != nil {
log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err)
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
cloneErr = fmt.Errorf("parse apt config: %w", err)
break
}
cloner := aptclone.New(repo.SourceURL, destDir, cfg)
cloner.OnProgress = func(file string, bytes int64) {
log.Printf("clone job %d: %s (%d bytes)", jobID, file, bytes)
}
cloneErr = cloner.Clone(ctx)
} }
if err := cloner.Clone(ctx); err != nil { if cloneErr != nil {
errMsg := err.Error() errMsg := cloneErr.Error()
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobFailed, &errMsg) _ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobFailed, &errMsg)
log.Printf("clone job %d: failed: %v", jobID, err) log.Printf("clone job %d: failed: %v", jobID, cloneErr)
return return
} }
if err := rpm.RegenerateMetadata(destDir); err != nil {
log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err)
}
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobCompleted, nil) _ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobCompleted, nil)
log.Printf("clone job %d: completed", jobID) log.Printf("clone job %d: completed", jobID)
} }

View file

@ -2,10 +2,12 @@ package core
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"log" "log"
"path/filepath" "path/filepath"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm" rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store" "github.com/syonad/clonepack/internal/store"
) )
@ -24,10 +26,13 @@ func NewRepoService(s store.RepoStore, dataDir string) *RepoService {
var validSyncModes = map[string]bool{"auto": true, "manual": true} var validSyncModes = map[string]bool{"auto": true, "manual": true}
type CreateRepoInput struct { type CreateRepoInput struct {
Name string `json:"name"` Name string `json:"name"`
Type string `json:"type"` Type string `json:"type"`
SourceURL string `json:"source_url"` SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode"` SyncMode string `json:"sync_mode"`
AptSuite string `json:"apt_suite,omitempty"`
AptComponents []string `json:"apt_components,omitempty"`
AptArchitectures []string `json:"apt_architectures,omitempty"`
} }
func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Repo, error) { func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Repo, error) {
@ -46,7 +51,30 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re
return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", in.SyncMode) return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", in.SyncMode)
} }
r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode} var configJSON string
if in.Type == "apt" {
if in.AptSuite == "" {
return nil, fmt.Errorf("apt_suite is required for apt repositories")
}
if len(in.AptComponents) == 0 {
in.AptComponents = []string{"main"}
}
if len(in.AptArchitectures) == 0 {
in.AptArchitectures = []string{"amd64"}
}
cfg := aptclone.Config{
Suite: in.AptSuite,
Components: in.AptComponents,
Architectures: in.AptArchitectures,
}
b, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("marshal apt config: %w", err)
}
configJSON = string(b)
}
r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode, Config: configJSON}
id, err := s.store.CreateRepo(ctx, r) id, err := s.store.CreateRepo(ctx, r)
if err != nil { if err != nil {
return nil, err return nil, err
@ -60,15 +88,85 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re
} }
func (s *RepoService) initStorage(repo *store.Repo) { func (s *RepoService) initStorage(repo *store.Repo) {
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), repo.Type)
switch repo.Type { switch repo.Type {
case "rpm": case "rpm":
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), "rpm")
if err := rpmclone.InitEmptyRepo(localDir); err != nil { if err := rpmclone.InitEmptyRepo(localDir); err != nil {
log.Printf("init storage for repo %d: %v", repo.ID, err) log.Printf("init storage for repo %d: %v", repo.ID, err)
} }
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
log.Printf("init storage for repo %d: parse config: %v", repo.ID, err)
return
}
if err := aptclone.InitEmptyRepo(localDir, cfg); err != nil {
log.Printf("init storage for repo %d: %v", repo.ID, err)
}
} }
} }
type UpdateRepoInput struct {
SourceURL *string
SyncMode *string
AptSuite *string
AptComponents []string
AptArchitectures []string
}
func (s *RepoService) Update(ctx context.Context, id int64, in UpdateRepoInput) (*store.Repo, error) {
repo, err := s.store.GetRepo(ctx, id)
if err != nil {
return nil, err
}
if in.SourceURL != nil {
if *in.SourceURL == "" {
return nil, fmt.Errorf("source_url cannot be empty")
}
repo.SourceURL = *in.SourceURL
}
if in.SyncMode != nil {
if !validSyncModes[*in.SyncMode] {
return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", *in.SyncMode)
}
repo.SyncMode = *in.SyncMode
}
if repo.Type == "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) {
var cfg aptclone.Config
if repo.Config != "" {
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return nil, fmt.Errorf("parse existing apt config: %w", err)
}
}
if in.AptSuite != nil {
if *in.AptSuite == "" {
return nil, fmt.Errorf("apt_suite cannot be empty")
}
cfg.Suite = *in.AptSuite
}
if in.AptComponents != nil {
cfg.Components = in.AptComponents
}
if in.AptArchitectures != nil {
cfg.Architectures = in.AptArchitectures
}
b, err := json.Marshal(cfg)
if err != nil {
return nil, fmt.Errorf("marshal apt config: %w", err)
}
repo.Config = string(b)
} else if repo.Type != "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) {
return nil, fmt.Errorf("apt fields only apply to apt repositories")
}
if err := s.store.UpdateRepo(ctx, id, repo.SourceURL, repo.SyncMode, repo.Config); err != nil {
return nil, err
}
return s.store.GetRepo(ctx, id)
}
func (s *RepoService) List(ctx context.Context) ([]store.Repo, error) { func (s *RepoService) List(ctx context.Context) ([]store.Repo, error) {
return s.store.ListRepos(ctx) return s.store.ListRepos(ctx)
} }
@ -77,6 +175,10 @@ func (s *RepoService) Get(ctx context.Context, id int64) (*store.Repo, error) {
return s.store.GetRepo(ctx, id) return s.store.GetRepo(ctx, id)
} }
func (s *RepoService) GetByName(ctx context.Context, name string) (*store.Repo, error) {
return s.store.GetRepoByName(ctx, name)
}
func (s *RepoService) Delete(ctx context.Context, id int64) error { func (s *RepoService) Delete(ctx context.Context, id int64) error {
return s.store.DeleteRepo(ctx, id) return s.store.DeleteRepo(ctx, id)
} }

View file

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"compress/gzip" "compress/gzip"
"context" "context"
"encoding/json"
"encoding/xml" "encoding/xml"
"errors" "errors"
"fmt" "fmt"
@ -13,6 +14,7 @@ import (
"strings" "strings"
"time" "time"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm" rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store" "github.com/syonad/clonepack/internal/store"
) )
@ -46,60 +48,24 @@ type SnapshotDiff struct {
} }
func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label string) (int64, error) { func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label string) (int64, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { repo, err := s.repoStore.GetRepo(ctx, repoID)
if err != nil {
return 0, err return 0, err
} }
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml")) var pkgs []store.SnapshotPackage
switch repo.Type {
case "rpm":
pkgs, err = s.snapshotPackagesRPM(localDir)
case "apt":
pkgs, err = s.snapshotPackagesAPT(localDir, repo.Config)
default:
return 0, fmt.Errorf("snapshots not supported for repo type %q", repo.Type)
}
if err != nil { if err != nil {
return 0, fmt.Errorf("read repomd.xml: %w", err) return 0, err
}
var repomd rpmclone.RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return 0, fmt.Errorf("parse repomd.xml: %w", err)
}
var primaryHref string
for _, entry := range repomd.Data {
if entry.Type == "primary" {
primaryHref = entry.Location.Href
break
}
}
if primaryHref == "" {
return 0, fmt.Errorf("no primary entry in repomd.xml")
}
gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref)))
if err != nil {
return 0, fmt.Errorf("read primary.xml.gz: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return 0, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary rpmclone.PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return 0, fmt.Errorf("parse primary.xml: %w", err)
}
pkgs := make([]store.SnapshotPackage, len(primary.Packages))
for i, p := range primary.Packages {
pkgs[i] = store.SnapshotPackage{
Name: p.Name,
Version: p.Version.Ver + "-" + p.Version.Rel,
Arch: p.Arch,
Location: p.Location.Href,
Checksum: p.Checksum.Value,
ChecksumType: p.Checksum.Type,
Size: p.Size.Package,
}
} }
snapID, err := s.snapshotStore.CreateSnapshot(ctx, repoID, label) snapID, err := s.snapshotStore.CreateSnapshot(ctx, repoID, label)
@ -114,6 +80,94 @@ func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label
return snapID, nil return snapID, nil
} }
func (s *SnapshotService) snapshotPackagesRPM(localDir string) ([]store.SnapshotPackage, error) {
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml"))
if err != nil {
return nil, fmt.Errorf("read repomd.xml: %w", err)
}
var repomd rpmclone.RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return nil, fmt.Errorf("parse repomd.xml: %w", err)
}
var primaryHref string
for _, entry := range repomd.Data {
if entry.Type == "primary" {
primaryHref = entry.Location.Href
break
}
}
if primaryHref == "" {
return nil, fmt.Errorf("no primary entry in repomd.xml")
}
gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref)))
if err != nil {
return nil, fmt.Errorf("read primary.xml.gz: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return nil, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary rpmclone.PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return nil, fmt.Errorf("parse primary.xml: %w", err)
}
pkgs := make([]store.SnapshotPackage, len(primary.Packages))
for i, p := range primary.Packages {
pkgs[i] = store.SnapshotPackage{
Name: p.Name,
Version: p.Version.Ver + "-" + p.Version.Rel,
Arch: p.Arch,
Location: p.Location.Href,
Checksum: p.Checksum.Value,
ChecksumType: p.Checksum.Type,
Size: p.Size.Package,
}
}
return pkgs, nil
}
func (s *SnapshotService) snapshotPackagesAPT(localDir, configJSON string) ([]store.SnapshotPackage, error) {
var cfg aptclone.Config
if err := json.Unmarshal([]byte(configJSON), &cfg); err != nil {
return nil, fmt.Errorf("parse apt config: %w", err)
}
seen := make(map[string]struct{})
var pkgs []store.SnapshotPackage
for _, component := range cfg.Components {
for _, arch := range cfg.Architectures {
pkgsPath := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch, "Packages")
data, err := os.ReadFile(pkgsPath)
if err != nil {
// Not yet populated — skip silently.
continue
}
debs, err := aptclone.ParsePackages(data)
if err != nil {
return nil, fmt.Errorf("parse Packages for %s/%s: %w", component, arch, err)
}
for _, d := range debs {
if _, dup := seen[d.Filename]; dup {
continue
}
seen[d.Filename] = struct{}{}
pkgs = append(pkgs, store.SnapshotPackage{
Name: d.Package,
Version: d.Version,
Arch: d.Architecture,
Location: d.Filename,
Checksum: d.SHA256,
ChecksumType: "sha256",
Size: d.Size,
})
}
}
}
return pkgs, nil
}
func (s *SnapshotService) List(ctx context.Context, repoID int64) ([]store.Snapshot, error) { func (s *SnapshotService) List(ctx context.Context, repoID int64) ([]store.Snapshot, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return nil, err return nil, err
@ -212,16 +266,18 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64
return err return err
} }
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
snapSet := make(map[string]store.SnapshotPackage, len(pkgs)) snapSet := make(map[string]store.SnapshotPackage, len(pkgs))
for _, p := range pkgs { for _, p := range pkgs {
snapSet[p.Location] = p snapSet[p.Location] = p
} }
ext := packageExt(repo.Type)
var errs []error var errs []error
if err := filepath.WalkDir(localDir, func(path string, d os.DirEntry, err error) error { if err := filepath.WalkDir(localDir, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() || filepath.Ext(path) != ".rpm" { if err != nil || d.IsDir() || filepath.Ext(path) != ext {
return err return err
} }
rel, _ := filepath.Rel(localDir, path) rel, _ := filepath.Rel(localDir, path)
@ -251,10 +307,28 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64
return err return err
} }
if repo.Type == "rpm" { switch repo.Type {
case "rpm":
if err := rpmclone.RegenerateMetadata(localDir); err != nil { if err := rpmclone.RegenerateMetadata(localDir); err != nil {
return fmt.Errorf("regenerate metadata: %w", err) return fmt.Errorf("regenerate metadata: %w", err)
} }
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return fmt.Errorf("parse apt config: %w", err)
}
if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
} }
return nil return nil
} }
func packageExt(repoType string) string {
switch repoType {
case "apt":
return ".deb"
default:
return ".rpm"
}
}

View file

@ -2,6 +2,7 @@ package core
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"log" "log"
@ -11,6 +12,7 @@ import (
"strings" "strings"
"time" "time"
aptclone "github.com/syonad/clonepack/internal/clone/apt"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm" rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store" "github.com/syonad/clonepack/internal/store"
) )
@ -54,18 +56,54 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
if err != nil { if err != nil {
return err return err
} }
if repo.Type != "rpm" {
return fmt.Errorf("sync only supported for rpm repos") localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
type candidate struct {
Name string
Version string
Arch string
Location string
Checksum string
ChecksumType string
Size int64
}
var candidates []candidate
switch repo.Type {
case "rpm":
scanner := rpmclone.NewScanner(repo.SourceURL, localDir)
pkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
for _, p := range pkgs {
candidates = append(candidates, candidate{
Name: p.Name, Version: p.Version, Arch: p.Arch,
Location: p.Location, Checksum: p.Checksum, ChecksumType: p.ChecksumType, Size: p.Size,
})
}
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
return fmt.Errorf("parse apt config for repo %d: %w", repoID, err)
}
scanner := aptclone.NewScanner(repo.SourceURL, localDir, cfg)
pkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
for _, p := range pkgs {
candidates = append(candidates, candidate{
Name: p.Package, Version: p.Version, Arch: p.Architecture,
Location: p.Filename, Checksum: p.SHA256, ChecksumType: "sha256", Size: p.Size,
})
}
default:
return fmt.Errorf("sync only supported for rpm and apt repos")
} }
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") if len(candidates) == 0 {
scanner := rpmclone.NewScanner(repo.SourceURL, localDir)
newPkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
if len(newPkgs) == 0 {
log.Printf("sync: repo %d is up to date", repoID) log.Printf("sync: repo %d is up to date", repoID)
return nil return nil
} }
@ -82,19 +120,19 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
blockedNames[b.Name] = true blockedNames[b.Name] = true
} }
} }
filtered := newPkgs[:0] filtered := candidates[:0]
for _, p := range newPkgs { for _, p := range candidates {
if !blockedLocations[p.Location] && !blockedNames[p.Name] { if !blockedLocations[p.Location] && !blockedNames[p.Name] {
filtered = append(filtered, p) filtered = append(filtered, p)
} }
} }
newPkgs = filtered candidates = filtered
if len(newPkgs) == 0 { if len(candidates) == 0 {
log.Printf("sync: repo %d is up to date (all new packages are blocked)", repoID) log.Printf("sync: repo %d is up to date (all new packages are blocked)", repoID)
return nil return nil
} }
log.Printf("sync: repo %d has %d new package(s)", repoID, len(newPkgs)) log.Printf("sync: repo %d has %d new package(s)", repoID, len(candidates))
switch repo.SyncMode { switch repo.SyncMode {
case "auto": case "auto":
@ -103,8 +141,8 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
return fmt.Errorf("start clone for repo %d: %w", repoID, err) return fmt.Errorf("start clone for repo %d: %w", repoID, err)
} }
case "manual": case "manual":
pending := make([]store.PendingPackage, len(newPkgs)) pending := make([]store.PendingPackage, len(candidates))
for i, p := range newPkgs { for i, p := range candidates {
pending[i] = store.PendingPackage{ pending[i] = store.PendingPackage{
RepoID: repoID, RepoID: repoID,
Name: p.Name, Name: p.Name,
@ -148,7 +186,7 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in
wanted[id] = true wanted[id] = true
} }
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type)
var downloadErrors []error var downloadErrors []error
var anyApproved bool var anyApproved bool
for _, pkg := range all { for _, pkg := range all {
@ -173,14 +211,22 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in
} }
if anyApproved { if anyApproved {
if repo.Type == "rpm" { switch repo.Type {
case "rpm":
if err := rpmclone.RegenerateMetadata(localDir); err != nil { if err := rpmclone.RegenerateMetadata(localDir); err != nil {
log.Printf("metadata regeneration for repo %d failed: %v", repoID, err) log.Printf("metadata regeneration for repo %d failed: %v", repoID, err)
} }
} label := "auto-" + time.Now().UTC().Format(time.RFC3339)
label := "auto-" + time.Now().UTC().Format(time.RFC3339) if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil {
if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil { log.Printf("auto-snapshot for repo %d failed: %v", repoID, err)
log.Printf("auto-snapshot for repo %d failed: %v", repoID, err) }
case "apt":
var cfg aptclone.Config
if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil {
log.Printf("parse apt config for repo %d: %v", repoID, err)
} else if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil {
log.Printf("metadata regeneration for repo %d failed: %v", repoID, err)
}
} }
} }

View file

@ -27,6 +27,9 @@ var migration005 string
//go:embed migrations/000006_blocked_packages_name.up.sql //go:embed migrations/000006_blocked_packages_name.up.sql
var migration006 string var migration006 string
//go:embed migrations/000007_repo_config.up.sql
var migration007 string
func Open(cfg config.DBConfig) (*sql.DB, error) { func Open(cfg config.DBConfig) (*sql.DB, error) {
db, err := sql.Open("sqlite", cfg.Path) db, err := sql.Open("sqlite", cfg.Path)
if err != nil { if err != nil {
@ -66,6 +69,7 @@ func runMigrations(db *sql.DB) error {
{4, migration004}, {4, migration004},
{5, migration005}, {5, migration005},
{6, migration006}, {6, migration006},
{7, migration007},
} }
for _, m := range migrations { for _, m := range migrations {

View file

@ -0,0 +1,4 @@
-- SQLite ne supporte pas DROP COLUMN avant 3.35
CREATE TABLE repos_backup AS SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos;
DROP TABLE repos;
ALTER TABLE repos_backup RENAME TO repos;

View file

@ -0,0 +1 @@
ALTER TABLE repos ADD COLUMN config TEXT NOT NULL DEFAULT '';

View file

@ -16,6 +16,7 @@ type Repo struct {
SourceURL string `db:"source_url"` SourceURL string `db:"source_url"`
Frozen bool `db:"frozen"` Frozen bool `db:"frozen"`
SyncMode string `db:"sync_mode"` SyncMode string `db:"sync_mode"`
Config string `db:"config"`
CreatedAt time.Time `db:"created_at"` CreatedAt time.Time `db:"created_at"`
} }
@ -23,8 +24,10 @@ type RepoStore interface {
CreateRepo(ctx context.Context, r *Repo) (int64, error) CreateRepo(ctx context.Context, r *Repo) (int64, error)
ListRepos(ctx context.Context) ([]Repo, error) ListRepos(ctx context.Context) ([]Repo, error)
GetRepo(ctx context.Context, id int64) (*Repo, error) GetRepo(ctx context.Context, id int64) (*Repo, error)
GetRepoByName(ctx context.Context, name string) (*Repo, error)
DeleteRepo(ctx context.Context, id int64) error DeleteRepo(ctx context.Context, id int64) error
UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error
UpdateRepo(ctx context.Context, id int64, sourceURL, syncMode, config string) error
} }
type SQLiteRepoStore struct { type SQLiteRepoStore struct {
@ -37,8 +40,8 @@ func NewRepoStore(db *sql.DB) *SQLiteRepoStore {
func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error) { func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error) {
res, err := s.db.ExecContext(ctx, res, err := s.db.ExecContext(ctx,
`INSERT INTO repos (name, type, source_url, frozen, sync_mode) VALUES (?, ?, ?, ?, ?)`, `INSERT INTO repos (name, type, source_url, frozen, sync_mode, config) VALUES (?, ?, ?, ?, ?, ?)`,
r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, r.Config,
) )
if err != nil { if err != nil {
return 0, err return 0, err
@ -48,7 +51,7 @@ func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error
func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) { func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) {
rows, err := s.db.QueryContext(ctx, rows, err := s.db.QueryContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos ORDER BY created_at DESC`) `SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos ORDER BY created_at DESC`)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -57,7 +60,7 @@ func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) {
var repos []Repo var repos []Repo
for rows.Next() { for rows.Next() {
var r Repo var r Repo
if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt); err != nil { if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt); err != nil {
return nil, err return nil, err
} }
repos = append(repos, r) repos = append(repos, r)
@ -68,8 +71,22 @@ func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) {
func (s *SQLiteRepoStore) GetRepo(ctx context.Context, id int64) (*Repo, error) { func (s *SQLiteRepoStore) GetRepo(ctx context.Context, id int64) (*Repo, error) {
var r Repo var r Repo
err := s.db.QueryRowContext(ctx, err := s.db.QueryRowContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos WHERE id = ?`, id, `SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos WHERE id = ?`, id,
).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt) ).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
return &r, nil
}
func (s *SQLiteRepoStore) GetRepoByName(ctx context.Context, name string) (*Repo, error) {
var r Repo
err := s.db.QueryRowContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos WHERE name = ?`, name,
).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt)
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound return nil, ErrNotFound
} }
@ -94,6 +111,21 @@ func (s *SQLiteRepoStore) DeleteRepo(ctx context.Context, id int64) error {
return nil return nil
} }
func (s *SQLiteRepoStore) UpdateRepo(ctx context.Context, id int64, sourceURL, syncMode, config string) error {
res, err := s.db.ExecContext(ctx,
`UPDATE repos SET source_url = ?, sync_mode = ?, config = ? WHERE id = ?`,
sourceURL, syncMode, config, id,
)
if err != nil {
return err
}
n, _ := res.RowsAffected()
if n == 0 {
return ErrNotFound
}
return nil
}
func (s *SQLiteRepoStore) UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error { func (s *SQLiteRepoStore) UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error {
res, err := s.db.ExecContext(ctx, `UPDATE repos SET sync_mode = ? WHERE id = ?`, mode, id) res, err := s.db.ExecContext(ctx, `UPDATE repos SET sync_mode = ? WHERE id = ?`, mode, id)
if err != nil { if err != nil {