From c4776d81ddfea16b969e4c7d2a2d6e3e392073c7 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sat, 25 Apr 2026 19:03:26 +0200 Subject: [PATCH] add apt gestion Signed-off-by: GnomeZworc --- README.md | 72 +++++-- client/client.go | 51 +++-- cmd/clonepack/repo.go | 96 +++++++-- internal/api/proxy_handler.go | 18 +- internal/api/repo_handler.go | 54 ++++- internal/api/router.go | 3 +- internal/api/types.go | 36 ++-- internal/clone/apt/cloner.go | 184 ++++++++++++++++++ internal/clone/apt/config.go | 8 + internal/clone/apt/helpers.go | 44 +++++ internal/clone/apt/metadata_gen.go | 148 ++++++++++++++ internal/clone/apt/packages.go | 128 ++++++++++++ internal/clone/apt/release.go | 154 +++++++++++++++ internal/clone/apt/scanner.go | 145 ++++++++++++++ internal/core/clone.go | 50 +++-- internal/core/repo.go | 114 ++++++++++- internal/core/snapshot.go | 178 ++++++++++++----- internal/core/sync.go | 92 ++++++--- internal/store/db.go | 4 + .../migrations/000007_repo_config.down.sql | 4 + .../migrations/000007_repo_config.up.sql | 1 + internal/store/repo.go | 44 ++++- 22 files changed, 1465 insertions(+), 163 deletions(-) create mode 100644 internal/clone/apt/cloner.go create mode 100644 internal/clone/apt/config.go create mode 100644 internal/clone/apt/helpers.go create mode 100644 internal/clone/apt/metadata_gen.go create mode 100644 internal/clone/apt/packages.go create mode 100644 internal/clone/apt/release.go create mode 100644 internal/clone/apt/scanner.go create mode 100644 internal/store/migrations/000007_repo_config.down.sql create mode 100644 internal/store/migrations/000007_repo_config.up.sql diff --git a/README.md b/README.md index 8ef6d9b..646f9b0 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,18 @@ ClonePack est un système de clonage et de gel d'artefacts. Il permet de synchro ## Types d'artefacts supportés - Enterprise Linux (RPM/YUM/DNF) -- Debian (APT) — *à venir* +- Debian/Ubuntu (APT) - Docker images — *à venir* - Binaires — *à venir* ## Fonctionnalités -- Clonage de dépôts externes avec vérification SHA256 +- Clonage de dépôts RPM et APT avec vérification SHA256 - Scan régulier des sources pour détecter les nouveaux paquets - Validation manuelle ou automatique des mises à jour -- Snapshots métadonnées avec diff et rollback -- Proxy miroir HTTP (consommable directement par `yum`/`apt`) +- Blocklist permanente par paquet (toutes versions) +- Snapshots avec diff et rollback (RPM) +- Proxy miroir HTTP accessible par ID ou par nom de dépôt --- @@ -85,11 +86,21 @@ Toutes les commandes CLI appellent l'API REST. L'URL du serveur se configure ave --type rpm \ --source-url https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os +# Créer un dépôt APT +./bin/clonepack repo create \ + --name debian-bookworm \ + --type apt \ + --source-url https://deb.debian.org/debian \ + --suite bookworm \ + --components main,contrib \ + --arch amd64 + # Créer avec validation manuelle ./bin/clonepack repo create \ - --name rocky9-baseos \ - --type rpm \ - --source-url https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os \ + --name debian-bookworm \ + --type apt \ + --source-url https://deb.debian.org/debian \ + --suite bookworm \ --sync-mode manual # Lister les dépôts @@ -98,10 +109,19 @@ Toutes les commandes CLI appellent l'API REST. L'URL du serveur se configure ave # Détail d'un dépôt ./bin/clonepack repo get 1 +# Modifier un dépôt (source URL, sync mode, config APT) +./bin/clonepack repo update 1 --source-url https://archive.debian.org/debian +./bin/clonepack repo update 1 --suite bookworm-backports +./bin/clonepack repo update 1 --components main,contrib,non-free +./bin/clonepack repo update 1 --sync-mode manual + # Supprimer un dépôt ./bin/clonepack repo delete 1 ``` +Les flags `--suite`, `--components` et `--arch` ne s'appliquent qu'aux dépôts de type `apt`. +Seuls les flags fournis sont modifiés — les autres champs restent inchangés. + ### Clonage ```bash @@ -171,34 +191,35 @@ Les paquets bloqués n'apparaissent plus jamais dans la liste pending, quelle qu ./bin/clonepack repo snapshot delete 1 3 ``` -Un snapshot automatique est créé après chaque `sync-approve`. +Un snapshot automatique est créé après chaque `sync-approve` sur un dépôt RPM. --- ## Proxy miroir -ClonePack expose chaque dépôt cloné comme un miroir HTTP à l'adresse : +ClonePack expose chaque dépôt cloné comme un miroir HTTP. L'accès est possible par **ID numérique** ou par **nom du dépôt** : ``` http://:/mirror// +http://:/mirror// ``` +Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si un fichier est absent, le serveur retourne 404. + **Configuration YUM/DNF** (`/etc/yum.repos.d/clonepack.repo`) : ```ini [rocky9-baseos] name=Rocky Linux 9 BaseOS via ClonePack -baseurl=http://clonepack:8080/mirror/1 +baseurl=http://clonepack:8080/mirror/rocky9-baseos enabled=1 gpgcheck=0 ``` **Configuration APT** (`/etc/apt/sources.list.d/clonepack.list`) : ``` -deb [trusted=yes] http://clonepack:8080/mirror/2 bookworm main +deb [trusted=yes] http://clonepack:8080/mirror/debian-bookworm bookworm main contrib ``` -Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si un fichier est absent, le serveur retourne 404. - --- ## API REST @@ -209,6 +230,7 @@ Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si | `POST` | `/api/v1/repos` | Créer un dépôt | | `GET` | `/api/v1/repos` | Lister les dépôts | | `GET` | `/api/v1/repos/{id}` | Détail d'un dépôt | +| `PATCH` | `/api/v1/repos/{id}` | Modifier un dépôt | | `DELETE` | `/api/v1/repos/{id}` | Supprimer un dépôt | | `POST` | `/api/v1/repos/{id}/clone` | Démarrer le clonage | | `GET` | `/api/v1/repos/{id}/clone/status` | Statut du clonage | @@ -225,4 +247,26 @@ Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si | `DELETE` | `/api/v1/repos/{id}/snapshots/{snap_id}` | Supprimer un snapshot | | `GET` | `/api/v1/repos/{id}/snapshots/diff?from=X&to=Y` | Diff deux snapshots | | `POST` | `/api/v1/repos/{id}/snapshots/{snap_id}/rollback` | Rollback | -| `GET` | `/mirror/{id}/*` | Proxy miroir HTTP | +| `GET` | `/mirror/{id_ou_nom}/*` | Proxy miroir HTTP | + +**Exemple de création d'un dépôt APT via REST :** +```bash +curl -X POST http://localhost:8080/api/v1/repos \ + -H "Content-Type: application/json" \ + -d '{ + "name": "debian-bookworm", + "type": "apt", + "source_url": "https://deb.debian.org/debian", + "apt_suite": "bookworm", + "apt_components": ["main", "contrib"], + "apt_architectures": ["amd64"], + "sync_mode": "manual" + }' +``` + +**Exemple de modification via REST :** +```bash +curl -X PATCH http://localhost:8080/api/v1/repos/1 \ + -H "Content-Type: application/json" \ + -d '{"source_url": "https://archive.debian.org/debian", "apt_suite": "buster"}' +``` diff --git a/client/client.go b/client/client.go index e33523d..3ec86e8 100644 --- a/client/client.go +++ b/client/client.go @@ -25,20 +25,26 @@ func New(baseURL string) *Client { } type CreateRepoInput struct { - Name string `json:"name"` - Type string `json:"type"` - SourceURL string `json:"source_url"` - SyncMode string `json:"sync_mode,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + SyncMode string `json:"sync_mode,omitempty"` + AptSuite string `json:"apt_suite,omitempty"` + AptComponents []string `json:"apt_components,omitempty"` + AptArchitectures []string `json:"apt_architectures,omitempty"` } type Repo struct { - ID int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - SourceURL string `json:"source_url"` - Frozen bool `json:"frozen"` - SyncMode string `json:"sync_mode"` - CreatedAt time.Time `json:"created_at"` + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + Frozen bool `json:"frozen"` + SyncMode string `json:"sync_mode"` + AptSuite string `json:"apt_suite,omitempty"` + AptComponents []string `json:"apt_components,omitempty"` + AptArchitectures []string `json:"apt_architectures,omitempty"` + CreatedAt time.Time `json:"created_at"` } type listReposResponse struct { @@ -84,6 +90,29 @@ func (c *Client) GetRepo(ctx context.Context, id int64) (*Repo, error) { return &repo, nil } +type UpdateRepoInput struct { + SourceURL *string `json:"source_url,omitempty"` + SyncMode *string `json:"sync_mode,omitempty"` + AptSuite *string `json:"apt_suite,omitempty"` + AptComponents []string `json:"apt_components,omitempty"` + AptArchitectures []string `json:"apt_architectures,omitempty"` +} + +func (c *Client) UpdateRepo(ctx context.Context, id int64, in UpdateRepoInput) (*Repo, error) { + body, _ := json.Marshal(in) + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, + fmt.Sprintf("%s/api/v1/repos/%d", c.baseURL, id), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + var repo Repo + if err := c.do(req, &repo); err != nil { + return nil, err + } + return &repo, nil +} + func (c *Client) DeleteRepo(ctx context.Context, id int64) error { req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fmt.Sprintf("%s/api/v1/repos/%d", c.baseURL, id), nil) if err != nil { diff --git a/cmd/clonepack/repo.go b/cmd/clonepack/repo.go index b20059c..e96da3b 100644 --- a/cmd/clonepack/repo.go +++ b/cmd/clonepack/repo.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strconv" + "strings" "text/tabwriter" "time" @@ -63,9 +64,12 @@ var repoCmd = &cobra.Command{ } var ( - repoName string - repoType string - repoSourceURL string + repoName string + repoType string + repoSourceURL string + repoAptSuite string + repoAptComponents []string + repoAptArchs []string ) var repoCreateCmd = &cobra.Command{ @@ -74,10 +78,13 @@ var repoCreateCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { c := client.New(apiURL) repo, err := c.CreateRepo(cmd.Context(), client.CreateRepoInput{ - Name: repoName, - Type: repoType, - SourceURL: repoSourceURL, - SyncMode: repoSyncMode, + Name: repoName, + Type: repoType, + SourceURL: repoSourceURL, + SyncMode: repoSyncMode, + AptSuite: repoAptSuite, + AptComponents: repoAptComponents, + AptArchitectures: repoAptArchs, }) if err != nil { return err @@ -87,6 +94,54 @@ var repoCreateCmd = &cobra.Command{ }, } +var ( + updateSourceURL string + updateSyncMode string + updateAptSuite string + updateComponents []string + updateArchs []string +) + +var repoUpdateCmd = &cobra.Command{ + Use: "update ", + Short: "Update a repository's source URL, sync mode, or APT config", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return fmt.Errorf("invalid id: %s", args[0]) + } + + in := client.UpdateRepoInput{} + if cmd.Flags().Changed("source-url") { + in.SourceURL = &updateSourceURL + } + if cmd.Flags().Changed("sync-mode") { + in.SyncMode = &updateSyncMode + } + if cmd.Flags().Changed("suite") { + in.AptSuite = &updateAptSuite + } + if cmd.Flags().Changed("components") { + in.AptComponents = updateComponents + } + if cmd.Flags().Changed("arch") { + in.AptArchitectures = updateArchs + } + + c := client.New(apiURL) + repo, err := c.UpdateRepo(cmd.Context(), id, in) + if errors.Is(err, client.ErrNotFound) { + return fmt.Errorf("repo %d not found", id) + } + if err != nil { + return err + } + printRepos([]client.Repo{*repo}) + return nil + }, +} + var repoListCmd = &cobra.Command{ Use: "list", Short: "List all repositories", @@ -149,11 +204,21 @@ var repoDeleteCmd = &cobra.Command{ func printRepos(repos []client.Repo) { w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "ID\tNAME\tTYPE\tSOURCE URL\tFROZEN\tSYNC MODE\tCREATED AT") + fmt.Fprintln(w, "ID\tNAME\tTYPE\tSOURCE URL\tFROZEN\tSYNC MODE\tAPT CONFIG\tCREATED AT") for _, r := range repos { - fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%v\t%s\t%s\n", + aptInfo := "" + if r.Type == "apt" { + aptInfo = r.AptSuite + if len(r.AptComponents) > 0 { + aptInfo += " [" + strings.Join(r.AptComponents, ",") + "]" + } + if len(r.AptArchitectures) > 0 { + aptInfo += " (" + strings.Join(r.AptArchitectures, ",") + ")" + } + } + fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%v\t%s\t%s\t%s\n", r.ID, r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, - r.CreatedAt.Format(time.RFC3339), + aptInfo, r.CreatedAt.Format(time.RFC3339), ) } w.Flush() @@ -481,17 +546,26 @@ func init() { repoCreateCmd.Flags().StringVar(&repoType, "type", "", "Type: apt|rpm|docker|binary (required)") repoCreateCmd.Flags().StringVar(&repoSourceURL, "source-url", "", "Upstream source URL (required)") repoCreateCmd.Flags().StringVar(&repoSyncMode, "sync-mode", "auto", "Sync mode: auto|manual") + repoCreateCmd.Flags().StringVar(&repoAptSuite, "suite", "", "APT suite/codename (required for apt, e.g. bookworm)") + repoCreateCmd.Flags().StringSliceVar(&repoAptComponents, "components", nil, "APT components (default: main)") + repoCreateCmd.Flags().StringSliceVar(&repoAptArchs, "arch", nil, "APT architectures (default: amd64)") repoCreateCmd.MarkFlagRequired("name") repoCreateCmd.MarkFlagRequired("type") repoCreateCmd.MarkFlagRequired("source-url") + repoUpdateCmd.Flags().StringVar(&updateSourceURL, "source-url", "", "New upstream source URL") + repoUpdateCmd.Flags().StringVar(&updateSyncMode, "sync-mode", "", "New sync mode: auto|manual") + repoUpdateCmd.Flags().StringVar(&updateAptSuite, "suite", "", "New APT suite/codename") + repoUpdateCmd.Flags().StringSliceVar(&updateComponents, "components", nil, "New APT components") + repoUpdateCmd.Flags().StringSliceVar(&updateArchs, "arch", nil, "New APT architectures") + repoSyncApproveCmd.Flags().BoolVar(&syncApproveAll, "all", false, "Approuver tous les paquets en attente") repoSnapshotCreateCmd.Flags().StringVar(&snapshotLabel, "label", "", "Snapshot label") repoSnapshotCmd.AddCommand(repoSnapshotCreateCmd, repoSnapshotListCmd, repoSnapshotShowCmd, repoSnapshotDeleteCmd, repoSnapshotDiffCmd, repoSnapshotRollbackCmd) - repoCmd.AddCommand(repoCreateCmd, repoListCmd, repoGetCmd, repoDeleteCmd, repoCloneCmd, + repoCmd.AddCommand(repoCreateCmd, repoUpdateCmd, repoListCmd, repoGetCmd, repoDeleteCmd, repoCloneCmd, repoSyncCmd, repoSyncListCmd, repoSyncApproveCmd, repoSyncRejectCmd, repoSyncBlockCmd, repoSyncUnblockCmd, repoSyncBlockListCmd, repoSnapshotCmd) diff --git a/internal/api/proxy_handler.go b/internal/api/proxy_handler.go index cac9794..f104a5f 100644 --- a/internal/api/proxy_handler.go +++ b/internal/api/proxy_handler.go @@ -21,13 +21,15 @@ func NewProxyHandler(repoSvc *core.RepoService, dataDir string) *ProxyHandler { } func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) { - repoID, err := strconv.ParseInt(chi.URLParam(r, "repo_id"), 10, 64) - if err != nil { - http.Error(w, "invalid repo_id", http.StatusBadRequest) - return - } + ref := chi.URLParam(r, "repo_ref") - repo, err := h.repoSvc.Get(r.Context(), repoID) + var repo *store.Repo + var err error + if id, parseErr := strconv.ParseInt(ref, 10, 64); parseErr == nil { + repo, err = h.repoSvc.Get(r.Context(), id) + } else { + repo, err = h.repoSvc.GetByName(r.Context(), ref) + } if errors.Is(err, store.ErrNotFound) { http.NotFound(w, r) return @@ -37,7 +39,7 @@ func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) { return } - localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repoID, repo.Type) - prefix := fmt.Sprintf("/mirror/%d", repoID) + localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repo.ID, repo.Type) + prefix := fmt.Sprintf("/mirror/%s", ref) http.StripPrefix(prefix, http.FileServer(http.Dir(localDir))).ServeHTTP(w, r) } diff --git a/internal/api/repo_handler.go b/internal/api/repo_handler.go index 9b791e7..d14f756 100644 --- a/internal/api/repo_handler.go +++ b/internal/api/repo_handler.go @@ -7,6 +7,7 @@ import ( "strconv" "github.com/go-chi/chi/v5" + aptclone "github.com/syonad/clonepack/internal/clone/apt" "github.com/syonad/clonepack/internal/core" "github.com/syonad/clonepack/internal/store" ) @@ -27,10 +28,13 @@ func (h *RepoHandler) Create(w http.ResponseWriter, r *http.Request) { } repo, err := h.svc.Create(r.Context(), core.CreateRepoInput{ - Name: req.Name, - Type: req.Type, - SourceURL: req.SourceURL, - SyncMode: req.SyncMode, + Name: req.Name, + Type: req.Type, + SourceURL: req.SourceURL, + SyncMode: req.SyncMode, + AptSuite: req.AptSuite, + AptComponents: req.AptComponents, + AptArchitectures: req.AptArchitectures, }) if err != nil { Error(w, http.StatusUnprocessableEntity, err.Error()) @@ -72,6 +76,37 @@ func (h *RepoHandler) Get(w http.ResponseWriter, r *http.Request) { JSON(w, http.StatusOK, repoToResponse(repo)) } +func (h *RepoHandler) Update(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + + var req UpdateRepoRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + Error(w, http.StatusBadRequest, "invalid JSON") + return + } + + repo, err := h.svc.Update(r.Context(), id, core.UpdateRepoInput{ + SourceURL: req.SourceURL, + SyncMode: req.SyncMode, + AptSuite: req.AptSuite, + AptComponents: req.AptComponents, + AptArchitectures: req.AptArchitectures, + }) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } + if err != nil { + Error(w, http.StatusUnprocessableEntity, err.Error()) + return + } + JSON(w, http.StatusOK, repoToResponse(repo)) +} + func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) { id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) if err != nil { @@ -90,7 +125,7 @@ func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) { } func repoToResponse(r *store.Repo) RepoResponse { - return RepoResponse{ + resp := RepoResponse{ ID: r.ID, Name: r.Name, Type: r.Type, @@ -99,4 +134,13 @@ func repoToResponse(r *store.Repo) RepoResponse { SyncMode: r.SyncMode, CreatedAt: r.CreatedAt, } + if r.Type == "apt" && r.Config != "" { + var cfg aptclone.Config + if err := json.Unmarshal([]byte(r.Config), &cfg); err == nil { + resp.AptSuite = cfg.Suite + resp.AptComponents = cfg.Components + resp.AptArchitectures = cfg.Architectures + } + } + return resp } diff --git a/internal/api/router.go b/internal/api/router.go index 55cb3b9..ee14133 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -13,7 +13,7 @@ func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler r.Use(middleware.Recoverer) r.Use(middleware.RequestID) - r.Get("/mirror/{repo_id}/*", proxyHandler.ServeFile) + r.Get("/mirror/{repo_ref}/*", proxyHandler.ServeFile) r.Get("/health", func(w http.ResponseWriter, r *http.Request) { JSON(w, http.StatusOK, map[string]string{"status": "ok"}) @@ -24,6 +24,7 @@ func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler r.Post("/", repoHandler.Create) r.Get("/", repoHandler.List) r.Get("/{id}", repoHandler.Get) + r.Patch("/{id}", repoHandler.Update) r.Delete("/{id}", repoHandler.Delete) r.Post("/{id}/clone", cloneHandler.StartClone) diff --git a/internal/api/types.go b/internal/api/types.go index e2f7c26..a7d7742 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -3,20 +3,26 @@ package api import "time" type CreateRepoRequest struct { - Name string `json:"name"` - Type string `json:"type"` - SourceURL string `json:"source_url"` - SyncMode string `json:"sync_mode,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + SyncMode string `json:"sync_mode,omitempty"` + AptSuite string `json:"apt_suite,omitempty"` + AptComponents []string `json:"apt_components,omitempty"` + AptArchitectures []string `json:"apt_architectures,omitempty"` } type RepoResponse struct { - ID int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - SourceURL string `json:"source_url"` - Frozen bool `json:"frozen"` - SyncMode string `json:"sync_mode"` - CreatedAt time.Time `json:"created_at"` + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + Frozen bool `json:"frozen"` + SyncMode string `json:"sync_mode"` + AptSuite string `json:"apt_suite,omitempty"` + AptComponents []string `json:"apt_components,omitempty"` + AptArchitectures []string `json:"apt_architectures,omitempty"` + CreatedAt time.Time `json:"created_at"` } type ListReposResponse struct { @@ -24,6 +30,14 @@ type ListReposResponse struct { Total int `json:"total"` } +type UpdateRepoRequest struct { + SourceURL *string `json:"source_url,omitempty"` + SyncMode *string `json:"sync_mode,omitempty"` + AptSuite *string `json:"apt_suite,omitempty"` + AptComponents []string `json:"apt_components,omitempty"` + AptArchitectures []string `json:"apt_architectures,omitempty"` +} + type ErrorResponse struct { Error string `json:"error"` } diff --git a/internal/clone/apt/cloner.go b/internal/clone/apt/cloner.go new file mode 100644 index 0000000..fced28f --- /dev/null +++ b/internal/clone/apt/cloner.go @@ -0,0 +1,184 @@ +package apt + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + upstreamInRelease = ".upstream-InRelease" + upstreamPackagesGz = ".upstream-Packages.gz" +) + +// Cloner mirrors an APT repository into a local directory. +type Cloner struct { + SourceURL string + DestDir string + Config Config + HTTPClient *http.Client + OnProgress func(file string, bytes int64) +} + +// New creates a Cloner with a 30-minute HTTP timeout. +func New(sourceURL, destDir string, cfg Config) *Cloner { + return &Cloner{ + SourceURL: strings.TrimRight(sourceURL, "/"), + DestDir: destDir, + Config: cfg, + HTTPClient: &http.Client{Timeout: 30 * time.Minute}, + } +} + +// Clone performs a full mirror of the configured APT repository. +func (c *Cloner) Clone(ctx context.Context) error { + suite := c.Config.Suite + + // 1. Create base directories. + if err := os.MkdirAll(filepath.Join(c.DestDir, "dists", suite), 0o755); err != nil { + return fmt.Errorf("create dists dir: %w", err) + } + if err := os.MkdirAll(filepath.Join(c.DestDir, "pool"), 0o755); err != nil { + return fmt.Errorf("create pool dir: %w", err) + } + + // 2. Fetch InRelease and save as .upstream-InRelease. + inReleaseURL := c.SourceURL + "/dists/" + suite + "/InRelease" + inReleaseData, err := fetchBytes(ctx, c.HTTPClient, inReleaseURL) + if err != nil { + return fmt.Errorf("fetch InRelease: %w", err) + } + upstreamIRPath := filepath.Join(c.DestDir, "dists", suite, upstreamInRelease) + if err := atomicWrite(upstreamIRPath, inReleaseData); err != nil { + return fmt.Errorf("save upstream InRelease: %w", err) + } + + // 3. Parse InRelease. + rf, err := ParseRelease(inReleaseData) + if err != nil { + return fmt.Errorf("parse InRelease: %w", err) + } + + // 4. For each component/arch: find Packages.gz in SHA256 section, download, parse debs. + var allDebs []DebPackage + for _, component := range c.Config.Components { + for _, arch := range c.Config.Architectures { + if err := ctx.Err(); err != nil { + return err + } + pkgs, err := c.fetchComponentPackages(ctx, suite, component, arch, rf) + if err != nil { + return fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err) + } + allDebs = append(allDebs, pkgs...) + } + } + + // 5. Download all .deb files. + for _, pkg := range allDebs { + if err := ctx.Err(); err != nil { + return err + } + if err := c.downloadDeb(ctx, pkg); err != nil { + return fmt.Errorf("download %s: %w", pkg.Filename, err) + } + } + + // 6. Regenerate local metadata. + if err := RegenerateMetadata(c.DestDir, c.Config); err != nil { + return fmt.Errorf("regenerate metadata: %w", err) + } + + return nil +} + +// fetchComponentPackages downloads Packages.gz for one component/arch pair, +// saves it as .upstream-Packages.gz, and returns the parsed package list. +func (c *Cloner) fetchComponentPackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) { + wantPath := component + "/binary-" + arch + "/Packages.gz" + + // Verify this path is listed in the upstream SHA256 section. + var found bool + for _, e := range rf.SHA256 { + if e.Path == wantPath { + found = true + break + } + } + if !found { + // Not an error — this component/arch may just not exist upstream. + return nil, nil + } + + pkgGzURL := c.SourceURL + "/dists/" + suite + "/" + wantPath + data, err := fetchBytes(ctx, c.HTTPClient, pkgGzURL) + if err != nil { + return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err) + } + + // Save as .upstream-Packages.gz so RegenerateMetadata can find it later. + destDir := filepath.Join(c.DestDir, "dists", suite, component, "binary-"+arch) + if err := os.MkdirAll(destDir, 0o755); err != nil { + return nil, err + } + upstreamPath := filepath.Join(destDir, upstreamPackagesGz) + if err := atomicWrite(upstreamPath, data); err != nil { + return nil, fmt.Errorf("save %s: %w", upstreamPath, err) + } + + if c.OnProgress != nil { + c.OnProgress(wantPath, int64(len(data))) + } + + // Decompress and parse. + gz, err := gzip.NewReader(strings.NewReader(string(data))) + if err != nil { + return nil, fmt.Errorf("open gzip for %s: %w", wantPath, err) + } + defer gz.Close() + + raw, err := io.ReadAll(gz) + if err != nil { + return nil, fmt.Errorf("decompress %s: %w", wantPath, err) + } + + pkgs, err := ParsePackages(raw) + if err != nil { + return nil, fmt.Errorf("parse Packages for %s: %w", wantPath, err) + } + return pkgs, nil +} + +// downloadDeb downloads a single .deb file into the local pool tree. +func (c *Cloner) downloadDeb(ctx context.Context, pkg DebPackage) error { + url := c.SourceURL + "/" + pkg.Filename + dest := filepath.Join(c.DestDir, filepath.FromSlash(pkg.Filename)) + + // Skip if already on disk. + if _, err := os.Stat(dest); err == nil { + return nil + } + + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + + data, err := fetchBytes(ctx, c.HTTPClient, url) + if err != nil { + return err + } + if err := atomicWrite(dest, data); err != nil { + return err + } + + if c.OnProgress != nil { + c.OnProgress(pkg.Filename, int64(len(data))) + } + return nil +} diff --git a/internal/clone/apt/config.go b/internal/clone/apt/config.go new file mode 100644 index 0000000..50ea5ec --- /dev/null +++ b/internal/clone/apt/config.go @@ -0,0 +1,8 @@ +package apt + +// Config describes which parts of an APT repository to mirror. +type Config struct { + Suite string `json:"suite"` + Components []string `json:"components"` + Architectures []string `json:"architectures"` +} diff --git a/internal/clone/apt/helpers.go b/internal/clone/apt/helpers.go new file mode 100644 index 0000000..4c1fc9d --- /dev/null +++ b/internal/clone/apt/helpers.go @@ -0,0 +1,44 @@ +package apt + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" +) + +// fetchBytes performs a GET request and returns the response body. +func fetchBytes(ctx context.Context, httpClient *http.Client, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + return io.ReadAll(resp.Body) +} + +// atomicWrite writes data to path via a temporary file and an atomic rename. +func atomicWrite(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return err + } + return nil +} diff --git a/internal/clone/apt/metadata_gen.go b/internal/clone/apt/metadata_gen.go new file mode 100644 index 0000000..65a4ffa --- /dev/null +++ b/internal/clone/apt/metadata_gen.go @@ -0,0 +1,148 @@ +package apt + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" +) + +// InitEmptyRepo creates the directory structure for an APT mirror and writes an +// empty Release file so APT clients don't get a 404 before the first clone. +func InitEmptyRepo(localDir string, cfg Config) error { + if err := os.MkdirAll(filepath.Join(localDir, "pool"), 0o755); err != nil { + return err + } + for _, component := range cfg.Components { + for _, arch := range cfg.Architectures { + dir := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + } + } + releasePath := filepath.Join(localDir, "dists", cfg.Suite, "Release") + if _, err := os.Stat(releasePath); os.IsNotExist(err) { + releaseData := GenerateRelease(cfg.Suite, cfg.Suite, cfg.Components, cfg.Architectures, nil) + if err := atomicWrite(releasePath, releaseData); err != nil { + return fmt.Errorf("write empty Release: %w", err) + } + } + return nil +} + +// RegenerateMetadata rebuilds local Packages, Packages.gz, and Release files +// for all configured component/arch pairs, keeping only packages present on disk. +func RegenerateMetadata(localDir string, cfg Config) error { + suite := cfg.Suite + var releaseEntries []ReleaseEntry + + for _, component := range cfg.Components { + for _, arch := range cfg.Architectures { + entries, err := regenerateComponentArch(localDir, suite, component, arch) + if err != nil { + return fmt.Errorf("regenerate %s/%s: %w", component, arch, err) + } + releaseEntries = append(releaseEntries, entries...) + } + } + + // Write dists/{suite}/Release. + releaseData := GenerateRelease(suite, suite, cfg.Components, cfg.Architectures, releaseEntries) + releasePath := filepath.Join(localDir, "dists", suite, "Release") + if err := os.MkdirAll(filepath.Dir(releasePath), 0o755); err != nil { + return fmt.Errorf("create dists dir: %w", err) + } + if err := atomicWrite(releasePath, releaseData); err != nil { + return fmt.Errorf("write Release: %w", err) + } + + return nil +} + +// regenerateComponentArch regenerates Packages and Packages.gz for one component/arch +// and returns the two ReleaseEntry values (plain then gz) for the Release file. +func regenerateComponentArch(localDir, suite, component, arch string) ([]ReleaseEntry, error) { + binDir := filepath.Join(localDir, "dists", suite, component, "binary-"+arch) + upstreamGzPath := filepath.Join(binDir, upstreamPackagesGz) + + // 1. Read upstream Packages.gz. + gzData, err := os.ReadFile(upstreamGzPath) + if err != nil { + // Not cloned yet — skip silently. + return nil, nil + } + + // 2. Decompress. + gz, err := gzip.NewReader(bytes.NewReader(gzData)) + if err != nil { + return nil, fmt.Errorf("open gzip %s: %w", upstreamGzPath, err) + } + rawPackages, err := io.ReadAll(gz) + gz.Close() + if err != nil { + return nil, fmt.Errorf("decompress %s: %w", upstreamGzPath, err) + } + + // 3. Parse all stanzas. + allPkgs, err := ParsePackages(rawPackages) + if err != nil { + return nil, fmt.Errorf("parse upstream Packages: %w", err) + } + + // 4. Filter: keep only packages present on disk. + checker := diskPresenceChecker(localDir) + localPkgs := FilterPackages(allPkgs, checker) + + // 5. Re-emit filtered stanzas. + plainBytes := emitPackages(localPkgs) + + // 6. Gzip the filtered Packages. + var gzBuf bytes.Buffer + gzw := gzip.NewWriter(&gzBuf) + if _, err := gzw.Write(plainBytes); err != nil { + return nil, fmt.Errorf("gzip write: %w", err) + } + if err := gzw.Close(); err != nil { + return nil, fmt.Errorf("gzip close: %w", err) + } + gzBytes := gzBuf.Bytes() + + // 7. Write Packages and Packages.gz. + if err := os.MkdirAll(binDir, 0o755); err != nil { + return nil, err + } + plainPath := filepath.Join(binDir, "Packages") + if err := atomicWrite(plainPath, plainBytes); err != nil { + return nil, fmt.Errorf("write Packages: %w", err) + } + gzPath := filepath.Join(binDir, "Packages.gz") + if err := atomicWrite(gzPath, gzBytes); err != nil { + return nil, fmt.Errorf("write Packages.gz: %w", err) + } + + // 8. Compute SHA256 hashes. + plainSum := sha256.Sum256(plainBytes) + gzSum := sha256.Sum256(gzBytes) + + // Release paths are relative to dists/{suite}/. + relBase := component + "/binary-" + arch + "/" + + return []ReleaseEntry{ + { + Hash: hex.EncodeToString(plainSum[:]), + Size: int64(len(plainBytes)), + Path: relBase + "Packages", + }, + { + Hash: hex.EncodeToString(gzSum[:]), + Size: int64(len(gzBytes)), + Path: relBase + "Packages.gz", + }, + }, nil +} + diff --git a/internal/clone/apt/packages.go b/internal/clone/apt/packages.go new file mode 100644 index 0000000..ad42796 --- /dev/null +++ b/internal/clone/apt/packages.go @@ -0,0 +1,128 @@ +package apt + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strconv" +) + +// DebPackage represents a single stanza from a Debian Packages file. +type DebPackage struct { + Package string + Version string + Architecture string + Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb + SHA256 string + Size int64 + Raw []byte // original stanza bytes for faithful re-emission +} + +// ParsePackages parses all stanzas from a Debian Packages control file. +// Stanzas are separated by blank lines. +func ParsePackages(data []byte) ([]DebPackage, error) { + // Normalise line endings. + data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) + + var pkgs []DebPackage + stanzas := bytes.Split(data, []byte("\n\n")) + + for _, stanza := range stanzas { + stanza = bytes.TrimSpace(stanza) + if len(stanza) == 0 { + continue + } + pkg, err := parseStanza(stanza) + if err != nil { + return nil, fmt.Errorf("parse stanza: %w", err) + } + pkgs = append(pkgs, pkg) + } + return pkgs, nil +} + +func parseStanza(stanza []byte) (DebPackage, error) { + var pkg DebPackage + pkg.Raw = stanza + + lines := bytes.Split(stanza, []byte("\n")) + for _, rawLine := range lines { + line := bytes.TrimRight(rawLine, " \t") + // Continuation lines (description body etc.) start with a space — skip. + if len(line) == 0 || line[0] == ' ' || line[0] == '\t' { + continue + } + + colonIdx := bytes.IndexByte(line, ':') + if colonIdx < 0 { + continue + } + key := string(bytes.TrimSpace(line[:colonIdx])) + value := string(bytes.TrimSpace(line[colonIdx+1:])) + + switch key { + case "Package": + pkg.Package = value + case "Version": + pkg.Version = value + case "Architecture": + pkg.Architecture = value + case "Filename": + pkg.Filename = value + case "SHA256": + pkg.SHA256 = value + case "Size": + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return DebPackage{}, fmt.Errorf("bad Size value %q: %w", value, err) + } + pkg.Size = n + } + } + + if pkg.Package == "" { + return DebPackage{}, fmt.Errorf("stanza missing Package field") + } + return pkg, nil +} + +// FilterPackages returns only the packages for which presentOnDisk returns true. +func FilterPackages(pkgs []DebPackage, presentOnDisk func(filename string) bool) []DebPackage { + var out []DebPackage + for _, p := range pkgs { + if presentOnDisk(p.Filename) { + out = append(out, p) + } + } + return out +} + +// emitPackages serialises a slice of DebPackage back to Packages-file format, +// using the original Raw bytes for each stanza. +func emitPackages(pkgs []DebPackage) []byte { + var buf bytes.Buffer + for i, pkg := range pkgs { + // Strip trailing whitespace from each line within the stanza. + lines := bytes.Split(pkg.Raw, []byte("\n")) + for j, l := range lines { + lines[j] = bytes.TrimRight(l, " \t") + } + buf.Write(bytes.Join(lines, []byte("\n"))) + if i < len(pkgs)-1 { + buf.WriteString("\n\n") + } else { + buf.WriteString("\n") + } + } + return buf.Bytes() +} + +// diskPresenceChecker returns a presentOnDisk func rooted at localDir. +func diskPresenceChecker(localDir string) func(string) bool { + return func(filename string) bool { + p := filepath.Join(localDir, filepath.FromSlash(filename)) + _, err := os.Stat(p) + return err == nil + } +} diff --git a/internal/clone/apt/release.go b/internal/clone/apt/release.go new file mode 100644 index 0000000..ae42b24 --- /dev/null +++ b/internal/clone/apt/release.go @@ -0,0 +1,154 @@ +package apt + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "time" +) + +// ReleaseFile represents a parsed Debian Release / InRelease file. +type ReleaseFile struct { + Origin string + Suite string + Codename string + Components string + Architectures string + SHA256 []ReleaseEntry +} + +// ReleaseEntry is a single line from the SHA256 section of a Release file. +type ReleaseEntry struct { + Hash string + Size int64 + Path string +} + +// ParseRelease strips PGP armor (if present) and parses an InRelease or Release file. +func ParseRelease(data []byte) (*ReleaseFile, error) { + text := stripPGPArmor(string(data)) + + rf := &ReleaseFile{} + inSHA256 := false + + for _, rawLine := range strings.Split(text, "\n") { + line := strings.TrimRight(rawLine, " \t\r") + + if inSHA256 { + if line == "" || (len(line) > 0 && line[0] != ' ' && line[0] != '\t') { + // A non-indented non-empty line ends the SHA256 block. + if line != "" { + inSHA256 = false + // Fall through to parse this line as a key-value pair. + } else { + inSHA256 = false + continue + } + } else { + // Indented line: " " + entry, err := parseReleaseEntry(line) + if err == nil { + rf.SHA256 = append(rf.SHA256, entry) + } + continue + } + } + + if strings.HasPrefix(line, "SHA256:") { + inSHA256 = true + continue + } + + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + + switch key { + case "Origin": + rf.Origin = value + case "Suite": + rf.Suite = value + case "Codename": + rf.Codename = value + case "Components": + rf.Components = value + case "Architectures": + rf.Architectures = value + } + } + + return rf, nil +} + +// parseReleaseEntry parses a single indented line from the SHA256 section. +// Format: " " +func parseReleaseEntry(line string) (ReleaseEntry, error) { + fields := strings.Fields(line) + if len(fields) != 3 { + return ReleaseEntry{}, fmt.Errorf("malformed SHA256 entry: %q", line) + } + size, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return ReleaseEntry{}, fmt.Errorf("malformed size in SHA256 entry: %w", err) + } + return ReleaseEntry{Hash: fields[0], Size: size, Path: fields[2]}, nil +} + +// stripPGPArmor removes the PGP signed-message wrapper if present. +// It returns only the signed body (between the header and the signature). +func stripPGPArmor(text string) string { + const beginSigned = "-----BEGIN PGP SIGNED MESSAGE-----" + const beginSig = "-----BEGIN PGP SIGNATURE-----" + + if !strings.Contains(text, beginSigned) { + return text + } + + // Drop the header lines (Hash: etc.) up to the first blank line. + after, found := strings.CutPrefix(text, beginSigned) + if !found { + // beginSigned not at start — find it + idx := strings.Index(text, beginSigned) + if idx < 0 { + return text + } + after = text[idx+len(beginSigned):] + } + + // Skip the armor header lines (e.g. "Hash: SHA512") until the blank line. + blankIdx := strings.Index(after, "\n\n") + if blankIdx < 0 { + return text + } + body := after[blankIdx+2:] + + // Cut off at the PGP signature block. + if sigIdx := strings.Index(body, beginSig); sigIdx >= 0 { + body = body[:sigIdx] + } + + return strings.TrimRight(body, "\n\r ") + "\n" +} + +// GenerateRelease generates a plain unsigned Release file. +func GenerateRelease(suite, codename string, components, architectures []string, entries []ReleaseEntry) []byte { + var buf bytes.Buffer + + fmt.Fprintf(&buf, "Origin: ClonePack Mirror\n") + fmt.Fprintf(&buf, "Suite: %s\n", suite) + fmt.Fprintf(&buf, "Codename: %s\n", codename) + fmt.Fprintf(&buf, "Components: %s\n", strings.Join(components, " ")) + fmt.Fprintf(&buf, "Architectures: %s\n", strings.Join(architectures, " ")) + fmt.Fprintf(&buf, "Date: %s\n", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 UTC")) + fmt.Fprintf(&buf, "SHA256:\n") + + for _, e := range entries { + fmt.Fprintf(&buf, " %s %d %s\n", e.Hash, e.Size, e.Path) + } + + return buf.Bytes() +} diff --git a/internal/clone/apt/scanner.go b/internal/clone/apt/scanner.go new file mode 100644 index 0000000..faff989 --- /dev/null +++ b/internal/clone/apt/scanner.go @@ -0,0 +1,145 @@ +package apt + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +// NewPackage is a package present upstream but absent from the local mirror. +type NewPackage struct { + Package string + Version string + Architecture string + Filename string // relative path, e.g. pool/main/c/curl/curl_7.88_amd64.deb + SHA256 string + Size int64 + Component string +} + +// Scanner compares an upstream APT repository against a local mirror and +// returns packages that are present upstream but missing on disk. +type Scanner struct { + SourceURL string + LocalDir string + Config Config + HTTPClient *http.Client +} + +// NewScanner creates a Scanner with a 5-minute HTTP timeout. +func NewScanner(sourceURL, localDir string, cfg Config) *Scanner { + return &Scanner{ + SourceURL: strings.TrimRight(sourceURL, "/"), + LocalDir: localDir, + Config: cfg, + HTTPClient: &http.Client{Timeout: 5 * time.Minute}, + } +} + +// Scan fetches the upstream package lists and returns packages not on disk. +func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) { + suite := sc.Config.Suite + + // 1. Fetch InRelease. + inReleaseURL := sc.SourceURL + "/dists/" + suite + "/InRelease" + inReleaseData, err := fetchBytes(ctx, sc.HTTPClient, inReleaseURL) + if err != nil { + return nil, fmt.Errorf("fetch InRelease: %w", err) + } + + rf, err := ParseRelease(inReleaseData) + if err != nil { + return nil, fmt.Errorf("parse InRelease: %w", err) + } + + // 2 & 3. Fetch Packages.gz for each component/arch, save it, then filter. + var missing []NewPackage + + for _, component := range sc.Config.Components { + for _, arch := range sc.Config.Architectures { + if err := ctx.Err(); err != nil { + return nil, err + } + pkgs, err := sc.fetchAndSavePackages(ctx, suite, component, arch, rf) + if err != nil { + return nil, fmt.Errorf("fetch packages for %s/%s: %w", component, arch, err) + } + + // 4. Filter: only return packages whose Filename is NOT on disk. + checker := diskPresenceChecker(sc.LocalDir) + for _, pkg := range pkgs { + if !checker(pkg.Filename) { + missing = append(missing, NewPackage{ + Package: pkg.Package, + Version: pkg.Version, + Architecture: pkg.Architecture, + Filename: pkg.Filename, + SHA256: pkg.SHA256, + Size: pkg.Size, + Component: component, + }) + } + } + } + } + + return missing, nil +} + +// fetchAndSavePackages fetches Packages.gz for a component/arch, saves it as +// .upstream-Packages.gz (so RegenerateMetadata can use it), and returns parsed packages. +func (sc *Scanner) fetchAndSavePackages(ctx context.Context, suite, component, arch string, rf *ReleaseFile) ([]DebPackage, error) { + wantPath := component + "/binary-" + arch + "/Packages.gz" + + // Check that this path is listed in the upstream SHA256 section. + var found bool + for _, e := range rf.SHA256 { + if e.Path == wantPath { + found = true + break + } + } + if !found { + return nil, nil + } + + pkgGzURL := sc.SourceURL + "/dists/" + suite + "/" + wantPath + data, err := fetchBytes(ctx, sc.HTTPClient, pkgGzURL) + if err != nil { + return nil, fmt.Errorf("fetch %s: %w", pkgGzURL, err) + } + + // Save as .upstream-Packages.gz. + destDir := filepath.Join(sc.LocalDir, "dists", suite, component, "binary-"+arch) + if err := os.MkdirAll(destDir, 0o755); err != nil { + return nil, err + } + upstreamPath := filepath.Join(destDir, upstreamPackagesGz) + if err := atomicWrite(upstreamPath, data); err != nil { + return nil, fmt.Errorf("save %s: %w", upstreamPath, err) + } + + // Decompress and parse. + gz, err := gzip.NewReader(strings.NewReader(string(data))) + if err != nil { + return nil, fmt.Errorf("open gzip: %w", err) + } + defer gz.Close() + + raw, err := io.ReadAll(gz) + if err != nil { + return nil, fmt.Errorf("decompress: %w", err) + } + + pkgs, err := ParsePackages(raw) + if err != nil { + return nil, fmt.Errorf("parse Packages: %w", err) + } + return pkgs, nil +} diff --git a/internal/core/clone.go b/internal/core/clone.go index 420d4e0..1901392 100644 --- a/internal/core/clone.go +++ b/internal/core/clone.go @@ -2,11 +2,13 @@ package core import ( "context" + "encoding/json" "errors" "fmt" "log" "path/filepath" + aptclone "github.com/syonad/clonepack/internal/clone/apt" "github.com/syonad/clonepack/internal/clone/rpm" "github.com/syonad/clonepack/internal/store" ) @@ -28,8 +30,8 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err if err != nil { return 0, err } - if repo.Type != "rpm" { - return 0, fmt.Errorf("clone is only supported for rpm repositories") + if repo.Type != "rpm" && repo.Type != "apt" { + return 0, fmt.Errorf("clone is only supported for rpm and apt repositories") } running, err := s.jobStore.HasRunningCloneJob(ctx, repoID) @@ -45,7 +47,7 @@ func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, err return 0, err } - go s.runClone(repoID, jobID, repo.SourceURL, repo.Type) + go s.runClone(repoID, jobID, repo) return jobID, nil } @@ -53,7 +55,7 @@ func (s *CloneService) GetLatestCloneJob(ctx context.Context, repoID int64) (*st return s.jobStore.GetLatestCloneJob(ctx, repoID) } -func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string) { +func (s *CloneService) runClone(repoID, jobID int64, repo *store.Repo) { ctx := context.Background() if err := s.jobStore.MarkCloneJobStarted(ctx, jobID); err != nil { @@ -61,23 +63,41 @@ func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string) return } - destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repoType) - cloner := rpm.New(sourceURL, destDir) - cloner.OnProgress = func(p rpm.Progress) { - log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone) + destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type) + var cloneErr error + + switch repo.Type { + case "rpm": + cloner := rpm.New(repo.SourceURL, destDir) + cloner.OnProgress = func(p rpm.Progress) { + log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone) + } + if err := cloner.Clone(ctx); err != nil { + cloneErr = err + } else if err := rpm.RegenerateMetadata(destDir); err != nil { + log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err) + } + + case "apt": + var cfg aptclone.Config + if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil { + cloneErr = fmt.Errorf("parse apt config: %w", err) + break + } + cloner := aptclone.New(repo.SourceURL, destDir, cfg) + cloner.OnProgress = func(file string, bytes int64) { + log.Printf("clone job %d: %s (%d bytes)", jobID, file, bytes) + } + cloneErr = cloner.Clone(ctx) } - if err := cloner.Clone(ctx); err != nil { - errMsg := err.Error() + if cloneErr != nil { + errMsg := cloneErr.Error() _ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobFailed, &errMsg) - log.Printf("clone job %d: failed: %v", jobID, err) + log.Printf("clone job %d: failed: %v", jobID, cloneErr) return } - if err := rpm.RegenerateMetadata(destDir); err != nil { - log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err) - } - _ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobCompleted, nil) log.Printf("clone job %d: completed", jobID) } diff --git a/internal/core/repo.go b/internal/core/repo.go index cfd5f42..561ade9 100644 --- a/internal/core/repo.go +++ b/internal/core/repo.go @@ -2,10 +2,12 @@ package core import ( "context" + "encoding/json" "fmt" "log" "path/filepath" + aptclone "github.com/syonad/clonepack/internal/clone/apt" rpmclone "github.com/syonad/clonepack/internal/clone/rpm" "github.com/syonad/clonepack/internal/store" ) @@ -24,10 +26,13 @@ func NewRepoService(s store.RepoStore, dataDir string) *RepoService { var validSyncModes = map[string]bool{"auto": true, "manual": true} type CreateRepoInput struct { - Name string `json:"name"` - Type string `json:"type"` - SourceURL string `json:"source_url"` - SyncMode string `json:"sync_mode"` + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + SyncMode string `json:"sync_mode"` + AptSuite string `json:"apt_suite,omitempty"` + AptComponents []string `json:"apt_components,omitempty"` + AptArchitectures []string `json:"apt_architectures,omitempty"` } func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Repo, error) { @@ -46,7 +51,30 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", in.SyncMode) } - r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode} + var configJSON string + if in.Type == "apt" { + if in.AptSuite == "" { + return nil, fmt.Errorf("apt_suite is required for apt repositories") + } + if len(in.AptComponents) == 0 { + in.AptComponents = []string{"main"} + } + if len(in.AptArchitectures) == 0 { + in.AptArchitectures = []string{"amd64"} + } + cfg := aptclone.Config{ + Suite: in.AptSuite, + Components: in.AptComponents, + Architectures: in.AptArchitectures, + } + b, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("marshal apt config: %w", err) + } + configJSON = string(b) + } + + r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode, Config: configJSON} id, err := s.store.CreateRepo(ctx, r) if err != nil { return nil, err @@ -60,15 +88,85 @@ func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Re } func (s *RepoService) initStorage(repo *store.Repo) { + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), repo.Type) switch repo.Type { case "rpm": - localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), "rpm") if err := rpmclone.InitEmptyRepo(localDir); err != nil { log.Printf("init storage for repo %d: %v", repo.ID, err) } + case "apt": + var cfg aptclone.Config + if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil { + log.Printf("init storage for repo %d: parse config: %v", repo.ID, err) + return + } + if err := aptclone.InitEmptyRepo(localDir, cfg); err != nil { + log.Printf("init storage for repo %d: %v", repo.ID, err) + } } } +type UpdateRepoInput struct { + SourceURL *string + SyncMode *string + AptSuite *string + AptComponents []string + AptArchitectures []string +} + +func (s *RepoService) Update(ctx context.Context, id int64, in UpdateRepoInput) (*store.Repo, error) { + repo, err := s.store.GetRepo(ctx, id) + if err != nil { + return nil, err + } + + if in.SourceURL != nil { + if *in.SourceURL == "" { + return nil, fmt.Errorf("source_url cannot be empty") + } + repo.SourceURL = *in.SourceURL + } + if in.SyncMode != nil { + if !validSyncModes[*in.SyncMode] { + return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", *in.SyncMode) + } + repo.SyncMode = *in.SyncMode + } + + if repo.Type == "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) { + var cfg aptclone.Config + if repo.Config != "" { + if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil { + return nil, fmt.Errorf("parse existing apt config: %w", err) + } + } + if in.AptSuite != nil { + if *in.AptSuite == "" { + return nil, fmt.Errorf("apt_suite cannot be empty") + } + cfg.Suite = *in.AptSuite + } + if in.AptComponents != nil { + cfg.Components = in.AptComponents + } + if in.AptArchitectures != nil { + cfg.Architectures = in.AptArchitectures + } + b, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("marshal apt config: %w", err) + } + repo.Config = string(b) + } else if repo.Type != "apt" && (in.AptSuite != nil || in.AptComponents != nil || in.AptArchitectures != nil) { + return nil, fmt.Errorf("apt fields only apply to apt repositories") + } + + if err := s.store.UpdateRepo(ctx, id, repo.SourceURL, repo.SyncMode, repo.Config); err != nil { + return nil, err + } + return s.store.GetRepo(ctx, id) +} + func (s *RepoService) List(ctx context.Context) ([]store.Repo, error) { return s.store.ListRepos(ctx) } @@ -77,6 +175,10 @@ func (s *RepoService) Get(ctx context.Context, id int64) (*store.Repo, error) { return s.store.GetRepo(ctx, id) } +func (s *RepoService) GetByName(ctx context.Context, name string) (*store.Repo, error) { + return s.store.GetRepoByName(ctx, name) +} + func (s *RepoService) Delete(ctx context.Context, id int64) error { return s.store.DeleteRepo(ctx, id) } diff --git a/internal/core/snapshot.go b/internal/core/snapshot.go index 15c515f..1bf743a 100644 --- a/internal/core/snapshot.go +++ b/internal/core/snapshot.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "context" + "encoding/json" "encoding/xml" "errors" "fmt" @@ -13,6 +14,7 @@ import ( "strings" "time" + aptclone "github.com/syonad/clonepack/internal/clone/apt" rpmclone "github.com/syonad/clonepack/internal/clone/rpm" "github.com/syonad/clonepack/internal/store" ) @@ -46,60 +48,24 @@ type SnapshotDiff struct { } func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label string) (int64, error) { - if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + repo, err := s.repoStore.GetRepo(ctx, repoID) + if err != nil { return 0, err } - localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type) - repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml")) + var pkgs []store.SnapshotPackage + switch repo.Type { + case "rpm": + pkgs, err = s.snapshotPackagesRPM(localDir) + case "apt": + pkgs, err = s.snapshotPackagesAPT(localDir, repo.Config) + default: + return 0, fmt.Errorf("snapshots not supported for repo type %q", repo.Type) + } if err != nil { - return 0, fmt.Errorf("read repomd.xml: %w", err) - } - - var repomd rpmclone.RepoMD - if err := xml.Unmarshal(repomdData, &repomd); err != nil { - return 0, fmt.Errorf("parse repomd.xml: %w", err) - } - - var primaryHref string - for _, entry := range repomd.Data { - if entry.Type == "primary" { - primaryHref = entry.Location.Href - break - } - } - if primaryHref == "" { - return 0, fmt.Errorf("no primary entry in repomd.xml") - } - - gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref))) - if err != nil { - return 0, fmt.Errorf("read primary.xml.gz: %w", err) - } - - gz, err := gzip.NewReader(bytes.NewReader(gzData)) - if err != nil { - return 0, fmt.Errorf("open gzip: %w", err) - } - defer gz.Close() - - var primary rpmclone.PrimaryMetadata - if err := xml.NewDecoder(gz).Decode(&primary); err != nil { - return 0, fmt.Errorf("parse primary.xml: %w", err) - } - - pkgs := make([]store.SnapshotPackage, len(primary.Packages)) - for i, p := range primary.Packages { - pkgs[i] = store.SnapshotPackage{ - Name: p.Name, - Version: p.Version.Ver + "-" + p.Version.Rel, - Arch: p.Arch, - Location: p.Location.Href, - Checksum: p.Checksum.Value, - ChecksumType: p.Checksum.Type, - Size: p.Size.Package, - } + return 0, err } snapID, err := s.snapshotStore.CreateSnapshot(ctx, repoID, label) @@ -114,6 +80,94 @@ func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label return snapID, nil } +func (s *SnapshotService) snapshotPackagesRPM(localDir string) ([]store.SnapshotPackage, error) { + repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml")) + if err != nil { + return nil, fmt.Errorf("read repomd.xml: %w", err) + } + var repomd rpmclone.RepoMD + if err := xml.Unmarshal(repomdData, &repomd); err != nil { + return nil, fmt.Errorf("parse repomd.xml: %w", err) + } + var primaryHref string + for _, entry := range repomd.Data { + if entry.Type == "primary" { + primaryHref = entry.Location.Href + break + } + } + if primaryHref == "" { + return nil, fmt.Errorf("no primary entry in repomd.xml") + } + gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref))) + if err != nil { + return nil, fmt.Errorf("read primary.xml.gz: %w", err) + } + gz, err := gzip.NewReader(bytes.NewReader(gzData)) + if err != nil { + return nil, fmt.Errorf("open gzip: %w", err) + } + defer gz.Close() + var primary rpmclone.PrimaryMetadata + if err := xml.NewDecoder(gz).Decode(&primary); err != nil { + return nil, fmt.Errorf("parse primary.xml: %w", err) + } + pkgs := make([]store.SnapshotPackage, len(primary.Packages)) + for i, p := range primary.Packages { + pkgs[i] = store.SnapshotPackage{ + Name: p.Name, + Version: p.Version.Ver + "-" + p.Version.Rel, + Arch: p.Arch, + Location: p.Location.Href, + Checksum: p.Checksum.Value, + ChecksumType: p.Checksum.Type, + Size: p.Size.Package, + } + } + return pkgs, nil +} + +func (s *SnapshotService) snapshotPackagesAPT(localDir, configJSON string) ([]store.SnapshotPackage, error) { + var cfg aptclone.Config + if err := json.Unmarshal([]byte(configJSON), &cfg); err != nil { + return nil, fmt.Errorf("parse apt config: %w", err) + } + + seen := make(map[string]struct{}) + var pkgs []store.SnapshotPackage + + for _, component := range cfg.Components { + for _, arch := range cfg.Architectures { + pkgsPath := filepath.Join(localDir, "dists", cfg.Suite, component, "binary-"+arch, "Packages") + data, err := os.ReadFile(pkgsPath) + if err != nil { + // Not yet populated — skip silently. + continue + } + debs, err := aptclone.ParsePackages(data) + if err != nil { + return nil, fmt.Errorf("parse Packages for %s/%s: %w", component, arch, err) + } + for _, d := range debs { + if _, dup := seen[d.Filename]; dup { + continue + } + seen[d.Filename] = struct{}{} + pkgs = append(pkgs, store.SnapshotPackage{ + Name: d.Package, + Version: d.Version, + Arch: d.Architecture, + Location: d.Filename, + Checksum: d.SHA256, + ChecksumType: "sha256", + Size: d.Size, + }) + } + } + } + return pkgs, nil +} + func (s *SnapshotService) List(ctx context.Context, repoID int64) ([]store.Snapshot, error) { if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { return nil, err @@ -212,16 +266,18 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64 return err } - localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type) snapSet := make(map[string]store.SnapshotPackage, len(pkgs)) for _, p := range pkgs { snapSet[p.Location] = p } + ext := packageExt(repo.Type) var errs []error + if err := filepath.WalkDir(localDir, func(path string, d os.DirEntry, err error) error { - if err != nil || d.IsDir() || filepath.Ext(path) != ".rpm" { + if err != nil || d.IsDir() || filepath.Ext(path) != ext { return err } rel, _ := filepath.Rel(localDir, path) @@ -251,10 +307,28 @@ func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64 return err } - if repo.Type == "rpm" { + switch repo.Type { + case "rpm": if err := rpmclone.RegenerateMetadata(localDir); err != nil { return fmt.Errorf("regenerate metadata: %w", err) } + case "apt": + var cfg aptclone.Config + if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil { + return fmt.Errorf("parse apt config: %w", err) + } + if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil { + return fmt.Errorf("regenerate metadata: %w", err) + } } return nil } + +func packageExt(repoType string) string { + switch repoType { + case "apt": + return ".deb" + default: + return ".rpm" + } +} diff --git a/internal/core/sync.go b/internal/core/sync.go index ddd3c54..0167371 100644 --- a/internal/core/sync.go +++ b/internal/core/sync.go @@ -2,6 +2,7 @@ package core import ( "context" + "encoding/json" "errors" "fmt" "log" @@ -11,6 +12,7 @@ import ( "strings" "time" + aptclone "github.com/syonad/clonepack/internal/clone/apt" rpmclone "github.com/syonad/clonepack/internal/clone/rpm" "github.com/syonad/clonepack/internal/store" ) @@ -54,18 +56,54 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error { if err != nil { return err } - if repo.Type != "rpm" { - return fmt.Errorf("sync only supported for rpm repos") + + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type) + + type candidate struct { + Name string + Version string + Arch string + Location string + Checksum string + ChecksumType string + Size int64 + } + var candidates []candidate + + switch repo.Type { + case "rpm": + scanner := rpmclone.NewScanner(repo.SourceURL, localDir) + pkgs, err := scanner.Scan(ctx) + if err != nil { + return fmt.Errorf("scan repo %d: %w", repoID, err) + } + for _, p := range pkgs { + candidates = append(candidates, candidate{ + Name: p.Name, Version: p.Version, Arch: p.Arch, + Location: p.Location, Checksum: p.Checksum, ChecksumType: p.ChecksumType, Size: p.Size, + }) + } + case "apt": + var cfg aptclone.Config + if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil { + return fmt.Errorf("parse apt config for repo %d: %w", repoID, err) + } + scanner := aptclone.NewScanner(repo.SourceURL, localDir, cfg) + pkgs, err := scanner.Scan(ctx) + if err != nil { + return fmt.Errorf("scan repo %d: %w", repoID, err) + } + for _, p := range pkgs { + candidates = append(candidates, candidate{ + Name: p.Package, Version: p.Version, Arch: p.Architecture, + Location: p.Filename, Checksum: p.SHA256, ChecksumType: "sha256", Size: p.Size, + }) + } + default: + return fmt.Errorf("sync only supported for rpm and apt repos") } - localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") - scanner := rpmclone.NewScanner(repo.SourceURL, localDir) - newPkgs, err := scanner.Scan(ctx) - if err != nil { - return fmt.Errorf("scan repo %d: %w", repoID, err) - } - - if len(newPkgs) == 0 { + if len(candidates) == 0 { log.Printf("sync: repo %d is up to date", repoID) return nil } @@ -82,19 +120,19 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error { blockedNames[b.Name] = true } } - filtered := newPkgs[:0] - for _, p := range newPkgs { + filtered := candidates[:0] + for _, p := range candidates { if !blockedLocations[p.Location] && !blockedNames[p.Name] { filtered = append(filtered, p) } } - newPkgs = filtered + candidates = filtered - if len(newPkgs) == 0 { + if len(candidates) == 0 { log.Printf("sync: repo %d is up to date (all new packages are blocked)", repoID) return nil } - log.Printf("sync: repo %d has %d new package(s)", repoID, len(newPkgs)) + log.Printf("sync: repo %d has %d new package(s)", repoID, len(candidates)) switch repo.SyncMode { case "auto": @@ -103,8 +141,8 @@ func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error { return fmt.Errorf("start clone for repo %d: %w", repoID, err) } case "manual": - pending := make([]store.PendingPackage, len(newPkgs)) - for i, p := range newPkgs { + pending := make([]store.PendingPackage, len(candidates)) + for i, p := range candidates { pending[i] = store.PendingPackage{ RepoID: repoID, Name: p.Name, @@ -148,7 +186,7 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in wanted[id] = true } - localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repo.Type) var downloadErrors []error var anyApproved bool for _, pkg := range all { @@ -173,14 +211,22 @@ func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []in } if anyApproved { - if repo.Type == "rpm" { + switch repo.Type { + case "rpm": if err := rpmclone.RegenerateMetadata(localDir); err != nil { log.Printf("metadata regeneration for repo %d failed: %v", repoID, err) } - } - label := "auto-" + time.Now().UTC().Format(time.RFC3339) - if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil { - log.Printf("auto-snapshot for repo %d failed: %v", repoID, err) + label := "auto-" + time.Now().UTC().Format(time.RFC3339) + if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil { + log.Printf("auto-snapshot for repo %d failed: %v", repoID, err) + } + case "apt": + var cfg aptclone.Config + if err := json.Unmarshal([]byte(repo.Config), &cfg); err != nil { + log.Printf("parse apt config for repo %d: %v", repoID, err) + } else if err := aptclone.RegenerateMetadata(localDir, cfg); err != nil { + log.Printf("metadata regeneration for repo %d failed: %v", repoID, err) + } } } diff --git a/internal/store/db.go b/internal/store/db.go index cb504f6..d9c6a20 100644 --- a/internal/store/db.go +++ b/internal/store/db.go @@ -27,6 +27,9 @@ var migration005 string //go:embed migrations/000006_blocked_packages_name.up.sql var migration006 string +//go:embed migrations/000007_repo_config.up.sql +var migration007 string + func Open(cfg config.DBConfig) (*sql.DB, error) { db, err := sql.Open("sqlite", cfg.Path) if err != nil { @@ -66,6 +69,7 @@ func runMigrations(db *sql.DB) error { {4, migration004}, {5, migration005}, {6, migration006}, + {7, migration007}, } for _, m := range migrations { diff --git a/internal/store/migrations/000007_repo_config.down.sql b/internal/store/migrations/000007_repo_config.down.sql new file mode 100644 index 0000000..c2371c5 --- /dev/null +++ b/internal/store/migrations/000007_repo_config.down.sql @@ -0,0 +1,4 @@ +-- SQLite ne supporte pas DROP COLUMN avant 3.35 +CREATE TABLE repos_backup AS SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos; +DROP TABLE repos; +ALTER TABLE repos_backup RENAME TO repos; diff --git a/internal/store/migrations/000007_repo_config.up.sql b/internal/store/migrations/000007_repo_config.up.sql new file mode 100644 index 0000000..b8fd3c8 --- /dev/null +++ b/internal/store/migrations/000007_repo_config.up.sql @@ -0,0 +1 @@ +ALTER TABLE repos ADD COLUMN config TEXT NOT NULL DEFAULT ''; diff --git a/internal/store/repo.go b/internal/store/repo.go index 952d0db..f118f60 100644 --- a/internal/store/repo.go +++ b/internal/store/repo.go @@ -16,6 +16,7 @@ type Repo struct { SourceURL string `db:"source_url"` Frozen bool `db:"frozen"` SyncMode string `db:"sync_mode"` + Config string `db:"config"` CreatedAt time.Time `db:"created_at"` } @@ -23,8 +24,10 @@ type RepoStore interface { CreateRepo(ctx context.Context, r *Repo) (int64, error) ListRepos(ctx context.Context) ([]Repo, error) GetRepo(ctx context.Context, id int64) (*Repo, error) + GetRepoByName(ctx context.Context, name string) (*Repo, error) DeleteRepo(ctx context.Context, id int64) error UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error + UpdateRepo(ctx context.Context, id int64, sourceURL, syncMode, config string) error } type SQLiteRepoStore struct { @@ -37,8 +40,8 @@ func NewRepoStore(db *sql.DB) *SQLiteRepoStore { func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error) { res, err := s.db.ExecContext(ctx, - `INSERT INTO repos (name, type, source_url, frozen, sync_mode) VALUES (?, ?, ?, ?, ?)`, - r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, + `INSERT INTO repos (name, type, source_url, frozen, sync_mode, config) VALUES (?, ?, ?, ?, ?, ?)`, + r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, r.Config, ) if err != nil { return 0, err @@ -48,7 +51,7 @@ func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) { rows, err := s.db.QueryContext(ctx, - `SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos ORDER BY created_at DESC`) + `SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos ORDER BY created_at DESC`) if err != nil { return nil, err } @@ -57,7 +60,7 @@ func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) { var repos []Repo for rows.Next() { var r Repo - if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt); err != nil { + if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt); err != nil { return nil, err } repos = append(repos, r) @@ -68,8 +71,22 @@ func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) { func (s *SQLiteRepoStore) GetRepo(ctx context.Context, id int64) (*Repo, error) { var r Repo err := s.db.QueryRowContext(ctx, - `SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos WHERE id = ?`, id, - ).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt) + `SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos WHERE id = ?`, id, + ).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + return &r, nil +} + +func (s *SQLiteRepoStore) GetRepoByName(ctx context.Context, name string) (*Repo, error) { + var r Repo + err := s.db.QueryRowContext(ctx, + `SELECT id, name, type, source_url, frozen, sync_mode, config, created_at FROM repos WHERE name = ?`, name, + ).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.Config, &r.CreatedAt) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } @@ -94,6 +111,21 @@ func (s *SQLiteRepoStore) DeleteRepo(ctx context.Context, id int64) error { return nil } +func (s *SQLiteRepoStore) UpdateRepo(ctx context.Context, id int64, sourceURL, syncMode, config string) error { + res, err := s.db.ExecContext(ctx, + `UPDATE repos SET source_url = ?, sync_mode = ?, config = ? WHERE id = ?`, + sourceURL, syncMode, config, id, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return ErrNotFound + } + return nil +} + func (s *SQLiteRepoStore) UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error { res, err := s.db.ExecContext(ctx, `UPDATE repos SET sync_mode = ? WHERE id = ?`, mode, id) if err != nil {