From 274ea454ddbac09489188acb3dc713099f5ca6dd Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sat, 25 Apr 2026 14:49:14 +0200 Subject: [PATCH] first with full handle over rpm Signed-off-by: GnomeZworc --- Makefile | 44 ++ README.md | 228 ++++++++ client/client.go | 388 ++++++++++++++ cmd/clonepack/main.go | 9 + cmd/clonepack/repo.go | 498 ++++++++++++++++++ cmd/clonepack/root.go | 20 + cmd/clonepack/serve.go | 60 +++ config/config.go | 31 ++ config/loader.go | 41 ++ go.mod | 33 ++ go.sum | 59 +++ internal/api/clone_handler.go | 83 +++ internal/api/proxy_handler.go | 43 ++ internal/api/repo_handler.go | 102 ++++ internal/api/respond.go | 16 + internal/api/router.go | 50 ++ internal/api/snapshot_handler.go | 196 +++++++ internal/api/snapshot_types.go | 43 ++ internal/api/sync_handler.go | 191 +++++++ internal/api/sync_types.go | 38 ++ internal/api/types.go | 44 ++ internal/clone/rpm/cloner.go | 195 +++++++ internal/clone/rpm/download.go | 71 +++ internal/clone/rpm/metadata_gen.go | 212 ++++++++ internal/clone/rpm/primary.go | 41 ++ internal/clone/rpm/repomd.go | 26 + internal/clone/rpm/scanner.go | 143 +++++ internal/core/clone.go | 83 +++ internal/core/errors.go | 5 + internal/core/repo.go | 82 +++ internal/core/snapshot.go | 260 +++++++++ internal/core/sync.go | 268 ++++++++++ internal/store/blocked_package.go | 78 +++ internal/store/clone_job.go | 109 ++++ internal/store/db.go | 88 ++++ .../store/migrations/000001_init.down.sql | 3 + internal/store/migrations/000001_init.up.sql | 25 + .../migrations/000002_clone_jobs.down.sql | 1 + .../store/migrations/000002_clone_jobs.up.sql | 11 + .../store/migrations/000003_sync.down.sql | 1 + internal/store/migrations/000003_sync.up.sql | 18 + .../000004_snapshot_packages.down.sql | 2 + .../000004_snapshot_packages.up.sql | 14 + .../000005_blocked_packages.down.sql | 1 + .../migrations/000005_blocked_packages.up.sql | 7 + .../000006_blocked_packages_name.down.sql | 4 + .../000006_blocked_packages_name.up.sql | 1 + internal/store/pending_package.go | 92 ++++ internal/store/repo.go | 107 ++++ internal/store/snapshot.go | 144 +++++ 50 files changed, 4309 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 client/client.go create mode 100644 cmd/clonepack/main.go create mode 100644 cmd/clonepack/repo.go create mode 100644 cmd/clonepack/root.go create mode 100644 cmd/clonepack/serve.go create mode 100644 config/config.go create mode 100644 config/loader.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/api/clone_handler.go create mode 100644 internal/api/proxy_handler.go create mode 100644 internal/api/repo_handler.go create mode 100644 internal/api/respond.go create mode 100644 internal/api/router.go create mode 100644 internal/api/snapshot_handler.go create mode 100644 internal/api/snapshot_types.go create mode 100644 internal/api/sync_handler.go create mode 100644 internal/api/sync_types.go create mode 100644 internal/api/types.go create mode 100644 internal/clone/rpm/cloner.go create mode 100644 internal/clone/rpm/download.go create mode 100644 internal/clone/rpm/metadata_gen.go create mode 100644 internal/clone/rpm/primary.go create mode 100644 internal/clone/rpm/repomd.go create mode 100644 internal/clone/rpm/scanner.go create mode 100644 internal/core/clone.go create mode 100644 internal/core/errors.go create mode 100644 internal/core/repo.go create mode 100644 internal/core/snapshot.go create mode 100644 internal/core/sync.go create mode 100644 internal/store/blocked_package.go create mode 100644 internal/store/clone_job.go create mode 100644 internal/store/db.go create mode 100644 internal/store/migrations/000001_init.down.sql create mode 100644 internal/store/migrations/000001_init.up.sql create mode 100644 internal/store/migrations/000002_clone_jobs.down.sql create mode 100644 internal/store/migrations/000002_clone_jobs.up.sql create mode 100644 internal/store/migrations/000003_sync.down.sql create mode 100644 internal/store/migrations/000003_sync.up.sql create mode 100644 internal/store/migrations/000004_snapshot_packages.down.sql create mode 100644 internal/store/migrations/000004_snapshot_packages.up.sql create mode 100644 internal/store/migrations/000005_blocked_packages.down.sql create mode 100644 internal/store/migrations/000005_blocked_packages.up.sql create mode 100644 internal/store/migrations/000006_blocked_packages_name.down.sql create mode 100644 internal/store/migrations/000006_blocked_packages_name.up.sql create mode 100644 internal/store/pending_package.go create mode 100644 internal/store/repo.go create mode 100644 internal/store/snapshot.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..26955ba --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +BINARY := clonepack +BUILD_DIR := bin +CMD := ./cmd/clonepack + +.PHONY: all build run test lint clean tidy help + +all: build + +## build: compile le binaire dans bin/clonepack +build: + @mkdir -p $(BUILD_DIR) + go build -o $(BUILD_DIR)/$(BINARY) $(CMD) + +## run: démarre le serveur +run: build + ./$(BUILD_DIR)/$(BINARY) serve + +## test: lance tous les tests +test: + go test ./... + +## test-v: tests avec sortie verbeuse +test-v: + go test -v ./... + +## lint: vérifie le code (nécessite golangci-lint) +lint: + golangci-lint run ./... + +## vet: analyse statique Go +vet: + go vet ./... + +## tidy: met à jour go.mod et go.sum +tidy: + go mod tidy + +## clean: supprime les artefacts de build +clean: + rm -rf $(BUILD_DIR) + +## help: affiche cette aide +help: + @grep -E '^## ' Makefile | sed 's/## / /' diff --git a/README.md b/README.md new file mode 100644 index 0000000..8ef6d9b --- /dev/null +++ b/README.md @@ -0,0 +1,228 @@ +# ClonePack + +ClonePack est un système de clonage et de gel d'artefacts. Il permet de synchroniser des dépôts externes, de les figer à un instant donné, et de les exposer localement via un miroir HTTP. + +## Types d'artefacts supportés + +- Enterprise Linux (RPM/YUM/DNF) +- Debian (APT) — *à venir* +- Docker images — *à venir* +- Binaires — *à venir* + +## Fonctionnalités + +- Clonage de dépôts externes avec vérification SHA256 +- Scan régulier des sources pour détecter les nouveaux paquets +- Validation manuelle ou automatique des mises à jour +- Snapshots métadonnées avec diff et rollback +- Proxy miroir HTTP (consommable directement par `yum`/`apt`) + +--- + +## Compilation + +**Prérequis** : Go 1.25+ + +```bash +make build # compile → bin/clonepack +make run # compile + démarre le serveur +make test # lance les tests +make vet # analyse statique +make clean # supprime bin/ +``` + +Le binaire est généré dans `bin/clonepack`. + +--- + +## Configuration + +ClonePack se configure via fichier YAML ou variables d'environnement (préfixe `CLONEPACK_`). + +**Fichier de config (optionnel) :** +```yaml +server: + host: 0.0.0.0 + port: 8080 + +db: + path: ./clonepack.db + +data_dir: ./data + +sync: + interval: 1h +``` + +**Variables d'environnement :** +```bash +CLONEPACK_SERVER_PORT=9090 +CLONEPACK_DB_PATH=/var/lib/clonepack/clonepack.db +CLONEPACK_DATA_DIR=/var/lib/clonepack/data +CLONEPACK_SYNC_INTERVAL=30m +``` + +**Démarrer le serveur :** +```bash +./bin/clonepack serve +./bin/clonepack serve --config /etc/clonepack/config.yaml +``` + +Par défaut le serveur écoute sur `http://0.0.0.0:8080`. + +--- + +## CLI + +Toutes les commandes CLI appellent l'API REST. L'URL du serveur se configure avec `--api-url` (défaut : `http://localhost:8080`). + +### Dépôts + +```bash +# Créer un dépôt RPM (sync auto) +./bin/clonepack repo create \ + --name rocky9-baseos \ + --type rpm \ + --source-url https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os + +# Créer avec validation manuelle +./bin/clonepack repo create \ + --name rocky9-baseos \ + --type rpm \ + --source-url https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os \ + --sync-mode manual + +# Lister les dépôts +./bin/clonepack repo list + +# Détail d'un dépôt +./bin/clonepack repo get 1 + +# Supprimer un dépôt +./bin/clonepack repo delete 1 +``` + +### Clonage + +```bash +# Lancer le clonage complet d'un dépôt (télécharge tous les paquets) +./bin/clonepack repo clone 1 +``` + +Le clonage est asynchrone : la commande affiche la progression du job jusqu'à completion. + +### Synchronisation + +```bash +# Déclencher un scan pour détecter les nouveaux paquets +./bin/clonepack repo sync 1 + +# Lister les paquets en attente de validation (mode manual) +./bin/clonepack repo sync-list 1 + +# Approuver des paquets spécifiques (téléchargement en arrière-plan) +./bin/clonepack repo sync-approve 1 42 43 44 + +# Approuver tous les paquets en attente +./bin/clonepack repo sync-approve 1 --all + +# Rejeter des paquets (ils réapparaîtront au prochain scan) +./bin/clonepack repo sync-reject 1 42 +``` + +Le scheduler tourne automatiquement selon l'intervalle configuré (`sync.interval`). En mode `auto`, les nouveaux paquets sont téléchargés directement. En mode `manual`, ils s'accumulent dans la liste d'attente. + +### Blocklist + +Les paquets bloqués n'apparaissent plus jamais dans la liste pending, quelle que soit leur version. + +```bash +# Bloquer définitivement un paquet (par ID de la liste pending) +# → bloque la version exacte ET toutes les futures versions du même paquet +./bin/clonepack repo sync-block 1 42 43 + +# Voir les paquets bloqués +./bin/clonepack repo sync-blocklist 1 + +# Débloquer (par ID de la liste blocked) +./bin/clonepack repo sync-unblock 1 1 2 +``` + +### Snapshots + +```bash +# Créer un snapshot manuellement +./bin/clonepack repo snapshot create 1 +./bin/clonepack repo snapshot create 1 --label "avant-mise-a-jour" + +# Lister les snapshots d'un dépôt +./bin/clonepack repo snapshot list 1 + +# Détail d'un snapshot (métadonnées + liste des paquets) +./bin/clonepack repo snapshot show 1 3 + +# Comparer deux snapshots +./bin/clonepack repo snapshot diff 1 2 3 + +# Rollback vers un snapshot (supprime les paquets intrus, re-télécharge les manquants) +./bin/clonepack repo snapshot rollback 1 2 + +# Supprimer un snapshot +./bin/clonepack repo snapshot delete 1 3 +``` + +Un snapshot automatique est créé après chaque `sync-approve`. + +--- + +## Proxy miroir + +ClonePack expose chaque dépôt cloné comme un miroir HTTP à l'adresse : + +``` +http://:/mirror// +``` + +**Configuration YUM/DNF** (`/etc/yum.repos.d/clonepack.repo`) : +```ini +[rocky9-baseos] +name=Rocky Linux 9 BaseOS via ClonePack +baseurl=http://clonepack:8080/mirror/1 +enabled=1 +gpgcheck=0 +``` + +**Configuration APT** (`/etc/apt/sources.list.d/clonepack.list`) : +``` +deb [trusted=yes] http://clonepack:8080/mirror/2 bookworm main +``` + +Le proxy sert uniquement ce qui a été cloné localement (mode local-only). Si un fichier est absent, le serveur retourne 404. + +--- + +## API REST + +| Méthode | Route | Action | +|---------|-------|--------| +| `GET` | `/health` | Santé du serveur | +| `POST` | `/api/v1/repos` | Créer un dépôt | +| `GET` | `/api/v1/repos` | Lister les dépôts | +| `GET` | `/api/v1/repos/{id}` | Détail d'un dépôt | +| `DELETE` | `/api/v1/repos/{id}` | Supprimer un dépôt | +| `POST` | `/api/v1/repos/{id}/clone` | Démarrer le clonage | +| `GET` | `/api/v1/repos/{id}/clone/status` | Statut du clonage | +| `POST` | `/api/v1/repos/{id}/sync/trigger` | Déclencher un scan | +| `GET` | `/api/v1/repos/{id}/sync/pending` | Paquets en attente | +| `POST` | `/api/v1/repos/{id}/sync/approve` | Approuver des paquets | +| `POST` | `/api/v1/repos/{id}/sync/reject` | Rejeter des paquets | +| `POST` | `/api/v1/repos/{id}/sync/block` | Bloquer définitivement des paquets | +| `POST` | `/api/v1/repos/{id}/sync/unblock` | Débloquer des paquets | +| `GET` | `/api/v1/repos/{id}/sync/blocked` | Lister les paquets bloqués | +| `POST` | `/api/v1/repos/{id}/snapshots` | Créer un snapshot | +| `GET` | `/api/v1/repos/{id}/snapshots` | Lister les snapshots | +| `GET` | `/api/v1/repos/{id}/snapshots/{snap_id}` | Détail d'un snapshot | +| `DELETE` | `/api/v1/repos/{id}/snapshots/{snap_id}` | Supprimer un snapshot | +| `GET` | `/api/v1/repos/{id}/snapshots/diff?from=X&to=Y` | Diff deux snapshots | +| `POST` | `/api/v1/repos/{id}/snapshots/{snap_id}/rollback` | Rollback | +| `GET` | `/mirror/{id}/*` | Proxy miroir HTTP | diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..e33523d --- /dev/null +++ b/client/client.go @@ -0,0 +1,388 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" +) + +var ErrNotFound = errors.New("repo not found") + +type Client struct { + baseURL string + httpClient *http.Client +} + +func New(baseURL string) *Client { + return &Client{ + baseURL: baseURL, + httpClient: &http.Client{Timeout: 30 * time.Second}, + } +} + +type CreateRepoInput struct { + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + SyncMode string `json:"sync_mode,omitempty"` +} + +type Repo struct { + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + Frozen bool `json:"frozen"` + SyncMode string `json:"sync_mode"` + CreatedAt time.Time `json:"created_at"` +} + +type listReposResponse struct { + Items []Repo `json:"items"` + Total int `json:"total"` +} + +func (c *Client) CreateRepo(ctx context.Context, in CreateRepoInput) (*Repo, error) { + body, _ := json.Marshal(in) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/v1/repos", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + var repo Repo + if err := c.do(req, &repo); err != nil { + return nil, err + } + return &repo, nil +} + +func (c *Client) ListRepos(ctx context.Context) ([]Repo, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/v1/repos", nil) + if err != nil { + return nil, err + } + var resp listReposResponse + if err := c.do(req, &resp); err != nil { + return nil, err + } + return resp.Items, nil +} + +func (c *Client) GetRepo(ctx context.Context, id int64) (*Repo, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/api/v1/repos/%d", c.baseURL, id), nil) + if err != nil { + return nil, err + } + var repo Repo + if err := c.do(req, &repo); err != nil { + return nil, err + } + return &repo, nil +} + +func (c *Client) DeleteRepo(ctx context.Context, id int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fmt.Sprintf("%s/api/v1/repos/%d", c.baseURL, id), nil) + if err != nil { + return err + } + return c.do(req, nil) +} + +type StartCloneResponse struct { + JobID int64 `json:"job_id"` + Status string `json:"status"` +} + +type CloneStatus struct { + JobID int64 `json:"job_id"` + RepoID int64 `json:"repo_id"` + Status string `json:"status"` + StartedAt *string `json:"started_at,omitempty"` + FinishedAt *string `json:"finished_at,omitempty"` + Error *string `json:"error,omitempty"` + CreatedAt string `json:"created_at"` +} + +func (c *Client) StartClone(ctx context.Context, repoID int64) (*StartCloneResponse, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/clone", c.baseURL, repoID), nil) + if err != nil { + return nil, err + } + var resp StartCloneResponse + if err := c.do(req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *Client) GetCloneStatus(ctx context.Context, repoID int64) (*CloneStatus, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s/api/v1/repos/%d/clone/status", c.baseURL, repoID), nil) + if err != nil { + return nil, err + } + var status CloneStatus + if err := c.do(req, &status); err != nil { + return nil, err + } + return &status, nil +} + +type PendingPackage struct { + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + Name string `json:"name"` + Version string `json:"version"` + Arch string `json:"arch"` + Location string `json:"location"` + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` + Size int64 `json:"size"` + CreatedAt time.Time `json:"created_at"` +} + +type listPendingResponse struct { + Items []PendingPackage `json:"items"` + Total int `json:"total"` +} + +type syncSelectionBody struct { + IDs []int64 `json:"ids"` +} + +func (c *Client) TriggerSync(ctx context.Context, repoID int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/sync/trigger", c.baseURL, repoID), nil) + if err != nil { + return err + } + return c.do(req, nil) +} + +func (c *Client) ListPending(ctx context.Context, repoID int64) ([]PendingPackage, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s/api/v1/repos/%d/sync/pending", c.baseURL, repoID), nil) + if err != nil { + return nil, err + } + var resp listPendingResponse + if err := c.do(req, &resp); err != nil { + return nil, err + } + return resp.Items, nil +} + +func (c *Client) ApprovePending(ctx context.Context, repoID int64, ids []int64) error { + body, _ := json.Marshal(syncSelectionBody{IDs: ids}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/sync/approve", c.baseURL, repoID), + bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req, nil) +} + +func (c *Client) RejectPending(ctx context.Context, repoID int64, ids []int64) error { + body, _ := json.Marshal(syncSelectionBody{IDs: ids}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/sync/reject", c.baseURL, repoID), + bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req, nil) +} + +type Snapshot struct { + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + Label string `json:"label"` + CreatedAt time.Time `json:"created_at"` +} + +type SnapshotPackage struct { + ID int64 `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Arch string `json:"arch"` + Location string `json:"location"` + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` + Size int64 `json:"size"` +} + +type SnapshotDetail struct { + Snapshot + Packages []SnapshotPackage `json:"packages"` +} + +type listSnapshotsResponse struct { + Items []Snapshot `json:"items"` + Total int `json:"total"` +} + +type SnapshotDiff struct { + From Snapshot `json:"from"` + To Snapshot `json:"to"` + Added []SnapshotPackage `json:"added"` + Removed []SnapshotPackage `json:"removed"` + Unchanged int `json:"unchanged"` +} + +func (c *Client) CreateSnapshot(ctx context.Context, repoID int64, label string) (*Snapshot, error) { + body, _ := json.Marshal(map[string]string{"label": label}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/snapshots", c.baseURL, repoID), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + var snap Snapshot + if err := c.do(req, &snap); err != nil { + return nil, err + } + return &snap, nil +} + +func (c *Client) ListSnapshots(ctx context.Context, repoID int64) ([]Snapshot, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s/api/v1/repos/%d/snapshots", c.baseURL, repoID), nil) + if err != nil { + return nil, err + } + var resp listSnapshotsResponse + if err := c.do(req, &resp); err != nil { + return nil, err + } + return resp.Items, nil +} + +func (c *Client) GetSnapshot(ctx context.Context, repoID, snapID int64) (*SnapshotDetail, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s/api/v1/repos/%d/snapshots/%d", c.baseURL, repoID, snapID), nil) + if err != nil { + return nil, err + } + var detail SnapshotDetail + if err := c.do(req, &detail); err != nil { + return nil, err + } + return &detail, nil +} + +func (c *Client) DeleteSnapshot(ctx context.Context, repoID, snapID int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, + fmt.Sprintf("%s/api/v1/repos/%d/snapshots/%d", c.baseURL, repoID, snapID), nil) + if err != nil { + return err + } + return c.do(req, nil) +} + +func (c *Client) DiffSnapshots(ctx context.Context, repoID, fromID, toID int64) (*SnapshotDiff, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s/api/v1/repos/%d/snapshots/diff?from=%d&to=%d", c.baseURL, repoID, fromID, toID), nil) + if err != nil { + return nil, err + } + var diff SnapshotDiff + if err := c.do(req, &diff); err != nil { + return nil, err + } + return &diff, nil +} + +func (c *Client) RollbackSnapshot(ctx context.Context, repoID, snapID int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/snapshots/%d/rollback", c.baseURL, repoID, snapID), nil) + if err != nil { + return err + } + return c.do(req, nil) +} + +type BlockedPackage struct { + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + Name string `json:"name"` + Location string `json:"location"` + CreatedAt time.Time `json:"created_at"` +} + +type listBlockedResponse struct { + Items []BlockedPackage `json:"items"` + Total int `json:"total"` +} + +func (c *Client) BlockPackages(ctx context.Context, repoID int64, pendingIDs []int64) error { + body, _ := json.Marshal(syncSelectionBody{IDs: pendingIDs}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/sync/block", c.baseURL, repoID), + bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req, nil) +} + +func (c *Client) UnblockPackages(ctx context.Context, repoID int64, blockedIDs []int64) error { + body, _ := json.Marshal(syncSelectionBody{IDs: blockedIDs}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/api/v1/repos/%d/sync/unblock", c.baseURL, repoID), + bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req, nil) +} + +func (c *Client) ListBlocked(ctx context.Context, repoID int64) ([]BlockedPackage, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s/api/v1/repos/%d/sync/blocked", c.baseURL, repoID), nil) + if err != nil { + return nil, err + } + var resp listBlockedResponse + if err := c.do(req, &resp); err != nil { + return nil, err + } + return resp.Items, nil +} + +type errorResponse struct { + Error string `json:"error"` +} + +func (c *Client) do(req *http.Request, out any) error { + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return ErrNotFound + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + var errResp errorResponse + json.NewDecoder(resp.Body).Decode(&errResp) + if errResp.Error != "" { + return fmt.Errorf("%s", errResp.Error) + } + return fmt.Errorf("unexpected status %d", resp.StatusCode) + } + if out != nil { + return json.NewDecoder(resp.Body).Decode(out) + } + return nil +} diff --git a/cmd/clonepack/main.go b/cmd/clonepack/main.go new file mode 100644 index 0000000..2e7006e --- /dev/null +++ b/cmd/clonepack/main.go @@ -0,0 +1,9 @@ +package main + +import "os" + +func main() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/cmd/clonepack/repo.go b/cmd/clonepack/repo.go new file mode 100644 index 0000000..b20059c --- /dev/null +++ b/cmd/clonepack/repo.go @@ -0,0 +1,498 @@ +package main + +import ( + "errors" + "fmt" + "os" + "strconv" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + "github.com/syonad/clonepack/client" +) + +var repoCloneCmd = &cobra.Command{ + Use: "clone ", + Short: "Clone an RPM repository", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return fmt.Errorf("invalid id: %s", args[0]) + } + c := client.New(apiURL) + + resp, err := c.StartClone(cmd.Context(), id) + if err != nil { + return fmt.Errorf("start clone: %w", err) + } + fmt.Printf("Clone job started: job_id=%d\n", resp.JobID) + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-cmd.Context().Done(): + return cmd.Context().Err() + case <-ticker.C: + status, err := c.GetCloneStatus(cmd.Context(), id) + if err != nil { + return fmt.Errorf("get status: %w", err) + } + fmt.Printf(" status: %s\n", status.Status) + switch status.Status { + case "completed": + fmt.Println("Clone completed successfully.") + return nil + case "failed": + errMsg := "(unknown error)" + if status.Error != nil { + errMsg = *status.Error + } + return fmt.Errorf("clone failed: %s", errMsg) + } + } + } + }, +} + +var repoCmd = &cobra.Command{ + Use: "repo", + Short: "Manage repositories", +} + +var ( + repoName string + repoType string + repoSourceURL string +) + +var repoCreateCmd = &cobra.Command{ + Use: "create", + Short: "Create a new repository", + RunE: func(cmd *cobra.Command, args []string) error { + c := client.New(apiURL) + repo, err := c.CreateRepo(cmd.Context(), client.CreateRepoInput{ + Name: repoName, + Type: repoType, + SourceURL: repoSourceURL, + SyncMode: repoSyncMode, + }) + if err != nil { + return err + } + printRepos([]client.Repo{*repo}) + return nil + }, +} + +var repoListCmd = &cobra.Command{ + Use: "list", + Short: "List all repositories", + RunE: func(cmd *cobra.Command, args []string) error { + c := client.New(apiURL) + repos, err := c.ListRepos(cmd.Context()) + if err != nil { + return err + } + if len(repos) == 0 { + fmt.Println("No repositories found.") + return nil + } + printRepos(repos) + return nil + }, +} + +var repoGetCmd = &cobra.Command{ + Use: "get ", + Short: "Get a repository by ID", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return fmt.Errorf("invalid id: %s", args[0]) + } + c := client.New(apiURL) + repo, err := c.GetRepo(cmd.Context(), id) + if errors.Is(err, client.ErrNotFound) { + return fmt.Errorf("repo %d not found", id) + } + if err != nil { + return err + } + printRepos([]client.Repo{*repo}) + return nil + }, +} + +var repoDeleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Delete a repository by ID", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return fmt.Errorf("invalid id: %s", args[0]) + } + c := client.New(apiURL) + if err := c.DeleteRepo(cmd.Context(), id); errors.Is(err, client.ErrNotFound) { + return fmt.Errorf("repo %d not found", id) + } else if err != nil { + return err + } + fmt.Printf("Deleted repo %d\n", id) + return nil + }, +} + +func printRepos(repos []client.Repo) { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tNAME\tTYPE\tSOURCE URL\tFROZEN\tSYNC MODE\tCREATED AT") + for _, r := range repos { + fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%v\t%s\t%s\n", + r.ID, r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, + r.CreatedAt.Format(time.RFC3339), + ) + } + w.Flush() +} + +var repoSyncCmd = &cobra.Command{ + Use: "sync ", + Short: "Trigger a sync scan for a repository", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + c := client.New(apiURL) + if err := c.TriggerSync(cmd.Context(), id); err != nil { + return err + } + fmt.Println("Scan started. Use 'sync-list' to check pending packages.") + return nil + }, +} + +var repoSyncListCmd = &cobra.Command{ + Use: "sync-list ", + Short: "List packages pending approval for a repository", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + c := client.New(apiURL) + pkgs, err := c.ListPending(cmd.Context(), id) + if err != nil { + return err + } + if len(pkgs) == 0 { + fmt.Println("No packages pending approval.") + return nil + } + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tNAME\tVERSION\tARCH\tSIZE") + for _, p := range pkgs { + fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%d\n", p.ID, p.Name, p.Version, p.Arch, p.Size) + } + w.Flush() + return nil + }, +} + +var syncApproveAll bool + +var repoSyncApproveCmd = &cobra.Command{ + Use: "sync-approve [pkg_id...]", + Short: "Approve pending packages for download", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + c := client.New(apiURL) + + var ids []int64 + if syncApproveAll { + pkgs, err := c.ListPending(cmd.Context(), repoID) + if err != nil { + return fmt.Errorf("list pending: %w", err) + } + if len(pkgs) == 0 { + fmt.Println("No packages pending approval.") + return nil + } + for _, p := range pkgs { + ids = append(ids, p.ID) + } + } else { + if len(args) < 2 { + return fmt.Errorf("provide package IDs or use --all") + } + for _, a := range args[1:] { + id, err := strconv.ParseInt(a, 10, 64) + if err != nil { + return fmt.Errorf("invalid pkg id: %s", a) + } + ids = append(ids, id) + } + } + + if err := c.ApprovePending(cmd.Context(), repoID, ids); err != nil { + return err + } + fmt.Printf("Approval started for %d package(s). Downloads running in background.\n", len(ids)) + return nil + }, +} + +var repoSyncRejectCmd = &cobra.Command{ + Use: "sync-reject ...", + Short: "Reject pending packages", + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + var ids []int64 + for _, a := range args[1:] { + id, _ := strconv.ParseInt(a, 10, 64) + ids = append(ids, id) + } + c := client.New(apiURL) + if err := c.RejectPending(cmd.Context(), repoID, ids); err != nil { + return err + } + fmt.Printf("Rejected %d package(s)\n", len(ids)) + return nil + }, +} + +var repoSyncBlockCmd = &cobra.Command{ + Use: "sync-block ...", + Short: "Bloquer définitivement des paquets (par ID pending)", + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + var ids []int64 + for _, a := range args[1:] { + id, err := strconv.ParseInt(a, 10, 64) + if err != nil { + return fmt.Errorf("invalid id: %s", a) + } + ids = append(ids, id) + } + c := client.New(apiURL) + if err := c.BlockPackages(cmd.Context(), repoID, ids); err != nil { + return err + } + fmt.Printf("%d paquet(s) bloqué(s) définitivement.\n", len(ids)) + return nil + }, +} + +var repoSyncUnblockCmd = &cobra.Command{ + Use: "sync-unblock ...", + Short: "Débloquer des paquets (par ID blocked)", + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + var ids []int64 + for _, a := range args[1:] { + id, _ := strconv.ParseInt(a, 10, 64) + ids = append(ids, id) + } + c := client.New(apiURL) + if err := c.UnblockPackages(cmd.Context(), repoID, ids); err != nil { + return err + } + fmt.Printf("%d paquet(s) débloqué(s).\n", len(ids)) + return nil + }, +} + +var repoSyncBlockListCmd = &cobra.Command{ + Use: "sync-blocklist ", + Short: "Lister les paquets bloqués définitivement", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + c := client.New(apiURL) + pkgs, err := c.ListBlocked(cmd.Context(), repoID) + if err != nil { + return err + } + if len(pkgs) == 0 { + fmt.Println("Aucun paquet bloqué.") + return nil + } + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tNOM\tLOCATION\tBLOQUÉ LE") + for _, p := range pkgs { + fmt.Fprintf(w, "%d\t%s\t%s\t%s\n", p.ID, p.Name, p.Location, p.CreatedAt.Format(time.RFC3339)) + } + w.Flush() + return nil + }, +} + +var repoSnapshotCmd = &cobra.Command{ + Use: "snapshot", + Short: "Manage repository snapshots", +} + +var repoSnapshotCreateCmd = &cobra.Command{ + Use: "create ", + Short: "Create a snapshot of a repository", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + c := client.New(apiURL) + snap, err := c.CreateSnapshot(cmd.Context(), repoID, snapshotLabel) + if err != nil { + return err + } + printSnapshots([]client.Snapshot{*snap}) + return nil + }, +} + +var repoSnapshotListCmd = &cobra.Command{ + Use: "list ", + Short: "List snapshots of a repository", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + c := client.New(apiURL) + snaps, err := c.ListSnapshots(cmd.Context(), repoID) + if err != nil { + return err + } + if len(snaps) == 0 { + fmt.Println("No snapshots found.") + return nil + } + printSnapshots(snaps) + return nil + }, +} + +var repoSnapshotShowCmd = &cobra.Command{ + Use: "show ", + Short: "Show snapshot details", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + snapID, _ := strconv.ParseInt(args[1], 10, 64) + c := client.New(apiURL) + detail, err := c.GetSnapshot(cmd.Context(), repoID, snapID) + if err != nil { + return err + } + printSnapshots([]client.Snapshot{detail.Snapshot}) + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "NAME\tVERSION\tARCH\tSIZE") + for _, p := range detail.Packages { + fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", p.Name, p.Version, p.Arch, p.Size) + } + w.Flush() + return nil + }, +} + +var repoSnapshotDeleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Delete a snapshot", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + snapID, _ := strconv.ParseInt(args[1], 10, 64) + c := client.New(apiURL) + if err := c.DeleteSnapshot(cmd.Context(), repoID, snapID); err != nil { + return err + } + fmt.Printf("Deleted snapshot %d\n", snapID) + return nil + }, +} + +var repoSnapshotDiffCmd = &cobra.Command{ + Use: "diff ", + Short: "Diff two snapshots", + Args: cobra.ExactArgs(3), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + fromID, _ := strconv.ParseInt(args[1], 10, 64) + toID, _ := strconv.ParseInt(args[2], 10, 64) + c := client.New(apiURL) + diff, err := c.DiffSnapshots(cmd.Context(), repoID, fromID, toID) + if err != nil { + return err + } + fmt.Printf("From: [%d] %s\nTo: [%d] %s\nUnchanged: %d\n", + diff.From.ID, diff.From.Label, diff.To.ID, diff.To.Label, diff.Unchanged) + if len(diff.Added) > 0 { + fmt.Printf("\nAdded (%d):\n", len(diff.Added)) + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, " NAME\tVERSION\tARCH") + for _, p := range diff.Added { + fmt.Fprintf(w, " %s\t%s\t%s\n", p.Name, p.Version, p.Arch) + } + w.Flush() + } + if len(diff.Removed) > 0 { + fmt.Printf("\nRemoved (%d):\n", len(diff.Removed)) + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, " NAME\tVERSION\tARCH") + for _, p := range diff.Removed { + fmt.Fprintf(w, " %s\t%s\t%s\n", p.Name, p.Version, p.Arch) + } + w.Flush() + } + return nil + }, +} + +var repoSnapshotRollbackCmd = &cobra.Command{ + Use: "rollback ", + Short: "Rollback repository to a snapshot", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + repoID, _ := strconv.ParseInt(args[0], 10, 64) + snapID, _ := strconv.ParseInt(args[1], 10, 64) + c := client.New(apiURL) + if err := c.RollbackSnapshot(cmd.Context(), repoID, snapID); err != nil { + return err + } + fmt.Println("Rollback completed.") + return nil + }, +} + +func printSnapshots(snaps []client.Snapshot) { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tREPO ID\tLABEL\tCREATED AT") + for _, s := range snaps { + fmt.Fprintf(w, "%d\t%d\t%s\t%s\n", s.ID, s.RepoID, s.Label, s.CreatedAt.Format(time.RFC3339)) + } + w.Flush() +} + +var snapshotLabel string +var repoSyncMode string + +func init() { + repoCreateCmd.Flags().StringVar(&repoName, "name", "", "Repository name (required)") + repoCreateCmd.Flags().StringVar(&repoType, "type", "", "Type: apt|rpm|docker|binary (required)") + repoCreateCmd.Flags().StringVar(&repoSourceURL, "source-url", "", "Upstream source URL (required)") + repoCreateCmd.Flags().StringVar(&repoSyncMode, "sync-mode", "auto", "Sync mode: auto|manual") + repoCreateCmd.MarkFlagRequired("name") + repoCreateCmd.MarkFlagRequired("type") + repoCreateCmd.MarkFlagRequired("source-url") + + repoSyncApproveCmd.Flags().BoolVar(&syncApproveAll, "all", false, "Approuver tous les paquets en attente") + + repoSnapshotCreateCmd.Flags().StringVar(&snapshotLabel, "label", "", "Snapshot label") + repoSnapshotCmd.AddCommand(repoSnapshotCreateCmd, repoSnapshotListCmd, repoSnapshotShowCmd, + repoSnapshotDeleteCmd, repoSnapshotDiffCmd, repoSnapshotRollbackCmd) + + repoCmd.AddCommand(repoCreateCmd, repoListCmd, repoGetCmd, repoDeleteCmd, repoCloneCmd, + repoSyncCmd, repoSyncListCmd, repoSyncApproveCmd, repoSyncRejectCmd, + repoSyncBlockCmd, repoSyncUnblockCmd, repoSyncBlockListCmd, + repoSnapshotCmd) +} diff --git a/cmd/clonepack/root.go b/cmd/clonepack/root.go new file mode 100644 index 0000000..2af7338 --- /dev/null +++ b/cmd/clonepack/root.go @@ -0,0 +1,20 @@ +package main + +import "github.com/spf13/cobra" + +var ( + cfgFile string + apiURL string +) + +var rootCmd = &cobra.Command{ + Use: "clonepack", + Short: "ClonePack artifact mirror manager", +} + +func init() { + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file path") + rootCmd.PersistentFlags().StringVar(&apiURL, "api-url", "http://localhost:8080", "ClonePack server URL") + rootCmd.AddCommand(serveCmd) + rootCmd.AddCommand(repoCmd) +} diff --git a/cmd/clonepack/serve.go b/cmd/clonepack/serve.go new file mode 100644 index 0000000..b299869 --- /dev/null +++ b/cmd/clonepack/serve.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + + "github.com/spf13/cobra" + "github.com/syonad/clonepack/config" + "github.com/syonad/clonepack/internal/api" + "github.com/syonad/clonepack/internal/core" + "github.com/syonad/clonepack/internal/store" +) + +var serveCmd = &cobra.Command{ + Use: "serve", + Short: "Start the ClonePack HTTP server", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load(cfgFile) + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + db, err := store.Open(cfg.DB) + if err != nil { + return fmt.Errorf("open store: %w", err) + } + + repoStore := store.NewRepoStore(db) + jobStore := store.NewCloneJobStore(db) + pendingStore := store.NewPendingPackageStore(db) + blockedStore := store.NewBlockedPackageStore(db) + snapshotStore := store.NewSnapshotStore(db) + + repoSvc := core.NewRepoService(repoStore, cfg.DataDir) + cloneSvc := core.NewCloneService(repoStore, jobStore, cfg.DataDir) + snapshotSvc := core.NewSnapshotService(snapshotStore, repoStore, cfg.DataDir) + syncSvc := core.NewSyncService(repoStore, pendingStore, blockedStore, cloneSvc, snapshotSvc, cfg.DataDir) + + repoH := api.NewRepoHandler(repoSvc) + cloneH := api.NewCloneHandler(cloneSvc) + syncH := api.NewSyncHandler(syncSvc) + snapshotH := api.NewSnapshotHandler(snapshotSvc) + proxyH := api.NewProxyHandler(repoSvc, cfg.DataDir) + router := api.NewRouter(repoH, cloneH, syncH, snapshotH, proxyH) + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + syncSvc.StartScheduler(ctx, cfg.Sync.IntervalDuration()) + + addr := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port) + log.Printf("ClonePack listening on %s", addr) + return http.ListenAndServe(addr, router) + }, +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..d3a4cb0 --- /dev/null +++ b/config/config.go @@ -0,0 +1,31 @@ +package config + +import "time" + +type Config struct { + Server ServerConfig `mapstructure:"server"` + DB DBConfig `mapstructure:"db"` + DataDir string `mapstructure:"data_dir"` + Sync SyncConfig `mapstructure:"sync"` +} + +type ServerConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` +} + +type DBConfig struct { + Path string `mapstructure:"path"` +} + +type SyncConfig struct { + Interval string `mapstructure:"interval"` +} + +func (s SyncConfig) IntervalDuration() time.Duration { + d, err := time.ParseDuration(s.Interval) + if err != nil || d <= 0 { + return time.Hour + } + return d +} diff --git a/config/loader.go b/config/loader.go new file mode 100644 index 0000000..2b00112 --- /dev/null +++ b/config/loader.go @@ -0,0 +1,41 @@ +package config + +import ( + "strings" + + "github.com/spf13/viper" +) + +func Load(cfgFile string) (*Config, error) { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + } else { + viper.SetConfigName("clonepack") + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("$HOME/.clonepack") + viper.AddConfigPath("/etc/clonepack") + } + + viper.SetEnvPrefix("CLONEPACK") + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.AutomaticEnv() + + viper.SetDefault("server.host", "0.0.0.0") + viper.SetDefault("server.port", 8080) + viper.SetDefault("db.path", "./clonepack.db") + viper.SetDefault("data_dir", "./data") + viper.SetDefault("sync.interval", "1h") + + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { + return nil, err + } + } + + var cfg Config + if err := viper.Unmarshal(&cfg); err != nil { + return nil, err + } + return &cfg, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..fa90285 --- /dev/null +++ b/go.mod @@ -0,0 +1,33 @@ +module github.com/syonad/clonepack + +go 1.25.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-chi/chi/v5 v5.2.5 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/golang-migrate/migrate/v4 v4.19.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.21.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.31.0 // indirect + modernc.org/libc v1.72.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.49.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..277e975 --- /dev/null +++ b/go.sum @@ -0,0 +1,59 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= +github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= +github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U= +modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= diff --git a/internal/api/clone_handler.go b/internal/api/clone_handler.go new file mode 100644 index 0000000..ad16ba9 --- /dev/null +++ b/internal/api/clone_handler.go @@ -0,0 +1,83 @@ +package api + +import ( + "errors" + "net/http" + "strconv" + "time" + + "github.com/go-chi/chi/v5" + "github.com/syonad/clonepack/internal/core" + "github.com/syonad/clonepack/internal/store" +) + +type CloneHandler struct { + svc *core.CloneService +} + +func NewCloneHandler(svc *core.CloneService) *CloneHandler { + return &CloneHandler{svc: svc} +} + +func (h *CloneHandler) StartClone(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + + jobID, err := h.svc.StartClone(r.Context(), repoID) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } + if errors.Is(err, core.ErrCloneAlreadyRunning) { + Error(w, http.StatusConflict, err.Error()) + return + } + if err != nil { + Error(w, http.StatusUnprocessableEntity, err.Error()) + return + } + + JSON(w, http.StatusAccepted, StartCloneResponse{JobID: jobID, Status: "running"}) +} + +func (h *CloneHandler) GetStatus(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + + job, err := h.svc.GetLatestCloneJob(r.Context(), repoID) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "no clone job found for this repo") + return + } + if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + + JSON(w, http.StatusOK, cloneJobToResponse(job)) +} + +func cloneJobToResponse(j *store.CloneJob) CloneStatusResponse { + resp := CloneStatusResponse{ + JobID: j.ID, + RepoID: j.RepoID, + Status: string(j.Status), + Error: j.Error, + CreatedAt: j.CreatedAt.Format(time.RFC3339), + } + if j.StartedAt != nil { + s := j.StartedAt.Format(time.RFC3339) + resp.StartedAt = &s + } + if j.FinishedAt != nil { + f := j.FinishedAt.Format(time.RFC3339) + resp.FinishedAt = &f + } + return resp +} diff --git a/internal/api/proxy_handler.go b/internal/api/proxy_handler.go new file mode 100644 index 0000000..cac9794 --- /dev/null +++ b/internal/api/proxy_handler.go @@ -0,0 +1,43 @@ +package api + +import ( + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/go-chi/chi/v5" + "github.com/syonad/clonepack/internal/core" + "github.com/syonad/clonepack/internal/store" +) + +type ProxyHandler struct { + repoSvc *core.RepoService + dataDir string +} + +func NewProxyHandler(repoSvc *core.RepoService, dataDir string) *ProxyHandler { + return &ProxyHandler{repoSvc: repoSvc, dataDir: dataDir} +} + +func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "repo_id"), 10, 64) + if err != nil { + http.Error(w, "invalid repo_id", http.StatusBadRequest) + return + } + + repo, err := h.repoSvc.Get(r.Context(), repoID) + if errors.Is(err, store.ErrNotFound) { + http.NotFound(w, r) + return + } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repoID, repo.Type) + prefix := fmt.Sprintf("/mirror/%d", repoID) + http.StripPrefix(prefix, http.FileServer(http.Dir(localDir))).ServeHTTP(w, r) +} diff --git a/internal/api/repo_handler.go b/internal/api/repo_handler.go new file mode 100644 index 0000000..9b791e7 --- /dev/null +++ b/internal/api/repo_handler.go @@ -0,0 +1,102 @@ +package api + +import ( + "encoding/json" + "errors" + "net/http" + "strconv" + + "github.com/go-chi/chi/v5" + "github.com/syonad/clonepack/internal/core" + "github.com/syonad/clonepack/internal/store" +) + +type RepoHandler struct { + svc *core.RepoService +} + +func NewRepoHandler(svc *core.RepoService) *RepoHandler { + return &RepoHandler{svc: svc} +} + +func (h *RepoHandler) Create(w http.ResponseWriter, r *http.Request) { + var req CreateRepoRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + Error(w, http.StatusBadRequest, "invalid JSON") + return + } + + repo, err := h.svc.Create(r.Context(), core.CreateRepoInput{ + Name: req.Name, + Type: req.Type, + SourceURL: req.SourceURL, + SyncMode: req.SyncMode, + }) + if err != nil { + Error(w, http.StatusUnprocessableEntity, err.Error()) + return + } + JSON(w, http.StatusCreated, repoToResponse(repo)) +} + +func (h *RepoHandler) List(w http.ResponseWriter, r *http.Request) { + repos, err := h.svc.List(r.Context()) + if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + + items := make([]RepoResponse, len(repos)) + for i, repo := range repos { + items[i] = repoToResponse(&repo) + } + JSON(w, http.StatusOK, ListReposResponse{Items: items, Total: len(items)}) +} + +func (h *RepoHandler) Get(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + + repo, err := h.svc.Get(r.Context(), id) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } + if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + JSON(w, http.StatusOK, repoToResponse(repo)) +} + +func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + + if err := h.svc.Delete(r.Context(), id); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func repoToResponse(r *store.Repo) RepoResponse { + return RepoResponse{ + ID: r.ID, + Name: r.Name, + Type: r.Type, + SourceURL: r.SourceURL, + Frozen: r.Frozen, + SyncMode: r.SyncMode, + CreatedAt: r.CreatedAt, + } +} diff --git a/internal/api/respond.go b/internal/api/respond.go new file mode 100644 index 0000000..30df7ca --- /dev/null +++ b/internal/api/respond.go @@ -0,0 +1,16 @@ +package api + +import ( + "encoding/json" + "net/http" +) + +func JSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) +} + +func Error(w http.ResponseWriter, status int, msg string) { + JSON(w, status, ErrorResponse{Error: msg}) +} diff --git a/internal/api/router.go b/internal/api/router.go new file mode 100644 index 0000000..55cb3b9 --- /dev/null +++ b/internal/api/router.go @@ -0,0 +1,50 @@ +package api + +import ( + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" +) + +func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler *SyncHandler, snapshotHandler *SnapshotHandler, proxyHandler *ProxyHandler) http.Handler { + r := chi.NewRouter() + r.Use(middleware.Logger) + r.Use(middleware.Recoverer) + r.Use(middleware.RequestID) + + r.Get("/mirror/{repo_id}/*", proxyHandler.ServeFile) + + r.Get("/health", func(w http.ResponseWriter, r *http.Request) { + JSON(w, http.StatusOK, map[string]string{"status": "ok"}) + }) + + r.Route("/api/v1", func(r chi.Router) { + r.Route("/repos", func(r chi.Router) { + r.Post("/", repoHandler.Create) + r.Get("/", repoHandler.List) + r.Get("/{id}", repoHandler.Get) + r.Delete("/{id}", repoHandler.Delete) + + r.Post("/{id}/clone", cloneHandler.StartClone) + r.Get("/{id}/clone/status", cloneHandler.GetStatus) + + r.Post("/{id}/sync/trigger", syncHandler.Trigger) + r.Get("/{id}/sync/pending", syncHandler.ListPending) + r.Post("/{id}/sync/approve", syncHandler.Approve) + r.Post("/{id}/sync/reject", syncHandler.Reject) + r.Post("/{id}/sync/block", syncHandler.Block) + r.Post("/{id}/sync/unblock", syncHandler.Unblock) + r.Get("/{id}/sync/blocked", syncHandler.ListBlocked) + + r.Post("/{id}/snapshots", snapshotHandler.Create) + r.Get("/{id}/snapshots", snapshotHandler.List) + r.Get("/{id}/snapshots/diff", snapshotHandler.Diff) + r.Get("/{id}/snapshots/{snap_id}", snapshotHandler.Get) + r.Delete("/{id}/snapshots/{snap_id}", snapshotHandler.Delete) + r.Post("/{id}/snapshots/{snap_id}/rollback", snapshotHandler.Rollback) + }) + }) + + return r +} diff --git a/internal/api/snapshot_handler.go b/internal/api/snapshot_handler.go new file mode 100644 index 0000000..831d489 --- /dev/null +++ b/internal/api/snapshot_handler.go @@ -0,0 +1,196 @@ +package api + +import ( + "encoding/json" + "errors" + "net/http" + "strconv" + "time" + + "github.com/go-chi/chi/v5" + "github.com/syonad/clonepack/internal/core" + "github.com/syonad/clonepack/internal/store" +) + +type SnapshotHandler struct { + svc *core.SnapshotService +} + +func NewSnapshotHandler(svc *core.SnapshotService) *SnapshotHandler { + return &SnapshotHandler{svc: svc} +} + +func (h *SnapshotHandler) Create(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + var req CreateSnapshotRequest + json.NewDecoder(r.Body).Decode(&req) + if req.Label == "" { + req.Label = "manual-" + time.Now().UTC().Format(time.RFC3339) + } + id, err := h.svc.TakeSnapshot(r.Context(), repoID, req.Label) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + snap, _, _ := h.svc.Get(r.Context(), repoID, id) + JSON(w, http.StatusCreated, snapshotToResponse(*snap)) +} + +func (h *SnapshotHandler) List(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + snaps, err := h.svc.List(r.Context(), repoID) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + items := make([]SnapshotResponse, len(snaps)) + for i, s := range snaps { + items[i] = snapshotToResponse(s) + } + JSON(w, http.StatusOK, ListSnapshotsResponse{Items: items, Total: len(items)}) +} + +func (h *SnapshotHandler) Get(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + snapID, err := strconv.ParseInt(chi.URLParam(r, "snap_id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid snap_id") + return + } + snap, pkgs, err := h.svc.Get(r.Context(), repoID, snapID) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "snapshot not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + resp := SnapshotDetailResponse{ + SnapshotResponse: snapshotToResponse(*snap), + Packages: make([]SnapshotPackageResponse, len(pkgs)), + } + for i, p := range pkgs { + resp.Packages[i] = snapshotPkgToResponse(p) + } + JSON(w, http.StatusOK, resp) +} + +func (h *SnapshotHandler) Delete(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + snapID, err := strconv.ParseInt(chi.URLParam(r, "snap_id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid snap_id") + return + } + if err := h.svc.Delete(r.Context(), repoID, snapID); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "snapshot not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *SnapshotHandler) Diff(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + fromID, err := strconv.ParseInt(r.URL.Query().Get("from"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid from param") + return + } + toID, err := strconv.ParseInt(r.URL.Query().Get("to"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid to param") + return + } + diff, err := h.svc.Diff(r.Context(), repoID, fromID, toID) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "snapshot not found") + return + } else if err != nil { + Error(w, http.StatusBadRequest, err.Error()) + return + } + resp := SnapshotDiffResponse{ + From: snapshotToResponse(*diff.From), + To: snapshotToResponse(*diff.To), + Unchanged: diff.Unchanged, + } + for _, p := range diff.Added { + resp.Added = append(resp.Added, snapshotPkgToResponse(p)) + } + for _, p := range diff.Removed { + resp.Removed = append(resp.Removed, snapshotPkgToResponse(p)) + } + JSON(w, http.StatusOK, resp) +} + +func (h *SnapshotHandler) Rollback(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + snapID, err := strconv.ParseInt(chi.URLParam(r, "snap_id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid snap_id") + return + } + if err := h.svc.Rollback(r.Context(), repoID, snapID); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "snapshot not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + JSON(w, http.StatusOK, map[string]string{"status": "rollback completed"}) +} + +func snapshotToResponse(s store.Snapshot) SnapshotResponse { + return SnapshotResponse{ + ID: s.ID, + RepoID: s.RepoID, + Label: s.Label, + CreatedAt: s.CreatedAt, + } +} + +func snapshotPkgToResponse(p store.SnapshotPackage) SnapshotPackageResponse { + return SnapshotPackageResponse{ + ID: p.ID, + Name: p.Name, + Version: p.Version, + Arch: p.Arch, + Location: p.Location, + Checksum: p.Checksum, + ChecksumType: p.ChecksumType, + Size: p.Size, + } +} diff --git a/internal/api/snapshot_types.go b/internal/api/snapshot_types.go new file mode 100644 index 0000000..8dc173d --- /dev/null +++ b/internal/api/snapshot_types.go @@ -0,0 +1,43 @@ +package api + +import "time" + +type SnapshotResponse struct { + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + Label string `json:"label"` + CreatedAt time.Time `json:"created_at"` +} + +type SnapshotDetailResponse struct { + SnapshotResponse + Packages []SnapshotPackageResponse `json:"packages"` +} + +type SnapshotPackageResponse struct { + ID int64 `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Arch string `json:"arch"` + Location string `json:"location"` + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` + Size int64 `json:"size"` +} + +type ListSnapshotsResponse struct { + Items []SnapshotResponse `json:"items"` + Total int `json:"total"` +} + +type CreateSnapshotRequest struct { + Label string `json:"label"` +} + +type SnapshotDiffResponse struct { + From SnapshotResponse `json:"from"` + To SnapshotResponse `json:"to"` + Added []SnapshotPackageResponse `json:"added"` + Removed []SnapshotPackageResponse `json:"removed"` + Unchanged int `json:"unchanged"` +} diff --git a/internal/api/sync_handler.go b/internal/api/sync_handler.go new file mode 100644 index 0000000..1b66735 --- /dev/null +++ b/internal/api/sync_handler.go @@ -0,0 +1,191 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "log" + "net/http" + "strconv" + + "github.com/go-chi/chi/v5" + "github.com/syonad/clonepack/internal/core" + "github.com/syonad/clonepack/internal/store" +) + +type SyncHandler struct { + svc *core.SyncService +} + +func NewSyncHandler(svc *core.SyncService) *SyncHandler { + return &SyncHandler{svc: svc} +} + +func (h *SyncHandler) Trigger(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + // Validate repo exists before returning. + if err := h.svc.ValidateRepo(r.Context(), repoID); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + go func() { + if err := h.svc.ScanRepo(context.Background(), repoID); err != nil { + log.Printf("scan repo %d: %v", repoID, err) + } + }() + JSON(w, http.StatusAccepted, map[string]string{"status": "scan started"}) +} + +func (h *SyncHandler) ListPending(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + pkgs, err := h.svc.ListPending(r.Context(), repoID) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + items := make([]PendingPackageResponse, len(pkgs)) + for i, p := range pkgs { + items[i] = pendingToResponse(p) + } + JSON(w, http.StatusOK, ListPendingResponse{Items: items, Total: len(items)}) +} + +func (h *SyncHandler) Approve(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + var req SyncSelectionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 { + Error(w, http.StatusBadRequest, "body must contain non-empty ids array") + return + } + if err := h.svc.ValidateRepo(r.Context(), repoID); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + ids := req.IDs + go func() { + if err := h.svc.ApprovePending(context.Background(), repoID, ids); err != nil { + log.Printf("approve pending for repo %d: %v", repoID, err) + } + }() + JSON(w, http.StatusAccepted, map[string]string{"status": "approval started"}) +} + +func (h *SyncHandler) Reject(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + var req SyncSelectionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 { + Error(w, http.StatusBadRequest, "body must contain non-empty ids array") + return + } + if err := h.svc.RejectPending(r.Context(), repoID, req.IDs); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *SyncHandler) Block(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + var req SyncSelectionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 { + Error(w, http.StatusBadRequest, "body must contain non-empty ids array") + return + } + if err := h.svc.BlockPackages(r.Context(), repoID, req.IDs); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *SyncHandler) Unblock(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + var req SyncSelectionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 { + Error(w, http.StatusBadRequest, "body must contain non-empty ids array") + return + } + if err := h.svc.UnblockPackages(r.Context(), repoID, req.IDs); errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *SyncHandler) ListBlocked(w http.ResponseWriter, r *http.Request) { + repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil { + Error(w, http.StatusBadRequest, "invalid id") + return + } + pkgs, err := h.svc.ListBlocked(r.Context(), repoID) + if errors.Is(err, store.ErrNotFound) { + Error(w, http.StatusNotFound, "repo not found") + return + } else if err != nil { + Error(w, http.StatusInternalServerError, err.Error()) + return + } + items := make([]BlockedPackageResponse, len(pkgs)) + for i, p := range pkgs { + items[i] = BlockedPackageResponse{ID: p.ID, RepoID: p.RepoID, Name: p.Name, Location: p.Location, CreatedAt: p.CreatedAt} + } + JSON(w, http.StatusOK, ListBlockedResponse{Items: items, Total: len(items)}) +} + +func pendingToResponse(p store.PendingPackage) PendingPackageResponse { + return PendingPackageResponse{ + ID: p.ID, + RepoID: p.RepoID, + Name: p.Name, + Version: p.Version, + Arch: p.Arch, + Location: p.Location, + Checksum: p.Checksum, + ChecksumType: p.ChecksumType, + Size: p.Size, + CreatedAt: p.CreatedAt, + } +} diff --git a/internal/api/sync_types.go b/internal/api/sync_types.go new file mode 100644 index 0000000..01e0734 --- /dev/null +++ b/internal/api/sync_types.go @@ -0,0 +1,38 @@ +package api + +import "time" + +type SyncSelectionRequest struct { + IDs []int64 `json:"ids"` +} + +type PendingPackageResponse struct { + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + Name string `json:"name"` + Version string `json:"version"` + Arch string `json:"arch"` + Location string `json:"location"` + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` + Size int64 `json:"size"` + CreatedAt time.Time `json:"created_at"` +} + +type ListPendingResponse struct { + Items []PendingPackageResponse `json:"items"` + Total int `json:"total"` +} + +type BlockedPackageResponse struct { + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + Name string `json:"name"` + Location string `json:"location"` + CreatedAt time.Time `json:"created_at"` +} + +type ListBlockedResponse struct { + Items []BlockedPackageResponse `json:"items"` + Total int `json:"total"` +} diff --git a/internal/api/types.go b/internal/api/types.go new file mode 100644 index 0000000..e2f7c26 --- /dev/null +++ b/internal/api/types.go @@ -0,0 +1,44 @@ +package api + +import "time" + +type CreateRepoRequest struct { + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + SyncMode string `json:"sync_mode,omitempty"` +} + +type RepoResponse struct { + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + Frozen bool `json:"frozen"` + SyncMode string `json:"sync_mode"` + CreatedAt time.Time `json:"created_at"` +} + +type ListReposResponse struct { + Items []RepoResponse `json:"items"` + Total int `json:"total"` +} + +type ErrorResponse struct { + Error string `json:"error"` +} + +type StartCloneResponse struct { + JobID int64 `json:"job_id"` + Status string `json:"status"` +} + +type CloneStatusResponse struct { + JobID int64 `json:"job_id"` + RepoID int64 `json:"repo_id"` + Status string `json:"status"` + StartedAt *string `json:"started_at,omitempty"` + FinishedAt *string `json:"finished_at,omitempty"` + Error *string `json:"error,omitempty"` + CreatedAt string `json:"created_at"` +} diff --git a/internal/clone/rpm/cloner.go b/internal/clone/rpm/cloner.go new file mode 100644 index 0000000..5e43e23 --- /dev/null +++ b/internal/clone/rpm/cloner.go @@ -0,0 +1,195 @@ +package rpm + +import ( + "compress/gzip" + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +type Progress struct { + File string + BytesDone int64 +} + +type ProgressFunc func(p Progress) + +type Cloner struct { + SourceURL string + DestDir string + HTTPClient *http.Client + OnProgress ProgressFunc +} + +func New(sourceURL, destDir string) *Cloner { + return &Cloner{ + SourceURL: strings.TrimRight(sourceURL, "/"), + DestDir: destDir, + HTTPClient: &http.Client{Timeout: 30 * time.Minute}, + } +} + +func (c *Cloner) Clone(ctx context.Context) error { + if err := os.MkdirAll(filepath.Join(c.DestDir, "repodata"), 0o755); err != nil { + return fmt.Errorf("create repodata dir: %w", err) + } + if err := os.MkdirAll(filepath.Join(c.DestDir, "Packages"), 0o755); err != nil { + return fmt.Errorf("create Packages dir: %w", err) + } + + repomd, err := c.fetchRepoMD(ctx) + if err != nil { + return fmt.Errorf("fetch repomd.xml: %w", err) + } + + var primaryEntry *RepoMDEntry + for i, entry := range repomd.Data { + if entry.Type == "primary" { + primaryEntry = &repomd.Data[i] + continue + } + if err := c.downloadMetadataFile(ctx, entry); err != nil { + return fmt.Errorf("download metadata %s: %w", entry.Type, err) + } + } + + if primaryEntry == nil { + return fmt.Errorf("no primary metadata found in repomd.xml") + } + + packages, err := c.fetchPrimary(ctx, *primaryEntry) + if err != nil { + return fmt.Errorf("fetch primary.xml: %w", err) + } + + for _, pkg := range packages { + if err := ctx.Err(); err != nil { + return err + } + if err := c.downloadPackage(ctx, pkg); err != nil { + return fmt.Errorf("download package %s: %w", pkg.Name, err) + } + } + + return nil +} + +func (c *Cloner) fetchRepoMD(ctx context.Context) (*RepoMD, error) { + url := c.SourceURL + "/repodata/repomd.xml" + data, err := c.fetchBytes(ctx, url) + if err != nil { + return nil, err + } + + dest := filepath.Join(c.DestDir, "repodata", "repomd.xml") + if err := writeFile(dest, data); err != nil { + return nil, err + } + + var repomd RepoMD + if err := xml.Unmarshal(data, &repomd); err != nil { + return nil, fmt.Errorf("parse repomd.xml: %w", err) + } + return &repomd, nil +} + +func (c *Cloner) fetchPrimary(ctx context.Context, entry RepoMDEntry) ([]Package, error) { + url := c.SourceURL + "/" + entry.Location.Href + data, err := c.fetchBytes(ctx, url) + if err != nil { + return nil, err + } + + if entry.Checksum.Type == "sha256" { + if err := verifyChecksum(data, entry.Checksum.Value); err != nil { + return nil, fmt.Errorf("primary.xml.gz: %w", err) + } + } + + dest := filepath.Join(c.DestDir, entry.Location.Href) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return nil, err + } + if err := writeFile(dest, data); err != nil { + return nil, err + } + + gz, err := gzip.NewReader(strings.NewReader(string(data))) + if err != nil { + return nil, fmt.Errorf("open gzip: %w", err) + } + defer gz.Close() + + var primary PrimaryMetadata + if err := xml.NewDecoder(gz).Decode(&primary); err != nil { + return nil, fmt.Errorf("parse primary.xml: %w", err) + } + return primary.Packages, nil +} + +func (c *Cloner) downloadMetadataFile(ctx context.Context, entry RepoMDEntry) error { + url := c.SourceURL + "/" + entry.Location.Href + dest := filepath.Join(c.DestDir, entry.Location.Href) + + n, err := c.downloadAndVerify(ctx, url, dest, entry.Checksum.Type, entry.Checksum.Value) + if err != nil { + return err + } + if c.OnProgress != nil { + c.OnProgress(Progress{File: entry.Location.Href, BytesDone: n}) + } + return nil +} + +func (c *Cloner) downloadPackage(ctx context.Context, pkg Package) error { + url := c.SourceURL + "/" + pkg.Location.Href + dest := filepath.Join(c.DestDir, pkg.Location.Href) + + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + + n, err := c.downloadAndVerify(ctx, url, dest, pkg.Checksum.Type, pkg.Checksum.Value) + if err != nil { + return err + } + if c.OnProgress != nil { + c.OnProgress(Progress{File: pkg.Location.Href, BytesDone: n}) + } + return nil +} + +func (c *Cloner) downloadAndVerify(ctx context.Context, url, destPath, checksumType, expectedChecksum string) (int64, error) { + return DownloadAndVerify(ctx, c.HTTPClient, url, destPath, checksumType, expectedChecksum) +} + +func (c *Cloner) fetchBytes(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + return io.ReadAll(resp.Body) +} + +func writeFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + diff --git a/internal/clone/rpm/download.go b/internal/clone/rpm/download.go new file mode 100644 index 0000000..bd643fa --- /dev/null +++ b/internal/clone/rpm/download.go @@ -0,0 +1,71 @@ +package rpm + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" +) + +func verifyChecksum(data []byte, expected string) error { + sum := sha256.Sum256(data) + got := hex.EncodeToString(sum[:]) + if got != expected { + return fmt.Errorf("checksum mismatch: got %s, want %s", got, expected) + } + return nil +} + +// DownloadAndVerify fetches url into destPath atomically (temp file + rename), +// verifies the SHA256 checksum if checksumType is "sha256", and returns bytes written. +func DownloadAndVerify(ctx context.Context, httpClient *http.Client, url, destPath, checksumType, expectedChecksum string) (int64, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, err + } + resp, err := httpClient.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + + tmpPath := destPath + ".tmp" + if err := os.MkdirAll(filepath.Dir(tmpPath), 0o755); err != nil { + return 0, err + } + + f, err := os.Create(tmpPath) + if err != nil { + return 0, err + } + + h := sha256.New() + n, err := io.Copy(io.MultiWriter(f, h), resp.Body) + f.Close() + if err != nil { + os.Remove(tmpPath) + return 0, err + } + + if checksumType == "sha256" && expectedChecksum != "" { + got := hex.EncodeToString(h.Sum(nil)) + if got != expectedChecksum { + os.Remove(tmpPath) + return 0, fmt.Errorf("checksum mismatch for %s: got %s, want %s", url, got, expectedChecksum) + } + } + + if err := os.Rename(tmpPath, destPath); err != nil { + os.Remove(tmpPath) + return 0, err + } + return n, nil +} diff --git a/internal/clone/rpm/metadata_gen.go b/internal/clone/rpm/metadata_gen.go new file mode 100644 index 0000000..2e84692 --- /dev/null +++ b/internal/clone/rpm/metadata_gen.go @@ -0,0 +1,212 @@ +package rpm + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "fmt" + "os" + "path/filepath" + "strings" +) + +const PrimarySourceFile = ".primary-source" +const localPrimaryHref = "repodata/primary.xml.gz" + +// filterablePackage captures location for filtering and inner XML for faithful re-emission. +type filterablePackage struct { + Type string `xml:"type,attr"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"http://linux.duke.edu/metadata/common location"` + Inner string `xml:",innerxml"` +} + +type filterableMetadata struct { + XMLName xml.Name `xml:"http://linux.duke.edu/metadata/common metadata"` + Packages []filterablePackage `xml:"http://linux.duke.edu/metadata/common package"` +} + +// RegenerateMetadata filters the upstream primary.xml.gz to only include packages +// present on disk, writes a new primary.xml.gz, and updates repomd.xml. +func RegenerateMetadata(localDir string) error { + sourcePath, err := upstreamPrimaryPath(localDir) + if err != nil { + return err + } + + gzData, err := os.ReadFile(sourcePath) + if err != nil { + return fmt.Errorf("read upstream primary: %w", err) + } + gz, err := gzip.NewReader(bytes.NewReader(gzData)) + if err != nil { + return fmt.Errorf("open gzip: %w", err) + } + var meta filterableMetadata + if err := xml.NewDecoder(gz).Decode(&meta); err != nil { + gz.Close() + return fmt.Errorf("parse primary.xml: %w", err) + } + gz.Close() + + var local []filterablePackage + for _, pkg := range meta.Packages { + dest := filepath.Join(localDir, filepath.FromSlash(pkg.Location.Href)) + if _, statErr := os.Stat(dest); statErr == nil { + local = append(local, pkg) + } + } + + var xmlBuf bytes.Buffer + xmlBuf.WriteString("\n") + fmt.Fprintf(&xmlBuf, + ``, + len(local)) + for _, pkg := range local { + fmt.Fprintf(&xmlBuf, `%s`, pkg.Type, pkg.Inner) + } + xmlBuf.WriteString(``) + xmlBytes := xmlBuf.Bytes() + + var gzBuf bytes.Buffer + gzw := gzip.NewWriter(&gzBuf) + if _, err := gzw.Write(xmlBytes); err != nil { + return fmt.Errorf("gzip write: %w", err) + } + if err := gzw.Close(); err != nil { + return fmt.Errorf("gzip close: %w", err) + } + gzBytes := gzBuf.Bytes() + + openSum := sha256.Sum256(xmlBytes) + gzSum := sha256.Sum256(gzBytes) + + outPath := filepath.Join(localDir, localPrimaryHref) + if err := atomicWrite(outPath, gzBytes); err != nil { + return fmt.Errorf("write primary.xml.gz: %w", err) + } + + return updateRepoMD(localDir, gzSum[:], openSum[:], int64(len(gzBytes)), int64(len(xmlBytes))) +} + +// upstreamPrimaryPath returns the full path to the original upstream primary.xml.gz, +// persisting it in .primary-source so subsequent calls always filter from the full list. +func upstreamPrimaryPath(localDir string) (string, error) { + markerPath := filepath.Join(localDir, "repodata", PrimarySourceFile) + + if data, err := os.ReadFile(markerPath); err == nil { + href := strings.TrimSpace(string(data)) + full := filepath.Join(localDir, filepath.FromSlash(href)) + if _, err := os.Stat(full); err == nil { + return full, nil + } + } + + repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml")) + if err != nil { + return "", fmt.Errorf("read repomd.xml: %w", err) + } + var repomd RepoMD + if err := xml.Unmarshal(repomdData, &repomd); err != nil { + return "", fmt.Errorf("parse repomd.xml: %w", err) + } + for _, entry := range repomd.Data { + if entry.Type == "primary" && entry.Location.Href != localPrimaryHref { + _ = os.WriteFile(markerPath, []byte(entry.Location.Href), 0o644) + return filepath.Join(localDir, filepath.FromSlash(entry.Location.Href)), nil + } + } + return "", fmt.Errorf("upstream primary source not found — repo must be cloned first") +} + +func updateRepoMD(localDir string, gzSum, openSum []byte, gzSize, openSize int64) error { + repomdPath := filepath.Join(localDir, "repodata", "repomd.xml") + repomdData, err := os.ReadFile(repomdPath) + if err != nil { + return fmt.Errorf("read repomd.xml: %w", err) + } + var repomd RepoMD + if err := xml.Unmarshal(repomdData, &repomd); err != nil { + return fmt.Errorf("parse repomd.xml: %w", err) + } + for i, entry := range repomd.Data { + if entry.Type == "primary" { + repomd.Data[i].Location.Href = localPrimaryHref + repomd.Data[i].Checksum = RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(gzSum)} + repomd.Data[i].OpenChecksum = RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(openSum)} + repomd.Data[i].Size = gzSize + repomd.Data[i].OpenSize = openSize + break + } + } + newXML, err := xml.MarshalIndent(repomd, "", " ") + if err != nil { + return fmt.Errorf("marshal repomd.xml: %w", err) + } + return atomicWrite(repomdPath, append([]byte(xml.Header), newXML...)) +} + +// InitEmptyRepo creates the directory structure and empty RPM metadata for a new repo. +// It is idempotent: if repomd.xml already exists it is left untouched. +func InitEmptyRepo(localDir string) error { + for _, d := range []string{ + filepath.Join(localDir, "repodata"), + filepath.Join(localDir, "Packages"), + } { + if err := os.MkdirAll(d, 0o755); err != nil { + return err + } + } + + repomdPath := filepath.Join(localDir, "repodata", "repomd.xml") + if _, err := os.Stat(repomdPath); err == nil { + return nil // already initialised + } + + emptyXML := []byte("\n" + + ``) + + var gzBuf bytes.Buffer + gzw := gzip.NewWriter(&gzBuf) + _, _ = gzw.Write(emptyXML) + _ = gzw.Close() + gzBytes := gzBuf.Bytes() + + openSum := sha256.Sum256(emptyXML) + gzSum := sha256.Sum256(gzBytes) + + if err := atomicWrite(filepath.Join(localDir, localPrimaryHref), gzBytes); err != nil { + return fmt.Errorf("write empty primary.xml.gz: %w", err) + } + + repomd := RepoMD{ + Data: []RepoMDEntry{{ + Type: "primary", + Location: RepoMDLocation{Href: localPrimaryHref}, + Checksum: RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(gzSum[:])}, + OpenChecksum: RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(openSum[:])}, + Size: int64(len(gzBytes)), + OpenSize: int64(len(emptyXML)), + }}, + } + newXML, err := xml.MarshalIndent(repomd, "", " ") + if err != nil { + return err + } + return atomicWrite(repomdPath, append([]byte(xml.Header), newXML...)) +} + +func atomicWrite(path string, data []byte) error { + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return err + } + return nil +} diff --git a/internal/clone/rpm/primary.go b/internal/clone/rpm/primary.go new file mode 100644 index 0000000..62aa1d8 --- /dev/null +++ b/internal/clone/rpm/primary.go @@ -0,0 +1,41 @@ +package rpm + +import "encoding/xml" + +const primaryNS = "http://linux.duke.edu/metadata/common" + +type PrimaryMetadata struct { + XMLName xml.Name `xml:"http://linux.duke.edu/metadata/common metadata"` + Packages []Package `xml:"http://linux.duke.edu/metadata/common package"` +} + +type Package struct { + Type string `xml:"type,attr"` + Name string `xml:"http://linux.duke.edu/metadata/common name"` + Arch string `xml:"http://linux.duke.edu/metadata/common arch"` + Version PackageVersion `xml:"http://linux.duke.edu/metadata/common version"` + Checksum PackageChecksum `xml:"http://linux.duke.edu/metadata/common checksum"` + Location PackageLocation `xml:"http://linux.duke.edu/metadata/common location"` + Size PackageSize `xml:"http://linux.duke.edu/metadata/common size"` +} + +type PackageVersion struct { + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` +} + +type PackageChecksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` +} + +type PackageLocation struct { + Href string `xml:"href,attr"` +} + +type PackageSize struct { + Package int64 `xml:"package,attr"` + Installed int64 `xml:"installed,attr"` + Archive int64 `xml:"archive,attr"` +} diff --git a/internal/clone/rpm/repomd.go b/internal/clone/rpm/repomd.go new file mode 100644 index 0000000..3802a8e --- /dev/null +++ b/internal/clone/rpm/repomd.go @@ -0,0 +1,26 @@ +package rpm + +import "encoding/xml" + +type RepoMD struct { + XMLName xml.Name `xml:"repomd"` + Data []RepoMDEntry `xml:"data"` +} + +type RepoMDEntry struct { + Type string `xml:"type,attr"` + Location RepoMDLocation `xml:"location"` + Checksum RepoMDChecksum `xml:"checksum"` + Size int64 `xml:"size"` + OpenChecksum RepoMDChecksum `xml:"open-checksum"` + OpenSize int64 `xml:"open-size"` +} + +type RepoMDLocation struct { + Href string `xml:"href,attr"` +} + +type RepoMDChecksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` +} diff --git a/internal/clone/rpm/scanner.go b/internal/clone/rpm/scanner.go new file mode 100644 index 0000000..374c038 --- /dev/null +++ b/internal/clone/rpm/scanner.go @@ -0,0 +1,143 @@ +package rpm + +import ( + "compress/gzip" + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +type NewPackage struct { + Name string + Version string + Arch string + Location string + Checksum string + ChecksumType string + Size int64 +} + +type Scanner struct { + SourceURL string + LocalDir string + HTTPClient *http.Client +} + +func NewScanner(sourceURL, localDir string) *Scanner { + return &Scanner{ + SourceURL: strings.TrimRight(sourceURL, "/"), + LocalDir: localDir, + HTTPClient: &http.Client{Timeout: 5 * time.Minute}, + } +} + +// Scan fetches the remote package list and returns packages not present on disk. +func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) { + repomd, err := sc.fetchRepoMD(ctx) + if err != nil { + return nil, fmt.Errorf("fetch repomd.xml: %w", err) + } + + var primaryEntry *RepoMDEntry + for i := range repomd.Data { + if repomd.Data[i].Type == "primary" { + primaryEntry = &repomd.Data[i] + break + } + } + if primaryEntry == nil { + return nil, fmt.Errorf("no primary entry in repomd.xml") + } + + packages, err := sc.fetchPrimary(ctx, *primaryEntry) + if err != nil { + return nil, fmt.Errorf("fetch primary.xml: %w", err) + } + + var missing []NewPackage + for _, pkg := range packages { + localPath := filepath.Join(sc.LocalDir, filepath.FromSlash(pkg.Location.Href)) + if _, err := os.Stat(localPath); os.IsNotExist(err) { + ver := pkg.Version.Ver + "-" + pkg.Version.Rel + if pkg.Version.Epoch != "0" && pkg.Version.Epoch != "" { + ver = pkg.Version.Epoch + ":" + ver + } + missing = append(missing, NewPackage{ + Name: pkg.Name, + Version: ver, + Arch: pkg.Arch, + Location: pkg.Location.Href, + Checksum: pkg.Checksum.Value, + ChecksumType: pkg.Checksum.Type, + Size: pkg.Size.Package, + }) + } + } + return missing, nil +} + +func (sc *Scanner) fetchRepoMD(ctx context.Context) (*RepoMD, error) { + data, err := sc.fetchBytes(ctx, sc.SourceURL+"/repodata/repomd.xml") + if err != nil { + return nil, err + } + var repomd RepoMD + if err := xml.Unmarshal(data, &repomd); err != nil { + return nil, fmt.Errorf("parse repomd.xml: %w", err) + } + return &repomd, nil +} + +func (sc *Scanner) fetchPrimary(ctx context.Context, entry RepoMDEntry) ([]Package, error) { + data, err := sc.fetchBytes(ctx, sc.SourceURL+"/"+entry.Location.Href) + if err != nil { + return nil, err + } + if entry.Checksum.Type == "sha256" { + if err := verifyChecksum(data, entry.Checksum.Value); err != nil { + return nil, fmt.Errorf("primary.xml.gz: %w", err) + } + } + + // Save upstream primary to disk so RegenerateMetadata can use it as source. + dest := filepath.Join(sc.LocalDir, filepath.FromSlash(entry.Location.Href)) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err == nil { + if err := writeFile(dest, data); err == nil { + marker := filepath.Join(sc.LocalDir, "repodata", PrimarySourceFile) + _ = os.WriteFile(marker, []byte(entry.Location.Href), 0o644) + } + } + + gz, err := gzip.NewReader(strings.NewReader(string(data))) + if err != nil { + return nil, fmt.Errorf("open gzip: %w", err) + } + defer gz.Close() + var primary PrimaryMetadata + if err := xml.NewDecoder(gz).Decode(&primary); err != nil { + return nil, fmt.Errorf("parse primary.xml: %w", err) + } + return primary.Packages, nil +} + +func (sc *Scanner) fetchBytes(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := sc.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + return io.ReadAll(resp.Body) +} diff --git a/internal/core/clone.go b/internal/core/clone.go new file mode 100644 index 0000000..420d4e0 --- /dev/null +++ b/internal/core/clone.go @@ -0,0 +1,83 @@ +package core + +import ( + "context" + "errors" + "fmt" + "log" + "path/filepath" + + "github.com/syonad/clonepack/internal/clone/rpm" + "github.com/syonad/clonepack/internal/store" +) + +var ErrCloneAlreadyRunning = errors.New("a clone job is already running for this repo") + +type CloneService struct { + repoStore store.RepoStore + jobStore store.CloneJobStore + dataDir string +} + +func NewCloneService(repoStore store.RepoStore, jobStore store.CloneJobStore, dataDir string) *CloneService { + return &CloneService{repoStore: repoStore, jobStore: jobStore, dataDir: dataDir} +} + +func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, error) { + repo, err := s.repoStore.GetRepo(ctx, repoID) + if err != nil { + return 0, err + } + if repo.Type != "rpm" { + return 0, fmt.Errorf("clone is only supported for rpm repositories") + } + + running, err := s.jobStore.HasRunningCloneJob(ctx, repoID) + if err != nil { + return 0, err + } + if running { + return 0, ErrCloneAlreadyRunning + } + + jobID, err := s.jobStore.CreateCloneJob(ctx, repoID) + if err != nil { + return 0, err + } + + go s.runClone(repoID, jobID, repo.SourceURL, repo.Type) + return jobID, nil +} + +func (s *CloneService) GetLatestCloneJob(ctx context.Context, repoID int64) (*store.CloneJob, error) { + return s.jobStore.GetLatestCloneJob(ctx, repoID) +} + +func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string) { + ctx := context.Background() + + if err := s.jobStore.MarkCloneJobStarted(ctx, jobID); err != nil { + log.Printf("clone job %d: failed to mark started: %v", jobID, err) + return + } + + destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repoType) + cloner := rpm.New(sourceURL, destDir) + cloner.OnProgress = func(p rpm.Progress) { + log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone) + } + + if err := cloner.Clone(ctx); err != nil { + errMsg := err.Error() + _ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobFailed, &errMsg) + log.Printf("clone job %d: failed: %v", jobID, err) + return + } + + if err := rpm.RegenerateMetadata(destDir); err != nil { + log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err) + } + + _ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobCompleted, nil) + log.Printf("clone job %d: completed", jobID) +} diff --git a/internal/core/errors.go b/internal/core/errors.go new file mode 100644 index 0000000..3922eee --- /dev/null +++ b/internal/core/errors.go @@ -0,0 +1,5 @@ +package core + +import "github.com/syonad/clonepack/internal/store" + +var ErrNotFound = store.ErrNotFound diff --git a/internal/core/repo.go b/internal/core/repo.go new file mode 100644 index 0000000..cfd5f42 --- /dev/null +++ b/internal/core/repo.go @@ -0,0 +1,82 @@ +package core + +import ( + "context" + "fmt" + "log" + "path/filepath" + + rpmclone "github.com/syonad/clonepack/internal/clone/rpm" + "github.com/syonad/clonepack/internal/store" +) + +var validTypes = map[string]bool{"apt": true, "rpm": true, "docker": true, "binary": true} + +type RepoService struct { + store store.RepoStore + dataDir string +} + +func NewRepoService(s store.RepoStore, dataDir string) *RepoService { + return &RepoService{store: s, dataDir: dataDir} +} + +var validSyncModes = map[string]bool{"auto": true, "manual": true} + +type CreateRepoInput struct { + Name string `json:"name"` + Type string `json:"type"` + SourceURL string `json:"source_url"` + SyncMode string `json:"sync_mode"` +} + +func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Repo, error) { + if in.Name == "" { + return nil, fmt.Errorf("name is required") + } + if !validTypes[in.Type] { + return nil, fmt.Errorf("invalid type: %s (must be apt, rpm, docker, or binary)", in.Type) + } + if in.SourceURL == "" { + return nil, fmt.Errorf("source_url is required") + } + if in.SyncMode == "" { + in.SyncMode = "auto" + } else if !validSyncModes[in.SyncMode] { + return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", in.SyncMode) + } + + r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode} + id, err := s.store.CreateRepo(ctx, r) + if err != nil { + return nil, err + } + repo, err := s.store.GetRepo(ctx, id) + if err != nil { + return nil, err + } + s.initStorage(repo) + return repo, nil +} + +func (s *RepoService) initStorage(repo *store.Repo) { + switch repo.Type { + case "rpm": + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), "rpm") + if err := rpmclone.InitEmptyRepo(localDir); err != nil { + log.Printf("init storage for repo %d: %v", repo.ID, err) + } + } +} + +func (s *RepoService) List(ctx context.Context) ([]store.Repo, error) { + return s.store.ListRepos(ctx) +} + +func (s *RepoService) Get(ctx context.Context, id int64) (*store.Repo, error) { + return s.store.GetRepo(ctx, id) +} + +func (s *RepoService) Delete(ctx context.Context, id int64) error { + return s.store.DeleteRepo(ctx, id) +} diff --git a/internal/core/snapshot.go b/internal/core/snapshot.go new file mode 100644 index 0000000..15c515f --- /dev/null +++ b/internal/core/snapshot.go @@ -0,0 +1,260 @@ +package core + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/xml" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + rpmclone "github.com/syonad/clonepack/internal/clone/rpm" + "github.com/syonad/clonepack/internal/store" +) + +type SnapshotService struct { + snapshotStore store.SnapshotStore + repoStore store.RepoStore + dataDir string + httpClient *http.Client +} + +func NewSnapshotService( + snapshotStore store.SnapshotStore, + repoStore store.RepoStore, + dataDir string, +) *SnapshotService { + return &SnapshotService{ + snapshotStore: snapshotStore, + repoStore: repoStore, + dataDir: dataDir, + httpClient: &http.Client{Timeout: 30 * time.Minute}, + } +} + +type SnapshotDiff struct { + From *store.Snapshot + To *store.Snapshot + Added []store.SnapshotPackage + Removed []store.SnapshotPackage + Unchanged int +} + +func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label string) (int64, error) { + if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + return 0, err + } + + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") + + repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml")) + if err != nil { + return 0, fmt.Errorf("read repomd.xml: %w", err) + } + + var repomd rpmclone.RepoMD + if err := xml.Unmarshal(repomdData, &repomd); err != nil { + return 0, fmt.Errorf("parse repomd.xml: %w", err) + } + + var primaryHref string + for _, entry := range repomd.Data { + if entry.Type == "primary" { + primaryHref = entry.Location.Href + break + } + } + if primaryHref == "" { + return 0, fmt.Errorf("no primary entry in repomd.xml") + } + + gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref))) + if err != nil { + return 0, fmt.Errorf("read primary.xml.gz: %w", err) + } + + gz, err := gzip.NewReader(bytes.NewReader(gzData)) + if err != nil { + return 0, fmt.Errorf("open gzip: %w", err) + } + defer gz.Close() + + var primary rpmclone.PrimaryMetadata + if err := xml.NewDecoder(gz).Decode(&primary); err != nil { + return 0, fmt.Errorf("parse primary.xml: %w", err) + } + + pkgs := make([]store.SnapshotPackage, len(primary.Packages)) + for i, p := range primary.Packages { + pkgs[i] = store.SnapshotPackage{ + Name: p.Name, + Version: p.Version.Ver + "-" + p.Version.Rel, + Arch: p.Arch, + Location: p.Location.Href, + Checksum: p.Checksum.Value, + ChecksumType: p.Checksum.Type, + Size: p.Size.Package, + } + } + + snapID, err := s.snapshotStore.CreateSnapshot(ctx, repoID, label) + if err != nil { + return 0, err + } + if len(pkgs) > 0 { + if err := s.snapshotStore.AddSnapshotPackages(ctx, snapID, pkgs); err != nil { + return 0, err + } + } + return snapID, nil +} + +func (s *SnapshotService) List(ctx context.Context, repoID int64) ([]store.Snapshot, error) { + if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + return nil, err + } + return s.snapshotStore.ListSnapshots(ctx, repoID) +} + +func (s *SnapshotService) Get(ctx context.Context, repoID, id int64) (*store.Snapshot, []store.SnapshotPackage, error) { + snap, err := s.snapshotStore.GetSnapshot(ctx, id) + if err != nil { + return nil, nil, err + } + if snap.RepoID != repoID { + return nil, nil, store.ErrNotFound + } + pkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, id) + if err != nil { + return nil, nil, err + } + return snap, pkgs, nil +} + +func (s *SnapshotService) Delete(ctx context.Context, repoID, id int64) error { + snap, err := s.snapshotStore.GetSnapshot(ctx, id) + if err != nil { + return err + } + if snap.RepoID != repoID { + return store.ErrNotFound + } + return s.snapshotStore.DeleteSnapshot(ctx, id) +} + +func (s *SnapshotService) Diff(ctx context.Context, repoID, fromID, toID int64) (*SnapshotDiff, error) { + from, err := s.snapshotStore.GetSnapshot(ctx, fromID) + if err != nil { + return nil, err + } + to, err := s.snapshotStore.GetSnapshot(ctx, toID) + if err != nil { + return nil, err + } + if from.RepoID != repoID || to.RepoID != repoID { + return nil, fmt.Errorf("snapshots do not belong to repo %d", repoID) + } + + fromPkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, fromID) + if err != nil { + return nil, err + } + toPkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, toID) + if err != nil { + return nil, err + } + + fromSet := make(map[string]struct{}, len(fromPkgs)) + for _, p := range fromPkgs { + fromSet[p.Name+"|"+p.Version+"|"+p.Arch] = struct{}{} + } + toSet := make(map[string]struct{}, len(toPkgs)) + for _, p := range toPkgs { + toSet[p.Name+"|"+p.Version+"|"+p.Arch] = struct{}{} + } + + diff := &SnapshotDiff{From: from, To: to} + for _, p := range toPkgs { + if _, ok := fromSet[p.Name+"|"+p.Version+"|"+p.Arch]; ok { + diff.Unchanged++ + } else { + diff.Added = append(diff.Added, p) + } + } + for _, p := range fromPkgs { + if _, ok := toSet[p.Name+"|"+p.Version+"|"+p.Arch]; !ok { + diff.Removed = append(diff.Removed, p) + } + } + return diff, nil +} + +func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64) error { + repo, err := s.repoStore.GetRepo(ctx, repoID) + if err != nil { + return err + } + snap, err := s.snapshotStore.GetSnapshot(ctx, snapshotID) + if err != nil { + return err + } + if snap.RepoID != repoID { + return store.ErrNotFound + } + + pkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, snapshotID) + if err != nil { + return err + } + + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") + + snapSet := make(map[string]store.SnapshotPackage, len(pkgs)) + for _, p := range pkgs { + snapSet[p.Location] = p + } + + var errs []error + if err := filepath.WalkDir(localDir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || filepath.Ext(path) != ".rpm" { + return err + } + rel, _ := filepath.Rel(localDir, path) + rel = filepath.ToSlash(rel) + if _, ok := snapSet[rel]; !ok { + if rmErr := os.Remove(path); rmErr != nil { + errs = append(errs, rmErr) + } + } + return nil + }); err != nil { + errs = append(errs, err) + } + + for _, p := range pkgs { + dest := filepath.Join(localDir, filepath.FromSlash(p.Location)) + if _, err := os.Stat(dest); err == nil { + continue + } + url := strings.TrimRight(repo.SourceURL, "/") + "/" + p.Location + if _, dlErr := rpmclone.DownloadAndVerify(ctx, s.httpClient, url, dest, p.ChecksumType, p.Checksum); dlErr != nil { + errs = append(errs, fmt.Errorf("re-download %s: %w", p.Name, dlErr)) + } + } + + if err := errors.Join(errs...); err != nil { + return err + } + + if repo.Type == "rpm" { + if err := rpmclone.RegenerateMetadata(localDir); err != nil { + return fmt.Errorf("regenerate metadata: %w", err) + } + } + return nil +} diff --git a/internal/core/sync.go b/internal/core/sync.go new file mode 100644 index 0000000..ddd3c54 --- /dev/null +++ b/internal/core/sync.go @@ -0,0 +1,268 @@ +package core + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + rpmclone "github.com/syonad/clonepack/internal/clone/rpm" + "github.com/syonad/clonepack/internal/store" +) + +type SyncService struct { + repoStore store.RepoStore + pendingStore store.PendingPackageStore + blockedStore store.BlockedPackageStore + cloneSvc *CloneService + snapshotSvc *SnapshotService + dataDir string + httpClient *http.Client +} + +func NewSyncService( + repoStore store.RepoStore, + pendingStore store.PendingPackageStore, + blockedStore store.BlockedPackageStore, + cloneSvc *CloneService, + snapshotSvc *SnapshotService, + dataDir string, +) *SyncService { + return &SyncService{ + repoStore: repoStore, + pendingStore: pendingStore, + blockedStore: blockedStore, + cloneSvc: cloneSvc, + snapshotSvc: snapshotSvc, + dataDir: dataDir, + httpClient: &http.Client{Timeout: 5 * time.Minute}, + } +} + +func (s *SyncService) ValidateRepo(ctx context.Context, repoID int64) error { + _, err := s.repoStore.GetRepo(ctx, repoID) + return err +} + +func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error { + repo, err := s.repoStore.GetRepo(ctx, repoID) + if err != nil { + return err + } + if repo.Type != "rpm" { + return fmt.Errorf("sync only supported for rpm repos") + } + + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") + scanner := rpmclone.NewScanner(repo.SourceURL, localDir) + newPkgs, err := scanner.Scan(ctx) + if err != nil { + return fmt.Errorf("scan repo %d: %w", repoID, err) + } + + if len(newPkgs) == 0 { + log.Printf("sync: repo %d is up to date", repoID) + return nil + } + + blocked, err := s.blockedStore.ListBlocked(ctx, repoID) + if err != nil { + return fmt.Errorf("list blocked for repo %d: %w", repoID, err) + } + blockedLocations := make(map[string]bool, len(blocked)) + blockedNames := make(map[string]bool, len(blocked)) + for _, b := range blocked { + blockedLocations[b.Location] = true + if b.Name != "" { + blockedNames[b.Name] = true + } + } + filtered := newPkgs[:0] + for _, p := range newPkgs { + if !blockedLocations[p.Location] && !blockedNames[p.Name] { + filtered = append(filtered, p) + } + } + newPkgs = filtered + + if len(newPkgs) == 0 { + log.Printf("sync: repo %d is up to date (all new packages are blocked)", repoID) + return nil + } + log.Printf("sync: repo %d has %d new package(s)", repoID, len(newPkgs)) + + switch repo.SyncMode { + case "auto": + _, err = s.cloneSvc.StartClone(ctx, repoID) + if err != nil && err != ErrCloneAlreadyRunning { + return fmt.Errorf("start clone for repo %d: %w", repoID, err) + } + case "manual": + pending := make([]store.PendingPackage, len(newPkgs)) + for i, p := range newPkgs { + pending[i] = store.PendingPackage{ + RepoID: repoID, + Name: p.Name, + Version: p.Version, + Arch: p.Arch, + Location: p.Location, + Checksum: p.Checksum, + ChecksumType: p.ChecksumType, + Size: p.Size, + } + } + if err := s.pendingStore.UpsertPending(ctx, pending); err != nil { + return fmt.Errorf("upsert pending for repo %d: %w", repoID, err) + } + default: + return fmt.Errorf("unknown sync_mode %q for repo %d", repo.SyncMode, repoID) + } + return nil +} + +func (s *SyncService) ListPending(ctx context.Context, repoID int64) ([]store.PendingPackage, error) { + if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + return nil, err + } + return s.pendingStore.ListPending(ctx, repoID) +} + +func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []int64) error { + repo, err := s.repoStore.GetRepo(ctx, repoID) + if err != nil { + return err + } + + all, err := s.pendingStore.ListPending(ctx, repoID) + if err != nil { + return err + } + + wanted := make(map[int64]bool, len(ids)) + for _, id := range ids { + wanted[id] = true + } + + localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm") + var downloadErrors []error + var anyApproved bool + for _, pkg := range all { + if !wanted[pkg.ID] { + continue + } + url := strings.TrimRight(repo.SourceURL, "/") + "/" + pkg.Location + dest := filepath.Join(localDir, filepath.FromSlash(pkg.Location)) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + downloadErrors = append(downloadErrors, fmt.Errorf("mkdirall %s: %w", pkg.Location, err)) + continue + } + if _, err := rpmclone.DownloadAndVerify(ctx, s.httpClient, url, dest, pkg.ChecksumType, pkg.Checksum); err != nil { + downloadErrors = append(downloadErrors, fmt.Errorf("download %s: %w", pkg.Name, err)) + continue + } + // Supprime immédiatement de la liste — même si la suite échoue, le paquet est acquis. + if err := s.pendingStore.DeletePending(ctx, []int64{pkg.ID}); err != nil { + log.Printf("delete pending %d: %v", pkg.ID, err) + } + anyApproved = true + } + + if anyApproved { + if repo.Type == "rpm" { + if err := rpmclone.RegenerateMetadata(localDir); err != nil { + log.Printf("metadata regeneration for repo %d failed: %v", repoID, err) + } + } + label := "auto-" + time.Now().UTC().Format(time.RFC3339) + if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil { + log.Printf("auto-snapshot for repo %d failed: %v", repoID, err) + } + } + + return errors.Join(downloadErrors...) +} + +func (s *SyncService) BlockPackages(ctx context.Context, repoID int64, pendingIDs []int64) error { + if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + return err + } + all, err := s.pendingStore.ListPending(ctx, repoID) + if err != nil { + return err + } + wanted := make(map[int64]bool, len(pendingIDs)) + for _, id := range pendingIDs { + wanted[id] = true + } + var toBlock []store.BlockedPackage + var toDelete []int64 + for _, p := range all { + if wanted[p.ID] { + toBlock = append(toBlock, store.BlockedPackage{Name: p.Name, Location: p.Location}) + toDelete = append(toDelete, p.ID) + } + } + if len(toBlock) == 0 { + return nil + } + if err := s.blockedStore.BlockPackages(ctx, repoID, toBlock); err != nil { + return err + } + return s.pendingStore.DeletePending(ctx, toDelete) +} + +func (s *SyncService) UnblockPackages(ctx context.Context, repoID int64, ids []int64) error { + if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + return err + } + return s.blockedStore.UnblockPackages(ctx, repoID, ids) +} + +func (s *SyncService) ListBlocked(ctx context.Context, repoID int64) ([]store.BlockedPackage, error) { + if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + return nil, err + } + return s.blockedStore.ListBlocked(ctx, repoID) +} + +func (s *SyncService) RejectPending(ctx context.Context, repoID int64, ids []int64) error { + if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil { + return err + } + return s.pendingStore.DeletePending(ctx, ids) +} + +func (s *SyncService) StartScheduler(ctx context.Context, interval time.Duration) { + go func() { + log.Printf("sync scheduler started (interval=%s)", interval) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + log.Printf("sync scheduler stopped") + return + case <-ticker.C: + s.runScheduledScan(ctx) + } + } + }() +} + +func (s *SyncService) runScheduledScan(ctx context.Context) { + repos, err := s.repoStore.ListRepos(ctx) + if err != nil { + log.Printf("sync scheduler: list repos error: %v", err) + return + } + for _, repo := range repos { + if err := s.ScanRepo(ctx, repo.ID); err != nil { + log.Printf("sync scheduler: scan repo %d error: %v", repo.ID, err) + } + } +} diff --git a/internal/store/blocked_package.go b/internal/store/blocked_package.go new file mode 100644 index 0000000..9e15c26 --- /dev/null +++ b/internal/store/blocked_package.go @@ -0,0 +1,78 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" +) + +type BlockedPackage struct { + ID int64 + RepoID int64 + Name string + Location string + CreatedAt time.Time +} + +type BlockedPackageStore interface { + BlockPackages(ctx context.Context, repoID int64, pkgs []BlockedPackage) error + UnblockPackages(ctx context.Context, repoID int64, ids []int64) error + ListBlocked(ctx context.Context, repoID int64) ([]BlockedPackage, error) +} + +type SQLiteBlockedPackageStore struct { + db *sql.DB +} + +func NewBlockedPackageStore(db *sql.DB) *SQLiteBlockedPackageStore { + return &SQLiteBlockedPackageStore{db: db} +} + +func (s *SQLiteBlockedPackageStore) BlockPackages(ctx context.Context, repoID int64, pkgs []BlockedPackage) error { + for _, p := range pkgs { + _, err := s.db.ExecContext(ctx, + `INSERT OR IGNORE INTO blocked_packages (repo_id, name, location) VALUES (?, ?, ?)`, + repoID, p.Name, p.Location) + if err != nil { + return err + } + } + return nil +} + +func (s *SQLiteBlockedPackageStore) UnblockPackages(ctx context.Context, repoID int64, ids []int64) error { + if len(ids) == 0 { + return nil + } + placeholders := strings.Join(strings.Fields(strings.Repeat("? ", len(ids))), ", ") + query := fmt.Sprintf("DELETE FROM blocked_packages WHERE repo_id = ? AND id IN (%s)", placeholders) + args := make([]any, 0, len(ids)+1) + args = append(args, repoID) + for _, id := range ids { + args = append(args, id) + } + _, err := s.db.ExecContext(ctx, query, args...) + return err +} + +func (s *SQLiteBlockedPackageStore) ListBlocked(ctx context.Context, repoID int64) ([]BlockedPackage, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, repo_id, name, location, created_at FROM blocked_packages WHERE repo_id = ? ORDER BY id ASC`, + repoID) + if err != nil { + return nil, err + } + defer rows.Close() + + var pkgs []BlockedPackage + for rows.Next() { + var p BlockedPackage + if err := rows.Scan(&p.ID, &p.RepoID, &p.Name, &p.Location, &p.CreatedAt); err != nil { + return nil, err + } + pkgs = append(pkgs, p) + } + return pkgs, rows.Err() +} diff --git a/internal/store/clone_job.go b/internal/store/clone_job.go new file mode 100644 index 0000000..60f755e --- /dev/null +++ b/internal/store/clone_job.go @@ -0,0 +1,109 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "time" +) + +type CloneJobStatus string + +const ( + CloneJobPending CloneJobStatus = "pending" + CloneJobRunning CloneJobStatus = "running" + CloneJobCompleted CloneJobStatus = "completed" + CloneJobFailed CloneJobStatus = "failed" +) + +type CloneJob struct { + ID int64 + RepoID int64 + Status CloneJobStatus + StartedAt *time.Time + FinishedAt *time.Time + Error *string + CreatedAt time.Time +} + +type CloneJobStore interface { + CreateCloneJob(ctx context.Context, repoID int64) (int64, error) + GetCloneJob(ctx context.Context, id int64) (*CloneJob, error) + GetLatestCloneJob(ctx context.Context, repoID int64) (*CloneJob, error) + HasRunningCloneJob(ctx context.Context, repoID int64) (bool, error) + MarkCloneJobStarted(ctx context.Context, id int64) error + MarkCloneJobFinished(ctx context.Context, id int64, status CloneJobStatus, errMsg *string) error +} + +type SQLiteCloneJobStore struct { + db *sql.DB +} + +func NewCloneJobStore(db *sql.DB) *SQLiteCloneJobStore { + return &SQLiteCloneJobStore{db: db} +} + +func (s *SQLiteCloneJobStore) CreateCloneJob(ctx context.Context, repoID int64) (int64, error) { + res, err := s.db.ExecContext(ctx, `INSERT INTO clone_jobs (repo_id) VALUES (?)`, repoID) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func (s *SQLiteCloneJobStore) GetCloneJob(ctx context.Context, id int64) (*CloneJob, error) { + row := s.db.QueryRowContext(ctx, + `SELECT id, repo_id, status, started_at, finished_at, error, created_at FROM clone_jobs WHERE id = ?`, id) + return scanCloneJob(row) +} + +func (s *SQLiteCloneJobStore) GetLatestCloneJob(ctx context.Context, repoID int64) (*CloneJob, error) { + row := s.db.QueryRowContext(ctx, + `SELECT id, repo_id, status, started_at, finished_at, error, created_at FROM clone_jobs WHERE repo_id = ? ORDER BY created_at DESC LIMIT 1`, repoID) + return scanCloneJob(row) +} + +func (s *SQLiteCloneJobStore) HasRunningCloneJob(ctx context.Context, repoID int64) (bool, error) { + var count int + err := s.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM clone_jobs WHERE repo_id = ? AND status IN ('pending','running')`, repoID, + ).Scan(&count) + return count > 0, err +} + +func (s *SQLiteCloneJobStore) MarkCloneJobStarted(ctx context.Context, id int64) error { + _, err := s.db.ExecContext(ctx, + `UPDATE clone_jobs SET status='running', started_at=CURRENT_TIMESTAMP WHERE id=?`, id) + return err +} + +func (s *SQLiteCloneJobStore) MarkCloneJobFinished(ctx context.Context, id int64, status CloneJobStatus, errMsg *string) error { + _, err := s.db.ExecContext(ctx, + `UPDATE clone_jobs SET status=?, finished_at=CURRENT_TIMESTAMP, error=? WHERE id=?`, + status, errMsg, id) + return err +} + +func scanCloneJob(row *sql.Row) (*CloneJob, error) { + var j CloneJob + var startedAt, finishedAt sql.NullTime + var errMsg sql.NullString + + err := row.Scan(&j.ID, &j.RepoID, &j.Status, &startedAt, &finishedAt, &errMsg, &j.CreatedAt) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + if startedAt.Valid { + j.StartedAt = &startedAt.Time + } + if finishedAt.Valid { + j.FinishedAt = &finishedAt.Time + } + if errMsg.Valid { + j.Error = &errMsg.String + } + return &j, nil +} diff --git a/internal/store/db.go b/internal/store/db.go new file mode 100644 index 0000000..cb504f6 --- /dev/null +++ b/internal/store/db.go @@ -0,0 +1,88 @@ +package store + +import ( + "database/sql" + _ "embed" + "fmt" + + "github.com/syonad/clonepack/config" + _ "modernc.org/sqlite" +) + +//go:embed migrations/000001_init.up.sql +var migration001 string + +//go:embed migrations/000002_clone_jobs.up.sql +var migration002 string + +//go:embed migrations/000003_sync.up.sql +var migration003 string + +//go:embed migrations/000004_snapshot_packages.up.sql +var migration004 string + +//go:embed migrations/000005_blocked_packages.up.sql +var migration005 string + +//go:embed migrations/000006_blocked_packages_name.up.sql +var migration006 string + +func Open(cfg config.DBConfig) (*sql.DB, error) { + db, err := sql.Open("sqlite", cfg.Path) + if err != nil { + return nil, fmt.Errorf("open db: %w", err) + } + + if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil { + return nil, fmt.Errorf("set WAL mode: %w", err) + } + if _, err := db.Exec("PRAGMA foreign_keys=ON"); err != nil { + return nil, fmt.Errorf("enable foreign keys: %w", err) + } + + if err := runMigrations(db); err != nil { + return nil, fmt.Errorf("migrations: %w", err) + } + + return db, nil +} + +func runMigrations(db *sql.DB) error { + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + )`) + if err != nil { + return err + } + + migrations := []struct { + version int + sql string + }{ + {1, migration001}, + {2, migration002}, + {3, migration003}, + {4, migration004}, + {5, migration005}, + {6, migration006}, + } + + for _, m := range migrations { + var count int + row := db.QueryRow("SELECT COUNT(*) FROM schema_migrations WHERE version = ?", m.version) + if err := row.Scan(&count); err != nil { + return err + } + if count > 0 { + continue + } + if _, err := db.Exec(m.sql); err != nil { + return fmt.Errorf("migration %d: %w", m.version, err) + } + if _, err := db.Exec("INSERT INTO schema_migrations (version) VALUES (?)", m.version); err != nil { + return err + } + } + return nil +} diff --git a/internal/store/migrations/000001_init.down.sql b/internal/store/migrations/000001_init.down.sql new file mode 100644 index 0000000..364037c --- /dev/null +++ b/internal/store/migrations/000001_init.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS snapshots; +DROP TABLE IF EXISTS artifacts; +DROP TABLE IF EXISTS repos; diff --git a/internal/store/migrations/000001_init.up.sql b/internal/store/migrations/000001_init.up.sql new file mode 100644 index 0000000..1adbbe9 --- /dev/null +++ b/internal/store/migrations/000001_init.up.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS repos ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + type TEXT NOT NULL CHECK(type IN ('apt','rpm','docker','binary')), + source_url TEXT NOT NULL, + frozen BOOLEAN NOT NULL DEFAULT FALSE, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS artifacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE, + name TEXT NOT NULL, + version TEXT NOT NULL, + path TEXT NOT NULL, + checksum TEXT NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS snapshots ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE, + label TEXT NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/internal/store/migrations/000002_clone_jobs.down.sql b/internal/store/migrations/000002_clone_jobs.down.sql new file mode 100644 index 0000000..e7f7aa8 --- /dev/null +++ b/internal/store/migrations/000002_clone_jobs.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS clone_jobs; diff --git a/internal/store/migrations/000002_clone_jobs.up.sql b/internal/store/migrations/000002_clone_jobs.up.sql new file mode 100644 index 0000000..0777ba6 --- /dev/null +++ b/internal/store/migrations/000002_clone_jobs.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS clone_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE, + status TEXT NOT NULL CHECK(status IN ('pending','running','completed','failed')) DEFAULT 'pending', + started_at DATETIME, + finished_at DATETIME, + error TEXT, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_clone_jobs_repo_id ON clone_jobs(repo_id); diff --git a/internal/store/migrations/000003_sync.down.sql b/internal/store/migrations/000003_sync.down.sql new file mode 100644 index 0000000..712f587 --- /dev/null +++ b/internal/store/migrations/000003_sync.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pending_packages; diff --git a/internal/store/migrations/000003_sync.up.sql b/internal/store/migrations/000003_sync.up.sql new file mode 100644 index 0000000..8d82e81 --- /dev/null +++ b/internal/store/migrations/000003_sync.up.sql @@ -0,0 +1,18 @@ +ALTER TABLE repos ADD COLUMN sync_mode TEXT NOT NULL DEFAULT 'auto' + CHECK(sync_mode IN ('auto', 'manual')); + +CREATE TABLE IF NOT EXISTS pending_packages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE, + name TEXT NOT NULL, + version TEXT NOT NULL, + arch TEXT NOT NULL, + location TEXT NOT NULL, + checksum TEXT NOT NULL, + checksum_type TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(repo_id, name, version, arch) +); + +CREATE INDEX IF NOT EXISTS idx_pending_packages_repo_id ON pending_packages(repo_id); diff --git a/internal/store/migrations/000004_snapshot_packages.down.sql b/internal/store/migrations/000004_snapshot_packages.down.sql new file mode 100644 index 0000000..23edf07 --- /dev/null +++ b/internal/store/migrations/000004_snapshot_packages.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_snapshot_packages_snapshot_id; +DROP TABLE IF EXISTS snapshot_packages; diff --git a/internal/store/migrations/000004_snapshot_packages.up.sql b/internal/store/migrations/000004_snapshot_packages.up.sql new file mode 100644 index 0000000..1ba4151 --- /dev/null +++ b/internal/store/migrations/000004_snapshot_packages.up.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS snapshot_packages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + snapshot_id INTEGER NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE, + name TEXT NOT NULL, + version TEXT NOT NULL, + arch TEXT NOT NULL, + location TEXT NOT NULL, + checksum TEXT NOT NULL, + checksum_type TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_snapshot_packages_snapshot_id + ON snapshot_packages(snapshot_id); diff --git a/internal/store/migrations/000005_blocked_packages.down.sql b/internal/store/migrations/000005_blocked_packages.down.sql new file mode 100644 index 0000000..ff314aa --- /dev/null +++ b/internal/store/migrations/000005_blocked_packages.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS blocked_packages; diff --git a/internal/store/migrations/000005_blocked_packages.up.sql b/internal/store/migrations/000005_blocked_packages.up.sql new file mode 100644 index 0000000..bfeabd7 --- /dev/null +++ b/internal/store/migrations/000005_blocked_packages.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS blocked_packages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE, + location TEXT NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(repo_id, location) +); diff --git a/internal/store/migrations/000006_blocked_packages_name.down.sql b/internal/store/migrations/000006_blocked_packages_name.down.sql new file mode 100644 index 0000000..e60e088 --- /dev/null +++ b/internal/store/migrations/000006_blocked_packages_name.down.sql @@ -0,0 +1,4 @@ +-- SQLite ne supporte pas DROP COLUMN avant 3.35 ; recréation de la table sans la colonne name +CREATE TABLE blocked_packages_backup AS SELECT id, repo_id, location, created_at FROM blocked_packages; +DROP TABLE blocked_packages; +ALTER TABLE blocked_packages_backup RENAME TO blocked_packages; diff --git a/internal/store/migrations/000006_blocked_packages_name.up.sql b/internal/store/migrations/000006_blocked_packages_name.up.sql new file mode 100644 index 0000000..9a7a08d --- /dev/null +++ b/internal/store/migrations/000006_blocked_packages_name.up.sql @@ -0,0 +1 @@ +ALTER TABLE blocked_packages ADD COLUMN name TEXT NOT NULL DEFAULT ''; diff --git a/internal/store/pending_package.go b/internal/store/pending_package.go new file mode 100644 index 0000000..f954a41 --- /dev/null +++ b/internal/store/pending_package.go @@ -0,0 +1,92 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" +) + +type PendingPackage struct { + ID int64 + RepoID int64 + Name string + Version string + Arch string + Location string + Checksum string + ChecksumType string + Size int64 + CreatedAt time.Time +} + +type PendingPackageStore interface { + UpsertPending(ctx context.Context, pkgs []PendingPackage) error + ListPending(ctx context.Context, repoID int64) ([]PendingPackage, error) + DeletePending(ctx context.Context, ids []int64) error + DeleteAllPending(ctx context.Context, repoID int64) error +} + +type SQLitePendingPackageStore struct { + db *sql.DB +} + +func NewPendingPackageStore(db *sql.DB) *SQLitePendingPackageStore { + return &SQLitePendingPackageStore{db: db} +} + +func (s *SQLitePendingPackageStore) UpsertPending(ctx context.Context, pkgs []PendingPackage) error { + for _, p := range pkgs { + _, err := s.db.ExecContext(ctx, + `INSERT OR IGNORE INTO pending_packages + (repo_id, name, version, arch, location, checksum, checksum_type, size) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + p.RepoID, p.Name, p.Version, p.Arch, p.Location, p.Checksum, p.ChecksumType, p.Size, + ) + if err != nil { + return err + } + } + return nil +} + +func (s *SQLitePendingPackageStore) ListPending(ctx context.Context, repoID int64) ([]PendingPackage, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, repo_id, name, version, arch, location, checksum, checksum_type, size, created_at + FROM pending_packages WHERE repo_id = ? ORDER BY created_at ASC`, repoID) + if err != nil { + return nil, err + } + defer rows.Close() + + var pkgs []PendingPackage + for rows.Next() { + var p PendingPackage + if err := rows.Scan(&p.ID, &p.RepoID, &p.Name, &p.Version, &p.Arch, + &p.Location, &p.Checksum, &p.ChecksumType, &p.Size, &p.CreatedAt); err != nil { + return nil, err + } + pkgs = append(pkgs, p) + } + return pkgs, rows.Err() +} + +func (s *SQLitePendingPackageStore) DeletePending(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + placeholders := strings.Join(strings.Fields(strings.Repeat("? ", len(ids))), ", ") + query := fmt.Sprintf("DELETE FROM pending_packages WHERE id IN (%s)", placeholders) + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + _, err := s.db.ExecContext(ctx, query, args...) + return err +} + +func (s *SQLitePendingPackageStore) DeleteAllPending(ctx context.Context, repoID int64) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM pending_packages WHERE repo_id = ?`, repoID) + return err +} diff --git a/internal/store/repo.go b/internal/store/repo.go new file mode 100644 index 0000000..952d0db --- /dev/null +++ b/internal/store/repo.go @@ -0,0 +1,107 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "time" +) + +var ErrNotFound = errors.New("not found") + +type Repo struct { + ID int64 `db:"id"` + Name string `db:"name"` + Type string `db:"type"` + SourceURL string `db:"source_url"` + Frozen bool `db:"frozen"` + SyncMode string `db:"sync_mode"` + CreatedAt time.Time `db:"created_at"` +} + +type RepoStore interface { + CreateRepo(ctx context.Context, r *Repo) (int64, error) + ListRepos(ctx context.Context) ([]Repo, error) + GetRepo(ctx context.Context, id int64) (*Repo, error) + DeleteRepo(ctx context.Context, id int64) error + UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error +} + +type SQLiteRepoStore struct { + db *sql.DB +} + +func NewRepoStore(db *sql.DB) *SQLiteRepoStore { + return &SQLiteRepoStore{db: db} +} + +func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error) { + res, err := s.db.ExecContext(ctx, + `INSERT INTO repos (name, type, source_url, frozen, sync_mode) VALUES (?, ?, ?, ?, ?)`, + r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode, + ) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos ORDER BY created_at DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var repos []Repo + for rows.Next() { + var r Repo + if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt); err != nil { + return nil, err + } + repos = append(repos, r) + } + return repos, rows.Err() +} + +func (s *SQLiteRepoStore) GetRepo(ctx context.Context, id int64) (*Repo, error) { + var r Repo + err := s.db.QueryRowContext(ctx, + `SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos WHERE id = ?`, id, + ).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + return &r, nil +} + +func (s *SQLiteRepoStore) DeleteRepo(ctx context.Context, id int64) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM repos WHERE id = ?`, id) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + return nil +} + +func (s *SQLiteRepoStore) UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error { + res, err := s.db.ExecContext(ctx, `UPDATE repos SET sync_mode = ? WHERE id = ?`, mode, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return ErrNotFound + } + return nil +} diff --git a/internal/store/snapshot.go b/internal/store/snapshot.go new file mode 100644 index 0000000..db47b9d --- /dev/null +++ b/internal/store/snapshot.go @@ -0,0 +1,144 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "time" +) + +type Snapshot struct { + ID int64 + RepoID int64 + Label string + CreatedAt time.Time +} + +type SnapshotPackage struct { + ID int64 + SnapshotID int64 + Name string + Version string + Arch string + Location string + Checksum string + ChecksumType string + Size int64 +} + +type SnapshotStore interface { + CreateSnapshot(ctx context.Context, repoID int64, label string) (int64, error) + AddSnapshotPackages(ctx context.Context, snapshotID int64, pkgs []SnapshotPackage) error + ListSnapshots(ctx context.Context, repoID int64) ([]Snapshot, error) + GetSnapshot(ctx context.Context, id int64) (*Snapshot, error) + GetSnapshotPackages(ctx context.Context, snapshotID int64) ([]SnapshotPackage, error) + DeleteSnapshot(ctx context.Context, id int64) error +} + +type SQLiteSnapshotStore struct { + db *sql.DB +} + +func NewSnapshotStore(db *sql.DB) *SQLiteSnapshotStore { + return &SQLiteSnapshotStore{db: db} +} + +func (s *SQLiteSnapshotStore) CreateSnapshot(ctx context.Context, repoID int64, label string) (int64, error) { + res, err := s.db.ExecContext(ctx, `INSERT INTO snapshots (repo_id, label) VALUES (?, ?)`, repoID, label) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func (s *SQLiteSnapshotStore) AddSnapshotPackages(ctx context.Context, snapshotID int64, pkgs []SnapshotPackage) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + stmt, err := tx.PrepareContext(ctx, `INSERT INTO snapshot_packages + (snapshot_id, name, version, arch, location, checksum, checksum_type, size) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + defer stmt.Close() + + for _, p := range pkgs { + if _, err := stmt.ExecContext(ctx, snapshotID, p.Name, p.Version, p.Arch, p.Location, p.Checksum, p.ChecksumType, p.Size); err != nil { + return err + } + } + return tx.Commit() +} + +func (s *SQLiteSnapshotStore) ListSnapshots(ctx context.Context, repoID int64) ([]Snapshot, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, repo_id, label, created_at FROM snapshots WHERE repo_id = ? ORDER BY id DESC`, repoID) + if err != nil { + return nil, err + } + defer rows.Close() + + var snaps []Snapshot + for rows.Next() { + var snap Snapshot + if err := rows.Scan(&snap.ID, &snap.RepoID, &snap.Label, &snap.CreatedAt); err != nil { + return nil, err + } + snaps = append(snaps, snap) + } + return snaps, rows.Err() +} + +func (s *SQLiteSnapshotStore) GetSnapshot(ctx context.Context, id int64) (*Snapshot, error) { + var snap Snapshot + err := s.db.QueryRowContext(ctx, + `SELECT id, repo_id, label, created_at FROM snapshots WHERE id = ?`, id). + Scan(&snap.ID, &snap.RepoID, &snap.Label, &snap.CreatedAt) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + return &snap, nil +} + +func (s *SQLiteSnapshotStore) GetSnapshotPackages(ctx context.Context, snapshotID int64) ([]SnapshotPackage, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, snapshot_id, name, version, arch, location, checksum, checksum_type, size + FROM snapshot_packages WHERE snapshot_id = ?`, snapshotID) + if err != nil { + return nil, err + } + defer rows.Close() + + var pkgs []SnapshotPackage + for rows.Next() { + var p SnapshotPackage + if err := rows.Scan(&p.ID, &p.SnapshotID, &p.Name, &p.Version, &p.Arch, + &p.Location, &p.Checksum, &p.ChecksumType, &p.Size); err != nil { + return nil, err + } + pkgs = append(pkgs, p) + } + return pkgs, rows.Err() +} + +func (s *SQLiteSnapshotStore) DeleteSnapshot(ctx context.Context, id int64) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM snapshots WHERE id = ?`, id) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + return nil +}