syonad/clonepack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

first with full handle over rpm

Browse source 
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-04-25 14:49:14 +02:00
commit 274ea454dd
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
50 changed files with 4309 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

83
internal/api/clone_handler.go Normal file
View file

@ -0,0 +1,83 @@
package api
import (
"errors"
"net/http"
"strconv"
"time"
"github.com/go-chi/chi/v5"
"github.com/syonad/clonepack/internal/core"
"github.com/syonad/clonepack/internal/store"
)
type CloneHandler struct {
svc *core.CloneService
}
func NewCloneHandler(svc *core.CloneService) *CloneHandler {
return &CloneHandler{svc: svc}
}
func (h *CloneHandler) StartClone(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
jobID, err := h.svc.StartClone(r.Context(), repoID)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
}
if errors.Is(err, core.ErrCloneAlreadyRunning) {
Error(w, http.StatusConflict, err.Error())
return
}
if err != nil {
Error(w, http.StatusUnprocessableEntity, err.Error())
return
}
JSON(w, http.StatusAccepted, StartCloneResponse{JobID: jobID, Status: "running"})
}
func (h *CloneHandler) GetStatus(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
job, err := h.svc.GetLatestCloneJob(r.Context(), repoID)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "no clone job found for this repo")
return
}
if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
JSON(w, http.StatusOK, cloneJobToResponse(job))
}
func cloneJobToResponse(j *store.CloneJob) CloneStatusResponse {
resp := CloneStatusResponse{
JobID: j.ID,
RepoID: j.RepoID,
Status: string(j.Status),
Error: j.Error,
CreatedAt: j.CreatedAt.Format(time.RFC3339),
}
if j.StartedAt != nil {
s := j.StartedAt.Format(time.RFC3339)
resp.StartedAt = &s
}
if j.FinishedAt != nil {
f := j.FinishedAt.Format(time.RFC3339)
resp.FinishedAt = &f
}
return resp
}

43
internal/api/proxy_handler.go Normal file
View file

@ -0,0 +1,43 @@
package api
import (
"errors"
"fmt"
"net/http"
"strconv"
"github.com/go-chi/chi/v5"
"github.com/syonad/clonepack/internal/core"
"github.com/syonad/clonepack/internal/store"
)
type ProxyHandler struct {
repoSvc *core.RepoService
dataDir string
}
func NewProxyHandler(repoSvc *core.RepoService, dataDir string) *ProxyHandler {
return &ProxyHandler{repoSvc: repoSvc, dataDir: dataDir}
}
func (h *ProxyHandler) ServeFile(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "repo_id"), 10, 64)
if err != nil {
http.Error(w, "invalid repo_id", http.StatusBadRequest)
return
}
repo, err := h.repoSvc.Get(r.Context(), repoID)
if errors.Is(err, store.ErrNotFound) {
http.NotFound(w, r)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
localDir := fmt.Sprintf("%s/repos/%d/%s", h.dataDir, repoID, repo.Type)
prefix := fmt.Sprintf("/mirror/%d", repoID)
http.StripPrefix(prefix, http.FileServer(http.Dir(localDir))).ServeHTTP(w, r)
}

102
internal/api/repo_handler.go Normal file
View file

@ -0,0 +1,102 @@
package api
import (
"encoding/json"
"errors"
"net/http"
"strconv"
"github.com/go-chi/chi/v5"
"github.com/syonad/clonepack/internal/core"
"github.com/syonad/clonepack/internal/store"
)
type RepoHandler struct {
svc *core.RepoService
}
func NewRepoHandler(svc *core.RepoService) *RepoHandler {
return &RepoHandler{svc: svc}
}
func (h *RepoHandler) Create(w http.ResponseWriter, r *http.Request) {
var req CreateRepoRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
Error(w, http.StatusBadRequest, "invalid JSON")
return
}
repo, err := h.svc.Create(r.Context(), core.CreateRepoInput{
Name: req.Name,
Type: req.Type,
SourceURL: req.SourceURL,
SyncMode: req.SyncMode,
})
if err != nil {
Error(w, http.StatusUnprocessableEntity, err.Error())
return
}
JSON(w, http.StatusCreated, repoToResponse(repo))
}
func (h *RepoHandler) List(w http.ResponseWriter, r *http.Request) {
repos, err := h.svc.List(r.Context())
if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
items := make([]RepoResponse, len(repos))
for i, repo := range repos {
items[i] = repoToResponse(&repo)
}
JSON(w, http.StatusOK, ListReposResponse{Items: items, Total: len(items)})
}
func (h *RepoHandler) Get(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
repo, err := h.svc.Get(r.Context(), id)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
}
if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
JSON(w, http.StatusOK, repoToResponse(repo))
}
func (h *RepoHandler) Delete(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
if err := h.svc.Delete(r.Context(), id); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
func repoToResponse(r *store.Repo) RepoResponse {
return RepoResponse{
ID: r.ID,
Name: r.Name,
Type: r.Type,
SourceURL: r.SourceURL,
Frozen: r.Frozen,
SyncMode: r.SyncMode,
CreatedAt: r.CreatedAt,
}
}

16
internal/api/respond.go Normal file
View file

@ -0,0 +1,16 @@
package api
import (
"encoding/json"
"net/http"
)
func JSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(v)
}
func Error(w http.ResponseWriter, status int, msg string) {
JSON(w, status, ErrorResponse{Error: msg})
}

50
internal/api/router.go Normal file
View file

@ -0,0 +1,50 @@
package api
import (
"net/http"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
)
func NewRouter(repoHandler *RepoHandler, cloneHandler *CloneHandler, syncHandler *SyncHandler, snapshotHandler *SnapshotHandler, proxyHandler *ProxyHandler) http.Handler {
r := chi.NewRouter()
r.Use(middleware.Logger)
r.Use(middleware.Recoverer)
r.Use(middleware.RequestID)
r.Get("/mirror/{repo_id}/*", proxyHandler.ServeFile)
r.Get("/health", func(w http.ResponseWriter, r *http.Request) {
JSON(w, http.StatusOK, map[string]string{"status": "ok"})
})
r.Route("/api/v1", func(r chi.Router) {
r.Route("/repos", func(r chi.Router) {
r.Post("/", repoHandler.Create)
r.Get("/", repoHandler.List)
r.Get("/{id}", repoHandler.Get)
r.Delete("/{id}", repoHandler.Delete)
r.Post("/{id}/clone", cloneHandler.StartClone)
r.Get("/{id}/clone/status", cloneHandler.GetStatus)
r.Post("/{id}/sync/trigger", syncHandler.Trigger)
r.Get("/{id}/sync/pending", syncHandler.ListPending)
r.Post("/{id}/sync/approve", syncHandler.Approve)
r.Post("/{id}/sync/reject", syncHandler.Reject)
r.Post("/{id}/sync/block", syncHandler.Block)
r.Post("/{id}/sync/unblock", syncHandler.Unblock)
r.Get("/{id}/sync/blocked", syncHandler.ListBlocked)
r.Post("/{id}/snapshots", snapshotHandler.Create)
r.Get("/{id}/snapshots", snapshotHandler.List)
r.Get("/{id}/snapshots/diff", snapshotHandler.Diff)
r.Get("/{id}/snapshots/{snap_id}", snapshotHandler.Get)
r.Delete("/{id}/snapshots/{snap_id}", snapshotHandler.Delete)
r.Post("/{id}/snapshots/{snap_id}/rollback", snapshotHandler.Rollback)
})
})
return r
}

196
internal/api/snapshot_handler.go Normal file
View file

@ -0,0 +1,196 @@
package api
import (
"encoding/json"
"errors"
"net/http"
"strconv"
"time"
"github.com/go-chi/chi/v5"
"github.com/syonad/clonepack/internal/core"
"github.com/syonad/clonepack/internal/store"
)
type SnapshotHandler struct {
svc *core.SnapshotService
}
func NewSnapshotHandler(svc *core.SnapshotService) *SnapshotHandler {
return &SnapshotHandler{svc: svc}
}
func (h *SnapshotHandler) Create(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var req CreateSnapshotRequest
json.NewDecoder(r.Body).Decode(&req)
if req.Label == "" {
req.Label = "manual-" + time.Now().UTC().Format(time.RFC3339)
}
id, err := h.svc.TakeSnapshot(r.Context(), repoID, req.Label)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
snap, _, _ := h.svc.Get(r.Context(), repoID, id)
JSON(w, http.StatusCreated, snapshotToResponse(*snap))
}
func (h *SnapshotHandler) List(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
snaps, err := h.svc.List(r.Context(), repoID)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
items := make([]SnapshotResponse, len(snaps))
for i, s := range snaps {
items[i] = snapshotToResponse(s)
}
JSON(w, http.StatusOK, ListSnapshotsResponse{Items: items, Total: len(items)})
}
func (h *SnapshotHandler) Get(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
snapID, err := strconv.ParseInt(chi.URLParam(r, "snap_id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid snap_id")
return
}
snap, pkgs, err := h.svc.Get(r.Context(), repoID, snapID)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "snapshot not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
resp := SnapshotDetailResponse{
SnapshotResponse: snapshotToResponse(*snap),
Packages: make([]SnapshotPackageResponse, len(pkgs)),
}
for i, p := range pkgs {
resp.Packages[i] = snapshotPkgToResponse(p)
}
JSON(w, http.StatusOK, resp)
}
func (h *SnapshotHandler) Delete(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
snapID, err := strconv.ParseInt(chi.URLParam(r, "snap_id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid snap_id")
return
}
if err := h.svc.Delete(r.Context(), repoID, snapID); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "snapshot not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *SnapshotHandler) Diff(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
fromID, err := strconv.ParseInt(r.URL.Query().Get("from"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid from param")
return
}
toID, err := strconv.ParseInt(r.URL.Query().Get("to"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid to param")
return
}
diff, err := h.svc.Diff(r.Context(), repoID, fromID, toID)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "snapshot not found")
return
} else if err != nil {
Error(w, http.StatusBadRequest, err.Error())
return
}
resp := SnapshotDiffResponse{
From: snapshotToResponse(*diff.From),
To: snapshotToResponse(*diff.To),
Unchanged: diff.Unchanged,
}
for _, p := range diff.Added {
resp.Added = append(resp.Added, snapshotPkgToResponse(p))
}
for _, p := range diff.Removed {
resp.Removed = append(resp.Removed, snapshotPkgToResponse(p))
}
JSON(w, http.StatusOK, resp)
}
func (h *SnapshotHandler) Rollback(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
snapID, err := strconv.ParseInt(chi.URLParam(r, "snap_id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid snap_id")
return
}
if err := h.svc.Rollback(r.Context(), repoID, snapID); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "snapshot not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
JSON(w, http.StatusOK, map[string]string{"status": "rollback completed"})
}
func snapshotToResponse(s store.Snapshot) SnapshotResponse {
return SnapshotResponse{
ID: s.ID,
RepoID: s.RepoID,
Label: s.Label,
CreatedAt: s.CreatedAt,
}
}
func snapshotPkgToResponse(p store.SnapshotPackage) SnapshotPackageResponse {
return SnapshotPackageResponse{
ID: p.ID,
Name: p.Name,
Version: p.Version,
Arch: p.Arch,
Location: p.Location,
Checksum: p.Checksum,
ChecksumType: p.ChecksumType,
Size: p.Size,
}
}

43
internal/api/snapshot_types.go Normal file
View file

@ -0,0 +1,43 @@
package api
import "time"
type SnapshotResponse struct {
ID int64 `json:"id"`
RepoID int64 `json:"repo_id"`
Label string `json:"label"`
CreatedAt time.Time `json:"created_at"`
}
type SnapshotDetailResponse struct {
SnapshotResponse
Packages []SnapshotPackageResponse `json:"packages"`
}
type SnapshotPackageResponse struct {
ID int64 `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Arch string `json:"arch"`
Location string `json:"location"`
Checksum string `json:"checksum"`
ChecksumType string `json:"checksum_type"`
Size int64 `json:"size"`
}
type ListSnapshotsResponse struct {
Items []SnapshotResponse `json:"items"`
Total int `json:"total"`
}
type CreateSnapshotRequest struct {
Label string `json:"label"`
}
type SnapshotDiffResponse struct {
From SnapshotResponse `json:"from"`
To SnapshotResponse `json:"to"`
Added []SnapshotPackageResponse `json:"added"`
Removed []SnapshotPackageResponse `json:"removed"`
Unchanged int `json:"unchanged"`
}

191
internal/api/sync_handler.go Normal file
View file

@ -0,0 +1,191 @@
package api
import (
"context"
"encoding/json"
"errors"
"log"
"net/http"
"strconv"
"github.com/go-chi/chi/v5"
"github.com/syonad/clonepack/internal/core"
"github.com/syonad/clonepack/internal/store"
)
type SyncHandler struct {
svc *core.SyncService
}
func NewSyncHandler(svc *core.SyncService) *SyncHandler {
return &SyncHandler{svc: svc}
}
func (h *SyncHandler) Trigger(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
// Validate repo exists before returning.
if err := h.svc.ValidateRepo(r.Context(), repoID); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
go func() {
if err := h.svc.ScanRepo(context.Background(), repoID); err != nil {
log.Printf("scan repo %d: %v", repoID, err)
}
}()
JSON(w, http.StatusAccepted, map[string]string{"status": "scan started"})
}
func (h *SyncHandler) ListPending(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
pkgs, err := h.svc.ListPending(r.Context(), repoID)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
items := make([]PendingPackageResponse, len(pkgs))
for i, p := range pkgs {
items[i] = pendingToResponse(p)
}
JSON(w, http.StatusOK, ListPendingResponse{Items: items, Total: len(items)})
}
func (h *SyncHandler) Approve(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var req SyncSelectionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 {
Error(w, http.StatusBadRequest, "body must contain non-empty ids array")
return
}
if err := h.svc.ValidateRepo(r.Context(), repoID); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
ids := req.IDs
go func() {
if err := h.svc.ApprovePending(context.Background(), repoID, ids); err != nil {
log.Printf("approve pending for repo %d: %v", repoID, err)
}
}()
JSON(w, http.StatusAccepted, map[string]string{"status": "approval started"})
}
func (h *SyncHandler) Reject(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var req SyncSelectionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 {
Error(w, http.StatusBadRequest, "body must contain non-empty ids array")
return
}
if err := h.svc.RejectPending(r.Context(), repoID, req.IDs); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *SyncHandler) Block(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var req SyncSelectionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 {
Error(w, http.StatusBadRequest, "body must contain non-empty ids array")
return
}
if err := h.svc.BlockPackages(r.Context(), repoID, req.IDs); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *SyncHandler) Unblock(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var req SyncSelectionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.IDs) == 0 {
Error(w, http.StatusBadRequest, "body must contain non-empty ids array")
return
}
if err := h.svc.UnblockPackages(r.Context(), repoID, req.IDs); errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *SyncHandler) ListBlocked(w http.ResponseWriter, r *http.Request) {
repoID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
pkgs, err := h.svc.ListBlocked(r.Context(), repoID)
if errors.Is(err, store.ErrNotFound) {
Error(w, http.StatusNotFound, "repo not found")
return
} else if err != nil {
Error(w, http.StatusInternalServerError, err.Error())
return
}
items := make([]BlockedPackageResponse, len(pkgs))
for i, p := range pkgs {
items[i] = BlockedPackageResponse{ID: p.ID, RepoID: p.RepoID, Name: p.Name, Location: p.Location, CreatedAt: p.CreatedAt}
}
JSON(w, http.StatusOK, ListBlockedResponse{Items: items, Total: len(items)})
}
func pendingToResponse(p store.PendingPackage) PendingPackageResponse {
return PendingPackageResponse{
ID: p.ID,
RepoID: p.RepoID,
Name: p.Name,
Version: p.Version,
Arch: p.Arch,
Location: p.Location,
Checksum: p.Checksum,
ChecksumType: p.ChecksumType,
Size: p.Size,
CreatedAt: p.CreatedAt,
}
}

38
internal/api/sync_types.go Normal file
View file

@ -0,0 +1,38 @@
package api
import "time"
type SyncSelectionRequest struct {
IDs []int64 `json:"ids"`
}
type PendingPackageResponse struct {
ID int64 `json:"id"`
RepoID int64 `json:"repo_id"`
Name string `json:"name"`
Version string `json:"version"`
Arch string `json:"arch"`
Location string `json:"location"`
Checksum string `json:"checksum"`
ChecksumType string `json:"checksum_type"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"created_at"`
}
type ListPendingResponse struct {
Items []PendingPackageResponse `json:"items"`
Total int `json:"total"`
}
type BlockedPackageResponse struct {
ID int64 `json:"id"`
RepoID int64 `json:"repo_id"`
Name string `json:"name"`
Location string `json:"location"`
CreatedAt time.Time `json:"created_at"`
}
type ListBlockedResponse struct {
Items []BlockedPackageResponse `json:"items"`
Total int `json:"total"`
}

44
internal/api/types.go Normal file
View file

@ -0,0 +1,44 @@
package api
import "time"
type CreateRepoRequest struct {
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode,omitempty"`
}
type RepoResponse struct {
ID int64 `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
Frozen bool `json:"frozen"`
SyncMode string `json:"sync_mode"`
CreatedAt time.Time `json:"created_at"`
}
type ListReposResponse struct {
Items []RepoResponse `json:"items"`
Total int `json:"total"`
}
type ErrorResponse struct {
Error string `json:"error"`
}
type StartCloneResponse struct {
JobID int64 `json:"job_id"`
Status string `json:"status"`
}
type CloneStatusResponse struct {
JobID int64 `json:"job_id"`
RepoID int64 `json:"repo_id"`
Status string `json:"status"`
StartedAt *string `json:"started_at,omitempty"`
FinishedAt *string `json:"finished_at,omitempty"`
Error *string `json:"error,omitempty"`
CreatedAt string `json:"created_at"`
}

195
internal/clone/rpm/cloner.go Normal file
View file

@ -0,0 +1,195 @@
package rpm
import (
"compress/gzip"
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
type Progress struct {
File string
BytesDone int64
}
type ProgressFunc func(p Progress)
type Cloner struct {
SourceURL string
DestDir string
HTTPClient *http.Client
OnProgress ProgressFunc
}
func New(sourceURL, destDir string) *Cloner {
return &Cloner{
SourceURL: strings.TrimRight(sourceURL, "/"),
DestDir: destDir,
HTTPClient: &http.Client{Timeout: 30 * time.Minute},
}
}
func (c *Cloner) Clone(ctx context.Context) error {
if err := os.MkdirAll(filepath.Join(c.DestDir, "repodata"), 0o755); err != nil {
return fmt.Errorf("create repodata dir: %w", err)
}
if err := os.MkdirAll(filepath.Join(c.DestDir, "Packages"), 0o755); err != nil {
return fmt.Errorf("create Packages dir: %w", err)
}
repomd, err := c.fetchRepoMD(ctx)
if err != nil {
return fmt.Errorf("fetch repomd.xml: %w", err)
}
var primaryEntry *RepoMDEntry
for i, entry := range repomd.Data {
if entry.Type == "primary" {
primaryEntry = &repomd.Data[i]
continue
}
if err := c.downloadMetadataFile(ctx, entry); err != nil {
return fmt.Errorf("download metadata %s: %w", entry.Type, err)
}
}
if primaryEntry == nil {
return fmt.Errorf("no primary metadata found in repomd.xml")
}
packages, err := c.fetchPrimary(ctx, *primaryEntry)
if err != nil {
return fmt.Errorf("fetch primary.xml: %w", err)
}
for _, pkg := range packages {
if err := ctx.Err(); err != nil {
return err
}
if err := c.downloadPackage(ctx, pkg); err != nil {
return fmt.Errorf("download package %s: %w", pkg.Name, err)
}
}
return nil
}
func (c *Cloner) fetchRepoMD(ctx context.Context) (*RepoMD, error) {
url := c.SourceURL + "/repodata/repomd.xml"
data, err := c.fetchBytes(ctx, url)
if err != nil {
return nil, err
}
dest := filepath.Join(c.DestDir, "repodata", "repomd.xml")
if err := writeFile(dest, data); err != nil {
return nil, err
}
var repomd RepoMD
if err := xml.Unmarshal(data, &repomd); err != nil {
return nil, fmt.Errorf("parse repomd.xml: %w", err)
}
return &repomd, nil
}
func (c *Cloner) fetchPrimary(ctx context.Context, entry RepoMDEntry) ([]Package, error) {
url := c.SourceURL + "/" + entry.Location.Href
data, err := c.fetchBytes(ctx, url)
if err != nil {
return nil, err
}
if entry.Checksum.Type == "sha256" {
if err := verifyChecksum(data, entry.Checksum.Value); err != nil {
return nil, fmt.Errorf("primary.xml.gz: %w", err)
}
}
dest := filepath.Join(c.DestDir, entry.Location.Href)
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return nil, err
}
if err := writeFile(dest, data); err != nil {
return nil, err
}
gz, err := gzip.NewReader(strings.NewReader(string(data)))
if err != nil {
return nil, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return nil, fmt.Errorf("parse primary.xml: %w", err)
}
return primary.Packages, nil
}
func (c *Cloner) downloadMetadataFile(ctx context.Context, entry RepoMDEntry) error {
url := c.SourceURL + "/" + entry.Location.Href
dest := filepath.Join(c.DestDir, entry.Location.Href)
n, err := c.downloadAndVerify(ctx, url, dest, entry.Checksum.Type, entry.Checksum.Value)
if err != nil {
return err
}
if c.OnProgress != nil {
c.OnProgress(Progress{File: entry.Location.Href, BytesDone: n})
}
return nil
}
func (c *Cloner) downloadPackage(ctx context.Context, pkg Package) error {
url := c.SourceURL + "/" + pkg.Location.Href
dest := filepath.Join(c.DestDir, pkg.Location.Href)
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return err
}
n, err := c.downloadAndVerify(ctx, url, dest, pkg.Checksum.Type, pkg.Checksum.Value)
if err != nil {
return err
}
if c.OnProgress != nil {
c.OnProgress(Progress{File: pkg.Location.Href, BytesDone: n})
}
return nil
}
func (c *Cloner) downloadAndVerify(ctx context.Context, url, destPath, checksumType, expectedChecksum string) (int64, error) {
return DownloadAndVerify(ctx, c.HTTPClient, url, destPath, checksumType, expectedChecksum)
}
func (c *Cloner) fetchBytes(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := c.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
return io.ReadAll(resp.Body)
}
func writeFile(path string, data []byte) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, data, 0o644)
}

71
internal/clone/rpm/download.go Normal file
View file

@ -0,0 +1,71 @@
package rpm
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
)
func verifyChecksum(data []byte, expected string) error {
sum := sha256.Sum256(data)
got := hex.EncodeToString(sum[:])
if got != expected {
return fmt.Errorf("checksum mismatch: got %s, want %s", got, expected)
}
return nil
}
// DownloadAndVerify fetches url into destPath atomically (temp file + rename),
// verifies the SHA256 checksum if checksumType is "sha256", and returns bytes written.
func DownloadAndVerify(ctx context.Context, httpClient *http.Client, url, destPath, checksumType, expectedChecksum string) (int64, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return 0, err
}
resp, err := httpClient.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
tmpPath := destPath + ".tmp"
if err := os.MkdirAll(filepath.Dir(tmpPath), 0o755); err != nil {
return 0, err
}
f, err := os.Create(tmpPath)
if err != nil {
return 0, err
}
h := sha256.New()
n, err := io.Copy(io.MultiWriter(f, h), resp.Body)
f.Close()
if err != nil {
os.Remove(tmpPath)
return 0, err
}
if checksumType == "sha256" && expectedChecksum != "" {
got := hex.EncodeToString(h.Sum(nil))
if got != expectedChecksum {
os.Remove(tmpPath)
return 0, fmt.Errorf("checksum mismatch for %s: got %s, want %s", url, got, expectedChecksum)
}
}
if err := os.Rename(tmpPath, destPath); err != nil {
os.Remove(tmpPath)
return 0, err
}
return n, nil
}

212
internal/clone/rpm/metadata_gen.go Normal file
View file

@ -0,0 +1,212 @@
package rpm
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strings"
)
const PrimarySourceFile = ".primary-source"
const localPrimaryHref = "repodata/primary.xml.gz"
// filterablePackage captures location for filtering and inner XML for faithful re-emission.
type filterablePackage struct {
Type string `xml:"type,attr"`
Location struct {
Href string `xml:"href,attr"`
} `xml:"http://linux.duke.edu/metadata/common location"`
Inner string `xml:",innerxml"`
}
type filterableMetadata struct {
XMLName xml.Name `xml:"http://linux.duke.edu/metadata/common metadata"`
Packages []filterablePackage `xml:"http://linux.duke.edu/metadata/common package"`
}
// RegenerateMetadata filters the upstream primary.xml.gz to only include packages
// present on disk, writes a new primary.xml.gz, and updates repomd.xml.
func RegenerateMetadata(localDir string) error {
sourcePath, err := upstreamPrimaryPath(localDir)
if err != nil {
return err
}
gzData, err := os.ReadFile(sourcePath)
if err != nil {
return fmt.Errorf("read upstream primary: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return fmt.Errorf("open gzip: %w", err)
}
var meta filterableMetadata
if err := xml.NewDecoder(gz).Decode(&meta); err != nil {
gz.Close()
return fmt.Errorf("parse primary.xml: %w", err)
}
gz.Close()
var local []filterablePackage
for _, pkg := range meta.Packages {
dest := filepath.Join(localDir, filepath.FromSlash(pkg.Location.Href))
if _, statErr := os.Stat(dest); statErr == nil {
local = append(local, pkg)
}
}
var xmlBuf bytes.Buffer
xmlBuf.WriteString("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
fmt.Fprintf(&xmlBuf,
`<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%d">`,
len(local))
for _, pkg := range local {
fmt.Fprintf(&xmlBuf, `<package type="%s">%s</package>`, pkg.Type, pkg.Inner)
}
xmlBuf.WriteString(`</metadata>`)
xmlBytes := xmlBuf.Bytes()
var gzBuf bytes.Buffer
gzw := gzip.NewWriter(&gzBuf)
if _, err := gzw.Write(xmlBytes); err != nil {
return fmt.Errorf("gzip write: %w", err)
}
if err := gzw.Close(); err != nil {
return fmt.Errorf("gzip close: %w", err)
}
gzBytes := gzBuf.Bytes()
openSum := sha256.Sum256(xmlBytes)
gzSum := sha256.Sum256(gzBytes)
outPath := filepath.Join(localDir, localPrimaryHref)
if err := atomicWrite(outPath, gzBytes); err != nil {
return fmt.Errorf("write primary.xml.gz: %w", err)
}
return updateRepoMD(localDir, gzSum[:], openSum[:], int64(len(gzBytes)), int64(len(xmlBytes)))
}
// upstreamPrimaryPath returns the full path to the original upstream primary.xml.gz,
// persisting it in .primary-source so subsequent calls always filter from the full list.
func upstreamPrimaryPath(localDir string) (string, error) {
markerPath := filepath.Join(localDir, "repodata", PrimarySourceFile)
if data, err := os.ReadFile(markerPath); err == nil {
href := strings.TrimSpace(string(data))
full := filepath.Join(localDir, filepath.FromSlash(href))
if _, err := os.Stat(full); err == nil {
return full, nil
}
}
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml"))
if err != nil {
return "", fmt.Errorf("read repomd.xml: %w", err)
}
var repomd RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return "", fmt.Errorf("parse repomd.xml: %w", err)
}
for _, entry := range repomd.Data {
if entry.Type == "primary" && entry.Location.Href != localPrimaryHref {
_ = os.WriteFile(markerPath, []byte(entry.Location.Href), 0o644)
return filepath.Join(localDir, filepath.FromSlash(entry.Location.Href)), nil
}
}
return "", fmt.Errorf("upstream primary source not found — repo must be cloned first")
}
func updateRepoMD(localDir string, gzSum, openSum []byte, gzSize, openSize int64) error {
repomdPath := filepath.Join(localDir, "repodata", "repomd.xml")
repomdData, err := os.ReadFile(repomdPath)
if err != nil {
return fmt.Errorf("read repomd.xml: %w", err)
}
var repomd RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return fmt.Errorf("parse repomd.xml: %w", err)
}
for i, entry := range repomd.Data {
if entry.Type == "primary" {
repomd.Data[i].Location.Href = localPrimaryHref
repomd.Data[i].Checksum = RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(gzSum)}
repomd.Data[i].OpenChecksum = RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(openSum)}
repomd.Data[i].Size = gzSize
repomd.Data[i].OpenSize = openSize
break
}
}
newXML, err := xml.MarshalIndent(repomd, "", " ")
if err != nil {
return fmt.Errorf("marshal repomd.xml: %w", err)
}
return atomicWrite(repomdPath, append([]byte(xml.Header), newXML...))
}
// InitEmptyRepo creates the directory structure and empty RPM metadata for a new repo.
// It is idempotent: if repomd.xml already exists it is left untouched.
func InitEmptyRepo(localDir string) error {
for _, d := range []string{
filepath.Join(localDir, "repodata"),
filepath.Join(localDir, "Packages"),
} {
if err := os.MkdirAll(d, 0o755); err != nil {
return err
}
}
repomdPath := filepath.Join(localDir, "repodata", "repomd.xml")
if _, err := os.Stat(repomdPath); err == nil {
return nil // already initialised
}
emptyXML := []byte("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
`<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="0"></metadata>`)
var gzBuf bytes.Buffer
gzw := gzip.NewWriter(&gzBuf)
_, _ = gzw.Write(emptyXML)
_ = gzw.Close()
gzBytes := gzBuf.Bytes()
openSum := sha256.Sum256(emptyXML)
gzSum := sha256.Sum256(gzBytes)
if err := atomicWrite(filepath.Join(localDir, localPrimaryHref), gzBytes); err != nil {
return fmt.Errorf("write empty primary.xml.gz: %w", err)
}
repomd := RepoMD{
Data: []RepoMDEntry{{
Type: "primary",
Location: RepoMDLocation{Href: localPrimaryHref},
Checksum: RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(gzSum[:])},
OpenChecksum: RepoMDChecksum{Type: "sha256", Value: hex.EncodeToString(openSum[:])},
Size: int64(len(gzBytes)),
OpenSize: int64(len(emptyXML)),
}},
}
newXML, err := xml.MarshalIndent(repomd, "", " ")
if err != nil {
return err
}
return atomicWrite(repomdPath, append([]byte(xml.Header), newXML...))
}
func atomicWrite(path string, data []byte) error {
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return err
}
if err := os.Rename(tmp, path); err != nil {
os.Remove(tmp)
return err
}
return nil
}

41
internal/clone/rpm/primary.go Normal file
View file

@ -0,0 +1,41 @@
package rpm
import "encoding/xml"
const primaryNS = "http://linux.duke.edu/metadata/common"
type PrimaryMetadata struct {
XMLName xml.Name `xml:"http://linux.duke.edu/metadata/common metadata"`
Packages []Package `xml:"http://linux.duke.edu/metadata/common package"`
}
type Package struct {
Type string `xml:"type,attr"`
Name string `xml:"http://linux.duke.edu/metadata/common name"`
Arch string `xml:"http://linux.duke.edu/metadata/common arch"`
Version PackageVersion `xml:"http://linux.duke.edu/metadata/common version"`
Checksum PackageChecksum `xml:"http://linux.duke.edu/metadata/common checksum"`
Location PackageLocation `xml:"http://linux.duke.edu/metadata/common location"`
Size PackageSize `xml:"http://linux.duke.edu/metadata/common size"`
}
type PackageVersion struct {
Epoch string `xml:"epoch,attr"`
Ver string `xml:"ver,attr"`
Rel string `xml:"rel,attr"`
}
type PackageChecksum struct {
Type string `xml:"type,attr"`
Value string `xml:",chardata"`
}
type PackageLocation struct {
Href string `xml:"href,attr"`
}
type PackageSize struct {
Package int64 `xml:"package,attr"`
Installed int64 `xml:"installed,attr"`
Archive int64 `xml:"archive,attr"`
}

26
internal/clone/rpm/repomd.go Normal file
View file

@ -0,0 +1,26 @@
package rpm
import "encoding/xml"
type RepoMD struct {
XMLName xml.Name `xml:"repomd"`
Data []RepoMDEntry `xml:"data"`
}
type RepoMDEntry struct {
Type string `xml:"type,attr"`
Location RepoMDLocation `xml:"location"`
Checksum RepoMDChecksum `xml:"checksum"`
Size int64 `xml:"size"`
OpenChecksum RepoMDChecksum `xml:"open-checksum"`
OpenSize int64 `xml:"open-size"`
}
type RepoMDLocation struct {
Href string `xml:"href,attr"`
}
type RepoMDChecksum struct {
Type string `xml:"type,attr"`
Value string `xml:",chardata"`
}

143
internal/clone/rpm/scanner.go Normal file
View file

@ -0,0 +1,143 @@
package rpm
import (
"compress/gzip"
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
type NewPackage struct {
Name string
Version string
Arch string
Location string
Checksum string
ChecksumType string
Size int64
}
type Scanner struct {
SourceURL string
LocalDir string
HTTPClient *http.Client
}
func NewScanner(sourceURL, localDir string) *Scanner {
return &Scanner{
SourceURL: strings.TrimRight(sourceURL, "/"),
LocalDir: localDir,
HTTPClient: &http.Client{Timeout: 5 * time.Minute},
}
}
// Scan fetches the remote package list and returns packages not present on disk.
func (sc *Scanner) Scan(ctx context.Context) ([]NewPackage, error) {
repomd, err := sc.fetchRepoMD(ctx)
if err != nil {
return nil, fmt.Errorf("fetch repomd.xml: %w", err)
}
var primaryEntry *RepoMDEntry
for i := range repomd.Data {
if repomd.Data[i].Type == "primary" {
primaryEntry = &repomd.Data[i]
break
}
}
if primaryEntry == nil {
return nil, fmt.Errorf("no primary entry in repomd.xml")
}
packages, err := sc.fetchPrimary(ctx, *primaryEntry)
if err != nil {
return nil, fmt.Errorf("fetch primary.xml: %w", err)
}
var missing []NewPackage
for _, pkg := range packages {
localPath := filepath.Join(sc.LocalDir, filepath.FromSlash(pkg.Location.Href))
if _, err := os.Stat(localPath); os.IsNotExist(err) {
ver := pkg.Version.Ver + "-" + pkg.Version.Rel
if pkg.Version.Epoch != "0" && pkg.Version.Epoch != "" {
ver = pkg.Version.Epoch + ":" + ver
}
missing = append(missing, NewPackage{
Name: pkg.Name,
Version: ver,
Arch: pkg.Arch,
Location: pkg.Location.Href,
Checksum: pkg.Checksum.Value,
ChecksumType: pkg.Checksum.Type,
Size: pkg.Size.Package,
})
}
}
return missing, nil
}
func (sc *Scanner) fetchRepoMD(ctx context.Context) (*RepoMD, error) {
data, err := sc.fetchBytes(ctx, sc.SourceURL+"/repodata/repomd.xml")
if err != nil {
return nil, err
}
var repomd RepoMD
if err := xml.Unmarshal(data, &repomd); err != nil {
return nil, fmt.Errorf("parse repomd.xml: %w", err)
}
return &repomd, nil
}
func (sc *Scanner) fetchPrimary(ctx context.Context, entry RepoMDEntry) ([]Package, error) {
data, err := sc.fetchBytes(ctx, sc.SourceURL+"/"+entry.Location.Href)
if err != nil {
return nil, err
}
if entry.Checksum.Type == "sha256" {
if err := verifyChecksum(data, entry.Checksum.Value); err != nil {
return nil, fmt.Errorf("primary.xml.gz: %w", err)
}
}
// Save upstream primary to disk so RegenerateMetadata can use it as source.
dest := filepath.Join(sc.LocalDir, filepath.FromSlash(entry.Location.Href))
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err == nil {
if err := writeFile(dest, data); err == nil {
marker := filepath.Join(sc.LocalDir, "repodata", PrimarySourceFile)
_ = os.WriteFile(marker, []byte(entry.Location.Href), 0o644)
}
}
gz, err := gzip.NewReader(strings.NewReader(string(data)))
if err != nil {
return nil, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return nil, fmt.Errorf("parse primary.xml: %w", err)
}
return primary.Packages, nil
}
func (sc *Scanner) fetchBytes(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := sc.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
return io.ReadAll(resp.Body)
}

83
internal/core/clone.go Normal file
View file

@ -0,0 +1,83 @@
package core
import (
"context"
"errors"
"fmt"
"log"
"path/filepath"
"github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
var ErrCloneAlreadyRunning = errors.New("a clone job is already running for this repo")
type CloneService struct {
repoStore store.RepoStore
jobStore store.CloneJobStore
dataDir string
}
func NewCloneService(repoStore store.RepoStore, jobStore store.CloneJobStore, dataDir string) *CloneService {
return &CloneService{repoStore: repoStore, jobStore: jobStore, dataDir: dataDir}
}
func (s *CloneService) StartClone(ctx context.Context, repoID int64) (int64, error) {
repo, err := s.repoStore.GetRepo(ctx, repoID)
if err != nil {
return 0, err
}
if repo.Type != "rpm" {
return 0, fmt.Errorf("clone is only supported for rpm repositories")
}
running, err := s.jobStore.HasRunningCloneJob(ctx, repoID)
if err != nil {
return 0, err
}
if running {
return 0, ErrCloneAlreadyRunning
}
jobID, err := s.jobStore.CreateCloneJob(ctx, repoID)
if err != nil {
return 0, err
}
go s.runClone(repoID, jobID, repo.SourceURL, repo.Type)
return jobID, nil
}
func (s *CloneService) GetLatestCloneJob(ctx context.Context, repoID int64) (*store.CloneJob, error) {
return s.jobStore.GetLatestCloneJob(ctx, repoID)
}
func (s *CloneService) runClone(repoID, jobID int64, sourceURL, repoType string) {
ctx := context.Background()
if err := s.jobStore.MarkCloneJobStarted(ctx, jobID); err != nil {
log.Printf("clone job %d: failed to mark started: %v", jobID, err)
return
}
destDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), repoType)
cloner := rpm.New(sourceURL, destDir)
cloner.OnProgress = func(p rpm.Progress) {
log.Printf("clone job %d: %s (%d bytes)", jobID, p.File, p.BytesDone)
}
if err := cloner.Clone(ctx); err != nil {
errMsg := err.Error()
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobFailed, &errMsg)
log.Printf("clone job %d: failed: %v", jobID, err)
return
}
if err := rpm.RegenerateMetadata(destDir); err != nil {
log.Printf("clone job %d: metadata regeneration failed: %v", jobID, err)
}
_ = s.jobStore.MarkCloneJobFinished(ctx, jobID, store.CloneJobCompleted, nil)
log.Printf("clone job %d: completed", jobID)
}

5
internal/core/errors.go Normal file
View file

@ -0,0 +1,5 @@
package core
import "github.com/syonad/clonepack/internal/store"
var ErrNotFound = store.ErrNotFound

82
internal/core/repo.go Normal file
View file

@ -0,0 +1,82 @@
package core
import (
"context"
"fmt"
"log"
"path/filepath"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
var validTypes = map[string]bool{"apt": true, "rpm": true, "docker": true, "binary": true}
type RepoService struct {
store store.RepoStore
dataDir string
}
func NewRepoService(s store.RepoStore, dataDir string) *RepoService {
return &RepoService{store: s, dataDir: dataDir}
}
var validSyncModes = map[string]bool{"auto": true, "manual": true}
type CreateRepoInput struct {
Name string `json:"name"`
Type string `json:"type"`
SourceURL string `json:"source_url"`
SyncMode string `json:"sync_mode"`
}
func (s *RepoService) Create(ctx context.Context, in CreateRepoInput) (*store.Repo, error) {
if in.Name == "" {
return nil, fmt.Errorf("name is required")
}
if !validTypes[in.Type] {
return nil, fmt.Errorf("invalid type: %s (must be apt, rpm, docker, or binary)", in.Type)
}
if in.SourceURL == "" {
return nil, fmt.Errorf("source_url is required")
}
if in.SyncMode == "" {
in.SyncMode = "auto"
} else if !validSyncModes[in.SyncMode] {
return nil, fmt.Errorf("invalid sync_mode: %s (must be auto or manual)", in.SyncMode)
}
r := &store.Repo{Name: in.Name, Type: in.Type, SourceURL: in.SourceURL, SyncMode: in.SyncMode}
id, err := s.store.CreateRepo(ctx, r)
if err != nil {
return nil, err
}
repo, err := s.store.GetRepo(ctx, id)
if err != nil {
return nil, err
}
s.initStorage(repo)
return repo, nil
}
func (s *RepoService) initStorage(repo *store.Repo) {
switch repo.Type {
case "rpm":
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repo.ID), "rpm")
if err := rpmclone.InitEmptyRepo(localDir); err != nil {
log.Printf("init storage for repo %d: %v", repo.ID, err)
}
}
}
func (s *RepoService) List(ctx context.Context) ([]store.Repo, error) {
return s.store.ListRepos(ctx)
}
func (s *RepoService) Get(ctx context.Context, id int64) (*store.Repo, error) {
return s.store.GetRepo(ctx, id)
}
func (s *RepoService) Delete(ctx context.Context, id int64) error {
return s.store.DeleteRepo(ctx, id)
}

260
internal/core/snapshot.go Normal file
View file

@ -0,0 +1,260 @@
package core
import (
"bytes"
"compress/gzip"
"context"
"encoding/xml"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"time"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
type SnapshotService struct {
snapshotStore store.SnapshotStore
repoStore store.RepoStore
dataDir string
httpClient *http.Client
}
func NewSnapshotService(
snapshotStore store.SnapshotStore,
repoStore store.RepoStore,
dataDir string,
) *SnapshotService {
return &SnapshotService{
snapshotStore: snapshotStore,
repoStore: repoStore,
dataDir: dataDir,
httpClient: &http.Client{Timeout: 30 * time.Minute},
}
}
type SnapshotDiff struct {
From *store.Snapshot
To *store.Snapshot
Added []store.SnapshotPackage
Removed []store.SnapshotPackage
Unchanged int
}
func (s *SnapshotService) TakeSnapshot(ctx context.Context, repoID int64, label string) (int64, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return 0, err
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
repomdData, err := os.ReadFile(filepath.Join(localDir, "repodata", "repomd.xml"))
if err != nil {
return 0, fmt.Errorf("read repomd.xml: %w", err)
}
var repomd rpmclone.RepoMD
if err := xml.Unmarshal(repomdData, &repomd); err != nil {
return 0, fmt.Errorf("parse repomd.xml: %w", err)
}
var primaryHref string
for _, entry := range repomd.Data {
if entry.Type == "primary" {
primaryHref = entry.Location.Href
break
}
}
if primaryHref == "" {
return 0, fmt.Errorf("no primary entry in repomd.xml")
}
gzData, err := os.ReadFile(filepath.Join(localDir, filepath.FromSlash(primaryHref)))
if err != nil {
return 0, fmt.Errorf("read primary.xml.gz: %w", err)
}
gz, err := gzip.NewReader(bytes.NewReader(gzData))
if err != nil {
return 0, fmt.Errorf("open gzip: %w", err)
}
defer gz.Close()
var primary rpmclone.PrimaryMetadata
if err := xml.NewDecoder(gz).Decode(&primary); err != nil {
return 0, fmt.Errorf("parse primary.xml: %w", err)
}
pkgs := make([]store.SnapshotPackage, len(primary.Packages))
for i, p := range primary.Packages {
pkgs[i] = store.SnapshotPackage{
Name: p.Name,
Version: p.Version.Ver + "-" + p.Version.Rel,
Arch: p.Arch,
Location: p.Location.Href,
Checksum: p.Checksum.Value,
ChecksumType: p.Checksum.Type,
Size: p.Size.Package,
}
}
snapID, err := s.snapshotStore.CreateSnapshot(ctx, repoID, label)
if err != nil {
return 0, err
}
if len(pkgs) > 0 {
if err := s.snapshotStore.AddSnapshotPackages(ctx, snapID, pkgs); err != nil {
return 0, err
}
}
return snapID, nil
}
func (s *SnapshotService) List(ctx context.Context, repoID int64) ([]store.Snapshot, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return nil, err
}
return s.snapshotStore.ListSnapshots(ctx, repoID)
}
func (s *SnapshotService) Get(ctx context.Context, repoID, id int64) (*store.Snapshot, []store.SnapshotPackage, error) {
snap, err := s.snapshotStore.GetSnapshot(ctx, id)
if err != nil {
return nil, nil, err
}
if snap.RepoID != repoID {
return nil, nil, store.ErrNotFound
}
pkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, id)
if err != nil {
return nil, nil, err
}
return snap, pkgs, nil
}
func (s *SnapshotService) Delete(ctx context.Context, repoID, id int64) error {
snap, err := s.snapshotStore.GetSnapshot(ctx, id)
if err != nil {
return err
}
if snap.RepoID != repoID {
return store.ErrNotFound
}
return s.snapshotStore.DeleteSnapshot(ctx, id)
}
func (s *SnapshotService) Diff(ctx context.Context, repoID, fromID, toID int64) (*SnapshotDiff, error) {
from, err := s.snapshotStore.GetSnapshot(ctx, fromID)
if err != nil {
return nil, err
}
to, err := s.snapshotStore.GetSnapshot(ctx, toID)
if err != nil {
return nil, err
}
if from.RepoID != repoID || to.RepoID != repoID {
return nil, fmt.Errorf("snapshots do not belong to repo %d", repoID)
}
fromPkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, fromID)
if err != nil {
return nil, err
}
toPkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, toID)
if err != nil {
return nil, err
}
fromSet := make(map[string]struct{}, len(fromPkgs))
for _, p := range fromPkgs {
fromSet[p.Name+"|"+p.Version+"|"+p.Arch] = struct{}{}
}
toSet := make(map[string]struct{}, len(toPkgs))
for _, p := range toPkgs {
toSet[p.Name+"|"+p.Version+"|"+p.Arch] = struct{}{}
}
diff := &SnapshotDiff{From: from, To: to}
for _, p := range toPkgs {
if _, ok := fromSet[p.Name+"|"+p.Version+"|"+p.Arch]; ok {
diff.Unchanged++
} else {
diff.Added = append(diff.Added, p)
}
}
for _, p := range fromPkgs {
if _, ok := toSet[p.Name+"|"+p.Version+"|"+p.Arch]; !ok {
diff.Removed = append(diff.Removed, p)
}
}
return diff, nil
}
func (s *SnapshotService) Rollback(ctx context.Context, repoID, snapshotID int64) error {
repo, err := s.repoStore.GetRepo(ctx, repoID)
if err != nil {
return err
}
snap, err := s.snapshotStore.GetSnapshot(ctx, snapshotID)
if err != nil {
return err
}
if snap.RepoID != repoID {
return store.ErrNotFound
}
pkgs, err := s.snapshotStore.GetSnapshotPackages(ctx, snapshotID)
if err != nil {
return err
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
snapSet := make(map[string]store.SnapshotPackage, len(pkgs))
for _, p := range pkgs {
snapSet[p.Location] = p
}
var errs []error
if err := filepath.WalkDir(localDir, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() || filepath.Ext(path) != ".rpm" {
return err
}
rel, _ := filepath.Rel(localDir, path)
rel = filepath.ToSlash(rel)
if _, ok := snapSet[rel]; !ok {
if rmErr := os.Remove(path); rmErr != nil {
errs = append(errs, rmErr)
}
}
return nil
}); err != nil {
errs = append(errs, err)
}
for _, p := range pkgs {
dest := filepath.Join(localDir, filepath.FromSlash(p.Location))
if _, err := os.Stat(dest); err == nil {
continue
}
url := strings.TrimRight(repo.SourceURL, "/") + "/" + p.Location
if _, dlErr := rpmclone.DownloadAndVerify(ctx, s.httpClient, url, dest, p.ChecksumType, p.Checksum); dlErr != nil {
errs = append(errs, fmt.Errorf("re-download %s: %w", p.Name, dlErr))
}
}
if err := errors.Join(errs...); err != nil {
return err
}
if repo.Type == "rpm" {
if err := rpmclone.RegenerateMetadata(localDir); err != nil {
return fmt.Errorf("regenerate metadata: %w", err)
}
}
return nil
}

268
internal/core/sync.go Normal file
View file

@ -0,0 +1,268 @@
package core
import (
"context"
"errors"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
rpmclone "github.com/syonad/clonepack/internal/clone/rpm"
"github.com/syonad/clonepack/internal/store"
)
type SyncService struct {
repoStore store.RepoStore
pendingStore store.PendingPackageStore
blockedStore store.BlockedPackageStore
cloneSvc *CloneService
snapshotSvc *SnapshotService
dataDir string
httpClient *http.Client
}
func NewSyncService(
repoStore store.RepoStore,
pendingStore store.PendingPackageStore,
blockedStore store.BlockedPackageStore,
cloneSvc *CloneService,
snapshotSvc *SnapshotService,
dataDir string,
) *SyncService {
return &SyncService{
repoStore: repoStore,
pendingStore: pendingStore,
blockedStore: blockedStore,
cloneSvc: cloneSvc,
snapshotSvc: snapshotSvc,
dataDir: dataDir,
httpClient: &http.Client{Timeout: 5 * time.Minute},
}
}
func (s *SyncService) ValidateRepo(ctx context.Context, repoID int64) error {
_, err := s.repoStore.GetRepo(ctx, repoID)
return err
}
func (s *SyncService) ScanRepo(ctx context.Context, repoID int64) error {
repo, err := s.repoStore.GetRepo(ctx, repoID)
if err != nil {
return err
}
if repo.Type != "rpm" {
return fmt.Errorf("sync only supported for rpm repos")
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
scanner := rpmclone.NewScanner(repo.SourceURL, localDir)
newPkgs, err := scanner.Scan(ctx)
if err != nil {
return fmt.Errorf("scan repo %d: %w", repoID, err)
}
if len(newPkgs) == 0 {
log.Printf("sync: repo %d is up to date", repoID)
return nil
}
blocked, err := s.blockedStore.ListBlocked(ctx, repoID)
if err != nil {
return fmt.Errorf("list blocked for repo %d: %w", repoID, err)
}
blockedLocations := make(map[string]bool, len(blocked))
blockedNames := make(map[string]bool, len(blocked))
for _, b := range blocked {
blockedLocations[b.Location] = true
if b.Name != "" {
blockedNames[b.Name] = true
}
}
filtered := newPkgs[:0]
for _, p := range newPkgs {
if !blockedLocations[p.Location] && !blockedNames[p.Name] {
filtered = append(filtered, p)
}
}
newPkgs = filtered
if len(newPkgs) == 0 {
log.Printf("sync: repo %d is up to date (all new packages are blocked)", repoID)
return nil
}
log.Printf("sync: repo %d has %d new package(s)", repoID, len(newPkgs))
switch repo.SyncMode {
case "auto":
_, err = s.cloneSvc.StartClone(ctx, repoID)
if err != nil && err != ErrCloneAlreadyRunning {
return fmt.Errorf("start clone for repo %d: %w", repoID, err)
}
case "manual":
pending := make([]store.PendingPackage, len(newPkgs))
for i, p := range newPkgs {
pending[i] = store.PendingPackage{
RepoID: repoID,
Name: p.Name,
Version: p.Version,
Arch: p.Arch,
Location: p.Location,
Checksum: p.Checksum,
ChecksumType: p.ChecksumType,
Size: p.Size,
}
}
if err := s.pendingStore.UpsertPending(ctx, pending); err != nil {
return fmt.Errorf("upsert pending for repo %d: %w", repoID, err)
}
default:
return fmt.Errorf("unknown sync_mode %q for repo %d", repo.SyncMode, repoID)
}
return nil
}
func (s *SyncService) ListPending(ctx context.Context, repoID int64) ([]store.PendingPackage, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return nil, err
}
return s.pendingStore.ListPending(ctx, repoID)
}
func (s *SyncService) ApprovePending(ctx context.Context, repoID int64, ids []int64) error {
repo, err := s.repoStore.GetRepo(ctx, repoID)
if err != nil {
return err
}
all, err := s.pendingStore.ListPending(ctx, repoID)
if err != nil {
return err
}
wanted := make(map[int64]bool, len(ids))
for _, id := range ids {
wanted[id] = true
}
localDir := filepath.Join(s.dataDir, "repos", fmt.Sprintf("%d", repoID), "rpm")
var downloadErrors []error
var anyApproved bool
for _, pkg := range all {
if !wanted[pkg.ID] {
continue
}
url := strings.TrimRight(repo.SourceURL, "/") + "/" + pkg.Location
dest := filepath.Join(localDir, filepath.FromSlash(pkg.Location))
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
downloadErrors = append(downloadErrors, fmt.Errorf("mkdirall %s: %w", pkg.Location, err))
continue
}
if _, err := rpmclone.DownloadAndVerify(ctx, s.httpClient, url, dest, pkg.ChecksumType, pkg.Checksum); err != nil {
downloadErrors = append(downloadErrors, fmt.Errorf("download %s: %w", pkg.Name, err))
continue
}
// Supprime immédiatement de la liste — même si la suite échoue, le paquet est acquis.
if err := s.pendingStore.DeletePending(ctx, []int64{pkg.ID}); err != nil {
log.Printf("delete pending %d: %v", pkg.ID, err)
}
anyApproved = true
}
if anyApproved {
if repo.Type == "rpm" {
if err := rpmclone.RegenerateMetadata(localDir); err != nil {
log.Printf("metadata regeneration for repo %d failed: %v", repoID, err)
}
}
label := "auto-" + time.Now().UTC().Format(time.RFC3339)
if _, err := s.snapshotSvc.TakeSnapshot(context.Background(), repoID, label); err != nil {
log.Printf("auto-snapshot for repo %d failed: %v", repoID, err)
}
}
return errors.Join(downloadErrors...)
}
func (s *SyncService) BlockPackages(ctx context.Context, repoID int64, pendingIDs []int64) error {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return err
}
all, err := s.pendingStore.ListPending(ctx, repoID)
if err != nil {
return err
}
wanted := make(map[int64]bool, len(pendingIDs))
for _, id := range pendingIDs {
wanted[id] = true
}
var toBlock []store.BlockedPackage
var toDelete []int64
for _, p := range all {
if wanted[p.ID] {
toBlock = append(toBlock, store.BlockedPackage{Name: p.Name, Location: p.Location})
toDelete = append(toDelete, p.ID)
}
}
if len(toBlock) == 0 {
return nil
}
if err := s.blockedStore.BlockPackages(ctx, repoID, toBlock); err != nil {
return err
}
return s.pendingStore.DeletePending(ctx, toDelete)
}
func (s *SyncService) UnblockPackages(ctx context.Context, repoID int64, ids []int64) error {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return err
}
return s.blockedStore.UnblockPackages(ctx, repoID, ids)
}
func (s *SyncService) ListBlocked(ctx context.Context, repoID int64) ([]store.BlockedPackage, error) {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return nil, err
}
return s.blockedStore.ListBlocked(ctx, repoID)
}
func (s *SyncService) RejectPending(ctx context.Context, repoID int64, ids []int64) error {
if _, err := s.repoStore.GetRepo(ctx, repoID); err != nil {
return err
}
return s.pendingStore.DeletePending(ctx, ids)
}
func (s *SyncService) StartScheduler(ctx context.Context, interval time.Duration) {
go func() {
log.Printf("sync scheduler started (interval=%s)", interval)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
log.Printf("sync scheduler stopped")
return
case <-ticker.C:
s.runScheduledScan(ctx)
}
}
}()
}
func (s *SyncService) runScheduledScan(ctx context.Context) {
repos, err := s.repoStore.ListRepos(ctx)
if err != nil {
log.Printf("sync scheduler: list repos error: %v", err)
return
}
for _, repo := range repos {
if err := s.ScanRepo(ctx, repo.ID); err != nil {
log.Printf("sync scheduler: scan repo %d error: %v", repo.ID, err)
}
}
}

78
internal/store/blocked_package.go Normal file
View file

@ -0,0 +1,78 @@
package store
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
)
type BlockedPackage struct {
ID int64
RepoID int64
Name string
Location string
CreatedAt time.Time
}
type BlockedPackageStore interface {
BlockPackages(ctx context.Context, repoID int64, pkgs []BlockedPackage) error
UnblockPackages(ctx context.Context, repoID int64, ids []int64) error
ListBlocked(ctx context.Context, repoID int64) ([]BlockedPackage, error)
}
type SQLiteBlockedPackageStore struct {
db *sql.DB
}
func NewBlockedPackageStore(db *sql.DB) *SQLiteBlockedPackageStore {
return &SQLiteBlockedPackageStore{db: db}
}
func (s *SQLiteBlockedPackageStore) BlockPackages(ctx context.Context, repoID int64, pkgs []BlockedPackage) error {
for _, p := range pkgs {
_, err := s.db.ExecContext(ctx,
`INSERT OR IGNORE INTO blocked_packages (repo_id, name, location) VALUES (?, ?, ?)`,
repoID, p.Name, p.Location)
if err != nil {
return err
}
}
return nil
}
func (s *SQLiteBlockedPackageStore) UnblockPackages(ctx context.Context, repoID int64, ids []int64) error {
if len(ids) == 0 {
return nil
}
placeholders := strings.Join(strings.Fields(strings.Repeat("? ", len(ids))), ", ")
query := fmt.Sprintf("DELETE FROM blocked_packages WHERE repo_id = ? AND id IN (%s)", placeholders)
args := make([]any, 0, len(ids)+1)
args = append(args, repoID)
for _, id := range ids {
args = append(args, id)
}
_, err := s.db.ExecContext(ctx, query, args...)
return err
}
func (s *SQLiteBlockedPackageStore) ListBlocked(ctx context.Context, repoID int64) ([]BlockedPackage, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT id, repo_id, name, location, created_at FROM blocked_packages WHERE repo_id = ? ORDER BY id ASC`,
repoID)
if err != nil {
return nil, err
}
defer rows.Close()
var pkgs []BlockedPackage
for rows.Next() {
var p BlockedPackage
if err := rows.Scan(&p.ID, &p.RepoID, &p.Name, &p.Location, &p.CreatedAt); err != nil {
return nil, err
}
pkgs = append(pkgs, p)
}
return pkgs, rows.Err()
}

109
internal/store/clone_job.go Normal file
View file

@ -0,0 +1,109 @@
package store
import (
"context"
"database/sql"
"errors"
"time"
)
type CloneJobStatus string
const (
CloneJobPending CloneJobStatus = "pending"
CloneJobRunning CloneJobStatus = "running"
CloneJobCompleted CloneJobStatus = "completed"
CloneJobFailed CloneJobStatus = "failed"
)
type CloneJob struct {
ID int64
RepoID int64
Status CloneJobStatus
StartedAt *time.Time
FinishedAt *time.Time
Error *string
CreatedAt time.Time
}
type CloneJobStore interface {
CreateCloneJob(ctx context.Context, repoID int64) (int64, error)
GetCloneJob(ctx context.Context, id int64) (*CloneJob, error)
GetLatestCloneJob(ctx context.Context, repoID int64) (*CloneJob, error)
HasRunningCloneJob(ctx context.Context, repoID int64) (bool, error)
MarkCloneJobStarted(ctx context.Context, id int64) error
MarkCloneJobFinished(ctx context.Context, id int64, status CloneJobStatus, errMsg *string) error
}
type SQLiteCloneJobStore struct {
db *sql.DB
}
func NewCloneJobStore(db *sql.DB) *SQLiteCloneJobStore {
return &SQLiteCloneJobStore{db: db}
}
func (s *SQLiteCloneJobStore) CreateCloneJob(ctx context.Context, repoID int64) (int64, error) {
res, err := s.db.ExecContext(ctx, `INSERT INTO clone_jobs (repo_id) VALUES (?)`, repoID)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func (s *SQLiteCloneJobStore) GetCloneJob(ctx context.Context, id int64) (*CloneJob, error) {
row := s.db.QueryRowContext(ctx,
`SELECT id, repo_id, status, started_at, finished_at, error, created_at FROM clone_jobs WHERE id = ?`, id)
return scanCloneJob(row)
}
func (s *SQLiteCloneJobStore) GetLatestCloneJob(ctx context.Context, repoID int64) (*CloneJob, error) {
row := s.db.QueryRowContext(ctx,
`SELECT id, repo_id, status, started_at, finished_at, error, created_at FROM clone_jobs WHERE repo_id = ? ORDER BY created_at DESC LIMIT 1`, repoID)
return scanCloneJob(row)
}
func (s *SQLiteCloneJobStore) HasRunningCloneJob(ctx context.Context, repoID int64) (bool, error) {
var count int
err := s.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM clone_jobs WHERE repo_id = ? AND status IN ('pending','running')`, repoID,
).Scan(&count)
return count > 0, err
}
func (s *SQLiteCloneJobStore) MarkCloneJobStarted(ctx context.Context, id int64) error {
_, err := s.db.ExecContext(ctx,
`UPDATE clone_jobs SET status='running', started_at=CURRENT_TIMESTAMP WHERE id=?`, id)
return err
}
func (s *SQLiteCloneJobStore) MarkCloneJobFinished(ctx context.Context, id int64, status CloneJobStatus, errMsg *string) error {
_, err := s.db.ExecContext(ctx,
`UPDATE clone_jobs SET status=?, finished_at=CURRENT_TIMESTAMP, error=? WHERE id=?`,
status, errMsg, id)
return err
}
func scanCloneJob(row *sql.Row) (*CloneJob, error) {
var j CloneJob
var startedAt, finishedAt sql.NullTime
var errMsg sql.NullString
err := row.Scan(&j.ID, &j.RepoID, &j.Status, &startedAt, &finishedAt, &errMsg, &j.CreatedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
if startedAt.Valid {
j.StartedAt = &startedAt.Time
}
if finishedAt.Valid {
j.FinishedAt = &finishedAt.Time
}
if errMsg.Valid {
j.Error = &errMsg.String
}
return &j, nil
}

88
internal/store/db.go Normal file
View file

@ -0,0 +1,88 @@
package store
import (
"database/sql"
_ "embed"
"fmt"
"github.com/syonad/clonepack/config"
_ "modernc.org/sqlite"
)
//go:embed migrations/000001_init.up.sql
var migration001 string
//go:embed migrations/000002_clone_jobs.up.sql
var migration002 string
//go:embed migrations/000003_sync.up.sql
var migration003 string
//go:embed migrations/000004_snapshot_packages.up.sql
var migration004 string
//go:embed migrations/000005_blocked_packages.up.sql
var migration005 string
//go:embed migrations/000006_blocked_packages_name.up.sql
var migration006 string
func Open(cfg config.DBConfig) (*sql.DB, error) {
db, err := sql.Open("sqlite", cfg.Path)
if err != nil {
return nil, fmt.Errorf("open db: %w", err)
}
if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
return nil, fmt.Errorf("set WAL mode: %w", err)
}
if _, err := db.Exec("PRAGMA foreign_keys=ON"); err != nil {
return nil, fmt.Errorf("enable foreign keys: %w", err)
}
if err := runMigrations(db); err != nil {
return nil, fmt.Errorf("migrations: %w", err)
}
return db, nil
}
func runMigrations(db *sql.DB) error {
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
)`)
if err != nil {
return err
}
migrations := []struct {
version int
sql string
}{
{1, migration001},
{2, migration002},
{3, migration003},
{4, migration004},
{5, migration005},
{6, migration006},
}
for _, m := range migrations {
var count int
row := db.QueryRow("SELECT COUNT(*) FROM schema_migrations WHERE version = ?", m.version)
if err := row.Scan(&count); err != nil {
return err
}
if count > 0 {
continue
}
if _, err := db.Exec(m.sql); err != nil {
return fmt.Errorf("migration %d: %w", m.version, err)
}
if _, err := db.Exec("INSERT INTO schema_migrations (version) VALUES (?)", m.version); err != nil {
return err
}
}
return nil
}

3
internal/store/migrations/000001_init.down.sql Normal file
View file

@ -0,0 +1,3 @@
DROP TABLE IF EXISTS snapshots;
DROP TABLE IF EXISTS artifacts;
DROP TABLE IF EXISTS repos;

25
internal/store/migrations/000001_init.up.sql Normal file
View file

@ -0,0 +1,25 @@
CREATE TABLE IF NOT EXISTS repos (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
type TEXT NOT NULL CHECK(type IN ('apt','rpm','docker','binary')),
source_url TEXT NOT NULL,
frozen BOOLEAN NOT NULL DEFAULT FALSE,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS artifacts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
name TEXT NOT NULL,
version TEXT NOT NULL,
path TEXT NOT NULL,
checksum TEXT NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
label TEXT NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);

1
internal/store/migrations/000002_clone_jobs.down.sql Normal file
View file

@ -0,0 +1 @@
DROP TABLE IF EXISTS clone_jobs;

11
internal/store/migrations/000002_clone_jobs.up.sql Normal file
View file

@ -0,0 +1,11 @@
CREATE TABLE IF NOT EXISTS clone_jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
status TEXT NOT NULL CHECK(status IN ('pending','running','completed','failed')) DEFAULT 'pending',
started_at DATETIME,
finished_at DATETIME,
error TEXT,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_clone_jobs_repo_id ON clone_jobs(repo_id);

1
internal/store/migrations/000003_sync.down.sql Normal file
View file

@ -0,0 +1 @@
DROP TABLE IF EXISTS pending_packages;

18
internal/store/migrations/000003_sync.up.sql Normal file
View file

@ -0,0 +1,18 @@
ALTER TABLE repos ADD COLUMN sync_mode TEXT NOT NULL DEFAULT 'auto'
CHECK(sync_mode IN ('auto', 'manual'));
CREATE TABLE IF NOT EXISTS pending_packages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
name TEXT NOT NULL,
version TEXT NOT NULL,
arch TEXT NOT NULL,
location TEXT NOT NULL,
checksum TEXT NOT NULL,
checksum_type TEXT NOT NULL,
size INTEGER NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(repo_id, name, version, arch)
);
CREATE INDEX IF NOT EXISTS idx_pending_packages_repo_id ON pending_packages(repo_id);

2
internal/store/migrations/000004_snapshot_packages.down.sql Normal file
View file

@ -0,0 +1,2 @@
DROP INDEX IF EXISTS idx_snapshot_packages_snapshot_id;
DROP TABLE IF EXISTS snapshot_packages;

14
internal/store/migrations/000004_snapshot_packages.up.sql Normal file
View file

@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS snapshot_packages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
snapshot_id INTEGER NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE,
name TEXT NOT NULL,
version TEXT NOT NULL,
arch TEXT NOT NULL,
location TEXT NOT NULL,
checksum TEXT NOT NULL,
checksum_type TEXT NOT NULL,
size INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_snapshot_packages_snapshot_id
ON snapshot_packages(snapshot_id);

1
internal/store/migrations/000005_blocked_packages.down.sql Normal file
View file

@ -0,0 +1 @@
DROP TABLE IF EXISTS blocked_packages;

7
internal/store/migrations/000005_blocked_packages.up.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS blocked_packages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
location TEXT NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(repo_id, location)
);

4
internal/store/migrations/000006_blocked_packages_name.down.sql Normal file
View file

@ -0,0 +1,4 @@
-- SQLite ne supporte pas DROP COLUMN avant 3.35 ; recréation de la table sans la colonne name
CREATE TABLE blocked_packages_backup AS SELECT id, repo_id, location, created_at FROM blocked_packages;
DROP TABLE blocked_packages;
ALTER TABLE blocked_packages_backup RENAME TO blocked_packages;

1
internal/store/migrations/000006_blocked_packages_name.up.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE blocked_packages ADD COLUMN name TEXT NOT NULL DEFAULT '';

92
internal/store/pending_package.go Normal file
View file

@ -0,0 +1,92 @@
package store
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
)
type PendingPackage struct {
ID int64
RepoID int64
Name string
Version string
Arch string
Location string
Checksum string
ChecksumType string
Size int64
CreatedAt time.Time
}
type PendingPackageStore interface {
UpsertPending(ctx context.Context, pkgs []PendingPackage) error
ListPending(ctx context.Context, repoID int64) ([]PendingPackage, error)
DeletePending(ctx context.Context, ids []int64) error
DeleteAllPending(ctx context.Context, repoID int64) error
}
type SQLitePendingPackageStore struct {
db *sql.DB
}
func NewPendingPackageStore(db *sql.DB) *SQLitePendingPackageStore {
return &SQLitePendingPackageStore{db: db}
}
func (s *SQLitePendingPackageStore) UpsertPending(ctx context.Context, pkgs []PendingPackage) error {
for _, p := range pkgs {
_, err := s.db.ExecContext(ctx,
`INSERT OR IGNORE INTO pending_packages
(repo_id, name, version, arch, location, checksum, checksum_type, size)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
p.RepoID, p.Name, p.Version, p.Arch, p.Location, p.Checksum, p.ChecksumType, p.Size,
)
if err != nil {
return err
}
}
return nil
}
func (s *SQLitePendingPackageStore) ListPending(ctx context.Context, repoID int64) ([]PendingPackage, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT id, repo_id, name, version, arch, location, checksum, checksum_type, size, created_at
FROM pending_packages WHERE repo_id = ? ORDER BY created_at ASC`, repoID)
if err != nil {
return nil, err
}
defer rows.Close()
var pkgs []PendingPackage
for rows.Next() {
var p PendingPackage
if err := rows.Scan(&p.ID, &p.RepoID, &p.Name, &p.Version, &p.Arch,
&p.Location, &p.Checksum, &p.ChecksumType, &p.Size, &p.CreatedAt); err != nil {
return nil, err
}
pkgs = append(pkgs, p)
}
return pkgs, rows.Err()
}
func (s *SQLitePendingPackageStore) DeletePending(ctx context.Context, ids []int64) error {
if len(ids) == 0 {
return nil
}
placeholders := strings.Join(strings.Fields(strings.Repeat("? ", len(ids))), ", ")
query := fmt.Sprintf("DELETE FROM pending_packages WHERE id IN (%s)", placeholders)
args := make([]any, len(ids))
for i, id := range ids {
args[i] = id
}
_, err := s.db.ExecContext(ctx, query, args...)
return err
}
func (s *SQLitePendingPackageStore) DeleteAllPending(ctx context.Context, repoID int64) error {
_, err := s.db.ExecContext(ctx, `DELETE FROM pending_packages WHERE repo_id = ?`, repoID)
return err
}

107
internal/store/repo.go Normal file
View file

@ -0,0 +1,107 @@
package store
import (
"context"
"database/sql"
"errors"
"time"
)
var ErrNotFound = errors.New("not found")
type Repo struct {
ID int64 `db:"id"`
Name string `db:"name"`
Type string `db:"type"`
SourceURL string `db:"source_url"`
Frozen bool `db:"frozen"`
SyncMode string `db:"sync_mode"`
CreatedAt time.Time `db:"created_at"`
}
type RepoStore interface {
CreateRepo(ctx context.Context, r *Repo) (int64, error)
ListRepos(ctx context.Context) ([]Repo, error)
GetRepo(ctx context.Context, id int64) (*Repo, error)
DeleteRepo(ctx context.Context, id int64) error
UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error
}
type SQLiteRepoStore struct {
db *sql.DB
}
func NewRepoStore(db *sql.DB) *SQLiteRepoStore {
return &SQLiteRepoStore{db: db}
}
func (s *SQLiteRepoStore) CreateRepo(ctx context.Context, r *Repo) (int64, error) {
res, err := s.db.ExecContext(ctx,
`INSERT INTO repos (name, type, source_url, frozen, sync_mode) VALUES (?, ?, ?, ?, ?)`,
r.Name, r.Type, r.SourceURL, r.Frozen, r.SyncMode,
)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func (s *SQLiteRepoStore) ListRepos(ctx context.Context) ([]Repo, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos ORDER BY created_at DESC`)
if err != nil {
return nil, err
}
defer rows.Close()
var repos []Repo
for rows.Next() {
var r Repo
if err := rows.Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt); err != nil {
return nil, err
}
repos = append(repos, r)
}
return repos, rows.Err()
}
func (s *SQLiteRepoStore) GetRepo(ctx context.Context, id int64) (*Repo, error) {
var r Repo
err := s.db.QueryRowContext(ctx,
`SELECT id, name, type, source_url, frozen, sync_mode, created_at FROM repos WHERE id = ?`, id,
).Scan(&r.ID, &r.Name, &r.Type, &r.SourceURL, &r.Frozen, &r.SyncMode, &r.CreatedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
return &r, nil
}
func (s *SQLiteRepoStore) DeleteRepo(ctx context.Context, id int64) error {
res, err := s.db.ExecContext(ctx, `DELETE FROM repos WHERE id = ?`, id)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return err
}
if n == 0 {
return ErrNotFound
}
return nil
}
func (s *SQLiteRepoStore) UpdateRepoSyncMode(ctx context.Context, id int64, mode string) error {
res, err := s.db.ExecContext(ctx, `UPDATE repos SET sync_mode = ? WHERE id = ?`, mode, id)
if err != nil {
return err
}
n, _ := res.RowsAffected()
if n == 0 {
return ErrNotFound
}
return nil
}

144
internal/store/snapshot.go Normal file
View file

@ -0,0 +1,144 @@
package store
import (
"context"
"database/sql"
"errors"
"time"
)
type Snapshot struct {
ID int64
RepoID int64
Label string
CreatedAt time.Time
}
type SnapshotPackage struct {
ID int64
SnapshotID int64
Name string
Version string
Arch string
Location string
Checksum string
ChecksumType string
Size int64
}
type SnapshotStore interface {
CreateSnapshot(ctx context.Context, repoID int64, label string) (int64, error)
AddSnapshotPackages(ctx context.Context, snapshotID int64, pkgs []SnapshotPackage) error
ListSnapshots(ctx context.Context, repoID int64) ([]Snapshot, error)
GetSnapshot(ctx context.Context, id int64) (*Snapshot, error)
GetSnapshotPackages(ctx context.Context, snapshotID int64) ([]SnapshotPackage, error)
DeleteSnapshot(ctx context.Context, id int64) error
}
type SQLiteSnapshotStore struct {
db *sql.DB
}
func NewSnapshotStore(db *sql.DB) *SQLiteSnapshotStore {
return &SQLiteSnapshotStore{db: db}
}
func (s *SQLiteSnapshotStore) CreateSnapshot(ctx context.Context, repoID int64, label string) (int64, error) {
res, err := s.db.ExecContext(ctx, `INSERT INTO snapshots (repo_id, label) VALUES (?, ?)`, repoID, label)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func (s *SQLiteSnapshotStore) AddSnapshotPackages(ctx context.Context, snapshotID int64, pkgs []SnapshotPackage) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
stmt, err := tx.PrepareContext(ctx, `INSERT INTO snapshot_packages
(snapshot_id, name, version, arch, location, checksum, checksum_type, size)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
defer stmt.Close()
for _, p := range pkgs {
if _, err := stmt.ExecContext(ctx, snapshotID, p.Name, p.Version, p.Arch, p.Location, p.Checksum, p.ChecksumType, p.Size); err != nil {
return err
}
}
return tx.Commit()
}
func (s *SQLiteSnapshotStore) ListSnapshots(ctx context.Context, repoID int64) ([]Snapshot, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT id, repo_id, label, created_at FROM snapshots WHERE repo_id = ? ORDER BY id DESC`, repoID)
if err != nil {
return nil, err
}
defer rows.Close()
var snaps []Snapshot
for rows.Next() {
var snap Snapshot
if err := rows.Scan(&snap.ID, &snap.RepoID, &snap.Label, &snap.CreatedAt); err != nil {
return nil, err
}
snaps = append(snaps, snap)
}
return snaps, rows.Err()
}
func (s *SQLiteSnapshotStore) GetSnapshot(ctx context.Context, id int64) (*Snapshot, error) {
var snap Snapshot
err := s.db.QueryRowContext(ctx,
`SELECT id, repo_id, label, created_at FROM snapshots WHERE id = ?`, id).
Scan(&snap.ID, &snap.RepoID, &snap.Label, &snap.CreatedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
return &snap, nil
}
func (s *SQLiteSnapshotStore) GetSnapshotPackages(ctx context.Context, snapshotID int64) ([]SnapshotPackage, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT id, snapshot_id, name, version, arch, location, checksum, checksum_type, size
FROM snapshot_packages WHERE snapshot_id = ?`, snapshotID)
if err != nil {
return nil, err
}
defer rows.Close()
var pkgs []SnapshotPackage
for rows.Next() {
var p SnapshotPackage
if err := rows.Scan(&p.ID, &p.SnapshotID, &p.Name, &p.Version, &p.Arch,
&p.Location, &p.Checksum, &p.ChecksumType, &p.Size); err != nil {
return nil, err
}
pkgs = append(pkgs, p)
}
return pkgs, rows.Err()
}
func (s *SQLiteSnapshotStore) DeleteSnapshot(ctx context.Context, id int64) error {
res, err := s.db.ExecContext(ctx, `DELETE FROM snapshots WHERE id = ?`, id)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return err
}
if n == 0 {
return ErrNotFound
}
return nil
}