Skip to content

Commit 20802d4

Browse files
dominikvagnerdominikvagner
authored
Fixes 4742: snapshot and bulk delete endpoints and task (#845)
* Fixes 4742: add delete snapshot and bulk delete endpoints and task * fix: lint errs and regen openapi * fix: update snapshots of templates after deletion * fix: update last snapshot refs in repo config * fix: update latest distribution on snap deletion * fix: change handler checks, cleanup * fix: openapi method for snapshot bulk delete * fix: error message check in unit tests * fix: add require pulp check to delete snapshots task * fix: unit test config mock
1 parent 663e860 commit 20802d4

31 files changed

+1953
-69
lines changed

‎api/docs.go

+118
Original file line numberDiff line numberDiff line change
@@ -925,6 +925,118 @@ const docTemplate = `{
925925
}
926926
}
927927
},
928+
"/repositories/{repo_uuid}/snapshots/bulk_delete/": {
929+
"post": {
930+
"description": "This enables deleting specified snapshots from a repository.",
931+
"tags": [
932+
"snapshots"
933+
],
934+
"summary": "Bulk delete a snapshots",
935+
"operationId": "bulkDeleteSnapshots",
936+
"parameters": [
937+
{
938+
"type": "string",
939+
"description": "Repository UUID.",
940+
"name": "repo_uuid",
941+
"in": "path",
942+
"required": true
943+
},
944+
{
945+
"description": "Identifiers of the snapshots",
946+
"name": "body",
947+
"in": "body",
948+
"required": true,
949+
"schema": {
950+
"$ref": "#/definitions/api.UUIDListRequest"
951+
}
952+
}
953+
],
954+
"responses": {
955+
"204": {
956+
"description": "Snapshots were successfully deleted"
957+
},
958+
"400": {
959+
"description": "Bad Request",
960+
"schema": {
961+
"$ref": "#/definitions/errors.ErrorResponse"
962+
}
963+
},
964+
"401": {
965+
"description": "Unauthorized",
966+
"schema": {
967+
"$ref": "#/definitions/errors.ErrorResponse"
968+
}
969+
},
970+
"404": {
971+
"description": "Not Found",
972+
"schema": {
973+
"$ref": "#/definitions/errors.ErrorResponse"
974+
}
975+
},
976+
"500": {
977+
"description": "Internal Server Error",
978+
"schema": {
979+
"$ref": "#/definitions/errors.ErrorResponse"
980+
}
981+
}
982+
}
983+
}
984+
},
985+
"/repositories/{repo_uuid}/snapshots/{snapshot_uuid}": {
986+
"delete": {
987+
"description": "This enables deleting a specific snapshot.",
988+
"tags": [
989+
"snapshots"
990+
],
991+
"summary": "Delete a snapshot",
992+
"operationId": "deleteSnapshot",
993+
"parameters": [
994+
{
995+
"type": "string",
996+
"description": "Repository UUID.",
997+
"name": "repo_uuid",
998+
"in": "path",
999+
"required": true
1000+
},
1001+
{
1002+
"type": "string",
1003+
"description": "Snapshot UUID.",
1004+
"name": "snapshot_uuid",
1005+
"in": "path",
1006+
"required": true
1007+
}
1008+
],
1009+
"responses": {
1010+
"204": {
1011+
"description": "Snapshot was successfully deleted"
1012+
},
1013+
"400": {
1014+
"description": "Bad Request",
1015+
"schema": {
1016+
"$ref": "#/definitions/errors.ErrorResponse"
1017+
}
1018+
},
1019+
"401": {
1020+
"description": "Unauthorized",
1021+
"schema": {
1022+
"$ref": "#/definitions/errors.ErrorResponse"
1023+
}
1024+
},
1025+
"404": {
1026+
"description": "Not Found",
1027+
"schema": {
1028+
"$ref": "#/definitions/errors.ErrorResponse"
1029+
}
1030+
},
1031+
"500": {
1032+
"description": "Internal Server Error",
1033+
"schema": {
1034+
"$ref": "#/definitions/errors.ErrorResponse"
1035+
}
1036+
}
1037+
}
1038+
}
1039+
},
9281040
"/repositories/{uuid}": {
9291041
"get": {
9301042
"description": "Get repository information.",
@@ -2735,6 +2847,12 @@ const docTemplate = `{
27352847
"name": "repository_uuids",
27362848
"in": "query"
27372849
},
2850+
{
2851+
"type": "string",
2852+
"description": "Filter templates by associated snapshots using a comma separated list of snapshot UUIDs",
2853+
"name": "snapshot_uuids",
2854+
"in": "query"
2855+
},
27382856
{
27392857
"type": "string",
27402858
"description": "Sort the response data based on specific parameters. Sort criteria can include ` + "`" + `name` + "`" + `, ` + "`" + `arch` + "`" + `, and ` + "`" + `version` + "`" + `.",

‎api/openapi.json

+161
Original file line numberDiff line numberDiff line change
@@ -2850,6 +2850,159 @@
28502850
]
28512851
}
28522852
},
2853+
"/repositories/{repo_uuid}/snapshots/bulk_delete/": {
2854+
"post": {
2855+
"description": "This enables deleting specified snapshots from a repository.",
2856+
"operationId": "bulkDeleteSnapshots",
2857+
"parameters": [
2858+
{
2859+
"description": "Repository UUID.",
2860+
"in": "path",
2861+
"name": "repo_uuid",
2862+
"required": true,
2863+
"schema": {
2864+
"type": "string"
2865+
}
2866+
}
2867+
],
2868+
"requestBody": {
2869+
"content": {
2870+
"*/*": {
2871+
"schema": {
2872+
"$ref": "#/components/schemas/api.UUIDListRequest"
2873+
}
2874+
}
2875+
},
2876+
"description": "Identifiers of the snapshots",
2877+
"required": true,
2878+
"x-originalParamName": "body"
2879+
},
2880+
"responses": {
2881+
"204": {
2882+
"description": "Snapshots were successfully deleted"
2883+
},
2884+
"400": {
2885+
"content": {
2886+
"application/json": {
2887+
"schema": {
2888+
"$ref": "#/components/schemas/errors.ErrorResponse"
2889+
}
2890+
}
2891+
},
2892+
"description": "Bad Request"
2893+
},
2894+
"401": {
2895+
"content": {
2896+
"application/json": {
2897+
"schema": {
2898+
"$ref": "#/components/schemas/errors.ErrorResponse"
2899+
}
2900+
}
2901+
},
2902+
"description": "Unauthorized"
2903+
},
2904+
"404": {
2905+
"content": {
2906+
"application/json": {
2907+
"schema": {
2908+
"$ref": "#/components/schemas/errors.ErrorResponse"
2909+
}
2910+
}
2911+
},
2912+
"description": "Not Found"
2913+
},
2914+
"500": {
2915+
"content": {
2916+
"application/json": {
2917+
"schema": {
2918+
"$ref": "#/components/schemas/errors.ErrorResponse"
2919+
}
2920+
}
2921+
},
2922+
"description": "Internal Server Error"
2923+
}
2924+
},
2925+
"summary": "Bulk delete a snapshots",
2926+
"tags": [
2927+
"snapshots"
2928+
]
2929+
}
2930+
},
2931+
"/repositories/{repo_uuid}/snapshots/{snapshot_uuid}": {
2932+
"delete": {
2933+
"description": "This enables deleting a specific snapshot.",
2934+
"operationId": "deleteSnapshot",
2935+
"parameters": [
2936+
{
2937+
"description": "Repository UUID.",
2938+
"in": "path",
2939+
"name": "repo_uuid",
2940+
"required": true,
2941+
"schema": {
2942+
"type": "string"
2943+
}
2944+
},
2945+
{
2946+
"description": "Snapshot UUID.",
2947+
"in": "path",
2948+
"name": "snapshot_uuid",
2949+
"required": true,
2950+
"schema": {
2951+
"type": "string"
2952+
}
2953+
}
2954+
],
2955+
"responses": {
2956+
"204": {
2957+
"description": "Snapshot was successfully deleted"
2958+
},
2959+
"400": {
2960+
"content": {
2961+
"application/json": {
2962+
"schema": {
2963+
"$ref": "#/components/schemas/errors.ErrorResponse"
2964+
}
2965+
}
2966+
},
2967+
"description": "Bad Request"
2968+
},
2969+
"401": {
2970+
"content": {
2971+
"application/json": {
2972+
"schema": {
2973+
"$ref": "#/components/schemas/errors.ErrorResponse"
2974+
}
2975+
}
2976+
},
2977+
"description": "Unauthorized"
2978+
},
2979+
"404": {
2980+
"content": {
2981+
"application/json": {
2982+
"schema": {
2983+
"$ref": "#/components/schemas/errors.ErrorResponse"
2984+
}
2985+
}
2986+
},
2987+
"description": "Not Found"
2988+
},
2989+
"500": {
2990+
"content": {
2991+
"application/json": {
2992+
"schema": {
2993+
"$ref": "#/components/schemas/errors.ErrorResponse"
2994+
}
2995+
}
2996+
},
2997+
"description": "Internal Server Error"
2998+
}
2999+
},
3000+
"summary": "Delete a snapshot",
3001+
"tags": [
3002+
"snapshots"
3003+
]
3004+
}
3005+
},
28533006
"/repositories/{uuid}": {
28543007
"delete": {
28553008
"description": "This enables deleting a specific repository.",
@@ -5187,6 +5340,14 @@
51875340
"type": "string"
51885341
}
51895342
},
5343+
{
5344+
"description": "Filter templates by associated snapshots using a comma separated list of snapshot UUIDs",
5345+
"in": "query",
5346+
"name": "snapshot_uuids",
5347+
"schema": {
5348+
"type": "string"
5349+
}
5350+
},
51905351
{
51915352
"description": "Sort the response data based on specific parameters. Sort criteria can include `name`, `arch`, and `version`.",
51925353
"in": "query",

‎cmd/content-sources/main.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,8 @@ func kafkaConsumer(ctx context.Context, wg *sync.WaitGroup, metrics *m.Metrics)
109109
wrk := worker.NewTaskWorkerPool(&pgqueue, metrics)
110110
wrk.RegisterHandler(config.IntrospectTask, tasks.IntrospectHandler)
111111
wrk.RegisterHandler(config.RepositorySnapshotTask, tasks.SnapshotHandler)
112-
wrk.RegisterHandler(config.DeleteRepositorySnapshotsTask, tasks.DeleteSnapshotHandler)
112+
wrk.RegisterHandler(config.DeleteRepositorySnapshotsTask, tasks.DeleteRepositorySnapshotsHandler)
113+
wrk.RegisterHandler(config.DeleteSnapshotsTask, tasks.DeleteSnapshotsHandler)
113114
wrk.RegisterHandler(config.DeleteTemplatesTask, tasks.DeleteTemplateHandler)
114115
wrk.RegisterHandler(config.UpdateTemplateContentTask, tasks.UpdateTemplateContentHandler)
115116
wrk.RegisterHandler(config.UpdateRepositoryTask, tasks.UpdateRepositoryHandler)

‎db/migrations.latest

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
1-
20241104115955
2-
1+
20241113084850

‎db/migrations/20241113084850_add_deleted_at_to_snapshots.down.sql

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
BEGIN;
2+
3+
ALTER TABLE snapshots DROP COLUMN IF EXISTS deleted_at;
4+
5+
COMMIT;

‎db/migrations/20241113084850_add_deleted_at_to_snapshots.up.sql

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
BEGIN;
2+
3+
ALTER TABLE snapshots
4+
ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMP WITH TIME ZONE;
5+
6+
COMMIT;

‎mk/go-rules.mk

+1-2
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ test-unit: ## Run tests for ci
5050

5151
.PHONY: test-integration
5252
test-integration: ## Run tests for ci
53-
CONFIG_PATH="$(PROJECT_DIR)/configs/" go test $(MOD_VENDOR) ./test/integration/...
53+
CONFIG_PATH="$(PROJECT_DIR)/configs/" go test -timeout 15m $(MOD_VENDOR) ./test/integration/...
5454

5555
DB_CONNECT_INFO := dbname=$(DATABASE_NAME) sslmode=disable user=$(DATABASE_USER) host=$(DATABASE_HOST) password=$(DATABASE_PASSWORD)
5656
.PHONY: test-db-migrations
@@ -66,4 +66,3 @@ test-db-migrations: ## CI test for checking that db migrations work. Use careful
6666
# Add dependencies from binaries to all the the sources
6767
# so any change is detected for the build rule
6868
$(patsubst cmd/%,$(GO_OUTPUT)/%,$(wildcard cmd/*)): $(shell find $(PROJECT_DIR)/cmd -type f -name '*.go') $(shell find $(PROJECT_DIR)/pkg -type f -name '*.go')
69-

‎pkg/api/templates.go

+1
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ type TemplateFilterData struct {
7272
Version string `json:"version"` // Filter templates by version using an exact match.
7373
Search string `json:"search"` // Search string based query to optionally filter on
7474
RepositoryUUIDs []string `json:"repository_uuids"` // List templates that contain one or more of these Repositories
75+
SnapshotUUIDs []string `json:"snapshot_uuids"` // List templates that contain one or more of these Snapshots
7576
UseLatest bool `json:"use_latest"` // List templates that have use_latest set to true
7677
}
7778

‎pkg/config/tasks.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package config
33
const (
44
RepositorySnapshotTask = "snapshot" // Task to create a snapshot for a repository config
55
DeleteRepositorySnapshotsTask = "delete-repository-snapshots" // Task to delete all snapshots for a repository config
6+
DeleteSnapshotsTask = "delete-snapshots" // Task to delete all snapshots marked for deletion
67
IntrospectTask = "introspect" // Task to introspect repository
78
DeleteTemplatesTask = "delete-templates" // Task to delete all content templates marked for deletion
89
UpdateTemplateContentTask = "update-template-content" // Task to update the pulp distributions of a template's snapshots
@@ -19,7 +20,7 @@ const (
1920
TaskStatusPending = "pending" // Task is waiting to be started
2021
)
2122

22-
var RequeueableTasks = []string{DeleteTemplatesTask, DeleteRepositorySnapshotsTask, UpdateTemplateContentTask}
23+
var RequeueableTasks = []string{DeleteTemplatesTask, DeleteRepositorySnapshotsTask, UpdateTemplateContentTask, DeleteSnapshotsTask}
2324

2425
var CancellableTasks = []string{IntrospectTask, RepositorySnapshotTask, UpdateTemplateContentTask}
2526

@@ -37,4 +38,4 @@ var TasksToCleanup = []string{
3738
}
3839

3940
// TasksToCleanupIfCompleted tasks that will get deleted if older than 10 days, only if status is completed
40-
var TasksToCleanupIfCompleted = []string{DeleteRepositorySnapshotsTask, DeleteTemplatesTask}
41+
var TasksToCleanupIfCompleted = []string{DeleteRepositorySnapshotsTask, DeleteTemplatesTask, DeleteSnapshotsTask}

‎pkg/dao/interfaces.go

+5
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ type RepositoryConfigDao interface {
7777
FetchByRepoUuid(ctx context.Context, orgID string, repoUuid string) (api.RepositoryResponse, error)
7878
InternalOnly_FetchRepoConfigsForRepoUUID(ctx context.Context, uuid string) []api.RepositoryResponse
7979
UpdateLastSnapshotTask(ctx context.Context, taskUUID string, orgID string, repoUUID string) error
80+
UpdateLastSnapshot(ctx context.Context, orgID string, repoConfigUUID string, snapUUID string) error
8081
InternalOnly_RefreshRedHatRepo(ctx context.Context, request api.RepositoryRequest, label string) (*api.RepositoryResponse, error)
8182
FetchWithoutOrgID(ctx context.Context, uuid string) (api.RepositoryResponse, error)
8283
BulkExport(ctx context.Context, orgID string, reposToExport api.RepositoryExportRequest) ([]api.RepositoryExportResponse, error)
@@ -110,7 +111,11 @@ type SnapshotDao interface {
110111
List(ctx context.Context, orgID string, repoConfigUuid string, paginationData api.PaginationData, filterData api.FilterData) (api.SnapshotCollectionResponse, int64, error)
111112
ListByTemplate(ctx context.Context, orgID string, template api.TemplateResponse, repositorySearch string, paginationData api.PaginationData) (api.SnapshotCollectionResponse, int64, error)
112113
FetchForRepoConfigUUID(ctx context.Context, repoConfigUUID string) ([]models.Snapshot, error)
114+
FetchModel(ctx context.Context, uuid string, includeSoftDel bool) (models.Snapshot, error)
115+
SoftDelete(ctx context.Context, snapUUID string) error
113116
Delete(ctx context.Context, snapUUID string) error
117+
BulkDelete(ctx context.Context, uuids []string) []error
118+
ClearDeletedAt(ctx context.Context, snapUUID string) error
114119
FetchLatestSnapshot(ctx context.Context, repoConfigUUID string) (api.SnapshotResponse, error)
115120
FetchLatestSnapshotModel(ctx context.Context, repoConfigUUID string) (models.Snapshot, error)
116121
FetchSnapshotsByDateAndRepository(ctx context.Context, orgID string, request api.ListSnapshotByDateRequest) (api.ListSnapshotByDateResponse, error)

0 commit comments

Comments
 (0)