feat: Add database migrations (#125)

This commit implements the database migrations.
The database migrations bring more consistency to the user on the
deployment as Archivista will handle changes on the SQL Schema.

The implementation uses the Versioned Migrations
(https://entgo.io/docs/versioned-migrations/).
The main reason is that the ent Offline Mode
(https://entgo.io/docs/migrate#offline-mode), which seems safer,
is migrating to Versioned Migrations.

The proposal strategy is to run the migrations during the
container start and fail early in case of issues before the
archivista application is done in the entry point.

During the development, the developers can generate a new migration
using the `make db-migrations` if there are changes in the schema
files.

This commit also implements an automatic check to avoid it for code
review. It was added to checks (`db-migrations.yml`)

Signed-off-by: Kairo de Araujo <[email protected]>

.github/workflows/db-migrations.yml

@@ -0,0 +1,47 @@
+# Copyright 2023 The Archivista Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: db-migrations
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  db-migrations:
+    name: db-migrations
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: '1.19.x'
+
+      - name: Check DB Migrations
+        run: |
+          curl -sSf https://atlasgo.sh | sh
+          before=$(find ent/migrate/migrations/ -type f | wc -l | awk '{ print $1 }')
+          make db-migrations
+          after=$(find ent/migrate/migrations/ -type f | wc -l | awk '{ print $1 }')
+          if [[ $before -lt $after ]]; then echo "missing 'make db-migrations'"; exit 1; fi

.github/workflows/verify-licence.yml

@@ -40,4 +40,4 @@ jobs:
         - name: Check license headers
           run: |
             set -e
-            addlicense --check -l apache -c 'The Archivista Contributors' -v ./
+            addlicense --check -l apache -c 'The Archivista Contributors' --ignore "ent/migrate/migrations/**" -v ./

CONTRIBUTING.md

@@ -34,12 +34,12 @@ the development process:
  on your terminal. If this command is unsuccessful, you will need to find the standard
  method for installing it for your system. For installing `make` on Windows, please see
  [here](https://gnuwin32.sourceforge.net/packages/make.html).
- 
-* Go v1.19: Archivista is written in [Go](https://golang.org/), so you 
+
+* Go v1.19: Archivista is written in [Go](https://golang.org/), so you
  will need this installed in order to compile and run the source code.
 
 * pre-commit: It is a framework for managing and maintaining multi-language
- pre-commit hooks. It is used to run some checks on the code before it is 
+ pre-commit hooks. It is used to run some checks on the code before it is
  committed. You can install it by following the instructions
  [here](https://pre-commit.com/#install).
 
@@ -59,7 +59,7 @@ the development process:
 
 [Fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo>) the repository on GitHub and
 [clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) it to
-your local machine: 
+your local machine:
 ```console
     git clone [email protected]:YOUR-USERNAME/archivista.git
 ```
@@ -106,6 +106,14 @@ To clean all Archivista containers in your environment execute the command:
     make clean
 ```
 
+### Changes in the Archivista SQL Schema `ent/schema`
+
+If there are changes in the SQL Schema you must to create the database migration.
+To do this, you must run the following command:
+
+```console
+    make db-migrations
+```
 
 ### Running Tests
 

Dockerfile

@@ -14,13 +14,17 @@
 
 FROM golang:1.21.5-alpine@sha256:feceecc0e1d73d085040a8844de11a2858ba4a0c58c16a672f1736daecc2a4ff AS build
 WORKDIR /src
-RUN apk update && apk add --no-cache file git
+RUN apk update && apk add --no-cache file git curl
+RUN curl -sSf https://atlasgo.sh | sh
 ENV GOMODCACHE /root/.cache/gocache
 RUN --mount=target=. --mount=target=/root/.cache,type=cache \
     CGO_ENABLED=0 go build -o /out/archivista -ldflags '-s -d -w' ./cmd/archivista; \
     file /out/archivista | grep "statically linked"
 
 FROM alpine
 COPY --from=build /out/archivista /bin/archivista
+COPY --from=build /usr/local/bin/atlas /bin/atlas
+ADD entrypoint.sh /bin/entrypoint.sh
+ADD ent/migrate/migrations /archivista/migrations
 RUN mkdir /tmp/archivista
-ENTRYPOINT ["/bin/archivista"]
+ENTRYPOINT ["sh", "/bin/entrypoint.sh"]

Dockerfile-dev

@@ -14,7 +14,8 @@
 
 FROM golang:1.21.5-alpine@sha256:feceecc0e1d73d085040a8844de11a2858ba4a0c58c16a672f1736daecc2a4ff AS build
 WORKDIR /src
-RUN apk update && apk add --no-cache file git
+RUN apk update && apk add --no-cache file git curl
+RUN curl -sSf https://atlasgo.sh | sh
 ENV GOMODCACHE /root/.cache/gocache
 RUN go install github.com/githubnemo/CompileDaemon@latest
-ENTRYPOINT CompileDaemon -log-prefix=false -build="go build -o /out/archivista ./cmd/archivista" -command="/out/archivista"
+ENTRYPOINT ["sh", "entrypoint-dev.sh"]

Makefile

@@ -49,5 +49,12 @@ lint:  ## Run linter
 	@go vet ./...
 
 
+
+.PHONY: db-migrations
+db-migrations:  ## Run the migrations for the database
+	@atlas migrate diff mysql --dir "file://ent/migrate/migrations/mysql" --to "ent://ent/schema" --dev-url "docker://mysql/8/dev"
+	@atlas migrate diff pgsql --dir "file://ent/migrate/migrations/pgsql" --to "ent://ent/schema" --dev-url "docker://postgres/16/dev?search_path=public"
+
+
 help:  ## Show this help
 	@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

ent/migrate/migrations/mysql/20231214165639_mysql.sql

@@ -0,0 +1,18 @@
+-- Create "statements" table
+CREATE TABLE `statements` (`id` bigint NOT NULL AUTO_INCREMENT, `predicate` varchar(255) NOT NULL, PRIMARY KEY (`id`), INDEX `statement_predicate` (`predicate`)) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "attestation_collections" table
+CREATE TABLE `attestation_collections` (`id` bigint NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL, `statement_attestation_collections` bigint NOT NULL, PRIMARY KEY (`id`), INDEX `attestationcollection_name` (`name`), UNIQUE INDEX `statement_attestation_collections` (`statement_attestation_collections`), CONSTRAINT `attestation_collections_statements_attestation_collections` FOREIGN KEY (`statement_attestation_collections`) REFERENCES `statements` (`id`) ON UPDATE NO ACTION ON DELETE NO ACTION) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "attestations" table
+CREATE TABLE `attestations` (`id` bigint NOT NULL AUTO_INCREMENT, `type` varchar(255) NOT NULL, `attestation_collection_attestations` bigint NOT NULL, PRIMARY KEY (`id`), INDEX `attestation_type` (`type`), INDEX `attestations_attestation_collections_attestations` (`attestation_collection_attestations`), CONSTRAINT `attestations_attestation_collections_attestations` FOREIGN KEY (`attestation_collection_attestations`) REFERENCES `attestation_collections` (`id`) ON UPDATE NO ACTION ON DELETE NO ACTION) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "dsses" table
+CREATE TABLE `dsses` (`id` bigint NOT NULL AUTO_INCREMENT, `gitoid_sha256` varchar(255) NOT NULL, `payload_type` varchar(255) NOT NULL, `dsse_statement` bigint NULL, PRIMARY KEY (`id`), INDEX `dsses_statements_statement` (`dsse_statement`), UNIQUE INDEX `gitoid_sha256` (`gitoid_sha256`), CONSTRAINT `dsses_statements_statement` FOREIGN KEY (`dsse_statement`) REFERENCES `statements` (`id`) ON UPDATE NO ACTION ON DELETE SET NULL) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "payload_digests" table
+CREATE TABLE `payload_digests` (`id` bigint NOT NULL AUTO_INCREMENT, `algorithm` varchar(255) NOT NULL, `value` varchar(255) NOT NULL, `dsse_payload_digests` bigint NULL, PRIMARY KEY (`id`), INDEX `payload_digests_dsses_payload_digests` (`dsse_payload_digests`), INDEX `payloaddigest_value` (`value`), CONSTRAINT `payload_digests_dsses_payload_digests` FOREIGN KEY (`dsse_payload_digests`) REFERENCES `dsses` (`id`) ON UPDATE NO ACTION ON DELETE SET NULL) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "signatures" table
+CREATE TABLE `signatures` (`id` bigint NOT NULL AUTO_INCREMENT, `key_id` varchar(255) NOT NULL, `signature` text NOT NULL, `dsse_signatures` bigint NULL, PRIMARY KEY (`id`), INDEX `signature_key_id` (`key_id`), INDEX `signatures_dsses_signatures` (`dsse_signatures`), CONSTRAINT `signatures_dsses_signatures` FOREIGN KEY (`dsse_signatures`) REFERENCES `dsses` (`id`) ON UPDATE NO ACTION ON DELETE SET NULL) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "subjects" table
+CREATE TABLE `subjects` (`id` bigint NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL, `statement_subjects` bigint NULL, PRIMARY KEY (`id`), INDEX `subject_name` (`name`), INDEX `subjects_statements_subjects` (`statement_subjects`), CONSTRAINT `subjects_statements_subjects` FOREIGN KEY (`statement_subjects`) REFERENCES `statements` (`id`) ON UPDATE NO ACTION ON DELETE SET NULL) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "subject_digests" table
+CREATE TABLE `subject_digests` (`id` bigint NOT NULL AUTO_INCREMENT, `algorithm` varchar(255) NOT NULL, `value` varchar(255) NOT NULL, `subject_subject_digests` bigint NULL, PRIMARY KEY (`id`), INDEX `subject_digests_subjects_subject_digests` (`subject_subject_digests`), INDEX `subjectdigest_value` (`value`), CONSTRAINT `subject_digests_subjects_subject_digests` FOREIGN KEY (`subject_subject_digests`) REFERENCES `subjects` (`id`) ON UPDATE NO ACTION ON DELETE SET NULL) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+-- Create "timestamps" table
+CREATE TABLE `timestamps` (`id` bigint NOT NULL AUTO_INCREMENT, `type` varchar(255) NOT NULL, `timestamp` timestamp NOT NULL, `signature_timestamps` bigint NULL, PRIMARY KEY (`id`), INDEX `timestamps_signatures_timestamps` (`signature_timestamps`), CONSTRAINT `timestamps_signatures_timestamps` FOREIGN KEY (`signature_timestamps`) REFERENCES `signatures` (`id`) ON UPDATE NO ACTION ON DELETE SET NULL) CHARSET utf8mb4 COLLATE utf8mb4_bin;

ent/migrate/migrations/mysql/atlas.sum

@@ -0,0 +1,2 @@
+h1:uz2B9rkm7QWMNJE8Lp0NAr4JOcWFeRO6+MgSc/SBwqY=
+20231214165639_mysql.sql h1:rqSqoXUOtdEpJT04rLCzyyPJC+NYRjQj9jXHZW+Oq9Q=

ent/migrate/migrations/pgsql/20231214165922_pgsql.sql

@@ -0,0 +1,36 @@
+-- Create "statements" table
+CREATE TABLE "statements" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "predicate" character varying NOT NULL, PRIMARY KEY ("id"));
+-- Create index "statement_predicate" to table: "statements"
+CREATE INDEX "statement_predicate" ON "statements" ("predicate");
+-- Create "attestation_collections" table
+CREATE TABLE "attestation_collections" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "name" character varying NOT NULL, "statement_attestation_collections" bigint NOT NULL, PRIMARY KEY ("id"), CONSTRAINT "attestation_collections_statements_attestation_collections" FOREIGN KEY ("statement_attestation_collections") REFERENCES "statements" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION);
+-- Create index "attestation_collections_statement_attestation_collections_key" to table: "attestation_collections"
+CREATE UNIQUE INDEX "attestation_collections_statement_attestation_collections_key" ON "attestation_collections" ("statement_attestation_collections");
+-- Create index "attestationcollection_name" to table: "attestation_collections"
+CREATE INDEX "attestationcollection_name" ON "attestation_collections" ("name");
+-- Create "attestations" table
+CREATE TABLE "attestations" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "type" character varying NOT NULL, "attestation_collection_attestations" bigint NOT NULL, PRIMARY KEY ("id"), CONSTRAINT "attestations_attestation_collections_attestations" FOREIGN KEY ("attestation_collection_attestations") REFERENCES "attestation_collections" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION);
+-- Create index "attestation_type" to table: "attestations"
+CREATE INDEX "attestation_type" ON "attestations" ("type");
+-- Create "dsses" table
+CREATE TABLE "dsses" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "gitoid_sha256" character varying NOT NULL, "payload_type" character varying NOT NULL, "dsse_statement" bigint NULL, PRIMARY KEY ("id"), CONSTRAINT "dsses_statements_statement" FOREIGN KEY ("dsse_statement") REFERENCES "statements" ("id") ON UPDATE NO ACTION ON DELETE SET NULL);
+-- Create index "dsses_gitoid_sha256_key" to table: "dsses"
+CREATE UNIQUE INDEX "dsses_gitoid_sha256_key" ON "dsses" ("gitoid_sha256");
+-- Create "payload_digests" table
+CREATE TABLE "payload_digests" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "algorithm" character varying NOT NULL, "value" character varying NOT NULL, "dsse_payload_digests" bigint NULL, PRIMARY KEY ("id"), CONSTRAINT "payload_digests_dsses_payload_digests" FOREIGN KEY ("dsse_payload_digests") REFERENCES "dsses" ("id") ON UPDATE NO ACTION ON DELETE SET NULL);
+-- Create index "payloaddigest_value" to table: "payload_digests"
+CREATE INDEX "payloaddigest_value" ON "payload_digests" ("value");
+-- Create "signatures" table
+CREATE TABLE "signatures" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "key_id" character varying NOT NULL, "signature" character varying NOT NULL, "dsse_signatures" bigint NULL, PRIMARY KEY ("id"), CONSTRAINT "signatures_dsses_signatures" FOREIGN KEY ("dsse_signatures") REFERENCES "dsses" ("id") ON UPDATE NO ACTION ON DELETE SET NULL);
+-- Create index "signature_key_id" to table: "signatures"
+CREATE INDEX "signature_key_id" ON "signatures" ("key_id");
+-- Create "subjects" table
+CREATE TABLE "subjects" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "name" character varying NOT NULL, "statement_subjects" bigint NULL, PRIMARY KEY ("id"), CONSTRAINT "subjects_statements_subjects" FOREIGN KEY ("statement_subjects") REFERENCES "statements" ("id") ON UPDATE NO ACTION ON DELETE SET NULL);
+-- Create index "subject_name" to table: "subjects"
+CREATE INDEX "subject_name" ON "subjects" ("name");
+-- Create "subject_digests" table
+CREATE TABLE "subject_digests" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "algorithm" character varying NOT NULL, "value" character varying NOT NULL, "subject_subject_digests" bigint NULL, PRIMARY KEY ("id"), CONSTRAINT "subject_digests_subjects_subject_digests" FOREIGN KEY ("subject_subject_digests") REFERENCES "subjects" ("id") ON UPDATE NO ACTION ON DELETE SET NULL);
+-- Create index "subjectdigest_value" to table: "subject_digests"
+CREATE INDEX "subjectdigest_value" ON "subject_digests" ("value");
+-- Create "timestamps" table
+CREATE TABLE "timestamps" ("id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, "type" character varying NOT NULL, "timestamp" timestamptz NOT NULL, "signature_timestamps" bigint NULL, PRIMARY KEY ("id"), CONSTRAINT "timestamps_signatures_timestamps" FOREIGN KEY ("signature_timestamps") REFERENCES "signatures" ("id") ON UPDATE NO ACTION ON DELETE SET NULL);

ent/migrate/migrations/pgsql/atlas.sum

@@ -0,0 +1,2 @@
+h1:0dyKf7c5dDT+Vk7LvKiAeEQAH9aBJcynyXc0Ru44eDo=
+20231214165922_pgsql.sql h1:AZaZD6/98yKVFL6svm7gPpKlD1EL6fk8juLh+SW765I=

entrypoint-dev.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright 2023 The Archivista Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+env
+if [[ -z $ARCHIVISTA_SQL_STORE_BACKEND ]]; then
+    SQL_TYPE="MYSQL"
+else
+    SQL_TYPE=$(echo "$ARCHIVISTA_SQL_STORE_BACKEND" | tr '[:lower:]' '[:upper:]')
+fi
+
+case $SQL_TYPE in
+    MYSQL)
+        if [[ -z $ARCHIVISTA_SQL_STORE_CONNECTION_STRING ]]; then
+            ARCHIVISTA_SQL_STORE_CONNECTION_STRING="root:example@db/testify"
+        fi
+        echo "Running migrations for MySQL"
+        atlas migrate apply --dir "file:///src/ent/migrate/migrations/mysql" --url "mysql://$ARCHIVISTA_SQL_STORE_CONNECTION_STRING"
+        atlas_rc=$?
+        ;;
+    PSQL)
+        echo "Running migrations for Postgres"
+        atlas migrate apply --dir "file:///src/ent/migrate/migrations/pgsql" --url "$ARCHIVISTA_SQL_STORE_CONNECTION_STRING"
+        atlas_rc=$?
+        ;;
+    *)
+        echo "Unknown SQL backend: $ARCHIVISTA_SQL_STORE_BACKEND"
+        exit 1
+        ;;
+    esac
+
+if [[ $atlas_rc -ne 0 ]]; then
+    echo "Failed to apply migrations"
+    exit 1
+fi
+
+CompileDaemon -log-prefix=false -build="go build -o /out/archivista ./cmd/archivista" -command="/out/archivista"

entrypoint.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright 2023 The Archivista Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -z $ARCHIVISTA_SQL_STORE_BACKEND ]]; then
+    SQL_TYPE="MYSQL"
+else
+    SQL_TYPE=$(echo "$ARCHIVISTA_SQL_STORE_BACKEND" | tr '[:lower:]' '[:upper:]')
+fi
+case $SQL_TYPE in
+    MYSQL)
+        if [[ -z $ARCHIVISTA_SQL_STORE_CONNECTION_STRING ]]; then
+            ARCHIVISTA_SQL_STORE_CONNECTION_STRING="root:example@db/testify"
+        fi
+        echo "Running migrations for MySQL"
+        atlas migrate apply --dir "file:///archivista/migrations/mysql" --url "mysql://$ARCHIVISTA_SQL_STORE_CONNECTION_STRING"
+        atlas_rc=$?
+        ;;
+    PSQL)
+        echo "Running migrations for Postgres"
+        atlas migrate apply --dir "file:///archivista/migrations/pgsql" --url "$ARCHIVISTA_SQL_STORE_CONNECTION_STRING"
+        atlas_rc=$?
+        ;;
+    *)
+        echo "Unknown SQL backend: $ARCHIVISTA_SQL_STORE_BACKEND"
+        exit 1
+        ;;
+    esac
+
+if [[ $atlas_rc -ne 0 ]]; then
+    echo "Failed to apply migrations"
+    exit 1
+fi
+
+/bin/archivista