feat: support scheduled cleaning of old data (beta)

Author: bethesque

Dockerfile

@@ -1,5 +1,15 @@
 FROM ruby:2.6.6-alpine
 
+ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.11/supercronic-linux-amd64 \
+    SUPERCRONIC=supercronic-linux-amd64 \
+    SUPERCRONIC_SHA1SUM=a2e2d47078a8dafc5949491e5ea7267cc721d67c
+
+RUN wget "$SUPERCRONIC_URL" \
+ && echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - \
+ && chmod +x "$SUPERCRONIC" \
+ && mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" \
+ && ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic
+
 # Installation path
 ENV HOME=/pact_broker
 
@@ -30,6 +40,8 @@ COPY pact_broker $HOME/
 # Start Puma
 ENV RACK_ENV=production
 ENV PACT_BROKER_PORT_ENVIRONMENT_VARIABLE_NAME=PACT_BROKER_PORT
+ENV PACT_BROKER_DATABASE_BETA_CLEAN_ENABLED=false
+ENV PACT_BROKER_DATABASE_BETA_CLEAN_CRON_SCHEDULE="5 * * * *"
 ENV PACT_BROKER_PORT=9292
 USER ruby
 ENTRYPOINT ["./entrypoint.sh"]

docker-compose-dev.yml

@@ -5,8 +5,8 @@ services:
     image: postgres
     healthcheck:
       test: psql postgres --command "select 1" -U postgres
-    ports:
-      - "5432:5432"
+    # ports:
+    #   - "5432:5432"
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: password
@@ -22,14 +22,16 @@ services:
       PACT_BROKER_PORT_ENVIRONMENT_VARIABLE_NAME: PORT
       PACT_BROKER_DATABASE_URL_ENVIRONMENT_VARIABLE_NAME: DATABASE_URL
       DATABASE_URL: "postgres://postgres:password@postgres/postgres"
+      PACT_BROKER_DATABASE_BETA_CLEAN_ENABLED: "true"
+      PACT_BROKER_DATABASE_BETA_CLEAN_CRON_SCHEDULE: "* * * * *"
       # PACT_BROKER_DATABASE_USERNAME: postgres
       # PACT_BROKER_DATABASE_PASSWORD: password
       # PACT_BROKER_DATABASE_HOST: postgres
       # PACT_BROKER_DATABASE_NAME: postgres
       # PACT_BROKER_PORT: "9292"
       PORT: '9393'
       PACT_BROKER_LOG_LEVEL: INFO
-      PACT_BROKER_SQL_LOG_LEVEL: INFO
+      PACT_BROKER_SQL_LOG_LEVEL: DEBUG
     entrypoint: /custom-entrypoint.sh
     volumes:
       - ./script/docker/docker-compose-entrypoint.sh:/custom-entrypoint.sh

pact_broker/Rakefile

@@ -1,21 +1,40 @@
 require_relative 'database_connection'
-require 'logger'
+require_relative 'docker_configuration'
+require_relative 'logger'
 require 'pact_broker/tasks'
 
+def create_database_configuration
+  dc = PactBroker::DockerConfiguration.new(ENV, OpenStruct.new)
+  create_database_connection_from_config($logger, dc.database_configuration.tap { |it| puts it })
+end
+
 PactBroker::DB::MigrationTask.new do | task |
-  task.database_connection = create_database_connection(Logger.new($stdout))
+  task.database_connection = create_database_configuration
 end
 
 PactBroker::DB::VersionTask.new do | task |
-  task.database_connection = create_database_connection(Logger.new($stdout))
+  task.database_connection = create_database_configuration
 end
 
 PactBroker::DB::CleanTask.new do | task |
-  task.database_connection = create_database_connection(Logger.new($stdout))
+  task.database_connection = create_database_configuration
+
+  if (ENV['PACT_BROKER_DATABASE_CLEAN_KEEP_VERSION_SELECTORS'] || '') != ''
+    require 'json'
+    task.keep_version_selectors = JSON.parse(ENV['PACT_BROKER_DATABASE_CLEAN_KEEP_VERSION_SELECTORS'])
+  end
+
+  if (ENV['PACT_BROKER_DATABASE_CLEAN_VERSION_DELETION_LIMIT'] || '') != ''
+    task.version_deletion_limit = ENV['PACT_BROKER_DATABASE_CLEAN_VERSION_DELETION_LIMIT'].to_i
+  end
+
+  if (ENV['PACT_BROKER_DATABASE_CLEAN_DRY_RUN'] || '') != ''
+    task.dry_run = ENV['PACT_BROKER_DATABASE_CLEAN_DRY_RUN'] == 'true'
+  end
 end
 
 PactBroker::DB::DeleteOverwrittenDataTask.new do | task |
-  task.database_connection = create_database_connection(Logger.new($stdout))
+  task.database_connection = create_database_configuration
   if ENV['PACT_BROKER_DELETE_OVERWRITTEN_DATA_AGE']
     task.age_in_days = ENV['PACT_BROKER_DELETE_OVERWRITTEN_DATA_AGE'].to_i
   else

pact_broker/clean.sh

@@ -0,0 +1,3 @@
+#/bin/sh
+
+bundle exec rake pact_broker:db:clean

pact_broker/entrypoint.sh

@@ -1,3 +1,9 @@
 #!/bin/sh
 
+if [ "${PACT_BROKER_DATABASE_BETA_CLEAN_ENABLED}" = "true" ]; then
+  echo "Creating crontab with schedule ${PACT_BROKER_DATABASE_BETA_CLEAN_CRON_SCHEDULE} to clean database"
+  echo "${PACT_BROKER_DATABASE_BETA_CLEAN_CRON_SCHEDULE} /pact_broker/clean.sh" >> /pact_broker/crontab
+  /usr/local/bin/supercronic -quiet -passthrough-logs /pact_broker/crontab &
+fi
+
 bundle exec puma

pact_broker/logger.rb

@@ -1,4 +1,5 @@
 require 'logger'
+require 'semantic_logger'
 
 log_level = begin
   Kernel.const_get('Logger').const_get(ENV['PACT_BROKER_LOG_LEVEL'] || 'WARN')