Merge pull request #1290 from Freika/dev

0.27.0

Author: Freika

.app_version

@@ -1 +1 @@
-0.26.7
+0.27.0

.github/workflows/build_and_push.yml

@@ -51,20 +51,46 @@ jobs:
       - name: Set Docker tags
         id: docker_meta
         run: |
-          VERSION=${GITHUB_REF#refs/tags/}
+          # Debug output
+          echo "GITHUB_REF: $GITHUB_REF"
+          echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
+
+          # Extract version from GITHUB_REF or use GITHUB_REF_NAME
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          else
+            VERSION=$GITHUB_REF_NAME
+          fi
+
+          # Additional safety check - if VERSION is empty, use a default
+          if [ -z "$VERSION" ]; then
+            VERSION="rc"
+          fi
+
+          echo "Using VERSION: $VERSION"
+
           TAGS="freikin/dawarich:${VERSION}"
 
+          # Set platforms based on release type
+          PLATFORMS="linux/amd64,linux/arm64,linux/arm/v8,linux/arm/v7,linux/arm/v6"
+
           # Add :rc tag for pre-releases
           if [ "${{ github.event.release.prerelease }}" = "true" ]; then
             TAGS="${TAGS},freikin/dawarich:rc"
+            # For RC builds, only use amd64
+            PLATFORMS="linux/amd64"
           fi
 
           # Add :latest tag only if release is not a pre-release
           if [ "${{ github.event.release.prerelease }}" != "true" ]; then
             TAGS="${TAGS},freikin/dawarich:latest"
           fi
 
+          echo "Final TAGS: $TAGS"
+          echo "PLATFORMS: $PLATFORMS"
+
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "platforms=${PLATFORMS}" >> $GITHUB_OUTPUT
 
       - name: Build and push
         uses: docker/build-push-action@v5
@@ -74,6 +100,6 @@ jobs:
           push: true
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v8,linux/arm/v7,linux/arm/v6
+          platforms: ${{ steps.docker_meta.outputs.platforms }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache

.gitignore

@@ -72,3 +72,7 @@
 /config/credentials/staging.yml.enc
 
 Makefile
+
+/db/*.sqlite3
+/db/*.sqlite3-shm
+/db/*.sqlite3-wal

CHANGELOG.md

@@ -4,8 +4,54 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+# 0.27.0 - 2025-05-31
 
-# 0.26.7 - 2025-05-26
+⚠️ This release includes a breaking change. ⚠️
+
+Starting 0.27.0, Dawarich is using SolidQueue and SolidCache to run background jobs and cache data. Before updating, make sure your Sidekiq queues (https://your_dawarich_app/sidekiq) are empty.
+
+Moving to SolidQueue and SolidCache will require creating new SQLite databases, which will be created automatically when you start the app. They will be stored in the `dawarich_db_data` volume.
+
+Background jobs interface is now available at `/jobs` page.
+
+Please, update your `docker-compose.yml` and add the following:
+
+```diff
+  dawarich_app:
+    image: freikin/dawarich:latest
+    container_name: dawarich_app
+    volumes:
+      - dawarich_public:/var/app/public
+      - dawarich_watched:/var/app/tmp/imports/watched
+      - dawarich_storage:/var/app/storage
++     - dawarich_db_data:/dawarich_db_data
+...
+    environment:
+      ...
+      DATABASE_NAME: dawarich_development
+      # SQLite database paths for secondary databases
++     QUEUE_DATABASE_PATH: /dawarich_db_data/dawarich_development_queue.sqlite3
++     CACHE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cache.sqlite3
++     CABLE_DATABASE_PATH: /dawarich_db_data/dawarich_development_cable.sqlite3
+```
+
+
+## Fixed
+
+- Enable caching in development for the docker image to improve performance.
+
+## Changed
+
+- SolidCache is now being used for caching instead of Redis.
+- SolidQueue is now being used for background jobs instead of Sidekiq.
+- SolidCable is now being used as ActionCable adapter.
+- Background jobs are now being run as Puma plugin instead of separate Docker container.
+- The `rc` docker image is now being built for amd64 architecture only to speed up the build process.
+- Deleting an import with many points now works significantly faster.
+
+
+
+# 0.26.7 - 2025-05-29
 
 ## Fixed
 

Gemfile

@@ -20,6 +20,7 @@ gem 'httparty'
 gem 'importmap-rails'
 gem 'kaminari'
 gem 'lograge'
+gem 'mission_control-jobs'
 gem 'oj'
 gem 'pg'
 gem 'prometheus_exporter'
@@ -35,13 +36,17 @@ gem 'rswag-api'
 gem 'rswag-ui'
 gem 'sentry-ruby'
 gem 'sentry-rails'
+gem 'sqlite3', '~> 2.6'
 gem 'stackprof'
 gem 'sidekiq'
 gem 'sidekiq-cron'
 gem 'sidekiq-limit_fetch'
 gem 'sprockets-rails'
 gem 'stimulus-rails'
 gem 'strong_migrations'
+gem 'solid_cable', '~> 3.0'
+gem 'solid_cache', '1.0.7'
+gem 'solid_queue', '~> 1.1'
 gem 'tailwindcss-rails'
 gem 'turbo-rails'
 gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]

Gemfile.lock

@@ -232,7 +232,17 @@ GEM
     mini_mime (1.1.5)
     mini_portile2 (2.8.9)
     minitest (5.25.5)
-    msgpack (1.8.0)
+    mission_control-jobs (1.0.2)
+      actioncable (>= 7.1)
+      actionpack (>= 7.1)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      importmap-rails (>= 1.2.1)
+      irb (~> 1.13)
+      railties (>= 7.1)
+      stimulus-rails
+      turbo-rails
+    msgpack (1.7.3)
     multi_json (1.15.0)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
@@ -442,13 +452,35 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
+    solid_cable (3.0.8)
+      actioncable (>= 7.2)
+      activejob (>= 7.2)
+      activerecord (>= 7.2)
+      railties (>= 7.2)
+    solid_cache (1.0.7)
+      activejob (>= 7.2)
+      activerecord (>= 7.2)
+      railties (>= 7.2)
+    solid_queue (1.1.5)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
+      thor (~> 1.3.1)
     sprockets (4.2.1)
       concurrent-ruby (~> 1.0)
       rack (>= 2.2.4, < 4)
     sprockets-rails (3.5.2)
       actionpack (>= 6.1)
       activesupport (>= 6.1)
       sprockets (>= 3.0.0)
+    sqlite3 (2.6.0-aarch64-linux-gnu)
+    sqlite3 (2.6.0-arm-linux-gnu)
+    sqlite3 (2.6.0-arm64-darwin)
+    sqlite3 (2.6.0-x86-linux-gnu)
+    sqlite3 (2.6.0-x86_64-darwin)
+    sqlite3 (2.6.0-x86_64-linux-gnu)
     stackprof (0.2.27)
     stimulus-rails (1.3.4)
       railties (>= 6.0.0)
@@ -532,6 +564,7 @@ DEPENDENCIES
   jwt
   kaminari
   lograge
+  mission_control-jobs
   oj
   pg
   prometheus_exporter
@@ -558,7 +591,11 @@ DEPENDENCIES
   sidekiq-cron
   sidekiq-limit_fetch
   simplecov
+  solid_cable (~> 3.0)
+  solid_cache (= 1.0.7)
+  solid_queue (~> 1.1)
   sprockets-rails
+  sqlite3 (~> 2.6)
   stackprof
   stimulus-rails
   strong_migrations

Procfile

@@ -1,2 +1,2 @@
 web: bundle exec puma -C config/puma.rb
-worker: bundle exec sidekiq -C config/sidekiq.yml
+worker: bundle exec bin/jobs

app/jobs/application_job.rb

@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 class ApplicationJob < ActiveJob::Base
   # Automatically retry jobs that encountered a deadlock
   # retry_on ActiveRecord::Deadlocked
 
+  retry_on Exception, wait: :polynomially_longer, attempts: 25
+
   # Most jobs are safe to ignore if the underlying records are no longer available
   # discard_on ActiveJob::DeserializationError
 end

app/jobs/jobs/clean_finished_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Jobs::CleanFinishedJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    SolidQueue::Job.clear_finished_in_batches
+  end
+end

app/services/imports/destroy.rb

@@ -9,7 +9,10 @@ def initialize(user, import)
   end
 
   def call
-    @import.destroy!
+    ActiveRecord::Base.transaction do
+      @import.points.delete_all
+      @import.destroy!
+    end
 
     Stats::BulkCalculator.new(@user.id).call
   end

app/services/jobs/create.rb

@@ -21,6 +21,7 @@ def call
         raise InvalidJobName, 'Invalid job name'
       end
 
+    # TODO: bulk enqueue reverse geocoding with ActiveJob
     points.find_each(&:async_reverse_geocode)
   end
 end

bin/jobs

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/environment"
+require "solid_queue/cli"
+
+SolidQueue::Cli.start(ARGV)

config/cable.yml

@@ -1,11 +1,21 @@
+# Async adapter only works within the same process, so for manually triggering cable updates from a console,
+# and seeing results in the browser, you must do so from the web console (running inside the dev process),
+# not a terminal started via bin/rails console! Add "console" to any action or any ERB template view
+# to make the web console appear.
+
+default: &default
+  adapter: solid_cable
+  connects_to:
+    database:
+      writing: cable
+  polling_interval: 0.1.seconds
+  message_retention: 1.day
+
 development:
-  adapter: redis
-  url: <%= ENV['REDIS_URL'] %>
+  <<: *default
 
 test:
   adapter: test
 
 production:
-  adapter: redis
-  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
-  channel_prefix: dawarich_production
+  <<: *default

config/cache.yml

@@ -0,0 +1,15 @@
+default: &default
+  store_options:
+    # Cap age of oldest cache entry to fulfill retention policies
+    max_age: <%= 60.days.to_i %>
+    max_size: <%= 256.megabytes %>
+    namespace: <%= Rails.env %>
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+
+production:
+  <<: *default

config/database.yml

@@ -9,18 +9,64 @@ default: &default
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 10 } %>
   timeout: 5000
 
+sqlite_default: &sqlite_default
+  adapter: sqlite3
+  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 10 } %>
+  timeout: 5000
+
 development:
-  <<: *default
-  database: <%= ENV['DATABASE_NAME'] || 'dawarich_development' %>
+  primary:
+    <<: *default
+    database: <%= ENV['DATABASE_NAME'] || 'dawarich_development' %>
+  queue:
+    <<: *sqlite_default
+    database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %>
+    migrations_paths: db/queue_migrate
+  cache:
+    <<: *sqlite_default
+    database: <%= ENV['CACHE_DATABASE_PATH'] || 'db/cache.sqlite3' %>
+    migrations_paths: db/cache_migrate
+  cable:
+    <<: *sqlite_default
+    database: <%= ENV['CABLE_DATABASE_PATH'] || 'db/cable.sqlite3' %>
+    migrations_paths: db/cable_migrate
 
 test:
-  <<: *default
-  database: <%= ENV['DATABASE_NAME'] || 'dawarich_test' %>
+  primary:
+    <<: *default
+    database: <%= ENV['DATABASE_NAME'] || 'dawarich_test' %>
 
 production:
-  <<: *default
-  database: <%= ENV['DATABASE_NAME'] || 'dawarich_production' %>
+  primary:
+    <<: *default
+    database: <%= ENV['DATABASE_NAME'] || 'dawarich_production' %>
+  queue:
+    <<: *sqlite_default
+    database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %>
+    migrations_paths: db/queue_migrate
+  cable:
+    <<: *sqlite_default
+    database: <%= ENV['CABLE_DATABASE_PATH'] || 'db/cable.sqlite3' %>
+    migrations_paths: db/cable_migrate
+  cache:
+    <<: *sqlite_default
+    database: <%= ENV['CACHE_DATABASE_PATH'] %>
+    migrations_paths: db/cache_migrate
 
 staging:
-  <<: *default
-  database: <%= ENV['DATABASE_NAME'] || 'dawarich_staging' %>
+  primary:
+    <<: *default
+    database: <%= ENV['DATABASE_NAME'] || 'dawarich_staging' %>
+    password: <%= ENV['DATABASE_PASSWORD'] %>
+  cache:
+    <<: *sqlite_default
+    database: <%= ENV['CACHE_DATABASE_PATH'] || 'db/cache.sqlite3' %>
+    migrations_paths: db/cache_migrate
+  queue:
+    <<: *sqlite_default
+    database: <%= ENV['QUEUE_DATABASE_PATH'] || 'db/queue.sqlite3' %>
+    migrations_paths: db/queue_migrate
+  cable:
+    <<: *sqlite_default
+    database: <%= ENV['CABLE_DATABASE_PATH'] || 'db/cable.sqlite3' %>
+    migrations_paths: db/cable_migrate