Skip to content

Commit a886e69

Browse files
pglombardopglombardo
authored
New pwpush-worker Docker container (#2601)
* Add SolidQueue and background jobs * Add worker to Procfiles * Docker container build * Worker container: updated entrypoint
1 parent b7376ed commit a886e69

16 files changed

+564
-3
lines changed

‎.github/workflows/docker-containers.yml

+44
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,47 @@ jobs:
105105
tags: ${{ steps.meta.outputs.tags }}
106106
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/pwpush-public-gateway:buildcache
107107
cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/pwpush-public-gateway:buildcache,mode=max,ignore-error=${{env.DOCKER_PUSH == 'false'}}
108+
109+
worker-container:
110+
needs: pwpush-container
111+
runs-on: ubuntu-latest
112+
steps:
113+
- name: Checkout
114+
uses: actions/checkout@v4
115+
116+
- name: Set up QEMU
117+
uses: docker/setup-qemu-action@v3
118+
119+
- name: Set up Docker Buildx
120+
uses: docker/setup-buildx-action@v3
121+
122+
- name: Populate Docker metadata
123+
id: meta
124+
uses: docker/metadata-action@v5
125+
with:
126+
images: ${{ secrets.DOCKER_USERNAME }}/pwpush-worker
127+
tags: |
128+
type=match,pattern=stable
129+
type=schedule,pattern=nightly
130+
type=semver,pattern={{version}}
131+
type=semver,pattern={{major}}.{{minor}}
132+
type=semver,pattern={{major}}
133+
134+
- name: Login to DockerHub
135+
uses: docker/login-action@v3
136+
with:
137+
username: ${{ secrets.DOCKER_USERNAME }}
138+
password: ${{ secrets.DOCKER_PASSWORD }}
139+
140+
- name: Build and push Docker image
141+
uses: docker/build-push-action@v6
142+
with:
143+
file: ./containers/docker/Dockerfile.worker
144+
platforms: linux/amd64,linux/arm64
145+
provenance: false
146+
push: true
147+
labels: ${{ steps.meta.outputs.labels }}
148+
tags: ${{ steps.meta.outputs.tags }}
149+
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/pwpush-worker:buildcache
150+
cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/pwpush-worker:buildcache,mode=max,ignore-error=${{env.DOCKER_PUSH == 'false'}}
151+

‎Gemfile

+2
Original file line numberDiff line numberDiff line change
@@ -126,3 +126,5 @@ gem "version", git: "https://github.com/pglombardo/version.git", branch: "master
126126
gem "administrate", "~> 0.20.1"
127127
gem "rqrcode", "~> 2.2"
128128
gem "turnout2024", require: "turnout"
129+
130+
gem "solid_queue", "~> 1.0"

‎Gemfile.lock

+14
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ GEM
186186
rubocop (>= 1)
187187
smart_properties
188188
erubi (1.13.0)
189+
et-orbi (1.2.11)
190+
tzinfo
189191
execjs (2.9.1)
190192
faraday (1.10.4)
191193
faraday-em_http (~> 1.0)
@@ -215,6 +217,9 @@ GEM
215217
ffi (1.17.0)
216218
foreman (0.88.1)
217219
forwardable (1.3.3)
220+
fugit (1.11.1)
221+
et-orbi (~> 1, >= 1.2.11)
222+
raabro (~> 1.4)
218223
gettext (3.4.9)
219224
erubi
220225
locale (>= 2.0.5)
@@ -392,6 +397,7 @@ GEM
392397
public_suffix (6.0.1)
393398
puma (6.4.3)
394399
nio4r (~> 2.0)
400+
raabro (1.4.0)
395401
racc (1.8.1)
396402
rack (3.1.7)
397403
rack-accept (0.4.5)
@@ -525,6 +531,13 @@ GEM
525531
devise (>= 3.2, < 6)
526532
singleton (0.2.0)
527533
smart_properties (1.17.0)
534+
solid_queue (1.0.0)
535+
activejob (>= 7.1)
536+
activerecord (>= 7.1)
537+
concurrent-ruby (>= 1.3.1)
538+
fugit (~> 1.11.0)
539+
railties (>= 7.1)
540+
thor (~> 1.3.1)
528541
sprockets (4.2.1)
529542
concurrent-ruby (~> 1.0)
530543
rack (>= 2.2.4, < 4)
@@ -646,6 +659,7 @@ DEPENDENCIES
646659
sass-rails (~> 6.0, >= 6.0.0)
647660
selenium-webdriver
648661
simple_token_authentication
662+
solid_queue (~> 1.0)
649663
sqlite3
650664
standardrb (~> 1.0)
651665
stimulus-rails

‎Procfile

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
release: bundle exec rails db:migrate
22
web: bundle exec puma -C config/puma.rb
33
console: bundle exec rails console
4+
worker: bundle exec rake solid_queue:start

‎Procfile.dev

+1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
web: bin/rails server -p 5100
22
js: yarn watch
3+
worker: bundle exec rake solid_queue:start

‎app/jobs/clean_up_pushes_job.rb

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
class CleanUpPushesJob < ApplicationJob
2+
queue_as :default
3+
4+
# Delete Anonymous Expired Pushes
5+
#
6+
# When a push expires, the payload is deleted but the metadata record still exists. This
7+
# includes information such as creation date, audit logs, duration etc.. When the record
8+
# was created by an anonymous user, this data is no longer needed and we delete it (we
9+
# don't want it).
10+
#
11+
# If a user attempts to retrieve a secret link that doesn't exist anymore, we still show
12+
# the standard "This secret link has expired" message. This strategy provides two benefits:
13+
#
14+
# 1. It hides the fact that if a secret ever exists or not (more secure)
15+
# 2. It allows us to delete data that we don't want
16+
#
17+
# This task will run through all expired and anonymous records and delete them entirely.
18+
#
19+
# Because of the above, expired and anonymous secret URLs still will show the same
20+
# expiration message
21+
#
22+
# Note: This applies to anonymous pushes. For logged-in user records, we don't do this
23+
# to maintain user audit logs.
24+
#
25+
def perform(*args)
26+
# Log task start
27+
logger.info("--> #{self.class.name}: Starting...")
28+
29+
counter = 0
30+
31+
Password.includes(:views)
32+
.where(expired: true)
33+
.where(user_id: nil)
34+
.find_each do |push|
35+
counter += 1
36+
push.destroy
37+
end
38+
39+
if Settings.enable_file_pushes
40+
FilePush.includes(:views)
41+
.where(expired: true)
42+
.where(user_id: nil)
43+
.find_each do |push|
44+
counter += 1
45+
push.destroy
46+
end
47+
end
48+
49+
if Settings.enable_url_pushes
50+
Url.includes(:views)
51+
.where(expired: true)
52+
.where(user_id: nil)
53+
.find_each do |push|
54+
counter += 1
55+
push.destroy
56+
end
57+
end
58+
59+
logger.info(" -> #{counter} total anonymous expired pushes deleted.")
60+
61+
# Log completion
62+
logger.info(" -> #{self.class.name}: Finished.")
63+
end
64+
65+
private
66+
67+
def logger
68+
@logger ||= if ENV.key?(PWP_WORKER)
69+
# We are running inside the pwpush-worker container. Log to STDOUT
70+
Logger.new($stdout)
71+
else
72+
Logger.new(Rails.root.join("log", "recurring.log"))
73+
end
74+
end
75+
end

‎app/jobs/expire_pushes_job.rb

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
class ExpirePushesJob < ApplicationJob
2+
queue_as :default
3+
4+
##
5+
# This job is responsible for scanning all unexpired password pushes and
6+
# conditionally expiring them. This is a preemptive measure to expire pushes
7+
# periodically. It saves some CPU and DB calls on live requests.
8+
#
9+
def perform(*args)
10+
# Log task start
11+
logger.info("--> #{self.class.name}: Starting...")
12+
13+
counter = 0
14+
expiration_count = 0
15+
16+
Password.where(expired: false).find_each do |push|
17+
counter += 1
18+
push.validate!
19+
expiration_count += 1 if push.expired
20+
end
21+
22+
logger.info(" -> Finished validating #{counter} unexpired password pushes. #{expiration_count} total pushes expired...")
23+
24+
if Settings.enable_file_pushes
25+
counter = 0
26+
expiration_count = 0
27+
FilePush.where(expired: false).find_each do |push|
28+
counter += 1
29+
push.validate!
30+
expiration_count += 1 if push.expired
31+
end
32+
logger.info(" -> Finished validating #{counter} unexpired File pushes. #{expiration_count} total pushes expired...")
33+
end
34+
35+
if Settings.enable_url_pushes
36+
counter = 0
37+
expiration_count = 0
38+
Url.where(expired: false).find_each do |push|
39+
counter += 1
40+
push.validate!
41+
expiration_count += 1 if push.expired
42+
end
43+
logger.info(" -> Finished validating #{counter} unexpired URL pushes. #{expiration_count} total pushes expired...")
44+
end
45+
46+
# Log results
47+
logger.info(" -> #{self.class.name}: #{counter} anonymous and expired pushes have been deleted.")
48+
49+
# Log completion
50+
logger.info(" -> #{self.class.name}: Finished.")
51+
end
52+
53+
private
54+
55+
def logger
56+
@logger ||= if ENV.key?(PWP_WORKER)
57+
# We are running inside the pwpush-worker container. Log to STDOUT
58+
# so that docker logs works to investigate problems.
59+
Logger.new($stdout)
60+
else
61+
Logger.new(Rails.root.join("log", "recurring.log"))
62+
end
63+
end
64+
end

‎bin/jobs

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/env ruby
2+
3+
require_relative "../config/environment"
4+
require "solid_queue/cli"
5+
6+
SolidQueue::Cli.start(ARGV)

‎config/environments/production.rb

+3-2
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,9 @@
7979
# config.cache_store = :mem_cache_store
8080

8181
# Use a real queuing backend for Active Job (and separate queues per environment).
82-
# config.active_job.queue_adapter = :resque
83-
# config.active_job.queue_name_prefix = "password_pusher_production"
82+
config.active_job.queue_adapter = :solid_queue
83+
84+
# config.active_job.queue_name_prefix = "pwp_prod"
8485

8586
config.action_mailer.perform_caching = true
8687

‎config/queue.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
default: &default
2+
dispatchers:
3+
- polling_interval: 1
4+
batch_size: 500
5+
workers:
6+
- queues: "*"
7+
threads: 3
8+
processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %>
9+
polling_interval: 0.1
10+
11+
development:
12+
<<: *default
13+
14+
test:
15+
<<: *default
16+
17+
production:
18+
<<: *default

‎config/recurring.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# production:
2+
# periodic_cleanup:
3+
# class: CleanSoftDeletedRecordsJob
4+
# queue: background
5+
# args: [ 1000, { batch_size: 500 } ]
6+
# schedule: every hour
7+
# periodic_command:
8+
# command: "SoftDeletedRecord.due.delete_all"
9+
# priority: 2
10+
# schedule: at 5am every day
11+
12+
production:
13+
expire_pushes:
14+
class: ExpirePushesJob
15+
queue: background
16+
schedule: every 2 hours
17+
18+
# expire_pulls:
19+
# class: ExpirePullsJob
20+
# queue: background
21+
# schedule: every 3 hours
22+
23+
cleanup_pushes:
24+
class: CleanUpPushesJob
25+
queue: background
26+
schedule: every day at 5am
27+
28+
development:
29+
expire_pushes:
30+
class: ExpirePushesJob
31+
queue: background
32+
schedule: every 4 hours
33+
34+
# expire_pulls:
35+
# class: ExpirePullsJob
36+
# queue: background
37+
# schedule: every 5 hours
38+
39+
cleanup_pushes:
40+
class: CleanUpPushesJob
41+
queue: background
42+
schedule: every 7 hours

‎config/routes/admin.rb

+1
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@
99

1010
root to: "users#index"
1111
end
12+
# mount MissionControl::Jobs::Engine, at: "/admin/jobs"
1213
end
1314
end

‎containers/docker/Dockerfile.worker

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
FROM pglombardo/pwpush:latest
2+
3+
ENV PWP_WORKER=true
4+
5+
USER pwpusher
6+
ENTRYPOINT ["containers/docker/worker-entrypoint.sh"]

‎containers/docker/worker-entrypoint.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
set -e
3+
4+
export RAILS_ENV=production
5+
6+
echo ""
7+
if [ -z "$DATABASE_URL" ]
8+
then
9+
echo "DATABASE_URL not specified. This worker container only works with a PostgreSQL, MySQL or MariaDB database."
10+
echo "Please specify DATABASE_URL and use the same settings & environment variables as you do with other pwpush containers."
11+
echo ""
12+
echo "See https://docs.pwpush.com/docs/database_url/ for more information on DATABASE_URL."
13+
exit 1
14+
elif [[ "$DATABASE_URL" == sqlite3://* ]]; then
15+
echo "Error: sqlite3 isn't supported for the pwpush-worker container."
16+
exit 1
17+
else
18+
echo "According to DATABASE_URL database backend is set to $(echo $DATABASE_URL|cut -d ":" -f 1):..."
19+
fi
20+
echo ""
21+
22+
echo "Password Pusher: migrating database to latest..."
23+
bundle exec rake db:migrate
24+
25+
echo "Password Pusher: starting background workers..."
26+
bundle exec solid_queue:start
27+
28+
exec "$@"

0 commit comments

Comments
 (0)