From d4c855062896916bce274af03bfa2db5a2541801 Mon Sep 17 00:00:00 2001 From: Dimas Ciputra Date: Sun, 21 Aug 2022 15:55:44 +0200 Subject: [PATCH] Update docker compose (#256) * Fix min_qg_version query in plugins.xml (#224) * fix min_qg_version query * added 0 patch value for qgis version against max_qg_version * only add patch if it has major.minor version * Update docker-compose and dockerfile * Update nginx configuration * Update smtp * Add celery beat * Add feedjack update celery task * Add metabase configuration * Update test.yaml * Add test docker-compose Co-authored-by: sumandari --- .dockerignore | 1 + .github/workflows/test.yaml | 16 +- .gitignore | 2 + dockerize/.env.template | 6 + dockerize/Makefile | 46 +++- .../docker-compose.override.template.yml | 55 ++++ dockerize/docker-compose.override.test.yml | 19 ++ dockerize/docker-compose.yml | 252 ++++++++++-------- dockerize/docker/Dockerfile | 32 ++- dockerize/docker/REQUIREMENTS.txt | 3 +- dockerize/sites-enabled/default.conf | 17 ++ qgis-app/plugins/celery.py | 8 +- qgis-app/plugins/tasks/__init__.py | 3 +- .../plugins/tasks/generate_plugins_xml.py | 8 +- qgis-app/plugins/tasks/update_feedjack.py | 10 + qgis-app/plugins/tests/test_view.py | 27 ++ qgis-app/plugins/views.py | 45 +++- qgis-app/settings_docker.py | 23 +- 18 files changed, 423 insertions(+), 150 deletions(-) create mode 100644 .dockerignore create mode 100644 dockerize/.env.template create mode 100644 dockerize/docker-compose.override.template.yml create mode 100644 dockerize/docker-compose.override.test.yml create mode 100644 qgis-app/plugins/tasks/update_feedjack.py create mode 100644 qgis-app/plugins/tests/test_view.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7ab91dc3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +dockerize/postgres_data diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index afc17f03..8166e470 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -48,23 +48,29 @@ jobs: - uses: actions/checkout@v2 - name: Run docker-compose build - run: docker-compose build + run: docker-compose build devweb - - name: Run the containers - run: docker-compose up -d db devweb + - name: Run docker-compose services + working-directory: dockerize + run: | + cp docker-compose.override.test.yml docker-compose.override.yml + make devweb-test + make wait-db + make create-test-db - name: Run Coverage test run: | - cat << EOF | docker-compose exec -T devweb bash + cat << EOF | docker-compose -p qgis-plugins exec -T devweb bash pip install coverage python manage.py makemigrations python manage.py migrate + python manage.py collectstatic --noinput --verbosity 0 coverage run manage.py test coverage xml EOF - name: Upload coverage to codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 4267cc6f..073c345f 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ qgis-app/api/tests/*/ # whoosh_index qgis-app/whoosh_index/ +docker-compose.override.yml +.env diff --git a/dockerize/.env.template b/dockerize/.env.template new file mode 100644 index 00000000..cd4fbcfe --- /dev/null +++ b/dockerize/.env.template @@ -0,0 +1,6 @@ +RABBITMQ_HOST=rabbitmq +DATABASE_NAME=gis +DATABASE_USERNAME=docker +DATABASE_PASSWORD=docker +DATABASE_HOST=db +DJANGO_SETTINGS_MODULE=settings_docker diff --git a/dockerize/Makefile b/dockerize/Makefile index aa3360a8..bdbb9ff0 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -15,6 +15,13 @@ build: @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) build +build-dev: + @echo + @echo "------------------------------------------------------------------" + @echo "Building in development mode only" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) build devweb + db: @echo @echo "------------------------------------------------------------------" @@ -22,19 +29,33 @@ db: @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) up -d db +metabase: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running metabase in production mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) up -d metabase + web: db @echo @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d web + @docker-compose -p $(PROJECT_ID) up -d uwsgi web smtp worker beat + +devweb-test: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in TESTING mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb devweb: db @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb + @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat devweb-runserver: devweb @echo @@ -100,6 +121,20 @@ rm-only: kill @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) rm +maillogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp logs in production mode" + @echo "------------------------------------------------------------------" + @docker-compose exec smtp tail -f /var/log/mail.log + +mailerrorlogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp error logs in production mode" + @echo "------------------------------------------------------------------" + @docker-compose exec smtp tail -f /var/log/mail.err + dbrestore: @echo @echo "------------------------------------------------------------------" @@ -119,3 +154,10 @@ dbrestore: @docker exec -t $(PROJECT_ID)-db pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)-db su - postgres -c "psql gis" @docker-compose -p $(PROJECT_ID) start web @echo "starting web container" + +wait-db: + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "until pg_isready; do sleep 5; done" + +create-test-db: + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -c 'create database test_db;'" + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -d test_db -c 'create extension postgis;'" diff --git a/dockerize/docker-compose.override.template.yml b/dockerize/docker-compose.override.template.yml new file mode 100644 index 00000000..bb0fdd8a --- /dev/null +++ b/dockerize/docker-compose.override.template.yml @@ -0,0 +1,55 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" + + beat: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + worker: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + uwsgi: + container_name: qgis-plugins-uwsgi + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod + + db: + volumes: + - ./postgres_data:/var/lib/postgresql + - ./backups:/backups + + web: + volumes: + - ./sites-enabled:/etc/nginx/conf.d:ro + - ./static:/home/web/static:ro + - ./media:/home/web/media:ro + ports: + - "62201:8080" diff --git a/dockerize/docker-compose.override.test.yml b/dockerize/docker-compose.override.test.yml new file mode 100644 index 00000000..e10fe9a1 --- /dev/null +++ b/dockerize/docker-compose.override.test.yml @@ -0,0 +1,19 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 2ee672fb..24854bc0 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,117 +1,145 @@ -db: - container_name: qgis-plugins-db - image: kartoza/postgis:9.6-2.4 - environment: - - ALLOW_IP_RANGE=0.0.0.0/0 - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - volumes: - - ./backups:/backups - restart: unless-stopped +version: '3' +volumes: + backups-data: + static-data: + media-data: + logs-data: + nginx-conf: + db-backups: + rabbitmq: + celerybeat-schedule: + postgres-data: +services: -web: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-web - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - DEBUG=False - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - command: uwsgi --ini /uwsgi.conf + smtp: + image: catatnight/postfix:latest + hostname: postfix + environment: + - maildomain=qgis.org + - smtp_user=noreply:docker + restart: unless-stopped -devweb: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-devweb - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - ports: - # for django test server - - "62202:8080" - # for ssh - - "62203:22" + db: + container_name: qgis-plugins-db + image: kartoza/postgis:14-3.3 + environment: + - ALLOW_IP_RANGE=0.0.0.0/0 + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - PASSWORD_AUTHENTICATION=${PASSWORD_AUTHENTICATION:-md5} + volumes: + - postgres-data:/var/lib/postgresql + - backups-data:/backups + restart: unless-stopped -rabbitmq: - image: library/rabbitmq:3.6 - hostname: rabbitmq - environment: - - RABBIT_PASSWORD=rabbit_test_password - - USER=rabbit_user - - RABBITMQ_NODENAME=rabbit - restart: unless-stopped + uwsgi: &uwsgi-common + container_name: qgis-plugins-uwsgi + image: kartoza/qgis-plugins-uwsgi + hostname: uwsgi + environment: + - DATABASE_NAME=${DATABASE_NAME:-gis} + - DATABASE_USERNAME=${DATABASE_USERNAME:-docker} + - DATABASE_PASSWORD=${DATABASE_PASSWORD:-docker} + - DATABASE_HOST=${DATABASE_HOST:-db} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-settings_docker} + - VIRTUAL_HOST=${VIRTUAL_HOST:-plugins.kartoza.com} + - VIRTUAL_PORT=${VIRTUAL_PORT:-8080} + - DEBUG=${DEBUG:-False} + - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} + - BROKER_URL=amqp://rabbitmq:5672 + volumes: + - static-data:/home/web/static:rw + - media-data:/home/web/media:rw + - celerybeat-schedule:/home/web/celerybeat-schedule:rw + links: + - db:db + - rabbitmq:rabbitmq + restart: unless-stopped + user: root -worker: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-worker - build: docker - hostname: uwsgi - working_dir: /home/web/django_project - command: celery -A plugins worker -l info - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq + # This is the entry point for a development server. + # Run with --no-deps to run attached to the services + # from prod environment if wanted + devweb: + <<: *uwsgi-common + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev -nginx: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-nginx - image: nginx - hostname: nginx - volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro - # I dont use volumes_from as I want to use the ro modifier - - ./static:/home/web/static:ro - - ./static:/home/web/media:ro - - ./logs:/var/log/nginx - links: - - web:uwsgi - ports: - - "62201:8080" - restart: unless-stopped + rabbitmq: + image: rabbitmq:3.7-alpine + hostname: rabbitmq + volumes: + - rabbitmq:/var/lib/rabbitmq + restart: unless-stopped + + beat: + <<: *uwsgi-common + container_name: qgis-plugins-beat + working_dir: /home/web/django_project + entrypoint: [ ] + command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + + worker: + <<: *uwsgi-common + container_name: qgis-plugins-worker + links: + - db + - rabbitmq + - beat + working_dir: /home/web/django_project + entrypoint: [] + command: celery -A plugins worker -l INFO + + beat: + <<: *uwsgi-common + container_name: qgis-plugins-beat + working_dir: /home/web/django_project + entrypoint: [ ] + command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + + web: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-web + image: nginx + hostname: web + volumes: + - nginx-conf:/etc/nginx/conf.d:ro + - static-data:/home/web/static:ro + - media-data:/home/web/media:ro + links: + - uwsgi:uwsgi + - metabase:metabase + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + restart: unless-stopped + + dbbackups: + image: kartoza/pg-backup:14-3.3 + hostname: pg-backups + volumes: + - db-backups:/backups + links: + - db:db + environment: + # take care to let the project name below match that + # declared in the top of the makefile + - DUMPPREFIX=${DUMPPREFIX:-QGIS_PLUGINS} + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_HOST=${DATABASE_HOST:-db} + - PGDATABASE=${DATABASE_NAME:-gis} + restart: unless-stopped + + metabase: + image: metabase/metabase:latest + environment: + - MB_DB_TYPE=postgres + - MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker} + links: + - db diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index a77ab97b..8cc74ab1 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -1,18 +1,34 @@ #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ # Note this base image is based on debian -FROM kartoza/django-base:3.7 +FROM kartoza/django-base:3.7 as prod MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl RUN apt-get clean all RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev -ADD REQUIREMENTS.txt /REQUIREMENTS.txt + +RUN rm -rf /uwsgi.conf +ADD dockerize/docker/uwsgi.conf /uwsgi.conf +ADD qgis-app /home/web/django_project +ADD dockerize/docker/REQUIREMENTS.txt /REQUIREMENTS.txt RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt RUN pip install uwsgi +# Open port 8080 as we will be running our uwsgi socket on that +EXPOSE 8080 + +RUN mkdir -p /var/log/uwsgi + +WORKDIR /home/web/django_project +CMD ["uwsgi", "--ini", "/uwsgi.conf"] + + +FROM prod as dev + +# This section taken on 2 July 2015 from # https://docs.docker.com/examples/running_ssh_service/ # Sudo is needed by pycharm when it tries to pip install packages -RUN apt-get install -y openssh-server sudo +RUN apt-get update && apt-get install -y openssh-server sudo RUN mkdir /var/run/sshd RUN echo 'root:docker' | chpasswd RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config @@ -23,15 +39,13 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so ENV NOTVISIBLE "in users profile" RUN echo "export VISIBLE=now" >> /etc/profile -RUN rm -rf /uwsgi.conf -ADD uwsgi.conf /uwsgi.conf +# -------------------------------------------------------- +# Open ports as needed +# -------------------------------------------------------- -# Open port 8080 as we will be running our uwsgi socket on that +# Open port 8080 as we will be running our django dev server on EXPOSE 8080 - # Open port 22 as we will be using a remote interpreter from pycharm EXPOSE 22 -RUN mkdir -p /var/log/uwsgi -WORKDIR /home/web/django_project CMD ["/usr/sbin/sshd", "-D"] diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index e05c7661..70bb9dd9 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -35,7 +35,8 @@ django-haystack git+https://github.com/dimasciput/feedjack.git feedparser==5.2.1 -celery==4.3.1 +celery==5.2.7 +django-celery-beat==2.3.0 requests==2.23.0 markdown==3.2.1 diff --git a/dockerize/sites-enabled/default.conf b/dockerize/sites-enabled/default.conf index e870a2a6..8cf7cec7 100644 --- a/dockerize/sites-enabled/default.conf +++ b/dockerize/sites-enabled/default.conf @@ -28,6 +28,8 @@ server { } # max upload size, adjust to taste client_max_body_size 15M; + + # Django media location /media { # your Django project's media files - amend as required @@ -52,6 +54,15 @@ server { alias /home/web/archive; expires 21d; # cache for 6h } + + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + # Finally, send all non-media requests to the Django server. location / { uwsgi_pass uwsgi; @@ -74,4 +85,10 @@ server { uwsgi_param SERVER_PORT $server_port; uwsgi_param SERVER_NAME $server_name; } + + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } } diff --git a/qgis-app/plugins/celery.py b/qgis-app/plugins/celery.py index 030207ab..7a5c4468 100644 --- a/qgis-app/plugins/celery.py +++ b/qgis-app/plugins/celery.py @@ -1,6 +1,10 @@ from __future__ import absolute_import import os from celery import Celery +import logging + +logger = logging.getLogger('plugins') + # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings_docker') @@ -16,7 +20,3 @@ # Load task modules from all registered Django app configs. app.autodiscover_tasks() - -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) diff --git a/qgis-app/plugins/tasks/__init__.py b/qgis-app/plugins/tasks/__init__.py index c06d2250..a0df9010 100644 --- a/qgis-app/plugins/tasks/__init__.py +++ b/qgis-app/plugins/tasks/__init__.py @@ -1 +1,2 @@ -from plugins.tasks.generate_plugins_xml import * \ No newline at end of file +from plugins.tasks.generate_plugins_xml import * # noqa +from plugins.tasks.update_feedjack import * # noqa diff --git a/qgis-app/plugins/tasks/generate_plugins_xml.py b/qgis-app/plugins/tasks/generate_plugins_xml.py index 42493392..ad0d1281 100644 --- a/qgis-app/plugins/tasks/generate_plugins_xml.py +++ b/qgis-app/plugins/tasks/generate_plugins_xml.py @@ -1,17 +1,23 @@ import os import requests from celery import shared_task +from celery.utils.log import get_task_logger from preferences import preferences from django.conf import settings +logger = get_task_logger(__name__) + + @shared_task -def generate_plugins_xml(site = ''): +def generate_plugins_xml(site=''): """ Fetch the xml list of plugins from the plugin site. :param site: site domain where the plugins will be fetched, default to http://plugins.qgis.org """ + logger.info('generate_plugins_xml : {}'.format(site)) + if not site: if settings.DEFAULT_PLUGINS_SITE: site = settings.DEFAULT_PLUGINS_SITE diff --git a/qgis-app/plugins/tasks/update_feedjack.py b/qgis-app/plugins/tasks/update_feedjack.py new file mode 100644 index 00000000..e12d14bd --- /dev/null +++ b/qgis-app/plugins/tasks/update_feedjack.py @@ -0,0 +1,10 @@ +from celery import shared_task +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +@shared_task +def update_feedjack(): + import subprocess + subprocess.call(['python', 'manage.py', 'feedjackupdate']) diff --git a/qgis-app/plugins/tests/test_view.py b/qgis-app/plugins/tests/test_view.py new file mode 100644 index 00000000..67b9fc32 --- /dev/null +++ b/qgis-app/plugins/tests/test_view.py @@ -0,0 +1,27 @@ +from django.test import TestCase + +from plugins.views import _add_patch_version + + +class TestTruncateVersion(TestCase): + """Test _add_patch_version function""" + + def test__add_patch_version_with_3_segment_version_number(self): + version = '1.2.3' + self.assertEqual(_add_patch_version(version, '99'), '1.2.3') + + def test__add_patch_version_with_2_segment_version_number(self): + version = '1.2' + self.assertEqual(_add_patch_version(version, '00'), '1.2.00') + + def test__add_patch_version_with_1_segment_version_number(self): + version = '1' + self.assertEqual(_add_patch_version(version, '99'), '1') + + def test__add_patch_version_with_None(self): + version = None + self.assertEqual(_add_patch_version(version, '99'), None) + + def test__add_patch_version_with_empty_string(self): + version = '' + self.assertEqual(_add_patch_version(version, '99'), '') diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index d625e2fd..45919175 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -909,6 +909,25 @@ def version_detail(request, package_name, version): from django.views.decorators.cache import cache_page +def _add_patch_version(version: str, additional_patch: str ) -> str: + """To add patch number in version. + + e.g qgis version = 3.16 we add patch number (99) in versioning -> 3.16.99 + We use this versioning to query against PluginVersion min_qg_version, + so that the query result will include all PluginVersion with + minimum QGIS version 3.16 regardless of the patch number. + """ + + if not version: + return version + separator = '.' + v = version.split(separator) + if len(v) == 2: + two_first_segment = separator.join(v[:2]) + version = f'{two_first_segment}.{additional_patch}' + return version + + @cache_page(60 * 15) def xml_plugins(request, qg_version=None, stable_only=None, package_name=None): """ @@ -930,10 +949,10 @@ def xml_plugins(request, qg_version=None, stable_only=None, package_name=None): object_list = [] if qg_version: - filters.update({'pluginversion__min_qg_version__lte' : qg_version}) - version_filters.update({'min_qg_version__lte' : qg_version}) - filters.update({'pluginversion__max_qg_version__gte' : qg_version}) - version_filters.update({'max_qg_version__gte' : qg_version}) + filters.update({'pluginversion__min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + version_filters.update({'min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + filters.update({'pluginversion__max_qg_version__gte' : _add_patch_version(qg_version, '0')}) + version_filters.update({'max_qg_version__gte' : _add_patch_version(qg_version, '0')}) # Get all versions for the given plugin) @@ -1007,10 +1026,10 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non object_list = [] if qg_version: - filters.update({'pluginversion__min_qg_version__lte' : qg_version}) - version_filters.update({'min_qg_version__lte' : qg_version}) - filters.update({'pluginversion__max_qg_version__gte' : qg_version}) - version_filters.update({'max_qg_version__gte' : qg_version}) + filters.update({'pluginversion__min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + version_filters.update({'min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + filters.update({'pluginversion__max_qg_version__gte' : _add_patch_version(qg_version, '0')}) + version_filters.update({'max_qg_version__gte' : _add_patch_version(qg_version, '0')}) # Get all versions for the given plugin if package_name: @@ -1051,8 +1070,8 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non FROM %(pv_table)s pv WHERE ( pv.approved = True - AND pv."max_qg_version" >= '%(qg_version)s' - AND pv."min_qg_version" <= '%(qg_version)s' + AND pv."max_qg_version" >= '%(qg_version_with_patch_0)s' + AND pv."min_qg_version" <= '%(qg_version_with_patch_99)s' AND pv.experimental = %(experimental)s ) ORDER BY pv.plugin_id, pv.version DESC @@ -1062,7 +1081,8 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non object_list_new = PluginVersion.objects.raw(sql % { 'pv_table': PluginVersion._meta.db_table, 'p_table': Plugin._meta.db_table, - 'qg_version': qg_version, + 'qg_version_with_patch_0': _add_patch_version(qg_version, '0'), + 'qg_version_with_patch_99': _add_patch_version(qg_version, '99'), 'experimental': 'False', 'trusted_users_ids': str(trusted_users_ids), }) @@ -1074,7 +1094,8 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non object_list_new += [o for o in PluginVersion.objects.raw(sql % { 'pv_table': PluginVersion._meta.db_table, 'p_table': Plugin._meta.db_table, - 'qg_version': qg_version, + 'qg_version_with_patch_0': _add_patch_version(qg_version, '0'), + 'qg_version_with_patch_99': _add_patch_version(qg_version, '99'), 'experimental': 'True', 'trusted_users_ids': str(trusted_users_ids), })] diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index 2200839a..896c8a93 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -1,3 +1,5 @@ +from celery.schedules import crontab + from settings import * import ast import os @@ -43,6 +45,8 @@ # full text search postgres 'django.contrib.postgres', + 'feedjack', + # ABP: 'plugins', 'django.contrib.humanize', @@ -61,8 +65,6 @@ 'tinymce', 'rpc4django', - 'feedjack', - 'preferences', 'rest_framework', @@ -80,7 +82,7 @@ 'layerdefinitions', # models (sharing .model3 file feature) 'models', - 'wavefronts' + 'wavefronts', ] DATABASES = { @@ -125,3 +127,18 @@ 'TEST_REQUEST_DEFAULT_FORMAT': 'json', } +CELERY_RESULT_BACKEND = 'rpc://' +CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672') +CELERY_BEAT_SCHEDULE = { + 'generate_plugins_xml': { + 'task': 'plugins.tasks.generate_plugins_xml.generate_plugins_xml', + 'schedule': crontab(minute='*/10'), # Execute every 10 minutes. + 'kwargs': { + 'site': 'https://plugins.qgis.org/' + } + }, + 'update_feedjack': { + 'task': 'plugins.tasks.update_feedjack.update_feedjack', + 'schedule': crontab(minute='*/30'), # Execute every 30 minutes. + } +}