Celery

.env.example

@@ -0,0 +1 @@
+CLOUDAMQP_URL=amqp://guest:[email protected]:5672/

.gitignore

@@ -124,6 +124,7 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
-
+.vscode
+celerybeat*
 # Pyre type checker
 .pyre/

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.6.6  
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /app
+COPY requirements.txt ./  
+RUN apt-get update && apt-get -y install ghostscript && apt-get clean 
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt  
+COPY . /app  

api_covid19/tasks.py

@@ -0,0 +1,35 @@
+
+"""
+========================
+Celery Tasks
+========================
+For more celery crontab info visit: 
+    https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#crontab-schedules
+"""
+
+# celery imports
+from celery.decorators import task
+from celery.task.schedules import crontab
+from celery.decorators import periodic_task
+
+from .models import *
+from .scripts.fetch_data import run
+
+@periodic_task(run_every=(crontab(minute=0, hour='*/2')), name="update_info: Scrap https://www.gob.mx/ website every 2 hrs")
+def update_info():
+    """
+    This is a real world task.
+    """
+    print("Updating data...")
+    run()
+
+@periodic_task(run_every=(crontab()), name="example_task: This is example task running every minute")
+def example_task():
+    """
+    This is a example task
+    """
+    # do some stuff
+    # use models
+    # make requests
+    print("Running every minute...")
+    run() # just for test. May be deleted later

codeandomexico/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery_app import app as celery_app

codeandomexico/celery_app.py

@@ -0,0 +1,23 @@
+
+"""
+=======================
+Celery config
+=======================
+First steps with Django
+https://docs.celeryproject.org/en/latest/django/first-steps-with-django.html
+"""
+
+from __future__ import absolute_import
+from django.conf import settings
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'codeandomexico.settings')
+
+app = Celery(broker=settings.CELERY_BROKER_URL, backend='rpc://', namespace='CELERY')
+app.config_from_object('django.conf:settings')
+# app.autodiscover_tasks(settings.INSTALLED_APPS) 
+
+if __name__ == '__main__':
+    app.start()

codeandomexico/settings.py

@@ -82,6 +82,16 @@
     }
 }
 
+# Django Celery
+# https://pypi.org/project/django-celery/
+
+CELERY_BROKER_URL = os.environ.get('CLOUDAMQP_URL')
+CELERY_IMPORTS = ('api_covid19.tasks',)
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_TIMEZONE = "America/Mexico_City"
+
 
 # Password validation
 # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators

docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3'
+services:
+  ###
+  # services
+  ###
+  web:
+    build: .
+    network_mode: host
+    command: gunicorn codeandomexico.wsgi --log-file -
+    volumes:
+        - .:/app
+    ports:
+        - "8000:8000"
+    tty: true
+    env_file:
+      - ./.env
+    stdin_open: true
+
+  worker:
+      build: .
+      depends_on:
+        - rabbitmq
+      network_mode: host
+      command:  celery worker -A codeandomexico -B --loglevel=INFO
+      env_file:
+        - ./.env
+      volumes:
+        - .:/app
+
+  rabbitmq:
+    image: rabbitmq:3-management
+    restart: always
+    ports:
+      - "15672:15672"
+      - "5672:5672"
+    network_mode: host

requirements.txt

@@ -1,13 +1,39 @@
+amqp==2.5.2
+asgiref==3.2.7
 beautifulsoup4==4.8.2
+billiard==3.6.3.0
 bs4==0.0.1
 camelot-py==0.7.3
+celery==4.4.2
+certifi==2019.11.28
+chardet==3.0.4
+click==7.1.1
 Django==3.0.4
 django-appconf==1.0.3
 django-compressor==2.3
 django-extensions==2.2.3
+et-xmlfile==1.0.1
+gunicorn==20.0.4
+idna==2.8
+importlib-metadata==1.5.0
+jdcal==1.4.1
+kombu==4.6.8
+numpy==1.18.2
+opencv-python==4.2.0.32
+openpyxl==3.0.3
 pandas==1.0.1
 pdfminer.six==20200124
+pycryptodome==3.9.7
 PyPDF2==1.26.0
+python-dateutil==2.8.1
 pytz==2019.3
+rcssmin==1.0.6
 requests==2.22.0
-six==1.14.0
+rjsmin==1.1.0
+six==1.14.0
+sortedcontainers==2.1.0
+soupsieve==2.0
+sqlparse==0.3.1
+urllib3==1.25.8
+vine==1.3.0
+zipp==3.1.0