Roslin qc

.gitignore

@@ -7,4 +7,5 @@ db*
 scripts/*.json
 conda*
 input.json
+scripts/*.json*
 *.swp

Makefile

@@ -217,7 +217,7 @@ check-env:
 	BEAGLE_AUTH_LDAP_SERVER_URI \
 	BEAGLE_LIMS_PASSWORD \
 	BEAGLE_LIMS_USERNAME; do \
-	[ -z "$$(printenv BEAGLE_LIMS_USERNAME)" ] && echo ">>> env variable $$i is not set; some features may not work" || : ; done
+	[ -z "$$(printenv $$i)" ] && echo ">>> env variable $$i is not set; some features may not work" || : ; done
 
 # start the RabbitMQ server in the background
 rabbitmq-start: $(LOG_DIR_ABS)
@@ -273,7 +273,7 @@ celery-start:
 
 # check that the Celery processes are running
 celery-check:
-	-ps auxww | grep 'celery' | grep -v 'grep' | grep -v 'make' | grep '$(CURDIR)'
+	ps auxww | grep 'celery' | grep -v 'grep' | grep -v 'make' | grep '$(CURDIR)'
 
 # kill all the Celery processes started from this dir
 celery-stop:
@@ -291,11 +291,14 @@ start-services: check-env
 	$(MAKE) celery-start
 
 stop-services:
-	$(MAKE) celery-stop
-	$(MAKE) rabbitmq-stop
-	$(MAKE) db-stop
-
+	-$(MAKE) celery-stop
+	-$(MAKE) rabbitmq-stop
+	-$(MAKE) db-stop
 
+check-services:
+	$(MAKE) celery-check
+	$(MAKE) db-check
+	$(MAKE) rabbitmq-check
 
 # ~~~~~ Set Up Django ~~~~~ #
 export DJANGO_BEAGLE_IP:=localhost
@@ -311,15 +314,20 @@ export DJANGO_RIDGEBACK_IP:=localhost
 export DJANGO_RIDGEBACK_PORT:=7001
 export BEAGLE_RIDGEBACK_URL=http://$(DJANGO_RIDGEBACK_IP):$(DJANGO_RIDGEBACK_PORT)
 export DJ_DEBUG_LOG:=$(LOG_DIR_ABS)/dj.debug.log
-
+export BEAGLE_LOG_PATH:=$(LOG_DIR_ABS)/beagle-server.log
+export DUMP_JSON:=False
 
 # initialize the Django app in the database
 # do this after setting up the db above
 django-init:
-	python manage.py makemigrations --merge
+	python manage.py makemigrations # --merge
 	python manage.py migrate
+	$(MAKE) django-load-fixtures
 	python manage.py createsuperuser
+
+django-load-fixtures:
 	python manage.py loaddata \
+	beagle_etl.operator.json \
 	file_system.filegroup.json \
 	file_system.filetype.json \
 	file_system.storage.json \
@@ -336,6 +344,7 @@ test-lims: check-env
 
 # start the Django development server
 runserver: check-env
+	if $(MAKE) check-services ; then : ; else echo ">>> ERROR: some services are not started"; exit 1 ; fi
 	python manage.py runserver $(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)
 
 MIGRATION_ARGS?=
@@ -377,9 +386,10 @@ REQID:=07264_G
 TOKEN:=IUzI1NiJ9.eyJ0
 # import files data about samples in a request from the IGO LIMS
 import:
+	token=$$( jq -r '.token' "$(AUTH_FILE)" ) && \
 	curl -H "Content-Type: application/json" \
 	-X POST \
-	-H "Authorization: Bearer $(TOKEN)" \
+	-H "Authorization: Bearer $$token" \
 	--data '{"request_ids":["$(REQID)"]}' \
 	http://$(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)/v0/etl/import-requests/
 
@@ -430,6 +440,7 @@ files-request:
 	--data '{"request_ids":["$(REQID)"]}' \
 	http://$(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)/v0/fs/files/
 # python ./beagle_cli.py files list --metadata='requestId:$(REQID)'
+# http://localhost:6991/v0/fs/files/?metadata=requestId:DemoRequest1
 
 # get info on a single file
 REQFILE:=b37.fasta
@@ -440,22 +451,26 @@ file-get:
 	http://$(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)/v0/fs/files/?filename=$(REQFILE)
 
 # start a Roslin run for a given request in the Beagle db
-run-request:
+PIPELINE:=roslin
+run-request: $(AUTH_FILE)
+	token=$$( jq -r '.token' "$(AUTH_FILE)" ) && \
 	curl -H "Content-Type: application/json" \
 	-X POST \
-	-H "Authorization: Bearer $(TOKEN)" \
-	--data '{"request_ids":["$(REQID)"], "pipeline_name": "roslin"}' \
+	-H "Authorization: Bearer $$token" \
+	--data '{"request_ids":["$(REQID)"], "pipeline_name": "$(PIPELINE)"}' \
 	http://$(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)/v0/run/request/
 
 # send a pipeline input to the API to start running
 REQJSON:=fixtures/tests/run_roslin.json
 run-request-api: $(AUTH_FILE)
-	@token=$$( jq -r '.token' "$(AUTH_FILE)" ) && \
+	token=$$( jq -r '.token' "$(AUTH_FILE)" ) && \
 	curl -H "Content-Type: application/json" \
 	-X POST \
 	-H "Authorization: Bearer $$token" \
 	--data @$(REQJSON) \
 	http://$(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)/v0/run/api/
+# http://localhost:6991/v0/run/api/?metadata=requestId:10277_C
+# http://localhost:6991/v0/run/api/?requestId%3D10277_C/
 
 # make a demo Roslin input.json file from the template fixture;
 # need to update the fixture with the app ID of the demo pipeline that was loaded in the database
@@ -471,12 +486,32 @@ $(DEMO_INPUT): $(INPUT_TEMPLATE) $(AUTH_FILE)
 .PHONY: $(DEMO_INPUT)
 
 # submit a demo Roslin run using the dev Roslin pipeline entry in the database
-demo-run: register-dev-pipeline $(DEMO_INPUT)
+# submit using the API endpoint; bypasses the Operator
+demo-run-api: register-dev-pipeline $(DEMO_INPUT)
 	@python manage.py loaddata fixtures/tests/juno_roslin_demo2.file.json
 	@python manage.py loaddata fixtures/tests/juno_roslin_demo2.filemetadata.json
 	@python manage.py loaddata fixtures/tests/roslin_reference_files.json
 	@$(MAKE) run-request-api REQID=DemoRequest1 REQJSON=$(DEMO_INPUT)
 
+demo-roslin-qc:
+	python manage.py loaddata fixtures/tests/ca18b090-03ad-4bef-acd3-52600f8e62eb.run.full.json && \
+	token=$$( jq -r '.token' "$(AUTH_FILE)" ) && \
+	curl -H "Content-Type: application/json" \
+	-X POST \
+	-H "Authorization: Bearer $$token" \
+	--data '{"request_ids":[], "run_ids":["ca18b090-03ad-4bef-acd3-52600f8e62eb"], "pipeline_name": "roslin-qc"}' \
+	http://$(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)/v0/run/request/
+# python manage.py loaddata fixtures/tests/8f44d2f8-15c0-4d97-a966-6ad0b916bb41.run.ALN-REQ-ID.json && \
+
+# run the update-request endpoint for a request ID in order to update the metadata about a request
+update-request:
+	@token=$$( jq -r '.token' "$(AUTH_FILE)" ) && \
+	curl -H "Content-Type: application/json" \
+	-X POST \
+	-H "Authorization: Bearer $$token" \
+	--data '{"request_ids":["$(REQID)"], "pipeline_name": "roslin"}' \
+	http://$(DJANGO_BEAGLE_IP):$(DJANGO_BEAGLE_PORT)/v0/etl/update-requests/
+
 # check if the ports needed for services and servers are already in use on this system
 ifeq ($(UNAME), Darwin)
 # On macOS High Sierra, use this command: lsof -nP -i4TCP:$PORT | grep LISTEN
@@ -514,5 +549,3 @@ PORT=
 port-check:
 	ss -lntup | grep ':$(PORT)'
 endif
-
-

beagle/settings.py

@@ -316,3 +316,4 @@
 PROJECT_DIR = os.path.dirname(os.path.realpath(__file__))
 ROOT_DIR = os.path.dirname(PROJECT_DIR)
 TEST_FIXTURE_DIR = os.path.join(ROOT_DIR, "fixtures", "tests")
+DUMP_JSON = os.environ.get('DUMP_JSON') == 'True'

beagle_etl/fixtures/beagle_etl.operator.json

@@ -18,5 +18,15 @@
             "class_name": "AccessOperator",
             "slug": "access"
         }
+    },
+    {
+        "model": "beagle_etl.operator",
+        "pk": 3,
+        "fields": {
+            "active": true,
+            "recipes": "[]",
+            "class_name": "RoslinQcOperator",
+            "slug": "roslin-qc"
+        }
     }
 ]