Add new high memory autoreducer node to docker-compose

Dockerfile.autoreducer

@@ -1,9 +1,9 @@
 FROM registry.access.redhat.com/ubi9/ubi
 
 # install various dependencies
-RUN curl http://packages.sns.gov/distros/rhel/9/sns/sns.repo -o /etc/dnf.repos.d/sns.repo || echo "Cannot see packages.sns.gov"
 RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
 RUN dnf updateinfo
+RUN dnf install -y procps-ng # pgrep is used for health check
 
 # rpm needs to special groups first
 RUN groupadd snswheel
@@ -12,16 +12,10 @@ RUN useradd snsdata -G snswheel
 # put the developer configuration file in place
 RUN mkdir -p /etc/autoreduce/
 RUN mkdir -p /opt/postprocessing/log/
-RUN curl https://raw.githubusercontent.com/neutrons/post_processing_agent/v3.0/configuration/post_process_consumer.conf.development -o /etc/autoreduce/post_processing.conf
-RUN dnf install -y jq
-RUN contents="$(jq 'del(.processors)' /etc/autoreduce/post_processing.conf)" && \
-echo -E "${contents}" > /etc/autoreduce/post_processing.conf
-
-# This configuration allows it to run with docker-compose from https://github.com/neutrons/data_workflow
-RUN sed -i 's/localhost/activemq/' /etc/autoreduce/post_processing.conf
+COPY tests/configuration/post_process_consumer.conf /etc/autoreduce/post_processing.conf
 
 # install postprocessing
-RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.0/postprocessing-3.0.0-1.el9.noarch.rpm
+RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.1/postprocessing-3.1.0-1.el9.noarch.rpm
 
 # install the fake test data
 ARG DATA_TARBALL=/tmp/SNSdata.tar.gz

Dockerfile.autoreducer.himem

@@ -0,0 +1,39 @@
+FROM registry.access.redhat.com/ubi9/ubi
+
+# install various dependencies
+RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN dnf updateinfo
+RUN dnf install -y procps-ng # pgrep is used for health check
+
+# rpm needs to special groups first
+RUN groupadd snswheel
+RUN useradd snsdata -G snswheel
+
+# put the developer configuration file in place
+RUN mkdir -p /etc/autoreduce/
+RUN mkdir -p /opt/postprocessing/log/
+COPY tests/configuration/post_process_consumer.himem.conf /etc/autoreduce/post_processing.conf
+
+# install postprocessing
+RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.1/postprocessing-3.1.0-1.el9.noarch.rpm
+
+# install the fake test data
+ARG DATA_TARBALL=/tmp/SNSdata.tar.gz
+COPY SNSdata.tar.gz ${DATA_TARBALL}
+RUN ls ${DATA_TARBALL}
+RUN mkdir /SNS
+RUN cd /SNS && tar xzf ${DATA_TARBALL}
+
+# add fake ONCat ingest scripts
+RUN touch /opt/postprocessing/scripts/oncat_ingest.py && \
+    touch /opt/postprocessing/scripts/oncat_reduced_ingest.py
+
+# create startup script
+RUN echo "#!/bin/bash" > /usr/bin/run_postprocessing && \
+    echo "/opt/postprocessing/queueProcessor.py &" >> /usr/bin/run_postprocessing && \
+    echo "sleep 1" >> /usr/bin/run_postprocessing && \
+    echo "tail -F /opt/postprocessing/log/postprocessing.log" >> /usr/bin/run_postprocessing && \
+    chmod +x /usr/bin/run_postprocessing
+
+# start the service
+CMD run_postprocessing

docker-compose.yml

@@ -115,15 +115,25 @@ services:
       context: .
       dockerfile: Dockerfile.autoreducer
       network: host
-    ports:
-      - "8888:8888"
     hostname: autoreducer
     healthcheck:
       test: ["CMD", "pgrep", "python"]
     depends_on:
       activemq:
         condition: service_healthy
 
+  autoreducer_himem:
+    build:
+      context: .
+      dockerfile: Dockerfile.autoreducer.himem
+      network: host
+    hostname: autoreducer.himem
+    healthcheck:
+      test: ["CMD", "pgrep", "python"]
+    depends_on:
+      activemq:
+        condition: service_healthy
+
   amq_test_gen:
     restart: always
     build:

tests/configuration/post_process_consumer.conf

@@ -0,0 +1,28 @@
+{
+    "failover_uri": "failover:(tcp://localhost:61613)?randomize=false,startupMaxReconnectAttempts=100,initialReconnectDelay=1000,maxReconnectDelay=5000,maxReconnectAttempts=-1",
+    "brokers": [["activemq", 61613]],
+    "amq_user": "icat",
+    "amq_pwd": "icat",
+    "sw_dir": "/opt/postprocessing",
+    "python_dir": "/opt/postprocessing/postprocessing",
+    "start_script": "python3",
+    "task_script": "PostProcessAdmin.py",
+    "task_script_queue_arg": "-q",
+    "task_script_data_arg": "-d",
+    "log_file": "/opt/postprocessing/log/postprocessing.log",
+    "postprocess_error": "POSTPROCESS.ERROR",
+    "reduction_started": "REDUCTION.STARTED",
+    "reduction_complete": "REDUCTION.COMPLETE",
+    "reduction_error": "REDUCTION.ERROR",
+    "reduction_disabled": "REDUCTION.DISABLED",
+    "heart_beat": "/topic/SNS.COMMON.STATUS.AUTOREDUCE.0",
+    "dev_output_dir": "",
+    "reduction_data_ready": "REDUCTION.DATA_READY",
+    "communication_only": 0,
+    "max_procs": 5,
+    "processors": ["oncat_processor.ONCatProcessor",
+                   "oncat_reduced_processor.ONCatProcessor",
+                   "create_reduction_script_processor.CreateReductionScriptProcessor",
+                   "reduction_processor.ReductionProcessor"
+                   ]
+}

tests/configuration/post_process_consumer.himem.conf

@@ -0,0 +1,24 @@
+{
+    "failover_uri": "failover:(tcp://localhost:61613)?randomize=false,startupMaxReconnectAttempts=100,initialReconnectDelay=1000,maxReconnectDelay=5000,maxReconnectAttempts=-1",
+    "brokers": [["activemq", 61613]],
+    "amq_user": "icat",
+    "amq_pwd": "icat",
+    "sw_dir": "/opt/postprocessing",
+    "python_dir": "/opt/postprocessing/postprocessing",
+    "start_script": "python3",
+    "task_script": "PostProcessAdmin.py",
+    "task_script_queue_arg": "-q",
+    "task_script_data_arg": "-d",
+    "log_file": "/opt/postprocessing/log/postprocessing.log",
+    "postprocess_error": "POSTPROCESS.ERROR",
+    "reduction_started": "REDUCTION.STARTED",
+    "reduction_complete": "REDUCTION.COMPLETE",
+    "reduction_error": "REDUCTION.ERROR",
+    "reduction_disabled": "REDUCTION.DISABLED",
+    "heart_beat": "/topic/SNS.COMMON.STATUS.AUTOREDUCE.0",
+    "dev_output_dir": "",
+    "reduction_data_ready": "REDUCTION.DATA_READY",
+    "communication_only": 0,
+    "max_procs": 5,
+    "processors": ["reduction_processor.ReductionProcessorHighMemory"]
+}

tests/test_DASMONPageView.py

@@ -35,7 +35,7 @@ def testVerifyDASMONPageView(self, dasmon_diagnostics):
         tree = etree.parse(StringIO(dasmon_diagnostics.text), parser)
         table_content = tree.xpath("//tr/td//text()")
         # verify number of entries in the tables
-        expected_number_of_entries = 39
+        expected_number_of_entries = 43
         assert len(table_content) == expected_number_of_entries
         # -- DASMON diagnostics
         status = table_content[1]