Skip to content

Commit 90a7993

Browse files
rosswhitfieldrosswhitfield
committed
Add new high memory autoreducer node to docker-compose
1 parent 2dd77ec commit 90a7993

File tree

6 files changed

+107
-12
lines changed

6 files changed

+107
-12
lines changed

‎Dockerfile.autoreducer

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
FROM registry.access.redhat.com/ubi9/ubi
22

33
# install various dependencies
4-
RUN curl http://packages.sns.gov/distros/rhel/9/sns/sns.repo -o /etc/dnf.repos.d/sns.repo || echo "Cannot see packages.sns.gov"
54
RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
65
RUN dnf updateinfo
6+
RUN dnf install -y procps-ng # pgrep is used for health check
77

88
# rpm needs to special groups first
99
RUN groupadd snswheel
@@ -12,16 +12,10 @@ RUN useradd snsdata -G snswheel
1212
# put the developer configuration file in place
1313
RUN mkdir -p /etc/autoreduce/
1414
RUN mkdir -p /opt/postprocessing/log/
15-
RUN curl https://raw.githubusercontent.com/neutrons/post_processing_agent/v3.0/configuration/post_process_consumer.conf.development -o /etc/autoreduce/post_processing.conf
16-
RUN dnf install -y jq
17-
RUN contents="$(jq 'del(.processors)' /etc/autoreduce/post_processing.conf)" && \
18-
echo -E "${contents}" > /etc/autoreduce/post_processing.conf
19-
20-
# This configuration allows it to run with docker-compose from https://github.com/neutrons/data_workflow
21-
RUN sed -i 's/localhost/activemq/' /etc/autoreduce/post_processing.conf
15+
COPY tests/configuration/post_process_consumer.conf /etc/autoreduce/post_processing.conf
2216

2317
# install postprocessing
24-
RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.0/postprocessing-3.0.0-1.el9.noarch.rpm
18+
RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.1/postprocessing-3.1.0-1.el9.noarch.rpm
2519

2620
# install the fake test data
2721
ARG DATA_TARBALL=/tmp/SNSdata.tar.gz

‎Dockerfile.autoreducer.himem

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
FROM registry.access.redhat.com/ubi9/ubi
2+
3+
# install various dependencies
4+
RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
5+
RUN dnf updateinfo
6+
RUN dnf install -y procps-ng # pgrep is used for health check
7+
8+
# rpm needs to special groups first
9+
RUN groupadd snswheel
10+
RUN useradd snsdata -G snswheel
11+
12+
# put the developer configuration file in place
13+
RUN mkdir -p /etc/autoreduce/
14+
RUN mkdir -p /opt/postprocessing/log/
15+
COPY tests/configuration/post_process_consumer.himem.conf /etc/autoreduce/post_processing.conf
16+
17+
# install postprocessing
18+
RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.1/postprocessing-3.1.0-1.el9.noarch.rpm
19+
20+
# install the fake test data
21+
ARG DATA_TARBALL=/tmp/SNSdata.tar.gz
22+
COPY SNSdata.tar.gz ${DATA_TARBALL}
23+
RUN ls ${DATA_TARBALL}
24+
RUN mkdir /SNS
25+
RUN cd /SNS && tar xzf ${DATA_TARBALL}
26+
27+
# add fake ONCat ingest scripts
28+
RUN touch /opt/postprocessing/scripts/oncat_ingest.py && \
29+
touch /opt/postprocessing/scripts/oncat_reduced_ingest.py
30+
31+
# create startup script
32+
RUN echo "#!/bin/bash" > /usr/bin/run_postprocessing && \
33+
echo "/opt/postprocessing/queueProcessor.py &" >> /usr/bin/run_postprocessing && \
34+
echo "sleep 1" >> /usr/bin/run_postprocessing && \
35+
echo "tail -F /opt/postprocessing/log/postprocessing.log" >> /usr/bin/run_postprocessing && \
36+
chmod +x /usr/bin/run_postprocessing
37+
38+
# start the service
39+
CMD run_postprocessing

‎docker-compose.yml

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,15 +115,25 @@ services:
115115
context: .
116116
dockerfile: Dockerfile.autoreducer
117117
network: host
118-
ports:
119-
- "8888:8888"
120118
hostname: autoreducer
121119
healthcheck:
122120
test: ["CMD", "pgrep", "python"]
123121
depends_on:
124122
activemq:
125123
condition: service_healthy
126124

125+
autoreducer_himem:
126+
build:
127+
context: .
128+
dockerfile: Dockerfile.autoreducer.himem
129+
network: host
130+
hostname: autoreducer.himem
131+
healthcheck:
132+
test: ["CMD", "pgrep", "python"]
133+
depends_on:
134+
activemq:
135+
condition: service_healthy
136+
127137
amq_test_gen:
128138
restart: always
129139
build:

‎tests/configuration/post_process_consumer.conf

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"failover_uri": "failover:(tcp://localhost:61613)?randomize=false,startupMaxReconnectAttempts=100,initialReconnectDelay=1000,maxReconnectDelay=5000,maxReconnectAttempts=-1",
3+
"brokers": [["activemq", 61613]],
4+
"amq_user": "icat",
5+
"amq_pwd": "icat",
6+
"sw_dir": "/opt/postprocessing",
7+
"python_dir": "/opt/postprocessing/postprocessing",
8+
"start_script": "python3",
9+
"task_script": "PostProcessAdmin.py",
10+
"task_script_queue_arg": "-q",
11+
"task_script_data_arg": "-d",
12+
"log_file": "/opt/postprocessing/log/postprocessing.log",
13+
"postprocess_error": "POSTPROCESS.ERROR",
14+
"reduction_started": "REDUCTION.STARTED",
15+
"reduction_complete": "REDUCTION.COMPLETE",
16+
"reduction_error": "REDUCTION.ERROR",
17+
"reduction_disabled": "REDUCTION.DISABLED",
18+
"heart_beat": "/topic/SNS.COMMON.STATUS.AUTOREDUCE.0",
19+
"dev_output_dir": "",
20+
"reduction_data_ready": "REDUCTION.DATA_READY",
21+
"communication_only": 0,
22+
"max_procs": 5,
23+
"processors": ["oncat_processor.ONCatProcessor",
24+
"oncat_reduced_processor.ONCatProcessor",
25+
"create_reduction_script_processor.CreateReductionScriptProcessor",
26+
"reduction_processor.ReductionProcessor"
27+
]
28+
}

‎tests/configuration/post_process_consumer.himem.conf

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"failover_uri": "failover:(tcp://localhost:61613)?randomize=false,startupMaxReconnectAttempts=100,initialReconnectDelay=1000,maxReconnectDelay=5000,maxReconnectAttempts=-1",
3+
"brokers": [["activemq", 61613]],
4+
"amq_user": "icat",
5+
"amq_pwd": "icat",
6+
"sw_dir": "/opt/postprocessing",
7+
"python_dir": "/opt/postprocessing/postprocessing",
8+
"start_script": "python3",
9+
"task_script": "PostProcessAdmin.py",
10+
"task_script_queue_arg": "-q",
11+
"task_script_data_arg": "-d",
12+
"log_file": "/opt/postprocessing/log/postprocessing.log",
13+
"postprocess_error": "POSTPROCESS.ERROR",
14+
"reduction_started": "REDUCTION.STARTED",
15+
"reduction_complete": "REDUCTION.COMPLETE",
16+
"reduction_error": "REDUCTION.ERROR",
17+
"reduction_disabled": "REDUCTION.DISABLED",
18+
"heart_beat": "/topic/SNS.COMMON.STATUS.AUTOREDUCE.0",
19+
"dev_output_dir": "",
20+
"reduction_data_ready": "REDUCTION.DATA_READY",
21+
"communication_only": 0,
22+
"max_procs": 5,
23+
"processors": ["reduction_processor.ReductionProcessorHighMemory"]
24+
}

‎tests/test_DASMONPageView.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ def testVerifyDASMONPageView(self, dasmon_diagnostics):
3535
tree = etree.parse(StringIO(dasmon_diagnostics.text), parser)
3636
table_content = tree.xpath("//tr/td//text()")
3737
# verify number of entries in the tables
38-
expected_number_of_entries = 39
38+
expected_number_of_entries = 43
3939
assert len(table_content) == expected_number_of_entries
4040
# -- DASMON diagnostics
4141
status = table_content[1]

0 commit comments

Comments
 (0)