From 3a8aa65fa7895d248f9187e140e2c865f6be646a Mon Sep 17 00:00:00 2001 From: skyler-kv Date: Wed, 23 Oct 2024 11:21:35 +0000 Subject: [PATCH 1/5] fixed python2 and 3 compatibility issues --- contrib/datawave-quickstart/bin/query.sh | 39 ++++++++++++------- .../ingest-examples/tvmaze-api-query.sh | 15 +++++-- contrib/datawave-quickstart/docker/Dockerfile | 2 +- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/contrib/datawave-quickstart/bin/query.sh b/contrib/datawave-quickstart/bin/query.sh index ced895a3d66..f9743987f60 100644 --- a/contrib/datawave-quickstart/bin/query.sh +++ b/contrib/datawave-quickstart/bin/query.sh @@ -131,21 +131,32 @@ function setQueryIdFromResponse() { } function prettyPrintJson() { - local PY=$( which python ) - if [ -n "${PY}" ] ; then - echo "${1}" | ${PY} -c 'from __future__ import print_function;import sys,json;data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))' - local exitStatus=$? - echo - if [ "${exitStatus}" != "0" ] ; then - printRawResponse "${1}" - warn "Python encountered error. Printed response without formatting" - echo + PY_CMD='"from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True)"' + PY3=$(which python3 2>/dev/null) + PY2=$(which python2 2>/dev/null) + if [ -n "${PY3}" ] ; then + echo "${1}" | "${PY3}" -c "${PY_CMD}" + local exitStatus=$? + echo + if [ "${exitStatus}" != "0" ] ; then + printRawResponse "${1}" + warn "Python encountered error. Printed response without formatting" + echo + fi + elif [ -n "${PY2}" ] ; then + echo "${1}" | "${PY2}" -c "${PY_CMD}" + local exitStatus=$? + echo + if [ "${exitStatus}" != "0" ] ; then + printRawResponse "${1}" + warn "Python encountered error. Printed response without formatting" + echo + fi + else + printRawResponse "${1}" + warn "Couldn't find Python in your environment. Json response was printed without formatting" + echo fi - else - printRawResponse "${1}" - warn "Couldn't find python in your environment. Json response was printed without formatting" - echo - fi } function printRawResponse() { diff --git a/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh b/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh index af1da19187e..c86792b708b 100755 --- a/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh +++ b/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh @@ -38,10 +38,17 @@ TVMAZE_RESPONSE_STATUS=$( echo ${CURL_RESPONSE} | tr -d '\n' | sed -e 's/.*HTTP_ [ "${TVMAZE_RESPONSE_STATUS}" != "200" ] && error "api.tvmaze.com returned invalid response status: ${TVMAZE_RESPONSE_STATUS}" && exit 1 [ -z "${TVMAZE_RESPONSE_BODY}" ] && error "Response body is empty!" && exit 1 -if [ "${PRETTY}" == true ] ; then - echo "${TVMAZE_RESPONSE_BODY}" | python -c 'from __future__ import print_function;import sys,json;data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))' -else +PY_CMD="-c 'from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))' " +PY3="/usr/bin/python3" +PY2="/usr/bin/python2" + if [ "${PRETTY}" == true ] ; then + if [ -d "${PY3}" ]; then + echo "${TVMAZE_RESPONSE_BODY}" | "${PY3}" ${PY_CMD} + elif [ -d "${PY2}" ]; then + echo "${TVMAZE_RESPONSE_BODY}" | "${PY2}" ${PY_CMD} + fi + else echo "${TVMAZE_RESPONSE_BODY}" -fi + fi exit 0 \ No newline at end of file diff --git a/contrib/datawave-quickstart/docker/Dockerfile b/contrib/datawave-quickstart/docker/Dockerfile index 3d2e2bd238f..7396b0c7527 100644 --- a/contrib/datawave-quickstart/docker/Dockerfile +++ b/contrib/datawave-quickstart/docker/Dockerfile @@ -35,7 +35,7 @@ COPY . /opt/datawave # Install dependencies, configure password-less/zero-prompt SSH... -RUN dnf -y install gcc-c++ openssl openssh openssh-server openssh-clients openssl-libs which bc wget git java-11-openjdk-devel iproute && \ +RUN dnf -y install gcc-c++ openssl python3 openssh openssh-server openssh-clients openssl-libs which bc wget git java-11-openjdk-devel iproute && \ dnf clean all && \ ssh-keygen -q -N "" -t rsa -f ~/.ssh/id_rsa && \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \ From 7e90b33a1b5d627a02c9b41ac8db7e5ae0592174 Mon Sep 17 00:00:00 2001 From: skyler-kv Date: Wed, 23 Oct 2024 13:53:20 +0000 Subject: [PATCH 2/5] consolidated code and fixed indentation formatting --- contrib/datawave-quickstart/bin/query.sh | 41 ++++++++----------- .../ingest-examples/tvmaze-api-query.sh | 16 ++++---- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/contrib/datawave-quickstart/bin/query.sh b/contrib/datawave-quickstart/bin/query.sh index f9743987f60..c0a561cd7f0 100644 --- a/contrib/datawave-quickstart/bin/query.sh +++ b/contrib/datawave-quickstart/bin/query.sh @@ -132,31 +132,22 @@ function setQueryIdFromResponse() { function prettyPrintJson() { PY_CMD='"from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True)"' - PY3=$(which python3 2>/dev/null) - PY2=$(which python2 2>/dev/null) - if [ -n "${PY3}" ] ; then - echo "${1}" | "${PY3}" -c "${PY_CMD}" - local exitStatus=$? - echo - if [ "${exitStatus}" != "0" ] ; then - printRawResponse "${1}" - warn "Python encountered error. Printed response without formatting" - echo - fi - elif [ -n "${PY2}" ] ; then - echo "${1}" | "${PY2}" -c "${PY_CMD}" - local exitStatus=$? - echo - if [ "${exitStatus}" != "0" ] ; then - printRawResponse "${1}" - warn "Python encountered error. Printed response without formatting" - echo - fi - else - printRawResponse "${1}" - warn "Couldn't find Python in your environment. Json response was printed without formatting" - echo - fi + PY3=$(which python3 2>/dev/null) + PY2=$(which python2 2>/dev/null) + if [[ ( -n "${PY3}" ) || ( -n "${PY2}" ) ]] ; then + echo "${1}" | ( "${PY3}" -c "${PY_CMD}" || "${PY2}" -c "${PY_CMD}" ) + local exitStatus=$? + echo + if [ "${exitStatus}" != "0" ] ; then + printRawResponse "${1}" + warn "Python encountered error. Printed response without formatting" + echo + fi + else + printRawResponse "${1}" + warn "Couldn't find Python in your environment. Json response was printed without formatting" + echo + fi } function printRawResponse() { diff --git a/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh b/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh index c86792b708b..eaccd31d981 100755 --- a/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh +++ b/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh @@ -41,14 +41,14 @@ TVMAZE_RESPONSE_STATUS=$( echo ${CURL_RESPONSE} | tr -d '\n' | sed -e 's/.*HTTP_ PY_CMD="-c 'from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))' " PY3="/usr/bin/python3" PY2="/usr/bin/python2" - if [ "${PRETTY}" == true ] ; then - if [ -d "${PY3}" ]; then - echo "${TVMAZE_RESPONSE_BODY}" | "${PY3}" ${PY_CMD} - elif [ -d "${PY2}" ]; then - echo "${TVMAZE_RESPONSE_BODY}" | "${PY2}" ${PY_CMD} - fi - else - echo "${TVMAZE_RESPONSE_BODY}" +if [ "${PRETTY}" == true ] ; then + if [ -d "${PY3}" ]; then + echo "${TVMAZE_RESPONSE_BODY}" | "${PY3}" ${PY_CMD} + elif [ -d "${PY2}" ]; then + echo "${TVMAZE_RESPONSE_BODY}" | "${PY2}" ${PY_CMD} fi +else + echo "${TVMAZE_RESPONSE_BODY}" +fi exit 0 \ No newline at end of file From b70c3972a8f38307738e32bede9231c2c3130d8d Mon Sep 17 00:00:00 2001 From: skyler-kv Date: Wed, 23 Oct 2024 16:39:47 +0000 Subject: [PATCH 3/5] more consolidationgit status --- contrib/datawave-quickstart/bin/query.sh | 19 ++----------------- .../ingest-examples/tvmaze-api-query.sh | 10 ++-------- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/contrib/datawave-quickstart/bin/query.sh b/contrib/datawave-quickstart/bin/query.sh index c0a561cd7f0..c5f89615de6 100644 --- a/contrib/datawave-quickstart/bin/query.sh +++ b/contrib/datawave-quickstart/bin/query.sh @@ -131,23 +131,8 @@ function setQueryIdFromResponse() { } function prettyPrintJson() { - PY_CMD='"from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True)"' - PY3=$(which python3 2>/dev/null) - PY2=$(which python2 2>/dev/null) - if [[ ( -n "${PY3}" ) || ( -n "${PY2}" ) ]] ; then - echo "${1}" | ( "${PY3}" -c "${PY_CMD}" || "${PY2}" -c "${PY_CMD}" ) - local exitStatus=$? - echo - if [ "${exitStatus}" != "0" ] ; then - printRawResponse "${1}" - warn "Python encountered error. Printed response without formatting" - echo - fi - else - printRawResponse "${1}" - warn "Couldn't find Python in your environment. Json response was printed without formatting" - echo - fi + PY_CMD='from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True)' + echo "${1}" | ( "${PY3}" -c "${PY_CMD}" 2>/dev/null || "${PY2}" -c "${PY_CMD}" 2>/dev/null ) || ( warn "Python encountered error. Printed response without formatting" && printRawResponse "${1}" ) } function printRawResponse() { diff --git a/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh b/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh index eaccd31d981..69cf2d47fa5 100755 --- a/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh +++ b/contrib/datawave-quickstart/bin/services/datawave/ingest-examples/tvmaze-api-query.sh @@ -38,15 +38,9 @@ TVMAZE_RESPONSE_STATUS=$( echo ${CURL_RESPONSE} | tr -d '\n' | sed -e 's/.*HTTP_ [ "${TVMAZE_RESPONSE_STATUS}" != "200" ] && error "api.tvmaze.com returned invalid response status: ${TVMAZE_RESPONSE_STATUS}" && exit 1 [ -z "${TVMAZE_RESPONSE_BODY}" ] && error "Response body is empty!" && exit 1 -PY_CMD="-c 'from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))' " -PY3="/usr/bin/python3" -PY2="/usr/bin/python2" +PY_CMD='from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))' if [ "${PRETTY}" == true ] ; then - if [ -d "${PY3}" ]; then - echo "${TVMAZE_RESPONSE_BODY}" | "${PY3}" ${PY_CMD} - elif [ -d "${PY2}" ]; then - echo "${TVMAZE_RESPONSE_BODY}" | "${PY2}" ${PY_CMD} - fi + echo "${TVMAZE_RESPONSE_BODY}" | ( python3 -c "${PY_CMD}" 2>/dev/null || python2 -c "${PY_CMD}" 2>/dev/null ) || ( warn "Unable to pretty print, Python not detected" && echo "${TVMAZE_RESPONSE_BODY}" ) else echo "${TVMAZE_RESPONSE_BODY}" fi From 61b7241e7b6a6f8a22d2fa3fbaa62dfc2cd62f4a Mon Sep 17 00:00:00 2001 From: skyler-kv Date: Wed, 23 Oct 2024 16:49:07 +0000 Subject: [PATCH 4/5] replacing py3 & 2 w python3 & 2 --- contrib/datawave-quickstart/bin/query.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/datawave-quickstart/bin/query.sh b/contrib/datawave-quickstart/bin/query.sh index c5f89615de6..45e2605a8d2 100644 --- a/contrib/datawave-quickstart/bin/query.sh +++ b/contrib/datawave-quickstart/bin/query.sh @@ -131,8 +131,8 @@ function setQueryIdFromResponse() { } function prettyPrintJson() { - PY_CMD='from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True)' - echo "${1}" | ( "${PY3}" -c "${PY_CMD}" 2>/dev/null || "${PY2}" -c "${PY_CMD}" 2>/dev/null ) || ( warn "Python encountered error. Printed response without formatting" && printRawResponse "${1}" ) + PY_CMD='"from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True)"' + echo "${1}" | ( python3 -c "${PY_CMD}" 2>/dev/null || python2 -c "${PY_CMD}" 2>/dev/null ) || ( warn "Python encountered error. Printed response without formatting" && printRawResponse "${1}" ) } function printRawResponse() { From 607d9196aa1e14b0386b345cac0cbc243df83a6c Mon Sep 17 00:00:00 2001 From: skyler-kv Date: Thu, 24 Oct 2024 15:57:29 +0000 Subject: [PATCH 5/5] closed parenthesis --- contrib/datawave-quickstart/bin/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/datawave-quickstart/bin/query.sh b/contrib/datawave-quickstart/bin/query.sh index 45e2605a8d2..d49aa64b4f1 100644 --- a/contrib/datawave-quickstart/bin/query.sh +++ b/contrib/datawave-quickstart/bin/query.sh @@ -131,7 +131,7 @@ function setQueryIdFromResponse() { } function prettyPrintJson() { - PY_CMD='"from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True)"' + PY_CMD='from __future__ import print_function; import sys,json; data=json.loads(sys.stdin.read()); print(json.dumps(data, indent=2, sort_keys=True))' echo "${1}" | ( python3 -c "${PY_CMD}" 2>/dev/null || python2 -c "${PY_CMD}" 2>/dev/null ) || ( warn "Python encountered error. Printed response without formatting" && printRawResponse "${1}" ) }