From 692c066015a698d45eb11f18132cb1cb7809fd24 Mon Sep 17 00:00:00 2001
From: Malte Sander <malte.sander.it@gmail.com>
Date: Mon, 7 Apr 2025 10:45:37 +0200
Subject: [PATCH 1/5] consolidate docker image with fix image size PRs
---
airflow/Dockerfile | 99 ++++++++++++++++++++++++++--------------------
1 file changed, 56 insertions(+), 43 deletions(-)
diff --git a/airflow/Dockerfile b/airflow/Dockerfile
index b0723fd73..41deca85a 100644
--- a/airflow/Dockerfile
+++ b/airflow/Dockerfile
@@ -28,6 +28,7 @@ ARG PRODUCT
ARG STATSD_EXPORTER
ARG PYTHON
ARG TARGETARCH
+ARG STACKABLE_USER_UID
COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt
COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/
@@ -38,24 +39,24 @@ ENV AIRFLOW_EXTRAS=async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch
RUN microdnf update && \
microdnf install \
- cyrus-sasl-devel \
- # Needed by ./configure to build gevent, see snippet [1] at the end of file
- diffutils \
- # Needed to build gevent, see snippet [1] at the end of file
- make \
- gcc \
- gcc-c++ \
- libpq-devel \
- openldap-devel \
- openssl-devel \
- python${PYTHON} \
- python${PYTHON}-devel \
- python${PYTHON}-pip \
- python${PYTHON}-wheel \
- # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
- unixODBC \
- # Needed to modify the SBOM
- jq && \
+ cyrus-sasl-devel \
+ # Needed by ./configure to build gevent, see snippet [1] at the end of file
+ diffutils \
+ # Needed to build gevent, see snippet [1] at the end of file
+ make \
+ gcc \
+ gcc-c++ \
+ libpq-devel \
+ openldap-devel \
+ openssl-devel \
+ python${PYTHON} \
+ python${PYTHON}-devel \
+ python${PYTHON}-pip \
+ python${PYTHON}-wheel \
+ # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
+ unixODBC \
+ # Needed to modify the SBOM
+ jq && \
microdnf clean all && \
rm -rf /var/cache/yum
@@ -85,9 +86,17 @@ else
end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
EOF
-WORKDIR /stackable
COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
+COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
+
+RUN <<EOF
+mkdir -pv /stackable/airflow
+mkdir -pv /stackable/airflow/dags
+mkdir -pv /stackable/airflow/logs
+chmod --recursive g=u /stackable
+EOF
+
FROM stackable/image/vector AS airflow-main-image
@@ -99,22 +108,26 @@ ARG TARGETARCH
ARG STACKABLE_USER_UID
LABEL name="Apache Airflow" \
- maintainer="info@stackable.tech" \
- vendor="Stackable GmbH" \
- version="${PRODUCT}" \
- release="${RELEASE}" \
- summary="The Stackable image for Apache Airflow." \
- description="This image is deployed by the Stackable Operator for Apache Airflow."
-
-COPY airflow/licenses /licenses
-COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
-COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
+ maintainer="info@stackable.tech" \
+ vendor="Stackable GmbH" \
+ version="${PRODUCT}" \
+ release="${RELEASE}" \
+ summary="The Stackable image for Apache Airflow." \
+ description="This image is deployed by the Stackable Operator for Apache Airflow."
ENV HOME=/stackable
ENV AIRFLOW_USER_HOME_DIR=/stackable
ENV PATH=$PATH:/bin:$HOME/app/bin
ENV AIRFLOW_HOME=$HOME/airflow
+COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
+COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/git-sync ${HOME}/git-sync
+
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
+
+COPY airflow/licenses /licenses
+
# Update image and install needed packages
RUN <<EOF
microdnf update
@@ -142,33 +155,33 @@ rm -rf /var/cache/yum
# Get the correct `tini` binary for our architecture.
# It is used as an init alternative in the entrypoint
curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
+
+# fix missing permissions
chmod a+x /entrypoint.sh
chmod a+x /run-airflow.sh
chmod +x /usr/bin/tini
+EOF
-mkdir -pv ${AIRFLOW_HOME}
-mkdir -pv ${AIRFLOW_HOME}/dags
-mkdir -pv ${AIRFLOW_HOME}/logs
+# ----------------------------------------
+# Checks
+# This section is to run final checks to ensure the created final images
+# adhere to several minimal requirements like:
+# - check file permissions and ownerships
+# ----------------------------------------
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
-chown -R ${STACKABLE_USER_UID}:0 /stackable
-chmod -R g=u /stackable
+# Check that permissions and ownership in /stackable are set correctly
+# This will fail and stop the build if any mismatches are found.
+RUN <<EOF
+/bin/check-permissions-ownership.sh ${HOME} ${STACKABLE_USER_UID} 0
EOF
# ----------------------------------------
-# Attention: We are changing the group of all files in /stackable directly above
-# If you do any file based actions (copying / creating etc.) below this comment you
-# absolutely need to make sure that the correct permissions are applied!
-# chown ${STACKABLE_USER_UID}:0
+# Attention: Do not perform any file based actions (copying/creating etc.) below this comment because the permissions would not be checked.
# ----------------------------------------
USER ${STACKABLE_USER_UID}
WORKDIR /stackable
-COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
-COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
-
ENTRYPOINT ["/usr/bin/tini", "--", "/run-airflow.sh"]
CMD []
From b1fcb9d9525c7ed92d8d8ccfdeb041bd7018ec2b Mon Sep 17 00:00:00 2001
From: Malte Sander <malte.sander.it@gmail.com>
Date: Mon, 7 Apr 2025 10:47:05 +0200
Subject: [PATCH 2/5] adapted changelog
---
CHANGELOG.md | 3 +++
1 file changed, 3 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6a75fa8f5..c2b071f6b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file.
### Added
+- airflow: check for correct permissions and ownerships in /stackable folder via
+ `check-permissions-ownership.sh` provided in stackable-base image ([#1054]).
- spark-connect-client: A new image for Spark connect tests and demos ([#1034])
- nifi: check for correct permissions and ownerships in /stackable folder via
`check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
@@ -26,6 +28,7 @@ All notable changes to this project will be documented in this file.
[#1042]: https://github.com/stackabletech/docker-images/pull/1042
[#1044]: https://github.com/stackabletech/docker-images/pull/1044
[#1050]: https://github.com/stackabletech/docker-images/pull/1050
+[#1054]: https://github.com/stackabletech/docker-images/pull/1054
## [25.3.0] - 2025-03-21
From 9c8db9d54839cc2093a51f7b0386656477bd6048 Mon Sep 17 00:00:00 2001
From: Malte Sander <malte.sander.it@gmail.com>
Date: Mon, 7 Apr 2025 10:48:05 +0200
Subject: [PATCH 3/5] fix comment
---
airflow/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airflow/Dockerfile b/airflow/Dockerfile
index 41deca85a..84c67067e 100644
--- a/airflow/Dockerfile
+++ b/airflow/Dockerfile
@@ -169,7 +169,7 @@ EOF
# - check file permissions and ownerships
# ----------------------------------------
-# Check that permissions and ownership in /stackable are set correctly
+# Check that permissions and ownership in ${HOME} are set correctly
# This will fail and stop the build if any mismatches are found.
RUN <<EOF
/bin/check-permissions-ownership.sh ${HOME} ${STACKABLE_USER_UID} 0
From 970d43e5d8957437201051157cefaac26852843a Mon Sep 17 00:00:00 2001
From: Malte Sander <malte.sander.it@gmail.com>
Date: Mon, 7 Apr 2025 15:08:29 +0200
Subject: [PATCH 4/5] fix intendation
---
airflow/Dockerfile | 36 ++++++++++++++++++------------------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/airflow/Dockerfile b/airflow/Dockerfile
index 84c67067e..951f07ea9 100644
--- a/airflow/Dockerfile
+++ b/airflow/Dockerfile
@@ -39,24 +39,24 @@ ENV AIRFLOW_EXTRAS=async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch
RUN microdnf update && \
microdnf install \
- cyrus-sasl-devel \
- # Needed by ./configure to build gevent, see snippet [1] at the end of file
- diffutils \
- # Needed to build gevent, see snippet [1] at the end of file
- make \
- gcc \
- gcc-c++ \
- libpq-devel \
- openldap-devel \
- openssl-devel \
- python${PYTHON} \
- python${PYTHON}-devel \
- python${PYTHON}-pip \
- python${PYTHON}-wheel \
- # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
- unixODBC \
- # Needed to modify the SBOM
- jq && \
+ cyrus-sasl-devel \
+ # Needed by ./configure to build gevent, see snippet [1] at the end of file
+ diffutils \
+ # Needed to build gevent, see snippet [1] at the end of file
+ make \
+ gcc \
+ gcc-c++ \
+ libpq-devel \
+ openldap-devel \
+ openssl-devel \
+ python${PYTHON} \
+ python${PYTHON}-devel \
+ python${PYTHON}-pip \
+ python${PYTHON}-wheel \
+ # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
+ unixODBC \
+ # Needed to modify the SBOM
+ jq && \
microdnf clean all && \
rm -rf /var/cache/yum
From b5c03340fc661d68d15f7abb1ee6c1a31a0e3bcc Mon Sep 17 00:00:00 2001
From: Malte Sander <malte.sander.it@gmail.com>
Date: Wed, 9 Apr 2025 08:47:36 +0200
Subject: [PATCH 5/5] improve check permissions script performance (@siggi)
---
shared/checks/check-permissions-ownership.sh | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/shared/checks/check-permissions-ownership.sh b/shared/checks/check-permissions-ownership.sh
index 7b34009f7..f5cff9e71 100755
--- a/shared/checks/check-permissions-ownership.sh
+++ b/shared/checks/check-permissions-ownership.sh
@@ -29,27 +29,29 @@ EXPECTED_GID=$3
error_flag=0
# Check ownership
-while IFS= read -r -d '' file; do
- uid=$(stat -c "%u" "$file")
- gid=$(stat -c "%g" "$file")
+while IFS= read -r -d '' entry; do
+ uid=${entry%% *}
+ remainder=${entry#* }
+ gid=${remainder%% *}
if [[ "$uid" -ne "$EXPECTED_UID" || "$gid" -ne "$EXPECTED_GID" ]]; then
+ file=${remainder#* }
echo "Ownership mismatch: $file (Expected: $EXPECTED_UID:$EXPECTED_GID, Found: $uid:$gid)"
error_flag=1
fi
-done < <(find "$DIRECTORY" -print0)
+done < <(find "$DIRECTORY" -printf "%U %G %p\0")
# Check permissions
-while IFS= read -r -d '' file; do
- perms=$(stat -c "%A" "$file")
- owner_perms="${perms:1:3}"
- group_perms="${perms:4:3}"
+while IFS= read -r -d '' entry; do
+ owner_perms="${entry:1:3}"
+ group_perms="${entry:4:3}"
if [[ "$owner_perms" != "$group_perms" ]]; then
+ file="${entry:11}"
echo "Permission mismatch: $file (Owner: $owner_perms, Group: $group_perms)"
error_flag=1
fi
-done < <(find "$DIRECTORY" -print0)
+done < <(find "$DIRECTORY" -printf "%M %p\0")
if [[ $error_flag -ne 0 ]]; then
echo "Permission and Ownership checks failed for $DIRECTORY!"