diff --git a/.scripts/upload_new_boost_version.sh b/.scripts/upload_new_boost_version.sh new file mode 100755 index 000000000..5e43d9e15 --- /dev/null +++ b/.scripts/upload_new_boost_version.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# This script mirrors the boostorg/boost source bundle for the given version to Nexus. +# The boost source bundle is architecture independent. +# It contains its own build system (b2) which is also built from source before building boost itself, so we don't need to worry about architecture specific builds. +# This artifact is used by the hadoop/boost local image. + + +set -euo pipefail + +VERSION=${1:?"Missing version number argument (arg 1)"} +NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} + +read -r -s -p "Nexus Password: " NEXUS_PASSWORD +echo "" + +# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory +# Find the directory name of the script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# the temp directory used, within $DIR +WORK_DIR=$(mktemp -d -p "$DIR") + +# check if tmp dir was created +if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then + echo "Could not create temp dir" + exit 1 +fi + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +cd "$WORK_DIR" || exit + +# boost does not currently publish signatures or SBOMs +BOOST_UNDERSCORE="$(echo "${VERSION}" | tr '.' '_')" +BOOST_TARBALL="boost_${BOOST_UNDERSCORE}.tar.bz2" +DOWNLOAD_URL="https://archives.boost.io/release/$VERSION/source/$BOOST_TARBALL" + +echo "Downloading boost" +if ! curl --fail -Ls -O "$DOWNLOAD_URL"; then + echo "Failed to download from $DOWNLOAD_URL" + exit 1 +fi + +FILE_NAME=$(basename "$DOWNLOAD_URL") + +echo "Uploading boost to Nexus" +if ! curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$FILE_NAME" 'https://repo.stackable.tech/repository/packages/boost/'; then + echo "Failed to upload boost to Nexus" + exit 1 +fi + +echo "Successfully uploaded new version of boost ($VERSION) to Nexus" +echo "https://repo.stackable.tech/service/rest/repository/browse/packages/boost/" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa86a1e2..c87288d73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ All notable changes to this project will be documented in this file. ### Changed - hbase: Update `hbase-opa-authorizer` from `0.1.0` to `0.2.0` and then `0.3.0` ([#1446], [#1454]). +- hadoop: Build the boost library from source instead of installing the EPEL package ([#1475]). + This is a requirement for the adoption of UBI 10 as base image because the package is not available there. ### Fixed @@ -25,6 +27,7 @@ All notable changes to this project will be documented in this file. [#1463]: https://github.com/stackabletech/docker-images/pull/1463 [#1466]: https://github.com/stackabletech/docker-images/pull/1466 [#1474]: https://github.com/stackabletech/docker-images/pull/1474 +[#1475]: https://github.com/stackabletech/docker-images/pull/1475 ## [26.3.0] - 2026-03-16 diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 2757b4e8c..f5ebf2b14 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -90,6 +90,7 @@ LABEL \ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/*-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/boost /stackable/boost COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS_VERSION}.jar /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS_VERSION}.jar COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS_VERSION}-src.tar.gz /stackable @@ -105,9 +106,11 @@ microdnf update # tar is required for `kubectl cp` which can be used to copy the log files # or profiler flamegraph from the Pod # It is already installed in the base image but leaving here for documentation purposes +# libstdc++ is a runtime dependency for boost microdnf install \ fuse \ fuse-libs \ + libstdc++ \ tar microdnf clean all rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt @@ -132,8 +135,13 @@ chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar # Set correct permissions and ownerships -chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" -chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" +chown --recursive ${STACKABLE_USER_UID}:0 \ +/stackable/hadoop /stackable/jmx /stackable/async-profiler \ +"/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" \ +/stackable/boost +chmod --recursive g=u /stackable/jmx /stackable/async-profiler \ +"/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" \ +/stackable/boost # Workaround for https://issues.apache.org/jira/browse/HADOOP-12845 # The problem is that our stackable-devel image does contain the openssl-devel package @@ -162,7 +170,7 @@ EOF USER ${STACKABLE_USER_UID} ENV HOME=/stackable -ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server +ENV LD_LIBRARY_PATH=/stackable/boost/lib:/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server ENV PATH="${PATH}":/stackable/hadoop/bin ENV HADOOP_HOME=/stackable/hadoop ENV HADOOP_CONF_DIR=/stackable/config @@ -174,6 +182,8 @@ ENV ASYNC_PROFILER_HOME=/stackable/async-profiler # if HADOOP_YARN_HOME does not exist at all, so we set it here to a sensible default. ENV HADOOP_YARN_HOME=/stackable/hadoop ENV HADOOP_MAPRED_HOME=/stackable/hadoop +ENV BOOST_ROOT=/stackable/boost +ENV CPATH=/stackable/boost/include WORKDIR /stackable/hadoop CMD ["echo", "This image is not meant to be 'run' directly."] diff --git a/hadoop/boost/Dockerfile b/hadoop/boost/Dockerfile new file mode 100644 index 000000000..69ff112b6 --- /dev/null +++ b/hadoop/boost/Dockerfile @@ -0,0 +1,40 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# check=error=true + +# The boost library is a runtime dependency of the Hadoop native code +# but is not available in the RedHat UBI10 or EPEL10 repositories, so we need to build +# it ourselves. +# +# NOTE: We use a published source bundle instead of the patchable workflow +# because boost uses git submodules for its build system +# and patchable doesn't support these. +# The source bundle contains everything needed to build b2 (the build system) +# and boost except for dependencies which are automatically discovered. +# +FROM local-image/stackable-base AS boost-builder + +ARG BOOST_VERSION + +WORKDIR /tmp + +RUN <