Skip to content

Commit 01cd442

Browse files
tchughesivtchughesiv
authored
chore: Add rpms & milvus-lite / ibis binaries to hermeto fetcher for offline builds (#5333)
offline install: use generic fetcher for milvus-lite wheel Signed-off-by: Tommy Hughes <tohughes@redhat.com>
1 parent 18cbd7f commit 01cd442

10 files changed

+527
-32
lines changed

‎sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,12 @@ FROM registry.access.redhat.com/ubi9/python-311:1
22
ARG RELEASE
33

44
ENV IBIS_VERSION="9.5.0"
5-
ENV MILVUS_LITE_VERSION="2.4.12"
65

76
USER 0
87
RUN yum install -y ninja-build llvm-devel cmake llvm-toolset ncurses-devel rust cargo
98
RUN if [[ -z "$RELEASE" ]] ; then npm install -g yarn yalc && rm -rf .npm ; fi
109
USER 1001
1110

12-
# milvus dep
13-
RUN pip install milvus-lite==${MILVUS_LITE_VERSION}
14-
1511
# ibis-framework must build from a clone of its git repo... versioning requirement
1612
## RuntimeError: Unable to detect version control system. Checked: Git. Not installed: Mercurial, Darcs, Subversion, Bazaar, Fossil, Pijul.
1713
RUN git clone -b ${IBIS_VERSION} --single-branch https://github.com/ibis-project/ibis ${APP_ROOT}/src/ibis

‎sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,25 @@
1-
FROM yum-builder:dev
1+
FROM yarn-builder:latest
22

33
ENV APACHE_ARROW_VERSION="17.0.0"
4+
ENV MILVUS_LITE_VERSION="2.4.12"
5+
ENV IBIS_VERSION="9.5.0"
46

57
ENV ARROW_HOME=${APP_ROOT}/arrow-dist
68
ENV LD_LIBRARY_PATH=${ARROW_HOME}/lib:$LD_LIBRARY_PATH
79
ENV CMAKE_PREFIX_PATH=${ARROW_HOME}:$CMAKE_PREFIX_PATH
810
ENV THIRD_PARTY_PATH=/tmp/hermeto-generic-output/deps/generic
911

12+
######
13+
# unecessary for konflux build
14+
ENV RPM_PATH=/tmp/hermeto-rpm-output/deps/rpm/x86_64
15+
USER 0
16+
RUN useradd mockbuild
17+
RUN groupadd mock
18+
RUN usermod -G mock mockbuild
19+
RUN rpm -Uvh ${RPM_PATH}/*/*.rpm
20+
USER 1001
21+
######
22+
1023
# configured for Arrow version 17.0.0
1124
RUN mkdir ${APP_ROOT}/src/arrow ${ARROW_HOME} ${APP_ROOT}/src/arrow-build && \
1225
tar xfz ${THIRD_PARTY_PATH}/apache-arrow-${APACHE_ARROW_VERSION}.tar.gz -C ${APP_ROOT}/src/arrow --strip-components=1 && \
@@ -97,6 +110,8 @@ RUN mkdir ${APP_ROOT}/src/arrow ${ARROW_HOME} ${APP_ROOT}/src/arrow-build && \
97110
cd ${APP_ROOT}/src/arrow/python && \
98111
PYARROW_PARALLEL=4 python setup.py build_ext --bundle-arrow-cpp bdist_wheel && \
99112
pip install dist/pyarrow-*.whl && \
113+
pip install ${THIRD_PARTY_PATH}/milvus_lite-${MILVUS_LITE_VERSION}-py3-none-manylinux2014_x86_64.whl && \
114+
pip install ${THIRD_PARTY_PATH}/ibis_framework-${IBIS_VERSION}-py3-none-any.whl[duckdb] && \
100115
\
101116
cd ${APP_ROOT}/src && \
102117
rm -rf ${APP_ROOT}/src/arrow-build ${APP_ROOT}/src/arrow ${ARROW_HOME}
@@ -113,12 +128,6 @@ RUN python -c "import pyarrow.substrait as substrait; print(substrait.__name__)"
113128
RUN source /tmp/hermeto.env && \
114129
pip install "numpy>=1.16.6,<2"
115130

116-
RUN cd ${APP_ROOT}/src/ibis && \
117-
source /tmp/hermeto.env && \
118-
pip install .[duckdb] && \
119-
cd ${APP_ROOT}/src && \
120-
rm -rf ${APP_ROOT}/src/ibis
121-
122131
# This section only necessary when building from local feast source ... e.g. ".[minimal]"
123132
########################
124133
COPY --chown=default .git ${APP_ROOT}/src/.git

‎sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,25 @@
1-
FROM yum-builder:release
1+
FROM registry.access.redhat.com/ubi9/python-311:1
22

33
ENV APACHE_ARROW_VERSION="17.0.0"
4+
ENV MILVUS_LITE_VERSION="2.4.12"
5+
ENV IBIS_VERSION="9.5.0"
46

57
ENV ARROW_HOME=${APP_ROOT}/arrow-dist
68
ENV LD_LIBRARY_PATH=${ARROW_HOME}/lib:$LD_LIBRARY_PATH
79
ENV CMAKE_PREFIX_PATH=${ARROW_HOME}:$CMAKE_PREFIX_PATH
810
ENV THIRD_PARTY_PATH=/tmp/hermeto-generic-output/deps/generic
911

12+
######
13+
# unecessary for konflux build
14+
ENV RPM_PATH=/tmp/hermeto-rpm-output/deps/rpm/x86_64
15+
USER 0
16+
RUN useradd mockbuild
17+
RUN groupadd mock
18+
RUN usermod -G mock mockbuild
19+
RUN rpm -Uvh ${RPM_PATH}/*/*.rpm
20+
USER 1001
21+
######
22+
1023
# configured for Arrow version 17.0.0
1124
RUN mkdir ${APP_ROOT}/src/arrow ${ARROW_HOME} ${APP_ROOT}/src/arrow-build && \
1225
tar xfz ${THIRD_PARTY_PATH}/apache-arrow-${APACHE_ARROW_VERSION}.tar.gz -C ${APP_ROOT}/src/arrow --strip-components=1 && \
@@ -97,6 +110,8 @@ RUN mkdir ${APP_ROOT}/src/arrow ${ARROW_HOME} ${APP_ROOT}/src/arrow-build && \
97110
cd ${APP_ROOT}/src/arrow/python && \
98111
PYARROW_PARALLEL=4 python setup.py build_ext --bundle-arrow-cpp bdist_wheel && \
99112
pip install dist/pyarrow-*.whl && \
113+
pip install ${THIRD_PARTY_PATH}/milvus_lite-${MILVUS_LITE_VERSION}-py3-none-manylinux2014_x86_64.whl && \
114+
pip install ${THIRD_PARTY_PATH}/ibis_framework-${IBIS_VERSION}-py3-none-any.whl[duckdb] && \
100115
\
101116
cd ${APP_ROOT}/src && \
102117
rm -rf ${APP_ROOT}/src/arrow-build ${APP_ROOT}/src/arrow ${ARROW_HOME}
@@ -113,12 +128,6 @@ RUN python -c "import pyarrow.substrait as substrait; print(substrait.__name__)"
113128
RUN source /tmp/hermeto.env && \
114129
pip install "numpy>=1.16.6,<2"
115130

116-
RUN cd ${APP_ROOT}/src/ibis && \
117-
source /tmp/hermeto.env && \
118-
pip install .[duckdb] && \
119-
cd ${APP_ROOT}/src && \
120-
rm -rf ${APP_ROOT}/src/ibis
121-
122131
COPY requirements.txt requirements.txt
123132
RUN source /tmp/hermeto.env && \
124133
pip install -r requirements.txt

‎sdk/python/feast/infra/feature_servers/multicloud/offline/artifacts.lock.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
metadata:
33
version: "1.0"
44
artifacts:
5+
- download_url: "https://files.pythonhosted.org/packages/44/43/b3f6e9defd1f3927b972beac7abe3d5b4a3bdb287e3bad69618e2e76cf0a/milvus_lite-2.4.12-py3-none-manylinux2014_x86_64.whl"
6+
checksum: "sha256:334037ebbab60243b5d8b43d54ca2f835d81d48c3cda0c6a462605e588deb05d"
7+
filename: "milvus_lite-2.4.12-py3-none-manylinux2014_x86_64.whl"
8+
- download_url: "https://files.pythonhosted.org/packages/dd/a9/899888a3b49ee07856a0bab673652a82ea89999451a51fba4d99e65868f7/ibis_framework-9.5.0-py3-none-any.whl"
9+
checksum: "sha256:145fe30d94f111cff332580c275ce77725c5ff7086eede93af0b371649d009c0"
10+
filename: "ibis_framework-9.5.0-py3-none-any.whl"
511
- download_url: "https://github.com/apache/arrow/archive/apache-arrow-17.0.0.tar.gz"
612
checksum: "sha256:8379554d89f19f2c8db63620721cabade62541f47a4e706dfb0a401f05a713ef"
713
filename: "apache-arrow-17.0.0.tar.gz"

‎sdk/python/feast/infra/feature_servers/multicloud/offline/hermeto-generic-artifacts.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,12 @@ APACHE_ARCHIVE_SHA256_CHECKSUM="8379554d89f19f2c8db63620721cabade62541f47a4e706d
1111
SUBSTRAIT_VERSION="0.44.0"
1212
SUBSTRAIT_ARCHIVE_SHA256_CHECKSUM="f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2"
1313

14+
MILVUS_LITE_VERSION="2.4.12"
15+
MILVUS_LITE_ARCHIVE_SHA256_CHECKSUM="334037ebbab60243b5d8b43d54ca2f835d81d48c3cda0c6a462605e588deb05d"
16+
17+
IBIS_VERSION="9.5.0"
18+
IBIS_ARCHIVE_SHA256_CHECKSUM="145fe30d94f111cff332580c275ce77725c5ff7086eede93af0b371649d009c0"
19+
1420
PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
1521
DESTDIR=${PROJECT_ROOT_DIR}/sdk/python/feast/infra/feature_servers/multicloud/offline
1622
artifacts_file=${DESTDIR}/artifacts.lock.yaml
@@ -45,6 +51,8 @@ metadata:
4551
version: "1.0"
4652
artifacts:' > ${artifacts_file}
4753

54+
set_dependency "https://files.pythonhosted.org/packages/44/43/b3f6e9defd1f3927b972beac7abe3d5b4a3bdb287e3bad69618e2e76cf0a/milvus_lite-${MILVUS_LITE_VERSION}-py3-none-manylinux2014_x86_64.whl" "${MILVUS_LITE_ARCHIVE_SHA256_CHECKSUM}" "milvus_lite-${MILVUS_LITE_VERSION}-py3-none-manylinux2014_x86_64.whl"
55+
set_dependency "https://files.pythonhosted.org/packages/dd/a9/899888a3b49ee07856a0bab673652a82ea89999451a51fba4d99e65868f7/ibis_framework-${IBIS_VERSION}-py3-none-any.whl" "${IBIS_ARCHIVE_SHA256_CHECKSUM}" "ibis_framework-${IBIS_VERSION}-py3-none-any.whl"
4856
set_dependency "https://github.com/apache/arrow/archive/${ARROW_ARCHIVE}" "${APACHE_ARCHIVE_SHA256_CHECKSUM}" "${ARROW_ARCHIVE}"
4957

5058
source ${versions_file}

‎sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ cd ${PROJECT_ROOT_DIR}
99
rm -rf ./offline_build
1010
mkdir offline_build
1111

12-
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
12+
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.25.0'
1313
# not needed for downstream build from release
1414
###############################
1515
# yarn builder

‎sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-release-build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ cd ${PROJECT_ROOT_DIR}
99
rm -rf ./offline_build
1010
mkdir offline_build
1111

12-
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
12+
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.25.0'
1313
hermeto fetch-deps \
1414
--output ${OFFLINE_BUILD_DIR}/hermeto-output \
1515
'{

‎sdk/python/feast/infra/feature_servers/multicloud/offline/offline-build.sh

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,19 @@ cd ${PROJECT_ROOT_DIR}
99
rm -rf ./offline_build
1010
mkdir offline_build
1111

12-
# yum builder
12+
# yarn builder
1313
docker build \
14-
--tag yum-builder:dev \
15-
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum \
14+
--tag yarn-builder \
15+
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yarn \
1616
sdk/python/feast/infra/feature_servers/multicloud/offline
1717

18-
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
18+
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.25.0'
19+
hermeto fetch-deps \
20+
--source sdk/python/feast/infra/feature_servers/multicloud/offline \
21+
--output ${OFFLINE_BUILD_DIR}/hermeto-rpm-output \
22+
--dev-package-managers \
23+
rpm
24+
1925
hermeto fetch-deps \
2026
--source sdk/python/feast/infra/feature_servers/multicloud/offline \
2127
--output ${OFFLINE_BUILD_DIR}/hermeto-generic-output \
@@ -67,6 +73,7 @@ docker build \
6773
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
6874
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
6975
--volume ${OFFLINE_BUILD_DIR}/hermeto-generic-output:/tmp/hermeto-generic-output:Z \
76+
--volume ${OFFLINE_BUILD_DIR}/hermeto-rpm-output:/tmp/hermeto-rpm-output:Z \
7077
--tag feature-server:sdist-build \
7178
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist \
7279
${PROJECT_ROOT_DIR}

‎sdk/python/feast/infra/feature_servers/multicloud/offline/offline-release-build.sh

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,18 @@ cd ${PROJECT_ROOT_DIR}
99
rm -rf ./offline_build
1010
mkdir offline_build
1111

12-
# yum builder
13-
docker build \
14-
--build-arg RELEASE=true \
15-
--tag yum-builder:release \
16-
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum \
17-
sdk/python/feast/infra/feature_servers/multicloud/offline
12+
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.25.0'
13+
hermeto fetch-deps \
14+
--source sdk/python/feast/infra/feature_servers/multicloud/offline \
15+
--output ${OFFLINE_BUILD_DIR}/hermeto-rpm-output \
16+
--dev-package-managers \
17+
rpm
1818

19-
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
2019
hermeto fetch-deps \
2120
--source sdk/python/feast/infra/feature_servers/multicloud/offline \
2221
--output ${OFFLINE_BUILD_DIR}/hermeto-generic-output \
2322
generic
2423

25-
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
2624
hermeto fetch-deps \
2725
--output ${OFFLINE_BUILD_DIR}/hermeto-output \
2826
'{
@@ -46,6 +44,7 @@ docker build \
4644
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
4745
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
4846
--volume ${OFFLINE_BUILD_DIR}/hermeto-generic-output:/tmp/hermeto-generic-output:Z \
47+
--volume ${OFFLINE_BUILD_DIR}/hermeto-rpm-output:/tmp/hermeto-rpm-output:Z \
4948
--tag feature-server:sdist-release-build \
5049
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release \
5150
sdk/python/feast/infra/feature_servers/multicloud

0 commit comments

Comments
 (0)