diff --git a/.github/workflows/update-recall.yml b/.github/workflows/update-recall.yml index 934072c..e69de29 100644 --- a/.github/workflows/update-recall.yml +++ b/.github/workflows/update-recall.yml @@ -1,33 +0,0 @@ -name: Update Recall - -on: - workflow_dispatch: - inputs: - tags: - description: 'Release Tags' - -jobs: - update_recall: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - push: true - tags: beclab/r4recall:${{ github.event.inputs.tags }} - file: Dockerfile.recall - platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/update_r4rank.yaml b/.github/workflows/update_r4rank.yaml index 66eb1f2..efa0b82 100644 --- a/.github/workflows/update_r4rank.yaml +++ b/.github/workflows/update_r4rank.yaml @@ -1,81 +1,36 @@ name: Update Rank on: - push: - tags: - - '*' workflow_dispatch: inputs: tags: description: 'Release Tags' - required: true + push: + tags: + - '*' jobs: - build_and_push_amd64: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Build and push Docker image for amd64 - uses: docker/build-push-action@v3 - with: - push: true - tags: beclab/r4rank:${{ github.ref_name }}-amd64 - file: Dockerfile.r4rank.amd64 - platforms: linux/amd64 - - build_and_push_arm64: + update_rank: runs-on: ubuntu-latest steps: - name: Check out the repo uses: actions/checkout@v3 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Build and push Docker image for arm64 + + - name: Build and push Docker image uses: docker/build-push-action@v3 with: push: true - tags: beclab/r4rank:${{ github.ref_name }}-arm64 - file: Dockerfile.r4rank.arm64 - platforms: linux/arm64 - - create_and_push_manifest: - runs-on: ubuntu-latest - needs: [build_and_push_amd64, build_and_push_arm64] - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Create and push manifest - run: | - docker manifest create beclab/r4rank:${{ github.ref_name }} \ - beclab/r4rank:${{ github.ref_name }}-amd64 \ - beclab/r4rank:${{ github.ref_name }}-arm64 - docker manifest push beclab/r4rank:${{ github.ref_name }} + tags: beclab/r4rank:${{ github.event.inputs.tags }} + file: Dockerfile.r4rank + platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/update_r4train.yaml b/.github/workflows/update_r4train.yaml index 675e82c..435cf5e 100644 --- a/.github/workflows/update_r4train.yaml +++ b/.github/workflows/update_r4train.yaml @@ -1,81 +1,36 @@ name: Update Train on: - push: - tags: - - '*' workflow_dispatch: inputs: tags: description: 'Release Tags' - required: true + push: + tags: + - '*' jobs: - build_and_push_amd64: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Build and push Docker image for amd64 - uses: docker/build-push-action@v3 - with: - push: true - tags: beclab/r4train:${{ github.ref_name }}-amd64 - file: Dockerfile.r4train.amd64 - platforms: linux/amd64 - - build_and_push_arm64: + update_train: runs-on: ubuntu-latest steps: - name: Check out the repo uses: actions/checkout@v3 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Build and push Docker image for arm64 + + - name: Build and push Docker image uses: docker/build-push-action@v3 with: push: true - tags: beclab/r4train:${{ github.ref_name }}-arm64 - file: Dockerfile.r4train.arm64 - platforms: linux/arm64 - - create_and_push_manifest: - runs-on: ubuntu-latest - needs: [build_and_push_amd64, build_and_push_arm64] - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Create and push manifest - run: | - docker manifest create beclab/r4train:${{ github.ref_name }} \ - beclab/r4train:${{ github.ref_name }}-amd64 \ - beclab/r4train:${{ github.ref_name }}-arm64 - docker manifest push beclab/r4train:${{ github.ref_name }} + tags: beclab/r4train:${{ github.event.inputs.tags }} + file: Dockerfile.r4train + platforms: linux/amd64,linux/arm64 diff --git a/Dockerfile.r4rank b/Dockerfile.r4rank new file mode 100644 index 0000000..0cfd7b0 --- /dev/null +++ b/Dockerfile.r4rank @@ -0,0 +1,134 @@ +FROM ubuntu:jammy as builder +RUN mkdir -p /opt/rss-termius-v2-rank && \ + mkdir -p /opt/xgboost && \ + mkdir -p /opt/rank_model/ +RUN apt-get update && \ + apt-get install build-essential -y && \ + apt-get install cmake -y && \ + apt install git -y && \ + apt install curl -y && \ + apt install pkg-config -y && \ + apt-get install libssl-dev -y && \ + apt-get install libcpprest-dev -y && \ + cd /opt/xgboost && \ + git clone --recursive https://github.com/dmlc/xgboost . && \ + git checkout -b release_2.0.0 origin/release_2.0.0 && \ + git submodule init && \ + git submodule update && \ + mkdir build && \ + cd build && \ + cmake .. -DBUILD_STATIC_LIB=on && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm -r xgboost + + +RUN cd /opt && \ + curl -LJO https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz && \ + tar -xvzf googletest-1.14.0.tar.gz && \ + cd /opt/googletest-1.14.0 && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm -r googletest-1.14.0 && \ + rm googletest-1.14.0.tar.gz + + +RUN cd /opt && \ + curl -LJO https://github.com/mongodb/mongo-c-driver/releases/download/1.24.0/mongo-c-driver-1.24.0.tar.gz && \ + tar -xvzf mongo-c-driver-1.24.0.tar.gz && \ + cd mongo-c-driver-1.24.0/build && \ + cmake .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm mongo-c-driver-1.24.0.tar.gz && \ + rm -r mongo-c-driver-1.24.0 + +RUN cd /opt && \ + curl -LJO https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.8.1/mongo-cxx-driver-r3.8.1.tar.gz && \ + tar -xvzf mongo-cxx-driver-r3.8.1.tar.gz && \ + cd mongo-cxx-driver-r3.8.1/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm mongo-cxx-driver-r3.8.1.tar.gz && \ + rm -r mongo-cxx-driver-r3.8.1 + +RUN cd /opt && \ + curl -LO https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.tar.gz && \ + tar -xvzf v0.1.99.tar.gz && \ + cd sentencepiece-0.1.99 && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm -r sentencepiece-0.1.99 && \ + rm v0.1.99.tar.gz + +RUN cd /opt && \ + mkdir -p /usr/local/include/cppitertools && \ + curl -LO https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.1.tar.gz && \ + tar xzvf v2.1.tar.gz && \ + cd cppitertools-2.1/ && \ + cp *.hpp /usr/local/include/cppitertools && \ + cp -r internal /usr/local/include/cppitertools && \ + cd /opt && \ + rm v2.1.tar.gz && \ + rm -r cppitertools-2.1 + +ADD train-rank/src /opt/rss-terminus-v2-rank/src +ADD train-rank/test /opt/rss-terminus-v2-rank/test +COPY train-rank/CMakeLists.txt /opt/rss-terminus-v2-rank/CMakeLists.txt +RUN cd /opt/rss-terminus-v2-rank && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make && \ + cp bertv2rank /usr/local/bin && \ + cd /opt && \ + rm -r rss-termius-v2-rank + +RUN if [ -f /usr/lib/x86_64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + +RUN if [ -f /usr/lib/aarch64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + +FROM ubuntu:jammy +# FROM alpine + + +COPY --from=builder /usr/local/bin/bertv2rank /usr/local/bin/bertv2rank +COPY --from=builder /usr/local/lib/libmongocxx.so._noabi /usr/local/lib/libmongocxx.so._noabi +COPY --from=builder /usr/local/lib/libbsoncxx.so._noabi /usr/local/lib/libbsoncxx.so._noabi +COPY --from=builder /usr/local/lib/libmongoc-1.0.so.0 /usr/local/lib/libmongoc-1.0.so.0 +COPY --from=builder /usr/local/lib/libbson-1.0.so.0 /usr/local/lib/libbson-1.0.so.0 + +COPY --from=builder /usr/local/lib/libgomp.so.1 /usr/local/lib/libgomp.so.1 +COPY --from=builder /usr/local/lib/libcpprest.so.2.10 /usr/local/lib/libcpprest.so.2.10 +COPY --from=builder /usr/local/lib/libbrotlidec.so.1 /usr/local/lib/libbrotlidec.so.1 +COPY --from=builder /usr/local/lib/libbrotlienc.so.1 /usr/local/lib/libbrotlienc.so.1 +COPY --from=builder /usr/local/lib/libbrotlicommon.so.1 /usr/local/lib/libbrotlicommon.so.1 +COPY --from=builder /usr/local/lib/libicuuc.so.70 /usr/local/lib/libicuuc.so.70 +COPY --from=builder /usr/local/lib/libicudata.so.70 /usr/local/lib/libicudata.so.70 +RUN mkdir /opt/rank_model/ +CMD ["/usr/local/bin/bertv2rank"] +# ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/Dockerfile.r4train b/Dockerfile.r4train new file mode 100644 index 0000000..7920cdb --- /dev/null +++ b/Dockerfile.r4train @@ -0,0 +1,133 @@ +FROM ubuntu:jammy as builder +RUN mkdir -p /opt/rss-termius-v2-rank && \ + mkdir -p /opt/xgboost && \ + mkdir -p /opt/rank_model/ +RUN apt-get update && \ + apt-get install build-essential -y && \ + apt-get install cmake -y && \ + apt install git -y && \ + apt install curl -y && \ + apt install pkg-config -y && \ + apt-get install libssl-dev -y && \ + apt-get install libcpprest-dev -y && \ + cd /opt/xgboost && \ + git clone --recursive https://github.com/dmlc/xgboost . && \ + git checkout -b release_2.0.0 origin/release_2.0.0 && \ + git submodule init && \ + git submodule update && \ + mkdir build && \ + cd build && \ + cmake .. -DBUILD_STATIC_LIB=on && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm -r xgboost + + +RUN cd /opt && \ + curl -LJO https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz && \ + tar -xvzf googletest-1.14.0.tar.gz && \ + cd /opt/googletest-1.14.0 && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm -r googletest-1.14.0 && \ + rm googletest-1.14.0.tar.gz + + +RUN cd /opt && \ + curl -LJO https://github.com/mongodb/mongo-c-driver/releases/download/1.24.0/mongo-c-driver-1.24.0.tar.gz && \ + tar -xvzf mongo-c-driver-1.24.0.tar.gz && \ + cd mongo-c-driver-1.24.0/build && \ + cmake .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm mongo-c-driver-1.24.0.tar.gz && \ + rm -r mongo-c-driver-1.24.0 + +RUN cd /opt && \ + curl -LJO https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.8.1/mongo-cxx-driver-r3.8.1.tar.gz && \ + tar -xvzf mongo-cxx-driver-r3.8.1.tar.gz && \ + cd mongo-cxx-driver-r3.8.1/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm mongo-cxx-driver-r3.8.1.tar.gz && \ + rm -r mongo-cxx-driver-r3.8.1 + +RUN cd /opt && \ + curl -LO https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.tar.gz && \ + tar -xvzf v0.1.99.tar.gz && \ + cd sentencepiece-0.1.99 && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make -j$(nproc) && \ + make install && \ + cd /opt && \ + rm -r sentencepiece-0.1.99 && \ + rm v0.1.99.tar.gz + +RUN cd /opt && \ + mkdir -p /usr/local/include/cppitertools && \ + curl -LO https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.1.tar.gz && \ + tar xzvf v2.1.tar.gz && \ + cd cppitertools-2.1/ && \ + cp *.hpp /usr/local/include/cppitertools && \ + cp -r internal /usr/local/include/cppitertools && \ + cd /opt && \ + rm v2.1.tar.gz && \ + rm -r cppitertools-2.1 + +ADD train-rank/src /opt/rss-terminus-v2-rank/src +ADD train-rank/test /opt/rss-terminus-v2-rank/test +COPY train-rank/CMakeLists.txt /opt/rss-terminus-v2-rank/CMakeLists.txt +RUN cd /opt/rss-terminus-v2-rank && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make && \ + cp bertv2train /usr/local/bin && \ + cd /opt && \ + rm -r rss-termius-v2-rank + +RUN if [ -f /usr/lib/x86_64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + +RUN if [ -f /usr/lib/aarch64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + +FROM ubuntu:jammy +# FROM alpine + + +COPY --from=builder /usr/local/bin/bertv2train /usr/local/bin/bertv2train +COPY --from=builder /usr/local/lib/libmongocxx.so._noabi /usr/local/lib/libmongocxx.so._noabi +COPY --from=builder /usr/local/lib/libbsoncxx.so._noabi /usr/local/lib/libbsoncxx.so._noabi +COPY --from=builder /usr/local/lib/libmongoc-1.0.so.0 /usr/local/lib/libmongoc-1.0.so.0 +COPY --from=builder /usr/local/lib/libbson-1.0.so.0 /usr/local/lib/libbson-1.0.so.0 +COPY --from=builder /usr/local/lib/libgomp.so.1 /usr/local/lib/libgomp.so.1 +COPY --from=builder /usr/local/lib/libcpprest.so.2.10 /usr/local/lib/libcpprest.so.2.10 +COPY --from=builder /usr/local/lib/libbrotlidec.so.1 /usr/local/lib/libbrotlidec.so.1 +COPY --from=builder /usr/local/lib/libbrotlienc.so.1 /usr/local/lib/libbrotlienc.so.1 +COPY --from=builder /usr/local/lib/libbrotlicommon.so.1 /usr/local/lib/libbrotlicommon.so.1 +COPY --from=builder /usr/local/lib/libicuuc.so.70 /usr/local/lib/libicuuc.so.70 +COPY --from=builder /usr/local/lib/libicudata.so.70 /usr/local/lib/libicudata.so.70 +RUN mkdir /opt/rank_model/ +CMD ["/usr/local/bin/bertv2train"] +# ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/train-rank/infra/bertv2rank_deploy/Dockerfile b/train-rank/infra/bertv2rank_deploy/Dockerfile index 2f16c2a..0cfd7b0 100644 --- a/train-rank/infra/bertv2rank_deploy/Dockerfile +++ b/train-rank/infra/bertv2rank_deploy/Dockerfile @@ -1,15 +1,15 @@ -FROM ubuntu:jammy as builder +FROM ubuntu:jammy as builder RUN mkdir -p /opt/rss-termius-v2-rank && \ mkdir -p /opt/xgboost && \ mkdir -p /opt/rank_model/ RUN apt-get update && \ apt-get install build-essential -y && \ apt-get install cmake -y && \ - apt-get install libcpprest-dev -y && \ apt install git -y && \ apt install curl -y && \ apt install pkg-config -y && \ apt-get install libssl-dev -y && \ + apt-get install libcpprest-dev -y && \ cd /opt/xgboost && \ git clone --recursive https://github.com/dmlc/xgboost . && \ git checkout -b release_2.0.0 origin/release_2.0.0 && \ @@ -96,6 +96,22 @@ RUN cd /opt/rss-terminus-v2-rank && \ cd /opt && \ rm -r rss-termius-v2-rank +RUN if [ -f /usr/lib/x86_64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + +RUN if [ -f /usr/lib/aarch64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + FROM ubuntu:jammy # FROM alpine @@ -105,13 +121,14 @@ COPY --from=builder /usr/local/lib/libmongocxx.so._noabi /usr/local/lib/libmong COPY --from=builder /usr/local/lib/libbsoncxx.so._noabi /usr/local/lib/libbsoncxx.so._noabi COPY --from=builder /usr/local/lib/libmongoc-1.0.so.0 /usr/local/lib/libmongoc-1.0.so.0 COPY --from=builder /usr/local/lib/libbson-1.0.so.0 /usr/local/lib/libbson-1.0.so.0 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/lib/x86_64-linux-gnu/libgomp.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libicuuc.so.70 /usr/lib/x86_64-linux-gnu/libicuuc.so.70 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/lib/x86_64-linux-gnu/libicudata.so.70 +COPY --from=builder /usr/local/lib/libgomp.so.1 /usr/local/lib/libgomp.so.1 +COPY --from=builder /usr/local/lib/libcpprest.so.2.10 /usr/local/lib/libcpprest.so.2.10 +COPY --from=builder /usr/local/lib/libbrotlidec.so.1 /usr/local/lib/libbrotlidec.so.1 +COPY --from=builder /usr/local/lib/libbrotlienc.so.1 /usr/local/lib/libbrotlienc.so.1 +COPY --from=builder /usr/local/lib/libbrotlicommon.so.1 /usr/local/lib/libbrotlicommon.so.1 +COPY --from=builder /usr/local/lib/libicuuc.so.70 /usr/local/lib/libicuuc.so.70 +COPY --from=builder /usr/local/lib/libicudata.so.70 /usr/local/lib/libicudata.so.70 RUN mkdir /opt/rank_model/ CMD ["/usr/local/bin/bertv2rank"] +# ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/train-rank/infra/bertv2train_deploy/Dockerfile b/train-rank/infra/bertv2train_deploy/Dockerfile index baa22a8..7920cdb 100755 --- a/train-rank/infra/bertv2train_deploy/Dockerfile +++ b/train-rank/infra/bertv2train_deploy/Dockerfile @@ -2,7 +2,6 @@ FROM ubuntu:jammy as builder RUN mkdir -p /opt/rss-termius-v2-rank && \ mkdir -p /opt/xgboost && \ mkdir -p /opt/rank_model/ - RUN apt-get update && \ apt-get install build-essential -y && \ apt-get install cmake -y && \ @@ -97,6 +96,22 @@ RUN cd /opt/rss-terminus-v2-rank && \ cd /opt && \ rm -r rss-termius-v2-rank +RUN if [ -f /usr/lib/x86_64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/x86_64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + +RUN if [ -f /usr/lib/aarch64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \ + if [ -f /usr/lib/aarch64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi + FROM ubuntu:jammy # FROM alpine @@ -106,13 +121,13 @@ COPY --from=builder /usr/local/lib/libmongocxx.so._noabi /usr/local/lib/libmong COPY --from=builder /usr/local/lib/libbsoncxx.so._noabi /usr/local/lib/libbsoncxx.so._noabi COPY --from=builder /usr/local/lib/libmongoc-1.0.so.0 /usr/local/lib/libmongoc-1.0.so.0 COPY --from=builder /usr/local/lib/libbson-1.0.so.0 /usr/local/lib/libbson-1.0.so.0 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/lib/x86_64-linux-gnu/libgomp.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libicuuc.so.70 /usr/lib/x86_64-linux-gnu/libicuuc.so.70 -COPY --from=builder /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/lib/x86_64-linux-gnu/libicudata.so.70 +COPY --from=builder /usr/local/lib/libgomp.so.1 /usr/local/lib/libgomp.so.1 +COPY --from=builder /usr/local/lib/libcpprest.so.2.10 /usr/local/lib/libcpprest.so.2.10 +COPY --from=builder /usr/local/lib/libbrotlidec.so.1 /usr/local/lib/libbrotlidec.so.1 +COPY --from=builder /usr/local/lib/libbrotlienc.so.1 /usr/local/lib/libbrotlienc.so.1 +COPY --from=builder /usr/local/lib/libbrotlicommon.so.1 /usr/local/lib/libbrotlicommon.so.1 +COPY --from=builder /usr/local/lib/libicuuc.so.70 /usr/local/lib/libicuuc.so.70 +COPY --from=builder /usr/local/lib/libicudata.so.70 /usr/local/lib/libicudata.so.70 RUN mkdir /opt/rank_model/ CMD ["/usr/local/bin/bertv2train"] # ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file