Merge pull request #18 from beclab/feat/rank-arm-build

fix:through change .so file path for build success
beclab · Jun 20, 2024 · 4f88e49 · 4f88e49
2 parents 824957e + ef9024a
commit 4f88e49
Show file tree

Hide file tree

Showing 7 changed files with 338 additions and 162 deletions.
diff --git a/.github/workflows/update-recall.yml b/.github/workflows/update-recall.yml
@@ -1,33 +0,0 @@
-name: Update Recall
-
-on:
-  workflow_dispatch:
-    inputs:
-      tags:
-        description: 'Release Tags'
-
-jobs:
-  update_recall:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Build and push Docker image 
-        uses: docker/build-push-action@v3
-        with:
-            push: true
-            tags: beclab/r4recall:${{ github.event.inputs.tags }}
-            file: Dockerfile.recall
-            platforms: linux/amd64,linux/arm64

diff --git a/.github/workflows/update_r4rank.yaml b/.github/workflows/update_r4rank.yaml
@@ -1,81 +1,36 @@
 name: Update Rank
 
 on:
-  push:
-    tags:
-      - '*'
   workflow_dispatch:
     inputs:
       tags:
         description: 'Release Tags'
-        required: true
+  push:
+    tags:
+      - '*'
 
 jobs:
-  build_and_push_amd64:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Build and push Docker image for amd64
-        uses: docker/build-push-action@v3
-        with:
-            push: true
-            tags: beclab/r4rank:${{ github.ref_name }}-amd64
-            file: Dockerfile.r4rank.amd64
-            platforms: linux/amd64
-
-  build_and_push_arm64:
+  update_rank:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
-
+      
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-
+        
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Build and push Docker image for arm64
+      
+      - name: Build and push Docker image 
         uses: docker/build-push-action@v3
         with:
             push: true
-            tags: beclab/r4rank:${{ github.ref_name }}-arm64
-            file: Dockerfile.r4rank.arm64
-            platforms: linux/arm64
-
-  create_and_push_manifest:
-    runs-on: ubuntu-latest
-    needs: [build_and_push_amd64, build_and_push_arm64]
-    steps:
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Create and push manifest
-        run: |
-          docker manifest create beclab/r4rank:${{ github.ref_name }} \
-            beclab/r4rank:${{ github.ref_name }}-amd64 \
-            beclab/r4rank:${{ github.ref_name }}-arm64
-          docker manifest push beclab/r4rank:${{ github.ref_name }}
+            tags: beclab/r4rank:${{ github.event.inputs.tags }}
+            file: Dockerfile.r4rank
+            platforms: linux/amd64,linux/arm64
diff --git a/.github/workflows/update_r4train.yaml b/.github/workflows/update_r4train.yaml
@@ -1,81 +1,36 @@
 name: Update Train
 
 on:
-  push:
-    tags:
-      - '*'
   workflow_dispatch:
     inputs:
       tags:
         description: 'Release Tags'
-        required: true
+  push:
+    tags:
+      - '*'
 
 jobs:
-  build_and_push_amd64:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Build and push Docker image for amd64
-        uses: docker/build-push-action@v3
-        with:
-            push: true
-            tags: beclab/r4train:${{ github.ref_name }}-amd64
-            file: Dockerfile.r4train.amd64
-            platforms: linux/amd64
-
-  build_and_push_arm64:
+  update_train:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
-
+      
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-
+        
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Build and push Docker image for arm64
+      
+      - name: Build and push Docker image 
         uses: docker/build-push-action@v3
         with:
             push: true
-            tags: beclab/r4train:${{ github.ref_name }}-arm64
-            file: Dockerfile.r4train.arm64
-            platforms: linux/arm64
-
-  create_and_push_manifest:
-    runs-on: ubuntu-latest
-    needs: [build_and_push_amd64, build_and_push_arm64]
-    steps:
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Create and push manifest
-        run: |
-          docker manifest create beclab/r4train:${{ github.ref_name }} \
-            beclab/r4train:${{ github.ref_name }}-amd64 \
-            beclab/r4train:${{ github.ref_name }}-arm64
-          docker manifest push beclab/r4train:${{ github.ref_name }}
+            tags: beclab/r4train:${{ github.event.inputs.tags }}
+            file: Dockerfile.r4train
+            platforms: linux/amd64,linux/arm64
diff --git a/Dockerfile.r4rank b/Dockerfile.r4rank
@@ -0,0 +1,134 @@
+FROM ubuntu:jammy as builder
+RUN mkdir -p /opt/rss-termius-v2-rank && \
+    mkdir -p /opt/xgboost && \
+    mkdir -p /opt/rank_model/
+RUN apt-get update && \
+    apt-get install build-essential -y && \
+    apt-get install cmake -y && \
+    apt install git -y && \
+    apt install curl -y && \
+    apt install pkg-config -y && \
+    apt-get install libssl-dev -y && \
+    apt-get install libcpprest-dev -y && \
+    cd /opt/xgboost && \
+    git clone --recursive https://github.com/dmlc/xgboost . && \
+    git checkout -b release_2.0.0 origin/release_2.0.0 && \
+    git submodule init && \
+    git submodule update && \
+    mkdir build && \
+    cd build && \
+    cmake ..  -DBUILD_STATIC_LIB=on && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm -r xgboost 
+
+
+RUN cd /opt && \
+    curl -LJO https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz  && \
+    tar -xvzf googletest-1.14.0.tar.gz && \
+    cd /opt/googletest-1.14.0 && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm -r googletest-1.14.0 && \
+    rm googletest-1.14.0.tar.gz
+
+
+RUN cd /opt && \
+    curl -LJO https://github.com/mongodb/mongo-c-driver/releases/download/1.24.0/mongo-c-driver-1.24.0.tar.gz && \
+    tar -xvzf mongo-c-driver-1.24.0.tar.gz && \
+    cd mongo-c-driver-1.24.0/build && \
+    cmake .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm mongo-c-driver-1.24.0.tar.gz && \
+    rm -r mongo-c-driver-1.24.0
+
+RUN cd /opt && \
+    curl -LJO  https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.8.1/mongo-cxx-driver-r3.8.1.tar.gz && \
+    tar -xvzf  mongo-cxx-driver-r3.8.1.tar.gz && \
+    cd mongo-cxx-driver-r3.8.1/build && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm mongo-cxx-driver-r3.8.1.tar.gz && \
+    rm -r mongo-cxx-driver-r3.8.1
+
+RUN cd /opt && \
+    curl -LO https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.tar.gz && \
+    tar -xvzf v0.1.99.tar.gz && \
+    cd sentencepiece-0.1.99 && \
+    mkdir build && \
+    cd build  && \
+    cmake .. && \
+    make -j$(nproc) && \
+    make install && \
+    cd /opt && \
+    rm -r sentencepiece-0.1.99 && \
+    rm v0.1.99.tar.gz
+
+RUN cd /opt && \
+    mkdir -p /usr/local/include/cppitertools && \
+    curl -LO https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.1.tar.gz && \
+    tar xzvf v2.1.tar.gz && \
+    cd cppitertools-2.1/ && \
+    cp *.hpp /usr/local/include/cppitertools && \
+    cp -r internal /usr/local/include/cppitertools && \
+    cd /opt && \
+    rm  v2.1.tar.gz  && \
+    rm -r cppitertools-2.1
+
+ADD train-rank/src /opt/rss-terminus-v2-rank/src
+ADD train-rank/test /opt/rss-terminus-v2-rank/test
+COPY train-rank/CMakeLists.txt /opt/rss-terminus-v2-rank/CMakeLists.txt
+RUN cd /opt/rss-terminus-v2-rank && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    make && \
+    cp bertv2rank /usr/local/bin && \
+    cd /opt && \
+    rm -r rss-termius-v2-rank
+
+RUN if [ -f /usr/lib/x86_64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/x86_64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/x86_64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/x86_64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/x86_64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi
+
+RUN if [ -f /usr/lib/aarch64-linux-gnu/libgomp.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libgomp.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 ]; then cp /usr/lib/aarch64-linux-gnu/libcpprest.so.2.10 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlidec.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlienc.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 ]; then cp /usr/lib/aarch64-linux-gnu/libbrotlicommon.so.1 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/aarch64-linux-gnu/libicuuc.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicuuc.so.70 /usr/local/lib/; fi && \
+    if [ -f /usr/lib/aarch64-linux-gnu/libicudata.so.70 ]; then cp /usr/lib/aarch64-linux-gnu/libicudata.so.70 /usr/local/lib/; fi
+
+FROM ubuntu:jammy
+# FROM alpine
+
+
+COPY --from=builder /usr/local/bin/bertv2rank /usr/local/bin/bertv2rank
+COPY  --from=builder /usr/local/lib/libmongocxx.so._noabi /usr/local/lib/libmongocxx.so._noabi
+COPY  --from=builder /usr/local/lib/libbsoncxx.so._noabi  /usr/local/lib/libbsoncxx.so._noabi
+COPY  --from=builder  /usr/local/lib/libmongoc-1.0.so.0  /usr/local/lib/libmongoc-1.0.so.0
+COPY   --from=builder  /usr/local/lib/libbson-1.0.so.0  /usr/local/lib/libbson-1.0.so.0
+
+COPY --from=builder /usr/local/lib/libgomp.so.1 /usr/local/lib/libgomp.so.1
+COPY --from=builder /usr/local/lib/libcpprest.so.2.10  /usr/local/lib/libcpprest.so.2.10
+COPY --from=builder /usr/local/lib/libbrotlidec.so.1 /usr/local/lib/libbrotlidec.so.1
+COPY --from=builder /usr/local/lib/libbrotlienc.so.1 /usr/local/lib/libbrotlienc.so.1
+COPY --from=builder /usr/local/lib/libbrotlicommon.so.1  /usr/local/lib/libbrotlicommon.so.1
+COPY --from=builder /usr/local/lib/libicuuc.so.70  /usr/local/lib/libicuuc.so.70 
+COPY --from=builder /usr/local/lib/libicudata.so.70 /usr/local/lib/libicudata.so.70
+RUN mkdir /opt/rank_model/
+CMD ["/usr/local/bin/bertv2rank"]
+# ENTRYPOINT ["tail", "-f", "/dev/null"]