enhance: use already installed vcpkg (#28703)

issue #28686 1. Update Builder gpu image changes, see changes #28505 2. update azure-identity-cpp from beta to release Signed-off-by: PowderLi <min.li@zilliz.com>
2024-11-29 18:38:44 +08:00 · 2023-11-30 15:58:32 +08:00 · 2023-11-30 15:58:32 +08:00 · cac802ef7f
commit cac802ef7f
parent aae7e62729
10 changed files with 184 additions and 106 deletions
--- a/.env
+++ b/.env
@ -1,10 +1,10 @@
 IMAGE_REPO=milvusdb
 IMAGE_ARCH=amd64
 OS_NAME=ubuntu20.04
-DATE_VERSION=20231024-4faba61
-LATEST_DATE_VERSION=20231024-4faba61
-GPU_DATE_VERSION=20231123-90997a5
-LATEST_GPU_DATE_VERSION=20231123-90997a5
+DATE_VERSION=20231123-90997a5
+LATEST_DATE_VERSION=20231123-90997a5
+GPU_DATE_VERSION=20231025-a951c75
+LATEST_GPU_DATE_VERSION=20231025-a951c75
 MINIO_ADDRESS=minio:9000
 PULSAR_ADDRESS=pulsar://pulsar:6650
 ETCD_ENDPOINTS=etcd:2379
--- a/.github/workflows/publish-builder.yaml
+++ b/.github/workflows/publish-builder.yaml
@ -7,14 +7,12 @@ on:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/builder/cpu/**'
-      - 'build/docker/builder/gpu/**'
      - '.github/workflows/publish-builder.yaml'
      - '!**.md'
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/builder/cpu/**'
-      - 'build/docker/builder/gpu/**'
      - '.github/workflows/publish-builder.yaml'
      - '!**.md'

@ -55,7 +53,6 @@ jobs:
          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .      
      - name: Docker Build&Push
        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
        shell: bash
        run: |
          docker buildx ls
@ -64,7 +61,7 @@ jobs:
          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-latest -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
      - name: Update Builder Image Changes
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
        continue-on-error: true
        shell: bash
        run: |
@ -96,80 +93,3 @@ jobs:
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

-
-  publish-gpu-builder:
-    name: ${{ matrix.arch }} ${{ matrix.os }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 500
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ ubuntu20.04 ]
-        arch: [ amd64&arm64 ]
-    env:
-      OS_NAME: ${{ matrix.os }}
-      IMAGE_ARCH: ${{ matrix.arch }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get version from system time after release step
-        id: extracter
-        run: |
-          echo "::set-output name=version::$(date +%Y%m%d)"
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      # - name: Setup upterm session
-      #   uses: lhotari/action-upterm@v1
-      - name: Docker Pull
-        shell: bash
-        run: |
-          docker run --rm --privileged tonistiigi/binfmt:latest --install arm64
-          docker buildx ls
-          docker buildx create --use --name=milvus --driver docker-container
-      - name: Docker Build
-        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
-        shell: bash
-        run: |
-          docker buildx ls
-          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-      - name: Docker Build&Push
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
-        shell: bash
-        run: |
-          docker buildx ls
-          docker login -u ${{ secrets.DOCKERHUB_USER }} \
-                       -p ${{ secrets.DOCKERHUB_TOKEN }}
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-      - name: Update Builder Image Changes
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
-        shell: bash
-        run: |
-          sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
-          sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git add .env
-          git commit -m "Update Builder gpu image changes"
-      - name: Create Pull Request
-        id: cpr
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
-        continue-on-error: true
-        uses: peter-evans/create-pull-request@v3
-        with:
-          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
-          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
-          signoff: false
-          branch: update_builder_${{ github.sha }}
-          delete-branch: true
-          title: '[automated] Update Builder gpu image changes'
-          body: |
-            Update Builder gpu image changes
-            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
-            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
-      - name: Check outputs
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
-        run: |
-          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
-          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
--- a/.github/workflows/publish-gpu-builder.yaml
+++ b/.github/workflows/publish-gpu-builder.yaml
@ -0,0 +1,94 @@
+name: Publish Gpu Builder
+# TODO: do not trigger action for some document file update
+
+# This workflow is triggered on pushes or pull request to the repository.
+on:
+  push:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/gpu/**'
+      - '.github/workflows/publish-gpu-builder.yaml'
+      - '!**.md'
+  pull_request:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/gpu/**'
+      - '.github/workflows/publish-gpu-builder.yaml'
+      - '!**.md'
+
+jobs:
+  publish-gpu-builder:
+    name: ${{ matrix.arch }} ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 500
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu20.04]
+        arch: [amd64&arm64]
+    env:
+      OS_NAME: ${{ matrix.os }}
+      IMAGE_ARCH: ${{ matrix.arch }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get version from system time after release step
+        id: extracter
+        run: |
+          echo "::set-output name=version::$(date +%Y%m%d)"
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - name: Docker Pull
+        shell: bash
+        run: |
+          docker run --rm --privileged tonistiigi/binfmt:latest --install arm64
+          docker buildx ls
+          docker buildx create --use --name=milvus --driver docker-container
+      - name: Docker Build
+        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
+        shell: bash
+        run: |
+          docker buildx ls
+          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Docker Build&Push
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        shell: bash
+        run: |
+          docker buildx ls
+          docker login -u ${{ secrets.DOCKERHUB_USER }} \
+                       -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Update Builder Image Changes
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        continue-on-error: true
+        shell: bash
+        run: |
+          sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add .env
+          git commit -m "Update Builder gpu image changes"
+      - name: Create Pull Request
+        id: cpr
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        continue-on-error: true
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: update_gpu_builder_${{ github.sha }}
+          delete-branch: true
+          title: '[automated] Update Builder gpu image changes'
+          body: |
+            Update Builder gpu image changes
+            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
+            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
+      - name: Check outputs
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -14,6 +14,8 @@ services:
    build:
      context: .
      dockerfile: build/docker/builder/cpu/${OS_NAME}/Dockerfile
+      args:
+        TARGETARCH: ${IMAGE_ARCH}
      cache_from:
        - ${IMAGE_REPO}/milvus-env:${OS_NAME}-${LATEST_DATE_VERSION}
    platform: linux/${IMAGE_ARCH}
@ -52,6 +54,8 @@ services:
    build:
      context: .
      dockerfile: build/docker/builder/gpu/${OS_NAME}/Dockerfile
+      args:
+        TARGETARCH: ${IMAGE_ARCH}
      cache_from:
        - ${IMAGE_REPO}/milvus-env:gpu-${OS_NAME}-${LATEST_GPU_DATE_VERSION}
    # user: {{ CURRENT_ID }}
--- a/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp
+++ b/internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp
@ -21,18 +21,6 @@

 namespace azure {

-std::string
-GetTenantId() {
-    return std::getenv("AZURE_TENANT_ID");
-}
-std::string
-GetClientId() {
-    return std::getenv("AZURE_CLIENT_ID");
-}
-std::string
-GetTokenFilePath() {
-    return std::getenv("AZURE_FEDERATED_TOKEN_FILE");
-}
 std::string
 GetConnectionString(const std::string& access_key_id,
                    const std::string& access_key_value,
@ -78,9 +66,10 @@ AzureBlobChunkManager::AzureBlobChunkManager(
    bool useIAM) {
    requestTimeoutMs_ = requestTimeoutMs;
    if (useIAM) {
+        Azure::Identity::WorkloadIdentityCredentialOptions options;
        auto workloadIdentityCredential =
            std::make_shared<Azure::Identity::WorkloadIdentityCredential>(
-                GetTenantId(), GetClientId(), GetTokenFilePath());
+                options);
        client_ = std::make_shared<Azure::Storage::Blobs::BlobServiceClient>(
            "https://" + access_key_id + ".blob." + address + "/",
            workloadIdentityCredential);
--- a/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake
+++ b/internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake
@ -6,6 +6,7 @@ set(AZ_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/..")

 macro(az_vcpkg_integrate)
  message("Vcpkg integrate step.")
+
  # AUTO CMAKE_TOOLCHAIN_FILE:
  #   User can call `cmake -DCMAKE_TOOLCHAIN_FILE="path_to_the_toolchain"` as the most specific scenario.
  #   As the last alternative (default case), Azure SDK will automatically clone VCPKG folder and set toolchain from there.
@ -17,7 +18,7 @@ macro(az_vcpkg_integrate)
      message("AZURE_SDK_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.")
      # GET VCPKG FROM SOURCE
      #  User can set env var AZURE_SDK_VCPKG_COMMIT to pick the VCPKG commit to fetch
-      set(VCPKG_COMMIT_STRING 71d875654e32ee216b0b7e0dc684e589dffa1b1c) # default SDK tested commit
+      set(VCPKG_COMMIT_STRING dc3c55f092c96fb3f1dcdff84e6a99f947ea4165) # default SDK tested commit
      if(DEFINED ENV{AZURE_SDK_VCPKG_COMMIT})
        message("AZURE_SDK_VCPKG_COMMIT is defined. Using that instead of the default.")
        set(VCPKG_COMMIT_STRING "$ENV{AZURE_SDK_VCPKG_COMMIT}") # default SDK tested commit
@ -26,15 +27,15 @@ macro(az_vcpkg_integrate)
      include(FetchContent)
      FetchContent_Declare(
          vcpkg
-          GIT_REPOSITORY      https://github.com/milvus-io/vcpkg.git
+          GIT_REPOSITORY      https://github.com/microsoft/vcpkg.git
          GIT_TAG             ${VCPKG_COMMIT_STRING}
          )
      FetchContent_GetProperties(vcpkg)
      # make sure to pull vcpkg only once.
-      if(NOT vcpkg_POPULATED) 
+      if(NOT vcpkg_POPULATED)
          FetchContent_Populate(vcpkg)
      endif()
-      # use the vcpkg source path 
+      # use the vcpkg source path
      set(CMAKE_TOOLCHAIN_FILE "${vcpkg_SOURCE_DIR}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "")
    endif()
  endif()
@ -43,6 +44,7 @@ macro(az_vcpkg_integrate)
  if(DEFINED ENV{VCPKG_DEFAULT_TRIPLET} AND NOT DEFINED VCPKG_TARGET_TRIPLET)
    set(VCPKG_TARGET_TRIPLET "$ENV{VCPKG_DEFAULT_TRIPLET}" CACHE STRING "")
  endif()
+  message("Vcpkg integrate step - DONE.")
 endmacro()

 macro(az_vcpkg_portfile_prep targetName fileName contentToRemove)
@ -166,4 +168,4 @@ macro(az_vcpkg_export targetName macroNamePart dllImportExportHeaderPath)

  # Export all the installs above as package.
  export(PACKAGE "${targetName}-cpp")
-endmacro()
+endmacro()
--- a/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt
+++ b/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt
@ -4,4 +4,5 @@
 project(azure-blob-test)

 add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp ../AzureBlobChunkManager.cpp)
-target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs)
+find_package(GTest CONFIG REQUIRED)
+target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest)
--- a/internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp
+++ b/internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp
@ -1,7 +1,60 @@
 #include "../AzureBlobChunkManager.h"
+#include <azure/identity/workload_identity_credential.hpp>
+#include <gtest/gtest.h>

 using namespace azure;

+void
+SetTenantId(const char* value) {
+    setenv("AZURE_TENANT_ID", value, 1);
+}
+void
+SetClientId(const char* value) {
+    setenv("AZURE_CLIENT_ID", value, 1);
+}
+void
+SetTokenFilePath(const char* value) {
+    setenv("AZURE_FEDERATED_TOKEN_FILE", value, 1);
+}
+
+std::string
+GetTenantId() {
+    return std::getenv("AZURE_TENANT_ID");
+}
+std::string
+GetClientId() {
+    return std::getenv("AZURE_CLIENT_ID");
+}
+std::string
+GetTokenFilePath() {
+    return std::getenv("AZURE_FEDERATED_TOKEN_FILE");
+}
+
+class AzureBlobChunkManagerTest : public testing::Test {
+ protected:
+    void
+    SetUp() override {
+    }
+
+    // void TearDown() override {}
+};
+
+TEST(AzureBlobChunkManagerTest, Options) {
+    SetTenantId("tenant_id");
+    SetClientId("client_id");
+    SetTokenFilePath("token_file_path");
+
+    Azure::Identity::WorkloadIdentityCredentialOptions options;
+    Azure::Identity::WorkloadIdentityCredential const cred(options);
+    EXPECT_EQ(cred.GetCredentialName(), "WorkloadIdentityCredential");
+
+    EXPECT_EQ(options.TenantId, GetTenantId());
+    EXPECT_EQ(options.TenantId, "tenant_id");
+    EXPECT_EQ(options.ClientId, GetClientId());
+    EXPECT_EQ(options.AuthorityHost, "https://login.microsoftonline.com/");
+    EXPECT_EQ(options.TokenFilePath, GetTokenFilePath());
+}
+
 void
 print(Azure::Core::Diagnostics::Logger::Level level,
      std::string const& message) {
@ -10,7 +63,7 @@ print(Azure::Core::Diagnostics::Logger::Level level,
 }

 int
-main() {
+main0() {
    const char* containerName = "default";
    const char* blobName = "sample-blob";
    AzureBlobChunkManager::InitLog("info", print);
@ -59,3 +112,9 @@ main() {
    chunkManager.DeleteBucket(containerName);
    exit(EXIT_SUCCESS);
 }
+
+int
+main(int argc, char** argv) {
+    testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}
--- a/internal/core/src/storage/azure-blob-storage/vcpkg.json
+++ b/internal/core/src/storage/azure-blob-storage/vcpkg.json
@ -3,6 +3,7 @@
  "version-string": "1.0.0",
  "dependencies": [
    "azure-identity-cpp",
-    "azure-storage-blobs-cpp"
+    "azure-storage-blobs-cpp",
+    "gtest"
  ]
 }
--- a/scripts/azure_build.sh
+++ b/scripts/azure_build.sh
@ -39,6 +39,14 @@ fi
 AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
 ${SOURCE_DIR}"
+
+if [[ -f "/opt/vcpkg/scripts/buildsystems/vcpkg.cmake" ]]; then
+  AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
+  -DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake \
+  -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
+  ${SOURCE_DIR}"
+fi
+
 echo ${AZURE_CMAKE_CMD}
 ${AZURE_CMAKE_CMD}