milvus/.github/workflows/main.yaml

name: Build and test

# This workflow is triggered on pushes or pull request to the repository.
on:
  push:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'scripts/**'
      - 'internal/**'
      - 'pkg/**'
      - 'cmd/**'
      - 'build/**'
      - 'tests/integration/**' # run integration test
      - '.github/workflows/main.yaml'
      - '.env'
      - docker-compose.yml
      - Makefile
      - go.mod
      - '!**.md'
      - '!build/ci/jenkins/**'
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'scripts/**'
      - 'internal/**'
      - 'pkg/**'
      - 'cmd/**'
      - 'build/**'
      - 'tests/integration/**' # run integration test
      - '.github/workflows/main.yaml'
      - '.env'
      - docker-compose.yml
      - Makefile
      - go.mod
      - '!**.md'
      - '!build/ci/jenkins/**'

jobs:
  Build:
    name: Build and test AMD64 Ubuntu ${{ matrix.ubuntu }}
    runs-on: ubuntu-${{ matrix.ubuntu }}
    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        ubuntu: [20.04]
    env:
      UBUNTU: ${{ matrix.ubuntu }}
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        with:
          root-reserve-mb: 10240
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: 'Check Changed files'
        id: changed-files-cpp
        uses: tj-actions/changed-files@v35
        with:
          since_last_remote_commit: 'true'
          files: |
            **/*.cpp
            **/*.cc
            **/*.c
            **/*.h
            **/*.hpp
            **/*.CMakeLists.txt
            **/conanfile.*            
      - name: 'Setup Use USE_ASAN'
        if: steps.changed-files-cpp.outputs.any_changed == 'true'
        run: |
          echo "useasan=ON" >> $GITHUB_ENV
          echo "Setup USE_ASAN to true since cpp file(s) changed"          
      - name: 'Generate CCache Hash'
        env:
          CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt', 'internal/core/conanfile.py') }}
        run: |
          echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
          echo "Set CCache hash to ${CORE_HASH}"          
      - name: Cache CCache Volumes
        # uses: actions/cache@v3
        uses: pat-s/always-upload-cache@v3
        with:
          path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-ccache
          key: ubuntu${{ matrix.ubuntu }}-ccache-${{ env.corehash }}
          restore-keys: ubuntu${{ matrix.ubuntu }}-ccache-
      - name: Cache Conan Packages
        uses: pat-s/always-upload-cache@v3
        with:
          path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-conan
          key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
          restore-keys: ubuntu${{ matrix.ubuntu }}-conan-
      - name: Build
        run: |
          ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage"          
      - run: |
          zip -r code.zip . -x "./.docker/*" -x "./cmake_build/thirdparty/*"          
      - name: Archive code
        uses: actions/upload-artifact@v3
        with:
          name: code
          path: code.zip
  UT-Cpp:
    name: UT for Cpp
    needs: Build
    runs-on: ubuntu-${{ matrix.ubuntu }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        ubuntu: [20.04]
    env:
      UBUNTU: ${{ matrix.ubuntu }}
    steps:
      - name: Download code
        uses: actions/download-artifact@v3.0.1
        with:
          name: code
      - run: |
          unzip code.zip
          rm code.zip          
      - name: Cache Conan Packages
        uses: pat-s/always-upload-cache@v3
        with:
          path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-conan
          key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
          restore-keys: ubuntu${{ matrix.ubuntu }}-conan-
      - name: Start Service
        shell: bash
        run: |
          docker-compose up -d azurite          
      - name: UT
        run: |
          chmod +x build/builder.sh
          chmod +x scripts/*
          chmod +x internal/core/output/unittest/*
          ./build/builder.sh /bin/bash -c ./scripts/run_cpp_codecov.sh          
      - name: Archive result
        uses: actions/upload-artifact@v3
        with:
          name: cpp-result
          path: |
            ./go_coverage.txt
            ./lcov_output.info
            *.info
            *.out            
  UT-Go:
    name: UT for Go
    needs: Build
    runs-on: ubuntu-${{ matrix.ubuntu }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        ubuntu: [20.04]
    env:
      UBUNTU: ${{ matrix.ubuntu }}
    steps:
      - name: Download code
        uses: actions/download-artifact@v3.0.1
        with:
          name: code
      - run: |
          unzip code.zip
          rm code.zip          
      - name: Cache Go Mod Volumes
        uses: actions/cache@v3
        with:
          path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-go-mod
          key: ubuntu${{ matrix.ubuntu }}-go-mod-${{ hashFiles('**/go.sum') }}
          restore-keys: ubuntu${{ matrix.ubuntu }}-go-mod-
      - name: Start Service
        shell: bash
        run: |
          docker-compose up -d pulsar etcd minio azurite          
      - name: UT
        run: |
          chmod +x build/builder.sh
          chmod +x scripts/run_go_codecov.sh
          ./build/builder.sh /bin/bash -c ./scripts/run_go_codecov.sh          
      - name: Archive result
        uses: actions/upload-artifact@v3
        with:
          name: go-result
          path: |
            ./go_coverage.txt
            ./lcov_output.info
            *.info
            *.out
            .git            
  integration-test:
    name: Integration Test
    needs: Build
    runs-on: ubuntu-${{ matrix.ubuntu }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        ubuntu: [20.04]
    env:
      UBUNTU: ${{ matrix.ubuntu }}
    steps:
      - name: Download code
        uses: actions/download-artifact@v3.0.1
        with:
          name: code
      - run: |
          unzip code.zip
          rm code.zip          
      - name: Cache Go Mod Volumes
        uses: actions/cache@v3
        with:
          path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-go-mod
          key: ubuntu${{ matrix.ubuntu }}-go-mod-${{ hashFiles('**/go.sum') }}
          restore-keys: ubuntu${{ matrix.ubuntu }}-go-mod-
      - name: Start Service
        shell: bash
        run: |
          docker-compose up -d pulsar etcd minio          
      - name: IntegrationTest
        run: |
          chmod +x build/builder.sh
          chmod +x scripts/run_intergration_test.sh
          ./build/builder.sh /bin/bash -c ./scripts/run_intergration_test.sh          
      - name: Archive result
        uses: actions/upload-artifact@v3
        with:
          name: it-result
          path: |
            ./it_coverage.txt
            *.info
            *.out            
  codecov:
    name: Upload Code Coverage
    needs: [UT-Cpp, UT-Go, integration-test]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Download Cpp code coverage results
        uses: actions/download-artifact@v3.0.1
        with:
          name: cpp-result
      - name: Download Go code coverage results
        uses: actions/download-artifact@v3.0.1
        with:
          name: go-result
      - name: Download Integration Test coverage results
        uses: actions/download-artifact@v3.0.1
        with:
          name: it-result
      - name: Display structure of code coverage results
        run: |
          ls -lah          

      - name: Upload coverage to Codecov
        if: "github.repository == 'milvus-io/milvus'"
        uses: codecov/codecov-action@v3.1.1
        id: upload_cov
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./go_coverage.txt,./lcov_output.info,./it_coverage.txt
          name: ubuntu-20.04-unittests
          fail_ci_if_error: true
      - name: Retry Upload coverage to Codecov
        if: "${{ failure() }} && github.repository == 'milvus-io/milvus'"
        uses: codecov/codecov-action@v3.1.1
        id: retry_upload_cov
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./go_coverage.txt,./lcov_output.info,./it_coverage.txt
          name: ubuntu-${{ matrix.ubuntu }}-unittests
          fail_ci_if_error: true