DolphinScheduler/.github/workflows/py-ci.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Python API

on:
  push:
    branches:
      - dev
    paths:
      - 'dolphinscheduler-python/**'
  pull_request:

concurrency:
  group: py-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

defaults:
  run:
    working-directory: dolphinscheduler-python/pydolphinscheduler

# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE
env:
  DEPENDENCES: pip setuptools wheel tox

jobs:
  paths-filter:
    name: Python-Path-Filter
    runs-on: ubuntu-latest
    outputs:
      not-docs: ${{ steps.filter.outputs.not-docs }}
      py-change: ${{ steps.filter.outputs.py-change }}
    steps:
      - uses: actions/checkout@v2
      - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
        id: filter
        with:
          filters: |
            not-docs:
              - '!(docs/**)'
            py-change:
              - 'dolphinscheduler-python/pydolphinscheduler/**'            
  lint:
    name: Lint
    if: ${{ (needs.paths-filter.outputs.py-change == 'true') || (github.event_name == 'push') }}
    timeout-minutes: 15
    needs: paths-filter
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v4
        with:
          python-version: 3.7
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}          
      - name: Run All Lint Check
        run: |
          python -m tox -vv -e lint          
  pytest:
    name: Pytest
    timeout-minutes: 15
    needs: lint
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        # YAML parse `3.10` to `3.1`, so we have to add quotes for `'3.10'`, see also:
        # https://github.com/actions/setup-python/issues/160#issuecomment-724485470
        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev]
        os: [ubuntu-latest, macOS-latest, windows-latest]
        # Skip because dependence [py4j](https://pypi.org/project/py4j/) not work on those environments
        exclude:
          - os: windows-latest
            python-version: '3.10'
          - os: windows-latest
            python-version: 3.11-dev
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}          
      - name: Run All Tests
        run: |
          python -m tox -vv -e code-test          
  doc-build:
    name: Docs Build Test
    timeout-minutes: 15
    needs: lint
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        env-list: [doc-build, doc-build-multi]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v4
        with:
          python-version: 3.7
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}          
      - name: Run Build Docs Tests ${{ matrix.env-list }}
        run: |
          python -m tox -vv -e ${{ matrix.env-list }}          
  local-ci:
    name: Local CI
    timeout-minutes: 15
    needs:
      - pytest
      - doc-build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v4
        with:
          python-version: 3.7
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}          
      - name: Run Tests Build Docs
        run: |
          python -m tox -vv -e local-ci          
  integrate-test:
    name: Integrate Test
    if: ${{ (needs.paths-filter.outputs.not-docs == 'true') || (github.event_name == 'push') }}
    runs-on: ubuntu-latest
    needs: paths-filter
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Sanity Check
        uses: ./.github/actions/sanity-check
      - name: Cache local Maven repository
        uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-maven-
      # Switch to project root directory to run mvnw command
      - name: Build Image
        working-directory: ./
        run: |
          ./mvnw -B clean install \
          -Dmaven.test.skip \
          -Dmaven.javadoc.skip \
          -Dmaven.checkstyle.skip \
          -Pdocker,release -Ddocker.tag=ci \
          -pl dolphinscheduler-standalone-server -am          
      - name: Set up Python 3.7
        uses: actions/setup-python@v4
        with:
          python-version: 3.7
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}          
      - name: Run Integrate Tests
        run: |
          python -m tox -vv -e integrate-test          
  result:
    name: Python
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: [ paths-filter, local-ci, integrate-test ]
    if: always()
    steps:
      - name: Status
        # We need change CWD to current directory to avoid global default working directory not exists
        working-directory: ./
        run: |
          if [[ ${{ needs.paths-filter.outputs.not-docs }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
            echo "Only document change, skip both python unit and integrate test!"
            exit 0
          fi
          if [[ ${{ needs.paths-filter.outputs.py-change }} == 'false' && ${{ needs.integrate-test.result }} == 'success' && ${{ github.event_name }} == 'pull_request' ]]; then
            echo "No python code change, and integrate test pass!"
            exit 0
          fi
          if [[ ${{ needs.integrate-test.result }} != 'success' || ${{ needs.local-ci.result }} != 'success' ]]; then
            echo "py-ci Failed!"
            exit -1
          fi