DolphinScheduler/.github/workflows/py-ci.yml
Jiajie Zhong e4ab488426 [python] Fix tasks with multiple upstream and workflow query error (#10941)
* when task with more than one upstreams, mapper
   TaskDefinitionMapper method queryByName will return
   more than one record, and failed the mybatis result
   type, so we have to add `limit 1` to it to
* add multiple runs of example in integrate test
* Change from subprocess.Popen to subprocess.call_check
  in integrating test which will raise an error when failed

(cherry picked from commit 81930e5420)
2022-07-28 11:13:48 +08:00

192 lines
6.1 KiB
YAML

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Python API
on:
push:
branches:
- dev
paths:
- 'dolphinscheduler-python/**'
pull_request:
concurrency:
group: py-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
defaults:
run:
working-directory: dolphinscheduler-python/pydolphinscheduler
# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE
env:
DEPENDENCES: pip setuptools wheel tox
jobs:
paths-filter:
name: Python-Path-Filter
runs-on: ubuntu-latest
outputs:
not-docs: ${{ steps.filter.outputs.not-docs }}
py-change: ${{ steps.filter.outputs.py-change }}
steps:
- uses: actions/checkout@v2
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
id: filter
with:
filters: |
not-docs:
- '!(docs/**)'
py-change:
- 'dolphinscheduler-python/pydolphinscheduler/**'
lint:
name: Lint
if: ${{ (needs.paths-filter.outputs.py-change == 'true') || (github.event_name == 'push') }}
timeout-minutes: 15
needs: paths-filter
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Install Dependences
run: |
python -m pip install --upgrade ${{ env.DEPENDENCES }}
- name: Run All Lint Check
run: |
python -m tox -vv -e lint
pytest:
name: Pytest
timeout-minutes: 15
needs: lint
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
os: [ubuntu-18.04, macOS-latest, windows-latest]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install Dependences
run: |
python -m pip install --upgrade ${{ env.DEPENDENCES }}
- name: Run All Tests
run: |
python -m tox -vv -e code-test
doc-build:
name: Docs Build Test
timeout-minutes: 15
needs: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Install Dependences
run: |
python -m pip install --upgrade ${{ env.DEPENDENCES }}
- name: Run Tests Build Docs
run: |
python -m tox -vv -e doc-build-test
local-ci:
name: Local CI
timeout-minutes: 15
needs:
- pytest
- doc-build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Install Dependences
run: |
python -m pip install --upgrade ${{ env.DEPENDENCES }}
- name: Run Tests Build Docs
run: |
python -m tox -vv -e local-ci
integrate-test:
name: Integrate Test
if: ${{ (needs.paths-filter.outputs.not-docs == 'true') || (github.event_name == 'push') }}
runs-on: ubuntu-latest
needs: paths-filter
timeout-minutes: 30
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Sanity Check
uses: ./.github/actions/sanity-check
- name: Cache local Maven repository
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-maven-
# Switch to project root directory to run mvnw command
- name: Build Image
working-directory: ./
run: |
./mvnw -B clean install \
-Dmaven.test.skip \
-Dmaven.javadoc.skip \
-Dmaven.checkstyle.skip \
-Pdocker,release -Ddocker.tag=ci \
-pl dolphinscheduler-standalone-server -am
- name: Set up Python 3.7
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Install Dependences
run: |
python -m pip install --upgrade ${{ env.DEPENDENCES }}
- name: Run Integrate Tests
run: |
python -m tox -vv -e integrate-test
result:
name: Python
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [ paths-filter, local-ci, integrate-test ]
if: always()
steps:
- name: Status
# We need change CWD to current directory to avoid global default working directory not exists
working-directory: ./
run: |
if [[ ${{ needs.paths-filter.outputs.not-docs }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
echo "Only document change, skip both python unit and integrate test!"
exit 0
fi
if [[ ${{ needs.paths-filter.outputs.py-change }} == 'false' && ${{ needs.integrate-test.result }} == 'success' && ${{ github.event_name }} == 'pull_request' ]]; then
echo "No python code change, and integrate test pass!"
exit 0
fi
if [[ ${{ needs.integrate-test.result }} != 'success' || ${{ needs.local-ci.result }} != 'success' ]]; then
echo "py-ci Failed!"
exit -1
fi