mirror of
https://gitee.com/dolphinscheduler/DolphinScheduler.git
synced 2024-12-05 21:59:29 +08:00
e4ab488426
* when task with more than one upstreams, mapper
TaskDefinitionMapper method queryByName will return
more than one record, and failed the mybatis result
type, so we have to add `limit 1` to it to
* add multiple runs of example in integrate test
* Change from subprocess.Popen to subprocess.call_check
in integrating test which will raise an error when failed
(cherry picked from commit 81930e5420
)
192 lines
6.1 KiB
YAML
192 lines
6.1 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
name: Python API
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- dev
|
|
paths:
|
|
- 'dolphinscheduler-python/**'
|
|
pull_request:
|
|
|
|
concurrency:
|
|
group: py-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
defaults:
|
|
run:
|
|
working-directory: dolphinscheduler-python/pydolphinscheduler
|
|
|
|
# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE
|
|
env:
|
|
DEPENDENCES: pip setuptools wheel tox
|
|
|
|
jobs:
|
|
paths-filter:
|
|
name: Python-Path-Filter
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
not-docs: ${{ steps.filter.outputs.not-docs }}
|
|
py-change: ${{ steps.filter.outputs.py-change }}
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
|
id: filter
|
|
with:
|
|
filters: |
|
|
not-docs:
|
|
- '!(docs/**)'
|
|
py-change:
|
|
- 'dolphinscheduler-python/pydolphinscheduler/**'
|
|
lint:
|
|
name: Lint
|
|
if: ${{ (needs.paths-filter.outputs.py-change == 'true') || (github.event_name == 'push') }}
|
|
timeout-minutes: 15
|
|
needs: paths-filter
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Set up Python 3.7
|
|
uses: actions/setup-python@v2
|
|
with:
|
|
python-version: 3.7
|
|
- name: Install Dependences
|
|
run: |
|
|
python -m pip install --upgrade ${{ env.DEPENDENCES }}
|
|
- name: Run All Lint Check
|
|
run: |
|
|
python -m tox -vv -e lint
|
|
pytest:
|
|
name: Pytest
|
|
timeout-minutes: 15
|
|
needs: lint
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: [3.6, 3.7, 3.8, 3.9]
|
|
os: [ubuntu-18.04, macOS-latest, windows-latest]
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v2
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- name: Install Dependences
|
|
run: |
|
|
python -m pip install --upgrade ${{ env.DEPENDENCES }}
|
|
- name: Run All Tests
|
|
run: |
|
|
python -m tox -vv -e code-test
|
|
doc-build:
|
|
name: Docs Build Test
|
|
timeout-minutes: 15
|
|
needs: lint
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Set up Python 3.7
|
|
uses: actions/setup-python@v2
|
|
with:
|
|
python-version: 3.7
|
|
- name: Install Dependences
|
|
run: |
|
|
python -m pip install --upgrade ${{ env.DEPENDENCES }}
|
|
- name: Run Tests Build Docs
|
|
run: |
|
|
python -m tox -vv -e doc-build-test
|
|
local-ci:
|
|
name: Local CI
|
|
timeout-minutes: 15
|
|
needs:
|
|
- pytest
|
|
- doc-build
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Set up Python 3.7
|
|
uses: actions/setup-python@v2
|
|
with:
|
|
python-version: 3.7
|
|
- name: Install Dependences
|
|
run: |
|
|
python -m pip install --upgrade ${{ env.DEPENDENCES }}
|
|
- name: Run Tests Build Docs
|
|
run: |
|
|
python -m tox -vv -e local-ci
|
|
integrate-test:
|
|
name: Integrate Test
|
|
if: ${{ (needs.paths-filter.outputs.not-docs == 'true') || (github.event_name == 'push') }}
|
|
runs-on: ubuntu-latest
|
|
needs: paths-filter
|
|
timeout-minutes: 30
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Sanity Check
|
|
uses: ./.github/actions/sanity-check
|
|
- name: Cache local Maven repository
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: ~/.m2/repository
|
|
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
|
restore-keys: ${{ runner.os }}-maven-
|
|
# Switch to project root directory to run mvnw command
|
|
- name: Build Image
|
|
working-directory: ./
|
|
run: |
|
|
./mvnw -B clean install \
|
|
-Dmaven.test.skip \
|
|
-Dmaven.javadoc.skip \
|
|
-Dmaven.checkstyle.skip \
|
|
-Pdocker,release -Ddocker.tag=ci \
|
|
-pl dolphinscheduler-standalone-server -am
|
|
- name: Set up Python 3.7
|
|
uses: actions/setup-python@v2
|
|
with:
|
|
python-version: 3.7
|
|
- name: Install Dependences
|
|
run: |
|
|
python -m pip install --upgrade ${{ env.DEPENDENCES }}
|
|
- name: Run Integrate Tests
|
|
run: |
|
|
python -m tox -vv -e integrate-test
|
|
result:
|
|
name: Python
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
needs: [ paths-filter, local-ci, integrate-test ]
|
|
if: always()
|
|
steps:
|
|
- name: Status
|
|
# We need change CWD to current directory to avoid global default working directory not exists
|
|
working-directory: ./
|
|
run: |
|
|
if [[ ${{ needs.paths-filter.outputs.not-docs }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
|
echo "Only document change, skip both python unit and integrate test!"
|
|
exit 0
|
|
fi
|
|
if [[ ${{ needs.paths-filter.outputs.py-change }} == 'false' && ${{ needs.integrate-test.result }} == 'success' && ${{ github.event_name }} == 'pull_request' ]]; then
|
|
echo "No python code change, and integrate test pass!"
|
|
exit 0
|
|
fi
|
|
if [[ ${{ needs.integrate-test.result }} != 'success' || ${{ needs.local-ci.result }} != 'success' ]]; then
|
|
echo "py-ci Failed!"
|
|
exit -1
|
|
fi
|