test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-29 18:58:05 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[DSIP-38] Support upload/download plugins from maven repo (#16654)

Browse Source
This commit is contained in:
xiangzihao 2024-09-26 14:39:14 +08:00 committed by GitHub
parent f56a311849
commit c0f70b88cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
36 changed files with 591 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/api-test.yml vendored
View File

@ -83,7 +83,7 @@ jobs:
-Dmaven.test.skip=true \
-Dmaven.javadoc.skip=true \
-Dspotless.skip=true \
-Pdocker,release -Ddocker.tag=ci
-Pdocker,staging -Ddocker.tag=ci
- name: Export Docker Images
run: |
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
@ -91,7 +91,7 @@ jobs:
- uses: actions/upload-artifact@v4
name: Upload Docker Images
with:
name: standalone-image
name: standalone-image-api-test
path: /tmp/standalone-image.tar
retention-days: 1
api-test:
@ -140,7 +140,7 @@ jobs:
- uses: actions/download-artifact@v4
name: Download Docker Images
with:
name: standalone-image
name: standalone-image-api-test
path: /tmp
- name: Load Docker Images
run: |

2
.github/workflows/backend.yml vendored
View File

@ -89,7 +89,7 @@ jobs:
- name: Build and Package on ${{ matrix.java }}
run: |
./mvnw -B clean install \
-Prelease \
-Pstaging \
-Dmaven.test.skip=true \
-Dspotless.skip=true
- name: Check dependency license

2
.github/workflows/e2e-k8s.yml vendored
View File

@ -73,7 +73,7 @@ jobs:
-Dmaven.test.skip=true \
-Dmaven.javadoc.skip=true \
-Dspotless.skip=true \
-Pdocker,release -Ddocker.tag=ci
-Pdocker,staging -Ddocker.tag=ci
- name: Create k8s Kind Cluster
run: |
# install kubectl

6
.github/workflows/e2e.yml vendored
View File

@ -85,7 +85,7 @@ jobs:
-Dmaven.test.skip=true \
-Dmaven.javadoc.skip=true \
-Dspotless.skip=true \
-Pdocker,release -Ddocker.tag=ci
-Pdocker,staging -Ddocker.tag=ci
- name: Export Docker Images
run: |
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
@ -93,7 +93,7 @@ jobs:
- uses: actions/upload-artifact@v4
name: Upload Docker Images
with:
name: standalone-image
name: standalone-image-e2e
path: /tmp/standalone-image.tar
retention-days: 1
e2e:
@ -168,7 +168,7 @@ jobs:
- uses: actions/download-artifact@v4
name: Download Docker Images
with:
name: standalone-image
name: standalone-image-e2e
path: /tmp
- name: Load Docker Images
run: |

75
.github/workflows/publish-nexus.yaml vendored Normal file
View File

@ -0,0 +1,75 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Publish Snapshot
on:
release:
types:
- released
jobs:
publish-snapshot:
if: github.repository == 'apache/dolphinscheduler'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: true
- name: Maximize runner space
uses: ./.github/actions/maximize-build-space
with:
root-reserve-mb: 30720
temp-reserve-mb: 10240
remove-dotnet: 'true'
remove-android: 'true'
remove-haskell: 'true'
remove-codeql: 'true'
remove-docker-images: 'true'
- uses: actions/checkout@v4
- name: Cache local Maven repository
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
restore-keys: ${{ runner.os }}-maven-
- name: Setup JDK 8
uses: actions/setup-java@v4
with:
distribution: 'adopt'
java-version: 8
- name: Set environment variables
run: |
if [[ ${{ github.event_name }} == "schedule" ]]; then
echo "SETTINGS_PATH=./.github/workflows/publish-nexus/snapshot-asf-settings.xml" >> $SETTINGS_PATH
elif [[ ${{ github.event_name }} == "release" ]]; then
echo "SETTINGS_PATH=./.github/workflows/publish-nexus/release-asf-settings.xml" >> $SETTINGS_PATH
else
echo "unknown event name: ${{ github.event_name }}"
exit 2
fi
- name: Publish snapshot with dev branch
env:
ASF_USERNAME: ${{ secrets.NEXUS_USER }}
ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
run: |
./mvnw clean deploy \
-s ${{ env.SETTINGS_PATH }} \
-Dmaven.test.skip=true \
-Dspotless.skip=true \
-Pstaging

34
.github/workflows/publish-nexus/release-asf-settings.xml vendored Normal file
View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>apache.snapshots.https</id>
<username>${env.ASF_USERNAME}</username>
<password>${env.ASF_PASSWORD}</password>
</server>
<server>
<id>apache.releases.https</id>
<username>${env.ASF_USERNAME}</username>
<password>${env.ASF_PASSWORD}</password>
</server>
</servers>
</settings>

29
.github/workflows/publish-nexus/snapshot-asf-settings.xml vendored Normal file
View File

@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>apache.snapshots.https</id>
<username>${env.ASF_USERNAME}</username>
<password>${env.ASF_PASSWORD}</password>
</server>
</servers>
</settings>

1
.licenserc.yaml
View File

@ -48,6 +48,7 @@ header:
- 'org.mockito.plugins.MockMaker'
- tools/dependencies/known-dependencies.txt
- '**/banner.txt'
- '**/*-banner.txt'
- '.terraform.lock.hcl'
- deploy/kubernetes/dolphinscheduler/README.md.gotmpl
- .idea/vcs.xml

110
config/plugins_config Normal file
View File

@ -0,0 +1,110 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This mapping is used to resolve the Jar package name without version (or call artifactId)
#
# corresponding to the module in the user Config, helping Dolphinscheduler to load the correct Jar package.
# Don't modify the delimiter " -- ", just select the plugin you need
--alert-plugins--
dolphinscheduler-alert-aliyunVoice
dolphinscheduler-alert-dingtalk
dolphinscheduler-alert-email
dolphinscheduler-alert-feishu
dolphinscheduler-alert-http
dolphinscheduler-alert-pagerduty
dolphinscheduler-alert-prometheus
dolphinscheduler-alert-script
dolphinscheduler-alert-slack
dolphinscheduler-alert-telegram
dolphinscheduler-alert-webexteams
dolphinscheduler-alert-wechat
--end--
--datasource-plugins--
dolphinscheduler-datasource-aliyunserverlessspark
dolphinscheduler-datasource-athena
dolphinscheduler-datasource-azure-sql
dolphinscheduler-datasource-clickhouse
dolphinscheduler-datasource-dameng
dolphinscheduler-datasource-databend
dolphinscheduler-datasource-db2
dolphinscheduler-datasource-doris
dolphinscheduler-datasource-hana
dolphinscheduler-datasource-hive
dolphinscheduler-datasource-k8s
dolphinscheduler-datasource-kyuubi
dolphinscheduler-datasource-mysql
dolphinscheduler-datasource-oceanbase
dolphinscheduler-datasource-oracle
dolphinscheduler-datasource-postgresql
dolphinscheduler-datasource-presto
dolphinscheduler-datasource-redshift
dolphinscheduler-datasource-sagemaker
dolphinscheduler-datasource-snowflake
dolphinscheduler-datasource-spark
dolphinscheduler-datasource-sqlserver
dolphinscheduler-datasource-ssh
dolphinscheduler-datasource-starrocks
dolphinscheduler-datasource-trino
dolphinscheduler-datasource-vertica
dolphinscheduler-datasource-zeppelin
--end--
--storage-plugins--
dolphinscheduler-storage-abs
dolphinscheduler-storage-gcs
dolphinscheduler-storage-hdfs
dolphinscheduler-storage-obs
dolphinscheduler-storage-oss
dolphinscheduler-storage-s3
--end--
--task-plugins--
dolphinscheduler-task-aliyunserverlessspark
dolphinscheduler-task-chunjun
dolphinscheduler-task-datafactory
dolphinscheduler-task-dataquality
dolphinscheduler-task-datasync
dolphinscheduler-task-datax
dolphinscheduler-task-dinky
dolphinscheduler-task-dms
dolphinscheduler-task-dvc
dolphinscheduler-task-emr
dolphinscheduler-task-flink
dolphinscheduler-task-flink-stream
dolphinscheduler-task-hivecli
dolphinscheduler-task-http
dolphinscheduler-task-java
dolphinscheduler-task-jupyter
dolphinscheduler-task-k8s
dolphinscheduler-task-kubeflow
dolphinscheduler-task-linkis
dolphinscheduler-task-mlflow
dolphinscheduler-task-mr
dolphinscheduler-task-openmldb
dolphinscheduler-task-procedure
dolphinscheduler-task-python
dolphinscheduler-task-pytorch
dolphinscheduler-task-remoteshell
dolphinscheduler-task-sagemaker
dolphinscheduler-task-seatunnel
dolphinscheduler-task-shell
dolphinscheduler-task-spark
dolphinscheduler-task-sql
dolphinscheduler-task-sqoop
dolphinscheduler-task-zeppelin
--end--

19
docs/docs/en/guide/installation/pseudo-cluster.md
View File

@ -16,10 +16,27 @@ Pseudo-cluster deployment of DolphinScheduler requires external software support
- `pstree` for macOS
- `psmisc` for Fedora/Red/Hat/CentOS/Ubuntu/Debian
> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support.
## Download Plugins Dependencies
Starting from version 3.3.0, the binary package no longer provides plugin dependencies, and users need to download them by themselves. The plugin dependency package download address: [Plugin Dependency Package](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler)
You can also execute the following command to install plugin dependencies:
```shell
bash ./bin/install-plugins.sh 3.3.0
```
Usually, you do not need all connector plugins, you can specify the plugins you need by configuring `conf/plugins_config`. For example, if you only need the `dolphinscheduler-task-shell` plugin, you can modify the configuration file as follows:
```
--task-plugins--
dolphinscheduler-task-shell
--end--
```
## DolphinScheduler Startup Environment
> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support.
### Configure User Exemption and Permissions
Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`:

4
docs/docs/en/guide/installation/standalone.md
View File

@ -14,6 +14,10 @@ If you want to deploy DolphinScheduler in production, we recommend you follow [c
- JDK：download [JDK][jdk] (1.8 or 11), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment.
- Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/<version>). <!-- markdown-link-check-disable-line -->
## Download Plugin Dependencies
Please refer to the [Download Plugin Dependencies](../installation/pseudo-cluster.md) in pseudo-cluster deployment.
### Configure User Exemption and Permissions
Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`:

8
docs/docs/en/guide/upgrade/incompatible.md
View File

@ -28,8 +28,8 @@ This document records the incompatible updates between each version. You need to
## 3.3.0
* Remove the `udf-manage` function from the `resource center` ([#16209])
* Remove the `Pigeon` from the `Task Plugin` ([#16218])
* Uniformly name `process` in code as `workflow` ([#16515])
* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release ([#16543])
* Remove the `udf-manage` function from the `resource center` ([#16209])(https://github.com/apache/dolphinscheduler/pull/16209)
* Remove the `Pigeon` from the `Task Plugin` ([#16218])(https://github.com/apache/dolphinscheduler/pull/16218)
* Uniformly name `process` in code as `workflow` ([#16515])(https://github.com/apache/dolphinscheduler/pull/16515)
* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release ([#16543])(https://github.com/apache/dolphinscheduler/pull/16543)

3
docs/docs/en/guide/upgrade/upgrade.md
View File

@ -72,7 +72,8 @@ Execution result:
#### Upgrade version restriction
After version 3.3.X and later, we only support upgrading from 3.0.0. For versions lower than this, please download the historical version and upgrade to 3.0.0.
- After version 3.3.X and later, we only support upgrading from 3.0.0. For versions lower than this, please download the historical version and upgrade to 3.0.0.
- After version 3.3.X and later, binary packages no longer provide plugins dependencies by default, so when you use them for the first time, you need to download and install them yourself. For more information, please refer to [Pseudo-Cluster](../installation/pseudo-cluster.md).
#### Precautions after the upgrade

19
docs/docs/zh/guide/installation/pseudo-cluster.md
View File

@ -16,10 +16,27 @@
- macOS 安装`pstree`
- Fedora/Red/Hat/CentOS/Ubuntu/Debian 安装`psmisc`
> **_注意:_** DolphinScheduler 本身不依赖 Hadoop、Hive、Spark，但如果你运行的任务需要依赖他们，就需要有对应的环境支持
## 下载插件依赖
从 3.3.0 版本开始，二进制包不再提供插件依赖，需要用户自行下载。插件依赖包下载地址：[插件依赖包](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler)
你也可以执行以下命令来安装插件依赖:
```shell
bash ./bin/install-plugins.sh 3.3.0
```
通常你并不需要所有的连接器插件，可以通过配置 `conf/plugins_config` 来指定你所需要的插件，例如，你只需要 `dolphinscheduler-task-shell` 插件，那么您可以修改配置文件如下：
```
--task-plugins--
dolphinscheduler-task-shell
--end--
```
## 准备 DolphinScheduler 启动环境
> **_注意:_** DolphinScheduler 本身不依赖 Hadoop、Hive、Spark，但如果你运行的任务需要依赖他们，就需要有对应的环境支持
### 配置用户免密及权限
创建部署用户，并且一定要配置 `sudo` 免密。以创建 dolphinscheduler 用户为例

4
docs/docs/zh/guide/installation/standalone.md
View File

@ -12,6 +12,10 @@ Standalone 仅适用于 DolphinScheduler 的快速体验.
- JDK：下载[JDK][jdk] (1.8 or 11)，安装并配置 `JAVA_HOME` 环境变量，并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在，可以跳过这步。
- 二进制包：在[下载页面](https://dolphinscheduler.apache.org/en-us/download/<version>)下载 DolphinScheduler 二进制包 <!-- markdown-link-check-disable-line -->
## 下载插件依赖
请参考伪集群部署的[下载插件依赖](../installation/pseudo-cluster.md#下载插件依赖)
## 配置用户免密及权限
创建部署用户，并且一定要配置 `sudo` 免密。以创建 dolphinscheduler 用户为例

8
docs/docs/zh/guide/upgrade/incompatible.md
View File

@ -26,8 +26,8 @@
## 3.3.0
* 从 `资源中心` 中移除了 `udf-manage` 功能 ([#16209])
* 从 `任务插件` 中移除了 `Pigeon` 类型 ([#16218])
* 统一代码中的 `process` 为 `workflow` ([#16515])
* 在 3.3.0-release 中废弃了从 1.x 至 2.x 的升级代码 ([#16543])
* 从 `资源中心` 中移除了 `udf-manage` 功能 ([#16209])(https://github.com/apache/dolphinscheduler/pull/16209)
* 从 `任务插件` 中移除了 `Pigeon` 类型 ([#16218])(https://github.com/apache/dolphinscheduler/pull/16218)
* 统一代码中的 `process` 为 `workflow` ([#16515])(https://github.com/apache/dolphinscheduler/pull/16515)
* 在 3.3.0-release 中废弃了从 1.x 至 2.x 的升级代码 ([#16543])(https://github.com/apache/dolphinscheduler/pull/16543)

3
docs/docs/zh/guide/upgrade/upgrade.md
View File

@ -73,7 +73,8 @@ jar 包 并添加到 `./tools/libs` 目录下，设置以下环境变量
#### 升级版本限制
在 3.3.X 以及之后的版本，我们仅支持从 3.0.0 开始进行升级，低于此版本的请下载历史版本升级至 3.0.0。
- 在 3.3.X 以及之后的版本，我们仅支持从 3.0.0 开始进行升级，低于此版本的请下载历史版本升级至 3.0.0。
- 在 3.3.X 以及之后的版本，二进制包不再默认提供插件依赖，因此第一次使用时，需要自行下载安装。具体请参考请参照[伪集群部署(Pseudo-Cluster)](../installation/pseudo-cluster.md)
#### 升级后的注意事项

4
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
View File

@ -58,7 +58,9 @@ alert-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
fi
done

4
dolphinscheduler-api/src/main/bin/start.sh
View File

@ -60,7 +60,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
fi
done
$JAVA_HOME/bin/java $JAVA_OPTS \

127
dolphinscheduler-dist/pom.xml
View File

@ -27,6 +27,10 @@
<artifactId>dolphinscheduler-dist</artifactId>
<name>${project.artifactId}</name>
<properties>
<assembly.skipAssembly>${build.assembly.skip}</assembly.skipAssembly>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
@ -73,71 +77,70 @@
<build>
<finalName>apache-dolphinscheduler-${project.version}</finalName>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-bin</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-bin.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
<execution>
<id>src</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-src.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>${exec-maven-plugin.version}</version>
<configuration>
<skip>${build.assembly.skip}</skip>
</configuration>
<executions>
<execution>
<id>assembly-plugins</id>
<goals>
<goal>exec</goal>
</goals>
<phase>package</phase>
<configuration>
<executable>bash</executable>
<workingDirectory>${project.basedir}</workingDirectory>
<arguments>
<argument>src/main/assembly/assembly-plugins.sh</argument>
<argument>${build.plugins.skip}</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>release</id>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-bin</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-bin.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
<execution>
<id>src</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-src.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>${exec-maven-plugin.version}</version>
<executions>
<execution>
<id>assembly-plugins</id>
<goals>
<goal>exec</goal>
</goals>
<phase>package</phase>
<configuration>
<executable>bash</executable>
<workingDirectory>${project.basedir}</workingDirectory>
<arguments>
<argument>src/main/assembly/assembly-plugins.sh</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>docker</id>
<build>

18
dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh
View File

@ -17,6 +17,8 @@
#
set -xeo pipefail
PLUGINS_ASSEMBLY_SKIP=$1
DIST_DIR="$(pwd)/target"
BIN_TAR_FILE="$DIST_DIR/apache-dolphinscheduler-*-bin.tar.gz"
if [ ! -f $BIN_TAR_FILE ]; then
@ -36,12 +38,16 @@ storage-plugins
task-plugins
)
for plugin_path in ${PLUGINS_PATH[@]}
do
cd $BIN_DIR/plugins/$plugin_path
find ./* -name "*.jar" | xargs -I {} mv {} ./
ls -d */ | xargs -I {} rm -rf {}
done
if [ $PLUGINS_ASSEMBLY_SKIP == "true" ]; then
rm -rf $BIN_DIR/plugins/*
else
for plugin_path in ${PLUGINS_PATH[@]}
do
cd $BIN_DIR/plugins/$plugin_path
find ./* -name "*.jar" | xargs -I {} mv {} ./
ls -d */ | xargs -I {} rm -rf {}
done
fi
# move *-server/libs/*.jar to libs/ and create symbolic link in *-server/libs/
MODULES_PATH=(

15
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
View File

@ -107,6 +107,21 @@
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../config</directory>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/.././</directory>
<includes>
<include>mvnw</include>
<include>mvnw.cmd</include>
</includes>
<fileMode>0755</fileMode>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/release-docs</directory>
<outputDirectory>.</outputDirectory>

6
dolphinscheduler-dist/src/main/docker/alert-server.dockerfile
View File

@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
mv apache-dolphinscheduler-*-bin/* . ; \
rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333

6
dolphinscheduler-dist/src/main/docker/api-server.dockerfile
View File

@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
mv apache-dolphinscheduler-*-bin/* . ; \
rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333

6
dolphinscheduler-dist/src/main/docker/master-server.dockerfile
View File

@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
mv apache-dolphinscheduler-*-bin/* . ; \
rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333

6
dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile
View File

@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
mv apache-dolphinscheduler-*-bin/* . ; \
rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333

6
dolphinscheduler-dist/src/main/docker/tools.dockerfile
View File

@ -28,6 +28,10 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
# see doc: https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/upgrade.html
ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
mv apache-dolphinscheduler-*-bin/* . ; \
rm -rf apache-dolphinscheduler-*-bin
ENTRYPOINT [ "/bin/bash" ]

6
dolphinscheduler-dist/src/main/docker/worker-server.dockerfile
View File

@ -27,7 +27,11 @@ RUN apt update ; \
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME
COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
mv apache-dolphinscheduler-*-bin/* . ; \
rm -rf apache-dolphinscheduler-*-bin
EXPOSE 12345 25333

4
dolphinscheduler-master/src/main/bin/start.sh
View File

@ -60,7 +60,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
fi
done
$JAVA_HOME/bin/java $JAVA_OPTS \

4
dolphinscheduler-standalone-server/src/main/bin/start.sh
View File

@ -65,7 +65,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
fi
done
for jar in $(find $STANDALONE_HOME/libs/* -name "*.jar"); do

1
dolphinscheduler-standalone-server/src/main/resources/application.yaml
View File

@ -23,6 +23,7 @@ spring:
date-format: "yyyy-MM-dd HH:mm:ss"
banner:
charset: UTF-8
location: classpath:standalone-banner.txt
sql:
init:
schema-locations: classpath:sql/dolphinscheduler_h2.sql

0
dolphinscheduler-standalone-server/src/main/resources/banner.txt → dolphinscheduler-standalone-server/src/main/resources/standalone-banner.txt
View File

89
dolphinscheduler-ui/pom.xml
View File

@ -35,54 +35,49 @@
<frontend-maven-plugin.version>1.12.1</frontend-maven-plugin.version>
</properties>
<profiles>
<profile>
<id>release</id>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<configuration>
<skip>${build.ui.skip}</skip>
<pnpmInheritsProxyConfigFromMaven>false</pnpmInheritsProxyConfigFromMaven>
</configuration>
<executions>
<execution>
<id>install node and pnpm</id>
<goals>
<goal>install-node-and-pnpm</goal>
</goals>
<configuration>
<skip>${build.ui.skip}</skip>
<pnpmInheritsProxyConfigFromMaven>false</pnpmInheritsProxyConfigFromMaven>
<nodeVersion>${node.version}</nodeVersion>
<pnpmVersion>${pnpm.version}</pnpmVersion>
</configuration>
<executions>
<execution>
<id>install node and pnpm</id>
<goals>
<goal>install-node-and-pnpm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<pnpmVersion>${pnpm.version}</pnpmVersion>
</configuration>
</execution>
<execution>
<id>pnpm install</id>
<goals>
<goal>pnpm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>pnpm run build:prod</id>
<goals>
<goal>pnpm</goal>
</goals>
<configuration>
<arguments>run build:prod</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</execution>
<execution>
<id>pnpm install</id>
<goals>
<goal>pnpm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>pnpm run build:prod</id>
<goals>
<goal>pnpm</goal>
</goals>
<configuration>
<arguments>run build:prod</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</build>
</project>

4
dolphinscheduler-worker/src/main/bin/start.sh
View File

@ -60,7 +60,9 @@ task-plugins
)
for plugin in ${PLUGINS_PATH[@]}; do
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then
CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*"
fi
done
$JAVA_HOME/bin/java $JAVA_OPTS \

33
pom.xml
View File

@ -95,7 +95,9 @@
<docker.build.skip>true</docker.build.skip>
<docker.push.skip>true</docker.push.skip>
<skipDepCheck>true</skipDepCheck>
<build.ui.skip>false</build.ui.skip>
<build.ui.skip>true</build.ui.skip>
<build.plugins.skip>false</build.plugins.skip>
<build.assembly.skip>true</build.assembly.skip>
<spotless.skip>false</spotless.skip>
<skipUT>false</skipUT>
@ -777,7 +779,36 @@
<tag>HEAD</tag>
</scm>
<distributionManagement>
<repository>
<id>apache.releases.https</id>
<name>Apache Release Distribution Repository</name>
<url>https://repository.apache.org/service/local/staging/deploy/maven2</url>
</repository>
<snapshotRepository>
<id>apache.snapshots.https</id>
<name>Apache Development Snapshot Repository</name>
<url>https://repository.apache.org/content/repositories/snapshots</url>
</snapshotRepository>
</distributionManagement>
<profiles>
<profile>
<id>release</id>
<properties>
<build.ui.skip>false</build.ui.skip>
<build.plugins.skip>true</build.plugins.skip>
<build.assembly.skip>false</build.assembly.skip>
</properties>
</profile>
<profile>
<id>staging</id>
<properties>
<build.ui.skip>false</build.ui.skip>
<build.plugins.skip>false</build.plugins.skip>
<build.assembly.skip>false</build.assembly.skip>
</properties>
</profile>
<profile>
<id>docker</id>
<properties>

66
script/install-plugins.sh Executable file
View File

@ -0,0 +1,66 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
# This script is used to download the plugins required during the running process.
# All are downloaded by default. You can also choose what you need.
# You only need to configure the plug-in name in config/plugin_config.
# get ds home
DOLPHINSCHEDULER_HOME=$(cd $(dirname $0);cd ../;pwd)
# plugins default version is 3.3.0, you can also choose a custom version. eg: 3.3.0: bash install-plugins.sh 3.3.0
version=3.3.0
if [ -n "$1" ]; then
if [ "$1" == "dev" ]; then
version="dev-SNAPSHOT"
else
version="$1"
fi
fi
echo "Install Dolphinscheduler plugins, usage version is ${version}"
# create the plugins directory
if [ ! -d ${DOLPHINSCHEDULER_HOME}/plugins ]; then
mkdir -p ${DOLPHINSCHEDULER_HOME}/plugins
fi
plugin_dir=""
while read line; do
if [ -z "$line" ]; then
continue
fi
start_char=$(echo "$line" | cut -c 1)
if [ "$start_char" == "-" ]; then
plugin_dir=$(echo ${line//--/})
if [ "$plugin_dir" != "end" ]; then
mkdir -p ${DOLPHINSCHEDULER_HOME}/plugins/${plugin_dir}
fi
fi
if [ "$start_char" != "-" ] && [ "$start_char" != "#" ]; then
echo "installing plugin: " $line
${DOLPHINSCHEDULER_HOME}/mvnw dependency:get -DgroupId=org.apache.dolphinscheduler -DartifactId=${line} -Dversion=${version} -Ddest=${DOLPHINSCHEDULER_HOME}/plugins/${plugin_dir}
fi
done < ${DOLPHINSCHEDULER_HOME}/conf/plugins_config