test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-29 18:58:05 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[python] Add independent deployment for python gateway server (#7549)

Browse Source 
* [python] Add independent deployment to python gateway server

* Correct some error

* Add missing deployment change

* Add missing expose port

* Add missing standalone config

* Keyword change from rpc to socket

* try to fix

* Add python gateway server to scp-hosts.sh
This commit is contained in:
Jiajie Zhong 2022-01-07 15:16:05 +08:00 committed by GitHub
parent 62cac2d529
commit da4a85943c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 392 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
deploy/docker/docker-compose.yml
View File

@ -140,6 +140,29 @@ services:
networks:
- dolphinscheduler
dolphinscheduler-python-gateway:
image: ${HUB}/dolphinscheduler-python-gateway:${TAG}
ports:
- "54321:54321"
- "25333:25333"
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-schema-initializer:
condition: service_completed_successfully
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
networks:
dolphinscheduler:
driver: bridge

21
deploy/docker/docker-stack.yml
View File

@ -118,6 +118,27 @@ services:
mode: replicated
replicas: 1
dolphinscheduler-python-gateway:
image: apache/dolphinscheduler-python-gateway
ports:
- 54321:54321
- 25333:25333
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
networks:
dolphinscheduler:
driver: overlay

3
deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
View File

@ -44,6 +44,9 @@ Create default docker images' fullname.
{{- define "dolphinscheduler.image.fullname.tools" -}}
{{- .Values.image.registry }}/dolphinscheduler-tools:{{ .Values.image.tag | default .Chart.AppVersion -}}
{{- end -}}
{{- define "dolphinscheduler.image.fullname.python-gateway" -}}
{{- .Values.image.registry }}/dolphinscheduler-python-gateway:{{ .Values.image.tag | default .Chart.AppVersion -}}
{{- end -}}
{{/*
Create a default common labels.

5
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
View File

@ -50,6 +50,11 @@
<outputDirectory>logger-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-python/target/python-gateway-server</directory>
<outputDirectory>python-gateway-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server</directory>
<outputDirectory>standalone-server</outputDirectory>

48
dolphinscheduler-python/pom.xml
View File

@ -57,4 +57,52 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<excludes>
<exclude>*.yaml</exclude>
<exclude>*.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-python-gateway-server</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<finalName>python-gateway-server</finalName>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-python-gateway-server.xml</descriptor>
</descriptors>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>docker</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

64
dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml Normal file
View File

@ -0,0 +1,64 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>dolphinscheduler-python-gateway-server</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<baseDirectory>python-gateway-server</baseDirectory>
<fileSets>
<fileSet>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>*.yaml</include>
<include>*.xml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../script/env</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>dolphinscheduler_env.sh</include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>libs</outputDirectory>
</dependencySet>
</dependencySets>
</assembly>

32
dolphinscheduler-python/src/main/bin/start.sh Normal file
View File

@ -0,0 +1,32 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
source "$BIN_DIR/dolphinscheduler_env.sh"
JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
if [[ "$DOCKER" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
fi
java $JAVA_OPTS \
-cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
org.apache.dolphinscheduler.server.PythonGatewayServer

34
dolphinscheduler-python/src/main/docker/Dockerfile Normal file
View File

@ -0,0 +1,34 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM openjdk:8-jre-slim-buster
ENV DOCKER true
ENV TZ Asia/Shanghai
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
RUN apt update ; \
apt install -y curl sudo ; \
rm -rf /var/lib/apt/lists/*
WORKDIR $DOLPHINSCHEDULER_HOME
ADD ./target/python-gateway-server $DOLPHINSCHEDULER_HOME
EXPOSE 25333 54321
CMD [ "/bin/bash", "./bin/start.sh" ]

2
dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
View File

@ -503,7 +503,7 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
public void run() {
GatewayServer server = new GatewayServer(this);
GatewayServer.turnLoggingOn();
// Start server to accept python client RPC
// Start server to accept python client socket
server.start();
}

61
dolphinscheduler-python/src/main/resources/application.yaml Normal file
View File

@ -0,0 +1,61 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
spring:
application:
name: python-gateway-server
main:
banner-mode: off
datasource:
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true
username: sa
password: ""
jackson:
time-zone: GMT+8
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
messages:
basename: i18n/messages
jpa:
hibernate:
ddl-auto: none
server:
port: 54321
management:
endpoints:
web:
exposure:
include: '*'
metrics:
tags:
application: ${spring.application.name}
# Override by profile
---
spring:
config:
activate:
on-profile: postgresql
quartz:
properties:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate

57
dolphinscheduler-python/src/main/resources/logback-spring.xml Normal file
View File

@ -0,0 +1,57 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="PYTHONGATEWAYLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-python-gateway.log</file>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-python-gateway.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>64MB</maxFileSize>
</rollingPolicy>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<root level="INFO">
<if condition="${DOCKER:-false}">
<then>
<appender-ref ref="STDOUT"/>
</then>
</if>
<appender-ref ref="PYTHONGATEWAYLOGFILE"/>
</root>
</configuration>

9
dolphinscheduler-standalone-server/pom.xml
View File

@ -50,6 +50,11 @@
<artifactId>dolphinscheduler-log-server</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-python</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
@ -61,10 +66,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-python</artifactId>
</dependency>
</dependencies>
<build>

7
dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
View File

@ -60,6 +60,13 @@
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-python-gateway/target/python-gateway</directory>
<includes>
<include>libs</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/resources</directory>

4
script/dolphinscheduler-daemon.sh
View File

@ -16,7 +16,7 @@
# limitations under the License.
#
usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) <api-server|master-server|worker-server|alert-server|standalone-server> "
usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) <api-server|master-server|worker-server|alert-server|python-gateway-server|standalone-server> "
# if no args specified, show usage
if [ $# -le 1 ]; then
@ -62,6 +62,8 @@ elif [ "$command" = "alert-server" ]; then
:
elif [ "$command" = "standalone-server" ]; then
:
elif [ "$command" = "python-gateway-server" ]; then
:
else
echo "Error: No command named '$command' was found."
exit 1

5
script/env/install_env.sh vendored
View File

@ -48,6 +48,11 @@ alertServer=${alertServer:-"ds3"}
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
apiServers=${apiServers:-"ds1"}
# A comma separated list of machine hostname or IP would be installed Python gateway server, it
# must be a subset of configuration `ips`.
# Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1"
pythonGatewayServers=${pythonGatewayServers:-"ds1"}
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
# Do not set this configuration same as the current path (pwd)
installPath=${installPath:-"/tmp/dolphinscheduler"}

2
script/scp-hosts.sh
View File

@ -49,7 +49,7 @@ do
echo "scp dirs to $host/$installPath starting"
ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/"
for dsDir in bin master-server worker-server alert-server api-server ui
for dsDir in bin master-server worker-server alert-server api-server ui python-gateway-server
do
# if worker in workersGroupMap
if [[ "${workersGroupMap[${host}]}" ]]; then

7
script/start-all.sh
View File

@ -56,6 +56,13 @@ do
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
done
pythonGatewayHost=(${pythonGatewayServers//,/ })
for pythonGatewayServer in "${pythonGatewayHost[@]}"
do
echo "$pythonGatewayServer python gateway server is starting"
ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start python-gateway-server;"
done
# query server status
echo "query server status"
cd $installPath/; sh bin/status-all.sh

8
script/status-all.sh
View File

@ -74,3 +74,11 @@ do
apiState=`ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status api-server;"`
echo "$apiServer $apiState"
done
# python gateway server check state
pythonGatewayHost=(${pythonGatewayServers//,/ })
for pythonGatewayServer in "${pythonGatewayHost[@]}"
do
pythonGatewayState=`ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status python-gateway-server;"`
echo "$pythonGatewayServer $pythonGatewayState"
done

7
script/stop-all.sh
View File

@ -54,3 +54,10 @@ do
echo "$apiServer api server is stopping"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
done
pythonGatewayHost=(${pythonGatewayServers//,/ })
for pythonGatewayServer in "${pythonGatewayHost[@]}"
do
echo "$pythonGatewayServer python gateway server is stopping"
ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop python-gateway-server;"
done