From da4a85943c2c42b3b35ad0cb438e008da6509e6e Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Fri, 7 Jan 2022 15:16:05 +0800 Subject: [PATCH] [python] Add independent deployment for python gateway server (#7549) * [python] Add independent deployment to python gateway server * Correct some error * Add missing deployment change * Add missing expose port * Add missing standalone config * Keyword change from rpc to socket * try to fix * Add python gateway server to scp-hosts.sh --- deploy/docker/docker-compose.yml | 23 +++++++ deploy/docker/docker-stack.yml | 21 ++++++ .../dolphinscheduler/templates/_helpers.tpl | 3 + .../main/assembly/dolphinscheduler-bin.xml | 5 ++ dolphinscheduler-python/pom.xml | 48 ++++++++++++++ ...dolphinscheduler-python-gateway-server.xml | 64 +++++++++++++++++++ dolphinscheduler-python/src/main/bin/start.sh | 32 ++++++++++ .../src/main/docker/Dockerfile | 34 ++++++++++ .../server/PythonGatewayServer.java | 2 +- .../src/main/resources/application.yaml | 61 ++++++++++++++++++ .../src/main/resources/logback-spring.xml | 57 +++++++++++++++++ dolphinscheduler-standalone-server/pom.xml | 9 +-- .../dolphinscheduler-standalone-server.xml | 7 ++ script/dolphinscheduler-daemon.sh | 4 +- script/env/install_env.sh | 5 ++ script/scp-hosts.sh | 2 +- script/start-all.sh | 7 ++ script/status-all.sh | 8 +++ script/stop-all.sh | 7 ++ 19 files changed, 392 insertions(+), 7 deletions(-) create mode 100644 dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml create mode 100644 dolphinscheduler-python/src/main/bin/start.sh create mode 100644 dolphinscheduler-python/src/main/docker/Dockerfile create mode 100644 dolphinscheduler-python/src/main/resources/application.yaml create mode 100644 dolphinscheduler-python/src/main/resources/logback-spring.xml diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 2c557c4dd7..e59320b958 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -140,6 +140,29 @@ services: networks: - dolphinscheduler + dolphinscheduler-python-gateway: + image: ${HUB}/dolphinscheduler-python-gateway:${TAG} + ports: + - "54321:54321" + - "25333:25333" + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-schema-initializer: + condition: service_completed_successfully + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + networks: + - dolphinscheduler + networks: dolphinscheduler: driver: bridge diff --git a/deploy/docker/docker-stack.yml b/deploy/docker/docker-stack.yml index 02ee37be02..8d0ae92553 100644 --- a/deploy/docker/docker-stack.yml +++ b/deploy/docker/docker-stack.yml @@ -118,6 +118,27 @@ services: mode: replicated replicas: 1 + dolphinscheduler-python-gateway: + image: apache/dolphinscheduler-python-gateway + ports: + - 54321:54321 + - 25333:25333 + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + networks: dolphinscheduler: driver: overlay diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index c2684e029a..f8fef1167a 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -44,6 +44,9 @@ Create default docker images' fullname. {{- define "dolphinscheduler.image.fullname.tools" -}} {{- .Values.image.registry }}/dolphinscheduler-tools:{{ .Values.image.tag | default .Chart.AppVersion -}} {{- end -}} +{{- define "dolphinscheduler.image.fullname.python-gateway" -}} +{{- .Values.image.registry }}/dolphinscheduler-python-gateway:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} {{/* Create a default common labels. diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml index 661dbdb826..769393db7e 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml @@ -50,6 +50,11 @@ logger-server + + ${basedir}/../dolphinscheduler-python/target/python-gateway-server + python-gateway-server + + ${basedir}/../dolphinscheduler-standalone-server/target/standalone-server standalone-server diff --git a/dolphinscheduler-python/pom.xml b/dolphinscheduler-python/pom.xml index 7076689473..ecb3872ab9 100644 --- a/dolphinscheduler-python/pom.xml +++ b/dolphinscheduler-python/pom.xml @@ -57,4 +57,52 @@ + + + + + org.apache.maven.plugins + maven-jar-plugin + + + *.yaml + *.xml + + + + + maven-assembly-plugin + + + dolphinscheduler-python-gateway-server + package + + single + + + python-gateway-server + + src/main/assembly/dolphinscheduler-python-gateway-server.xml + + false + + + + + + + + + + docker + + + + org.codehaus.mojo + exec-maven-plugin + + + + + diff --git a/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml b/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml new file mode 100644 index 0000000000..49e03206ea --- /dev/null +++ b/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml @@ -0,0 +1,64 @@ + + + + dolphinscheduler-python-gateway-server + + dir + + false + python-gateway-server + + + ${basedir}/src/main/resources + + *.yaml + *.xml + + conf + + + ${basedir}/src/main/bin + bin + 0755 + 0755 + + + ${basedir}/../script/env + bin + + dolphinscheduler_env.sh + + 0755 + 0755 + + + ${basedir}/../dolphinscheduler-common/src/main/resources + + **/*.properties + + conf + + + + + libs + + + diff --git a/dolphinscheduler-python/src/main/bin/start.sh b/dolphinscheduler-python/src/main/bin/start.sh new file mode 100644 index 0000000000..f189684330 --- /dev/null +++ b/dolphinscheduler-python/src/main/bin/start.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)} + +source "$BIN_DIR/dolphinscheduler_env.sh" + +JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +if [[ "$DOCKER" == "true" ]]; then + JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" +fi + +java $JAVA_OPTS \ + -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ + org.apache.dolphinscheduler.server.PythonGatewayServer diff --git a/dolphinscheduler-python/src/main/docker/Dockerfile b/dolphinscheduler-python/src/main/docker/Dockerfile new file mode 100644 index 0000000000..7f50a8fc39 --- /dev/null +++ b/dolphinscheduler-python/src/main/docker/Dockerfile @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM openjdk:8-jre-slim-buster + +ENV DOCKER true +ENV TZ Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler + +RUN apt update ; \ + apt install -y curl sudo ; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR $DOLPHINSCHEDULER_HOME + +ADD ./target/python-gateway-server $DOLPHINSCHEDULER_HOME + +EXPOSE 25333 54321 + +CMD [ "/bin/bash", "./bin/start.sh" ] diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java index ef6deb45da..73d66b17de 100644 --- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java +++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java @@ -503,7 +503,7 @@ public class PythonGatewayServer extends SpringBootServletInitializer { public void run() { GatewayServer server = new GatewayServer(this); GatewayServer.turnLoggingOn(); - // Start server to accept python client RPC + // Start server to accept python client socket server.start(); } diff --git a/dolphinscheduler-python/src/main/resources/application.yaml b/dolphinscheduler-python/src/main/resources/application.yaml new file mode 100644 index 0000000000..af315b1f7f --- /dev/null +++ b/dolphinscheduler-python/src/main/resources/application.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: python-gateway-server + main: + banner-mode: off + datasource: + driver-class-name: org.h2.Driver + url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true + username: sa + password: "" + jackson: + time-zone: GMT+8 + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB + messages: + basename: i18n/messages + jpa: + hibernate: + ddl-auto: none + +server: + port: 54321 + +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} + +# Override by profile +--- +spring: + config: + activate: + on-profile: postgresql + quartz: + properties: + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + diff --git a/dolphinscheduler-python/src/main/resources/logback-spring.xml b/dolphinscheduler-python/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..4f06d160d6 --- /dev/null +++ b/dolphinscheduler-python/src/main/resources/logback-spring.xml @@ -0,0 +1,57 @@ + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + ${log.base}/dolphinscheduler-python-gateway.log + + INFO + + + ${log.base}/dolphinscheduler-python-gateway.%d{yyyy-MM-dd_HH}.%i.log + 168 + 64MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + + + diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml index 4c4b2fd902..61df690c4d 100644 --- a/dolphinscheduler-standalone-server/pom.xml +++ b/dolphinscheduler-standalone-server/pom.xml @@ -50,6 +50,11 @@ dolphinscheduler-log-server + + org.apache.dolphinscheduler + dolphinscheduler-python + + org.apache.curator curator-test @@ -61,10 +66,6 @@ - - org.apache.dolphinscheduler - dolphinscheduler-python - diff --git a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml index 8d767eec0c..90e100e5cf 100644 --- a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml +++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml @@ -60,6 +60,13 @@ . + + ${basedir}/../dolphinscheduler-python-gateway/target/python-gateway + + libs + + . + ${basedir}/src/main/resources diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 37263e3c75..ddfdbe6540 100755 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -16,7 +16,7 @@ # limitations under the License. # -usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) " +usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) " # if no args specified, show usage if [ $# -le 1 ]; then @@ -62,6 +62,8 @@ elif [ "$command" = "alert-server" ]; then : elif [ "$command" = "standalone-server" ]; then : +elif [ "$command" = "python-gateway-server" ]; then + : else echo "Error: No command named '$command' was found." exit 1 diff --git a/script/env/install_env.sh b/script/env/install_env.sh index 8b68c0ea00..2df11c14ae 100644 --- a/script/env/install_env.sh +++ b/script/env/install_env.sh @@ -48,6 +48,11 @@ alertServer=${alertServer:-"ds3"} # Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" apiServers=${apiServers:-"ds1"} +# A comma separated list of machine hostname or IP would be installed Python gateway server, it +# must be a subset of configuration `ips`. +# Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1" +pythonGatewayServers=${pythonGatewayServers:-"ds1"} + # The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. # Do not set this configuration same as the current path (pwd) installPath=${installPath:-"/tmp/dolphinscheduler"} diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 2397ede936..1b216ab41e 100755 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -49,7 +49,7 @@ do echo "scp dirs to $host/$installPath starting" ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/" - for dsDir in bin master-server worker-server alert-server api-server ui + for dsDir in bin master-server worker-server alert-server api-server ui python-gateway-server do # if worker in workersGroupMap if [[ "${workersGroupMap[${host}]}" ]]; then diff --git a/script/start-all.sh b/script/start-all.sh index 002f99d873..e7934da8c1 100755 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -56,6 +56,13 @@ do ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" done +pythonGatewayHost=(${pythonGatewayServers//,/ }) +for pythonGatewayServer in "${pythonGatewayHost[@]}" +do + echo "$pythonGatewayServer python gateway server is starting" + ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start python-gateway-server;" +done + # query server status echo "query server status" cd $installPath/; sh bin/status-all.sh diff --git a/script/status-all.sh b/script/status-all.sh index df19eb9c37..c3b4591a62 100755 --- a/script/status-all.sh +++ b/script/status-all.sh @@ -74,3 +74,11 @@ do apiState=`ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status api-server;"` echo "$apiServer $apiState" done + +# python gateway server check state +pythonGatewayHost=(${pythonGatewayServers//,/ }) +for pythonGatewayServer in "${pythonGatewayHost[@]}" +do + pythonGatewayState=`ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status python-gateway-server;"` + echo "$pythonGatewayServer $pythonGatewayState" +done diff --git a/script/stop-all.sh b/script/stop-all.sh index 6592374659..856340c390 100755 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -54,3 +54,10 @@ do echo "$apiServer api server is stopping" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" done + +pythonGatewayHost=(${pythonGatewayServers//,/ }) +for pythonGatewayServer in "${pythonGatewayHost[@]}" +do + echo "$pythonGatewayServer python gateway server is stopping" + ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop python-gateway-server;" +done