test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-30 03:08:01 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

add worker group config (#2681)

Browse Source 
* update README about DolphinScheduler

* Update issue templates

* update

* regularize api pom xml
update rpc maven compile to 1.8

* regularize api pom xml

* change commons.lang3.StringUtils to common.utils.StringUtils

* update pom.xml

* update

* correct equals method

* jasper-runtime is needed when api server start

* jasper-runtime jar is needed when api server start

* combine logback config of master/worker/alert/api server to one logback.xml

* remove tomcat runtime jar

* add UT

* add license

* remove jasper-runtime jar, not need anymore

* sovle jar conflict， remove servlet-api 2.5 jar

* remove servlet 2.5

* add embedded database h2 and update embedded zookeeper

* delete logs

* add license

* change log level to INFO

* simplify and optimize config

* add development state

* simplify common config

* add worker group config

* add worker group config

Co-authored-by: lidongdai <escheduler@outlook.com>
Co-authored-by: dailidong <dolphinschedule@gmail.com>
This commit is contained in:
dailidong 2020-05-12 16:49:40 +08:00 committed by GitHub
parent d99471806e
commit 06c4ddf8e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 21 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dolphinscheduler-server/src/main/resources/config/install_config.conf
View File

@ -118,7 +118,7 @@ apiServerPort="12345"
# install hosts # install hosts
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname # Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="ark0,ark1,ark2,ark3,ark4" ips="ds1,ds2,ds3,ds4,ds5"
# ssh port, default 22 # ssh port, default 22
# Note: if ssh port is not default, modify here # Note: if ssh port is not default, modify here
@ -126,19 +126,19 @@ sshPort="22"
# run master machine # run master machine
# Note: list of hosts hostname for deploying master # Note: list of hosts hostname for deploying master
masters="ark0,ark1" masters="ds1,ds2"
# run worker machine # run worker machine
# note: list of machine hostnames for deploying workers # note: need to write the worker group name of each worker, the default value is "default"
workers="ark2,ark3,ark4" workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default")
# run alert machine # run alert machine
# note: list of machine hostnames for deploying alert server # note: list of machine hostnames for deploying alert server
alertServer="ark3" alertServer="ds3"
# run api machine # run api machine
# note: list of machine hostnames for deploying api server # note: list of machine hostnames for deploying api server
apiServers="ark1" apiServers="ds1"
# whether to start monitoring self-starting scripts # whether to start monitoring self-starting scripts
monitorServerState="false" monitorServerState="false"

4
dolphinscheduler-server/src/main/resources/worker.properties
View File

@ -21,7 +21,7 @@
# worker heartbeat interval # worker heartbeat interval
#worker.heartbeat.interval=10 #worker.heartbeat.interval=10
# submit the number of tasks at a time # submit the number of tasks at a time TODO
#worker.fetch.task.num = 3 #worker.fetch.task.num = 3
# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 # only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
@ -34,4 +34,4 @@
#worker.listen.port: 1234 #worker.listen.port: 1234
# default worker group # default worker group
#worker.group=default worker.group=default

11
script/scp-hosts.sh
View File

@ -20,6 +20,12 @@ workDir=`dirname $0`
workDir=`cd ${workDir};pwd` workDir=`cd ${workDir};pwd`
source $workDir/../conf/config/install_config.conf source $workDir/../conf/config/install_config.conf
txt=""
if [[ "$OSTYPE" == "darwin"* ]]; then
# Mac OSX
txt="''"
fi
hostsArr=(${ips//,/ }) hostsArr=(${ips//,/ })
for host in ${hostsArr[@]} for host in ${hostsArr[@]}
do do
@ -33,6 +39,11 @@ do
for dsDir in bin conf lib script sql ui install.sh for dsDir in bin conf lib script sql ui install.sh
do do
# if worker in workersGroup
if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then
sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties
fi
echo "start to scp $dsDir to $host/$installPath" echo "start to scp $dsDir to $host/$installPath"
scp -P $sshPort -r $workDir/../$dsDir $host:$installPath scp -P $sshPort -r $workDir/../$dsDir $host:$installPath
done done

3
script/start-all.sh
View File

@ -28,8 +28,7 @@ do
done done
workersHost=(${workers//,/ }) for worker in ${!workersGroup[*]}
for worker in ${workersHost[@]}
do do
echo "$worker worker server is starting" echo "$worker worker server is starting"

3
script/stop-all.sh
View File

@ -29,8 +29,7 @@ do
done done
workersHost=(${workers//,/ }) for worker in ${!workersGroup[*]}
for worker in ${workersHost[@]}
do do
echo "$worker worker server is stopping" echo "$worker worker server is stopping"
ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"