connection pair DolphinScheduler connect to the metadata database
-dbhost="192.168.xx.xx:3306"
-
-# The username DolphinScheduler connect to the metadata database
-username="xx"
-
-# The password DolphinScheduler connect to the metadata database
-password="xx"
-
-# The database DolphinScheduler connect to the metadata database
-dbname="dolphinscheduler"
-
-# ---------------------------------------------------------
-# Registry Server
-# ---------------------------------------------------------
-# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
-registryPluginName="zookeeper"
-
-# Registry Server address.
-registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
-
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
zkRoot="/dolphinscheduler"
-
-# ---------------------------------------------------------
-# Worker Task Server
-# ---------------------------------------------------------
-# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
-taskPluginDir="lib/plugin/task"
-
-# resource storage type: HDFS, S3, NONE
-resourceStorageType="NONE"
-
-# resource store on HDFS/S3 path, resource file will store to this hdfs path, self configuration, please make sure the directory exists on hdfs and has read write permissions. "/dolphinscheduler" is recommended
-resourceUploadPath="/dolphinscheduler"
-
-# if resourceStorageType is HDFS，defaultFS write namenode address，HA, you need to put core-site.xml and hdfs-site.xml in the conf directory.
-# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
-# Note，S3 be sure to create the root directory /dolphinscheduler
-defaultFS="hdfs://mycluster:8020"
-
-# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore
-s3Endpoint="http://192.168.xx.xx:9010"
-s3AccessKey="xxxxxxxxxx"
-s3SecretKey="xxxxxxxxxx"
-
-# resourcemanager port, the default value is 8088 if not specified
-resourceManagerHttpAddressPort="8088"
-
-# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single node, keep this value empty
-yarnHaIps="192.168.xx.xx,192.168.xx.xx"
-
-# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single node, you only need to replace 'yarnIp1' to actual resourcemanager hostname
-singleYarnIp="yarnIp1"
-
-# who has permission to create directory under HDFS/S3 root path
-# Note: if kerberos is enabled, please config hdfsRootUser=
-hdfsRootUser="hdfs"
-
-# kerberos config
-# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore
-kerberosStartUp="false"
-# kdc krb5 config file path
-krb5ConfPath="$installPath/conf/krb5.conf"
-# keytab username,watch out the @ sign should followd by \\
-keytabUserName="hdfs-mycluster\\@ESZ.COM"
-# username keytab path
-keytabPath="$installPath/conf/hdfs.headless.keytab"
-# kerberos expire time, the unit is hour
-kerberosExpireTime="2"
-
-# use sudo or not
-sudoEnable="true"
-
-# worker tenant auto create
-workerTenantAutoCreate="false"
diff --git a/dolphinscheduler-server/src/main/resources/logback-master.xml b/dolphinscheduler-server/src/main/resources/logback-master.xml
index a61d891b10..d5d2ae3481 100644
--- a/dolphinscheduler-server/src/main/resources/logback-master.xml
+++ b/dolphinscheduler-server/src/main/resources/logback-master.xml
@@ -32,9 +32,6 @@
-
taskAppId
@@ -53,12 +50,8 @@
-
${log.base}/dolphinscheduler-master.log
-
${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log
168
@@ -71,11 +64,10 @@
UTF-8
-
-
\ No newline at end of file
+
diff --git a/dolphinscheduler-server/src/main/resources/logback-worker.xml b/dolphinscheduler-server/src/main/resources/logback-worker.xml
index 31719d5b53..1c45037e41 100644
--- a/dolphinscheduler-server/src/main/resources/logback-worker.xml
+++ b/dolphinscheduler-server/src/main/resources/logback-worker.xml
@@ -29,13 +29,9 @@
-
-
taskAppId
@@ -56,9 +52,6 @@
${log.base}/dolphinscheduler-worker.log
-
${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log
168
@@ -66,16 +59,15 @@
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
UTF-8
-
-
\ No newline at end of file
+
diff --git a/dolphinscheduler-server/src/main/resources/master.properties b/dolphinscheduler-server/src/main/resources/master.properties
deleted file mode 100644
index fbe8f6e2a6..0000000000
--- a/dolphinscheduler-server/src/main/resources/master.properties
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# master listen port
-#master.listen.port=5678
-
-# master execute thread number to limit process instances in parallel
-#master.exec.threads=100
-
-# master execute task number in parallel per process instance
-#master.exec.task.num=20
-
-# master dispatch task number per batch
-#master.dispatch.task.num=3
-
-# master host selector to select a suitable worker, default value: LowerWeight. Optional values include Random, RoundRobin, LowerWeight
-#master.host.selector=LowerWeight
-
-# master heartbeat interval, the unit is second
-#master.heartbeat.interval=10
-
-# master commit task retry times
-#master.task.commit.retryTimes=5
-
-# master commit task interval, the unit is millisecond
-#master.task.commit.interval=1000
-
-# master cache process definition, default: true
-#master.cache.process.definition=true
-
-# master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
-#master.max.cpuload.avg=-1
-
-# master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
-#master.reserved.memory=0.3
diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties
deleted file mode 100644
index 2c37e5309b..0000000000
--- a/dolphinscheduler-server/src/main/resources/worker.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# worker listener port
-#worker.listen.port=1234
-
-# worker execute thread number to limit task instances in parallel
-#worker.exec.threads=100
-
-# worker heartbeat interval, the unit is second
-#worker.heartbeat.interval=10
-
-# worker host weight to dispatch tasks, default value 100
-#worker.host.weight=100
-
-# worker tenant auto create
-#worker.tenant.auto.create=false
-
-# worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2
-#worker.max.cpuload.avg=-1
-
-# worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
-#worker.reserved.memory=0.3
-
-# default worker groups separated by comma, like 'worker.groups=default,test'
-#worker.groups=default
-
-# alert server listen host
-#alert.listen.host=localhost
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
index 3b3b5855ce..c8a4ae4545 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
@@ -65,8 +65,8 @@ public class ConditionsTaskTest {
MasterConfig config = new MasterConfig();
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
- config.setMasterTaskCommitRetryTimes(3);
- config.setMasterTaskCommitInterval(1000);
+ config.setTaskCommitRetryTimes(3);
+ config.setTaskCommitInterval(1000);
processService = Mockito.mock(ProcessService.class);
Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
index 9d33331997..1fa37cc7f5 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
@@ -85,8 +85,8 @@ public class DependentTaskTest {
springApplicationContext.setApplicationContext(applicationContext);
MasterConfig config = new MasterConfig();
- config.setMasterTaskCommitRetryTimes(3);
- config.setMasterTaskCommitInterval(1000);
+ config.setTaskCommitRetryTimes(3);
+ config.setTaskCommitInterval(1000);
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
processService = Mockito.mock(ProcessService.class);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
index 5b19664950..5fcfc77bcc 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
@@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -62,8 +61,8 @@ public class SubProcessTaskTest {
MasterConfig config = new MasterConfig();
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
- config.setMasterTaskCommitRetryTimes(3);
- config.setMasterTaskCommitInterval(1000);
+ config.setTaskCommitRetryTimes(3);
+ config.setTaskCommitInterval(1000);
PowerMockito.mockStatic(Stopper.class);
PowerMockito.when(Stopper.isRunning()).thenReturn(true);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
index 61f1d6d800..3221edb9aa 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
@@ -58,8 +58,8 @@ public class SwitchTaskTest {
MasterConfig config = new MasterConfig();
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
- config.setMasterTaskCommitRetryTimes(3);
- config.setMasterTaskCommitInterval(1000);
+ config.setTaskCommitRetryTimes(3);
+ config.setTaskCommitInterval(1000);
processService = Mockito.mock(ProcessService.class);
Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
@@ -153,4 +153,4 @@ public class SwitchTaskTest {
taskInstance.setTaskDefinitionVersion(1);
return taskInstance;
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
index 6e2dda06ef..911d0d7a4b 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
@@ -22,7 +22,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_D
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
-import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.powermock.api.mockito.PowerMockito.mock;
@@ -89,7 +88,7 @@ public class WorkflowExecuteThreadTest {
applicationContext = mock(ApplicationContext.class);
config = new MasterConfig();
- config.setMasterExecTaskNum(1);
+ config.setExecTaskNum(1);
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
processInstance = mock(ProcessInstance.class);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java
index 43d2469275..5459b9ad21 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java
@@ -21,11 +21,13 @@ import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.test.context.ContextConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+@ActiveProfiles("master")
@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(classes = {MasterConfig.class})
+@SpringBootTest(classes = MasterConfig.class)
public class MasterConfigTest {
@Autowired
@@ -33,7 +35,7 @@ public class MasterConfigTest {
@Test
public void getMasterDispatchTaskNumber() {
- int masterDispatchTaskNumber = masterConfig.getMasterDispatchTaskNumber();
- Assert.assertEquals(6, masterDispatchTaskNumber);
+ int masterDispatchTaskNumber = masterConfig.getDispatchTaskNumber();
+ Assert.assertEquals(3, masterDispatchTaskNumber);
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
index daee652c39..33234d9fe6 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
@@ -84,7 +84,7 @@ public class TaskExecuteProcessorTest {
// init task execution context
taskExecutionContext = getTaskExecutionContext();
workerConfig = new WorkerConfig();
- workerConfig.setWorkerExecThreads(1);
+ workerConfig.setExecThreads(1);
workerConfig.setListenPort(1234);
command = new Command();
command.setType(CommandType.TASK_EXECUTE_REQUEST);
@@ -120,7 +120,7 @@ public class TaskExecuteProcessorTest {
.thenReturn(workerManager);
PowerMockito.mockStatic(ThreadUtils.class);
- PowerMockito.when(ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", workerConfig.getWorkerExecThreads()))
+ PowerMockito.when(ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", workerConfig.getExecThreads()))
.thenReturn(workerExecService);
PowerMockito.mockStatic(JsonSerializer.class);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
index d12cefb6f6..245107a786 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
@@ -71,7 +71,7 @@ public class WorkerRegistryClientTest {
@Before
public void before() {
- given(workerConfig.getWorkerGroups()).willReturn(Sets.newHashSet("127.0.0.1"));
+ given(workerConfig.getGroups()).willReturn(Sets.newHashSet("127.0.0.1"));
//given(heartBeatExecutor.getWorkerGroups()).willReturn(Sets.newHashSet("127.0.0.1"));
//scheduleAtFixedRate
given(heartBeatExecutor.scheduleAtFixedRate(Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(TimeUnit.class))).willReturn(null);
diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml
index 5040a2e9b2..50b8736198 100644
--- a/dolphinscheduler-service/pom.xml
+++ b/dolphinscheduler-service/pom.xml
@@ -47,25 +47,28 @@
dolphinscheduler-registry-zookeeper
+
+ org.springframework.boot
+ spring-boot-starter-quartz
+
org.quartz-scheduler
quartz
-
- com.mchange
- c3p0
-
-
- com.mchange
- mchange-commons-java
-
-
- com.zaxxer
- HikariCP-java6
-
+
+ com.mchange
+ c3p0
+
+
+ com.mchange
+ mchange-commons-java
+
+
+ com.zaxxer
+ HikariCP-java6
+
-
org.quartz-scheduler
quartz-jobs
@@ -80,17 +83,27 @@
org.powermock
powermock-api-mockito2
test
-
-
- org.mockito
- mockito-core
-
-
+
- org.mockito
- mockito-core
- test
+ io.micrometer
+ micrometer-core
+ provided
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ *.yaml
+ *.properties
+
+
+
+
+
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/HikariConnectionProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/HikariConnectionProvider.java
deleted file mode 100644
index ec064538a1..0000000000
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/HikariConnectionProvider.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.service.quartz;
-
-import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-
-import org.quartz.utils.ConnectionProvider;
-
-import com.zaxxer.hikari.HikariDataSource;
-
-/**
- * hikari connection provider
- */
-public class HikariConnectionProvider implements ConnectionProvider {
-
- private final HikariDataSource dataSource;
-
- public HikariConnectionProvider() {
- this.dataSource = SpringApplicationContext.getBean(HikariDataSource.class);
- }
-
- @Override
- public Connection getConnection() throws SQLException {
- return dataSource.getConnection();
- }
-
- @Override
- public void shutdown() {
- dataSource.close();
- }
-
- @Override
- public void initialize() throws SQLException {
- //NOP
- }
-}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
index 3c9f165e47..2f811c3bf2 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
@@ -23,45 +23,33 @@ import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Schedule;
-import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.Date;
-import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
+import org.quartz.JobKey;
+import org.quartz.Scheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.util.Assert;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.quartz.QuartzJobBean;
import org.springframework.util.StringUtils;
-/**
- * process schedule job
- */
-public class ProcessScheduleJob implements Job {
+import io.micrometer.core.annotation.Counted;
+import io.micrometer.core.annotation.Timed;
- /**
- * logger of ProcessScheduleJob
- */
+public class ProcessScheduleJob extends QuartzJobBean {
private static final Logger logger = LoggerFactory.getLogger(ProcessScheduleJob.class);
- public ProcessService getProcessService() {
- return SpringApplicationContext.getBean(ProcessService.class);
- }
+ @Autowired
+ private ProcessService processService;
- /**
- * Called by the Scheduler when a Trigger fires that is associated with the Job
- *
- * @param context JobExecutionContext
- * @throws JobExecutionException if there is an exception while executing the job.
- */
+ @Counted(value = "quartz_job_executed")
+ @Timed(value = "quartz_job_execution", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
@Override
- public void execute(JobExecutionContext context) throws JobExecutionException {
-
- Assert.notNull(getProcessService(), "please call init() method first");
-
+ protected void executeInternal(JobExecutionContext context) {
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
int projectId = dataMap.getInt(Constants.PROJECT_ID);
@@ -74,14 +62,14 @@ public class ProcessScheduleJob implements Job {
logger.info("scheduled fire time :{}, fire time :{}, process id :{}", scheduledFireTime, fireTime, scheduleId);
// query schedule
- Schedule schedule = getProcessService().querySchedule(scheduleId);
+ Schedule schedule = processService.querySchedule(scheduleId);
if (schedule == null || ReleaseState.OFFLINE == schedule.getReleaseState()) {
logger.warn("process schedule does not exist in db or process schedule offline，delete schedule job in quartz, projectId:{}, scheduleId:{}", projectId, scheduleId);
- deleteJob(projectId, scheduleId);
+ deleteJob(context, projectId, scheduleId);
return;
}
- ProcessDefinition processDefinition = getProcessService().findProcessDefinitionByCode(schedule.getProcessDefinitionCode());
+ ProcessDefinition processDefinition = processService.findProcessDefinitionByCode(schedule.getProcessDefinitionCode());
// release state : online/offline
ReleaseState releaseState = processDefinition.getReleaseState();
if (releaseState == ReleaseState.OFFLINE) {
@@ -103,15 +91,22 @@ public class ProcessScheduleJob implements Job {
command.setProcessInstancePriority(schedule.getProcessInstancePriority());
command.setProcessDefinitionVersion(processDefinition.getVersion());
- getProcessService().createCommand(command);
+ processService.createCommand(command);
}
- /**
- * delete job
- */
- private void deleteJob(int projectId, int scheduleId) {
+ private void deleteJob(JobExecutionContext context, int projectId, int scheduleId) {
+ final Scheduler scheduler = context.getScheduler();
String jobName = QuartzExecutors.buildJobName(scheduleId);
String jobGroupName = QuartzExecutors.buildJobGroupName(projectId);
- QuartzExecutors.getInstance().deleteJob(jobName, jobGroupName);
+
+ JobKey jobKey = new JobKey(jobName, jobGroupName);
+ try {
+ if (scheduler.checkExists(jobKey)) {
+ logger.info("Try to delete job: {}, group name: {},", jobName, jobGroupName);
+ scheduler.deleteJob(jobKey);
+ }
+ } catch (Exception e) {
+ logger.error("Failed to delete job: {}", jobKey);
+ }
}
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
index 9f49dd7939..bca3060206 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
@@ -17,42 +17,11 @@
package org.apache.dolphinscheduler.service.quartz;
-import static org.apache.dolphinscheduler.common.Constants.ORG_POSTGRESQL_DRIVER;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_CLASS;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_DATASOURCE;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_ISCLUSTERED;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_TABLEPREFIX;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_USEPROPERTIES;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_SCHEDULER_INSTANCEID;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_SCHEDULER_INSTANCENAME;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_CLASS;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_THREADCOUNT;
-import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_THREADPRIORITY;
import static org.apache.dolphinscheduler.common.Constants.PROJECT_ID;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_ACQUIRETRIGGERSWITHINLOCK;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_CLUSTERCHECKININTERVAL;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_DATASOURCE;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_INSTANCEID;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_INSTANCENAME;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_JOB_GROUP_PRIFIX;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_JOB_PRIFIX;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_MISFIRETHRESHOLD;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_PROPERTIES_PATH;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_TABLE_PREFIX;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_THREADCOUNT;
-import static org.apache.dolphinscheduler.common.Constants.QUARTZ_THREADPRIORITY;
import static org.apache.dolphinscheduler.common.Constants.SCHEDULE;
import static org.apache.dolphinscheduler.common.Constants.SCHEDULE_ID;
-import static org.apache.dolphinscheduler.common.Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME;
-import static org.apache.dolphinscheduler.common.Constants.STRING_FALSE;
-import static org.apache.dolphinscheduler.common.Constants.STRING_TRUE;
import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
import static org.quartz.CronScheduleBuilder.cronSchedule;
@@ -61,21 +30,14 @@ import static org.quartz.TriggerBuilder.newTrigger;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringUtils;
-import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import java.util.Properties;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -84,137 +46,26 @@ import org.quartz.Job;
import org.quartz.JobDetail;
import org.quartz.JobKey;
import org.quartz.Scheduler;
-import org.quartz.SchedulerException;
import org.quartz.TriggerKey;
-import org.quartz.impl.StdSchedulerFactory;
-import org.quartz.impl.jdbcjobstore.JobStoreTX;
-import org.quartz.impl.jdbcjobstore.PostgreSQLDelegate;
-import org.quartz.impl.jdbcjobstore.StdJDBCDelegate;
-import org.quartz.impl.matchers.GroupMatcher;
-import org.quartz.simpl.SimpleThreadPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * single Quartz executors instance
- */
public class QuartzExecutors {
-
- /**
- * logger of QuartzExecutors
- */
private static final Logger logger = LoggerFactory.getLogger(QuartzExecutors.class);
- /**
- * read write lock
- */
private final ReadWriteLock lock = new ReentrantReadWriteLock();
- /**
- * A Scheduler maintains a registry of org.quartz.JobDetail and Trigger.
- */
- private static Scheduler scheduler;
-
- /**
- * load conf
- */
- private static Configuration conf;
-
private static final class Holder {
private static final QuartzExecutors instance = new QuartzExecutors();
}
private QuartzExecutors() {
- try {
- conf = new PropertiesConfiguration(QUARTZ_PROPERTIES_PATH);
- init();
- } catch (ConfigurationException e) {
- logger.warn("not loaded quartz configuration file, will used default value", e);
- }
}
- /**
- * thread safe and performance promote
- *
- * @return instance of Quartz Executors
- */
public static QuartzExecutors getInstance() {
return Holder.instance;
}
- /**
- * init
- *
- * Returns a client-usable handle to a Scheduler.
- */
- private void init() {
- try {
- StdSchedulerFactory schedulerFactory = new StdSchedulerFactory();
- Properties properties = new Properties();
-
- String dataSourceDriverClass = PropertyUtils.getString(SPRING_DATASOURCE_DRIVER_CLASS_NAME);
- if (dataSourceDriverClass.equals(ORG_POSTGRESQL_DRIVER)) {
- properties.setProperty(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, conf.getString(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, PostgreSQLDelegate.class.getName()));
- } else {
- properties.setProperty(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, conf.getString(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, StdJDBCDelegate.class.getName()));
- }
- properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCENAME, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCENAME, QUARTZ_INSTANCENAME));
- properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCEID, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCEID, QUARTZ_INSTANCEID));
- properties.setProperty(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON, conf.getString(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON, STRING_TRUE));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_USEPROPERTIES, conf.getString(ORG_QUARTZ_JOBSTORE_USEPROPERTIES, STRING_FALSE));
- properties.setProperty(ORG_QUARTZ_THREADPOOL_CLASS, conf.getString(ORG_QUARTZ_THREADPOOL_CLASS, SimpleThreadPool.class.getName()));
- properties.setProperty(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS, conf.getString(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS, STRING_TRUE));
- properties.setProperty(ORG_QUARTZ_THREADPOOL_THREADCOUNT, conf.getString(ORG_QUARTZ_THREADPOOL_THREADCOUNT, QUARTZ_THREADCOUNT));
- properties.setProperty(ORG_QUARTZ_THREADPOOL_THREADPRIORITY, conf.getString(ORG_QUARTZ_THREADPOOL_THREADPRIORITY, QUARTZ_THREADPRIORITY));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_CLASS, conf.getString(ORG_QUARTZ_JOBSTORE_CLASS, JobStoreTX.class.getName()));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_TABLEPREFIX, conf.getString(ORG_QUARTZ_JOBSTORE_TABLEPREFIX, QUARTZ_TABLE_PREFIX));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_ISCLUSTERED, conf.getString(ORG_QUARTZ_JOBSTORE_ISCLUSTERED, STRING_TRUE));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD, conf.getString(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD, QUARTZ_MISFIRETHRESHOLD));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL, conf.getString(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL, QUARTZ_CLUSTERCHECKININTERVAL));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK, conf.getString(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK, QUARTZ_ACQUIRETRIGGERSWITHINLOCK));
- properties.setProperty(ORG_QUARTZ_JOBSTORE_DATASOURCE, conf.getString(ORG_QUARTZ_JOBSTORE_DATASOURCE, QUARTZ_DATASOURCE));
- properties.setProperty(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS, conf.getString(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS, HikariConnectionProvider.class.getName()));
-
- schedulerFactory.initialize(properties);
- scheduler = schedulerFactory.getScheduler();
-
- } catch (SchedulerException e) {
- logger.error(e.getMessage(), e);
- System.exit(1);
- }
-
- }
-
- /**
- * Whether the scheduler has been started.
- *
- * @throws SchedulerException scheduler exception
- */
- public void start() throws SchedulerException {
- if (!scheduler.isStarted()) {
- scheduler.start();
- logger.info("Quartz service started");
- }
- }
-
- /**
- * stop all scheduled tasks
- *
- * Halts the Scheduler's firing of Triggers,
- * and cleans up all resources associated with the Scheduler.
- *
- * The scheduler cannot be re-started.
- *
- * @throws SchedulerException scheduler exception
- */
- public void shutdown() throws SchedulerException {
- if (!scheduler.isShutdown()) {
- // don't wait for the task to complete
- scheduler.shutdown();
- logger.info("Quartz service stopped, and halt all tasks");
- }
- }
-
/**
* add task trigger , if this task already exists, return this task with updated trigger
*
@@ -222,7 +73,7 @@ public class QuartzExecutors {
* @param projectId projectId
* @param schedule schedule
*/
- public void addJob(Class clazz, int projectId, final Schedule schedule) {
+ public void addJob(Scheduler scheduler, Class clazz, int projectId, final Schedule schedule) {
String jobName = QuartzExecutors.buildJobName(schedule.getId());
String jobGroupName = QuartzExecutors.buildJobGroupName(projectId);
Date startDate = schedule.getStartTime();
@@ -253,7 +104,7 @@ public class QuartzExecutors {
}
TriggerKey triggerKey = new TriggerKey(jobName, jobGroupName);
- /**
+ /*
* Instructs the Scheduler that upon a mis-fire
* situation, the CronTrigger wants to have it's
* next-fire-time updated to the next time in the schedule after the
@@ -295,86 +146,14 @@ public class QuartzExecutors {
}
}
- /**
- * delete job
- *
- * @param jobName job name
- * @param jobGroupName job group name
- * @return true if the Job was found and deleted.
- */
- public boolean deleteJob(String jobName, String jobGroupName) {
- lock.writeLock().lock();
- try {
- JobKey jobKey = new JobKey(jobName, jobGroupName);
- if (scheduler.checkExists(jobKey)) {
- logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName);
- return scheduler.deleteJob(jobKey);
- } else {
- return true;
- }
-
- } catch (SchedulerException e) {
- logger.error("delete job : {} failed", jobName, e);
- } finally {
- lock.writeLock().unlock();
- }
- return false;
- }
-
- /**
- * delete all jobs in job group
- *
- * @param jobGroupName job group name
- * @return true if all of the Jobs were found and deleted, false if
- * one or more were not deleted.
- */
- public boolean deleteAllJobs(String jobGroupName) {
- lock.writeLock().lock();
- try {
- logger.info("try to delete all jobs in job group: {}", jobGroupName);
- List jobKeys = new ArrayList<>();
- jobKeys.addAll(scheduler.getJobKeys(GroupMatcher.groupEndsWith(jobGroupName)));
-
- return scheduler.deleteJobs(jobKeys);
- } catch (SchedulerException e) {
- logger.error("delete all jobs in job group: {} failed", jobGroupName, e);
- } finally {
- lock.writeLock().unlock();
- }
- return false;
- }
-
- /**
- * build job name
- *
- * @param processId process id
- * @return job name
- */
public static String buildJobName(int processId) {
- StringBuilder sb = new StringBuilder(30);
- sb.append(QUARTZ_JOB_PRIFIX).append(UNDERLINE).append(processId);
- return sb.toString();
+ return QUARTZ_JOB_PRIFIX + UNDERLINE + processId;
}
- /**
- * build job group name
- *
- * @param projectId project id
- * @return job group name
- */
public static String buildJobGroupName(int projectId) {
- StringBuilder sb = new StringBuilder(30);
- sb.append(QUARTZ_JOB_GROUP_PRIFIX).append(UNDERLINE).append(projectId);
- return sb.toString();
+ return QUARTZ_JOB_GROUP_PRIFIX + UNDERLINE + projectId;
}
- /**
- * add params to map
- *
- * @param projectId project id
- * @param schedule schedule
- * @return data map
- */
public static Map buildDataMap(int projectId, Schedule schedule) {
Map dataMap = new HashMap<>(8);
dataMap.put(PROJECT_ID, projectId);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/SchedulerConfiguration.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/SchedulerConfiguration.java
new file mode 100644
index 0000000000..d2c1ffd223
--- /dev/null
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/SchedulerConfiguration.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.service.quartz;
+
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.quartz.impl.jdbcjobstore.PostgreSQLDelegate;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.scheduling.quartz.SchedulerFactoryBean;
+import org.springframework.scheduling.quartz.SpringBeanJobFactory;
+
+@Configuration
+public class SchedulerConfiguration {
+ @Value("${spring.datasource.driver-class-name}")
+ private String dataSourceDriverClass;
+
+ @Bean
+ public SchedulerFactoryBean schedulerFactory(DataSource dataSource) {
+ SchedulerFactoryBean schedulerFactory = new SchedulerFactoryBean();
+ schedulerFactory.setConfigLocation(new ClassPathResource("quartz.properties"));
+ if (dataSourceDriverClass.equals(org.postgresql.Driver.class.getName())) {
+ final Properties overrideProperties = new Properties();
+ overrideProperties.setProperty("org.quartz.jobStore.driverDelegateClass", PostgreSQLDelegate.class.getName());
+ schedulerFactory.setQuartzProperties(overrideProperties);
+ }
+ schedulerFactory.setDataSource(dataSource);
+ schedulerFactory.setJobFactory(new SpringBeanJobFactory());
+ return schedulerFactory;
+ }
+}
diff --git a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml
deleted file mode 100644
index 9bdf18f8cd..0000000000
--- a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
- ${log.base}/dolphinscheduler-zookeeper.log
-
- ${log.base}/dolphinscheduler-zookeeper.%d{yyyy-MM-dd_HH}.%i.log
- 20
- 64MB
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
-
-
-
-
-
diff --git a/dolphinscheduler-service/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties
index 5420755fe7..52d535dde3 100644
--- a/dolphinscheduler-service/src/main/resources/quartz.properties
+++ b/dolphinscheduler-service/src/main/resources/quartz.properties
@@ -15,40 +15,18 @@
# limitations under the License.
#
-#============================================================================
-# Configure Main Scheduler Properties
-#============================================================================
-#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
-
-#org.quartz.scheduler.instanceName = DolphinScheduler
-#org.quartz.scheduler.instanceId = AUTO
-#org.quartz.scheduler.makeSchedulerThreadDaemon = true
-#org.quartz.jobStore.useProperties = false
-
-#============================================================================
-# Configure ThreadPool
-#============================================================================
-
-#org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
-#org.quartz.threadPool.makeThreadsDaemons = true
-#org.quartz.threadPool.threadCount = 25
-#org.quartz.threadPool.threadPriority = 5
-
-#============================================================================
-# Configure JobStore
-#============================================================================
-
-#org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
-
-#org.quartz.jobStore.tablePrefix = QRTZ_
-#org.quartz.jobStore.isClustered = true
-#org.quartz.jobStore.misfireThreshold = 60000
-#org.quartz.jobStore.clusterCheckinInterval = 5000
-#org.quartz.jobStore.acquireTriggersWithinLock=true
-#org.quartz.jobStore.dataSource = myDs
-
-#============================================================================
-# Configure Datasources
-#============================================================================
-#org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.HikariConnectionProvider
+org.quartz.threadPool.threadPriority=5
+org.quartz.jobStore.isClustered=true
+org.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX
+org.quartz.scheduler.instanceId=AUTO
+org.quartz.jobStore.tablePrefix=QRTZ_
+org.quartz.jobStore.acquireTriggersWithinLock=true
+org.quartz.scheduler.instanceName=DolphinScheduler
+org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool
+org.quartz.jobStore.useProperties=false
+org.quartz.threadPool.makeThreadsDaemons=true
+org.quartz.threadPool.threadCount=25
+org.quartz.jobStore.misfireThreshold=60000
+org.quartz.scheduler.makeSchedulerThreadDaemon=true
+org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate
+org.quartz.jobStore.clusterCheckinInterval=5000
diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml
index f3bca7c190..36b6b25fe5 100644
--- a/dolphinscheduler-spi/pom.xml
+++ b/dolphinscheduler-spi/pom.xml
@@ -26,41 +26,23 @@
${project.artifactId}
-
com.fasterxml.jackson.core
jackson-annotations
-
com.fasterxml.jackson.core
jackson-databind
-
- com.fasterxml.jackson.core
- jackson-core
-
-
org.apache.commons
commons-collections4
-
- commons-beanutils
- commons-beanutils
- provided
-
-
- commons-codec
- commons-codec
- provided
-
org.slf4j
slf4j-api
- provided
com.baomidou
@@ -68,12 +50,6 @@
${mybatis-plus.version}
provided
-
-
- junit
- junit
- test
-
com.google.guava
guava
@@ -84,11 +60,5 @@
-
- org.ow2.asm
- asm
- provided
-
-
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java
index 7fa7a7bcb0..b1df15c397 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java
@@ -23,19 +23,6 @@ import java.util.Map;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
-/**
- * The base model of connection param
- *
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickhouseConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2ConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkConnectionParam}
- * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerConnectionParam}
- */
@JsonInclude(Include.NON_NULL)
public abstract class BaseConnectionParam implements ConnectionParam {
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractParameters.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractParameters.java
index 591e5a630b..0f0c7a3cb6 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractParameters.java
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractParameters.java
@@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.spi.task;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java
index b646b326c4..08ca581b3e 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java
@@ -150,29 +150,6 @@ public class PropertyUtils {
return defaultValue;
}
- /**
- * get property long value
- *
- * @param key key
- * @param defaultVal default value
- * @return property value
- */
- public static long getLong(String key, long defaultVal) {
- String val = getString(key);
- return val == null ? defaultVal : Long.parseLong(val);
- }
-
- /**
- * @param key key
- * @return property value
- */
- public static long getLong(String key) {
- return getLong(key, -1);
- }
-
- /**
- *
- */
public static void setValue(String key, String value) {
properties.setProperty(key, value);
}
diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml
index c47bcfdc3d..c27e9cefa7 100644
--- a/dolphinscheduler-standalone-server/pom.xml
+++ b/dolphinscheduler-standalone-server/pom.xml
@@ -36,6 +36,11 @@
org.apache.dolphinscheduler
dolphinscheduler-api
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-meter
+
+
org.apache.curator
curator-test
@@ -57,4 +62,17 @@
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ *.yaml
+
+
+
+
+
diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
index 3dfde101d6..830cd42edf 100644
--- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
+++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
@@ -24,12 +24,8 @@ import org.apache.dolphinscheduler.server.worker.WorkerServer;
import org.apache.curator.test.TestingServer;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.annotation.ComponentScan;
-@EnableAutoConfiguration
-@ComponentScan
public class StandaloneServer {
public static void main(String[] args) throws Exception {
final TestingServer server = new TestingServer(true);
@@ -41,6 +37,6 @@ public class StandaloneServer {
WorkerServer.class,
AlertServer.class,
PythonGatewayServer.class
- ).profiles("h2", "api").run(args);
+ ).profiles("master", "worker", "api", "alert", "h2", "standalone").run(args);
}
}
diff --git a/dolphinscheduler-server/src/main/resources/META-INF/services/org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessFactory b/dolphinscheduler-standalone-server/src/main/resources/application-standalone.yaml
similarity index 66%
rename from dolphinscheduler-server/src/main/resources/META-INF/services/org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessFactory
rename to dolphinscheduler-standalone-server/src/main/resources/application-standalone.yaml
index 95bc81431e..5c68f88cb8 100644
--- a/dolphinscheduler-server/src/main/resources/META-INF/services/org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessFactory
+++ b/dolphinscheduler-standalone-server/src/main/resources/application-standalone.yaml
@@ -15,8 +15,6 @@
# limitations under the License.
#
-org.apache.dolphinscheduler.server.master.runner.task.CommonTaskProcessFactory
-org.apache.dolphinscheduler.server.master.runner.task.ConditionTaskProcessFactory
-org.apache.dolphinscheduler.server.master.runner.task.DependentTaskProcessFactory
-org.apache.dolphinscheduler.server.master.runner.task.SubTaskProcessFactory
-org.apache.dolphinscheduler.server.master.runner.task.SwitchTaskProcessFactory
+spring:
+ application:
+ name: standalone-server
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml
index 2848b4de09..d38f2f439d 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml
@@ -90,11 +90,5 @@
org.powermock
powermock-api-mockito2
-
-
- junit
- junit
- test
-
diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml
index d32ed2e38f..24c1a31c39 100644
--- a/dolphinscheduler-task-plugin/pom.xml
+++ b/dolphinscheduler-task-plugin/pom.xml
@@ -42,12 +42,4 @@
dolphinscheduler-task-procedure
dolphinscheduler-task-pigeon
-
-
-
- com.google.auto.service
- auto-service
- true
-
-
diff --git a/install.sh b/install.sh
index 67e5e5680e..af43b27f94 100755
--- a/install.sh
+++ b/install.sh
@@ -21,72 +21,16 @@ workDir=`cd ${workDir};pwd`
source ${workDir}/conf/config/install_config.conf
-# 1.replace file
-echo "1.replace file"
-
-txt=""
-if [[ "$OSTYPE" == "darwin"* ]]; then
- # Mac OSX
- txt="''"
-fi
-
-datasourceDriverClassname="com.mysql.jdbc.Driver"
-if [ $dbtype == "postgresql" ];then
- datasourceDriverClassname="org.postgresql.Driver"
-fi
-
-# Change configuration in conf/config/dolphinscheduler_env.sh
-sed -i ${txt} "s@^export JAVA_HOME=.*@export JAVA_HOME=${javaHome}@g" conf/env/dolphinscheduler_env.sh
-
-# Change configuration in conf/datasource.properties
-sed -i ${txt} "s@^spring.datasource.driver-class-name=.*@spring.datasource.driver-class-name=${datasourceDriverClassname}@g" conf/datasource.properties
-sed -i ${txt} "s@^spring.datasource.url=.*@spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true@g" conf/datasource.properties
-sed -i ${txt} "s@^spring.datasource.username=.*@spring.datasource.username=${username}@g" conf/datasource.properties
-sed -i ${txt} "s@^spring.datasource.password=.*@spring.datasource.password=${password}@g" conf/datasource.properties
-
-# Change configuration in conf/common.properties
-sed -i ${txt} "s@^data.basedir.path=.*@data.basedir.path=${dataBasedirPath}@g" conf/common.properties
-sed -i ${txt} "s@^resource.storage.type=.*@resource.storage.type=${resourceStorageType}@g" conf/common.properties
-sed -i ${txt} "s@^resource.upload.path=.*@resource.upload.path=${resourceUploadPath}@g" conf/common.properties
-sed -i ${txt} "s@^hadoop.security.authentication.startup.state=.*@hadoop.security.authentication.startup.state=${kerberosStartUp}@g" conf/common.properties
-sed -i ${txt} "s@^java.security.krb5.conf.path=.*@java.security.krb5.conf.path=${krb5ConfPath}@g" conf/common.properties
-sed -i ${txt} "s@^login.user.keytab.username=.*@login.user.keytab.username=${keytabUserName}@g" conf/common.properties
-sed -i ${txt} "s@^login.user.keytab.path=.*@login.user.keytab.path=${keytabPath}@g" conf/common.properties
-sed -i ${txt} "s@^kerberos.expire.time=.*@kerberos.expire.time=${kerberosExpireTime}@g" conf/common.properties
-sed -i ${txt} "s@^hdfs.root.user=.*@hdfs.root.user=${hdfsRootUser}@g" conf/common.properties
-sed -i ${txt} "s@^fs.defaultFS=.*@fs.defaultFS=${defaultFS}@g" conf/common.properties
-sed -i ${txt} "s@^fs.s3a.endpoint=.*@fs.s3a.endpoint=${s3Endpoint}@g" conf/common.properties
-sed -i ${txt} "s@^fs.s3a.access.key=.*@fs.s3a.access.key=${s3AccessKey}@g" conf/common.properties
-sed -i ${txt} "s@^fs.s3a.secret.key=.*@fs.s3a.secret.key=${s3SecretKey}@g" conf/common.properties
-sed -i ${txt} "s@^resource.manager.httpaddress.port=.*@resource.manager.httpaddress.port=${resourceManagerHttpAddressPort}@g" conf/common.properties
-sed -i ${txt} "s@^yarn.resourcemanager.ha.rm.ids=.*@yarn.resourcemanager.ha.rm.ids=${yarnHaIps}@g" conf/common.properties
-sed -i ${txt} "s@^yarn.application.status.address=.*@yarn.application.status.address=http://${singleYarnIp}:%s/ws/v1/cluster/apps/%s@g" conf/common.properties
-sed -i ${txt} "s@^yarn.job.history.status.address=.*@yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s@g" conf/common.properties
-sed -i ${txt} "s@^sudo.enable=.*@sudo.enable=${sudoEnable}@g" conf/common.properties
-
-# The following configurations may be commented, so ddd #* to ensure sed work correct
-# Change configuration in conf/worker.properties
-sed -i ${txt} "s@^#*worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
-sed -i ${txt} "s@^#*alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
-
-# Change configuration in conf/application-api.properties
-sed -i ${txt} "s@^#*server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
-
-# Change configuration in conf/registry.properties
-sed -i ${txt} "s@^#*registry.plugin.dir=.*@registry.plugin.dir=${installPath}/${registryPluginDir}@g" conf/registry.properties
-sed -i ${txt} "s@^#*registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
-sed -i ${txt} "s@^#*registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
-
-# 2.create directory
-echo "2.create directory"
+# 1.create directory
+echo "1.create directory"
if [ ! -d $installPath ];then
sudo mkdir -p $installPath
sudo chown -R $deployUser:$deployUser $installPath
fi
-# 3.scp resources
-echo "3.scp resources"
+# 2.scp resources
+echo "2.scp resources"
sh ${workDir}/script/scp-hosts.sh
if [ $? -eq 0 ]
then
@@ -97,17 +41,17 @@ else
fi
-# 4.stop server
-echo "4.stop server"
+# 3.stop server
+echo "3.stop server"
sh ${workDir}/script/stop-all.sh
-# 5.delete zk node
-echo "5.delete zk node"
+# 4.delete zk node
+echo "4.delete zk node"
sh ${workDir}/script/remove-zk-node.sh $zkRoot
-# 6.startup
-echo "6.startup"
+# 5.startup
+echo "5.startup"
sh ${workDir}/script/start-all.sh
diff --git a/pom.xml b/pom.xml
index b6ae286c29..463629b3e1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,18 +23,11 @@
2.0.0-SNAPSHOT
pom
${project.artifactId}
- http://dolphinscheduler.apache.org
+ https://dolphinscheduler.apache.org
Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated
to solving the complex dependencies in data processing, making the scheduling system out of the box for data
processing.
-
-
- Apache License 2.0
- http://www.apache.org/licenses/LICENSE-2.0.txt
- repo
-
-
scm:git:https://github.com/apache/dolphinscheduler.git
scm:git:https://github.com/apache/dolphinscheduler.git
@@ -60,8 +53,8 @@
UTF-8
4.3.0
3.4.14
- 5.1.19.RELEASE
- 2.1.18.RELEASE
+ 5.3.12
+ 2.5.6
1.8
1.2.3
2.7.3
@@ -128,10 +121,33 @@
0.10.9
1.0.1
false
+ 4.1.53.Final
+ 3.2.0
+
+ io.netty
+ netty-bom
+ ${netty.version}
+ import
+ pom
+
+
+ org.springframework.boot
+ spring-boot-starter-parent
+ ${spring.boot.version}
+ pom
+ import
+
+
+
+ io.netty
+ netty-all
+ ${netty.version}
+
+
org.java-websocket
Java-WebSocket
@@ -170,14 +186,6 @@
${druid.version}
-
- org.springframework.boot
- spring-boot-starter-parent
- ${spring.boot.version}
- pom
- import
-
-
org.springframework
spring-core
@@ -254,6 +262,11 @@
dolphinscheduler-service
${project.version}
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-meter
+ ${project.version}
+
org.apache.dolphinscheduler
dolphinscheduler-spi
@@ -528,13 +541,11 @@
-
io.protostuff
protostuff-core
${protostuff.version}
-
io.protostuff
protostuff-runtime
@@ -562,7 +573,6 @@
org.mockito
mockito-core
${mockito.version}
- jar
test
@@ -570,7 +580,6 @@
org.powermock
powermock-module-junit4
${powermock.version}
- jar
test
@@ -578,14 +587,7 @@
org.powermock
powermock-api-mockito2
${powermock.version}
- jar
test
-
-
- org.mockito
- mockito-core
-
-
@@ -651,7 +653,6 @@
${logback.version}
-
org.apache.commons
commons-email
@@ -684,6 +685,10 @@
com.sun.jersey
jersey-json
+
+ junit
+ junit
+
@@ -761,13 +766,6 @@
${presto.jdbc.version}
-
- net.jcip
- jcip-annotations
- ${jcip.version}
- true
-
-
javax.servlet
servlet-api
@@ -832,27 +830,12 @@
py4j
${py4j.version}
-
-
- com.google.auto.service
- auto-service
- ${auto-service.version}
- true
-
- apache-dolphinscheduler-${project.version}
-
- org.apache.dolphinscheduler
- dolphinscheduler-maven-plugin
- 1.0.0
- true
-
-
ca.vanzyl.maven.plugins
provisio-maven-plugin
@@ -919,24 +902,16 @@
org.apache.maven.plugins
maven-dependency-plugin
${maven-dependency-plugin.version}
-
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ ${maven-jar-plugin.version}
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-maven-plugin
- true
-
-
-
-
-
-
-
ca.vanzyl.maven.plugins
provisio-maven-plugin
@@ -983,7 +958,6 @@
true
@{project.version}
${project.version}
-
@@ -994,7 +968,6 @@
-
org.apache.maven.plugins
maven-compiler-plugin
@@ -1116,10 +1089,20 @@
-
+
+
+ junit
+ junit
+ test
+
org.jacoco
org.jacoco.agent
@@ -1127,6 +1110,17 @@
runtime
test
+
+ org.springframework.boot
+ spring-boot-configuration-processor
+ true
+
+
+ com.google.auto.service
+ auto-service
+ ${auto-service.version}
+ provided
+
@@ -1146,5 +1140,6 @@
dolphinscheduler-standalone-server
dolphinscheduler-datasource-plugin
dolphinscheduler-python
+ dolphinscheduler-meter
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index e8074bfd4b..1220e904e8 100755
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -38,8 +38,6 @@ DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
source /etc/profile
source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
-export JAVA_HOME=$JAVA_HOME
-#export JAVA_HOME=/opt/soft/jdk
export HOSTNAME=`hostname`
export DOLPHINSCHEDULER_PID_DIR=$DOLPHINSCHEDULER_HOME/pid
@@ -61,36 +59,43 @@ cd $DOLPHINSCHEDULER_HOME
export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS"
+export dbtype=${dbtype:-"h2"}
+export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
+if [ "$METER_ENABLED" = "true" ]; then
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},meter"
+fi
+
if [ "$command" = "api-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-api.xml"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},api,${dbtype}"
elif [ "$command" = "master-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-master.xml"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},master,${dbtype}"
elif [ "$command" = "worker-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-worker.xml"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},worker,${dbtype}"
elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},alert,${dbtype}"
elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS"
-elif [ "$command" = "zookeeper-server" ]; then
- #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server
- LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml"
- CLASS=org.apache.dolphinscheduler.service.zk.ZKServer
elif [ "$command" = "standalone-server" ]; then
CLASS=org.apache.dolphinscheduler.server.StandaloneServer
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},standalone,${dbtype}"
else
echo "Error: No command named '$command' was found."
exit 1
diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt
index db3ec486ac..0aad1c054b 100755
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -1,22 +1,23 @@
-HikariCP-3.2.0.jar
+HdrHistogram-2.1.12.jar
+HikariCP-4.0.3.jar
+LatencyUtils-2.0.3.jar
activation-1.1.jar
animal-sniffer-annotations-1.14.jar
aopalliance-1.0.jar
-apache-el-8.5.54.jar
apacheds-i18n-2.0.0-M15.jar
apacheds-kerberos-codec-2.0.0-M15.jar
api-asn1-api-1.0.0-M20.jar
api-util-1.0.0-M20.jar
asm-3.1.jar
asm-6.2.1.jar
-aspectjweaver-1.9.6.jar
+aspectjweaver-1.9.7.jar
audience-annotations-0.5.0.jar
avro-1.7.4.jar
aws-java-sdk-1.7.4.jar
bonecp-0.8.0.RELEASE.jar
byte-buddy-1.9.16.jar
checker-compat-qual-2.0.0.jar
-classmate-1.4.0.jar
+classmate-1.5.1.jar
clickhouse-jdbc-0.1.52.jar
commons-email-1.5.jar
commons-cli-1.2.jar
@@ -24,7 +25,6 @@ commons-codec-1.11.jar
commons-collections-3.2.2.jar
commons-collections4-4.1.jar
commons-compress-1.4.1.jar
-commons-compiler-3.0.16.jar
commons-configuration-1.10.jar
commons-daemon-1.0.13.jar
commons-beanutils-1.9.4.jar
@@ -48,7 +48,7 @@ datanucleus-rdbms-4.1.7.jar
derby-10.14.2.0.jar
error_prone_annotations-2.1.3.jar
druid-1.2.4.jar
-gson-2.8.6.jar
+gson-2.8.8.jar
guava-24.1-jre.jar
guava-retrying-2.0.0.jar
guice-3.0.jar
@@ -69,8 +69,6 @@ hadoop-yarn-api-2.7.3.jar
hadoop-yarn-client-2.7.3.jar
hadoop-yarn-common-2.7.3.jar
hadoop-yarn-server-common-2.7.3.jar
-hamcrest-core-1.3.jar
-hibernate-validator-6.0.21.Final.jar
hive-common-2.1.0.jar
hive-jdbc-2.1.0.jar
hive-metastore-2.1.0.jar
@@ -88,26 +86,26 @@ jackson-annotations-2.10.5.jar
jackson-core-2.10.5.jar
jackson-core-asl-1.9.13.jar
jackson-databind-2.10.5.jar
-jackson-datatype-jdk8-2.9.10.jar
-jackson-datatype-jsr310-2.9.10.jar
+jackson-datatype-jdk8-2.12.5.jar
+jackson-datatype-jsr310-2.12.5.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-parameter-names-2.9.10.jar
+jackson-module-parameter-names-2.12.5.jar
jackson-xc-1.9.13.jar
+jakarta.annotation-api-1.3.5.jar
+jakarta.servlet-api-4.0.4.jar
+jakarta.websocket-api-1.1.2.jar
jamon-runtime-2.3.1.jar
-janino-3.0.16.jar
java-xmlbuilder-0.4.jar
javassist-3.26.0-GA.jar
-javax.activation-api-1.2.0.jar
javax.annotation-api-1.3.2.jar
+javax.activation-api-1.2.0.jar
javax.inject-1.jar
javax.jdo-3.2.0-m3.jar
javax.mail-1.6.2.jar
-javax.servlet-api-3.1.0.jar
javolution-5.5.1.jar
jaxb-api-2.3.1.jar
jaxb-impl-2.2.3-1.jar
-jboss-logging-3.3.3.Final.jar
jdo-api-3.0.1.jar
jersey-client-1.9.jar
jersey-core-1.9.jar
@@ -117,38 +115,40 @@ jersey-server-1.9.jar
jets3t-0.9.0.jar
jettison-1.1.jar
jetty-6.1.26.jar
-jetty-continuation-9.4.33.v20201020.jar
-jetty-http-9.4.33.v20201020.jar
-jetty-io-9.4.33.v20201020.jar
-jetty-security-9.4.33.v20201020.jar
-jetty-server-9.4.33.v20201020.jar
-jetty-servlet-9.4.33.v20201020.jar
-jetty-servlets-9.4.33.v20201020.jar
+jetty-continuation-9.4.44.v20210927.jar
+jetty-http-9.4.44.v20210927.jar
+jetty-io-9.4.44.v20210927.jar
+jetty-security-9.4.44.v20210927.jar
+jetty-server-9.4.44.v20210927.jar
+jetty-servlet-9.4.44.v20210927.jar
+jetty-servlets-9.4.44.v20210927.jar
jetty-util-6.1.26.jar
-jetty-util-9.4.33.v20201020.jar
-jetty-webapp-9.4.33.v20201020.jar
-jetty-xml-9.4.33.v20201020.jar
+jetty-util-9.4.44.v20210927.jar
+jetty-util-ajax-9.4.44.v20210927.jar
+jetty-webapp-9.4.44.v20210927.jar
+jetty-xml-9.4.44.v20210927.jar
jline-0.9.94.jar
jna-4.5.2.jar
jna-platform-4.5.2.jar
-joda-time-2.10.8.jar
+joda-time-2.5.jar
jpam-1.1.jar
jsch-0.1.42.jar
jsp-api-2.1.jar
jsqlparser-2.1.jar
jsr305-3.0.0.jar
jta-1.1.jar
-jul-to-slf4j-1.7.30.jar
-junit-4.12.jar
+jul-to-slf4j-1.7.32.jar
leveldbjni-all-1.8.jar
libfb303-0.9.3.jar
libthrift-0.9.3.jar
-log4j-1.2-api-2.11.2.jar
+log4j-1.2-api-2.14.1.jar
log4j-1.2.17.jar
logback-classic-1.2.3.jar
logback-core-1.2.3.jar
lz4-1.3.0.jar
mapstruct-1.2.0.Final.jar
+micrometer-core-1.7.5.jar
+micrometer-registry-prometheus-1.7.5.jar
mssql-jdbc-6.1.0.jre8.jar
mybatis-3.5.2.jar
mybatis-plus-3.2.0.jar
@@ -177,32 +177,40 @@ py4j-0.10.9.jar
quartz-2.3.0.jar
quartz-jobs-2.3.0.jar
reflections-0.9.12.jar
+simpleclient-0.10.0.jar
+simpleclient_common-0.10.0.jar
slf4j-api-1.7.5.jar
-snakeyaml-1.23.jar
+snakeyaml-1.28.jar
snappy-0.2.jar
snappy-java-1.0.4.1.jar
SparseBitSet-1.2.jar
-spring-aop-5.1.19.RELEASE.jar
-spring-beans-5.1.19.RELEASE.jar
-spring-boot-2.1.18.RELEASE.jar
-spring-boot-autoconfigure-2.1.18.RELEASE.jar
-spring-boot-starter-2.1.18.RELEASE.jar
-spring-boot-starter-aop-2.1.18.RELEASE.jar
-spring-boot-starter-jdbc-2.1.18.RELEASE.jar
-spring-boot-starter-jetty-2.1.18.RELEASE.jar
-spring-boot-starter-json-2.1.18.RELEASE.jar
-spring-boot-starter-logging-2.1.18.RELEASE.jar
-spring-boot-starter-web-2.1.18.RELEASE.jar
-spring-context-5.1.19.RELEASE.jar
-spring-core-5.1.19.RELEASE.jar
-spring-expression-5.1.19.RELEASE.jar
-spring-jcl-5.1.19.RELEASE.jar
-spring-jdbc-5.1.19.RELEASE.jar
+spring-aop-5.3.12.jar
+spring-beans-5.3.12.jar
+spring-boot-2.5.6.jar
+spring-boot-actuator-2.5.6.jar
+spring-boot-actuator-autoconfigure-2.5.6.jar
+spring-boot-autoconfigure-2.5.6.jar
+spring-boot-configuration-processor-2.5.6.jar
+spring-boot-starter-2.5.6.jar
+spring-boot-starter-actuator-2.5.6.jar
+spring-boot-starter-aop-2.5.6.jar
+spring-boot-starter-jdbc-2.5.6.jar
+spring-boot-starter-jetty-2.5.6.jar
+spring-boot-starter-json-2.5.6.jar
+spring-boot-starter-logging-2.5.6.jar
+spring-boot-starter-quartz-2.5.6.jar
+spring-boot-starter-web-2.5.6.jar
+spring-context-5.3.12.jar
+spring-context-support-5.3.12.jar
+spring-core-5.3.12.jar
+spring-expression-5.3.12.jar
+spring-jcl-5.3.12.jar
+spring-jdbc-5.3.12.jar
spring-plugin-core-1.2.0.RELEASE.jar
spring-plugin-metadata-1.2.0.RELEASE.jar
-spring-tx-5.1.19.RELEASE.jar
-spring-web-5.1.19.RELEASE.jar
-spring-webmvc-5.1.19.RELEASE.jar
+spring-tx-5.3.12.jar
+spring-web-5.3.12.jar
+spring-webmvc-5.3.12.jar
springfox-core-2.9.2.jar
springfox-schema-2.9.2.jar
springfox-spi-2.9.2.jar
@@ -213,11 +221,11 @@ springfox-swagger2-2.9.2.jar
swagger-annotations-1.5.20.jar
swagger-bootstrap-ui-1.9.3.jar
swagger-models-1.5.24.jar
+tomcat-embed-el-9.0.54.jar
tephra-api-0.6.0.jar
transaction-api-1.1.jar
-validation-api-2.0.1.Final.jar
xercesImpl-2.9.1.jar
-xml-apis-1.4.01.jar
+xml-apis-1.3.04.jar
xmlbeans-3.1.0.jar
xmlenc-0.52.jar
xz-1.0.jar