test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-01 19:58:29 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

Optimize server startup log (#15362)

Browse Source
This commit is contained in:
Wenjun Ruan 2023-12-27 20:20:21 +08:00 committed by GitHub
parent 5523a62825
commit d6dea4633c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 104 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java
View File

@ -18,7 +18,7 @@
package org.apache.dolphinscheduler.alert.rpc;
import org.apache.dolphinscheduler.alert.config.AlertConfig;
import org.apache.dolphinscheduler.extract.base.NettyRemotingServer;
import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory;
import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig;
import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery;
@ -31,7 +31,8 @@ import org.springframework.stereotype.Service;
public class AlertRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable {
public AlertRpcServer(AlertConfig alertConfig) {
super(new NettyRemotingServer(new NettyServerConfig(alertConfig.getPort())));
super(NettyRemotingServerFactory.buildNettyRemotingServer(
NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build()));
}
public void start() {

4
dolphinscheduler-api/src/main/resources/logback-spring.xml
View File

@ -23,7 +23,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@ -40,7 +40,7 @@
</rollingPolicy>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java
View File

@ -91,7 +91,7 @@ public class LoggerServiceTest {
@BeforeEach
public void setUp() {
nettyRemotingServer = new NettyRemotingServer(new NettyServerConfig(8080));
nettyRemotingServer = new NettyRemotingServer(NettyServerConfig.builder().listenPort(8080).build());
nettyRemotingServer.start();
SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery =
new SpringServerMethodInvokerDiscovery(nettyRemotingServer);

114
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
View File

@ -35,9 +35,6 @@ public final class Constants {
*/
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
public static final String REGISTRY_DOLPHINSCHEDULER_MASTERS = "/nodes/master";
public static final String REGISTRY_DOLPHINSCHEDULER_WORKERS = "/nodes/worker";
public static final String FORMAT_SS = "%s%s";
public static final String FORMAT_S_S = "%s/%s";
public static final String FORMAT_S_S_COLON = "%s:%s";
@ -191,11 +188,6 @@ public final class Constants {
*/
public static final String DOUBLE_SLASH = "//";
/**
* EQUAL SIGN
*/
public static final String EQUAL_SIGN = "=";
/**
* AT SIGN
*/
@ -235,11 +227,6 @@ public final class Constants {
*/
public static final int SOCKET_TIMEOUT = 60 * 1000;
/**
* registry session timeout
*/
public static final int REGISTRY_SESSION_TIMEOUT = 10 * 1000;
/**
* http header
*/
@ -300,22 +287,6 @@ public final class Constants {
*/
public static final int MAX_TASK_TIMEOUT = 24 * 3600;
/**
* worker host weight
*/
public static final int DEFAULT_WORKER_HOST_WEIGHT = 100;
/**
* unit convertor for minute to second
*/
public static final int MINUTE_2_SECOND_TIME_UNIT = 60;
/***
*
* rpc port
*/
public static final String RPC_PORT = "rpc.port";
/**
* forbid running task
*/
@ -356,21 +327,6 @@ public final class Constants {
public static final Duration SERVER_CLOSE_WAIT_TIME = Duration.ofSeconds(3);
/**
* one second mils
*/
public static final long SECOND_TIME_MILLIS = 1_000L;
/**
* master task instance cache-database refresh interval
*/
public static final long CACHE_REFRESH_TIME_MILLIS = 20 * 1_000L;
/**
* heartbeat for zk info length
*/
public static final int HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH = 14;
/**
* jar
*/
@ -408,39 +364,10 @@ public final class Constants {
*/
public static final int VERSION_FIRST = 1;
/**
* ACCEPTED
*/
public static final String ACCEPTED = "ACCEPTED";
/**
* SUCCEEDED
*/
public static final String SUCCEEDED = "SUCCEEDED";
/**
* ENDED
*/
public static final String ENDED = "ENDED";
/**
* NEW
*/
public static final String NEW = "NEW";
/**
* NEW_SAVING
*/
public static final String NEW_SAVING = "NEW_SAVING";
/**
* SUBMITTED
*/
public static final String SUBMITTED = "SUBMITTED";
/**
* FAILED
*/
public static final String FAILED = "FAILED";
/**
* KILLED
*/
public static final String KILLED = "KILLED";
/**
* RUNNING
*/
@ -449,25 +376,11 @@ public final class Constants {
* underline "_"
*/
public static final String UNDERLINE = "_";
/**
* application regex
*/
public static final String APPLICATION_REGEX = "application_\\d+_\\d+";
public static final String PID = SystemUtils.IS_OS_WINDOWS ? "handle" : "pid";
public static final char SUBTRACT_CHAR = '-';
public static final char ADD_CHAR = '+';
public static final char MULTIPLY_CHAR = '*';
public static final char DIVISION_CHAR = '/';
public static final char LEFT_BRACE_CHAR = '(';
public static final char RIGHT_BRACE_CHAR = ')';
public static final String ADD_STRING = "+";
public static final String STAR = "*";
public static final String DIVISION_STRING = "/";
public static final String LEFT_BRACE_STRING = "(";
public static final char P = 'P';
public static final char N = 'N';
public static final String SUBTRACT_STRING = "-";
public static final String GLOBAL_PARAMS = "globalParams";
public static final String LOCAL_PARAMS = "localParams";
public static final String SUBPROCESS_INSTANCE_ID = "subProcessInstanceId";
@ -482,9 +395,6 @@ public final class Constants {
public static final String QUEUE_NAME = "queueName";
public static final int LOG_QUERY_SKIP_LINE_NUMBER = 0;
public static final int LOG_QUERY_LIMIT = 4096;
public static final String BLOCKING_CONDITION = "blockingCondition";
public static final String ALERT_WHEN_BLOCKING = "alertWhenBlocking";
public static final String ALIAS = "alias";
public static final String CONTENT = "content";
public static final String DEPENDENT_SPLIT = ":||";
@ -527,11 +437,6 @@ public final class Constants {
public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE =
"hadoop.security.authentication.startup.state";
/**
* com.amazonaws.services.s3.enableV4
*/
public static final String AWS_S3_V4 = "com.amazonaws.services.s3.enableV4";
/**
* loginUserFromKeytab user
*/
@ -550,11 +455,6 @@ public final class Constants {
public static final String WORKFLOW_INSTANCE_ID_MDC_KEY = "workflowInstanceId";
public static final String TASK_INSTANCE_ID_MDC_KEY = "taskInstanceId";
/**
* task log info format
*/
public static final String TASK_LOG_INFO_FORMAT = "TaskLogInfo-%s";
/**
* double brackets left
*/
@ -647,10 +547,6 @@ public final class Constants {
* authorize writable perm
*/
public static final int AUTHORIZE_WRITABLE_PERM = 7;
/**
* authorize readable perm
*/
public static final int AUTHORIZE_READABLE_PERM = 4;
public static final String START_TIME = "start time";
public static final String END_TIME = "end time";
@ -682,8 +578,6 @@ public final class Constants {
*/
public static final String DATA_QUALITY_ERROR_OUTPUT_PATH = "data-quality.error.output.path";
public static final String CACHE_KEY_VALUE_ALL = "'all'";
/**
* use for k8s
*/
@ -784,12 +678,6 @@ public final class Constants {
*/
public static final String SUPPORT_HIVE_ONE_SESSION = "support.hive.oneSession";
public static final String PRINCIPAL = "principal";
public static final String ORACLE_DB_CONNECT_TYPE = "connectType";
public static final String KERBEROS_KRB5_CONF_PATH = "javaSecurityKrb5Conf";
public static final String KERBEROS_KEY_TAB_USERNAME = "loginUserKeytabUsername";
public static final String KERBEROS_KEY_TAB_PATH = "loginUserKeytabPath";
public static final Integer QUERY_ALL_ON_SYSTEM = 0;
public static final Integer QUERY_ALL_ON_PROJECT = 1;
public static final Integer QUERY_ALL_ON_WORKFLOW = 2;

16
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
View File

@ -41,9 +41,6 @@ import lombok.extern.slf4j.Slf4j;
import com.google.common.collect.Sets;
/**
* NetUtils
*/
@Slf4j
public class NetUtils {
@ -213,15 +210,17 @@ public class NetUtils {
.collect(Collectors.toList());
// Use the specified network interface if set
if (StringUtils.isNotBlank(specifyNetworkInterfaceName())) {
String specifyNetworkInterfaceName = specifyNetworkInterfaceName();
String specifiedNetworkInterfaceName = specifyNetworkInterfaceName();
if (StringUtils.isNotBlank(specifiedNetworkInterfaceName)) {
validNetworkInterfaces = validNetworkInterfaces.stream()
.filter(networkInterface -> specifyNetworkInterfaceName.equals(networkInterface.getDisplayName()))
.filter(networkInterface -> specifiedNetworkInterfaceName.equals(networkInterface.getDisplayName()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(validNetworkInterfaces)) {
throw new IllegalArgumentException(
"The specified network interface: " + specifyNetworkInterfaceName + " is not found");
"The specified network interface: " + specifiedNetworkInterfaceName + " is not found");
}
log.info("Use the specified network interface: {} -> {}", specifiedNetworkInterfaceName,
validNetworkInterfaces);
}
Set<String> restrictNetworkInterfaceName = restrictNetworkInterfaceName();
@ -307,9 +306,10 @@ public class NetUtils {
Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
while (interfaces.hasMoreElements()) {
NetworkInterface networkInterface = interfaces.nextElement();
log.info("Found NetworkInterface: {}", networkInterface);
log.debug("Found NetworkInterface: {}", networkInterface);
validNetworkInterfaces.add(networkInterface);
}
log.info("Get all NetworkInterfaces: {}", validNetworkInterfaces);
return validNetworkInterfaces;
}

4
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
View File

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.entity;
import static org.apache.dolphinscheduler.common.constants.Constants.MINUTE_2_SECOND_TIME_UNIT;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
@ -38,6 +37,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import java.io.Serializable;
import java.util.Date;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.Data;
@ -407,7 +407,7 @@ public class TaskInstance implements Serializable {
Date now = new Date();
long failedTimeInterval = DateUtils.differSec(now, getEndTime());
// task retry does not over time, return false
return getRetryInterval() * MINUTE_2_SECOND_TIME_UNIT < failedTimeInterval;
return TimeUnit.MINUTES.toSeconds(getRetryInterval()) < failedTimeInterval;
}
}

25
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java
View File

@ -73,9 +73,11 @@ public class NettyRemotingServer {
public NettyRemotingServer(final NettyServerConfig serverConfig) {
this.serverConfig = serverConfig;
ThreadFactory bossThreadFactory =
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerBossThread_%s").build();
new ThreadFactoryBuilder().setDaemon(true).setNameFormat(serverConfig.getServerName() + "BossThread_%s")
.build();
ThreadFactory workerThreadFactory =
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerWorkerThread_%s").build();
new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat(serverConfig.getServerName() + "WorkerThread_%s").build();
if (Epoll.isAvailable()) {
this.bossGroup = new EpollEventLoopGroup(1, bossThreadFactory);
this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), workerThreadFactory);
@ -108,16 +110,23 @@ public class NettyRemotingServer {
try {
future = serverBootstrap.bind(serverConfig.getListenPort()).sync();
} catch (Exception e) {
log.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e);
throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()));
log.error("{} bind fail {}, exit", serverConfig.getServerName(), e.getMessage(), e);
throw new RemoteException(
String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()));
}
if (future.isSuccess()) {
log.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort());
} else if (future.cause() != null) {
throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()),
log.info("{} bind success at port: {}", serverConfig.getServerName(), serverConfig.getListenPort());
return;
}
if (future.cause() != null) {
throw new RemoteException(
String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()),
future.cause());
} else {
throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()));
throw new RemoteException(
String.format("%s bind %s fail", serverConfig.getServerName(), serverConfig.getListenPort()));
}
}
}

13
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
View File

@ -17,11 +17,19 @@
package org.apache.dolphinscheduler.extract.base.config;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class NettyServerConfig {
private String serverName;
/**
* init the server connectable queue
*/
@ -55,9 +63,6 @@ public class NettyServerConfig {
/**
* listen port
*/
private int listenPort = 12346;
private int listenPort;
public NettyServerConfig(int listenPort) {
this.listenPort = listenPort;
}
}

3
dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactoryTest.java
View File

@ -34,7 +34,8 @@ public class SingletonJdkDynamicRpcClientProxyFactoryTest {
@BeforeEach
public void setUp() {
nettyRemotingServer = new NettyRemotingServer(new NettyServerConfig(12345));
nettyRemotingServer =
new NettyRemotingServer(NettyServerConfig.builder().serverName("ApiServer").listenPort(12345).build());
nettyRemotingServer.start();
new SpringServerMethodInvokerDiscovery(nettyRemotingServer)

4
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java
View File

@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.server.master.builder;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.dolphinscheduler.common.constants.Constants.MINUTE_2_SECOND_TIME_UNIT;
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
import org.apache.dolphinscheduler.common.utils.DateUtils;
@ -34,6 +33,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@ -84,7 +84,7 @@ public class TaskExecutionContextBuilder {
if (taskDefinition.getTimeoutNotifyStrategy() == TaskTimeoutStrategy.FAILED
|| taskDefinition.getTimeoutNotifyStrategy() == TaskTimeoutStrategy.WARNFAILED) {
taskExecutionContext.setTaskTimeout(
Math.min(taskDefinition.getTimeout() * MINUTE_2_SECOND_TIME_UNIT, Integer.MAX_VALUE));
(int) Math.min(TimeUnit.MINUTES.toSeconds(taskDefinition.getTimeout()), Integer.MAX_VALUE));
}
}
taskExecutionContext.setTaskParams(taskDefinition.getTaskParams());

40
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
View File

@ -163,23 +163,27 @@ public class MasterConfig implements Validator {
}
private void printConfig() {
log.info("Master config: listenPort -> {} ", listenPort);
log.info("Master config: fetchCommandNum -> {} ", fetchCommandNum);
log.info("Master config: preExecThreads -> {} ", preExecThreads);
log.info("Master config: execThreads -> {} ", execThreads);
log.info("Master config: dispatchTaskNumber -> {} ", dispatchTaskNumber);
log.info("Master config: hostSelector -> {} ", hostSelector);
log.info("Master config: heartbeatInterval -> {} ", heartbeatInterval);
log.info("Master config: taskCommitRetryTimes -> {} ", taskCommitRetryTimes);
log.info("Master config: taskCommitInterval -> {} ", taskCommitInterval);
log.info("Master config: stateWheelInterval -> {} ", stateWheelInterval);
log.info("Master config: maxCpuLoadAvg -> {} ", maxCpuLoadAvg);
log.info("Master config: reservedMemory -> {} ", reservedMemory);
log.info("Master config: failoverInterval -> {} ", failoverInterval);
log.info("Master config: killApplicationWhenTaskFailover -> {} ", killApplicationWhenTaskFailover);
log.info("Master config: registryDisconnectStrategy -> {} ", registryDisconnectStrategy);
log.info("Master config: masterAddress -> {} ", masterAddress);
log.info("Master config: masterRegistryPath -> {} ", masterRegistryPath);
log.info("Master config: workerGroupRefreshInterval -> {} ", workerGroupRefreshInterval);
String config =
"\n****************************Master Configuration**************************************" +
"\n listen-port -> " + listenPort +
"\n fetch-command-num -> " + fetchCommandNum +
"\n pre-exec-threads -> " + preExecThreads +
"\n exec-threads -> " + execThreads +
"\n dispatch-task-number -> " + dispatchTaskNumber +
"\n host-selector -> " + hostSelector +
"\n heartbeat-interval -> " + heartbeatInterval +
"\n task-commit-retry-times -> " + taskCommitRetryTimes +
"\n task-commit-interval -> " + taskCommitInterval +
"\n state-wheel-interval -> " + stateWheelInterval +
"\n max-cpu-load-avg -> " + maxCpuLoadAvg +
"\n reserved-memory -> " + reservedMemory +
"\n failover-interval -> " + failoverInterval +
"\n kill-application-when-task-failover -> " + killApplicationWhenTaskFailover +
"\n registry-disconnect-strategy -> " + registryDisconnectStrategy +
"\n master-address -> " + masterAddress +
"\n master-registry-path: " + masterRegistryPath +
"\n worker-group-refresh-interval: " + workerGroupRefreshInterval +
"\n****************************Master Configuration**************************************";
log.info(config);
}
}

3
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java
View File

@ -31,7 +31,8 @@ import org.springframework.stereotype.Component;
public class MasterRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable {
public MasterRpcServer(MasterConfig masterConfig) {
super(NettyRemotingServerFactory.buildNettyRemotingServer(new NettyServerConfig(masterConfig.getListenPort())));
super(NettyRemotingServerFactory.buildNettyRemotingServer(NettyServerConfig.builder()
.serverName("MasterRpcServer").listenPort(masterConfig.getListenPort()).build()));
}
public void start() {

8
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
View File

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.server.master.runner;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.StateEventType;
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
@ -37,6 +36,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
import java.util.Optional;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
@ -147,8 +147,7 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
continue;
}
long timeRemain = DateUtils.getRemainTime(processInstance.getStartTime(),
(long) processInstance.getTimeout()
* Constants.MINUTE_2_SECOND_TIME_UNIT);
TimeUnit.MINUTES.toSeconds(processInstance.getTimeout()));
if (timeRemain < 0) {
log.info("Workflow instance {} timeout, adding timeout event", processInstance.getId());
addProcessTimeoutEvent(processInstance);
@ -247,8 +246,7 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
TaskInstance taskInstance = taskInstanceOptional.get();
if (TimeoutFlag.OPEN == taskInstance.getTaskDefine().getTimeoutFlag()) {
long timeRemain = DateUtils.getRemainTime(taskInstance.getStartTime(),
(long) taskInstance.getTaskDefine().getTimeout()
* Constants.MINUTE_2_SECOND_TIME_UNIT);
TimeUnit.MINUTES.toSeconds(taskInstance.getTaskDefine().getTimeout()));
if (timeRemain < 0) {
log.info("Task instance is timeout, adding task timeout event and remove the check");
addTaskTimeoutEvent(taskInstance);

4
dolphinscheduler-master/src/main/resources/logback-spring.xml
View File

@ -23,7 +23,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@ -61,7 +61,7 @@
</rollingPolicy>
<encoder>
<pattern>
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>

3
dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java
View File

@ -52,7 +52,8 @@ public class RpcBenchMarkTest extends AbstractBaseBenchmark {
@Setup
public void before() {
nettyRemotingServer = new NettyRemotingServer(new NettyServerConfig(12345));
nettyRemotingServer = new NettyRemotingServer(
NettyServerConfig.builder().serverName("NettyRemotingServer").listenPort(12345).build());
nettyRemotingServer.start();
SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery =
new SpringServerMethodInvokerDiscovery(nettyRemotingServer);

4
dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
View File

@ -23,7 +23,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@ -38,7 +38,7 @@
</rollingPolicy>
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>

72
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java
View File

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.constants.DateConstants;
import java.time.Duration;
import java.util.Set;
import java.util.regex.Pattern;
import com.google.common.collect.Sets;
@ -35,11 +34,6 @@ public class TaskConstants {
public static final String FLINK_APPLICATION_REGEX = "JobID \\w+";
/**
* string false
*/
public static final String STRING_FALSE = "false";
/**
* exit code kill
*/
@ -146,11 +140,6 @@ public class TaskConstants {
public static final String RWXR_XR_X = "rwxr-xr-x";
/**
* date format of yyyyMMdd
*/
public static final String PARAMETER_FORMAT_DATE = "yyyyMMdd";
/**
* date format of yyyyMMddHHmmss
*/
@ -291,22 +280,9 @@ public class TaskConstants {
public static final char P = 'P';
public static final char N = 'N';
public static final String SUBTRACT_STRING = "-";
public static final String GLOBAL_PARAMS = "globalParams";
public static final String LOCAL_PARAMS = "localParams";
public static final String LOCAL_PARAMS_LIST = "localParamsList";
public static final String SUBPROCESS_INSTANCE_ID = "subProcessInstanceId";
public static final String PROCESS_INSTANCE_STATE = "processInstanceState";
public static final String PARENT_WORKFLOW_INSTANCE = "parentWorkflowInstance";
public static final String CONDITION_RESULT = "conditionResult";
public static final String SWITCH_RESULT = "switchResult";
public static final String DEPENDENCE = "dependence";
public static final String TASK_TYPE = "taskType";
public static final String TASK_LIST = "taskList";
public static final String QUEUE = "queue";
public static final String QUEUE_NAME = "queueName";
public static final int LOG_QUERY_SKIP_LINE_NUMBER = 0;
public static final int LOG_QUERY_LIMIT = 4096;
/**
* default display rows
*/
@ -327,33 +303,6 @@ public class TaskConstants {
*/
public static final String D = "-D";
/**
* jdbc url
*/
public static final String JDBC_MYSQL = "jdbc:mysql://";
public static final String JDBC_POSTGRESQL = "jdbc:postgresql://";
public static final String JDBC_HIVE_2 = "jdbc:hive2://";
public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://";
public static final String JDBC_DATABEND = "jdbc:databend://";
public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@";
public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//";
public static final String JDBC_SQLSERVER = "jdbc:sqlserver://";
public static final String JDBC_DB2 = "jdbc:db2://";
public static final String JDBC_PRESTO = "jdbc:presto://";
/**
* driver
*/
public static final String ORG_POSTGRESQL_DRIVER = "org.postgresql.Driver";
public static final String COM_MYSQL_CJ_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver";
public static final String COM_CLICKHOUSE_JDBC_DRIVER = "com.clickhouse.jdbc.ClickHouseDriver";
public static final String COM_DATABEND_JDBC_DRIVER = "com.databend.jdbc.DatabendDriver";
public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver";
public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver";
public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver";
/**
* datasource encryption salt
*/
@ -361,11 +310,6 @@ public class TaskConstants {
public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable";
public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt";
/**
* resource storage type
*/
// public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type";
/**
* kerberos
*/
@ -407,11 +351,6 @@ public class TaskConstants {
public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE =
"hadoop.security.authentication.startup.state";
/**
* Task Logger Thread's name
*/
public static final String TASK_LOGGER_THREAD_NAME = "TaskLogInfo";
/**
* hdfs/s3 configuration
* resource.storage.upload.base.path
@ -423,11 +362,6 @@ public class TaskConstants {
*/
public static final String DATA_QUALITY_JAR_NAME = "data-quality.jar.name";
/**
* data.quality.error.output.path
*/
public static final String DATA_QUALITY_ERROR_OUTPUT_PATH = "data-quality.error.output.path";
public static final String TASK_TYPE_CONDITIONS = "CONDITIONS";
public static final String TASK_TYPE_SWITCH = "SWITCH";
@ -442,14 +376,10 @@ public class TaskConstants {
public static final String TASK_TYPE_DATA_QUALITY = "DATA_QUALITY";
public static final String DEPLOY_MODE_KUBERNETES = "Kubernetes";
public static final Set<String> TASK_TYPE_SET_K8S = Sets.newHashSet("K8S", "KUBEFLOW");
public static final String TASK_TYPE_BLOCKING = "BLOCKING";
public static final String TASK_TYPE_STREAM = "STREAM";
/**
* azure config
*/
@ -483,7 +413,6 @@ public class TaskConstants {
* use for k8s task
*/
public static final String API_VERSION = "batch/v1";
public static final String IMAGE_PULL_POLICY = "Always";
public static final String RESTART_POLICY = "Never";
public static final String MEMORY = "memory";
public static final String CPU = "cpu";
@ -496,7 +425,6 @@ public class TaskConstants {
public static final int LOG_LINES = 500;
public static final String NAMESPACE_NAME = "name";
public static final String CLUSTER = "cluster";
public static final Pattern COMMAND_SPLIT_REGEX = Pattern.compile("[^\\s\"'`]+|\"([^\"]+)\"|'([^']+)'|`([^`]+)`");
/**
* spark / flink on k8s label name

34
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
View File

@ -17,10 +17,9 @@
package org.apache.dolphinscheduler.server.worker.config;
import static org.apache.dolphinscheduler.common.constants.Constants.REGISTRY_DOLPHINSCHEDULER_WORKERS;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.commons.lang3.StringUtils;
@ -81,22 +80,27 @@ public class WorkerConfig implements Validator {
workerConfig.setWorkerAddress(NetUtils.getAddr(workerConfig.getListenPort()));
}
workerConfig.setWorkerRegistryPath(REGISTRY_DOLPHINSCHEDULER_WORKERS + "/" + workerConfig.getWorkerAddress());
workerConfig.setWorkerRegistryPath(
RegistryNodeType.WORKER.getRegistryPath() + "/" + workerConfig.getWorkerAddress());
printConfig();
}
private void printConfig() {
log.info("Worker config: listenPort -> {}", listenPort);
log.info("Worker config: execThreads -> {}", execThreads);
log.info("Worker config: heartbeatInterval -> {}", heartbeatInterval);
log.info("Worker config: hostWeight -> {}", hostWeight);
log.info("Worker config: tenantAutoCreate -> {}", tenantAutoCreate);
log.info("Worker config: tenantDistributedUser -> {}", tenantDistributedUser);
log.info("Worker config: maxCpuLoadAvg -> {}", maxCpuLoadAvg);
log.info("Worker config: reservedMemory -> {}", reservedMemory);
log.info("Worker config: registryDisconnectStrategy -> {}", registryDisconnectStrategy);
log.info("Worker config: workerAddress -> {}", workerAddress);
log.info("Worker config: workerRegistryPath: {}", workerRegistryPath);
log.info("Worker config: taskExecuteThreadsFullPolicy: {}", taskExecuteThreadsFullPolicy);
String config =
"\n****************************Worker Configuration**************************************" +
"\n listen-port -> " + listenPort +
"\n exec-threads -> " + execThreads +
"\n heartbeat-interval -> " + heartbeatInterval +
"\n host-weight -> " + hostWeight +
"\n tenant-auto-create -> " + tenantAutoCreate +
"\n tenant-distributed-user -> " + tenantDistributedUser +
"\n max-cpu-load-avg -> " + maxCpuLoadAvg +
"\n reserved-memory -> " + reservedMemory +
"\n registry-disconnect-strategy -> " + registryDisconnectStrategy +
"\n task-execute-threads-full-policy: " + taskExecuteThreadsFullPolicy +
"\n address -> " + workerAddress +
"\n registry-path: " + workerRegistryPath +
"\n****************************Worker Configuration**************************************";
log.info(config);
}
}

2
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java
View File

@ -69,7 +69,7 @@ public class MessageRetryRunner extends BaseDaemonThread {
log.info("Message retry runner staring");
messageSenders.forEach(messageSender -> {
messageSenderMap.put(messageSender.getMessageType(), messageSender);
log.info("Injected message sender: {}", messageSender.getClass().getName());
log.info("Injected message sender: {}", messageSender.getClass().getSimpleName());
});
super.start();
log.info("Message retry runner started");

3
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java
View File

@ -33,7 +33,8 @@ import org.springframework.stereotype.Service;
public class WorkerRpcServer extends SpringServerMethodInvokerDiscovery implements Closeable {
public WorkerRpcServer(WorkerConfig workerConfig) {
super(NettyRemotingServerFactory.buildNettyRemotingServer(new NettyServerConfig(workerConfig.getListenPort())));
super(NettyRemotingServerFactory.buildNettyRemotingServer(NettyServerConfig.builder()
.serverName("WorkerRpcServer").listenPort(workerConfig.getListenPort()).build()));
}
public void start() {

4
dolphinscheduler-worker/src/main/resources/logback-spring.xml
View File

@ -23,7 +23,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@ -61,7 +61,7 @@
</rollingPolicy>
<encoder>
<pattern>
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
[WI-%X{workflowInstanceId:-0}][TI-%X{taskInstanceId:-0}] - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{10}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>