test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-02 12:17:43 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

[Improvement][Log] optimize task log path to reduce task running time (#6731)

Browse Source 
* [Improvement][Log] optimize task log path to reduce task running time
* Reduce the creation of log directories


Co-authored-by: Kirs <acm_master@163.com>
This commit is contained in:
chouc 2021-11-17 17:08:44 +08:00 committed by GitHub
parent 842c3857c1
commit f564687a89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 65 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
View File

@ -184,6 +184,10 @@ public final class Constants {
*/
public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss";
/**
* date format of yyyyMMdd
*/
public static final String YYYYMMDD = "yyyyMMdd";
/**
* date format of yyyyMMddHHmmss

20
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
View File

@ -92,7 +92,7 @@ public final class DateUtils {
/**
* get the formatted date string
*
* @param date date
* @param date date
* @param format e.g. yyyy-MM-dd HH:mm:ss
* @return date string
*/
@ -104,7 +104,7 @@ public final class DateUtils {
* get the formatted date string
*
* @param localDateTime local data time
* @param format yyyy-MM-dd HH:mm:ss
* @param format yyyy-MM-dd HH:mm:ss
* @return date string
*/
public static String format(LocalDateTime localDateTime, String format) {
@ -124,7 +124,7 @@ public final class DateUtils {
/**
* convert string to date and time
*
* @param date date
* @param date date
* @param format format
* @return date
*/
@ -177,7 +177,7 @@ public final class DateUtils {
* get the date of the specified date in the days before and after
*
* @param date date
* @param day day
* @param day day
* @return the date of the specified date in the days before and after
*/
public static Date getSomeDay(Date date, int day) {
@ -203,7 +203,7 @@ public final class DateUtils {
* compare two dates
*
* @param future future date
* @param old old date
* @param old old date
* @return true if future time greater than old time
*/
public static boolean compare(Date future, Date old) {
@ -329,7 +329,7 @@ public final class DateUtils {
/**
* get some hour of day
*
* @param date date
* @param date date
* @param offsetHour hours
* @return some hour of day
*/
@ -432,15 +432,15 @@ public final class DateUtils {
*/
public static Date getCurrentDate() {
return DateUtils.parse(DateUtils.getCurrentTime(),
Constants.YYYY_MM_DD_HH_MM_SS);
Constants.YYYY_MM_DD_HH_MM_SS);
}
/**
* get date
*
* @param date date
* @param date date
* @param calendarField calendarField
* @param amount amount
* @param amount amount
* @return date
*/
public static Date add(final Date date, final int calendarField, final int amount) {
@ -457,7 +457,7 @@ public final class DateUtils {
* starting from the current time, get how many seconds are left before the target time.
* targetTime = baseTime + intervalSeconds
*
* @param baseTime base time
* @param baseTime base time
* @param intervalSeconds a period of time
* @return the number of seconds
*/

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
View File

@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -71,12 +72,14 @@ public class LoggerUtils {
* @return task id format
*/
public static String buildTaskId(String affix,
Date firstSubmitTime,
Long processDefineCode,
int processDefineVersion,
int processInstId,
int taskId) {
// - [taskAppId=TASK-798_1-4084-15210]
return String.format(" - %s%s-%s_%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix, processDefineCode, processDefineVersion, processInstId, taskId);
// - [taskAppId=TASK-20211107-798_1-4084-15210]
String firstSubmitTimeStr = DateUtils.format(firstSubmitTime, Constants.YYYYMMDD);
return String.format(" - %s%s-%s-%s_%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix, firstSubmitTimeStr, processDefineCode, processDefineVersion, processInstId, taskId);
}
/**

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
View File

@ -47,7 +47,7 @@ public class TaskLogDiscriminator extends AbstractDiscriminator<ILoggingEvent> {
String prefix = LoggerUtils.TASK_LOGGER_INFO_PREFIX + "-";
if (loggerName.startsWith(prefix)) {
return loggerName.substring(prefix.length(),
loggerName.length() - 1).replace("-","/");
loggerName.length() - 1).replaceFirst("-","/");
} else {
return "unknown_task";
}

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
View File

@ -82,6 +82,7 @@ public class ConditionTaskProcessor extends BaseTaskProcessor {
);
logger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
taskInstance.getFirstSubmitTime(),
processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(),
@ -144,7 +145,7 @@ public class ConditionTaskProcessor extends BaseTaskProcessor {
}
private void initTaskParameters() {
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(),
taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(),
taskInstance.getId()));

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java
View File

@ -89,7 +89,8 @@ public class DependentTaskProcessor extends BaseTaskProcessor {
taskDefinition = processService.findTaskDefinition(
taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()
);
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(),
taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),
processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(),
taskInstance.getId()));

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
View File

@ -71,7 +71,7 @@ public class SwitchTaskProcessor extends BaseTaskProcessor {
taskDefinition = processService.findTaskDefinition(
taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()
);
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(),
taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance.getFirstSubmitTime(),processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(),
taskInstance.getId()));

46
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java
View File

@ -17,11 +17,14 @@
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;
import java.util.Optional;
import org.slf4j.LoggerFactory;
@ -32,6 +35,8 @@ import ch.qos.logback.core.spi.AppenderAttachable;
public class LogUtils {
public static final String LOG_TAILFIX = ".log";
private LogUtils() throws IllegalStateException {
throw new IllegalStateException("Utility class");
}
@ -39,30 +44,39 @@ public class LogUtils {
/**
* get task log path
*/
public static String getTaskLogPath(Long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) {
public static String getTaskLogPath(Date firstSubmitTime, Long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) {
// format /logs/YYYYMMDD/defintion-code_defintion_version-processInstanceId-taskInstanceId.log
final String taskLogFileName = new StringBuilder(String.valueOf(processDefineCode))
.append(Constants.UNDERLINE)
.append(processDefineVersion)
.append(Constants.SUBTRACT_CHAR)
.append(processInstanceId)
.append(Constants.SUBTRACT_CHAR)
.append(taskInstanceId)
.append(LOG_TAILFIX)
.toString();
// Optional.map will be skipped if null
return Optional.of(LoggerFactory.getILoggerFactory())
.map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT")))
.map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE")))
.map(e -> ((TaskLogDiscriminator) (e.getDiscriminator())))
.map(TaskLogDiscriminator::getLogBase)
.map(e -> Paths.get(e)
.toAbsolutePath()
.resolve(processDefineCode + "_" + processDefineVersion)
.resolve(String.valueOf(processInstanceId))
.resolve(taskInstanceId + ".log"))
.map(Path::toString)
.orElse("");
.map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT")))
.map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE")))
.map(e -> ((TaskLogDiscriminator) (e.getDiscriminator())))
.map(TaskLogDiscriminator::getLogBase)
.map(e -> Paths.get(e)
.toAbsolutePath()
.resolve(DateUtils.format(firstSubmitTime,Constants.YYYYMMDD))
.resolve(taskLogFileName))
.map(Path::toString)
.orElse("");
}
/**
* get task log path by TaskExecutionContext
*/
public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) {
return getTaskLogPath(taskExecutionContext.getProcessDefineCode(),
taskExecutionContext.getProcessDefineVersion(),
taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTaskInstanceId());
return getTaskLogPath(taskExecutionContext.getFirstSubmitTime(),taskExecutionContext.getProcessDefineCode(),
taskExecutionContext.getProcessDefineVersion(),
taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTaskInstanceId());
}
}

1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
View File

@ -169,6 +169,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
}
TaskRequest taskRequest = JSONUtils.parseObject(JSONUtils.toJsonString(taskExecutionContext), TaskRequest.class);
String taskLogName = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
taskExecutionContext.getFirstSubmitTime(),
taskExecutionContext.getProcessDefineCode(),
taskExecutionContext.getProcessDefineVersion(),
taskExecutionContext.getProcessInstanceId(),

2
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
View File

@ -121,7 +121,7 @@ public class TaskLogDiscriminatorTest {
}
});
Assert.assertEquals("1/1/", result);
Assert.assertEquals("1/1-", result);
}
@Test

8
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java
View File

@ -17,11 +17,14 @@
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;
import org.junit.Assert;
import org.junit.Test;
@ -38,11 +41,13 @@ public class LogUtilsTest {
@Test
public void testGetTaskLogPath() {
Date firstSubmitTime = new Date();
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
taskExecutionContext.setProcessInstanceId(100);
taskExecutionContext.setTaskInstanceId(1000);
taskExecutionContext.setProcessDefineCode(1L);
taskExecutionContext.setProcessDefineVersion(1);
taskExecutionContext.setFirstSubmitTime(firstSubmitTime);
Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT");
Assert.assertNotNull(rootLogger);
@ -60,7 +65,8 @@ public class LogUtilsTest {
Path logPath = Paths.get(".").toAbsolutePath().getParent()
.resolve(logBase)
.resolve("1_1").resolve("100").resolve("1000.log");
.resolve(DateUtils.format(firstSubmitTime, Constants.YYYYMMDD))
.resolve("1_1-100-1000.log");
Assert.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskExecutionContext));
}

1
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
View File

@ -108,6 +108,7 @@ public class TaskExecuteProcessorTest {
.thenReturn(workerConfig);
Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
taskExecutionContext.getFirstSubmitTime(),
taskExecutionContext.getProcessDefineCode(),
taskExecutionContext.getProcessDefineVersion(),
taskExecutionContext.getProcessInstanceId(),