test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-02 04:08:31 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

[Improvement][Task] Mask configYaml in task log (#14571)

Browse Source 
* [Improvement][Task] Mask configYaml in task log

Signed-off-by: Gallardot <gallardot@apache.org>
---------

Signed-off-by: Gallardot <gallardot@apache.org>
Co-authored-by: Aaron Wang <wangweirao16@gmail.com>
This commit is contained in:
Gallardot 2023-07-18 15:02:20 +08:00 committed by GitHub
parent 5a6f1c88a3
commit 9d10de6efa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 37 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
View File

@ -856,4 +856,9 @@ public final class Constants {
*/
public static final String DATABASES_QUERY = "show databases";
public static final String DATABASES_QUERY_PG = "SELECT datname FROM pg_database";
/**
* K8S sensitive param
*/
public static final String K8S_CONFIG_REGEX = "(?<=((?i)configYaml(\" : \"))).*?(?=(\",\\n))";
}

29
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
View File

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.common.log;
import static org.apache.dolphinscheduler.common.constants.Constants.K8S_CONFIG_REGEX;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
@ -62,4 +64,31 @@ public class SensitiveDataConverterTest {
Assertions.assertEquals(expectedMsg, maskedLog);
}
@Test
public void testK8SLogMsgConverter() {
String msg = "End initialize task {\n" +
" \"taskName\" : \"echo\",\n" +
" \"k8sTaskExecutionContext\" : {\n" +
" \"configYaml\" : \"apiVersion: v1 xxx client-key-data: ==\",\n" +
" \"namespace\" : \"abc\"\n" +
" },\n" +
" \"logBufferEnable\" : false\n" +
"}";
String maskMsg = "End initialize task {\n" +
" \"taskName\" : \"echo\",\n" +
" \"k8sTaskExecutionContext\" : {\n" +
" \"configYaml\" : \"**************************************\",\n" +
" \"namespace\" : \"abc\"\n" +
" },\n" +
" \"logBufferEnable\" : false\n" +
"}";
SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX);
final String maskedLog = SensitiveDataConverter.maskSensitiveData(msg);
logger.info("original parameter : {}", msg);
logger.info("masked parameter : {}", maskedLog);
Assertions.assertEquals(maskMsg, maskedLog);
}
}

3
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java
View File

@ -19,9 +19,11 @@ package org.apache.dolphinscheduler.server.worker.runner;
import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER;
import static org.apache.dolphinscheduler.common.constants.Constants.DRY_RUN_FLAG_YES;
import static org.apache.dolphinscheduler.common.constants.Constants.K8S_CONFIG_REGEX;
import static org.apache.dolphinscheduler.common.constants.Constants.SINGLE_SLASH;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
import org.apache.dolphinscheduler.common.log.remote.RemoteLogUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
@ -93,6 +95,7 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable {
this.taskPluginManager = taskPluginManager;
this.storageOperate = storageOperate;
this.workerRegistryClient = workerRegistryClient;
SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX);
}
protected abstract void executeTask(TaskCallBack taskCallBack);