[Feature-3222][datasource] Store password in ciphertext instead (#3330)

* fix bug

Delete invalid field: executorcores

Modify verification prompt

* fix bug

Delete invalid field: executorcores

Modify verification prompt

* fix bug

Delete invalid field: executorcores

Modify verification prompt

* dag  add close button

* reset last version

* reset last version

* dag add close buttion

dag add close buttion

* update  CLICK_SAVE_WORKFLOW_BUTTON  xpath

* updae CLICK_SAVE_WORKFLOW_BUTTON xpath

* updae CLICK_SAVE_WORKFLOW_BUTTON xpath

* updae CLICK_SAVE_WORKFLOW_BUTTON xpath

* Update CreateWorkflowLocator.java

modify submit workflow button

* Update CreateWorkflowLocator.java

* Update CreateWorkflowLocator.java

modify CLICK_ADD_BUTTON

* Update CreateWorkflowLocator.java

delete print

* Update CreateWorkflowLocator.java

1

* Update CreateWorkflowLocator.java

1

* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks

in addition

update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'

* Update dag.vue

* Update dag.vue

* Update dag.vue

* Update CreateWorkflowLocator.java

* Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks"

This reverts commit 3a2cba7a

* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks

in addition

update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'

* Store password in ciphertext instead

* Store password in ciphertext instead

* Store password in ciphertext instead

* Store password in ciphertext instead

* Store password in ciphertext instead

* update test unit

* add test unit

* add test unit

* update e2e test

* update unit test

* update unit test

Co-authored-by: dailidong <dailidong66@gmail.com>
Co-authored-by: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@@ -527,7 +527,7 @@ public class DataSourceService extends BaseService{
         parameterMap.put(Constants.DATABASE, database);
         parameterMap.put(Constants.JDBC_URL, jdbcUrl);
         parameterMap.put(Constants.USER, userName);
-        parameterMap.put(Constants.PASSWORD, password);
+        parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
         if (CommonUtils.getKerberosStartupState() &&
                 (type == DbType.HIVE || type == DbType.SPARK)){
             parameterMap.put(Constants.PRINCIPAL,principal);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbConnectType;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.dao.entity.DataSource;
 import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
@@ -103,7 +104,28 @@ public class DataSourceServiceTest {
     public void buildParameter(){
         String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im"
                 ,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,"");
-        String expected = "{\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
+        String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
         Assert.assertEquals(expected, param);
     }
+
+    @Test
+    public void buildParameterWithDecodePassword(){
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+        String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
+                ,"","test","123456", null,"");
+        String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
+        Assert.assertEquals(expected, param);
+
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+        param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
+                ,"","test","123456", null,"");
+        expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}";
+        Assert.assertEquals(expected, param);
+    }
+
+
+
+
+
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -991,4 +991,10 @@ public final class Constants {
     public static final String DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE = "dolphin.scheduler.network.interface.preferred";
 
 
+    /**
+     * datasource encryption salt
+     */
+    public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*";
+    public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable";
+    public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt";
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import org.apache.commons.codec.binary.Base64;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
 import org.apache.hadoop.conf.Configuration;
@@ -23,8 +24,8 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 
 /**
  * common utils
@@ -32,6 +33,8 @@ import java.net.URL;
 public class CommonUtils {
   private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
 
+  private static final Base64 BASE64 = new Base64();
+
   private CommonUtils() {
     throw new IllegalStateException("CommonUtils class");
   }
@@ -90,4 +93,45 @@ public class CommonUtils {
               PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
     }
   }
+
+  /**
+   * encode password
+   * @param password
+   * @return
+   */
+  public static String encodePassword(String password) {
+    if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; }
+    //if encryption is not turned on, return directly
+    boolean encryptionEnable =  PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
+    if ( !encryptionEnable){ return password; }
+
+    // Using Base64 + salt to process password
+    String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
+    String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8)))   ;
+    return  new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
+  }
+
+  /**
+   * decode password
+   * @param password
+   * @return
+   */
+  public static String decodePassword(String password) {
+    if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; }
+
+    //if encryption is not turned on, return directly
+    boolean encryptionEnable =  PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
+    if ( !encryptionEnable){ return password; }
+
+    // Using Base64 + salt to process password
+    String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
+    String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ;
+    if(!passwordWithSalt.startsWith(salt)){
+      logger.warn("There is a password and salt mismatch: {} ",password);
+      return password;
+    }
+    return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ;
+  }
+
+
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@@ -250,4 +250,14 @@ public class PropertyUtils {
         }
         return matchedProperties;
     }
+
+    /**
+     *
+     * @param key
+     * @param value
+     */
+    public static void setValue(String key, String value) {
+        properties.setProperty(key,value);
+    }
+
 }

dolphinscheduler-common/src/main/resources/common.properties

@@ -67,4 +67,8 @@ yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
 development.state=false
 
 # kerberos tgt expire time, unit is hours
-kerberos.expire.time=2
+kerberos.expire.time=2
+
+# datasource encryption salt
+datasource.encryption.enable=false
+datasource.encryption.salt=!@#$%^&*

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import org.apache.dolphinscheduler.common.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -89,4 +90,42 @@ public class CommonUtilsTest {
         }
         Assert.assertTrue(true);
     }
+
+    @Test
+    public void encodePassword() {
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+
+        Assert.assertEquals("",CommonUtils.encodePassword(""));
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456"));
+        Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@"));
+        Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@"));
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+
+        Assert.assertEquals("",CommonUtils.encodePassword(""));
+        Assert.assertEquals("123456",CommonUtils.encodePassword("123456"));
+        Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@"));
+        Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@"));
+
+    }
+
+    @Test
+    public void decodePassword() {
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
+
+        Assert.assertEquals("", CommonUtils.decodePassword(""));
+        Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg=="));
+        Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0="));
+        Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0="));
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
+
+        Assert.assertEquals("", CommonUtils.decodePassword(""));
+        Assert.assertEquals("123456", CommonUtils.decodePassword("123456"));
+        Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@"));
+        Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@"));
+    }
+
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import org.apache.dolphinscheduler.common.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -67,4 +68,17 @@ public class FileUtilsTest {
             Assert.assertTrue(false);
         }
     }
+
+    @Test
+    public void testSetValue() {
+        try {
+            PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+            Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
+            PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+            Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
+        } catch (Exception e) {
+            Assert.assertTrue(false);
+        }
+    }
+
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java

@@ -20,6 +20,7 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -182,8 +183,12 @@ public abstract class BaseDataSource {
     this.user = user;
   }
 
+  /**
+   * password need decode
+   * @return
+   */
   public String getPassword() {
-    return password;
+    return CommonUtils.decodePassword(password);
   }
 
   public void setPassword(String password) {

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java

@@ -82,6 +82,8 @@ public class MySQLDataSource extends BaseDataSource {
 
   @Override
   public String getPassword() {
+    // password need decode
+    password = super.getPassword();
     if(password.contains(sensitiveParam)){
       logger.warn("sensitive param : {} in password field is filtered", sensitiveParam);
       password = password.replace(sensitiveParam, "");

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java

@@ -17,6 +17,8 @@
 package org.apache.dolphinscheduler.dao.datasource;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -112,4 +114,51 @@ public class BaseDataSourceTest {
 
 
   }
+
+  @Test
+  public void testGetPassword() {
+    BaseDataSource dataSource = new BaseDataSource() {
+      @Override
+      public String driverClassSelector() {
+        return null;
+      }
+
+      @Override
+      public DbType dbTypeSelector() {
+        return null;
+      }
+    };
+
+    String password= "";
+    dataSource.setPassword(password);
+    Assert.assertEquals("", dataSource.getPassword());
+    password= "IUAjJCVeJipNVEl6TkRVMg==";
+    dataSource.setPassword(password);
+    Assert.assertNotNull(dataSource.getPassword());
+    Assert.assertNotNull(dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+    Assert.assertEquals("123456", dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    Assert.assertEquals("123456", dataSource.getPassword());
+    Assert.assertEquals("123456", dataSource.getPassword());
+    Assert.assertEquals("123456", dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+
+  }
+
+
+
+
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dolphinscheduler.dao.datasource;
 
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -46,6 +48,7 @@ public class MySQLDataSourceTest {
         Assert.assertEquals("test_pwd?", dataSource.getPassword());
     }
 
+
     @Test
     public void testFilterOther(){
         MySQLDataSource dataSource = new MySQLDataSource();
@@ -61,4 +64,36 @@ public class MySQLDataSourceTest {
         other = dataSource.filterOther("serverTimezone=Asia/Shanghai&autoDeserialize=true&characterEncoding=utf8");
         Assert.assertEquals("serverTimezone=Asia/Shanghai&characterEncoding=utf8", other);
     }
+
+    @Test
+    public void testGetPasswordWithDecodePassword(){
+        MySQLDataSource dataSource = new MySQLDataSource();
+        String password= "";
+        dataSource.setPassword(password);
+        Assert.assertEquals("", dataSource.getPassword());
+        password= "IUAjJCVeJipNVEl6TkRVMg==";
+        dataSource.setPassword(password);
+        Assert.assertNotNull(dataSource.getPassword());
+        Assert.assertNotNull(dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+        Assert.assertEquals("123456", dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        Assert.assertEquals("123456", dataSource.getPassword());
+        Assert.assertEquals("123456", dataSource.getPassword());
+        Assert.assertEquals("123456", dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+    }
+
 }

e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java

@@ -21,7 +21,7 @@ import org.openqa.selenium.By;
 public class ProcessInstanceLocator {
     // jump Process Instance page
     //click Process Instance name
-    public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[3]/div/ul/li[2]");
+    public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[4]/div/ul/li[2]");
 
     // click rerun button
     public static final By CLICK_RERUN_BUTTON = By.xpath("//tr[2]/td[14]/div[1]/button[2]");