test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-30 11:17:54 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Feature-3222][datasource] Store password in ciphertext instead (#3330)

Browse Source 
* fix bug

Delete invalid field: executorcores

Modify verification prompt

* fix bug

Delete invalid field: executorcores

Modify verification prompt

* fix bug

Delete invalid field: executorcores

Modify verification prompt

* dag  add close button

* reset last version

* reset last version

* dag add close buttion

dag add close buttion

* update  CLICK_SAVE_WORKFLOW_BUTTON  xpath

* updae CLICK_SAVE_WORKFLOW_BUTTON xpath

* updae CLICK_SAVE_WORKFLOW_BUTTON xpath

* updae CLICK_SAVE_WORKFLOW_BUTTON xpath

* Update CreateWorkflowLocator.java

modify submit workflow button

* Update CreateWorkflowLocator.java

* Update CreateWorkflowLocator.java

modify CLICK_ADD_BUTTON

* Update CreateWorkflowLocator.java

delete print

* Update CreateWorkflowLocator.java

1

* Update CreateWorkflowLocator.java

1

* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks

in addition

update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'

* Update dag.vue

* Update dag.vue

* Update dag.vue

* Update CreateWorkflowLocator.java

* Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks"

This reverts commit 3a2cba7a

* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks

in addition

update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'

* Store password in ciphertext instead

* Store password in ciphertext instead

* Store password in ciphertext instead

* Store password in ciphertext instead

* Store password in ciphertext instead

* update test unit

* add test unit

* add test unit

* update e2e test

* update unit test

* update unit test

Co-authored-by: dailidong <dailidong66@gmail.com>
Co-authored-by: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com>
This commit is contained in:
BoYiZhang 2020-08-01 14:28:12 +08:00 committed by GitHub
parent 6d903e266f
commit d70d8a6054
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 236 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
View File

@ -527,7 +527,7 @@ public class DataSourceService extends BaseService{
parameterMap.put(Constants.DATABASE, database);
parameterMap.put(Constants.JDBC_URL, jdbcUrl);
parameterMap.put(Constants.USER, userName);
parameterMap.put(Constants.PASSWORD, password);
parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
if (CommonUtils.getKerberosStartupState() &&
(type == DbType.HIVE || type == DbType.SPARK)){
parameterMap.put(Constants.PRINCIPAL,principal);

24
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
View File

@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
@ -103,7 +104,28 @@ public class DataSourceServiceTest {
public void buildParameter(){
String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im"
,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,"");
String expected = "{\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
Assert.assertEquals(expected, param);
}
@Test
public void buildParameterWithDecodePassword(){
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
,"","test","123456", null,"");
String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
Assert.assertEquals(expected, param);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
,"","test","123456", null,"");
expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}";
Assert.assertEquals(expected, param);
}
}

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
View File

@ -991,4 +991,10 @@ public final class Constants {
public static final String DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE = "dolphin.scheduler.network.interface.preferred";
/**
* datasource encryption salt
*/
public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*";
public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable";
public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt";
}

46
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.hadoop.conf.Configuration;
@ -23,8 +24,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.nio.charset.StandardCharsets;
/**
* common utils
@ -32,6 +33,8 @@ import java.net.URL;
public class CommonUtils {
private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
private static final Base64 BASE64 = new Base64();
private CommonUtils() {
throw new IllegalStateException("CommonUtils class");
}
@ -90,4 +93,45 @@ public class CommonUtils {
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
}
}
/**
* encode password
* @param password
* @return
*/
public static String encodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ;
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
}
/**
* decode password
* @param password
* @return
*/
public static String decodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ;
if(!passwordWithSalt.startsWith(salt)){
logger.warn("There is a password and salt mismatch: {} ",password);
return password;
}
return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
View File

@ -250,4 +250,14 @@ public class PropertyUtils {
}
return matchedProperties;
}
/**
*
* @param key
* @param value
*/
public static void setValue(String key, String value) {
properties.setProperty(key,value);
}
}

6
dolphinscheduler-common/src/main/resources/common.properties
View File

@ -67,4 +67,8 @@ yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
development.state=false
# kerberos tgt expire time, unit is hours
kerberos.expire.time=2
kerberos.expire.time=2
# datasource encryption salt
datasource.encryption.enable=false
datasource.encryption.salt=!@#$%^&*

39
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@ -89,4 +90,42 @@ public class CommonUtilsTest {
}
Assert.assertTrue(true);
}
@Test
public void encodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertEquals("",CommonUtils.encodePassword(""));
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456"));
Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@"));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertEquals("",CommonUtils.encodePassword(""));
Assert.assertEquals("123456",CommonUtils.encodePassword("123456"));
Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@"));
}
@Test
public void decodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
Assert.assertEquals("", CommonUtils.decodePassword(""));
Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg=="));
Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0="));
Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0="));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
Assert.assertEquals("", CommonUtils.decodePassword(""));
Assert.assertEquals("123456", CommonUtils.decodePassword("123456"));
Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@"));
}
}

14
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -67,4 +68,17 @@ public class FileUtilsTest {
Assert.assertTrue(false);
}
}
@Test
public void testSetValue() {
try {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
} catch (Exception e) {
Assert.assertTrue(false);
}
}
}

7
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
View File

@ -20,6 +20,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -182,8 +183,12 @@ public abstract class BaseDataSource {
this.user = user;
}
/**
* password need decode
* @return
*/
public String getPassword() {
return password;
return CommonUtils.decodePassword(password);
}
public void setPassword(String password) {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java
View File

@ -82,6 +82,8 @@ public class MySQLDataSource extends BaseDataSource {
@Override
public String getPassword() {
// password need decode
password = super.getPassword();
if(password.contains(sensitiveParam)){
logger.warn("sensitive param : {} in password field is filtered", sensitiveParam);
password = password.replace(sensitiveParam, "");

49
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java
View File

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.junit.Assert;
import org.junit.Test;
@ -112,4 +114,51 @@ public class BaseDataSourceTest {
}
@Test
public void testGetPassword() {
BaseDataSource dataSource = new BaseDataSource() {
@Override
public String driverClassSelector() {
return null;
}
@Override
public DbType dbTypeSelector() {
return null;
}
};
String password= "";
dataSource.setPassword(password);
Assert.assertEquals("", dataSource.getPassword());
password= "IUAjJCVeJipNVEl6TkRVMg==";
dataSource.setPassword(password);
Assert.assertNotNull(dataSource.getPassword());
Assert.assertNotNull(dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
}
}

35
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java
View File

@ -16,6 +16,8 @@
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.junit.Assert;
import org.junit.Test;
@ -46,6 +48,7 @@ public class MySQLDataSourceTest {
Assert.assertEquals("test_pwd?", dataSource.getPassword());
}
@Test
public void testFilterOther(){
MySQLDataSource dataSource = new MySQLDataSource();
@ -61,4 +64,36 @@ public class MySQLDataSourceTest {
other = dataSource.filterOther("serverTimezone=Asia/Shanghai&autoDeserialize=true&characterEncoding=utf8");
Assert.assertEquals("serverTimezone=Asia/Shanghai&characterEncoding=utf8", other);
}
@Test
public void testGetPasswordWithDecodePassword(){
MySQLDataSource dataSource = new MySQLDataSource();
String password= "";
dataSource.setPassword(password);
Assert.assertEquals("", dataSource.getPassword());
password= "IUAjJCVeJipNVEl6TkRVMg==";
dataSource.setPassword(password);
Assert.assertNotNull(dataSource.getPassword());
Assert.assertNotNull(dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
}
}

2
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java
View File

@ -21,7 +21,7 @@ import org.openqa.selenium.By;
public class ProcessInstanceLocator {
// jump Process Instance page
//click Process Instance name
public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[3]/div/ul/li[2]");
public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[4]/div/ul/li[2]");
// click rerun button
public static final By CLICK_RERUN_BUTTON = By.xpath("//tr[2]/td[14]/div[1]/button[2]");