test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-04 13:17:50 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

fix 3112:download udf resource need find the tenant code of resource owner (#3114)

Browse Source 
* fix 3112:download udf resource need find the tenant code of resource owner

* fix 2975:download resource need find the tenant code of resource owner

* remove useless code
This commit is contained in:
lgcareer 2020-07-02 14:42:22 +08:00 committed by GitHub
parent 07a6aec1fc
commit 02e22e8a74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java
View File

@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.server.entity;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
/**
* SQL Task ExecutionContext
@ -38,9 +38,9 @@ public class SQLTaskExecutionContext implements Serializable {
*/
private String connectionParams;
/**
* udf function list
* udf function tenant code map
*/
private List<UdfFunc> udfFuncList;
private Map<UdfFunc,String> udfFuncTenantCodeMap;
public int getWarningGroupId() {
@ -51,12 +51,12 @@ public class SQLTaskExecutionContext implements Serializable {
this.warningGroupId = warningGroupId;
}
public List<UdfFunc> getUdfFuncList() {
return udfFuncList;
public Map<UdfFunc, String> getUdfFuncTenantCodeMap() {
return udfFuncTenantCodeMap;
}
public void setUdfFuncList(List<UdfFunc> udfFuncList) {
this.udfFuncList = udfFuncList;
public void setUdfFuncTenantCodeMap(Map<UdfFunc, String> udfFuncTenantCodeMap) {
this.udfFuncTenantCodeMap = udfFuncTenantCodeMap;
}
public String getConnectionParams() {
@ -72,7 +72,7 @@ public class SQLTaskExecutionContext implements Serializable {
return "SQLTaskExecutionContext{" +
"warningGroupId=" + warningGroupId +
", connectionParams='" + connectionParams + '\'' +
", udfFuncList=" + udfFuncList +
", udfFuncTenantCodeMap=" + udfFuncTenantCodeMap +
'}';
}
}

10
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
View File

@ -324,7 +324,13 @@ public class TaskPriorityQueueConsumer extends Thread{
}
List<UdfFunc> udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray);
sqlTaskExecutionContext.setUdfFuncList(udfFuncList);
Map<UdfFunc,String> udfFuncMap = new HashMap<>();
for(UdfFunc udfFunc : udfFuncList) {
String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF);
udfFuncMap.put(udfFunc,tenantCode);
}
sqlTaskExecutionContext.setUdfFuncTenantCodeMap(udfFuncMap);
}
}
@ -366,7 +372,7 @@ public class TaskPriorityQueueConsumer extends Thread{
if (baseParam != null) {
List<ResourceInfo> projectResourceFiles = baseParam.getResourceFilesList();
if (projectResourceFiles != null) {
if (CollectionUtils.isNotEmpty(projectResourceFiles)) {
// filter the resources that the resource id equals 0
Set<ResourceInfo> oldVersionResources = projectResourceFiles.stream().filter(t -> t.getId() == 0).collect(Collectors.toSet());

65
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.commons.collections.MapUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
@ -24,10 +25,8 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.slf4j.Logger;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static org.apache.dolphinscheduler.common.utils.CollectionUtils.isNotEmpty;
@ -43,53 +42,44 @@ public class UDFUtils {
/**
* create function list
* @param udfFuncs udf functions
* @param tenantCode tenant code
* @param logger logger
* @param udfFuncTenantCodeMap key is udf function,value is tenant code
* @param logger logger
* @return create function list
*/
public static List<String> createFuncs(List<UdfFunc> udfFuncs, String tenantCode,Logger logger){
public static List<String> createFuncs(Map<UdfFunc,String> udfFuncTenantCodeMap, Logger logger){
if (CollectionUtils.isEmpty(udfFuncs)){
if (MapUtils.isEmpty(udfFuncTenantCodeMap)){
logger.info("can't find udf function resource");
return null;
}
// get hive udf jar path
String hiveUdfJarPath = HadoopUtils.getHdfsUdfDir(tenantCode);
logger.info("hive udf jar path : {}" , hiveUdfJarPath);
// is the root directory of udf defined
if (StringUtils.isEmpty(hiveUdfJarPath)) {
logger.error("not define hive udf jar path");
throw new RuntimeException("hive udf jar base path not defined ");
}
Set<String> resources = getFuncResouces(udfFuncs);
List<String> funcList = new ArrayList<>();
// build jar sql
buildJarSql(funcList, resources, hiveUdfJarPath);
buildJarSql(funcList, udfFuncTenantCodeMap);
// build temp function sql
buildTempFuncSql(funcList, udfFuncs);
buildTempFuncSql(funcList, udfFuncTenantCodeMap.keySet().stream().collect(Collectors.toList()));
return funcList;
}
/**
* build jar sql
* @param sqls sql list
* @param resources resource set
* @param uploadPath upload path
* @param sqls sql list
* @param udfFuncTenantCodeMap key is udf function,value is tenant code
*/
private static void buildJarSql(List<String> sqls, Set<String> resources, String uploadPath) {
private static void buildJarSql(List<String> sqls, Map<UdfFunc,String> udfFuncTenantCodeMap) {
String defaultFS = HadoopUtils.getInstance().getConfiguration().get(Constants.FS_DEFAULTFS);
if (!uploadPath.startsWith("hdfs:")) {
uploadPath = defaultFS + uploadPath;
Set<Map.Entry<UdfFunc,String>> entries = udfFuncTenantCodeMap.entrySet();
for (Map.Entry<UdfFunc,String> entry:entries){
String uploadPath = HadoopUtils.getHdfsUdfDir(entry.getValue());
if (!uploadPath.startsWith("hdfs:")) {
uploadPath = defaultFS + uploadPath;
}
sqls.add(String.format("add jar %s%s", uploadPath, entry.getKey().getResourceName()));
}
for (String resource : resources) {
sqls.add(String.format("add jar %s/%s", uploadPath, resource));
}
}
/**
@ -106,20 +96,5 @@ public class UDFUtils {
}
}
/**
* get the resource names of all functions
* @param udfFuncs udf function list
* @return
*/
private static Set<String> getFuncResouces(List<UdfFunc> udfFuncs) {
Set<String> resources = new HashSet<>();
for (UdfFunc udfFunc : udfFuncs) {
resources.add(udfFunc.getResourceName());
}
return resources;
}
}

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
View File

@ -132,8 +132,7 @@ public class SqlTask extends AbstractTask {
.map(this::getSqlAndSqlParamsMap)
.collect(Collectors.toList());
List<String> createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncList(),
taskExecutionContext.getTenantCode(),
List<String> createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncTenantCodeMap(),
logger);
// execute sql task