test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-29 18:58:05 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

merge from 1.3.1-release (#3161)

Browse Source 
* merge from 1.3.1-release

* [bugfix] remove Conflicting configuration sudo and user (#3038)

Co-authored-by: zhangchunyang1024 <zhangchunyang0123@gamil.com>

* remove some standard notices in order to cut down the NOTICE file. (#3101)

* #3084 remove some standard notices in order to cut down the NOTICE file.

* update exclude element sql/soft_version of rat plugin

* update soft_version

* merge from 1.3.1-release

* fix 3112:download udf resource need find the tenant code of resource owner (#3114)

* fix 3112:download udf resource need find the tenant code of resource owner

* fix 2975:download resource need find the tenant code of resource owner

* remove useless code

* merge from 1.3.1-release

* remove some notes (#3127)

Co-authored-by: lenboo <baoliang@analysys.com.cn>

* fix Permission denied if not found dolphinscheder-env.sh on 1.3.1 (#3128)

* fix Permission denied if not found dolphinscheder-env.sh

* update some comments on method

* add PropertyPlaceholderHelper on license (#3129)

* update license header location

* add PropertyPlaceholderHelper on license

* merge from 1.3.1-release

* change from 1.3.0  to 1.3.2

* add Set import

Co-authored-by: dailidong <dailidong66@gmail.com>
Co-authored-by: zhangchunyang <zhangchunyang0123@gmail.com>
Co-authored-by: zhangchunyang1024 <zhangchunyang0123@gamil.com>
Co-authored-by: lgcareer <18610854716@163.com>
Co-authored-by: lenboo <baoliang@analysys.com.cn>
This commit is contained in:
bao liang 2020-07-13 16:35:42 +08:00 committed by GitHub
parent de41ac5083
commit 657ea4752f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 179 additions and 933 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
LICENSE
View File

@ -215,3 +215,4 @@ The following components are provided under the Apache License. See project link
The text of each license is the standard Apache 2.0 license.
ScriptRunner from https://github.com/mybatis/mybatis-3 Apache 2.0
mvnw files from https://github.com/takari/maven-wrapper Apache 2.0
PropertyPlaceholderHelper from https://github.com/spring-projects/spring-framework Apache 2.0

14
NOTICE
View File

@ -72,4 +72,16 @@ Refactored SqlBuilder class (SQL, AbstractSQL)
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
limitations under the License.
Spring Framework ${version}
Copyright (c) 2002-${copyright} Pivotal, Inc.
This product is licensed to you under the Apache License, Version 2.0
(the "License"). You may not use this product except in compliance with
the License.
This product may include a number of subcomponents with separate
copyright notices and license terms. Your use of the source code for
these subcomponents is subject to the terms and conditions of the
subcomponent's license, as noted in the license.txt file.

2
dolphinscheduler-alert/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-alert</artifactId>
<name>${project.artifactId}</name>

2
dolphinscheduler-api/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-api</artifactId>
<name>${project.artifactId}</name>

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java
View File

@ -1,5 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -16,6 +14,8 @@ package org.apache.dolphinscheduler.api.dto.resources;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
/**
* directory
*/

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java
View File

@ -1,5 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -16,6 +14,8 @@ package org.apache.dolphinscheduler.api.dto.resources;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
/**
* file leaf
*/

16
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
View File

@ -1,11 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import java.util.ArrayList;
import java.util.List;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -22,6 +14,14 @@ import java.util.List;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import java.util.ArrayList;
import java.util.List;
/**
* resource component
*/

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java
View File

@ -1,8 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -19,6 +14,10 @@ import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
/**
* Visitor
*/

18
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java
View File

@ -50,20 +50,10 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
/**
* Intercept the execution of a handler. Called after HandlerMapping determined
* an appropriate handler object, but before HandlerAdapter invokes the handler.
* <p>DispatcherServlet processes a handler in an execution chain, consisting
* of any number of interceptors, with the handler itself at the end.
* With this method, each interceptor can decide to abort the execution chain,
* typically sending a HTTP error or writing a custom response.
* <p><strong>Note:</strong> special considerations apply for asynchronous
* request processing. For more details see
* {@link org.springframework.web.servlet.AsyncHandlerInterceptor}.
* @param request current HTTP request
* @param response current HTTP response
* @param handler chosen handler to execute, for type and/or instance evaluation
* @return {@code true} if the execution chain should proceed with the
* next interceptor or the handler itself. Else, DispatcherServlet assumes
* that this interceptor has already dealt with the response itself.
* @param request current HTTP request
* @param response current HTTP response
* @param handler chosen handler to execute, for type and/or instance evaluation
* @return boolean true or false
*/
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {

2
dolphinscheduler-common/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-common</artifactId>
<name>dolphinscheduler-common</name>

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
View File

@ -408,7 +408,7 @@ public class HadoopUtils implements Closeable {
* @param applicationId application id
* @return the return may be null or there may be other parse exceptions
*/
public ExecutionStatus getApplicationStatus(String applicationId) throws Exception{
public ExecutionStatus getApplicationStatus(String applicationId) throws Exception {
if (StringUtils.isEmpty(applicationId)) {
return null;
}

249
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java
View File

@ -18,266 +18,55 @@ package org.apache.dolphinscheduler.common.utils;
/**
* A collection of static utility methods to validate input.
* utility methods for validating input
*
* <p>This class is modelled after Google Guava's Preconditions class, and partly takes code
* from that class. We add this code to here base in order to reduce external
* dependencies.
*/
public final class Preconditions {
// ------------------------------------------------------------------------
// Null checks
// ------------------------------------------------------------------------
private Preconditions() {}
/**
* Ensures that the given object reference is not null.
* Upon violation, a {@code NullPointerException} with no message is thrown.
* if obj is null will throw NPE
*
* @param reference reference
* @param obj obj
* @param <T> T
* @return T
*/
public static <T> T checkNotNull(T reference) {
if (reference == null) {
public static <T> T checkNotNull(T obj) {
if (obj == null) {
throw new NullPointerException();
}
return reference;
return obj;
}
/**
* Ensures that the given object reference is not null.
* Upon violation, a {@code NullPointerException} with the given message is thrown.
* @param reference reference
* @param errorMessage errorMessage
* if obj is null will throw NullPointerException with error message
* @param obj obj
* @param errorMsg error message
* @param <T> T
* @return T
*/
public static <T> T checkNotNull(T reference, String errorMessage) {
if (reference == null) {
throw new NullPointerException(String.valueOf(errorMessage));
public static <T> T checkNotNull(T obj, String errorMsg) {
if (obj == null) {
throw new NullPointerException(errorMsg);
}
return reference;
return obj;
}
/**
* Ensures that the given object reference is not null.
* Upon violation, a {@code NullPointerException} with the given message is thrown.
*
* <p>The error message is constructed from a template and an arguments array, after
* a similar fashion as {@link String#format(String, Object...)}, but supporting only
* {@code %s} as a placeholder.
*
* @param reference The object reference
* @param errorMessageTemplate The message template for the {@code NullPointerException}
* that is thrown if the check fails. The template substitutes its
* {@code %s} placeholders with the error message arguments.
* @param errorMessageArgs The arguments for the error message, to be inserted into the
* message template for the {@code %s} placeholders.
*
* @param <T> T
* @return The object reference itself (generically typed).
*/
public static <T> T checkNotNull(T reference,
String errorMessageTemplate,
Object... errorMessageArgs) {
if (reference == null) {
throw new NullPointerException(format(errorMessageTemplate, errorMessageArgs));
}
return reference;
}
// ------------------------------------------------------------------------
// Boolean Condition Checking (Argument)
// ------------------------------------------------------------------------
/**
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if
* the condition is not met (evaluates to {@code false}).
* if condition is false will throw an IllegalArgumentException with the given message
*
* @param condition The condition to check
* @param condition condition
* @param errorMsg error message
*
* @throws IllegalArgumentException Thrown, if the condition is violated.
*/
public static void checkArgument(boolean condition) {
public static void checkArgument(boolean condition, Object errorMsg) {
if (!condition) {
throw new IllegalArgumentException();
throw new IllegalArgumentException(String.valueOf(errorMsg));
}
}
/**
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if
* the condition is not met (evaluates to {@code false}). The exception will have the
* given error message.
*
* @param condition The condition to check
* @param errorMessage The message for the {@code IllegalArgumentException} that is thrown if the check fails.
*
* @throws IllegalArgumentException Thrown, if the condition is violated.
*/
public static void checkArgument(boolean condition, Object errorMessage) {
if (!condition) {
throw new IllegalArgumentException(String.valueOf(errorMessage));
}
}
/**
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if
* the condition is not met (evaluates to {@code false}).
*
* @param condition The condition to check
* @param errorMessageTemplate The message template for the {@code IllegalArgumentException}
* that is thrown if the check fails. The template substitutes its
* {@code %s} placeholders with the error message arguments.
* @param errorMessageArgs The arguments for the error message, to be inserted into the
* message template for the {@code %s} placeholders.
*
* @throws IllegalArgumentException Thrown, if the condition is violated.
*/
public static void checkArgument(boolean condition,
String errorMessageTemplate,
Object... errorMessageArgs) {
if (!condition) {
throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs));
}
}
// ------------------------------------------------------------------------
// Boolean Condition Checking (State)
// ------------------------------------------------------------------------
/**
* Checks the given boolean condition, and throws an {@code IllegalStateException} if
* the condition is not met (evaluates to {@code false}).
*
* @param condition The condition to check
*
* @throws IllegalStateException Thrown, if the condition is violated.
*/
public static void checkState(boolean condition) {
if (!condition) {
throw new IllegalStateException();
}
}
/**
* Checks the given boolean condition, and throws an {@code IllegalStateException} if
* the condition is not met (evaluates to {@code false}). The exception will have the
* given error message.
*
* @param condition The condition to check
* @param errorMessage The message for the {@code IllegalStateException} that is thrown if the check fails.
*
* @throws IllegalStateException Thrown, if the condition is violated.
*/
public static void checkState(boolean condition, Object errorMessage) {
if (!condition) {
throw new IllegalStateException(String.valueOf(errorMessage));
}
}
/**
* Checks the given boolean condition, and throws an {@code IllegalStateException} if
* the condition is not met (evaluates to {@code false}).
*
* @param condition The condition to check
* @param errorMessageTemplate The message template for the {@code IllegalStateException}
* that is thrown if the check fails. The template substitutes its
* {@code %s} placeholders with the error message arguments.
* @param errorMessageArgs The arguments for the error message, to be inserted into the
* message template for the {@code %s} placeholders.
*
* @throws IllegalStateException Thrown, if the condition is violated.
*/
public static void checkState(boolean condition,
String errorMessageTemplate,
Object... errorMessageArgs) {
if (!condition) {
throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs));
}
}
/**
* Ensures that the given index is valid for an array, list or string of the given size.
*
* @param index index to check
* @param size size of the array, list or string
*
* @throws IllegalArgumentException Thrown, if size is negative.
* @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size
*/
public static void checkElementIndex(int index, int size) {
checkArgument(size >= 0, "Size was negative.");
if (index < 0 || index >= size) {
throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + size);
}
}
/**
* Ensures that the given index is valid for an array, list or string of the given size.
*
* @param index index to check
* @param size size of the array, list or string
* @param errorMessage The message for the {@code IndexOutOfBoundsException} that is thrown if the check fails.
*
* @throws IllegalArgumentException Thrown, if size is negative.
* @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size
*/
public static void checkElementIndex(int index, int size, String errorMessage) {
checkArgument(size >= 0, "Size was negative.");
if (index < 0 || index >= size) {
throw new IndexOutOfBoundsException(String.valueOf(errorMessage) + " Index: " + index + ", Size: " + size);
}
}
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
/**
* A simplified formatting method. Similar to {@link String#format(String, Object...)}, but
* with lower overhead (only String parameters, no locale, no format validation).
*
* <p>This method is taken quasi verbatim from the Guava Preconditions class.
*/
private static String format( String template, Object... args) {
final int numArgs = args == null ? 0 : args.length;
template = String.valueOf(template); // null -> "null"
// start substituting the arguments into the '%s' placeholders
StringBuilder builder = new StringBuilder(template.length() + 16 * numArgs);
int templateStart = 0;
int i = 0;
while (i < numArgs) {
int placeholderStart = template.indexOf("%s", templateStart);
if (placeholderStart == -1) {
break;
}
builder.append(template.substring(templateStart, placeholderStart));
builder.append(args[i++]);
templateStart = placeholderStart + 2;
}
builder.append(template.substring(templateStart));
// if we run out of placeholders, append the extra args in square braces
if (i < numArgs) {
builder.append(" [");
builder.append(args[i++]);
while (i < numArgs) {
builder.append(", ");
builder.append(args[i++]);
}
builder.append(']');
}
return builder.toString();
}
// ------------------------------------------------------------------------
/** Private constructor to prevent instantiation. */
private Preconditions() {}
}

117
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java
View File

@ -33,11 +33,9 @@ public class PreconditionsTest {
*/
@Test
public void testCheckNotNull() throws Exception {
String testReference = "test reference";
//test reference is not null
String testReference = "test object";
Assert.assertEquals(testReference, Preconditions.checkNotNull(testReference));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"reference is null"));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"%s is null",testReference));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"object is null"));
//test reference is null
try {
@ -51,120 +49,19 @@ public class PreconditionsTest {
} catch (NullPointerException ex) {
assertNull(ex.getMessage());
}
//test reference is null ,expect contains errorMessage
try {
Preconditions.checkNotNull(null,"reference is null");
Preconditions.checkNotNull(null,"object is null");
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString("reference is null"));
assertThat(ex.getMessage(), containsString("object is null"));
}
try {
Preconditions.checkNotNull("","reference is null");
Preconditions.checkNotNull("","object is null");
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString("reference is null"));
assertThat(ex.getMessage(), containsString("object is null"));
}
//test reference is null ,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkNotNull(null,"%s is null",testReference);
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString(testReference + " is null"));
}
try {
Preconditions.checkNotNull("","%s is null",testReference);
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString(testReference + " is null"));
}
}
/**
* Test checkArgument
*/
@Test
public void testCheckArgument() throws Exception {
int argument = 100;
//boolean condition is true
Preconditions.checkArgument(argument > 0 && argument < 200);
//boolean condition is false
try {
Preconditions.checkArgument(argument > 0 && argument < 50);
} catch (IllegalArgumentException ex) {
assertNull(ex.getMessage());
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkArgument(argument > 300, "argument is error");
} catch (IllegalArgumentException ex) {
assertThat(ex.getMessage(), containsString("argument is error"));
}
//boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkArgument(argument > 0 && argument < 99, "argument %s is error",argument);
} catch (IllegalArgumentException ex) {
assertThat(ex.getMessage(), containsString( "argument " + argument + " is error"));
}
}
/**
* Test checkState
*/
@Test
public void testCheckState() throws Exception {
int state = 1;
//boolean condition is true
Preconditions.checkState(state == 1);
Preconditions.checkState(state > -1);
//boolean condition is false
try {
Preconditions.checkState(state > 2);
} catch (IllegalStateException ex) {
assertNull(ex.getMessage());
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkState(state < 1, "state is error");
} catch (IllegalStateException ex) {
assertThat(ex.getMessage(), containsString("state is error"));
}
//boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkState(state < -1 , "state %s is error",state);
} catch (IllegalStateException ex) {
assertThat(ex.getMessage(), containsString( "state " + state + " is error"));
}
}
/**
* Test checkElementIndex
*/
@Test
public void testCheckElementIndex() throws Exception {
int index = 2;
int size = 30;
//boolean condition is true
Preconditions.checkElementIndex(index, size);
//boolean condition is false
try {
Preconditions.checkElementIndex(-1, 10);
} catch (IndexOutOfBoundsException ex) {
assertThat(ex.getMessage(), containsString("Index: -1, Size: 10"));
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkElementIndex(100, 50, "index is greater than size");
} catch (IndexOutOfBoundsException ex) {
assertThat(ex.getMessage(), containsString("index is greater than size Index: 100, Size: 50"));
}
}
}

2
dolphinscheduler-dao/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-dao</artifactId>
<name>${project.artifactId}</name>

16
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.dao.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.fasterxml.jackson.annotation.JsonFormat;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@ -27,10 +28,11 @@ import org.apache.dolphinscheduler.common.utils.*;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
* task instance
@ -213,7 +215,7 @@ public class TaskInstance implements Serializable {
@TableField(exist = false)
private List<String> resources;
private Map<String,String> resources;
@ -455,10 +457,14 @@ public class TaskInstance implements Serializable {
|| (this.getState().typeIsFailure() && !taskCanRetry());
}
public List<String> getResources() {
public Map<String, String> getResources() {
return resources;
}
public void setResources(Map<String, String> resources) {
this.resources = resources;
}
public boolean isSubProcess(){
return TaskType.SUB_PROCESS.equals(TaskType.valueOf(this.taskType));
}
@ -471,9 +477,7 @@ public class TaskInstance implements Serializable {
return TaskType.CONDITIONS.equals(TaskType.valueOf(this.taskType));
}
public void setResources(List<String> resources) {
this.resources = resources;
}
/**
* determine if you can try again

2
dolphinscheduler-dist/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

347
dolphinscheduler-dist/release-docs/NOTICE
View File

@ -132,18 +132,6 @@ granted provided that the copyright notice appears in all copies.
========================================================================
Apache Log4j NOTICE
========================================================================
Apache log4j
Copyright 2007 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Joda Time NOTICE
========================================================================
@ -438,17 +426,6 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
========================================================================
Apache Jakarta Commons Lang NOTICE
========================================================================
Apache Jakarta Commons Lang
Copyright 2001-2007 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache ZooKeeper NOTICE
========================================================================
@ -672,109 +649,6 @@ The licenses for these third party components are included in LICENSE.txt
========================================================================
Apache Commons CLI NOTICE
========================================================================
Apache Commons CLI
Copyright 2001-2009 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Collections NOTICE
========================================================================
Apache Commons Collections
Copyright 2001-2015 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Collections4 NOTICE
========================================================================
Apache Commons Collections
Copyright 2001-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Compress NOTICE
========================================================================
Apache Commons Compress
Copyright 2002-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Configuration NOTICE
========================================================================
Apache Commons Configuration
Copyright 2001-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Daemon NOTICE
========================================================================
Copyright 1999-2019 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Email NOTICE
========================================================================
Apache Commons Email
Copyright 2001-2017 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache HttpComponents Client NOTICE
========================================================================
Copyright 1999-2018 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons IO NOTICE
========================================================================
Copyright 2002-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Commons Logging NOTICE
========================================================================
@ -872,18 +746,6 @@ file.
========================================================================
Apache Commons Pool NOTICE
========================================================================
Apache Commons Pool
Copyright 2001-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Derby NOTICE
===========================================================================
@ -1176,18 +1038,6 @@ No other notice covers that jar file.
=========================================================================
Apache HttpClient NOTICE
=========================================================================
Apache HttpComponents Client
Copyright 1999-2015 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Httpcomponents core NOTICE
=========================================================================
@ -1229,28 +1079,6 @@ from the source code management (SCM) system project uses.
=========================================================================
Apache Avro NOTICE
=========================================================================
Apache Avro
Copyright 2009-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apach Commons IO NOTICE
=========================================================================
Apache Commons IO
Copyright 2002-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Junit4 NOTICE
=========================================================================
@ -1335,39 +1163,6 @@ Junit4 NOTICE
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
=========================================================================
Apache Thrift NOTICE
=========================================================================
Apache Thrift
Copyright 2006-2010 The Apache Software Foundation.
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Commons DBCP NOTICE
=========================================================================
Apache Commons DBCP
Copyright 2001-2010 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Commons Daemon NOTICE
=========================================================================
Apache Commons Daemon
Copyright 1999-2013 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Bonecp NOTICE
@ -1500,28 +1295,6 @@ Licensed under the Apache License, Version 2.0 (the "License");
=========================================================================
Apache EL NOTICE
=========================================================================
Apache Tomcat
Copyright 1999-2018 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Commons Net NOTICE
=========================================================================
Apache Commons Net
Copyright 2001-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Logging Log4j2 NOTICE
=========================================================================
@ -1586,50 +1359,6 @@ Portions of this software were originally based on the following:
=========================================================================
Apache Parquet Hadoop Bundle NOTICE
=========================================================================
Apache Parquet Hadoop Bundle
Copyright 2015 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Log4j 1.x NOTICE
=========================================================================
Apache Log4j 1.x Compatibility API
Copyright 1999-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache log4j NOTICE
=========================================================================
Apache log4j
Copyright 2007 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Java Data Objects (JDO) NOTICE
=========================================================================
Apache Java Data Objects (JDO)
Copyright 2005-2006 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Jackson XC NOTICE
=========================================================================
@ -1656,17 +1385,6 @@ see CREDITS file.
=========================================================================
Apache HttpClient Mime NOTICE
=========================================================================
Apache HttpClient Mime
Copyright 1999-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache HttpCore NOTICE
=========================================================================
@ -1693,17 +1411,6 @@ The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Hive Storage API NOTICE
=========================================================================
Apache Hive
Copyright 2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Hadoop NOTICE
=========================================================================
@ -2190,60 +1897,6 @@ Other developers who have contributed code are:
=========================================================================
Apache Yetus NOTICE
=========================================================================
Apache Yetus - Audience Annotations
Copyright 2015-2017 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Directory LDAP API Utilities NOTICE
=========================================================================
Apache Directory LDAP API Utilities
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Directory API ASN.1 API NOTICE
=========================================================================
Apache Directory API ASN.1 API
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
ApacheDS Protocol Kerberos Codec NOTICE
=========================================================================
ApacheDS Protocol Kerberos Codec
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
ApacheDS I18n NOTICE
=========================================================================
ApacheDS I18n
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
ANT NOTICE
=========================================================================

2
dolphinscheduler-microbench/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

2
dolphinscheduler-plugin-api/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-plugin-api</artifactId>
<name>${project.artifactId}</name>

2
dolphinscheduler-remote/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

55
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java
View File

@ -26,69 +26,34 @@ public class RemotingException extends Exception {
super();
}
/** Constructs a new runtime exception with the specified detail message.
* The cause is not initialized, and may subsequently be initialized by a
* call to {@link #initCause}.
/**
* Construct a new runtime exception with the detail message
*
* @param message the detail message. The detail message is saved for
* later retrieval by the {@link #getMessage()} method.
* @param message detail message
*/
public RemotingException(String message) {
super(message);
}
/**
* Constructs a new runtime exception with the specified detail message and
* cause. <p>Note that the detail message associated with
* {@code cause} is <i>not</i> automatically incorporated in
* this runtime exception's detail message.
* Construct a new runtime exception with the detail message and cause
*
* @param message the detail message (which is saved for later retrieval
* by the {@link #getMessage()} method).
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @param message the detail message
* @param cause the cause
* @since 1.4
*/
public RemotingException(String message, Throwable cause) {
super(message, cause);
}
/** Constructs a new runtime exception with the specified cause and a
* detail message of <tt>(cause==null ? null : cause.toString())</tt>
* (which typically contains the class and detail message of
* <tt>cause</tt>). This constructor is useful for runtime exceptions
* that are little more than wrappers for other throwables.
/**
* Construct a new runtime exception with throwable
*
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
* @param cause the cause
*/
public RemotingException(Throwable cause) {
super(cause);
}
/**
* Constructs a new runtime exception with the specified detail
* message, cause, suppression enabled or disabled, and writable
* stack trace enabled or disabled.
*
* @param message the detail message.
* @param cause the cause. (A {@code null} value is permitted,
* and indicates that the cause is nonexistent or unknown.)
* @param enableSuppression whether or not suppression is enabled
* or disabled
* @param writableStackTrace whether or not the stack trace should
* be writable
*
* @since 1.7
*/
protected RemotingException(String message, Throwable cause,
boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

30
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java
View File

@ -170,21 +170,6 @@ public class ResponseFuture {
}
}
@Override
public String toString() {
return "ResponseFuture{" +
"opaque=" + opaque +
", timeoutMillis=" + timeoutMillis +
", invokeCallback=" + invokeCallback +
", releaseSemaphore=" + releaseSemaphore +
", latch=" + latch +
", beginTimestamp=" + beginTimestamp +
", responseCommand=" + responseCommand +
", sendOk=" + sendOk +
", cause=" + cause +
'}';
}
/**
* scan future table
*/
@ -209,4 +194,19 @@ public class ResponseFuture {
}
}
}
@Override
public String toString() {
return "ResponseFuture{" +
"opaque=" + opaque +
", timeoutMillis=" + timeoutMillis +
", invokeCallback=" + invokeCallback +
", releaseSemaphore=" + releaseSemaphore +
", latch=" + latch +
", beginTimestamp=" + beginTimestamp +
", responseCommand=" + responseCommand +
", sendOk=" + sendOk +
", cause=" + cause +
'}';
}
}

2
dolphinscheduler-server/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-server</artifactId>
<name>dolphinscheduler-server</name>

16
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java
View File

@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.server.entity;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
/**
* SQL Task ExecutionContext
@ -38,9 +38,9 @@ public class SQLTaskExecutionContext implements Serializable {
*/
private String connectionParams;
/**
* udf function list
* udf function tenant code map
*/
private List<UdfFunc> udfFuncList;
private Map<UdfFunc,String> udfFuncTenantCodeMap;
public int getWarningGroupId() {
@ -51,12 +51,12 @@ public class SQLTaskExecutionContext implements Serializable {
this.warningGroupId = warningGroupId;
}
public List<UdfFunc> getUdfFuncList() {
return udfFuncList;
public Map<UdfFunc, String> getUdfFuncTenantCodeMap() {
return udfFuncTenantCodeMap;
}
public void setUdfFuncList(List<UdfFunc> udfFuncList) {
this.udfFuncList = udfFuncList;
public void setUdfFuncTenantCodeMap(Map<UdfFunc, String> udfFuncTenantCodeMap) {
this.udfFuncTenantCodeMap = udfFuncTenantCodeMap;
}
public String getConnectionParams() {
@ -72,7 +72,7 @@ public class SQLTaskExecutionContext implements Serializable {
return "SQLTaskExecutionContext{" +
"warningGroupId=" + warningGroupId +
", connectionParams='" + connectionParams + '\'' +
", udfFuncList=" + udfFuncList +
", udfFuncTenantCodeMap=" + udfFuncTenantCodeMap +
'}';
}
}

9
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java
View File

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
@ -171,9 +170,9 @@ public class TaskExecutionContext implements Serializable{
private String workerGroup;
/**
* resources full name
* resources full name and tenant code
*/
private List<String> resources;
private Map<String,String> resources;
/**
* sql TaskExecutionContext
@ -446,11 +445,11 @@ public class TaskExecutionContext implements Serializable{
this.dependenceTaskExecutionContext = dependenceTaskExecutionContext;
}
public List<String> getResources() {
public Map<String, String> getResources() {
return resources;
}
public void setResources(List<String> resources) {
public void setResources(Map<String, String> resources) {
this.resources = resources;
}

36
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
View File

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.consumer;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.SqoopJobType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.model.TaskNode;
@ -49,14 +50,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS;
/**
* TaskUpdateQueue consumer
*/
@ -330,7 +327,13 @@ public class TaskPriorityQueueConsumer extends Thread{
}
List<UdfFunc> udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray);
sqlTaskExecutionContext.setUdfFuncList(udfFuncList);
Map<UdfFunc,String> udfFuncMap = new HashMap<>();
for(UdfFunc udfFunc : udfFuncList) {
String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF);
udfFuncMap.put(udfFunc,tenantCode);
}
sqlTaskExecutionContext.setUdfFuncTenantCodeMap(udfFuncMap);
}
}
@ -364,20 +367,23 @@ public class TaskPriorityQueueConsumer extends Thread{
}
/**
* get resource full name list
* get resource map key is full name and value is tenantCode
*/
private List<String> getResourceFullNames(TaskNode taskNode) {
List<String> resourceFullNameList = new ArrayList<>();
private Map<String,String> getResourceFullNames(TaskNode taskNode) {
Map<String,String> resourceMap = new HashMap<>();
AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams());
if (baseParam != null) {
List<ResourceInfo> projectResourceFiles = baseParam.getResourceFilesList();
if (projectResourceFiles != null) {
if (CollectionUtils.isNotEmpty(projectResourceFiles)) {
// filter the resources that the resource id equals 0
Set<ResourceInfo> oldVersionResources = projectResourceFiles.stream().filter(t -> t.getId() == 0).collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(oldVersionResources)) {
resourceFullNameList.addAll(oldVersionResources.stream().map(resource -> resource.getRes()).collect(Collectors.toSet()));
oldVersionResources.forEach(
(t)->resourceMap.put(t.getRes(), processService.queryTenantCodeByResName(t.getRes(), ResourceType.FILE))
);
}
// get the resource id in order to get the resource names in batch
@ -388,13 +394,13 @@ public class TaskPriorityQueueConsumer extends Thread{
Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]);
List<Resource> resources = processService.listResourceByIds(resourceIds);
resourceFullNameList.addAll(resources.stream()
.map(resourceInfo -> resourceInfo.getFullName())
.collect(Collectors.toList()));
resources.forEach(
(t)->resourceMap.put(t.getFullName(),processService.queryTenantCodeByResName(t.getFullName(), ResourceType.FILE))
);
}
}
}
return resourceFullNameList;
return resourceMap;
}
}

53
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java
View File

@ -26,68 +26,21 @@ public class ExecuteException extends Exception{
super();
}
/**
* Constructs a new exception with the specified detail message. The
* cause is not initialized, and may subsequently be initialized by
* a call to {@link #initCause}.
*
* @param message the detail message. The detail message is saved for
* later retrieval by the {@link #getMessage()} method.
*/
public ExecuteException(String message) {
super(message);
}
/**
* Constructs a new exception with the specified detail message and
* cause. <p>Note that the detail message associated with
* {@code cause} is <i>not</i> automatically incorporated in
* this exception's detail message.
*
* @param message the detail message (which is saved for later retrieval
* by the {@link #getMessage()} method).
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public ExecuteException(String message, Throwable cause) {
super(message, cause);
}
/**
* Constructs a new exception with the specified cause and a detail
* message of <tt>(cause==null ? null : cause.toString())</tt> (which
* typically contains the class and detail message of <tt>cause</tt>).
* This constructor is useful for exceptions that are little more than
* wrappers for other throwables (for example, {@link
* java.security.PrivilegedActionException}).
*
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public ExecuteException(Throwable cause) {
super(cause);
}
/**
* Constructs a new exception with the specified detail message,
* cause, suppression enabled or disabled, and writable stack
* trace enabled or disabled.
*
* @param message the detail message.
* @param cause the cause. (A {@code null} value is permitted,
* and indicates that the cause is nonexistent or unknown.)
* @param enableSuppression whether or not suppression is enabled
* or disabled
* @param writableStackTrace whether or not the stack trace should
* be writable
* @since 1.7
*/
protected ExecuteException(String message, Throwable cause,
boolean enableSuppression,
boolean writableStackTrace) {

65
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.commons.collections.MapUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
@ -24,10 +25,8 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.slf4j.Logger;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static org.apache.dolphinscheduler.common.utils.CollectionUtils.isNotEmpty;
@ -43,53 +42,44 @@ public class UDFUtils {
/**
* create function list
* @param udfFuncs udf functions
* @param tenantCode tenant code
* @param logger logger
* @param udfFuncTenantCodeMap key is udf function,value is tenant code
* @param logger logger
* @return create function list
*/
public static List<String> createFuncs(List<UdfFunc> udfFuncs, String tenantCode,Logger logger){
public static List<String> createFuncs(Map<UdfFunc,String> udfFuncTenantCodeMap, Logger logger){
if (CollectionUtils.isEmpty(udfFuncs)){
if (MapUtils.isEmpty(udfFuncTenantCodeMap)){
logger.info("can't find udf function resource");
return null;
}
// get hive udf jar path
String hiveUdfJarPath = HadoopUtils.getHdfsUdfDir(tenantCode);
logger.info("hive udf jar path : {}" , hiveUdfJarPath);
// is the root directory of udf defined
if (StringUtils.isEmpty(hiveUdfJarPath)) {
logger.error("not define hive udf jar path");
throw new RuntimeException("hive udf jar base path not defined ");
}
Set<String> resources = getFuncResouces(udfFuncs);
List<String> funcList = new ArrayList<>();
// build jar sql
buildJarSql(funcList, resources, hiveUdfJarPath);
buildJarSql(funcList, udfFuncTenantCodeMap);
// build temp function sql
buildTempFuncSql(funcList, udfFuncs);
buildTempFuncSql(funcList, udfFuncTenantCodeMap.keySet().stream().collect(Collectors.toList()));
return funcList;
}
/**
* build jar sql
* @param sqls sql list
* @param resources resource set
* @param uploadPath upload path
* @param sqls sql list
* @param udfFuncTenantCodeMap key is udf function,value is tenant code
*/
private static void buildJarSql(List<String> sqls, Set<String> resources, String uploadPath) {
private static void buildJarSql(List<String> sqls, Map<UdfFunc,String> udfFuncTenantCodeMap) {
String defaultFS = HadoopUtils.getInstance().getConfiguration().get(Constants.FS_DEFAULTFS);
if (!uploadPath.startsWith("hdfs:")) {
uploadPath = defaultFS + uploadPath;
Set<Map.Entry<UdfFunc,String>> entries = udfFuncTenantCodeMap.entrySet();
for (Map.Entry<UdfFunc,String> entry:entries){
String uploadPath = HadoopUtils.getHdfsUdfDir(entry.getValue());
if (!uploadPath.startsWith("hdfs:")) {
uploadPath = defaultFS + uploadPath;
}
sqls.add(String.format("add jar %s%s", uploadPath, entry.getKey().getResourceName()));
}
for (String resource : resources) {
sqls.add(String.format("add jar %s/%s", uploadPath, resource));
}
}
/**
@ -106,20 +96,5 @@ public class UDFUtils {
}
}
/**
* get the resource names of all functions
* @param udfFuncs udf function list
* @return
*/
private static Set<String> getFuncResouces(List<UdfFunc> udfFuncs) {
Set<String> resources = new HashSet<>();
for (UdfFunc udfFunc : udfFuncs) {
resources.add(udfFunc.getResourceName());
}
return resources;
}
}

20
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
View File

@ -22,7 +22,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.Set;
import org.apache.commons.collections.MapUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.TaskNode;
@ -106,7 +108,6 @@ public class TaskExecuteThread implements Runnable {
// copy hdfs/minio file to local
downloadResource(taskExecutionContext.getExecutePath(),
taskExecutionContext.getResources(),
taskExecutionContext.getTenantCode(),
logger);
taskExecutionContext.setTaskParams(taskNode.getParams());
@ -227,22 +228,25 @@ public class TaskExecuteThread implements Runnable {
* @param logger
*/
private void downloadResource(String execLocalPath,
List<String> projectRes,
String tenantCode,
Map<String,String> projectRes,
Logger logger) throws Exception {
if (CollectionUtils.isEmpty(projectRes)){
if (MapUtils.isEmpty(projectRes)){
return;
}
for (String resource : projectRes) {
File resFile = new File(execLocalPath, resource);
Set<Map.Entry<String, String>> resEntries = projectRes.entrySet();
for (Map.Entry<String,String> resource : resEntries) {
String fullName = resource.getKey();
String tenantCode = resource.getValue();
File resFile = new File(execLocalPath, fullName);
if (!resFile.exists()) {
try {
// query the tenant code of the resource according to the name of the resource
String resHdfsPath = HadoopUtils.getHdfsResourceFileName(tenantCode, resource);
String resHdfsPath = HadoopUtils.getHdfsResourceFileName(tenantCode, fullName);
logger.info("get resource file from hdfs :{}", resHdfsPath);
HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + resource, false, true);
HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + fullName, false, true);
}catch (Exception e){
logger.error(e.getMessage(),e);
throw new RuntimeException(e.getMessage());

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
View File

@ -131,8 +131,7 @@ public class SqlTask extends AbstractTask {
.map(this::getSqlAndSqlParamsMap)
.collect(Collectors.toList());
List<String> createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncList(),
taskExecutionContext.getTenantCode(),
List<String> createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncTenantCodeMap(),
logger);
// execute sql task

2
dolphinscheduler-service/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

2
dolphinscheduler-ui/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

4
pom.xml
View File

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.3.2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<url>http://dolphinscheduler.apache.org</url>
@ -916,7 +916,7 @@
<exclude>**/dist/**</exclude>
<exclude>**/licenses/**</exclude>
<exclude>.github/**</exclude>
<exclude>sql/soft_version</exclude>
<exclude>**/sql/soft_version</exclude>
<exclude>**/common/utils/ScriptRunner.java</exclude>
<exclude>**/*.json</exclude>
<!-- document files -->

2
sql/soft_version
View File

@ -1 +1 @@
1.3.0
1.3.1