test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-30 03:08:01 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Improvement] The spark version is associated with the spark command (#7544)

Browse Source 
* [Improvement] The spark version is associated with the spark command
This commit is contained in:
J·Y 2022-01-04 14:21:57 +08:00 committed by GitHub
parent 4b22ad6cf6
commit a4168ae24e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java
View File

@ -34,18 +34,6 @@ import java.util.Map;
public class SparkTask extends AbstractYarnTask {
/**
* spark1 command
* usage: spark-submit [options] <app jar | python file> [app arguments]
*/
private static final String SPARK1_COMMAND = "${SPARK_HOME1}/bin/spark-submit";
/**
* spark2 command
* usage: spark-submit [options] <app jar | python file> [app arguments]
*/
private static final String SPARK2_COMMAND = "${SPARK_HOME2}/bin/spark-submit";
/**
* spark parameters
*/
@ -90,10 +78,10 @@ public class SparkTask extends AbstractYarnTask {
List<String> args = new ArrayList<>();
// spark version
String sparkCommand = SPARK2_COMMAND;
String sparkCommand = SparkVersion.SPARK2.getCommand();
if (SparkVersion.SPARK1.name().equals(sparkParameters.getSparkVersion())) {
sparkCommand = SPARK1_COMMAND;
sparkCommand = SparkVersion.SPARK1.getCommand();
}
args.add(sparkCommand);

24
dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkVersion.java
View File

@ -23,16 +23,22 @@ public enum SparkVersion {
* 0 SPARK1
* 1 SPARK2
*/
SPARK1(0, "SPARK1"),
SPARK2(1, "SPARK2");
SparkVersion(int code, String descp) {
this.code = code;
this.descp = descp;
}
SPARK1(0, "SPARK1", "${SPARK_HOME1}/bin/spark-submit"),
SPARK2(1, "SPARK2", "${SPARK_HOME2}/bin/spark-submit"),
;
private final int code;
private final String descp;
/**
* usage: spark-submit [options] <app jar | python file> [app arguments]
*/
private final String command;
SparkVersion(int code, String descp, String command) {
this.code = code;
this.descp = descp;
this.command = command;
}
public int getCode() {
return code;
@ -41,4 +47,8 @@ public enum SparkVersion {
public String getDescp() {
return descp;
}
public String getCommand() {
return command;
}
}