test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-03 04:39:00 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

Add FlinkArgsUtilsTest which is the UT of FlinkArgsUtils (#1557)

Browse Source 
1.Add FlinkArgsUtilsTest
2.Optimize FlinkArgsUtils and fix bug.
use StringUtils.isNotEmpty(queue) instead of StringUtils.isNotEmpty(others)

3.Add the UT path in pom
This commit is contained in:
zhukai 2019-12-26 14:15:58 +08:00 committed by lgcareer
parent 18e8703b70
commit 0f909fca12
3 changed files with 172 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java
View File

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -44,9 +45,11 @@ public class FlinkArgsUtils {
*/ */
public static List<String> buildArgs(FlinkParameters param) { public static List<String> buildArgs(FlinkParameters param) {
List<String> args = new ArrayList<>(); List<String> args = new ArrayList<>();
String deployMode = "cluster"; String deployMode = "cluster";
if (StringUtils.isNotEmpty(param.getDeployMode())) { String tmpDeployMode = param.getDeployMode();
deployMode = param.getDeployMode(); if (StringUtils.isNotEmpty(tmpDeployMode)) {
deployMode = tmpDeployMode;
} }
if (!"local".equals(deployMode)) { if (!"local".equals(deployMode)) {
@ -54,68 +57,70 @@ public class FlinkArgsUtils {
args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster
int slot = param.getSlot();
if (param.getSlot() != 0) { if (slot != 0) {
args.add(Constants.FLINK_YARN_SLOT); args.add(Constants.FLINK_YARN_SLOT);
args.add(String.format("%d", param.getSlot())); //-ys args.add(String.format("%d", slot)); //-ys
} }
if (StringUtils.isNotEmpty(param.getAppName())) { //-ynm String appName = param.getAppName();
if (StringUtils.isNotEmpty(appName)) { //-ynm
args.add(Constants.FLINK_APP_NAME); args.add(Constants.FLINK_APP_NAME);
args.add(param.getAppName()); args.add(appName);
} }
if (param.getTaskManager() != 0) { //-yn int taskManager = param.getTaskManager();
if (taskManager != 0) { //-yn
args.add(Constants.FLINK_TASK_MANAGE); args.add(Constants.FLINK_TASK_MANAGE);
args.add(String.format("%d", param.getTaskManager())); args.add(String.format("%d", taskManager));
} }
if (StringUtils.isNotEmpty(param.getJobManagerMemory())) { String jobManagerMemory = param.getJobManagerMemory();
if (StringUtils.isNotEmpty(jobManagerMemory)) {
args.add(Constants.FLINK_JOB_MANAGE_MEM); args.add(Constants.FLINK_JOB_MANAGE_MEM);
args.add(param.getJobManagerMemory()); //-yjm args.add(jobManagerMemory); //-yjm
} }
if (StringUtils.isNotEmpty(param.getTaskManagerMemory())) { // -ytm String taskManagerMemory = param.getTaskManagerMemory();
if (StringUtils.isNotEmpty(taskManagerMemory)) { // -ytm
args.add(Constants.FLINK_TASK_MANAGE_MEM); args.add(Constants.FLINK_TASK_MANAGE_MEM);
args.add(param.getTaskManagerMemory()); args.add(taskManagerMemory);
} }
args.add(Constants.FLINK_detach); //-d args.add(Constants.FLINK_detach); //-d
} }
if (param.getProgramType() != null) { ProgramType programType = param.getProgramType();
if (param.getProgramType() != ProgramType.PYTHON) { String mainClass = param.getMainClass();
if (StringUtils.isNotEmpty(param.getMainClass())) { if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) {
args.add(Constants.FLINK_MAIN_CLASS); //-c args.add(Constants.FLINK_MAIN_CLASS); //-c
args.add(param.getMainClass()); //main class args.add(param.getMainClass()); //main class
}
}
} }
if (param.getMainJar() != null) { ResourceInfo mainJar = param.getMainJar();
args.add(param.getMainJar().getRes()); if (mainJar != null) {
args.add(mainJar.getRes());
} }
if (StringUtils.isNotEmpty(param.getMainArgs())) { String mainArgs = param.getMainArgs();
args.add(param.getMainArgs()); if (StringUtils.isNotEmpty(mainArgs)) {
args.add(mainArgs);
} }
// --files --conf --libjar ... // --files --conf --libjar ...
if (StringUtils.isNotEmpty(param.getOthers())) { String others = param.getOthers();
String others = param.getOthers(); String queue = param.getQueue();
if (!others.contains("--qu")) { if (StringUtils.isNotEmpty(others)) {
if (StringUtils.isNotEmpty(param.getQueue()) && !deployMode.equals("local")) {
args.add(Constants.FLINK_QUEUE); if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) {
args.add(param.getQueue()); args.add(Constants.FLINK_QUEUE);
} args.add(param.getQueue());
} }
args.add(param.getOthers()); args.add(others);
} else if (StringUtils.isNotEmpty(param.getQueue()) && !deployMode.equals("local")) { } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) {
args.add(Constants.FLINK_QUEUE); args.add(Constants.FLINK_QUEUE);
args.add(param.getQueue()); args.add(param.getQueue());
} }
return args; return args;

131
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java Normal file
View File

@ -0,0 +1,131 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
/**
* Test FlinkArgsUtils
*/
public class FlinkArgsUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(FlinkArgsUtilsTest.class);
public String mode = "cluster";
public int slot = 2;
public String appName = "testFlink";
public int taskManager = 4;
public String taskManagerMemory = "2G";
public String jobManagerMemory = "4G";
public ProgramType programType = ProgramType.JAVA;
public String mainClass = "com.test";
public ResourceInfo mainJar = null;
public String mainArgs = "testArgs";
public String queue = "queue1";
public String others = "--input file:///home";
@Before
public void setUp() throws Exception {
ResourceInfo main = new ResourceInfo();
main.setRes("testflink-1.0.0-SNAPSHOT.jar");
mainJar = main;
}
/**
* Test buildArgs
*/
@Test
public void testBuildArgs() {
//Define params
FlinkParameters param = new FlinkParameters();
param.setDeployMode(mode);
param.setMainClass(mainClass);
param.setAppName(appName);
param.setSlot(slot);
param.setTaskManager(taskManager);
param.setJobManagerMemory(jobManagerMemory);
param.setTaskManagerMemory(taskManagerMemory);
param.setMainJar(mainJar);
param.setProgramType(programType);
param.setMainArgs(mainArgs);
param.setQueue(queue);
param.setOthers(others);
//Invoke buildArgs
List<String> result = FlinkArgsUtils.buildArgs(param);
for (String s : result) {
logger.info(s);
}
//Expected values and order
assertEquals(result.size(),20);
assertEquals(result.get(0),"-m");
assertEquals(result.get(1),"yarn-cluster");
assertEquals(result.get(2),"-ys");
assertSame(Integer.valueOf(result.get(3)),slot);
assertEquals(result.get(4),"-ynm");
assertEquals(result.get(5),appName);
assertEquals(result.get(6),"-yn");
assertSame(Integer.valueOf(result.get(7)),taskManager);
assertEquals(result.get(8),"-yjm");
assertEquals(result.get(9),jobManagerMemory);
assertEquals(result.get(10),"-ytm");
assertEquals(result.get(11),taskManagerMemory);
assertEquals(result.get(12),"-d");
assertEquals(result.get(13),"-c");
assertEquals(result.get(14),mainClass);
assertEquals(result.get(15),mainJar.getRes());
assertEquals(result.get(16),mainArgs);
assertEquals(result.get(17),"--qu");
assertEquals(result.get(18),queue);
assertEquals(result.get(19),others);
//Others param without --qu
FlinkParameters param1 = new FlinkParameters();
param1.setQueue(queue);
param1.setDeployMode(mode);
result = FlinkArgsUtils.buildArgs(param1);
assertEquals(result.size(),5);
}
}

1
pom.xml
View File

@ -655,6 +655,7 @@
<include>**/alert/utils/JSONUtilsTest.java</include> <include>**/alert/utils/JSONUtilsTest.java</include>
<include>**/alert/utils/PropertyUtilsTest.java</include> <include>**/alert/utils/PropertyUtilsTest.java</include>
<include>**/server/utils/SparkArgsUtilsTest.java</include> <include>**/server/utils/SparkArgsUtilsTest.java</include>
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
</includes> </includes>
<!-- <skip>true</skip> --> <!-- <skip>true</skip> -->
</configuration> </configuration>