test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-30 03:08:01 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

add http task timeout & test unit (#2955)

Browse Source 
* add http task timeout & test unit

* add http task timeout & test unit

Co-authored-by: zhanglong <zhanglong@ysstech.com>
Co-authored-by: dailidong <dailidong66@gmail.com>
This commit is contained in:
BoYiZhang 2020-06-12 18:52:19 +08:00 committed by GitHub
parent 284c50f66c
commit f56ae9e64a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 399 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java
View File

@ -56,6 +56,17 @@ public class HttpParameters extends AbstractParameters {
private String condition;
/**
* Connect Timeout
* Unit: ms
*/
private int connectTimeout ;
/**
* Socket Timeout
* Unit: ms
*/
private int socketTimeout ;
@Override
public boolean checkParameters() {
@ -106,4 +117,20 @@ public class HttpParameters extends AbstractParameters {
public void setCondition(String condition) {
this.condition = condition;
}
public int getConnectTimeout() {
return connectTimeout;
}
public void setConnectTimeout(int connectTimeout) {
this.connectTimeout = connectTimeout;
}
public int getSocketTimeout() {
return socketTimeout;
}
public void setSocketTimeout(int socketTimeout) {
this.socketTimeout = socketTimeout;
}
}

91
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.task;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.enums.HttpCheckCondition;
import org.apache.dolphinscheduler.common.enums.HttpMethod;
import org.apache.dolphinscheduler.common.process.HttpProperty;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.fastjson.JSON;
/**
* http parameter
*/
public class HttpParametersTest {
@Test
public void testGenerator(){
String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +
"\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);
Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
Assert.assertEquals(10000,httpParameters.getSocketTimeout());
Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());
Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
Assert.assertEquals("",httpParameters.getCondition());
}
@Test
public void testCheckParameters(){
String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +
"\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);
Assert.assertTrue( httpParameters.checkParameters());
Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
Assert.assertEquals(10000,httpParameters.getSocketTimeout());
Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());
Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
Assert.assertEquals("",httpParameters.getCondition());
}
@Test
public void testCheckValues() {
String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +
"\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);
Assert.assertTrue( httpParameters.checkParameters());
Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
Assert.assertEquals(10000,httpParameters.getSocketTimeout());
Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());
Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
Assert.assertEquals("",httpParameters.getCondition());
Assert.assertEquals(0,httpParameters.getLocalParametersMap().size());
Assert.assertEquals(0,httpParameters.getResourceFilesList().size());
}
}

9
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
View File

@ -66,13 +66,6 @@ public class HttpTask extends AbstractTask {
*/
private HttpParameters httpParameters;
/**
* Convert mill seconds to second unit
*/
protected static final int MAX_CONNECTION_MILLISECONDS = 60000;
/**
* application json
*/
@ -303,7 +296,7 @@ public class HttpTask extends AbstractTask {
* @return RequestConfig
*/
private RequestConfig requestConfig() {
return RequestConfig.custom().setSocketTimeout(MAX_CONNECTION_MILLISECONDS).setConnectTimeout(MAX_CONNECTION_MILLISECONDS).build();
return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build();
}
/**

208
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java Normal file
View File

@ -0,0 +1,208 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.task.http;
import static org.apache.dolphinscheduler.common.enums.CommandType.*;
import java.io.IOException;
import java.util.Date;
import org.apache.dolphinscheduler.common.enums.HttpCheckCondition;
import org.apache.dolphinscheduler.common.enums.HttpMethod;
import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import com.alibaba.fastjson.JSON;
@RunWith(PowerMockRunner.class)
@PrepareForTest(OSUtils.class)
@PowerMockIgnore({"javax.management.*","javax.net.ssl.*"})
public class HttpTaskTest {
private static final Logger logger = LoggerFactory.getLogger(HttpTaskTest.class);
private HttpTask httpTask;
private ProcessService processService;
private ShellCommandExecutor shellCommandExecutor;
private ApplicationContext applicationContext;
private TaskExecutionContext taskExecutionContext;
@Before
public void before() throws Exception {
taskExecutionContext = new TaskExecutionContext();
PowerMockito.mockStatic(OSUtils.class);
processService = PowerMockito.mock(ProcessService.class);
shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class);
applicationContext = PowerMockito.mock(ApplicationContext.class);
SpringApplicationContext springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
TaskProps props = new TaskProps();
props.setExecutePath("/tmp");
props.setTaskAppId(String.valueOf(System.currentTimeMillis()));
props.setTaskInstanceId(1);
props.setTenantCode("1");
props.setEnvFile(".dolphinscheduler_env.sh");
props.setTaskStartTime(new Date());
props.setTaskTimeout(0);
props.setTaskParams(
"{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\",\"httpMethod\":\"GET\"," +
"\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"https://github.com/\"," +
"\"connectTimeout\":\"1000\",\"socketTimeout\":\"1000\"}");
taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams());
Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp");
Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("1");
Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root");
Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date());
Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000);
Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dx");
httpTask = new HttpTask(taskExecutionContext, logger);
httpTask.init();
}
@Test
public void testGetParameters() {
Assert.assertNotNull(httpTask.getParameters());
}
@Test
public void testCheckParameters() {
Assert.assertTrue(httpTask.getParameters().checkParameters());
}
@Test
public void testGenerator(){
String paramJson = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\"," +
"\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";
HttpParameters httpParameters = JSON.parseObject(paramJson, HttpParameters.class);
Assert.assertEquals(10000,httpParameters.getConnectTimeout() );
Assert.assertEquals(10000,httpParameters.getSocketTimeout());
Assert.assertEquals("https://github.com/",httpParameters.getUrl());
Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());
Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());
Assert.assertEquals("",httpParameters.getCondition());
}
@Test
public void testHandle(){
boolean flag = true ;
try {
httpTask.handle();
} catch (Exception e) {
flag = false ;
e.printStackTrace();
}
Assert.assertTrue(flag);
}
@Test
public void testSendRequest(){
CloseableHttpClient client = httpTask.createHttpClient();
String statusCode = null;
String body = null;
try {
CloseableHttpResponse response = httpTask.sendRequest(client) ;
statusCode = String.valueOf(httpTask.getStatusCode(response));
body = httpTask.getResponseBody(response);
int exitStatusCode = httpTask.validResponse(body, statusCode);
Assert.assertNotEquals(-1,exitStatusCode);
} catch (IOException e) {
e.printStackTrace();
};
}
@Test
public void testValidResponse(){
String body = "body";
String statusCode = "200" ;
int exitStatusCode = httpTask.validResponse(body,statusCode);
Assert.assertNotEquals(-1,exitStatusCode);
}
@Test
public void testAppendMessage(){
httpTask.appendMessage("message");
Assert.assertEquals("message",httpTask.getOutput());
}
@Test
public void testCreateHttpClient(){
Assert.assertNotNull(httpTask.createHttpClient());
}
@Test
public void testCreateRequestBuilder(){
RequestBuilder requestBuilder = httpTask.createRequestBuilder();
Assert.assertEquals(RequestBuilder.get().getMethod(),requestBuilder.getMethod());
}
private ProcessInstance getProcessInstance() {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setCommandType(START_PROCESS);
processInstance.setScheduleTime(new Date());
return processInstance;
}
}

58
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue
View File

@ -85,6 +85,39 @@
</x-input>
</div>
</m-list-box>
<m-list-box >
<div slot="text">{{$t('Timeout Settings')}}</div>
<div slot="content">
<label class="label-box">
<div style="padding-top: 5px;">
<x-switch
v-model="timeoutSettings"
:disabled="isDetails"
></x-switch>
</div>
</label>
</div>
</m-list-box>
<div class="clearfix list" v-if = "timeoutSettings" >
<div class="text-box">
<span>{{$t('Connect Timeout')}}</span>
</div>
<div class="cont-box">
<span class="label-box" style="width: 193px;display: inline-block;" >
<x-input v-model='connectTimeout' maxlength="7" />
</span>
<span>{{$t('ms')}}</span>
<span class="text-b">{{$t('Socket Timeout')}}</span>
<span class="label-box" style="width: 193px;display: inline-block;" >
<x-input v-model='socketTimeout' maxlength="7" />
</span>
<span>{{$t('ms')}}</span>
</div>
</div>
<m-list-box>
<div slot="text">{{$t('Custom Parameters')}}</div>
<div slot="content">
@ -110,6 +143,10 @@
name: 'http',
data () {
return {
timeoutSettings: false,
connectTimeout : 60000 ,
socketTimeout : 60000 ,
url: '',
condition: '',
localParams: [],
@ -152,6 +189,14 @@
if (!this.$refs.refHttpParams._verifValue()) {
return false
}
if (!_.isNumber(parseInt(this.socketTimeout))) {
this.$message.warning(`${i18n.$t('Socket Timeout be a positive integer')}`)
return false
}
if (!_.isNumber(parseInt(this.connectTimeout))) {
this.$message.warning(`${i18n.$t('Connect timeout be a positive integer')}`)
return false
}
// storage
this.$emit('on-params', {
localParams: this.localParams,
@ -159,7 +204,9 @@
url: this.url,
httpMethod: this.httpMethod,
httpCheckCondition: this.httpCheckCondition,
condition: this.condition
condition: this.condition,
connectTimeout : this.connectTimeout ,
socketTimeout : this.socketTimeout
})
return true
}
@ -172,7 +219,9 @@
url: this.url,
httpMethod: this.httpMethod,
httpCheckCondition: this.httpCheckCondition,
condition: this.condition
condition: this.condition,
connectTimeout : this.connectTimeout ,
socketTimeout : this.socketTimeout
}
}
},
@ -193,6 +242,11 @@
this.httpMethod = o.params.httpMethod || 'GET'
this.httpCheckCondition = o.params.httpCheckCondition || 'DEFAULT'
this.condition = o.params.condition || ''
this.connectTimeout = o.params.connectTimeout
this.socketTimeout = o.params.socketTimeout
if(this.connectTimeout != 60000 || this.socketTimeout != 60000 ){
this.timeoutSettings = true
}
// backfill localParams
let localParams = o.params.localParams || []
if (localParams.length) {

8
dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
View File

@ -600,5 +600,11 @@ export default {
'Unauthorized or deleted resources': 'Unauthorized or deleted resources',
'Please delete all non-existent resources': 'Please delete all non-existent resources',
'Enable': 'Enable',
'Disable': 'Disable'
'Disable': 'Disable',
'Timeout Settings': 'Timeout Settings',
'Connect Timeout':'Connect Timeout',
'Socket Timeout':'Socket Timeout',
'Connect timeout be a positive integer': 'Connect timeout be a positive integer',
'Socket Timeout be a positive integer': 'Socket Timeout be a positive integer',
'ms':'ms'
}

8
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
View File

@ -600,5 +600,11 @@ export default {
'Unauthorized or deleted resources': '未授权或已删除资源',
'Please delete all non-existent resources': '请删除所有未授权或已删除资源',
'Enable': '启用',
'Disable': '停用'
'Disable': '停用',
'Timeout Settings': '超时设置',
'Connect Timeout':'连接超时',
'Socket Timeout':'Socket超时',
'Connect timeout be a positive integer': '连接超时必须为数字',
'Socket Timeout be a positive integer': 'Socket超时必须为数字',
'ms':'毫秒'
}

2
pom.xml
View File

@ -749,6 +749,7 @@
<include>**/common/shell/ShellExecutorTest.java</include>
<include>**/common/task/EntityTestUtils.java</include>
<include>**/common/task/FlinkParametersTest.java</include>
<include>**/common/task/HttpParametersTest.java</include>
<include>**/common/task/SqoopParameterEntityTest.java</include>
<include>**/common/threadutils/ThreadPoolExecutorsTest.java</include>
<include>**/common/threadutils/ThreadUtilsTest.java</include>
@ -821,6 +822,7 @@
<include>**/server/worker/task/spark/SparkTaskTest.java</include>
<include>**/server/worker/task/EnvFileTest.java</include>
<include>**/server/worker/task/spark/SparkTaskTest.java</include>
<include>**/server/worker/task/http/HttpTaskTest.java</include>
<include>**/server/worker/task/sqoop/SqoopTaskTest.java</include>
<include>**/server/worker/EnvFileTest.java</include>
<include>**/service/quartz/cron/CronUtilsTest.java</include>