Merge remote-tracking branch 'upstream/dev' into dev

This commit is contained in:
loushang 2019-12-30 18:47:50 +08:00
commit 2106374dc8
101 changed files with 4039 additions and 1044 deletions

51
.github/workflows/SonarCloud.yml vendored Normal file
View File

@ -0,0 +1,51 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: SonarCloud
on: [push, pull_request]
jobs:
sonarCloudTrigger:
name: SonarCloud Trigger
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-java@v1
with:
java-version: 8
- uses: actions/cache@v1
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube
restore-keys: |
${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube
${{ runner.os }}-maven-
- name: Maven clean
run: mvn clean
- name: Run SonarCloud analyse
run: >
mvn clean --batch-mode
org.jacoco:jacoco-maven-plugin:prepare-agent
verify
org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
-Dmaven.test.skip=true
-Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache
-Dsonar.projectKey=apache-dolphinscheduler
-Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

View File

@ -4,7 +4,8 @@ Dolphin Scheduler Official Website
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data

View File

@ -4,7 +4,7 @@ Dolphin Scheduler Official Website
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data

View File

@ -39,19 +39,15 @@ public class AlertServer {
private AlertSender alertSender;
private static volatile AlertServer instance;
private static AlertServer instance;
public AlertServer() {
}
public static AlertServer getInstance(){
public synchronized static AlertServer getInstance(){
if (null == instance) {
synchronized (AlertServer.class) {
if(null == instance) {
instance = new AlertServer();
}
}
instance = new AlertServer();
}
return instance;
}
@ -63,6 +59,7 @@ public class AlertServer {
Thread.sleep(Constants.ALERT_SCAN_INTERVEL);
} catch (InterruptedException e) {
logger.error(e.getMessage(),e);
Thread.currentThread().interrupt();
}
List<Alert> alerts = alertDao.listWaitExecutionAlert();
alertSender = new AlertSender(alerts, alertDao);

View File

@ -86,20 +86,24 @@ public class EnterpriseWeChatUtils {
String resp;
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace);
CloseableHttpResponse response = httpClient.execute(httpGet);
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, Constants.UTF_8);
EntityUtils.consume(entity);
} finally {
response.close();
}
HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace);
CloseableHttpResponse response = httpClient.execute(httpGet);
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, Constants.UTF_8);
EntityUtils.consume(entity);
} finally {
response.close();
}
Map<String, Object> map = JSON.parseObject(resp,
new TypeToken<Map<String, Object>>() {
}.getType());
return map.get("access_token").toString();
Map<String, Object> map = JSON.parseObject(resp,
new TypeToken<Map<String, Object>>() {
}.getType());
return map.get("access_token").toString();
} finally {
httpClient.close();
}
}
/**
@ -167,20 +171,25 @@ public class EnterpriseWeChatUtils {
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token);
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace);
httpPost.setEntity(new StringEntity(data, charset));
CloseableHttpResponse response = httpclient.execute(httpPost);
String resp;
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, charset);
EntityUtils.consume(entity);
HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace);
httpPost.setEntity(new StringEntity(data, charset));
CloseableHttpResponse response = httpClient.execute(httpPost);
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, charset);
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}",
enterpriseWeChatPushUrl, data, resp);
return resp;
} finally {
response.close();
httpClient.close();
}
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp);
return resp;
}
/**

View File

@ -154,8 +154,13 @@ public class AccessTokenService extends BaseService {
*/
public Map<String, Object> updateToken(int id,int userId, String expireTime, String token) {
Map<String, Object> result = new HashMap<>(5);
AccessToken accessToken = new AccessToken();
accessToken.setId(id);
AccessToken accessToken = accessTokenMapper.selectById(id);
if (accessToken == null) {
logger.error("access token not exist, access token id {}", id);
putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST);
return result;
}
accessToken.setUserId(userId);
accessToken.setExpireTime(DateUtils.stringToDate(expireTime));
accessToken.setToken(token);

View File

@ -193,6 +193,12 @@ public class AlertGroupService extends BaseService{
if (checkAdmin(loginUser, result)){
return result;
}
//check exist
AlertGroup alertGroup = alertGroupMapper.selectById(id);
if (alertGroup == null) {
putMsg(result, Status.ALERT_GROUP_NOT_EXIST);
return result;
}
userAlertGroupMapper.deleteByAlertgroupId(id);
alertGroupMapper.deleteById(id);

View File

@ -41,7 +41,7 @@ import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkDesc;
/**
* project service
*HttpTask./
**/
**/
@Service
public class ProjectService extends BaseService{
@ -121,7 +121,7 @@ public class ProjectService extends BaseService{
* @param loginUser login user
* @param project project
* @param projectName project name
* @return true if the login user havve permission to see the project
* @return true if the login user have permission to see the project
*/
public Map<String, Object> checkProjectAndAuth(User loginUser, Project project, String projectName) {
@ -143,7 +143,7 @@ public class ProjectService extends BaseService{
public boolean hasProjectAndPerm(User loginUser, Project project, Map<String, Object> result) {
boolean checkResult = false;
if (project == null) {
putMsg(result, Status.PROJECT_NOT_FOUNT, project.getName());
putMsg(result, Status.PROJECT_NOT_FOUNT, "");
} else if (!checkReadPermission(loginUser, project)) {
putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName());
} else {
@ -199,13 +199,14 @@ public class ProjectService extends BaseService{
if (checkResult != null) {
return checkResult;
}
List<ProcessDefinition> processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId);
if (!hasPerm(loginUser, project.getUserId())) {
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
List<ProcessDefinition> processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId);
if(processDefinitionList.size() > 0){
putMsg(result, Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL);
return result;
@ -227,7 +228,8 @@ public class ProjectService extends BaseService{
* @return check result
*/
private Map<String, Object> getCheckResult(User loginUser, Project project) {
Map<String, Object> checkResult = checkProjectAndAuth(loginUser, project, project.getName());
String projectName = project == null ? null:project.getName();
Map<String, Object> checkResult = checkProjectAndAuth(loginUser, project, projectName);
Status status = (Status) checkResult.get(Constants.STATUS);
if (status != Status.SUCCESS) {
return checkResult;
@ -247,6 +249,11 @@ public class ProjectService extends BaseService{
public Map<String, Object> update(User loginUser, Integer projectId, String projectName, String desc) {
Map<String, Object> result = new HashMap<>(5);
Map<String, Object> descCheck = checkDesc(desc);
if (descCheck.get(Constants.STATUS) != Status.SUCCESS) {
return descCheck;
}
Project project = projectMapper.selectById(projectId);
boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result);
if (!hasProjectAndPerm) {

View File

@ -115,12 +115,12 @@ public class QueueService extends BaseService {
}
if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result;
}
if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result;
}
@ -163,6 +163,16 @@ public class QueueService extends BaseService {
return result;
}
if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result;
}
if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result;
}
Queue queueObj = queueMapper.selectById(id);
if (queueObj == null) {
putMsg(result, Status.QUEUE_NOT_EXIST, id);
@ -222,12 +232,12 @@ public class QueueService extends BaseService {
Result result = new Result();
if (StringUtils.isEmpty(queue)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue");
return result;
}
if (StringUtils.isEmpty(queueName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName);
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName");
return result;
}

View File

@ -210,7 +210,6 @@ public class ResourcesService extends BaseService {
}
Resource resource = resourcesMapper.selectById(resourceId);
String originResourceName = resource.getAlias();
if (resource == null) {
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
@ -236,6 +235,7 @@ public class ResourcesService extends BaseService {
}
//get the file suffix
String originResourceName = resource.getAlias();
String suffix = originResourceName.substring(originResourceName.lastIndexOf("."));
//if the name without suffix then add it ,else use the origin name

View File

@ -187,6 +187,12 @@ public class UdfFuncService extends BaseService{
// verify udfFunc is exist
UdfFunc udf = udfFuncMapper.selectUdfById(udfFuncId);
if (udf == null) {
result.put(Constants.STATUS, Status.UDF_FUNCTION_NOT_EXIST);
result.put(Constants.MSG, Status.UDF_FUNCTION_NOT_EXIST.getMsg());
return result;
}
// if resource upload startup
if (!PropertyUtils.getResUploadStartupState()){
logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState());
@ -194,12 +200,6 @@ public class UdfFuncService extends BaseService{
return result;
}
if (udf == null) {
result.put(Constants.STATUS, Status.UDF_FUNCTION_NOT_EXIST);
result.put(Constants.MSG, Status.UDF_FUNCTION_NOT_EXIST.getMsg());
return result;
}
// verify udfFuncName is exist
if (!funcName.equals(udf.getFuncName())) {
if (checkUdfFuncNameExists(funcName)) {
@ -303,7 +303,7 @@ public class UdfFuncService extends BaseService{
@Transactional(rollbackFor = Exception.class)
public Result delete(int id) {
Result result = new Result();
udfFuncMapper.deleteById(id);
udfUserMapper.deleteByUdfFuncId(id);
putMsg(result, Status.SUCCESS);

View File

@ -221,9 +221,13 @@ public class UsersService extends BaseService {
return result;
}
Date now = new Date();
if (StringUtils.isNotEmpty(userName)) {
if (!CheckUtils.checkUserName(userName)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userName);
return result;
}
User tempUser = userMapper.queryByUserNameAccurately(userName);
if (tempUser != null && tempUser.getId() != userId) {
putMsg(result, Status.USER_NAME_EXIST);
@ -233,14 +237,30 @@ public class UsersService extends BaseService {
}
if (StringUtils.isNotEmpty(userPassword)) {
if (!CheckUtils.checkPassword(userPassword)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userPassword);
return result;
}
user.setUserPassword(EncryptionUtils.getMd5(userPassword));
}
if (StringUtils.isNotEmpty(email)) {
if (!CheckUtils.checkEmail(email)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,email);
return result;
}
user.setEmail(email);
}
if (StringUtils.isNotEmpty(phone)) {
if (!CheckUtils.checkPhone(phone)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,phone);
return result;
}
user.setPhone(phone);
}
user.setQueue(queue);
user.setPhone(phone);
Date now = new Date();
user.setUpdateTime(now);
//if user switches the tenant, the user's resources need to be copied to the new tenant
@ -318,10 +338,15 @@ public class UsersService extends BaseService {
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (!isAdmin(loginUser)) {
putMsg(result, Status.USER_NO_OPERATION_PERM, id);
return result;
}
//check exist
User tempUser = userMapper.selectById(id);
if (tempUser == null) {
putMsg(result, Status.USER_NOT_EXIST, id);
return result;
}
// delete user
User user = userMapper.queryTenantCodeByUserId(id);
@ -357,6 +382,12 @@ public class UsersService extends BaseService {
return result;
}
//check exist
User tempUser = userMapper.selectById(userId);
if (tempUser == null) {
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
//if the selected projectIds are empty, delete all items associated with the user
projectUserMapper.deleteProjectRelation(0, userId);
@ -443,6 +474,11 @@ public class UsersService extends BaseService {
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
}
User user = userMapper.selectById(userId);
if(user == null){
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
udfUserMapper.deleteByUserId(userId);
@ -485,6 +521,11 @@ public class UsersService extends BaseService {
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
}
User user = userMapper.selectById(userId);
if(user == null){
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
datasourceUserMapper.deleteByUserId(userId);

View File

@ -76,6 +76,11 @@ public class WorkerGroupService extends BaseService {
WorkerGroup workerGroup = null;
if(id != 0){
workerGroup = workerGroupMapper.selectById(id);
//check exist
if (workerGroup == null){
workerGroup = new WorkerGroup();
workerGroup.setCreateTime(now);
}
}else{
workerGroup = new WorkerGroup();
workerGroup.setCreateTime(now);

View File

@ -104,7 +104,7 @@ public class CheckUtils {
* @return true if phone regex valid, otherwise return false
*/
public static boolean checkPhone(String phone) {
return StringUtils.isEmpty(phone) || phone.length() <= 11;
return StringUtils.isEmpty(phone) || phone.length() == 11;
}

View File

@ -1,40 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
logging.config=classpath:combined_logback.xml
# server port
server.port=12345
# session config
server.servlet.session.timeout=7200
server.servlet.context-path=/dolphinscheduler/
# file size limit for upload
spring.servlet.multipart.max-file-size=1024MB
spring.servlet.multipart.max-request-size=1024MB
#post content
server.jetty.max-http-post-size=5000000
spring.messages.encoding=UTF-8
#i18n classpath folder , file prefix messages if have many files, use "," seperator
spring.messages.basename=i18n/messages
server.is-combined-server=true

View File

@ -0,0 +1,181 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.AccessToken;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class AccessTokenServiceTest {
private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceTest.class);
@InjectMocks
private AccessTokenService accessTokenService ;
@Mock
private AccessTokenMapper accessTokenMapper;
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testQueryAccessTokenList(){
IPage<AccessToken> tokenPage = new Page<>();
tokenPage.setRecords(getList());
tokenPage.setTotal(1L);
when(accessTokenMapper.selectAccessTokenPage(any(Page.class),eq("zhangsan"),eq(0))).thenReturn(tokenPage);
User user =new User();
Map<String, Object> result = accessTokenService.queryAccessTokenList(user,"zhangsan",1,10);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
PageInfo<AccessToken> pageInfo = (PageInfo<AccessToken>) result.get(Constants.DATA_LIST);
Assert.assertTrue(pageInfo.getTotalCount()>0);
}
@Test
public void testCreateToken(){
when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2);
Map<String, Object> result = accessTokenService.createToken(1,getDate(),"AccessTokenServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testGenerateToken(){
Map<String, Object> result = accessTokenService.generateToken(Integer.MAX_VALUE,getDate());
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
String token = (String) result.get(Constants.DATA_LIST);
Assert.assertNotNull(token);
}
@Test
public void testDelAccessTokenById(){
when(accessTokenMapper.selectById(1)).thenReturn(getEntity());
User userLogin = new User();
// not exist
Map<String, Object> result = accessTokenService.delAccessTokenById(userLogin,0);
logger.info(result.toString());
Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS));
// no operate
result = accessTokenService.delAccessTokenById(userLogin,1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
//success
userLogin.setId(1);
userLogin.setUserType(UserType.ADMIN_USER);
result = accessTokenService.delAccessTokenById(userLogin,1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdateToken(){
when(accessTokenMapper.selectById(1)).thenReturn(getEntity());
Map<String, Object> result = accessTokenService.updateToken(1,Integer.MAX_VALUE,getDate(),"token");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
// not exist
result = accessTokenService.updateToken(2,Integer.MAX_VALUE,getDate(),"token");
logger.info(result.toString());
Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS));
}
/**
* create entity
* @return
*/
private AccessToken getEntity(){
AccessToken accessToken = new AccessToken();
accessToken.setId(1);
accessToken.setUserId(1);
accessToken.setToken("AccessTokenServiceTest");
Date date = DateUtils.addDays(new Date(),30);
accessToken.setExpireTime(date);
return accessToken;
}
/**
* entity list
* @return
*/
private List<AccessToken> getList(){
List<AccessToken> list = new ArrayList<>();
list.add(getEntity());
return list;
}
/**
* get dateStr
* @return
*/
private String getDate(){
Date date = DateUtils.addDays(new Date(),30);
return org.apache.dolphinscheduler.common.utils.DateUtils.dateToString(date);
}
}

View File

@ -0,0 +1,219 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
@RunWith(MockitoJUnitRunner.class)
public class AlertGroupServiceTest {
private static final Logger logger = LoggerFactory.getLogger(AlertGroupServiceTest.class);
@InjectMocks
private AlertGroupService alertGroupService;
@Mock
private AlertGroupMapper alertGroupMapper;
@Mock
private UserAlertGroupMapper userAlertGroupMapper;
private String groupName = "AlertGroupServiceTest";
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testQueryAlertgroup(){
Mockito.when(alertGroupMapper.queryAllGroupList()).thenReturn(getList());
HashMap<String, Object> result= alertGroupService.queryAlertgroup();
logger.info(result.toString());
List<AlertGroup> alertGroups = (List<AlertGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(alertGroups));
}
@Test
public void testListPaging(){
IPage<AlertGroup> page = new Page<>(1,10);
page.setTotal(1L);
page.setRecords(getList());
Mockito.when(alertGroupMapper.queryAlertGroupPage(any(Page.class),eq(groupName))).thenReturn(page);
User user = new User();
// no operate
Map<String, Object> result = alertGroupService.listPaging(user,groupName,1,10);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
//success
user.setUserType(UserType.ADMIN_USER);
result = alertGroupService.listPaging(user,groupName,1,10);
logger.info(result.toString());
PageInfo<AlertGroup> pageInfo = (PageInfo<AlertGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testCreateAlertgroup(){
Mockito.when(alertGroupMapper.insert(any(AlertGroup.class))).thenReturn(2);
User user = new User();
//no operate
Map<String, Object> result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
user.setUserType(UserType.ADMIN_USER);
//success
result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdateAlertgroup(){
User user = new User();
// no operate
Map<String, Object> result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
user.setUserType(UserType.ADMIN_USER);
// not exist
result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS));
//success
Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity());
result = alertGroupService.updateAlertgroup(user,2,groupName, AlertType.SMS,groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testDelAlertgroupById(){
User user = new User();
// no operate
Map<String, Object> result = alertGroupService.delAlertgroupById(user,1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
user.setUserType(UserType.ADMIN_USER);
// not exist
result = alertGroupService.delAlertgroupById(user,2);
logger.info(result.toString());
Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS));
//success
Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity());
result = alertGroupService.delAlertgroupById(user,2);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testGrantUser(){
Map<String, Object> result = alertGroupService.grantUser(getLoginUser(),1,"123,321");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testVerifyGroupName(){
//group name not exist
Result result = alertGroupService.verifyGroupName(getLoginUser(), groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
Mockito.when(alertGroupMapper.queryByGroupName(groupName)).thenReturn(getList());
//group name exist
result = alertGroupService.verifyGroupName(getLoginUser(), groupName);
logger.info(result.toString());
Assert.assertEquals(Status.ALERT_GROUP_EXIST.getMsg(),result.getMsg());
}
/**
* create admin user
* @return
*/
private User getLoginUser(){
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setId(99999999);
return loginUser;
}
/**
* get list
* @return
*/
private List<AlertGroup> getList(){
List<AlertGroup> alertGroups = new ArrayList<>();
alertGroups.add(getEntity());
return alertGroups;
}
/**
* get entity
* @return
*/
private AlertGroup getEntity(){
AlertGroup alertGroup = new AlertGroup();
alertGroup.setId(1);
alertGroup.setGroupName(groupName);
alertGroup.setGroupType(AlertType.EMAIL);
return alertGroup;
}
}

View File

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class MonitorServiceTest {
private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class);
@InjectMocks
private MonitorService monitorService;
@Mock
private MonitorDBDao monitorDBDao;
@Test
public void testQueryDatabaseState(){
Mockito.when(monitorDBDao.queryDatabaseState()).thenReturn(getList());
Map<String,Object> result = monitorService.queryDatabaseState(null);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
List<MonitorRecord> monitorRecordList = (List<MonitorRecord>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(monitorRecordList));
}
@Test
public void testQueryMaster(){
//TODO need zk
// Map<String,Object> result = monitorService.queryMaster(null);
// logger.info(result.toString());
// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryZookeeperState(){
//TODO need zk
// Map<String,Object> result = monitorService.queryZookeeperState(null);
// logger.info(result.toString());
// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testGetServerListFromZK(){
//TODO need zk
// List<Server> serverList = monitorService.getServerListFromZK(true);
// logger.info(serverList.toString());
}
private List<MonitorRecord> getList(){
List<MonitorRecord> monitorRecordList = new ArrayList<>();
monitorRecordList.add(getEntity());
return monitorRecordList;
}
private MonitorRecord getEntity(){
MonitorRecord monitorRecord = new MonitorRecord();
monitorRecord.setDbType(DbType.MYSQL);
return monitorRecord;
}
private List<Server> getServerList(){
List<Server> servers = new ArrayList<>();
servers.add(new Server());
return servers;
}
}

View File

@ -0,0 +1,356 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class ProjectServiceTest {
private static final Logger logger = LoggerFactory.getLogger(ProjectServiceTest.class);
@InjectMocks
private ProjectService projectService;
@Mock
private ProjectMapper projectMapper;
@Mock
private UserMapper userMapper;
@Mock
private ProjectUserMapper projectUserMapper;
@Mock
private ProcessDefinitionMapper processDefinitionMapper;
private String projectName = "ProjectServiceTest";
private String userName = "ProjectServiceTest";
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testCreateProject(){
User loginUser = getLoginUser();
loginUser.setId(1);
Map<String, Object> result = projectService.createProject(loginUser, projectName, getDesc());
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
//project name exist
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject());
result = projectService.createProject(loginUser, projectName, projectName);
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS));
//success
Mockito.when(projectMapper.insert(Mockito.any(Project.class))).thenReturn(1);
result = projectService.createProject(loginUser, "test", "test");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryById(){
//not exist
Map<String, Object> result = projectService.queryById(Integer.MAX_VALUE);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
logger.info(result.toString());
//success
Mockito.when(projectMapper.selectById(1)).thenReturn(getProject());
result = projectService.queryById(1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testCheckProjectAndAuth(){
Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser());
User loginUser = getLoginUser();
Map<String, Object> result = projectService.checkProjectAndAuth(loginUser,null,projectName);
logger.info(result.toString());
Status status = (Status)result.get(Constants.STATUS);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
Project project = getProject();
//USER_NO_OPERATION_PROJECT_PERM
project.setUserId(2);
result = projectService.checkProjectAndAuth(loginUser,project,projectName);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS));
//success
project.setUserId(1);
result = projectService.checkProjectAndAuth(loginUser,project,projectName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testHasProjectAndPerm(){
// Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser());
User loginUser = getLoginUser();
Project project = getProject();
Map<String, Object> result = new HashMap<>();
// not exist user
User tempUser = new User();
tempUser.setId(Integer.MAX_VALUE);
boolean checkResult = projectService.hasProjectAndPerm(tempUser,project,result);
logger.info(result.toString());
Assert.assertFalse(checkResult);
//success
result = new HashMap<>();
project.setUserId(1);
checkResult = projectService.hasProjectAndPerm(loginUser,project,result);
logger.info(result.toString());
Assert.assertTrue(checkResult);
}
@Test
public void testQueryProjectListPaging(){
IPage<Project> page = new Page<>(1,10);
page.setRecords(getList());
page.setTotal(1L);
Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(1), Mockito.eq(projectName))).thenReturn(page);
User loginUser = getLoginUser();
// project owner
Map<String, Object> result = projectService.queryProjectListPaging(loginUser,10,1,projectName);
logger.info(result.toString());
PageInfo<Project> pageInfo = (PageInfo<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
//admin
Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(0), Mockito.eq(projectName))).thenReturn(page);
loginUser.setUserType(UserType.ADMIN_USER);
result = projectService.queryProjectListPaging(loginUser,10,1,projectName);
logger.info(result.toString());
pageInfo = (PageInfo<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testDeleteProject(){
Mockito.when(projectMapper.selectById(1)).thenReturn(getProject());
User loginUser = getLoginUser();
//PROJECT_NOT_FOUNT
Map<String, Object> result= projectService.deleteProject(loginUser,12);
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
loginUser.setId(2);
//USER_NO_OPERATION_PROJECT_PERM
result= projectService.deleteProject(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS));
//DELETE_PROJECT_ERROR_DEFINES_NOT_NULL
Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(getProcessDefinitions());
loginUser.setUserType(UserType.ADMIN_USER);
result= projectService.deleteProject(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL,result.get(Constants.STATUS));
//success
Mockito.when(projectMapper.deleteById(1)).thenReturn(1);
Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(new ArrayList<>());
result= projectService.deleteProject(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdate(){
User loginUser = getLoginUser();
Project project = getProject();
project.setId(2);
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project);
Mockito.when( projectMapper.selectById(1)).thenReturn(getProject());
// PROJECT_NOT_FOUNT
Map<String, Object> result = projectService.update(loginUser,12,projectName,"desc");
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS));
//PROJECT_ALREADY_EXISTS
result = projectService.update(loginUser,1,projectName,"desc");
logger.info(result.toString());
Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS));
//success
project.setUserId(1);
Mockito.when(projectMapper.updateById(Mockito.any(Project.class))).thenReturn(1);
result = projectService.update(loginUser,1,"test","desc");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testQueryAuthorizedProject(){
User loginUser = getLoginUser();
Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList());
//USER_NO_OPERATION_PERM
Map<String, Object> result = projectService.queryAuthorizedProject(loginUser,3);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
result = projectService.queryAuthorizedProject(loginUser,1);
logger.info(result.toString());
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
@Test
public void testQueryAllProjectList(){
Mockito.when(projectMapper.selectList(null)).thenReturn(getList());
Mockito.when(processDefinitionMapper.selectList(null)).thenReturn(getProcessDefinitions());
Map<String, Object> result = projectService.queryAllProjectList();
logger.info(result.toString());
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
@Test
public void testQueryUnauthorizedProject(){
// Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList());
Mockito.when(projectMapper.queryProjectExceptUserId(2)).thenReturn(getList());
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
Map<String, Object> result = projectService.queryUnauthorizedProject(loginUser,2);
logger.info(result.toString());
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
private Project getProject(){
Project project = new Project();
project.setId(1);
project.setName(projectName);
project.setUserId(1);
return project;
}
private List<Project> getList(){
List<Project> list = new ArrayList<>();
list.add(getProject());
return list;
}
/**
* create admin user
* @return
*/
private User getLoginUser(){
User loginUser = new User();
loginUser.setUserType(UserType.GENERAL_USER);
loginUser.setUserName(userName);
loginUser.setId(1);
return loginUser;
}
/**
* get project user
*/
private ProjectUser getProjectUser(){
ProjectUser projectUser = new ProjectUser();
projectUser.setProjectId(1);
projectUser.setUserId(1);
return projectUser;
}
private List<ProcessDefinition> getProcessDefinitions(){
List<ProcessDefinition> list = new ArrayList<>();
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setProjectId(1);
list.add(processDefinition);
return list;
}
private String getDesc(){
return "projectUserMapper.deleteProjectRelation(projectId,userId)projectUserMappe" +
".deleteProjectRelation(projectId,userId)projectUserMappe" +
"r.deleteProjectRelation(projectId,userId)projectUserMapper" +
".deleteProjectRelation(projectId,userId)projectUserMapper.deleteProjectRelation(projectId,userId)";
}
}

View File

@ -0,0 +1,208 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class QueueServiceTest {
private static final Logger logger = LoggerFactory.getLogger(QueueServiceTest.class);
@InjectMocks
private QueueService queueService;
@Mock
private QueueMapper queueMapper;
@Mock
private UserMapper userMapper;
private String queueName = "QueueServiceTest";
@Before
public void setUp() {
}
@After
public void after(){
}
@Test
public void testQueryList(){
Mockito.when(queueMapper.selectList(null)).thenReturn(getQueueList());
Map<String, Object> result = queueService.queryList(getLoginUser());
logger.info(result.toString());
List<Queue> queueList = (List<Queue>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(queueList));
}
@Test
public void testQueryListPage(){
IPage<Queue> page = new Page<>(1,10);
page.setTotal(1L);
page.setRecords(getQueueList());
Mockito.when(queueMapper.queryQueuePaging(Mockito.any(Page.class), Mockito.eq(queueName))).thenReturn(page);
Map<String, Object> result = queueService.queryList(getLoginUser(),queueName,1,10);
logger.info(result.toString());
PageInfo<Queue> pageInfo = (PageInfo<Queue>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testCreateQueue(){
// queue is null
Map<String, Object> result = queueService.createQueue(getLoginUser(),null,queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
// queueName is null
result = queueService.createQueue(getLoginUser(),queueName,null);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
// correct
result = queueService.createQueue(getLoginUser(),queueName,queueName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
}
@Test
public void testUpdateQueue(){
Mockito.when(queueMapper.selectById(1)).thenReturn(getQueue());
Mockito.when(queueMapper.queryAllQueueList("test", null)).thenReturn(getQueueList());
Mockito.when(queueMapper.queryAllQueueList(null, "test")).thenReturn(getQueueList());
Mockito.when(userMapper.queryUserListByQueue(queueName)).thenReturn(getUserList());
// not exist
Map<String, Object> result = queueService.updateQueue(getLoginUser(),0,"queue",queueName);
logger.info(result.toString());
Assert.assertEquals(Status.QUEUE_NOT_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode());
//no need update
result = queueService.updateQueue(getLoginUser(),1,queueName,queueName);
logger.info(result.toString());
Assert.assertEquals(Status.NEED_NOT_UPDATE_QUEUE.getCode(),((Status)result.get(Constants.STATUS)).getCode());
//queue exist
result = queueService.updateQueue(getLoginUser(),1,"test",queueName);
logger.info(result.toString());
Assert.assertEquals(Status.QUEUE_VALUE_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode());
// queueName exist
result = queueService.updateQueue(getLoginUser(),1,"test1","test");
logger.info(result.toString());
Assert.assertEquals(Status.QUEUE_NAME_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode());
//success
result = queueService.updateQueue(getLoginUser(),1,"test1","test1");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getCode(),((Status)result.get(Constants.STATUS)).getCode());
}
@Test
public void testVerifyQueue(){
Mockito.when(queueMapper.queryAllQueueList(queueName, null)).thenReturn(getQueueList());
Mockito.when(queueMapper.queryAllQueueList(null, queueName)).thenReturn(getQueueList());
//queue null
Result result = queueService.verifyQueue(null,queueName);
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode());
//queueName null
result = queueService.verifyQueue(queueName,null);
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode());
//exist queueName
result = queueService.verifyQueue(queueName,queueName);
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_NAME_EXIST.getCode());
//exist queue
result = queueService.verifyQueue(queueName,"test");
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_VALUE_EXIST.getCode());
// success
result = queueService.verifyQueue("test","test");
logger.info(result.toString());
Assert.assertEquals(result.getCode().intValue(), Status.SUCCESS.getCode());
}
/**
* create admin user
* @return
*/
private User getLoginUser(){
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setId(99999999);
return loginUser;
}
private List<User> getUserList(){
List<User> list = new ArrayList<>();
list.add(getLoginUser());
return list;
}
/**
* get queue
* @return
*/
private Queue getQueue(){
Queue queue = new Queue();
queue.setId(1);
queue.setQueue(queueName);
queue.setQueueName(queueName);
return queue;
}
private List<Queue> getQueueList(){
List<Queue> queueList = new ArrayList<>();
queueList.add(getQueue());
return queueList;
}
}

View File

@ -16,37 +16,130 @@
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.mock.web.MockCookie;
import org.springframework.mock.web.MockHttpServletRequest;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@RunWith(MockitoJUnitRunner.class)
public class SessionServiceTest {
private static final Logger logger = LoggerFactory.getLogger(SessionServiceTest.class);
@Autowired
@InjectMocks
private SessionService sessionService;
@Mock
private SessionMapper sessionMapper;
private String sessionId ="aaaaaaaaaaaaaaaaaa";
@Before
public void setUp() {
}
@After
public void after(){
}
/**
* create session
*/
@Test
public void createSession(){
public void testGetSession(){
User loginUser = new User();
loginUser.setId(1);
loginUser.setUserType(UserType.GENERAL_USER);
String session = sessionService.createSession(loginUser, "127.0.0.1");
Assert.assertTrue(StringUtils.isNotEmpty(session));
Mockito.when(sessionMapper.selectById(sessionId)).thenReturn(getSession());
// get sessionId from header
MockHttpServletRequest mockHttpServletRequest = new MockHttpServletRequest();
mockHttpServletRequest.addHeader(Constants.SESSION_ID,sessionId);
mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1");
//query
Session session = sessionService.getSession(mockHttpServletRequest);
Assert.assertNotNull(session);
logger.info("session ip {}",session.getIp());
// get sessionId from cookie
mockHttpServletRequest = new MockHttpServletRequest();
mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1");
MockCookie mockCookie = new MockCookie(Constants.SESSION_ID,sessionId);
mockHttpServletRequest.setCookies(mockCookie);
//query
session = sessionService.getSession(mockHttpServletRequest);
Assert.assertNotNull(session);
logger.info("session ip {}",session.getIp());
Assert.assertEquals(session.getIp(),"127.0.0.1");
}
/**
* create session
*/
@Test
public void testCreateSession(){
String ip = "127.0.0.1";
User user = new User();
user.setUserType(UserType.GENERAL_USER);
user.setId(1);
Mockito.when(sessionMapper.queryByUserId(1)).thenReturn(getSessions());
String sessionId = sessionService.createSession(user, ip);
logger.info("createSessionId is "+sessionId);
Assert.assertTrue(StringUtils.isNotEmpty(sessionId));
}
/**
* sign out
* remove ip restrictions
*/
@Test
public void testSignOut(){
int userId = 88888888;
String ip = "127.0.0.1";
User user = new User();
user.setId(userId);
sessionService.signOut(ip ,user);
}
private Session getSession(){
Session session = new Session();
session.setId(sessionId);
session.setIp("127.0.0.1");
session.setLastLoginTime(DateUtils.addDays(new Date(),40));
session.setUserId(1);
return session;
}
private List<Session> getSessions(){
List<Session> sessionList = new ArrayList<>();
sessionList.add(getSession());
return sessionList;
}
}

View File

@ -16,38 +16,241 @@
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
@RunWith(MockitoJUnitRunner.class)
public class TenantServiceTest {
private static final Logger logger = LoggerFactory.getLogger(TenantServiceTest.class);
@Autowired
@InjectMocks
private TenantService tenantService;
@Mock
private TenantMapper tenantMapper;
@Mock
private ProcessDefinitionMapper processDefinitionMapper;
@Mock
private ProcessInstanceMapper processInstanceMapper;
@Mock
private UserMapper userMapper;
private String tenantCode ="TenantServiceTest";
private String tenantName ="TenantServiceTest";
@Test
public void queryTenantList(){
public void testCreateTenant(){
User loginUser = getLoginUser();
Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList());
try {
//check tenantCode
Map<String, Object> result = tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR,result.get(Constants.STATUS));
//check exist
result = tenantService.createTenant(loginUser, tenantCode, tenantName, 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS));
// success
result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("create tenant error",e);
Assert.assertTrue(false);
}
}
@Test
public void testQueryTenantListPage(){
IPage<Tenant> page = new Page<>(1,10);
page.setRecords(getList());
page.setTotal(1L);
Mockito.when(tenantMapper.queryTenantPaging(Mockito.any(Page.class), Mockito.eq("TenantServiceTest"))).thenReturn(page);
Map<String, Object> result = tenantService.queryTenantList(getLoginUser(), "TenantServiceTest", 1, 10);
logger.info(result.toString());
PageInfo<Tenant> pageInfo = (PageInfo<Tenant>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
@Test
public void testUpdateTenant(){
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
try {
// id not exist
Map<String, Object> result = tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc");
logger.info(result.toString());
// success
Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS));
result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, "TenantServiceTest001", 1, "desc");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("update tenant error",e);
Assert.assertTrue(false);
}
}
@Test
public void testDeleteTenantById(){
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getInstanceList());
Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList());
Mockito.when( userMapper.queryUserListByTenant(3)).thenReturn(getUserList());
try {
//TENANT_NOT_EXIST
Map<String, Object> result = tenantService.deleteTenantById(getLoginUser(),12);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS));
//DELETE_TENANT_BY_ID_FAIL
result = tenantService.deleteTenantById(getLoginUser(),1);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL,result.get(Constants.STATUS));
//DELETE_TENANT_BY_ID_FAIL_DEFINES
Mockito.when(tenantMapper.queryById(2)).thenReturn(getTenant(2));
result = tenantService.deleteTenantById(getLoginUser(),2);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES,result.get(Constants.STATUS));
//DELETE_TENANT_BY_ID_FAIL_USERS
Mockito.when(tenantMapper.queryById(3)).thenReturn(getTenant(3));
result = tenantService.deleteTenantById(getLoginUser(),3);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_USERS,result.get(Constants.STATUS));
// success
Mockito.when(tenantMapper.queryById(4)).thenReturn(getTenant(4));
result = tenantService.deleteTenantById(getLoginUser(),4);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("delete tenant error",e);
Assert.assertTrue(false);
}
}
@Test
public void testQueryTenantList(){
Mockito.when( tenantMapper.selectList(null)).thenReturn(getList());
Map<String, Object> result = tenantService.queryTenantList(getLoginUser());
logger.info(result.toString());
List<Tenant> tenantList = (List<Tenant>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList));
}
@Test
public void testVerifyTenantCode(){
Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList());
// tenantCode not exist
Result result = tenantService.verifyTenantCode("s00000000000l887888885554444sfjdskfjslakslkdf");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
// tenantCode exist
result = tenantService.verifyTenantCode(getTenant().getTenantCode());
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NAME_EXIST.getMsg(),result.getMsg());
}
/**
* get user
* @return
*/
private User getLoginUser(){
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
Map<String, Object> map = tenantService.queryTenantList(loginUser);
Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS));
logger.info(map.toString());
return loginUser;
}
/**
* get list
* @return
*/
private List<Tenant> getList(){
List<Tenant> tenantList = new ArrayList<>();
tenantList.add(getTenant());
return tenantList;
}
/**
* get tenant
* @return
*/
private Tenant getTenant(){
return getTenant(1);
}
/**
* get tenant
* @return
*/
private Tenant getTenant(int id){
Tenant tenant = new Tenant();
tenant.setId(id);
tenant.setTenantCode(tenantCode);
tenant.setTenantName(tenantName);
return tenant;
}
private List<User> getUserList(){
List<User> userList = new ArrayList<>();
userList.add(getLoginUser());
return userList;
}
private List<ProcessInstance> getInstanceList(){
List<ProcessInstance> processInstances = new ArrayList<>();
ProcessInstance processInstance = new ProcessInstance();
processInstances.add(processInstance);
return processInstances;
}
private List<ProcessDefinition> getDefinitionsList(){
List<ProcessDefinition> processDefinitions = new ArrayList<>();
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinitions.add(processDefinition);
return processDefinitions;
}
}

View File

@ -16,39 +16,449 @@
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class UsersServiceTest {
private static final Logger logger = LoggerFactory.getLogger(UsersServiceTest.class);
@Autowired
@InjectMocks
private UsersService usersService;
@Mock
private UserMapper userMapper;
@Mock
private TenantMapper tenantMapper;
@Mock
private ProjectUserMapper projectUserMapper;
@Mock
private ResourceUserMapper resourcesUserMapper;
@Mock
private UDFUserMapper udfUserMapper;
@Mock
private DataSourceUserMapper datasourceUserMapper;
@Mock
private AlertGroupMapper alertGroupMapper;
private String queueName ="UsersServiceTestQueue";
@Before
public void before(){
}
@After
public void after(){
}
@Test
public void testCreateUser(){
User user = new User();
user.setUserType(UserType.ADMIN_USER);
String userName = "userTest0001~";
String userPassword = "userTest";
String email = "123@qq.com";
int tenantId = Integer.MAX_VALUE;
String phone= "13456432345";
try {
//userName error
Map<String, Object> result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
userName = "userTest0001";
userPassword = "userTest000111111111111111";
//password error
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
userPassword = "userTest0001";
email = "1q.com";
//email error
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
email = "122222@qq.com";
phone ="2233";
//phone error
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
phone = "13456432345";
//tenantId not exists
result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS));
//success
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
result = usersService.createUser(user, userName, userPassword, email, 1, phone, queueName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
logger.error(Status.CREATE_USER_ERROR.getMsg(),e);
Assert.assertTrue(false);
}
}
@Test
public void testQueryUser(){
String userName = "userTest0001";
String userPassword = "userTest0001";
when(userMapper.queryUserByNamePassword(userName,EncryptionUtils.getMd5(userPassword))).thenReturn(getGeneralUser());
User queryUser = usersService.queryUser(userName, userPassword);
logger.info(queryUser.toString());
Assert.assertTrue(queryUser!=null);
}
@Test
public void testQueryUserList(){
User user = new User();
//no operate
Map<String, Object> result = usersService.queryUserList(user);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
user.setUserType(UserType.ADMIN_USER);
when(userMapper.selectList(null )).thenReturn(getUserList());
result = usersService.queryUserList(user);
List<User> userList = (List<User>) result.get(Constants.DATA_LIST);
Assert.assertTrue(userList.size()>0);
}
@Test
public void testQueryUserListPage(){
User user = new User();
IPage<User> page = new Page<>(1,10);
page.setRecords(getUserList());
when(userMapper.queryUserPaging(any(Page.class), eq("userTest"))).thenReturn(page);
//no operate
Map<String, Object> result = usersService.queryUserList(user,"userTest",1,10);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
user.setUserType(UserType.ADMIN_USER);
result = usersService.queryUserList(user,"userTest",1,10);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
PageInfo<User> pageInfo = (PageInfo<User>) result.get(Constants.DATA_LIST);
Assert.assertTrue(pageInfo.getLists().size()>0);
}
@Test
public void testUpdateUser(){
String userName = "userTest0001";
String userPassword = "userTest0001";
try {
//user not exist
Map<String, Object> result = usersService.updateUser(0,userName,userPassword,"3443@qq.com",1,"13457864543","queue");
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
logger.info(result.toString());
//success
when(userMapper.selectById(1)).thenReturn(getUser());
result = usersService.updateUser(1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("update user error",e);
Assert.assertTrue(false);
}
}
@Test
public void testDeleteUserById(){
User loginUser = new User();
try {
when(userMapper.queryTenantCodeByUserId(1)).thenReturn(getUser());
when(userMapper.selectById(1)).thenReturn(getUser());
//no operate
Map<String, Object> result = usersService.deleteUserById(loginUser,3);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
// user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.deleteUserById(loginUser,3);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.deleteUserById(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
logger.error("delete user error",e);
Assert.assertTrue(false);
}
}
@Test
public void testGrantProject(){
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
String projectIds= "100000,120000";
Map<String, Object> result = usersService.grantProject(loginUser, 1, projectIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantProject(loginUser, 2, projectIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantProject(loginUser, 1, projectIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testGrantResources(){
String resourceIds = "100000,120000";
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
Map<String, Object> result = usersService.grantResources(loginUser, 1, resourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantResources(loginUser, 2, resourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantResources(loginUser, 1, resourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testGrantUDFFunction(){
String udfIds = "100000,120000";
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
Map<String, Object> result = usersService.grantUDFFunction(loginUser, 1, udfIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantUDFFunction(loginUser, 2, udfIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantUDFFunction(loginUser, 1, udfIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testGrantDataSource(){
String datasourceIds = "100000,120000";
when(userMapper.selectById(1)).thenReturn(getUser());
User loginUser = new User();
Map<String, Object> result = usersService.grantDataSource(loginUser, 1, datasourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//user not exist
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.grantDataSource(loginUser, 2, datasourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
result = usersService.grantDataSource(loginUser, 1, datasourceIds);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void getUserInfo(){
User loginUser = new User();
loginUser.setId(19);
loginUser.setUserType(UserType.GENERAL_USER);
Map<String, Object> map = usersService.getUserInfo(loginUser);
Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS));
logger.info(map.toString());
loginUser.setUserName("admin");
loginUser.setUserType(UserType.ADMIN_USER);
// get admin user
Map<String, Object> result = usersService.getUserInfo(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
User tempUser = (User) result.get(Constants.DATA_LIST);
//check userName
Assert.assertEquals("admin",tempUser.getUserName());
//get general user
loginUser.setUserType(null);
loginUser.setId(1);
when(userMapper.queryDetailsById(1)).thenReturn(getGeneralUser());
result = usersService.getUserInfo(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
tempUser = (User) result.get(Constants.DATA_LIST);
//check userName
Assert.assertEquals("userTest0001",tempUser.getUserName());
}
@Test
public void testQueryAllGeneralUsers(){
User loginUser = new User();
//no operate
Map<String, Object> result = usersService.queryAllGeneralUsers(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
when(userMapper.queryAllGeneralUser()).thenReturn(getUserList());
result = usersService.queryAllGeneralUsers(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<User> userList = (List<User>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(userList));
}
@Test
public void testVerifyUserName(){
//not exist user
Result result = usersService.verifyUserName("admin89899");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
//exist user
when(userMapper.queryByUserNameAccurately("userTest0001")).thenReturn(getUser());
result = usersService.verifyUserName("userTest0001");
logger.info(result.toString());
Assert.assertEquals(Status.USER_NAME_EXIST.getMsg(), result.getMsg());
}
@Test
public void testUnauthorizedUser(){
User loginUser = new User();
when(userMapper.selectList(null )).thenReturn(getUserList());
when( userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList());
//no operate
Map<String, Object> result = usersService.unauthorizedUser(loginUser, 2);
logger.info(result.toString());
loginUser.setUserType(UserType.ADMIN_USER);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
result = usersService.unauthorizedUser(loginUser, 2);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testAuthorizedUser(){
User loginUser = new User();
when(userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList());
//no operate
Map<String, Object> result = usersService.authorizedUser(loginUser, 2);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
result = usersService.authorizedUser(loginUser, 2);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<User> userList = (List<User>) result.get(Constants.DATA_LIST);
logger.info(result.toString());
Assert.assertTrue(CollectionUtils.isNotEmpty(userList));
}
/**
* get user
* @return
*/
private User getGeneralUser(){
User user = new User();
user.setUserType(UserType.GENERAL_USER);
user.setUserName("userTest0001");
user.setUserPassword("userTest0001");
return user;
}
private List<User> getUserList(){
List<User> userList = new ArrayList<>();
userList.add(getGeneralUser());
return userList;
}
/**
* get user
*/
private User getUser(){
User user = new User();
user.setUserType(UserType.ADMIN_USER);
user.setUserName("userTest0001");
user.setUserPassword("userTest0001");
return user;
}
private Tenant getTenant(){
Tenant tenant = new Tenant();
tenant.setId(1);
return tenant;
}
}

View File

@ -0,0 +1,173 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class WorkerGroupServiceTest {
private static final Logger logger = LoggerFactory.getLogger(WorkerGroupServiceTest.class);
@InjectMocks
private WorkerGroupService workerGroupService;
@Mock
private WorkerGroupMapper workerGroupMapper;
@Mock
private ProcessInstanceMapper processInstanceMapper;
private String groupName="groupName000001";
/**
* create or update a worker group
*/
@Test
public void testSaveWorkerGroup(){
User user = new User();
// general user add
user.setUserType(UserType.GENERAL_USER);
Map<String, Object> result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1");
logger.info(result.toString());
Assert.assertEquals( Status.USER_NO_OPERATION_PERM.getMsg(),(String) result.get(Constants.MSG));
//success
user.setUserType(UserType.ADMIN_USER);
result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
// group name exist
Mockito.when(workerGroupMapper.selectById(2)).thenReturn(getWorkerGroup(2));
Mockito.when(workerGroupMapper.queryWorkerGroupByName(groupName)).thenReturn(getList());
result = workerGroupService.saveWorkerGroup(user, 2, groupName, "127.0.0.1");
logger.info(result.toString());
Assert.assertEquals(Status.NAME_EXIST,result.get(Constants.STATUS));
}
/**
* query worker group paging
*/
@Test
public void testQueryAllGroupPaging(){
User user = new User();
// general user add
user.setUserType(UserType.GENERAL_USER);
Map<String, Object> result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName);
logger.info(result.toString());
Assert.assertEquals((String) result.get(Constants.MSG), Status.USER_NO_OPERATION_PERM.getMsg());
//success
user.setUserType(UserType.ADMIN_USER);
Page<WorkerGroup> page = new Page<>(1,10);
page.setRecords(getList());
page.setSize(1L);
Mockito.when(workerGroupMapper.queryListPaging(Mockito.any(Page.class), Mockito.eq(groupName))).thenReturn(page);
result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
PageInfo<WorkerGroup> pageInfo = (PageInfo<WorkerGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists()));
}
/**
* delete group by id
*/
@Test
public void testDeleteWorkerGroupById(){
//DELETE_WORKER_GROUP_BY_ID_FAIL
Mockito.when(processInstanceMapper.queryByWorkerGroupIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList());
Map<String, Object> result = workerGroupService.deleteWorkerGroupById(1);
logger.info(result.toString());
Assert.assertEquals(Status.DELETE_WORKER_GROUP_BY_ID_FAIL.getCode(),((Status) result.get(Constants.STATUS)).getCode());
//correct
result = workerGroupService.deleteWorkerGroupById(2);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
}
@Test
public void testQueryAllGroup(){
Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(getList());
Map<String, Object> result = workerGroupService.queryAllGroup();
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG));
List<WorkerGroup> workerGroupList = (List<WorkerGroup>) result.get(Constants.DATA_LIST);
Assert.assertTrue(workerGroupList.size()>0);
}
/**
* get processInstances
* @return
*/
private List<ProcessInstance> getProcessInstanceList(){
List<ProcessInstance> processInstances = new ArrayList<>();
processInstances.add(new ProcessInstance());
return processInstances;
}
/**
* get Group
* @return
*/
private WorkerGroup getWorkerGroup(int id){
WorkerGroup workerGroup = new WorkerGroup();
workerGroup.setName(groupName);
workerGroup.setId(id);
return workerGroup;
}
private WorkerGroup getWorkerGroup(){
return getWorkerGroup(1);
}
private List<WorkerGroup> getList(){
List<WorkerGroup> list = new ArrayList<>();
list.add(getWorkerGroup());
return list;
}
}

View File

@ -26,25 +26,10 @@ import java.util.regex.Pattern;
*/
public final class Constants {
/**
* zookeeper properties path
*/
public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties";
/**
* hadoop properties path
*/
public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties";
/**
* common properties path
*/
public static final String COMMON_PROPERTIES_PATH = "/common/common.properties";
/**
* dao properties path
*/
public static final String DAO_PROPERTIES_PATH = "application.properties";
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
/**
* fs.defaultFS
@ -227,6 +212,8 @@ public final class Constants {
public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout";
public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep";
public static final String ZOOKEEPER_RETRY_BASE_SLEEP = "zookeeper.retry.base.sleep";
public static final String ZOOKEEPER_RETRY_MAX_SLEEP = "zookeeper.retry.max.sleep";
public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime";
@ -469,7 +456,7 @@ public final class Constants {
/**
* task record configuration path
*/
public static final String APPLICATION_PROPERTIES = "application-dao.properties";
public static final String APPLICATION_PROPERTIES = "application.properties";
public static final String TASK_RECORD_URL = "task.record.datasource.url";

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* alert status
*/
@Getter
public enum AlertStatus {
/**
* 0 waiting executed; 1 execute successfully2 execute failed
@ -40,4 +38,12 @@ public enum AlertStatus {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* warning message notification method
*/
@Getter
public enum AlertType {
/**
* 0 email; 1 SMS
@ -39,4 +37,12 @@ public enum AlertType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* command types
*/
@Getter
public enum CommandType {
/**
@ -59,4 +57,12 @@ public enum CommandType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,38 +17,44 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* data base types
*/
@Getter
public enum DbType {
/**
* 0 mysql
* 1 postgresql
* 2 hive
* 3 spark
* 4 clickhouse
* 5 oracle
* 6 sqlserver
* 7 db2
*/
MYSQL(0, "mysql"),
POSTGRESQL(1, "postgresql"),
HIVE(2, "hive"),
SPARK(3, "spark"),
CLICKHOUSE(4, "clickhouse"),
ORACLE(5, "oracle"),
SQLSERVER(6, "sqlserver"),
DB2(7, "db2");
/**
* 0 mysql
* 1 postgresql
* 2 hive
* 3 spark
* 4 clickhouse
* 5 oracle
* 6 sqlserver
* 7 db2
*/
MYSQL(0, "mysql"),
POSTGRESQL(1, "postgresql"),
HIVE(2, "hive"),
SPARK(3, "spark"),
CLICKHOUSE(4, "clickhouse"),
ORACLE(5, "oracle"),
SQLSERVER(6, "sqlserver"),
DB2(7, "db2");
DbType(int code, String descp){
this.code = code;
this.descp = descp;
}
DbType(int code, String descp) {
this.code = code;
this.descp = descp;
}
@EnumValue
private final int code;
private final String descp;
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* running status for workflow and task nodes
*
*/
@Getter
public enum ExecutionStatus {
/**
@ -123,5 +121,11 @@ public enum ExecutionStatus {
return this == KILL || this == STOP ;
}
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* failure policy when some task node failed.
*/
@Getter
public enum FailureStrategy {
/**
@ -40,4 +38,12 @@ public enum FailureStrategy {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* have_script
@ -27,7 +26,6 @@ import lombok.Getter;
* have_map_variables
* have_alert
*/
@Getter
public enum Flag {
/**
* 0 no
@ -45,4 +43,12 @@ public enum Flag {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* define process and task priority
*/
@Getter
public enum Priority {
/**
* 0 highest priority
@ -46,4 +44,11 @@ public enum Priority {
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* process define release state
*/
@Getter
public enum ReleaseState {
/**
@ -50,4 +48,12 @@ public enum ReleaseState {
//For values out of enum scope
return null;
}
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* resource type
*/
@Getter
public enum ResourceType {
/**
* 0 file, 1 udf
@ -39,4 +37,12 @@ public enum ResourceType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* complement data run mode
*/
@Getter
public enum RunMode {
/**
* 0 serial run
@ -39,4 +37,12 @@ public enum RunMode {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* show type for email
*/
@Getter
public enum ShowType {
/**
* 0 TABLE;
@ -44,4 +42,12 @@ public enum ShowType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,9 +17,7 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
@Getter
public enum SparkVersion {
/**
@ -37,4 +35,12 @@ public enum SparkVersion {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* task node depend type
*/
@Getter
public enum TaskDependType {
/**
* 0 run current tasks only
@ -41,4 +39,12 @@ public enum TaskDependType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* task node type
*/
@Getter
public enum TaskType {
/**
* 0 SHELL
@ -61,4 +59,11 @@ public enum TaskType {
return !(taskType == TaskType.SUB_PROCESS || taskType == TaskType.DEPENDENT);
}
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* UDF type
*/
@Getter
public enum UdfType {
/**
* 0 hive; 1 spark
@ -38,4 +36,12 @@ public enum UdfType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* user type
*/
@Getter
public enum UserType {
/**
* 0 admin user; 1 general user
@ -39,5 +37,13 @@ public enum UserType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -17,12 +17,10 @@
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;
import lombok.Getter;
/**
* types for whether to send warning when process ending;
*/
@Getter
public enum WarningType {
/**
* 0 do not send warning;
@ -44,4 +42,12 @@ public enum WarningType {
@EnumValue
private final int code;
private final String descp;
public int getCode() {
return code;
}
public String getDescp() {
return descp;
}
}

View File

@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.queue;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -43,7 +44,7 @@ public class TaskQueueFactory {
String queueImplValue = CommonUtils.getQueueImplValue();
if (StringUtils.isNotBlank(queueImplValue)) {
logger.info("task queue impl use zookeeper ");
return TaskQueueZkImpl.getInstance();
return SpringApplicationContext.getBean(TaskQueueZkImpl.class);
}else{
logger.error("property dolphinscheduler.queue.impl can't be blank, system will exit ");
System.exit(-1);

View File

@ -17,22 +17,14 @@
package org.apache.dolphinscheduler.common.queue;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.Bytes;
import org.apache.dolphinscheduler.common.utils.IpUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.zk.DefaultEnsembleProvider;
import org.apache.dolphinscheduler.common.zk.ZookeeperConfig;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.apache.dolphinscheduler.common.zk.ZookeeperOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
@ -40,35 +32,13 @@ import java.util.*;
* A singleton of a task queue implemented with zookeeper
* tasks queue implemention
*/
@Service
public class TaskQueueZkImpl implements ITaskQueue {
private static final Logger logger = LoggerFactory.getLogger(TaskQueueZkImpl.class);
private static volatile TaskQueueZkImpl instance;
private CuratorFramework zkClient;
private ZookeeperConfig zookeeperConfig;
private CuratorFramework getZkClient() {
return zkClient;
}
private TaskQueueZkImpl(){
init();
}
public static TaskQueueZkImpl getInstance(){
if (null == instance) {
synchronized (TaskQueueZkImpl.class) {
if(null == instance) {
instance = new TaskQueueZkImpl();
}
}
}
return instance;
}
@Autowired
private ZookeeperOperator zookeeperOperator;
/**
* get all tasks from tasks queue
@ -78,14 +48,12 @@ public class TaskQueueZkImpl implements ITaskQueue {
@Override
public List<String> getAllTasks(String key) {
try {
List<String> list = getZkClient().getChildren().forPath(getTasksPath(key));
List<String> list = zookeeperOperator.getChildrenKeys(getTasksPath(key));
return list;
} catch (Exception e) {
logger.error("get all tasks from tasks queue exception",e);
}
return new ArrayList<String>();
return new ArrayList<>();
}
/**
@ -99,22 +67,8 @@ public class TaskQueueZkImpl implements ITaskQueue {
public boolean checkTaskExists(String key, String task) {
String taskPath = getTasksPath(key) + Constants.SINGLE_SLASH + task;
try {
Stat stat = zkClient.checkExists().forPath(taskPath);
return zookeeperOperator.isExisted(taskPath);
if(null == stat){
logger.info("check task:{} not exist in task queue",task);
return false;
}else{
logger.info("check task {} exists in task queue ",task);
return true;
}
} catch (Exception e) {
logger.info(String.format("task {} check exists in task queue exception ", task), e);
}
return false;
}
@ -128,9 +82,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
public boolean add(String key, String value){
try {
String taskIdPath = getTasksPath(key) + Constants.SINGLE_SLASH + value;
String result = getZkClient().create().withMode(CreateMode.PERSISTENT).forPath(taskIdPath, Bytes.toBytes(value));
logger.info("add task : {} to tasks queue , result success",result);
zookeeperOperator.persist(taskIdPath, value);
return true;
} catch (Exception e) {
logger.error("add task to tasks queue exception",e);
@ -153,8 +105,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
@Override
public List<String> poll(String key, int tasksNum) {
try{
CuratorFramework zk = getZkClient();
List<String> list = zk.getChildren().forPath(getTasksPath(key));
List<String> list = zookeeperOperator.getChildrenKeys(getTasksPath(key));
if(list != null && list.size() > 0){
@ -277,15 +228,12 @@ public class TaskQueueZkImpl implements ITaskQueue {
@Override
public void removeNode(String key, String nodeValue){
CuratorFramework zk = getZkClient();
String tasksQueuePath = getTasksPath(key) + Constants.SINGLE_SLASH;
String taskIdPath = tasksQueuePath + nodeValue;
logger.info("consume task {}", taskIdPath);
logger.info("removeNode task {}", taskIdPath);
try{
Stat stat = zk.checkExists().forPath(taskIdPath);
if(stat != null){
zk.delete().forPath(taskIdPath);
}
zookeeperOperator.remove(taskIdPath);
}catch(Exception e){
logger.error(String.format("delete task:%s from zookeeper fail, exception:" ,nodeValue) ,e);
}
@ -307,13 +255,10 @@ public class TaskQueueZkImpl implements ITaskQueue {
if(value != null && value.trim().length() > 0){
String path = getTasksPath(key) + Constants.SINGLE_SLASH;
CuratorFramework zk = getZkClient();
Stat stat = zk.checkExists().forPath(path + value);
if(null == stat){
String result = zk.create().withMode(CreateMode.PERSISTENT).forPath(path + value,Bytes.toBytes(value));
logger.info("add task:{} to tasks set result:{} ",value,result);
}else{
if(!zookeeperOperator.isExisted(path + value)){
zookeeperOperator.persist(path + value,value);
logger.info("add task:{} to tasks set ",value);
} else{
logger.info("task {} exists in tasks set ",value);
}
@ -336,15 +281,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
public void srem(String key, String value) {
try{
String path = getTasksPath(key) + Constants.SINGLE_SLASH;
CuratorFramework zk = getZkClient();
Stat stat = zk.checkExists().forPath(path + value);
if(null != stat){
zk.delete().forPath(path + value);
logger.info("delete task:{} from tasks set ",value);
}else{
logger.info("delete task:{} from tasks set fail, there is no this task",value);
}
zookeeperOperator.remove(path + value);
}catch(Exception e){
logger.error(String.format("delete task:" + value + " exception"),e);
@ -363,7 +300,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
Set<String> tasksSet = new HashSet<>();
try {
List<String> list = getZkClient().getChildren().forPath(getTasksPath(key));
List<String> list = zookeeperOperator.getChildrenKeys(getTasksPath(key));
for (String task : list) {
tasksSet.add(task);
@ -377,56 +314,6 @@ public class TaskQueueZkImpl implements ITaskQueue {
return tasksSet;
}
/**
* Init the task queue of zookeeper node
*/
private void init(){
initZkClient();
try {
String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL);
for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){
if(zkClient.checkExists().forPath(taskQueuePath) == null){
// create a persistent parent node
zkClient.create().creatingParentContainersIfNeeded()
.withMode(CreateMode.PERSISTENT).forPath(taskQueuePath);
logger.info("create tasks queue parent node success : {} ",taskQueuePath);
}
}
} catch (Exception e) {
logger.error("create zk node failure",e);
}
}
private void initZkClient() {
Configuration conf = null;
try {
conf = new PropertiesConfiguration(Constants.ZOOKEEPER_PROPERTIES_PATH);
} catch (ConfigurationException ex) {
logger.error("load zookeeper properties file failed, system exit");
System.exit(-1);
}
zkClient = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(conf.getString("zookeeper.quorum")))
.retryPolicy(new ExponentialBackoffRetry(conf.getInt("zookeeper.retry.base.sleep"), conf.getInt("zookeeper.retry.maxtime"), conf.getInt("zookeeper.retry.max.sleep")))
.sessionTimeoutMs(conf.getInt("zookeeper.session.timeout"))
.connectionTimeoutMs(conf.getInt("zookeeper.connection.timeout"))
.build();
zkClient.start();
try {
zkClient.blockUntilConnected();
} catch (final Exception ex) {
throw new RuntimeException(ex);
}
}
/**
* Clear the task queue of zookeeper node
*/
@ -437,16 +324,12 @@ public class TaskQueueZkImpl implements ITaskQueue {
String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL);
for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){
if(zkClient.checkExists().forPath(taskQueuePath) != null){
List<String> list = zkClient.getChildren().forPath(taskQueuePath);
if(zookeeperOperator.isExisted(taskQueuePath)){
List<String> list = zookeeperOperator.getChildrenKeys(taskQueuePath);
for (String task : list) {
zkClient.delete().forPath(taskQueuePath + Constants.SINGLE_SLASH + task);
zookeeperOperator.remove(taskQueuePath + Constants.SINGLE_SLASH + task);
logger.info("delete task from tasks queue : {}/{} ",taskQueuePath,task);
}
}
}

View File

@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Properties;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
import static org.apache.dolphinscheduler.common.Constants.HADOOP_PROPERTIES_PATH;
/**
* property utils
@ -51,7 +50,7 @@ public class PropertyUtils {
}
private void init(){
String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH};
String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH};
for (String fileName : propertyFiles) {
InputStream fis = null;
try {

View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
package org.apache.dolphinscheduler.common.utils;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;

View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.zk;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.curator.framework.recipes.cache.TreeCacheListener;
public abstract class AbstractListener implements TreeCacheListener {
@Override
public final void childEvent(final CuratorFramework client, final TreeCacheEvent event) throws Exception {
String path = null == event.getData() ? "" : event.getData().getPath();
if (path.isEmpty()) {
return;
}
dataChanged(client, event, path);
}
protected abstract void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path);
}

View File

@ -16,6 +16,22 @@
*/
package org.apache.dolphinscheduler.common.zk;
import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP;
import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP;
import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX;
import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH;
import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
@ -23,26 +39,9 @@ import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ResInfo;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import static org.apache.dolphinscheduler.common.Constants.*;
/**
* abstract zookeeper client
@ -70,8 +69,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
return;
}
byte[] bytes = zkClient.getData().forPath(znode);
String resInfoStr = new String(bytes);
String resInfoStr = super.get(znode);
String[] splits = resInfoStr.split(Constants.COMMA);
if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){
return;
@ -107,8 +105,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX;
String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo;
if(zkClient.checkExists().forPath(zNode) == null ||
zkClient.checkExists().forPath(deadServerPath) != null ){
if(!isExisted(zNode) || isExisted(deadServerPath)){
return true;
}
@ -118,14 +115,12 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
public void removeDeadServerByHost(String host, String serverType) throws Exception {
List<String> deadServers = zkClient.getChildren().forPath(getDeadZNodeParentPath());
List<String> deadServers = super.getChildrenKeys(getDeadZNodeParentPath());
for(String serverPath : deadServers){
if(serverPath.startsWith(serverType+UNDERLINE+host)){
String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath;
if(zkClient.checkExists().forPath(server) != null){
zkClient.delete().forPath(server);
logger.info("{} server {} deleted from zk dead server path success" , serverType , host);
}
String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath;
super.remove(server);
logger.info("{} server {} deleted from zk dead server path success" , serverType , host);
}
}
}
@ -143,8 +138,8 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
// create temporary sequence nodes for master znode
String parentPath = getZNodeParentPath(zkNodeType);
String serverPathPrefix = parentPath + "/" + OSUtils.getHost();
String registerPath = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(
serverPathPrefix + "_", heartbeatZKInfo.getBytes());
String registerPath = serverPathPrefix + UNDERLINE;
super.persistEphemeral(registerPath, heartbeatZKInfo);
logger.info("register {} node {} success" , zkNodeType.toString(), registerPath);
return registerPath;
}
@ -165,7 +160,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
}
registerPath = createZNodePath(zkNodeType);
// handle dead server
// handle dead server
handleDeadServer(registerPath, zkNodeType, Constants.DELETE_ZK_OP);
return registerPath;
@ -196,10 +191,10 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
}else if(opType.equals(ADD_ZK_OP)){
String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo;
if(zkClient.checkExists().forPath(deadServerPath) == null){
if(!super.isExisted(deadServerPath)){
//add dead server info to zk dead server path : /dead-servers/
zkClient.create().forPath(deadServerPath,(type + UNDERLINE + ipSeqNo).getBytes());
super.persist(deadServerPath,(type + UNDERLINE + ipSeqNo));
logger.info("{} server dead , and {} added to zk dead server path success" ,
zkNodeType.toString(), zNode);
@ -226,19 +221,13 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
List<String> childrenList = new ArrayList<>();
try {
// read master node parent path from conf
if(zkClient.checkExists().forPath(getZNodeParentPath(ZKNodeType.MASTER)) != null){
childrenList = zkClient.getChildren().forPath(getZNodeParentPath(ZKNodeType.MASTER));
if(super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))){
childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER));
}
} catch (Exception e) {
if(e.getMessage().contains("java.lang.IllegalStateException: instance must be started")){
logger.error("zookeeper service not started",e);
}else{
logger.error(e.getMessage(),e);
}
}finally {
return childrenList.size();
logger.error("getActiveMasterNum error",e);
}
return childrenList.size();
}
/**
@ -280,10 +269,9 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
Map<String, String> masterMap = new HashMap<>();
try {
String path = getZNodeParentPath(zkNodeType);
List<String> serverList = getZkClient().getChildren().forPath(path);
List<String> serverList = super.getChildrenKeys(path);
for(String server : serverList){
byte[] bytes = getZkClient().getData().forPath(path + "/" + server);
masterMap.putIfAbsent(server, new String(bytes));
masterMap.putIfAbsent(server, super.get(path + "/" + server));
}
} catch (Exception e) {
logger.error("get server list failed : " + e.getMessage(), e);
@ -307,7 +295,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
}
Map<String, String> serverMaps = getServerMaps(zkNodeType);
for(String hostKey : serverMaps.keySet()){
if(hostKey.startsWith(host)){
if(hostKey.startsWith(host + UNDERLINE)){
return true;
}
}
@ -430,27 +418,15 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{
*/
protected void initSystemZNode(){
try {
createNodePath(getMasterZNodeParentPath());
createNodePath(getWorkerZNodeParentPath());
createNodePath(getDeadZNodeParentPath());
persist(getMasterZNodeParentPath(), "");
persist(getWorkerZNodeParentPath(), "");
persist(getDeadZNodeParentPath(), "");
} catch (Exception e) {
logger.error("init system znode failed : " + e.getMessage(),e);
}
}
/**
* create zookeeper node path if not exists
* @param zNodeParentPath zookeeper parent path
* @throws Exception errors
*/
private void createNodePath(String zNodeParentPath) throws Exception {
if(null == zkClient.checkExists().forPath(zNodeParentPath)){
zkClient.create().creatingParentContainersIfNeeded()
.withMode(CreateMode.PERSISTENT).forPath(zNodeParentPath);
}
}
/**
* server self dead, stop all threads
* @param serverHost server host

View File

@ -16,8 +16,10 @@
*/
package org.apache.dolphinscheduler.common.zk;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.curator.framework.recipes.cache.TreeCacheListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,30 +36,37 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
private final Logger logger = LoggerFactory.getLogger(ZookeeperCachedOperator.class);
//kay is zk path, value is TreeCache
private ConcurrentHashMap<String, TreeCache> allCaches = new ConcurrentHashMap<>();
TreeCache treeCache;
/**
* @param cachePath zk path
* @param listener operator
* register a unified listener of /${dsRoot},
*/
public void registerListener(final String cachePath, final TreeCacheListener listener) {
TreeCache newCache = new TreeCache(zkClient, cachePath);
logger.info("add listener to zk path: {}", cachePath);
@Override
protected void registerListener() {
treeCache = new TreeCache(zkClient, getZookeeperConfig().getDsRoot());
logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot());
try {
newCache.start();
treeCache.start();
} catch (Exception e) {
logger.error("add listener to zk path: {} failed", cachePath);
logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot());
throw new RuntimeException(e);
}
newCache.getListenable().addListener(listener);
treeCache.getListenable().addListener((client, event) -> {
String path = null == event.getData() ? "" : event.getData().getPath();
if (path.isEmpty()) {
return;
}
dataChanged(client, event, path);
});
allCaches.put(cachePath, newCache);
}
//for sub class
protected void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path){}
public String getFromCache(final String cachePath, final String key) {
ChildData resultInCache = allCaches.get(checkNotNull(cachePath)).getCurrentData(key);
ChildData resultInCache = treeCache.getCurrentData(key);
if (null != resultInCache) {
return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8);
}
@ -65,18 +74,15 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
}
public TreeCache getTreeCache(final String cachePath) {
return allCaches.get(checkNotNull(cachePath));
return treeCache;
}
public void close() {
allCaches.forEach((path, cache) -> {
cache.close();
try {
Thread.sleep(500);
} catch (InterruptedException ignore) {
}
});
treeCache.close();
try {
Thread.sleep(500);
} catch (InterruptedException ignore) {
}
super.close();
}
}

View File

@ -24,7 +24,7 @@ import org.springframework.stereotype.Component;
* zookeeper conf
*/
@Component
@PropertySource("classpath:zookeeper.properties")
@PropertySource("classpath:common.properties")
public class ZookeeperConfig {
//zk connect config

View File

@ -57,11 +57,13 @@ public class ZookeeperOperator implements InitializingBean {
public void afterPropertiesSet() throws Exception {
this.zkClient = buildClient();
initStateLister();
//init();
registerListener();
}
//for subclass
//protected void init(){}
/**
* this method is for sub class,
*/
protected void registerListener(){}
public void initStateLister() {
checkNotNull(zkClient);
@ -127,9 +129,6 @@ public class ZookeeperOperator implements InitializingBean {
List<String> values;
try {
values = zkClient.getChildren().forPath(key);
if (CollectionUtils.isEmpty(values)) {
logger.warn("getChildrenKeys key : {} is empty", key);
}
return values;
} catch (InterruptedException ex) {
logger.error("getChildrenKeys key : {} InterruptedException", key);

View File

@ -18,6 +18,28 @@
#task queue implementation, default "zookeeper"
dolphinscheduler.queue.impl=zookeeper
#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
zookeeper.quorum=localhost:2181
#dolphinscheduler root directory
zookeeper.dolphinscheduler.root=/dolphinscheduler
#dolphinscheduler failover directory
zookeeper.session.timeout=300
zookeeper.connection.timeout=300
zookeeper.retry.base.sleep=100
zookeeper.retry.max.sleep=30000
zookeeper.retry.maxtime=5
# resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE
# Users who have permission to create directories under the HDFS root path
hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
data.store2hdfs.basepath=/dolphinscheduler
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
@ -27,14 +49,6 @@ data.download.basedir.path=/tmp/dolphinscheduler/download
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/tmp/dolphinscheduler/exec
# Users who have permission to create directories under the HDFS root path
hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
data.store2hdfs.basepath=/dolphinscheduler
# resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE
# whether kerberos starts
hadoop.security.authentication.startup.state=false
@ -57,3 +71,24 @@ resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties
# is development state? default "false"
development.state=true
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directorysupport s3for example : s3a://dolphinscheduler
fs.defaultFS=hdfs://mycluster:8020
# s3 needs3 endpoint
fs.s3a.endpoint=http://192.168.199.91:9010
# s3 needs3 access key
fs.s3a.access.key=A3DXS30FO22544RE
# s3 needs3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
#resourcemanager ha note this need ips , this empty if single
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s

View File

@ -1,35 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directorysupport s3for example : s3a://dolphinscheduler
fs.defaultFS=hdfs://mycluster:8020
# s3 needs3 endpoint
fs.s3a.endpoint=http://192.168.199.91:9010
# s3 needs3 access key
fs.s3a.access.key=A3DXS30FO22544RE
# s3 needs3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
#resourcemanager ha note this need ips , this empty if single
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s

View File

@ -18,6 +18,15 @@
#============================================================================
# Configure Main Scheduler Properties
#============================================================================
#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
org.quartz.dataSource.myDs.driver = org.postgresql.Driver
#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.user = test
org.quartz.dataSource.myDs.password = test
org.quartz.scheduler.instanceName = DolphinScheduler
org.quartz.scheduler.instanceId = AUTO
org.quartz.scheduler.makeSchedulerThreadDaemon = true
@ -37,8 +46,7 @@ org.quartz.threadPool.threadPriority = 5
#============================================================================
org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.tablePrefix = QRTZ_
org.quartz.jobStore.isClustered = true
org.quartz.jobStore.misfireThreshold = 60000
@ -49,11 +57,5 @@ org.quartz.jobStore.dataSource = myDs
# Configure Datasources
#============================================================================
org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider
#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
org.quartz.dataSource.myDs.driver = org.postgresql.Driver
#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.URL = jdbc:postgresql://192.168.xx.xx:5432/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.user = xx
org.quartz.dataSource.myDs.password = xx
org.quartz.dataSource.myDs.maxConnections = 10
org.quartz.dataSource.myDs.validationQuery = select 1

View File

@ -1,43 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
zookeeper.quorum=localhost:2181
#dolphinscheduler root directory
zookeeper.dolphinscheduler.root=/dolphinscheduler
#zookeeper server dirctory
#zookeeper.dolphinscheduler.dead.servers=/dolphinscheduler/dead-servers
#zookeeper.dolphinscheduler.masters=/dolphinscheduler/masters
#zookeeper.dolphinscheduler.workers=/dolphinscheduler/workers
#zookeeper lock dirctory
#zookeeper.dolphinscheduler.lock.masters=/dolphinscheduler/lock/masters
#zookeeper.dolphinscheduler.lock.workers=/dolphinscheduler/lock/workers
#dolphinscheduler failover directory
#zookeeper.dolphinscheduler.lock.failover.masters=/dolphinscheduler/lock/failover/masters
#zookeeper.dolphinscheduler.lock.failover.workers=/dolphinscheduler/lock/failover/workers
#zookeeper.dolphinscheduler.lock.failover.startup.masters=/dolphinscheduler/lock/failover/startup-masters
#dolphinscheduler failover directory
zookeeper.session.timeout=300
zookeeper.connection.timeout=300
zookeeper.retry.base.sleep=100
zookeeper.retry.max.sleep=30000
zookeeper.retry.maxtime=5

View File

@ -32,11 +32,9 @@ import static org.junit.Assert.*;
/**
* task queue test
*/
@Ignore
public class TaskQueueZKImplTest extends BaseTaskQueueTest {
@Before
public void before(){

View File

@ -105,7 +105,8 @@ public class ProcessDao {
/**
* task queue impl
*/
protected ITaskQueue taskQueue = TaskQueueFactory.getTaskQueueInstance();
@Autowired
private ITaskQueue taskQueue;
/**
* handle Command (construct ProcessInstance from Command) , wrapped in transaction
* @param logger logger

View File

@ -32,7 +32,6 @@ import java.util.Map;
/**
* alert
*/
@Data
@TableName("t_ds_alert")
public class Alert {
@ -217,6 +216,72 @@ public class Alert {
this.updateTime = updateTime;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Alert alert = (Alert) o;
if (id != alert.id) {
return false;
}
if (alertGroupId != alert.alertGroupId) {
return false;
}
if (!title.equals(alert.title)) {
return false;
}
if (showType != alert.showType) {
return false;
}
if (!content.equals(alert.content)) {
return false;
}
if (alertType != alert.alertType) {
return false;
}
if (alertStatus != alert.alertStatus) {
return false;
}
if (!log.equals(alert.log)) {
return false;
}
if (!receivers.equals(alert.receivers)) {
return false;
}
if (!receiversCc.equals(alert.receiversCc)) {
return false;
}
if (!createTime.equals(alert.createTime)) {
return false;
}
return updateTime.equals(alert.updateTime) && info.equals(alert.info);
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + title.hashCode();
result = 31 * result + showType.hashCode();
result = 31 * result + content.hashCode();
result = 31 * result + alertType.hashCode();
result = 31 * result + alertStatus.hashCode();
result = 31 * result + log.hashCode();
result = 31 * result + alertGroupId;
result = 31 * result + receivers.hashCode();
result = 31 * result + receiversCc.hashCode();
result = 31 * result + createTime.hashCode();
result = 31 * result + updateTime.hashCode();
result = 31 * result + info.hashCode();
return result;
}
@Override
public String toString() {
return "Alert{" +
@ -228,10 +293,10 @@ public class Alert {
", alertStatus=" + alertStatus +
", log='" + log + '\'' +
", alertGroupId=" + alertGroupId +
", createTime=" + createTime +
", updateTime=" + updateTime +
", receivers='" + receivers + '\'' +
", receiversCc='" + receiversCc + '\'' +
", createTime=" + createTime +
", updateTime=" + updateTime +
", info=" + info +
'}';
}

View File

@ -28,7 +28,6 @@ import java.util.Date;
/**
* command
*/
@Data
@TableName("t_ds_command")
public class Command {
@ -265,6 +264,79 @@ public class Command {
this.workerGroupId = workerGroupId;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Command command = (Command) o;
if (id != command.id) {
return false;
}
if (processDefinitionId != command.processDefinitionId) {
return false;
}
if (executorId != command.executorId) {
return false;
}
if (workerGroupId != command.workerGroupId) {
return false;
}
if (commandType != command.commandType) {
return false;
}
if (commandParam != null ? !commandParam.equals(command.commandParam) : command.commandParam != null) {
return false;
}
if (taskDependType != command.taskDependType) {
return false;
}
if (failureStrategy != command.failureStrategy) {
return false;
}
if (warningType != command.warningType) {
return false;
}
if (warningGroupId != null ? !warningGroupId.equals(command.warningGroupId) : command.warningGroupId != null) {
return false;
}
if (scheduleTime != null ? !scheduleTime.equals(command.scheduleTime) : command.scheduleTime != null) {
return false;
}
if (startTime != null ? !startTime.equals(command.startTime) : command.startTime != null) {
return false;
}
if (processInstancePriority != command.processInstancePriority) {
return false;
}
return !(updateTime != null ? !updateTime.equals(command.updateTime) : command.updateTime != null);
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + (commandType != null ? commandType.hashCode() : 0);
result = 31 * result + processDefinitionId;
result = 31 * result + executorId;
result = 31 * result + (commandParam != null ? commandParam.hashCode() : 0);
result = 31 * result + (taskDependType != null ? taskDependType.hashCode() : 0);
result = 31 * result + (failureStrategy != null ? failureStrategy.hashCode() : 0);
result = 31 * result + (warningType != null ? warningType.hashCode() : 0);
result = 31 * result + (warningGroupId != null ? warningGroupId.hashCode() : 0);
result = 31 * result + (scheduleTime != null ? scheduleTime.hashCode() : 0);
result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0);
result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0);
result = 31 * result + workerGroupId;
return result;
}
@Override
public String toString() {
return "Command{" +

View File

@ -33,13 +33,6 @@ public class CommandCount {
private int count;
@Override
public String toString(){
return "command count:" +
" commandType: "+ commandType.toString() +
" count: "+ count;
}
public CommandType getCommandType() {
return commandType;
}
@ -55,4 +48,37 @@ public class CommandCount {
public void setCount(int count) {
this.count = count;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
CommandCount that = (CommandCount) o;
if (count != that.count) {
return false;
}
return commandType == that.commandType;
}
@Override
public int hashCode() {
int result = commandType != null ? commandType.hashCode() : 0;
result = 31 * result + count;
return result;
}
@Override
public String toString() {
return "CommandCount{" +
"commandType=" + commandType +
", count=" + count +
'}';
}
}

View File

@ -49,7 +49,7 @@ public class PropertyUtils {
* init
*/
private void init(){
String[] propertyFiles = new String[]{Constants.DAO_PROPERTIES_PATH};
String[] propertyFiles = new String[]{Constants.APPLICATION_PROPERTIES};
for (String fileName : propertyFiles) {
InputStream fis = null;
try {

View File

@ -19,44 +19,55 @@ package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.Date;
import java.util.List;
import java.util.*;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
/**
* alert mapper test
*/
@RunWith(SpringRunner.class)
@SpringBootTest
@Transactional
@Rollback(true)
public class AlertMapperTest {
@Autowired
AlertMapper alertMapper;
private AlertMapper alertMapper;
/**
* insert
* @return Alert
* test insert
* @return
*/
private Alert insertOne(){
//insertOne
Alert alert = new Alert();
alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]");
alert.setLog("success");
alert.setReceivers("xx@aa.com");
alert.setAlertType(AlertType.EMAIL);
alert.setShowType(ShowType.TABLE);
alert.setAlertGroupId(1);
alert.setAlertStatus(AlertStatus.EXECUTION_SUCCESS);
alert.setCreateTime(new Date());
alert.setUpdateTime(new Date());
alertMapper.insert(alert);
return alert;
@Test
public void testInsert(){
Alert expectedAlert = createAlert();
assertNotNull(expectedAlert.getId());
assertThat(expectedAlert.getId(), greaterThan(0));
}
/**
* test select by id
* @return
*/
@Test
public void testSelectById(){
Alert expectedAlert = createAlert();
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
assertEquals(expectedAlert, actualAlert);
}
/**
@ -64,13 +75,18 @@ public class AlertMapperTest {
*/
@Test
public void testUpdate(){
//insertOne
Alert alert = insertOne();
//update
alert.setTitle("hello");
int update = alertMapper.updateById(alert);
Assert.assertEquals(update, 1);
alertMapper.deleteById(alert.getId());
Alert expectedAlert = createAlert();
expectedAlert.setAlertStatus(AlertStatus.EXECUTION_FAILURE);
expectedAlert.setLog("error");
expectedAlert.setUpdateTime(DateUtils.getCurrentDate());
alertMapper.updateById(expectedAlert);
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
assertEquals(expectedAlert, actualAlert);
}
/**
@ -78,33 +94,83 @@ public class AlertMapperTest {
*/
@Test
public void testDelete(){
Alert expectedAlert = createAlert();
Alert alert = insertOne();
int delete = alertMapper.deleteById(alert.getId());
Assert.assertEquals(delete, 1);
alertMapper.deleteById(expectedAlert.getId());
Alert actualAlert = alertMapper.selectById(expectedAlert.getId());
assertNull(actualAlert);
}
/**
* test query
*/
@Test
public void testQuery() {
Alert alert = insertOne();
//query
List<Alert> alerts = alertMapper.selectList(null);
Assert.assertNotEquals(alerts.size(), 0);
alertMapper.deleteById(alert.getId());
}
/**
* test list alert by status
*/
@Test
public void testListAlertByStatus() {
Alert alert = insertOne();
//query
List<Alert> alerts = alertMapper.listAlertByStatus(AlertStatus.EXECUTION_SUCCESS);
Assert.assertNotEquals(alerts.size(), 0);
alertMapper.deleteById(alert.getId());
Integer count = 10;
AlertStatus waitExecution = AlertStatus.WAIT_EXECUTION;
Map<Integer,Alert> expectedAlertMap = createAlertMap(count, waitExecution);
List<Alert> actualAlerts = alertMapper.listAlertByStatus(waitExecution);
for (Alert actualAlert : actualAlerts){
Alert expectedAlert = expectedAlertMap.get(actualAlert.getId());
if (expectedAlert != null){
assertEquals(expectedAlert,actualAlert);
}
}
}
/**
* create alert map
* @param count alert count
* @param alertStatus alert status
* @return alert map
*/
private Map<Integer,Alert> createAlertMap(Integer count,AlertStatus alertStatus){
Map<Integer,Alert> alertMap = new HashMap<>();
for (int i = 0 ; i < count ;i++){
Alert alert = createAlert(alertStatus);
alertMap.put(alert.getId(),alert);
}
return alertMap;
}
/**
* create alert
* @return alert
* @throws Exception
*/
private Alert createAlert(){
return createAlert(AlertStatus.WAIT_EXECUTION);
}
/**
* create alert
* @param alertStatus alert status
* @return alert
*/
private Alert createAlert(AlertStatus alertStatus){
Alert alert = new Alert();
alert.setShowType(ShowType.TABLE);
alert.setTitle("test alert");
alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]");
alert.setAlertType(AlertType.EMAIL);
alert.setAlertStatus(alertStatus);
alert.setLog("success");
alert.setReceivers("aa@aa.com");
alert.setReceiversCc("bb@aa.com");
alert.setCreateTime(DateUtils.getCurrentDate());
alert.setUpdateTime(DateUtils.getCurrentDate());
alertMapper.insert(alert);
return alert;
}
}

View File

@ -21,7 +21,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication
@ComponentScan("org.apache.dolphinscheduler.dao")
@ComponentScan("org.apache.dolphinscheduler")
public class Application {
public static void main(String[] args) {

View File

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.CommandCount;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@ -25,13 +26,25 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
/**
* command mapper test
*/
@RunWith(SpringRunner.class)
@SpringBootTest
@Transactional
@Rollback(true)
public class CommandMapperTest {
@ -41,23 +54,28 @@ public class CommandMapperTest {
@Autowired
ProcessDefinitionMapper processDefinitionMapper;
/**
* insert
* @return Command
* test insert
*/
private Command insertOne(){
//insertOne
Command command = new Command();
command.setCommandType(CommandType.START_PROCESS);
command.setProcessDefinitionId(1);
command.setExecutorId(4);
command.setProcessInstancePriority(Priority.MEDIUM);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWorkerGroupId(-1);
command.setWarningGroupId(1);
command.setUpdateTime(new Date());
commandMapper.insert(command);
return command;
@Test
public void testInsert(){
Command command = createCommand();
assertNotNull(command.getId());
assertThat(command.getId(),greaterThan(0));
}
/**
* test select by id
*/
@Test
public void testSelectById() {
Command expectedCommand = createCommand();
//query
Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertEquals(expectedCommand, actualCommand);
}
/**
@ -65,13 +83,18 @@ public class CommandMapperTest {
*/
@Test
public void testUpdate(){
//insertOne
Command command = insertOne();
//update
command.setStartTime(new Date());
int update = commandMapper.updateById(command);
Assert.assertEquals(update, 1);
commandMapper.deleteById(command.getId());
Command expectedCommand = createCommand();
// update the command time if current command if recover from waiting
expectedCommand.setUpdateTime(DateUtils.getCurrentDate());
commandMapper.updateById(expectedCommand);
Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertEquals(expectedCommand,actualCommand);
}
/**
@ -79,33 +102,37 @@ public class CommandMapperTest {
*/
@Test
public void testDelete(){
Command expectedCommand = createCommand();
Command Command = insertOne();
int delete = commandMapper.deleteById(Command.getId());
Assert.assertEquals(delete, 1);
commandMapper.deleteById(expectedCommand.getId());
Command actualCommand = commandMapper.selectById(expectedCommand.getId());
assertNull(actualCommand);
}
/**
* test query
*/
@Test
public void testQuery() {
Command command = insertOne();
//query
List<Command> commands = commandMapper.selectList(null);
Assert.assertNotEquals(commands.size(), 0);
commandMapper.deleteById(command.getId());
}
/**
* test query all
*/
@Test
public void testGetAll() {
Command command = insertOne();
List<Command> commands = commandMapper.selectList(null);
Assert.assertNotEquals(commands.size(), 0);
commandMapper.deleteById(command.getId());
Integer count = 10;
Map<Integer, Command> commandMap = createCommandMap(count);
List<Command> actualCommands = commandMapper.selectList(null);
assertThat(actualCommands.size(), greaterThanOrEqualTo(count));
for (Command actualCommand : actualCommands){
Command expectedCommand = commandMap.get(actualCommand.getId());
if (expectedCommand != null){
assertEquals(expectedCommand,actualCommand);
}
}
}
/**
@ -113,28 +140,14 @@ public class CommandMapperTest {
*/
@Test
public void testGetOneToRun() {
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setReleaseState(ReleaseState.ONLINE);
processDefinition.setName("ut test");
processDefinition.setProjectId(1);
processDefinition.setFlag(Flag.YES);
processDefinitionMapper.insert(processDefinition);
Command command = new Command();
command.setCommandType(CommandType.START_PROCESS);
command.setProcessDefinitionId(processDefinition.getId());
command.setExecutorId(4);
command.setProcessInstancePriority(Priority.MEDIUM);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWorkerGroupId(-1);
command.setWarningGroupId(1);
command.setUpdateTime(new Date());
commandMapper.insert(command);
ProcessDefinition processDefinition = createProcessDefinition();
Command command2 = commandMapper.getOneToRun();
Assert.assertNotEquals(command2, null);
commandMapper.deleteById(command.getId());
processDefinitionMapper.deleteById(processDefinition.getId());
Command expectedCommand = createCommand(CommandType.START_PROCESS,processDefinition.getId());
Command actualCommand = commandMapper.getOneToRun();
assertEquals(expectedCommand, actualCommand);
}
/**
@ -142,35 +155,122 @@ public class CommandMapperTest {
*/
@Test
public void testCountCommandState() {
Command command = insertOne();
Integer count = 10;
//insertOne
ProcessDefinition processDefinition = createProcessDefinition();
CommandCount expectedCommandCount = createCommandMap(count, CommandType.START_PROCESS, processDefinition.getId());
Integer[] projectIdArray = {processDefinition.getProjectId()};
Date startTime = DateUtils.stringToDate("2019-12-29 00:10:00");
Date endTime = DateUtils.stringToDate("2019-12-29 23:59:59");
List<CommandCount> actualCommandCounts = commandMapper.countCommandState(0, startTime, endTime, projectIdArray);
assertThat(actualCommandCounts.size(),greaterThanOrEqualTo(1));
Boolean flag = false;
for (CommandCount actualCommandCount : actualCommandCounts){
if (actualCommandCount.getCommandType().equals(expectedCommandCount.getCommandType())){
assertEquals(expectedCommandCount,actualCommandCount);
flag = true;
}
}
assertTrue(flag);
}
/**
* create command map
* @param count map count
* @param commandType comman type
* @param processDefinitionId process definition id
* @return command map
*/
private CommandCount createCommandMap(
Integer count,
CommandType commandType,
Integer processDefinitionId){
CommandCount commandCount = new CommandCount();
for (int i = 0 ;i < count ;i++){
createCommand(commandType,processDefinitionId);
}
commandCount.setCommandType(commandType);
commandCount.setCount(count);
return commandCount;
}
/**
* create process definition
* @return process definition
*/
private ProcessDefinition createProcessDefinition(){
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setName("def 1");
processDefinition.setProjectId(1010);
processDefinition.setUserId(101);
processDefinition.setUpdateTime(new Date());
processDefinition.setCreateTime(new Date());
processDefinition.setReleaseState(ReleaseState.ONLINE);
processDefinition.setName("ut test");
processDefinition.setProjectId(1);
processDefinition.setFlag(Flag.YES);
processDefinitionMapper.insert(processDefinition);
command.setProcessDefinitionId(processDefinition.getId());
commandMapper.updateById(command);
List<CommandCount> commandCounts = commandMapper.countCommandState(
4, null, null, new Integer[0]
);
Integer[] projectIdArray = new Integer[2];
projectIdArray[0] = processDefinition.getProjectId();
projectIdArray[1] = 200;
List<CommandCount> commandCounts2 = commandMapper.countCommandState(
4, null, null, projectIdArray
);
commandMapper.deleteById(command.getId());
processDefinitionMapper.deleteById(processDefinition.getId());
Assert.assertNotEquals(commandCounts.size(), 0);
Assert.assertNotEquals(commandCounts2.size(), 0);
return processDefinition;
}
/**
* create command map
* @param count map count
* @return command map
*/
private Map<Integer,Command> createCommandMap(Integer count){
Map<Integer,Command> commandMap = new HashMap<>();
for (int i = 0; i < count ;i++){
Command command = createCommand();
commandMap.put(command.getId(),command);
}
return commandMap;
}
/**
* create command
* @return
*/
private Command createCommand(){
return createCommand(CommandType.START_PROCESS,1);
}
/**
* create command
* @return Command
*/
private Command createCommand(CommandType commandType,Integer processDefinitionId){
Command command = new Command();
command.setCommandType(commandType);
command.setProcessDefinitionId(processDefinitionId);
command.setExecutorId(4);
command.setCommandParam("test command param");
command.setTaskDependType(TaskDependType.TASK_ONLY);
command.setFailureStrategy(FailureStrategy.CONTINUE);
command.setWarningType(WarningType.ALL);
command.setWarningGroupId(1);
command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00"));
command.setProcessInstancePriority(Priority.MEDIUM);
command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00"));
command.setWorkerGroupId(-1);
commandMapper.insert(command);
return command;
}
}

View File

@ -101,6 +101,333 @@
</plugins>
</build>
</profile>
<profile>
<id>nginx</id>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>dolphinscheduler-nginx</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-nginx.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
<execution>
<id>src</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assembly/dolphinscheduler-src.xml</descriptor>
</descriptors>
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<phase>verify</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>rpmbuild</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<excludeScope>provided</excludeScope>
</configuration>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<extensions>true</extensions>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>attached-rpm</goal>
</goals>
</execution>
</executions>
<configuration>
<name>apache-dolphinscheduler-incubating</name>
<release>1</release>
<distribution>apache dolphinscheduler incubating rpm</distribution>
<group>apache</group>
<packager>dolphinscheduler</packager>
<!-- <version>${project.version}</version> -->
<prefix>/opt/soft</prefix>
<defineStatements>
<!-- disable compile python when rpm build -->
<defineStatement>__os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')</defineStatement>
</defineStatements>
<mappings>
<mapping>
<directory>/opt/soft/${project.build.finalName}/conf</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../dolphinscheduler-alert/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.ftl</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-common/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-dao/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.yml</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-api/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-server/src/main/resources
</location>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>config/*.*</include>
</includes>
</source>
<source>
<location>
${basedir}/../script
</location>
<includes>
<include>env/*.*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/lib</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../dolphinscheduler-dist/target/lib
</location>
<includes>
<include>*.*</include>
</includes>
<excludes>
<exclude>servlet-api-*.jar</exclude>
<exclude>slf4j-log4j12-${slf4j.log4j12.version}.jar</exclude>
</excludes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/bin</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../script
</location>
<includes>
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../
</location>
<includes>
<include>*.sh</include>
<include>*.py</include>
<include>DISCLAIMER</include>
</includes>
</source>
<source>
<location>
${basedir}/../dolphinscheduler-ui
</location>
<includes>
<include>install-dolphinscheduler-ui.sh</include>
</includes>
</source>
<source>
<location>
${basedir}/release-docs
</location>
<includes>
<include>**/*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/dist</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../dolphinscheduler-ui/dist
</location>
<includes>
<include>**/*.*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/sql</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../sql
</location>
<includes>
<include>**/*.*</include>
</includes>
</source>
</sources>
</mapping>
<mapping>
<directory>/opt/soft/${project.build.finalName}/script</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<sources>
<source>
<location>
${basedir}/../script
</location>
<includes>
<include>**/*.*</include>
</includes>
</source>
</sources>
</mapping>
</mappings>
<preinstallScriptlet>
<script>mkdir -p /opt/soft</script>
</preinstallScriptlet>
<postremoveScriptlet>
<script>rm -rf /opt/soft/apache-dolphinscheduler-incubating-${project.version}</script>
</postremoveScriptlet>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>

View File

@ -94,26 +94,12 @@
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>config/*.*</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources/bin</directory>
<includes>
<include>*.*</include>
</includes>
<directoryMode>755</directoryMode>
<outputDirectory>bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
@ -177,7 +163,6 @@
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>config/*.*</include>
<include>env/*.*</include>
</includes>
<outputDirectory>./conf</outputDirectory>

View File

@ -0,0 +1,236 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>dolphinscheduler-nginx</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>${project.build.finalName}-dolphinscheduler-bin</baseDirectory>
<fileSets>
<!--alert start-->
<fileSet>
<directory>${basedir}/../dolphinscheduler-alert/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.ftl</include>
</includes>
<outputDirectory>./conf</outputDirectory>
</fileSet>
<!--alert end-->
<!--api start-->
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources/bin</directory>
<includes>
<include>*.*</include>
</includes>
<directoryMode>755</directoryMode>
<outputDirectory>bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-api/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<!--api end-->
<!--server start-->
<fileSet>
<directory>${basedir}/../dolphinscheduler-server/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>config/*.*</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-common/src/main/resources/bin</directory>
<includes>
<include>*.*</include>
</includes>
<directoryMode>755</directoryMode>
<outputDirectory>bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.json</include>
<include>**/*.yml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
<!--server end-->
<fileSet>
<directory>${basedir}/../dolphinscheduler-server/target/dolphinscheduler-server-${project.version}</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-api/target/dolphinscheduler-api-${project.version}</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-alert/target/dolphinscheduler-alert-${project.version}</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-ui/dist</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>./ui/dist</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-ui</directory>
<includes>
<include>install-dolphinscheduler-ui.sh</include>
</includes>
<outputDirectory>./ui</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../sql</directory>
<includes>
<include>**/*</include>
</includes>
<outputDirectory>./sql</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>*.*</include>
</includes>
<outputDirectory>./script</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>env/*.*</include>
</includes>
<outputDirectory>./conf</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../script</directory>
<includes>
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
</includes>
<outputDirectory>./bin</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/.././</directory>
<includes>
<include>*.sh</include>
<include>*.py</include>
<include>DISCLAIMER</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/release-docs</directory>
<useDefaultExcludes>true</useDefaultExcludes>
<includes>
<include>**/*</include>
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<excludes>
<exclude>javax.servlet:servlet-api</exclude>
<exclude>org.eclipse.jetty.aggregate:jetty-all</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>

View File

@ -23,18 +23,17 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread;
import org.apache.dolphinscheduler.server.quartz.ProcessScheduleJob;
import org.apache.dolphinscheduler.server.quartz.QuartzExecutors;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.zk.ZKMasterClient;
import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.annotation.ComponentScan;
@ -153,10 +152,8 @@ public class MasterServer implements IStoppable {
@Override
public void run() {
if (zkMasterClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
zkMasterClient.getAlertDao().sendServerStopedAlert(
1, OSUtils.getHost(), "Master-Server");
}
zkMasterClient.getAlertDao().sendServerStopedAlert(
1, OSUtils.getHost(), "Master-Server");
}
stop("shutdownhook");
}

View File

@ -18,13 +18,13 @@ package org.apache.dolphinscheduler.server.master.runner;
import org.apache.dolphinscheduler.common.queue.ITaskQueue;
import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.utils.BeanContext;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.AlertManager;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.zk.ZKMasterClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -90,15 +90,15 @@ public class AlertManager {
* process instance format
*/
private static final String PROCESS_INSTANCE_FORMAT =
"\"Id:%d\"," +
"\"Name:%s\"," +
"\"Job type: %s\"," +
"\"State: %s\"," +
"\"Recovery:%s\"," +
"\"Run time: %d\"," +
"\"Start time: %s\"," +
"\"End time: %s\"," +
"\"Host: %s\"" ;
"\"id:%d\"," +
"\"name:%s\"," +
"\"job type: %s\"," +
"\"state: %s\"," +
"\"recovery:%s\"," +
"\"run time: %d\"," +
"\"start time: %s\"," +
"\"end time: %s\"," +
"\"host: %s\"" ;
/**
* get process instance content
@ -234,7 +234,7 @@ public class AlertManager {
String cmdName = getCommandCnName(processInstance.getCommandType());
String success = processInstance.getState().typeIsSuccess() ? "success" :"failed";
alert.setTitle(cmdName + success);
alert.setTitle(cmdName + " " + success);
ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE;
alert.setShowType(showType);
String content = getContentProcessInstance(processInstance, taskInstances);

View File

@ -29,12 +29,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread;
import org.apache.dolphinscheduler.server.zk.ZKWorkerClient;
@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable {
public void run() {
// worker server exit alert
if (zkWorkerClient.getActiveMasterNum() <= 1) {
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server");
}
alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server");
}
stop("shutdownhook");
}
@ -290,22 +288,20 @@ public class WorkerServer implements IStoppable {
Runnable killProcessThread = new Runnable() {
@Override
public void run() {
Set<String> taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
while (Stopper.isRunning()){
try {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (InterruptedException e) {
logger.error("interrupted exception",e);
}
// if set is null , return
Set<String> taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
if (CollectionUtils.isNotEmpty(taskInfoSet)){
for (String taskInfo : taskInfoSet){
killTask(taskInfo, processDao);
removeKillInfoFromQueue(taskInfo);
}
}
taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
try {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (InterruptedException e) {
logger.error("interrupted exception",e);
Thread.currentThread().interrupt();
}
}
}
};

View File

@ -25,12 +25,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.zk.ZKWorkerClient;
import org.slf4j.Logger;
@ -155,6 +155,7 @@ public class FetchTaskThread implements Runnable{
//whether have tasks, if no tasks , no need lock //get all tasks
List<String> tasksQueueList = taskQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
if (CollectionUtils.isEmpty(tasksQueueList)){
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
continue;
}
// creating distributed locks, lock path /dolphinscheduler/lock/worker

View File

@ -16,10 +16,10 @@
*/
package org.apache.dolphinscheduler.server.worker.task;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.slf4j.Logger;
/**

View File

@ -23,10 +23,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.model.DependentItem;
import org.apache.dolphinscheduler.common.utils.DependentUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.DependentUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.slf4j.Logger;

View File

@ -30,10 +30,10 @@ import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.utils.Bytes;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.apache.http.HttpEntity;

View File

@ -29,10 +29,10 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.slf4j.Logger;

View File

@ -22,9 +22,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.python.PythonParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;

View File

@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;

View File

@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.common.task.sql.SqlType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.DataSource;
@ -43,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.utils.UDFUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;

View File

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.zk.AbstractListener;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory;
@ -31,9 +30,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.utils.ThreadUtils;
import org.slf4j.Logger;
@ -101,12 +97,6 @@ public class ZKMasterClient extends AbstractZKClient {
// init system znode
this.initSystemZNode();
// monitor master
this.listenerMaster();
// monitor worker
this.listenerWorker();
// register master
this.registerMaster();
@ -158,31 +148,22 @@ public class ZKMasterClient extends AbstractZKClient {
}
}
/**
* monitor master
* handle path events that this class cares about
* @param client zkClient
* @param event path event
* @param path zk path
*/
public void listenerMaster(){
registerListener(getZNodeParentPath(ZKNodeType.MASTER), new AbstractListener() {
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
switch (event.getType()) {
case NODE_ADDED:
logger.info("master node added : {}", path);
break;
case NODE_REMOVED:
String serverHost = getHostByEventDataPath(path);
if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) {
return;
}
removeZKNodePath(path, ZKNodeType.MASTER, true);
break;
default:
break;
}
}
});
}
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
if(path.startsWith(getZNodeParentPath(ZKNodeType.MASTER)+Constants.SINGLE_SLASH)){ //monitor master
handleMasterEvent(event,path);
}else if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ //monitor worker
handleWorkerEvent(event,path);
}
//other path event, ignore
}
/**
* remove zookeeper node path
@ -266,32 +247,45 @@ public class ZKMasterClient extends AbstractZKClient {
*/
private void alertServerDown(String serverHost, ZKNodeType zkNodeType) {
String serverType = zkNodeType.toString();
for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) {
alertDao.sendServerStopedAlert(1, serverHost, serverType);
String serverType = zkNodeType.toString();
alertDao.sendServerStopedAlert(1, serverHost, serverType);
}
/**
* monitor master
*/
public void handleMasterEvent(TreeCacheEvent event, String path){
switch (event.getType()) {
case NODE_ADDED:
logger.info("master node added : {}", path);
break;
case NODE_REMOVED:
String serverHost = getHostByEventDataPath(path);
if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) {
return;
}
removeZKNodePath(path, ZKNodeType.MASTER, true);
break;
default:
break;
}
}
/**
* monitor worker
*/
public void listenerWorker(){
registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() {
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
logger.info("worker node deleted : {}", path);
removeZKNodePath(path, ZKNodeType.WORKER, true);
break;
default:
break;
}
}
});
public void handleWorkerEvent(TreeCacheEvent event, String path){
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
logger.info("worker node deleted : {}", path);
removeZKNodePath(path, ZKNodeType.WORKER, true);
break;
default:
break;
}
}

View File

@ -19,20 +19,13 @@ package org.apache.dolphinscheduler.server.zk;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.common.zk.AbstractListener;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.utils.ThreadUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.util.concurrent.ThreadFactory;
/**
* zookeeper worker client
@ -61,9 +54,6 @@ public class ZKWorkerClient extends AbstractZKClient {
// init system znode
this.initSystemZNode();
// monitor worker
this.listenerWorker();
// register worker
this.registWorker();
}
@ -83,31 +73,38 @@ public class ZKWorkerClient extends AbstractZKClient {
System.exit(-1);
}
}
/**
* monitor worker
*/
private void listenerWorker(){
registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() {
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
//find myself dead
String serverHost = getHostByEventDataPath(path);
if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){
return;
}
break;
default:
break;
}
}
});
/**
* handle path events that this class cares about
* @param client zkClient
* @param event path event
* @param path zk path
*/
@Override
protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) {
if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){
handleWorkerEvent(event,path);
}
}
/**
* monitor worker
*/
public void handleWorkerEvent(TreeCacheEvent event, String path){
switch (event.getType()) {
case NODE_ADDED:
logger.info("worker node added : {}", path);
break;
case NODE_REMOVED:
//find myself dead
String serverHost = getHostByEventDataPath(path);
if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){
return;
}
break;
default:
break;
}
}
/**

View File

@ -1,18 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
logging.config=classpath:master_logback.xml

View File

@ -1,18 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
logging.config=classpath:worker_logback.xml

View File

@ -20,10 +20,10 @@ import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.LoggerUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskManager;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;

View File

@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.LoggerUtils;
import org.apache.dolphinscheduler.server.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskManager;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;

View File

@ -32,55 +32,120 @@
<node.version>v12.12.0</node.version>
<npm.version>6.11.3</npm.version>
</properties>
<profiles>
<profile>
<id>release</id>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<npmVersion>${npm.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install node-sass --unsafe-perm</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install node-sass --unsafe-perm</arguments>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>npm run build:release</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run build:release</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>nginx</id>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<npmVersion>${npm.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install node-sass --unsafe-perm</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install node-sass --unsafe-perm</arguments>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run build</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>${frontend-maven-plugin.version}</version>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<npmVersion>${npm.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install node-sass --unsafe-perm</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install node-sass --unsafe-perm</arguments>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>npm run build:release</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run build:release</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -257,8 +257,8 @@
_rtParam () {
return {
taskInstId: this.stateId || this.logId,
skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '0000' : 0}`),
limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}0000`)
skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '000' : 0}`),
limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}000`)
}
}
},
@ -316,7 +316,7 @@
}
.refresh-log {
>i {
font-size: 24px;
font-size: 20px;
vertical-align: middle;
transform: scale(1);
}
@ -353,7 +353,7 @@
font-weight: bold;
resize:none;
line-height: 1.6;
padding: 6px;
padding: 0px;
}
}
}

View File

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<div class="script-model">
<m-list-box>
<div slot="content">
<div class="from-mirror1">
<textarea
id="code-shell-mirror1"
name="code-shell-mirror1"
style="opacity: 0">
</textarea>
</div>
</div>
</m-list-box>
<a class="ans-modal-box-close">
<i class="ans-icon-min" @click="closeModal"></i>
</a>
</div>
</template>
<script>
import _ from 'lodash'
import i18n from '@/module/i18n'
import mListBox from './listBox'
import disabledState from '@/module/mixin/disabledState'
import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror'
let editor
export default {
name: 'shell',
data () {
return {
// script
rawScript: '',
}
},
mixins: [disabledState],
props: {
item: String
},
methods: {
/**
* Processing code highlighting
*/
_handlerEditor () {
// editor
let self =this
editor = codemirror('code-shell-mirror1', {
mode: 'shell',
readOnly: this.isDetails
})
editor.on("change",function(){
self.$emit('getSriptBoxValue',editor.getValue())
})
this.keypress = () => {
if (!editor.getOption('readOnly')) {
editor.showHint({
completeSingle: false
})
}
}
// Monitor keyboard
editor.on('keypress', this.keypress)
editor.setValue(this.rawScript)
return editor
},
closeModal() {
let self = this
self.$emit('closeAble')
}
},
watch: {},
created () {
let o = this.item
// Non-null objects represent backfill
if (!_.isEmpty(o)) {
this.rawScript = o
}
},
mounted () {
setTimeout(() => {
this._handlerEditor()
}, 200)
},
destroyed () {
if (editor) {
editor.toTextArea() // Uninstall
editor.off($('.code-shell-mirror1'), 'keypress', this.keypress)
}
},
components: { mListBox }
}
</script>
<style lang="scss" rel="stylesheet/scss" scope>
.script-model {
width:100%;
}
.from-mirror1 {
.CodeMirror {
min-height: 600px;
max-height: 700px;
}
}
</style>

View File

@ -21,10 +21,13 @@
<div slot="content">
<div class="from-mirror">
<textarea
id="code-shell-mirror"
name="code-shell-mirror"
style="opacity: 0">
id="code-shell-mirror"
name="code-shell-mirror"
style="opacity: 0">
</textarea>
<a class="ans-modal-box-max">
<i class="ans-icon-max" @click="setEditorVal"></i>
</a>
</div>
</div>
</m-list-box>
@ -55,6 +58,7 @@
import _ from 'lodash'
import i18n from '@/module/i18n'
import mListBox from './_source/listBox'
import mScriptBox from './_source/scriptBox'
import mResources from './_source/resources'
import mLocalParams from './_source/localParams'
import disabledState from '@/module/mixin/disabledState'
@ -85,8 +89,37 @@
_onLocalParams (a) {
this.localParams = a
},
setEditorVal() {
let self = this
let modal = self.$modal.dialog({
className: 'scriptModal',
closable: false,
showMask: true,
maskClosable: true,
onClose: function() {
},
render (h) {
return h(mScriptBox, {
on: {
getSriptBoxValue (val) {
editor.setValue(val)
},
closeAble () {
// this.$modal.destroy()
modal.remove()
}
},
props: {
item: editor.getValue()
}
})
}
})
},
/**
* return resourceList
*
*/
_onResourcesData (a) {
this.resourceList = a
@ -109,7 +142,6 @@
if (!this.$refs.refLocalParams._verifProp()) {
return false
}
// storage
this.$emit('on-params', {
resourceList: this.resourceList,
@ -138,7 +170,6 @@
// Monitor keyboard
editor.on('keypress', this.keypress)
editor.setValue(this.rawScript)
return editor
@ -176,6 +207,27 @@
editor.off($('.code-shell-mirror'), 'keypress', this.keypress)
}
},
components: { mLocalParams, mListBox, mResources }
components: { mLocalParams, mListBox, mResources, mScriptBox }
}
</script>
<style lang="scss" rel="stylesheet/scss" scope>
.scriptModal {
.ans-modal-box-content-wrapper {
width: 90%;
.ans-modal-box-close {
right: -12px;
top: -16px;
color: #fff;
}
}
}
.ans-modal-box-close {
z-index: 100;
}
.ans-modal-box-max {
position: absolute;
right: -12px;
top: -16px;
}
</style>

View File

@ -19,6 +19,12 @@
<div class="title-box">
<span>{{$t('Please set the parameters before starting')}}</span>
</div>
<div class="clearfix list">
<div class="text">
{{$t('Process Name')}}
</div>
<div style="line-height: 32px;">{{workflowName}}</div>
</div>
<div class="clearfix list">
<div class="text">
{{$t('Failure Strategy')}}
@ -174,6 +180,7 @@
processDefinitionId: 0,
failureStrategy: 'CONTINUE',
warningTypeList: warningTypeList,
workflowName: '',
warningType: '',
notifyGroupList: [],
warningGroupId: '',
@ -276,6 +283,7 @@
this.warningGroupId = ''
})
})
this.workflowName = this.item.name
},
computed: {},
components: { mEmail, mPriority, mWorkerGroups }

View File

@ -313,7 +313,23 @@
this.crontab = this.item.crontab
}
if(this.type == 'timing') {
let date = new Date()
let year = date.getFullYear()
let month = date.getMonth() + 1
let day = date.getDate()
if (month < 10) {
month = "0" + month;
}
if (day < 10) {
day = "0" + day;
}
let startDate = year + "-" + month + "-" + day + ' ' + '00:00:00'
let endDate = (year+100) + "-" + month + "-" + day + ' ' + '00:00:00'
let times = []
times[0] = startDate
times[1] = endDate
this.crontab = '0 0 * * * ? *'
this.scheduleTime = times
}
this.receivers = _.cloneDeep(this.receiversD)
this.receiversCc = _.cloneDeep(this.receiversCcD)

View File

@ -27,7 +27,7 @@ let pie = {
avoidLabelOverlap: true, // Whether to prevent the label overlap policy
hoverAnimation: true, // Whether to enable hover to enlarge the animation on the sector.
radius: ['30%', '60%'],
center: ['50%', '50%'],
center: ['53%', '60%'],
label: {
align: 'left',
normal: {

View File

@ -29,7 +29,7 @@
<th>{{$t('Number')}}</th>
<th>{{$t('State')}}</th>
</tr>
<tr v-for="(item,$index) in processStateList">
<tr v-for="(item,$index) in processStateList" :key="$index">
<td><span>{{$index+1}}</span></td>
<td><span><a href="javascript:" @click="searchParams.projectId && _goProcess(item.key)" :class="searchParams.projectId ?'links':''">{{item.value}}</a></span></td>
<td><span class="ellipsis" style="width: 98%;" :title="item.key">{{item.key}}</span></td>

View File

@ -218,30 +218,6 @@ keytabPath="$installPath/conf/hdfs.headless.keytab"
# zk root directory
zkRoot="/dolphinscheduler"
# used to record the zk directory of the hanging machine
zkDeadServers="$zkRoot/dead-servers"
# masters directory
zkMasters="$zkRoot/masters"
# workers directory
zkWorkers="$zkRoot/workers"
# zk master distributed lock
mastersLock="$zkRoot/lock/masters"
# zk worker distributed lock
workersLock="$zkRoot/lock/workers"
# zk master fault-tolerant distributed lock
mastersFailover="$zkRoot/lock/failover/masters"
# zk worker fault-tolerant distributed lock
workersFailover="$zkRoot/lock/failover/workers"
# zk master start fault tolerant distributed lock
mastersStartupFailover="$zkRoot/lock/failover/startup-masters"
# zk session timeout
zkSessionTimeout="300"
@ -321,11 +297,10 @@ apiMaxHttpPostSize="5000000"
# 1,replace file
echo "1,replace file"
if [ $dbtype == "mysql" ];then
sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties
@ -335,10 +310,10 @@ if [ $dbtype == "mysql" ];then
fi
if [ $dbtype == "postgresql" ];then
sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application-dao.properties
sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties
sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties
sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties
@ -349,46 +324,34 @@ fi
sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties
sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties
sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties
sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties
sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties
sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties
sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties
sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties
sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties
sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties
sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties
sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties
sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties
sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties
sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties
sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties
sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common/common.properties
sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties
sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties
sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties
sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common/common.properties
sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common/common.properties
sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties
sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.dead.servers.*#zookeeper.dolphinscheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.masters.*#zookeeper.dolphinscheduler.masters=${zkMasters}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.workers.*#zookeeper.dolphinscheduler.workers=${zkWorkers}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.masters.*#zookeeper.dolphinscheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.workers.*#zookeeper.dolphinscheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.masters.*#zookeeper.dolphinscheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.workers.*#zookeeper.dolphinscheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.startup.masters.*#zookeeper.dolphinscheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties
sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties
sed -i ${txt} "s#server.port.*#server.port=${masterPort}#g" conf/application-master.properties
sed -i ${txt} "s#server.port.*#server.port=${workerPort}#g" conf/application-worker.properties
sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties
sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties
sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties
sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties
sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties
sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties
sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties
sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties
sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties
sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties
sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties
sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties
sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/common.properties
sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties
sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties
sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties

28
pom.xml
View File

@ -108,6 +108,8 @@
<maven-javadoc-plugin.version>2.10.3</maven-javadoc-plugin.version>
<maven-source-plugin.version>2.4</maven-source-plugin.version>
<maven-surefire-plugin.version>2.18.1</maven-surefire-plugin.version>
<maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version>
<rpm-maven-plugion.version>2.2.0</rpm-maven-plugion.version>
<jacoco.version>0.8.4</jacoco.version>
<jcip.version>1.0</jcip.version>
<maven.deploy.skip>false</maven.deploy.skip>
@ -524,6 +526,14 @@
<pluginManagement>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>${rpm-maven-plugion.version}</version>
<inherited>false</inherited>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
@ -568,6 +578,12 @@
<version>${maven-source-plugin.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
</plugin>
</plugins>
</pluginManagement>
@ -650,6 +666,15 @@
<include>**/api/utils/CheckUtilsTest.java</include>
<include>**/api/utils/FileUtilsTest.java</include>
<include>**/api/enums/*.java</include>
<include>**/api/service/AccessTokenServiceTest.java</include>
<include>**/api/service/QueueServiceTest.java</include>
<include>**/api/service/MonitorServiceTest.java</include>
<include>**/api/service/SessionServiceTest.java</include>
<include>**/api/service/UsersServiceTest.java</include>
<include>**/api/service/TenantServiceTest.java</include>
<include>**/api/service/WorkerGroupServiceTest.java</include>
<include>**/api/service/AlertGroupServiceTest.java</include>
<include>**/api/service/ProjectServiceTest.java</include>
<include>**/alert/utils/ExcelUtilsTest.java</include>
<include>**/alert/utils/FuncUtilsTest.java</include>
<include>**/alert/utils/JSONUtilsTest.java</include>
@ -657,6 +682,9 @@
<include>**/server/utils/SparkArgsUtilsTest.java</include>
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include>
<include>**/dao/mapper/AlertGroupMapperTest.java</include>
<include>**/dao/mapper/AlertMapperTest.java</include>
<include>**/dao/mapper/CommandMapperTest.java</include>
</includes>
<!-- <skip>true</skip> -->
</configuration>

Some files were not shown because too many files have changed in this diff Show More