test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-03 12:48:46 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

Fix homepage user wrong statistics result for new user (#7242)

Browse Source 
* Fix homepage user wrong statistics result for new user

For now, when we create a new user, it would see homepage
statistic info as admin user like, cause `projectCodeArray`
would not filter when we render SQL. So I add prefix conditions
to handle this situation.

Also, I do some minimal refactor here. including change function
name no quit fit

fix: #7182

* Fix UT
This commit is contained in:
Jiajie Zhong 2021-12-07 18:50:06 +08:00 committed by GitHub
parent 0b9792db76
commit 29414a080f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 112 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java
View File

@ -49,7 +49,9 @@ public interface DataAnalysisService {
Map<String, Object> countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate);
/**
* statistics the process definition quantities of certain person
* statistics the process definition quantities of a certain person
*
* We only need projects which users have permission to see to determine whether the definition belongs to the user or not.
*
* @param loginUser login user
* @param projectCode project code

56
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
View File

@ -44,6 +44,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
@ -90,30 +91,30 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
/**
* statistical task instance status data
*
* @param loginUser login user
* @param loginUser login user
* @param projectCode project code
* @param startDate start date
* @param endDate end date
* @param startDate start date
* @param endDate end date
* @return task state count data
*/
@Override
public Map<String, Object> countTaskStateByProject(User loginUser, long projectCode, String startDate, String endDate) {
return countStateByProject(
loginUser,
projectCode,
startDate,
endDate,
(start, end, projectCodes) -> this.taskInstanceMapper.countTaskInstanceStateByUser(start, end, projectCodes));
loginUser,
projectCode,
startDate,
endDate,
(start, end, projectCodes) -> this.taskInstanceMapper.countTaskInstanceStateByProjectCodes(start, end, projectCodes));
}
/**
* statistical process instance status data
*
* @param loginUser login user
* @param loginUser login user
* @param projectCode project code
* @param startDate start date
* @param endDate end date
* @param startDate start date
* @param endDate end date
* @return process instance state count data
*/
@Override
@ -123,7 +124,7 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
projectCode,
startDate,
endDate,
(start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByUser(start, end, projectCodes));
(start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByProjectCodes(start, end, projectCodes));
// process state count needs to remove state of forced success
if (result.containsKey(Constants.STATUS) && result.get(Constants.STATUS).equals(Status.SUCCESS)) {
((TaskCountDto)result.get(Constants.DATA_LIST)).removeStateFromCountList(ExecutionStatus.FORCED_SUCCESS);
@ -131,6 +132,14 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
return result;
}
/**
* Wrapper function of counting process instance state and task state
*
* @param loginUser login user
* @param projectCode project code
* @param startDate start date
* @param endDate end date
*/
private Map<String, Object> countStateByProject(User loginUser, long projectCode, String startDate, String endDate
, TriFunction<Date, Date, Long[], List<ExecuteStatusCount>> instanceStateCounter) {
Map<String, Object> result = new HashMap<>();
@ -154,10 +163,13 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
}
}
List<ExecuteStatusCount> processInstanceStateCounts = new ArrayList<>();
Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser)
: new Long[] { projectCode };
List<ExecuteStatusCount> processInstanceStateCounts =
instanceStateCounter.apply(start, end, projectCodeArray);
: new Long[] {projectCode};
if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) {
processInstanceStateCounts = instanceStateCounter.apply(start, end, projectCodeArray);
}
if (processInstanceStateCounts != null) {
TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts);
@ -169,9 +181,11 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
/**
* statistics the process definition quantities of certain person
* statistics the process definition quantities of a certain person
* <p>
* We only need projects which users have permission to see to determine whether the definition belongs to the user or not.
*
* @param loginUser login user
* @param loginUser login user
* @param projectCode project code
* @return definition count data
*/
@ -187,10 +201,12 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
}
}
List<DefinitionGroupByUser> defineGroupByUsers = new ArrayList<>();
Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser)
: new Long[] { projectCode };
List<DefinitionGroupByUser> defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser(
loginUser.getId(), projectCodeArray, isAdmin(loginUser));
: new Long[] {projectCode};
if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) {
defineGroupByUsers = processDefinitionMapper.countDefinitionByProjectCodes(projectCodeArray);
}
DefineUserDto dto = new DefineUserDto(defineGroupByUsers);
result.put(Constants.DATA_LIST, dto);

12
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java
View File

@ -130,7 +130,7 @@ public class DataAnalysisServiceTest {
Mockito.when(projectMapper.queryByCode(1L)).thenReturn(getProject("test"));
//SUCCESS
Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(DateUtils.getScheduleDate(startDate),
Mockito.when(taskInstanceMapper.countTaskInstanceStateByProjectCodes(DateUtils.getScheduleDate(startDate),
DateUtils.getScheduleDate(endDate), new Long[]{1L})).thenReturn(getTaskInstanceStateCounts());
Mockito.when(projectMapper.selectById(Mockito.any())).thenReturn(getProject("test"));
Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), (Map<String, Object>)Mockito.any())).thenReturn(true);
@ -187,7 +187,7 @@ public class DataAnalysisServiceTest {
// when general user doesn't have any task then return all count are 0
user.setUserType(UserType.GENERAL_USER);
Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any())).thenReturn(
Mockito.when(taskInstanceMapper.countTaskInstanceStateByProjectCodes(any(), any(), any())).thenReturn(
Collections.emptyList());
result = dataAnalysisServiceImpl.countTaskStateByProject(user, 1, null, null);
assertThat(result.get(Constants.DATA_LIST)).extracting("totalCount").isEqualTo(0);
@ -206,7 +206,7 @@ public class DataAnalysisServiceTest {
// when instanceStateCounter return null, then return nothing
user.setUserType(UserType.GENERAL_USER);
PowerMockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any())).thenReturn(null);
PowerMockito.when(taskInstanceMapper.countTaskInstanceStateByProjectCodes(any(), any(), any())).thenReturn(null);
result = dataAnalysisServiceImpl.countTaskStateByProject(user, 1, null, null);
Assert.assertNull(result.get(Constants.DATA_LIST));
}
@ -230,7 +230,7 @@ public class DataAnalysisServiceTest {
Mockito.when(projectService.checkProjectAndAuth(any(), any(), anyLong())).thenReturn(result);
//SUCCESS
Mockito.when(processInstanceMapper.countInstanceStateByUser(DateUtils.getScheduleDate(startDate),
Mockito.when(processInstanceMapper.countInstanceStateByProjectCodes(DateUtils.getScheduleDate(startDate),
DateUtils.getScheduleDate(endDate), new Long[]{1L})).thenReturn(getTaskInstanceStateCounts());
Mockito.when(projectService.hasProjectAndPerm(Mockito.any(), Mockito.any(), (Map<String, Object>)Mockito.any())).thenReturn(true);
@ -246,8 +246,8 @@ public class DataAnalysisServiceTest {
putMsg(result, Status.SUCCESS, null);
Mockito.when(projectService.checkProjectAndAuth(any(), any(), anyLong())).thenReturn(result);
Mockito.when(processDefinitionMapper.countDefinitionGroupByUser(Mockito.anyInt(), Mockito.any(Long[].class),
Mockito.anyBoolean())).thenReturn(new ArrayList<DefinitionGroupByUser>());
Mockito.when(processDefinitionMapper.countDefinitionByProjectCodes(
Mockito.any(Long[].class))).thenReturn(new ArrayList<DefinitionGroupByUser>());
result = dataAnalysisServiceImpl.countDefinitionByUser(user, 0);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}

25
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java
View File

@ -77,7 +77,7 @@ public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
* verify process definition by name
*
* @param projectCode projectCode
* @param name name
* @param name name
* @return process definition
*/
ProcessDefinition verifyByDefineName(@Param("projectCode") long projectCode,
@ -87,7 +87,7 @@ public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
* query process definition by name
*
* @param projectCode projectCode
* @param name name
* @param name name
* @return process definition
*/
ProcessDefinition queryByDefineName(@Param("projectCode") long projectCode,
@ -104,11 +104,11 @@ public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
/**
* process definition page
*
* @param page page
* @param searchVal searchVal
* @param userId userId
* @param page page
* @param searchVal searchVal
* @param userId userId
* @param projectCode projectCode
* @param isAdmin isAdmin
* @param isAdmin isAdmin
* @return process definition IPage
*/
IPage<ProcessDefinition> queryDefineListPaging(IPage<ProcessDefinition> page,
@ -142,17 +142,14 @@ public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
List<ProcessDefinition> queryDefinitionListByTenant(@Param("tenantId") int tenantId);
/**
* count process definition group by user
* Statistics process definition group by project codes list
* <p>
* We only need project codes to determine whether the definition belongs to the user or not.
*
* @param userId userId
* @param projectCodes projectCodes
* @param isAdmin isAdmin
* @return process definition list
* @return definition group by user
*/
List<DefinitionGroupByUser> countDefinitionGroupByUser(
@Param("userId") Integer userId,
@Param("projectCodes") Long[] projectCodes,
@Param("isAdmin") boolean isAdmin);
List<DefinitionGroupByUser> countDefinitionByProjectCodes(@Param("projectCodes") Long[] projectCodes);
/**
* list all resource ids

70
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java
View File

@ -46,7 +46,7 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
/**
* query process instance by host and stateArray
*
* @param host host
* @param host host
* @param stateArray stateArray
* @return process instance list
*/
@ -57,7 +57,7 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* query process instance by tenantId and stateArray
*
* @param tenantId tenantId
* @param states states array
* @param states states array
* @return process instance list
*/
List<ProcessInstance> queryByTenantIdAndStatus(@Param("tenantId") int tenantId,
@ -65,7 +65,7 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
/**
* @param workerGroupName workerGroupName
* @param states states array
* @param states states array
* @return process instance list
*/
List<ProcessInstance> queryByWorkerGroupNameAndStatus(@Param("workerGroupName") String workerGroupName,
@ -88,15 +88,15 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
/**
* process instance page
*
* @param page page
* @param projectCode projectCode
* @param page page
* @param projectCode projectCode
* @param processDefinitionCode processDefinitionCode
* @param searchVal searchVal
* @param executorId executorId
* @param statusArray statusArray
* @param host host
* @param startTime startTime
* @param endTime endTime
* @param searchVal searchVal
* @param executorId executorId
* @param statusArray statusArray
* @param host host
* @param startTime startTime
* @param endTime endTime
* @return process instance page
*/
IPage<ProcessInstance> queryProcessInstanceListPaging(Page<ProcessInstance> page,
@ -112,7 +112,7 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
/**
* set failover by host and state array
*
* @param host host
* @param host host
* @param stateArray stateArray
* @return set result
*/
@ -123,7 +123,7 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* update process instance by state
*
* @param originState originState
* @param destState destState
* @param destState destState
* @return update result
*/
int updateProcessInstanceByState(@Param("originState") ExecutionStatus originState,
@ -133,7 +133,7 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* update process instance by tenantId
*
* @param originTenantId originTenantId
* @param destTenantId destTenantId
* @param destTenantId destTenantId
* @return update result
*/
int updateProcessInstanceByTenantId(@Param("originTenantId") int originTenantId,
@ -143,30 +143,32 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* update process instance by worker groupId
*
* @param originWorkerGroupName originWorkerGroupName
* @param destWorkerGroupName destWorkerGroupName
* @param destWorkerGroupName destWorkerGroupName
* @return update result
*/
int updateProcessInstanceByWorkerGroupName(@Param("originWorkerGroupName") String originWorkerGroupName,
@Param("destWorkerGroupName") String destWorkerGroupName);
/**
* count process instance state by user
* Statistics process instance state by given project codes list
* <p>
* We only need project codes to determine whether the process instance belongs to the user or not.
*
* @param startTime startTime
* @param endTime endTime
* @param startTime startTime
* @param endTime endTime
* @param projectCodes projectCodes
* @return ExecuteStatusCount list
*/
List<ExecuteStatusCount> countInstanceStateByUser(
@Param("startTime") Date startTime,
@Param("endTime") Date endTime,
@Param("projectCodes") Long[] projectCodes);
List<ExecuteStatusCount> countInstanceStateByProjectCodes(
@Param("startTime") Date startTime,
@Param("endTime") Date endTime,
@Param("projectCodes") Long[] projectCodes);
/**
* query process instance by processDefinitionCode
*
* @param processDefinitionCode processDefinitionCode
* @param size size
* @param size size
* @return process instance list
*/
List<ProcessInstance> queryByProcessDefineCode(@Param("processDefinitionCode") Long processDefinitionCode,
@ -176,8 +178,8 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* query last scheduler process instance
*
* @param definitionCode definitionCode
* @param startTime startTime
* @param endTime endTime
* @param startTime startTime
* @param endTime endTime
* @return process instance
*/
ProcessInstance queryLastSchedulerProcess(@Param("processDefinitionCode") Long definitionCode,
@ -188,9 +190,9 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* query last running process instance
*
* @param definitionCode definitionCode
* @param startTime startTime
* @param endTime endTime
* @param stateArray stateArray
* @param startTime startTime
* @param endTime endTime
* @param stateArray stateArray
* @return process instance
*/
ProcessInstance queryLastRunningProcess(@Param("processDefinitionCode") Long definitionCode,
@ -202,8 +204,8 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* query last manual process instance
*
* @param definitionCode definitionCode
* @param startTime startTime
* @param endTime endTime
* @param startTime startTime
* @param endTime endTime
* @return process instance
*/
ProcessInstance queryLastManualProcess(@Param("processDefinitionCode") Long definitionCode,
@ -231,7 +233,7 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* query process instance by processDefinitionCode and stateArray
*
* @param processDefinitionCode processDefinitionCode
* @param states states array
* @param states states array
* @return process instance list
*/
@ -239,12 +241,12 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
@Param("states") int[] states);
List<ProcessInstance> queryByProcessDefineCodeAndStatusAndNextId(@Param("processDefinitionCode") Long processDefinitionCode,
@Param("states") int[] states, @Param("id") int id);
@Param("states") int[] states, @Param("id") int id);
int updateGlobalParamsById(@Param("globalParams") String globalParams,
@Param("id") int id);
boolean updateNextProcessIdById(@Param("thisInstanceId") int thisInstanceId, @Param("runningInstanceId")int runningInstanceId);
boolean updateNextProcessIdById(@Param("thisInstanceId") int thisInstanceId, @Param("runningInstanceId") int runningInstanceId);
ProcessInstance loadNextProcess4Serial(@Param("processDefinitionCode") Long processDefinitionCode,@Param("state") int state);
ProcessInstance loadNextProcess4Serial(@Param("processDefinitionCode") Long processDefinitionCode, @Param("state") int state);
}

18
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java
View File

@ -52,14 +52,24 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
@Param("name") String name);
TaskInstance queryByInstanceIdAndCode(@Param("processInstanceId") int processInstanceId,
@Param("taskCode") Long taskCode);
@Param("taskCode") Long taskCode);
Integer countTask(@Param("projectCodes") Long[] projectCodes,
@Param("taskIds") int[] taskIds);
List<ExecuteStatusCount> countTaskInstanceStateByUser(@Param("startTime") Date startTime,
@Param("endTime") Date endTime,
@Param("projectCodes") Long[] projectCodes);
/**
* Statistics task instance group by given project codes list
* <p>
* We only need project codes to determine whether the task instance belongs to the user or not.
*
* @param startTime Statistics start time
* @param endTime Statistics end time
* @param projectCodes Project codes list to filter
* @return List of ExecuteStatusCount
*/
List<ExecuteStatusCount> countTaskInstanceStateByProjectCodes(@Param("startTime") Date startTime,
@Param("endTime") Date endTime,
@Param("projectCodes") Long[] projectCodes);
IPage<TaskInstance> queryTaskInstanceListPaging(IPage<TaskInstance> page,
@Param("projectCode") Long projectCode,

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
View File

@ -109,7 +109,7 @@
#{i}
</foreach>
</select>
<select id="countDefinitionGroupByUser" resultType="org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser">
<select id="countDefinitionByProjectCodes" resultType="org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser">
SELECT td.user_id as user_id, tu.user_name as user_name, count(0) as count
FROM t_ds_process_definition td
JOIN t_ds_user tu on tu.id=td.user_id

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml
View File

@ -147,7 +147,7 @@
where worker_group = #{originWorkerGroupName}
</update>
<select id="countInstanceStateByUser" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
<select id="countInstanceStateByProjectCodes" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
select t.state, count(0) as count
from t_ds_process_instance t
join t_ds_process_definition d on d.code=t.process_definition_code

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
View File

@ -69,7 +69,7 @@
</foreach>
</if>
</select>
<select id="countTaskInstanceStateByUser" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
<select id="countTaskInstanceStateByProjectCodes" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
select state, count(0) as count
from t_ds_task_instance t
left join t_ds_task_definition_log d on d.code=t.task_code and d.version=t.task_definition_version

6
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java
View File

@ -318,11 +318,7 @@ public class ProcessDefinitionMapperTest extends BaseDaoTest {
Long[] projectCodes = new Long[1];
projectCodes[0] = processDefinition.getProjectCode();
List<DefinitionGroupByUser> processDefinitions = processDefinitionMapper.countDefinitionGroupByUser(
processDefinition.getUserId(),
projectCodes,
user.getUserType() == UserType.ADMIN_USER
);
List<DefinitionGroupByUser> processDefinitions = processDefinitionMapper.countDefinitionByProjectCodes(projectCodes);
Assert.assertNotEquals(processDefinitions.size(), 0);
}

2
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java
View File

@ -266,7 +266,7 @@ public class ProcessInstanceMapperTest extends BaseDaoTest {
Long[] projectCodes = new Long[]{processDefinition.getProjectCode()};
List<ExecuteStatusCount> executeStatusCounts = processInstanceMapper.countInstanceStateByUser(null, null, projectCodes);
List<ExecuteStatusCount> executeStatusCounts = processInstanceMapper.countInstanceStateByProjectCodes(null, null, projectCodes);
Assert.assertNotEquals(executeStatusCounts.size(), 0);

7
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java
View File

@ -32,12 +32,7 @@ import java.util.List;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@ -342,7 +337,7 @@ public class TaskInstanceMapperTest extends BaseDaoTest {
taskInstanceMapper.updateById(task);
List<ExecuteStatusCount> count = taskInstanceMapper.countTaskInstanceStateByUser(
List<ExecuteStatusCount> count = taskInstanceMapper.countTaskInstanceStateByProjectCodes(
null, null,
new Long[]{definition.getProjectCode()}
);