From 8840dbd8b24860e78078bf23fa8d6334948f89a7 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Wed, 6 Sep 2023 14:28:36 +0800 Subject: [PATCH] [Pipeline] Add logging interface and optimize UI --- .../controller/user/PipelineController.java | 9 +++++ .../server/runner/PipelineResetRunner.java | 7 +++- .../repository/PipelineRepository.java | 2 +- .../service/service/PipelineService.java | 4 +++ .../service/impl/PipelineServiceImpl.java | 36 +++++++++++++++++++ .../datacap-web/src/i18n/langs/en/pipeline.ts | 3 +- .../src/i18n/langs/zhCn/pipeline.ts | 3 +- .../views/admin/pipeline/AdminPipeline.vue | 10 ++++++ .../views/admin/pipeline/PipelineGenerate.ts | 2 ++ 9 files changed, 72 insertions(+), 4 deletions(-) diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/user/PipelineController.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/user/PipelineController.java index fe03f71d..e98750cc 100644 --- a/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/user/PipelineController.java +++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/user/PipelineController.java @@ -6,6 +6,7 @@ import io.edurt.datacap.service.body.PipelineBody; import io.edurt.datacap.service.entity.PipelineEntity; import io.edurt.datacap.service.repository.PipelineRepository; import io.edurt.datacap.service.service.PipelineService; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; @@ -13,6 +14,8 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + @RestController() @RequestMapping(value = "/api/v1/pipeline") public class PipelineController @@ -39,4 +42,10 @@ public class PipelineController { return service.stop(id); } + + @GetMapping(value = "/log/{id}") + public CommonResponse> log(@PathVariable Long id) + { + return service.log(id); + } } diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/runner/PipelineResetRunner.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/runner/PipelineResetRunner.java index 9e1e6482..d7a96926 100644 --- a/core/datacap-server/src/main/java/io/edurt/datacap/server/runner/PipelineResetRunner.java +++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/runner/PipelineResetRunner.java @@ -1,5 +1,6 @@ package io.edurt.datacap.server.runner; +import com.clearspring.analytics.util.Lists; import io.edurt.datacap.service.entity.PipelineEntity; import io.edurt.datacap.service.repository.PipelineRepository; import io.edurt.datacap.service.service.PipelineService; @@ -45,7 +46,11 @@ public class PipelineResetRunner resetState = PipelineState.STOPPED; } - List pipelines = repository.findAllByStateIs(PipelineState.RUNNING); + List states = Lists.newArrayList(); + states.add(PipelineState.RUNNING); + states.add(PipelineState.CREATED); + states.add(PipelineState.QUEUE); + List pipelines = repository.findAllByStateIn(states); for (PipelineEntity pipeline : pipelines) { log.info("Reset pipeline [ {} ] user [ {} ]", pipeline.getName(), pipeline.getUser().getUsername()); pipeline.setState(resetState); diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/PipelineRepository.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/PipelineRepository.java index 66088041..1d118a6b 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/PipelineRepository.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/PipelineRepository.java @@ -14,5 +14,5 @@ public interface PipelineRepository { Page findAllByUser(UserEntity user, Pageable pageable); - List findAllByStateIs(PipelineState state); + List findAllByStateIn(List state); } diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/PipelineService.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/PipelineService.java index d116e0d5..6f077a7f 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/PipelineService.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/PipelineService.java @@ -3,10 +3,14 @@ package io.edurt.datacap.service.service; import io.edurt.datacap.common.response.CommonResponse; import io.edurt.datacap.service.body.PipelineBody; +import java.util.List; + public interface PipelineService extends BaseService { CommonResponse submit(PipelineBody configure); CommonResponse stop(Long id); + + CommonResponse> log(Long id); } diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/PipelineServiceImpl.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/PipelineServiceImpl.java index 30a083a1..f5cf987b 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/PipelineServiceImpl.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/PipelineServiceImpl.java @@ -1,5 +1,6 @@ package io.edurt.datacap.service.service.impl; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.inject.Injector; import com.google.inject.Key; @@ -27,6 +28,7 @@ import io.edurt.datacap.spi.executor.PipelineResponse; import io.edurt.datacap.spi.executor.PipelineState; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; @@ -34,6 +36,8 @@ import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.sql.Timestamp; import java.util.List; import java.util.Locale; @@ -260,6 +264,7 @@ public class PipelineServiceImpl service.shutdownNow(); } entity.setState(PipelineState.STOPPED); + entity.setMessage(null); this.repository.save(entity); // Consume queue data for execution @@ -273,6 +278,37 @@ public class PipelineServiceImpl return CommonResponse.success(true); } + /** + * Retrieves the log for a given pipeline ID. + * + * @param id the ID of the pipeline + * @return a response containing the log lines as a list of strings + */ + @Override + public CommonResponse> log(Long id) + { + Optional pipelineOptional = this.repository.findById(id); + if (!pipelineOptional.isPresent()) { + return CommonResponse.failure(String.format("Pipeline [ %s ] not found", id)); + } + + PipelineEntity entity = pipelineOptional.get(); + if (entity.getState().equals(PipelineState.QUEUE) + || entity.getState().equals(PipelineState.CREATED)) { + return CommonResponse.failure(String.format("Pipeline [ %s ] is not running", entity.getName())); + } + + List lines = Lists.newArrayList(); + try (FileInputStream stream = new FileInputStream(new File(String.format("%s/%s.log", entity.getWork(), entity.getName())))) { + IOUtils.readLines(stream, "UTF-8") + .forEach(lines::add); + } + catch (IOException e) { + log.error("Failed to read pipeline [ {} ] log ", entity.getName(), e); + } + return CommonResponse.success(lines); + } + private Properties merge(SourceEntity entity, List fields, Properties configure) { Properties properties = new Properties(); diff --git a/core/datacap-web/src/i18n/langs/en/pipeline.ts b/core/datacap-web/src/i18n/langs/en/pipeline.ts index 4b02fc4d..bfb009c4 100644 --- a/core/datacap-web/src/i18n/langs/en/pipeline.ts +++ b/core/datacap-web/src/i18n/langs/en/pipeline.ts @@ -5,7 +5,8 @@ export default { success: 'Success', failure: 'Failure', stop: 'Stopped', - timeout: 'Timeout' + timeout: 'Timeout', + queue: 'Queue' }, delete: { deleteTip1: 'This action cannot be undone.', diff --git a/core/datacap-web/src/i18n/langs/zhCn/pipeline.ts b/core/datacap-web/src/i18n/langs/zhCn/pipeline.ts index 89247e0e..61daf191 100644 --- a/core/datacap-web/src/i18n/langs/zhCn/pipeline.ts +++ b/core/datacap-web/src/i18n/langs/zhCn/pipeline.ts @@ -5,7 +5,8 @@ export default { success: '运行成功', failure: '运行失败', stop: '已停止', - timeout: '运行超时' + timeout: '运行超时', + queue: '排队中' }, delete: { deleteTip1: '此操作无法撤消.', diff --git a/core/datacap-web/src/views/admin/pipeline/AdminPipeline.vue b/core/datacap-web/src/views/admin/pipeline/AdminPipeline.vue index 3311a4ab..efc8f84b 100644 --- a/core/datacap-web/src/views/admin/pipeline/AdminPipeline.vue +++ b/core/datacap-web/src/views/admin/pipeline/AdminPipeline.vue @@ -55,6 +55,16 @@ @click="handlerVisibleMarkdownPreview(row.message, true)"> + + +