From a7de51bd0498f7866887db6a2ac08aec934fc082 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Tue, 8 Aug 2023 15:17:44 +0800 Subject: [PATCH 1/2] [Core] Add a data source scan task --- .../configure/ScheduleRunnerConfigure.java | 60 ++++++++++++++ .../ScheduleTaskRunnerConfigure.java | 48 ----------- .../controller/admin/ScheduleController.java | 32 ++++++++ .../admin/ScheduleTaskController.java | 32 -------- .../source/CheckScheduledRunnable.java | 79 +++++++++++++++++++ .../src/main/schema/1.13.0/schema.sql | 11 +++ ...edTaskEntity.java => ScheduledEntity.java} | 22 +++++- .../datacap/service/enums/ScheduledType.java | 7 ++ .../repository/ScheduledRepository.java | 12 +++ .../repository/ScheduledTaskRepository.java | 12 --- .../service/service/ScheduledService.java | 8 ++ .../service/service/ScheduledTaskService.java | 8 -- .../service/impl/ScheduledServiceImpl.java | 10 +++ .../impl/ScheduledTaskServiceImpl.java | 10 --- .../src/views/admin/source/AdminSource.vue | 18 ++++- 15 files changed, 252 insertions(+), 117 deletions(-) create mode 100644 core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleRunnerConfigure.java delete mode 100644 core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleTaskRunnerConfigure.java create mode 100644 core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleController.java delete mode 100644 core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleTaskController.java create mode 100644 core/datacap-server/src/main/java/io/edurt/datacap/server/scheduled/source/CheckScheduledRunnable.java rename core/datacap-service/src/main/java/io/edurt/datacap/service/entity/{ScheduledTaskEntity.java => ScheduledEntity.java} (64%) create mode 100644 core/datacap-service/src/main/java/io/edurt/datacap/service/enums/ScheduledType.java create mode 100644 core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledRepository.java delete mode 100644 core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledTaskRepository.java create mode 100644 core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledService.java delete mode 100644 core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledTaskService.java create mode 100644 core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledServiceImpl.java delete mode 100644 core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledTaskServiceImpl.java diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleRunnerConfigure.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleRunnerConfigure.java new file mode 100644 index 00000000..8e343eed --- /dev/null +++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleRunnerConfigure.java @@ -0,0 +1,60 @@ +package io.edurt.datacap.server.configure; + +import com.google.inject.Injector; +import io.edurt.datacap.schedule.ScheduledCronRegistrar; +import io.edurt.datacap.server.scheduled.SourceScheduledRunnable; +import io.edurt.datacap.server.scheduled.source.CheckScheduledRunnable; +import io.edurt.datacap.service.repository.ScheduledRepository; +import io.edurt.datacap.service.repository.SourceRepository; +import io.edurt.datacap.service.repository.TemplateSqlRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.CommandLineRunner; +import org.springframework.core.env.Environment; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class ScheduleRunnerConfigure + implements CommandLineRunner +{ + private final Injector injector; + private final ScheduledRepository scheduledRepository; + private final SourceRepository sourceRepository; + private final TemplateSqlRepository templateSqlRepository; + private final ScheduledCronRegistrar scheduledCronRegistrar; + private final RedisTemplate redisTemplate; + private final Environment environment; + + public ScheduleRunnerConfigure(Injector injector, ScheduledRepository scheduledRepository, SourceRepository sourceRepository, TemplateSqlRepository templateSqlRepository, ScheduledCronRegistrar scheduledCronRegistrar, RedisTemplate redisTemplate, Environment environment) + { + this.injector = injector; + this.scheduledRepository = scheduledRepository; + this.sourceRepository = sourceRepository; + this.templateSqlRepository = templateSqlRepository; + this.scheduledCronRegistrar = scheduledCronRegistrar; + this.redisTemplate = redisTemplate; + this.environment = environment; + } + + @Override + public void run(String... args) + { + this.scheduledRepository.findAllByActiveIsTrueAndIsSystemIsTrue() + .forEach(task -> { + log.info("Add new task " + task.getName() + " to scheduler"); + switch (task.getType()) { + case SOURCE_SYNCHRONIZE: + SourceScheduledRunnable scheduled = new SourceScheduledRunnable(task.getName(), this.injector, this.sourceRepository, templateSqlRepository, redisTemplate, environment); + this.scheduledCronRegistrar.addCronTask(scheduled, task.getExpression()); + break; + case SOURCE_CHECK: + CheckScheduledRunnable runnable = new CheckScheduledRunnable(task.getName(), this.injector, this.sourceRepository); + this.scheduledCronRegistrar.addCronTask(runnable, task.getExpression()); + break; + default: + log.warn("Unsupported task type " + task.getType()); + } + }); + } +} diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleTaskRunnerConfigure.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleTaskRunnerConfigure.java deleted file mode 100644 index 7926628e..00000000 --- a/core/datacap-server/src/main/java/io/edurt/datacap/server/configure/ScheduleTaskRunnerConfigure.java +++ /dev/null @@ -1,48 +0,0 @@ -package io.edurt.datacap.server.configure; - -import com.google.inject.Injector; -import io.edurt.datacap.schedule.ScheduledCronRegistrar; -import io.edurt.datacap.server.scheduled.SourceScheduledRunnable; -import io.edurt.datacap.service.repository.ScheduledTaskRepository; -import io.edurt.datacap.service.repository.SourceRepository; -import io.edurt.datacap.service.repository.TemplateSqlRepository; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.CommandLineRunner; -import org.springframework.core.env.Environment; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class ScheduleTaskRunnerConfigure - implements CommandLineRunner -{ - private final Injector injector; - private final ScheduledTaskRepository scheduledTaskRepository; - private final SourceRepository sourceRepository; - private final TemplateSqlRepository templateSqlRepository; - private final ScheduledCronRegistrar scheduledCronRegistrar; - private final RedisTemplate redisTemplate; - private final Environment environment; - - public ScheduleTaskRunnerConfigure(Injector injector, ScheduledTaskRepository scheduledTaskRepository, SourceRepository sourceRepository, TemplateSqlRepository templateSqlRepository, ScheduledCronRegistrar scheduledCronRegistrar, RedisTemplate redisTemplate, Environment environment) - { - this.injector = injector; - this.scheduledTaskRepository = scheduledTaskRepository; - this.sourceRepository = sourceRepository; - this.templateSqlRepository = templateSqlRepository; - this.scheduledCronRegistrar = scheduledCronRegistrar; - this.redisTemplate = redisTemplate; - this.environment = environment; - } - - @Override - public void run(String... args) - { - this.scheduledTaskRepository.findAllByActiveIsTrueAndIsSystemIsTrue().forEach(task -> { - log.info("Add new task " + task.getName() + " to scheduler"); - SourceScheduledRunnable scheduled = new SourceScheduledRunnable(task.getName(), this.injector, this.sourceRepository, templateSqlRepository, redisTemplate, environment); - this.scheduledCronRegistrar.addCronTask(scheduled, task.getExpression()); - }); - } -} diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleController.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleController.java new file mode 100644 index 00000000..57d5ae62 --- /dev/null +++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleController.java @@ -0,0 +1,32 @@ +package io.edurt.datacap.server.controller.admin; + +import io.edurt.datacap.common.response.CommonResponse; +import io.edurt.datacap.service.body.FilterBody; +import io.edurt.datacap.service.entity.PageEntity; +import io.edurt.datacap.service.entity.ScheduledEntity; +import io.edurt.datacap.service.repository.ScheduledRepository; +import io.edurt.datacap.service.service.ScheduledService; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping(value = "/api/v1/admin/schedule") +public class ScheduleController +{ + private final ScheduledRepository scheduledRepository; + private final ScheduledService scheduledService; + + public ScheduleController(ScheduledRepository scheduledRepository, ScheduledService scheduledService) + { + this.scheduledRepository = scheduledRepository; + this.scheduledService = scheduledService; + } + + @PostMapping(value = "list") + public CommonResponse> getAllByFilter(@RequestBody FilterBody filter) + { + return scheduledService.getAll(scheduledRepository, filter); + } +} diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleTaskController.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleTaskController.java deleted file mode 100644 index 65b757ed..00000000 --- a/core/datacap-server/src/main/java/io/edurt/datacap/server/controller/admin/ScheduleTaskController.java +++ /dev/null @@ -1,32 +0,0 @@ -package io.edurt.datacap.server.controller.admin; - -import io.edurt.datacap.common.response.CommonResponse; -import io.edurt.datacap.service.body.FilterBody; -import io.edurt.datacap.service.entity.PageEntity; -import io.edurt.datacap.service.entity.ScheduledTaskEntity; -import io.edurt.datacap.service.repository.ScheduledTaskRepository; -import io.edurt.datacap.service.service.ScheduledTaskService; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@RequestMapping(value = "/api/v1/admin/schedule") -public class ScheduleTaskController -{ - private final ScheduledTaskRepository scheduledTaskRepository; - private final ScheduledTaskService scheduledTaskService; - - public ScheduleTaskController(ScheduledTaskRepository scheduledTaskRepository, ScheduledTaskService scheduledTaskService) - { - this.scheduledTaskRepository = scheduledTaskRepository; - this.scheduledTaskService = scheduledTaskService; - } - - @PostMapping(value = "list") - public CommonResponse> getAllByFilter(@RequestBody FilterBody filter) - { - return scheduledTaskService.getAll(scheduledTaskRepository, filter); - } -} diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/scheduled/source/CheckScheduledRunnable.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/scheduled/source/CheckScheduledRunnable.java new file mode 100644 index 00000000..95ff8091 --- /dev/null +++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/scheduled/source/CheckScheduledRunnable.java @@ -0,0 +1,79 @@ +package io.edurt.datacap.server.scheduled.source; + +import com.google.inject.Injector; +import io.edurt.datacap.schedule.ScheduledRunnable; +import io.edurt.datacap.service.common.PluginUtils; +import io.edurt.datacap.service.entity.SourceEntity; +import io.edurt.datacap.service.repository.SourceRepository; +import io.edurt.datacap.spi.Plugin; +import io.edurt.datacap.spi.model.Configure; +import io.edurt.datacap.spi.model.Response; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Optional; + +@Slf4j +public class CheckScheduledRunnable + extends ScheduledRunnable +{ + private final Injector injector; + private final SourceRepository sourceRepository; + + public CheckScheduledRunnable(String name, Injector injector, SourceRepository sourceRepository) + { + super(name); + this.injector = injector; + this.sourceRepository = sourceRepository; + } + + @Override + public void run() + { + log.info("==================== {} started =================", this.getName()); + this.sourceRepository.findAll() + .forEach(entity -> { + log.info("Before check source {}", entity.getName()); + Optional pluginOptional = PluginUtils.getPluginByNameAndType(this.injector, entity.getType(), entity.getProtocol()); + if (!pluginOptional.isPresent()) { + log.warn("Check scheduled task <{}> source {} protocol {} is not available", this.getName(), entity.getType(), entity.getProtocol()); + } + else { + Plugin plugin = pluginOptional.get(); + plugin.connect(getConfigure(entity)); + Response response = plugin.execute(plugin.validator()); + if (response.getIsSuccessful()) { + entity.setAvailable(true); + if (response.getColumns().get(0) instanceof ArrayList) { + ArrayList versions = (ArrayList) response.getColumns().get(0); + entity.setVersion(versions.get(0).toString()); + } + else { + entity.setVersion(response.getColumns().get(0).toString()); + } + } + else { + entity.setAvailable(false); + entity.setVersion(null); + entity.setMessage(response.getMessage()); + } + this.sourceRepository.save(entity); + } + }); + } + + private Configure getConfigure(SourceEntity entity) + { + Configure configure = new Configure(); + configure.setHost(entity.getHost()); + configure.setPort(entity.getPort()); + configure.setUsername(Optional.ofNullable(entity.getUsername())); + configure.setPassword(Optional.ofNullable(entity.getPassword())); + Optional database = StringUtils.isNotEmpty(entity.getDatabase()) ? Optional.ofNullable(entity.getDatabase()) : Optional.empty(); + configure.setDatabase(database); + configure.setSsl(Optional.ofNullable(entity.getSsl())); + configure.setEnv(Optional.ofNullable(entity.getConfigures())); + return configure; + } +} diff --git a/core/datacap-server/src/main/schema/1.13.0/schema.sql b/core/datacap-server/src/main/schema/1.13.0/schema.sql index d0005a52..99bafb8b 100644 --- a/core/datacap-server/src/main/schema/1.13.0/schema.sql +++ b/core/datacap-server/src/main/schema/1.13.0/schema.sql @@ -5,3 +5,14 @@ ALTER TABLE `source` ADD COLUMN `available` BOOLEAN DEFAULT TRUE, ADD COLUMN `message` TEXT DEFAULT NULL, ADD COLUMN `update_time` DATETIME DEFAULT NULL; + +RENAME TABLE `scheduled_task` TO `datacap_scheduled`; + +ALTER TABLE `datacap_scheduled` + ADD COLUMN `type` VARCHAR(255) DEFAULT NULL; + +UPDATE `datacap_scheduled` +SET `type` = 'SOURCE_SYNCHRONIZE'; + +INSERT INTO datacap.datacap_scheduled (name, description, expression, active, is_system, type) +VALUES ('Check source available', 'Check the availability of the data source every 1 hour', '0 0 * * * ?', 1, 1, 'SOURCE_CHECK'); diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ScheduledTaskEntity.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ScheduledEntity.java similarity index 64% rename from core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ScheduledTaskEntity.java rename to core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ScheduledEntity.java index 5049a4ad..63795025 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ScheduledTaskEntity.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ScheduledEntity.java @@ -2,14 +2,21 @@ package io.edurt.datacap.service.entity; import com.fasterxml.jackson.annotation.JsonFormat; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.edurt.datacap.service.enums.ScheduledType; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EntityListeners; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -23,9 +30,10 @@ import java.sql.Timestamp; @NoArgsConstructor @AllArgsConstructor @Entity -@Table(name = "scheduled_task") +@Table(name = "datacap_scheduled") +@EntityListeners(AuditingEntityListener.class) @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) -public class ScheduledTaskEntity +public class ScheduledEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -46,11 +54,17 @@ public class ScheduledTaskEntity @Column(name = "is_system") private boolean isSystem; - @Column(name = "create_time", columnDefinition = "datetime(5) default CURRENT_TIMESTAMP()") + @Column(name = "type") + @Enumerated(EnumType.STRING) + private ScheduledType type; + + @Column(name = "create_time") @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @CreatedDate private Timestamp createTime; - @Column(name = "update_time", columnDefinition = "datetime(5) default CURRENT_TIMESTAMP()") + @Column(name = "update_time") @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @LastModifiedDate private Timestamp updateTime; } diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/enums/ScheduledType.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/enums/ScheduledType.java new file mode 100644 index 00000000..12afc317 --- /dev/null +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/enums/ScheduledType.java @@ -0,0 +1,7 @@ +package io.edurt.datacap.service.enums; + +public enum ScheduledType +{ + SOURCE_CHECK, // Check that the data source is available + SOURCE_SYNCHRONIZE // Synchronize table structure +} diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledRepository.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledRepository.java new file mode 100644 index 00000000..a75e9d22 --- /dev/null +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledRepository.java @@ -0,0 +1,12 @@ +package io.edurt.datacap.service.repository; + +import io.edurt.datacap.service.entity.ScheduledEntity; +import org.springframework.data.repository.PagingAndSortingRepository; + +import java.util.List; + +public interface ScheduledRepository + extends PagingAndSortingRepository +{ + List findAllByActiveIsTrueAndIsSystemIsTrue(); +} diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledTaskRepository.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledTaskRepository.java deleted file mode 100644 index 526d5753..00000000 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/repository/ScheduledTaskRepository.java +++ /dev/null @@ -1,12 +0,0 @@ -package io.edurt.datacap.service.repository; - -import io.edurt.datacap.service.entity.ScheduledTaskEntity; -import org.springframework.data.repository.PagingAndSortingRepository; - -import java.util.List; - -public interface ScheduledTaskRepository - extends PagingAndSortingRepository -{ - List findAllByActiveIsTrueAndIsSystemIsTrue(); -} diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledService.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledService.java new file mode 100644 index 00000000..7eeaf06e --- /dev/null +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledService.java @@ -0,0 +1,8 @@ +package io.edurt.datacap.service.service; + +import io.edurt.datacap.service.entity.ScheduledEntity; + +public interface ScheduledService + extends BaseService +{ +} diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledTaskService.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledTaskService.java deleted file mode 100644 index 19762ca2..00000000 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/ScheduledTaskService.java +++ /dev/null @@ -1,8 +0,0 @@ -package io.edurt.datacap.service.service; - -import io.edurt.datacap.service.entity.ScheduledTaskEntity; - -public interface ScheduledTaskService - extends BaseService -{ -} diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledServiceImpl.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledServiceImpl.java new file mode 100644 index 00000000..3f47cf38 --- /dev/null +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledServiceImpl.java @@ -0,0 +1,10 @@ +package io.edurt.datacap.service.service.impl; + +import io.edurt.datacap.service.service.ScheduledService; +import org.springframework.stereotype.Service; + +@Service +public class ScheduledServiceImpl + implements ScheduledService +{ +} diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledTaskServiceImpl.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledTaskServiceImpl.java deleted file mode 100644 index 5c5a0090..00000000 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ScheduledTaskServiceImpl.java +++ /dev/null @@ -1,10 +0,0 @@ -package io.edurt.datacap.service.service.impl; - -import io.edurt.datacap.service.service.ScheduledTaskService; -import org.springframework.stereotype.Service; - -@Service -public class ScheduledTaskServiceImpl - implements ScheduledTaskService -{ -} diff --git a/core/datacap-web/src/views/admin/source/AdminSource.vue b/core/datacap-web/src/views/admin/source/AdminSource.vue index e12cab74..0b5ffc30 100644 --- a/core/datacap-web/src/views/admin/source/AdminSource.vue +++ b/core/datacap-web/src/views/admin/source/AdminSource.vue @@ -13,7 +13,7 @@