test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-12-02 03:57:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Refactor] [Core] Refactoring plug-in configuration extraction mode (all part)

Browse Source
This commit is contained in:
qianmoQ 2023-01-28 14:59:53 +08:00
parent 279fb2980a
commit f2c43a98ae
10 changed files with 293 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
common/src/main/java/io/edurt/datacap/common/OptionalUtils.java
View File

@ -1,27 +1,27 @@
package io.edurt.datacap.common;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Optional;
public class OptionalUtils
{
private OptionalUtils() {}
public static boolean isEmpty(Optional<String> optional)
public static boolean checkEmpty(Optional<String> optional)
{
boolean flag = false;
if (optional.isPresent()) {
if (ObjectUtils.isEmpty(optional.get()) && StringUtils.isEmpty(optional.get())) {
flag = true;
}
flag = true;
}
return flag;
}
public static boolean isEmpty(Optional<String> optional)
{
return !checkEmpty(optional);
}
public static boolean isNotEmpty(Optional<String> optional)
{
return !isEmpty(optional);
return checkEmpty(optional);
}
}

33
server/src/main/etc/conf/plugins/native/redis.json Normal file
View File

@ -0,0 +1,33 @@
{
"name": "Redis",
"supportTime": "2022-12-01",
"configures": [
{
"field": "name",
"type": "String",
"required": true,
"message": "name is a required field, please be sure to enter"
},
{
"field": "host",
"type": "String",
"required": true,
"value": "127.0.0.1",
"message": "host is a required field, please be sure to enter"
},
{
"field": "port",
"type": "Number",
"required": true,
"min": 1,
"max": 65535,
"value": 6379,
"message": "port is a required field, please be sure to enter"
},
{
"field": "password",
"type": "String",
"group": "authorization"
}
]
}

108
server/src/main/java/io/edurt/datacap/server/common/IConfigureCommon.java
View File

@ -1,18 +1,23 @@
package io.edurt.datacap.server.common;
import com.google.common.base.Preconditions;
import io.edurt.datacap.server.entity.SourceEntity;
import io.edurt.datacap.server.plugin.configure.IConfigure;
import io.edurt.datacap.server.plugin.configure.IConfigureField;
import io.edurt.datacap.server.plugin.configure.IConfigureFieldName;
import io.edurt.datacap.server.plugin.configure.IConfigureFieldType;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.model.Configure;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
public class IConfigureCommon
{
@ -64,6 +69,109 @@ public class IConfigureCommon
return configure;
}
public static IConfigure preparedConfigure(IConfigure configure, SourceEntity source)
{
configure.getConfigures().forEach(v -> {
switch (v.getField()) {
case name:
v.setValue(source.getName());
break;
case host:
v.setValue(source.getHost());
break;
case port:
v.setValue(source.getPort());
break;
case username:
v.setValue(source.getUsername());
break;
case password:
v.setValue(source.getPassword());
break;
case database:
v.setValue(source.getDatabase());
break;
case ssl:
v.setValue(source.getSsl());
break;
case catalog:
v.setValue(source.getCatalog());
break;
case configures:
List<Map<String, Object>> fields = new ArrayList<>();
if (ObjectUtils.isNotEmpty(source.getConfigures())) {
source.getConfigures().entrySet().forEach(entry -> {
Map<String, Object> map = new LinkedHashMap<>();
map.put(IConfigureFieldName.field.name(), entry.getKey());
map.put(IConfigureFieldName.value.name(), entry.getValue());
fields.add(map);
});
}
v.setValue(fields);
break;
}
});
return configure;
}
public static SourceEntity preparedSourceEntity(List<IConfigureField> configures)
{
SourceEntity configure = new SourceEntity();
configures.forEach(v -> {
switch (v.getField()) {
case name:
configure.setName(IConfigureCommon.getStringValue(configures, IConfigureFieldName.name));
case host:
configure.setHost(IConfigureCommon.getStringValue(configures, IConfigureFieldName.host));
break;
case port:
configure.setPort(IConfigureCommon.getIntegerValue(configures, IConfigureFieldName.port));
break;
case username:
configure.setUsername(IConfigureCommon.getStringValue(configures, IConfigureFieldName.username));
break;
case password:
configure.setPassword(IConfigureCommon.getStringValue(configures, IConfigureFieldName.password));
break;
case database:
String database = IConfigureCommon.getStringValue(configures, IConfigureFieldName.database);
configure.setDatabase(database);
break;
case catalog:
String catalog = IConfigureCommon.getStringValue(configures, IConfigureFieldName.catalog);
configure.setCatalog(catalog);
break;
case ssl:
configure.setSsl(IConfigureCommon.getBooleanValue(configures, IConfigureFieldName.ssl));
break;
case configures:
configure.setConfigure(JSON.toJSON(IConfigureCommon.getMapValue(configures, IConfigureFieldName.configures)));
break;
}
});
return configure;
}
public static List<IConfigureField> filterNotEmpty(List<IConfigureField> configures)
{
return configures.stream().filter(v -> !isEmpty(v)).collect(Collectors.toList());
}
public static boolean isEmpty(IConfigureField field)
{
boolean flag = true;
switch (field.getType()) {
case String:
if (ObjectUtils.isNotEmpty(field.getValue())) {
flag = StringUtils.isEmpty(String.valueOf(field.getValue()));
}
break;
default:
flag = false;
}
return flag;
}
public static IConfigureField getConfigure(List<IConfigureField> configures, IConfigureFieldName key)
{
Optional<IConfigureField> configureFieldOptional = configures.stream().filter(v -> v.getField().equals(key)).findFirst();

2
server/src/main/java/io/edurt/datacap/server/controller/user/SourceController.java
View File

@ -34,12 +34,14 @@ public class SourceController
this.sourceService = sourceService;
}
@Deprecated
@PostMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
public Response<SourceEntity> save(@RequestBody @Validated(ValidationGroup.Crud.Create.class) SourceEntity configure)
{
return this.sourceService.saveOrUpdate(configure);
}
@Deprecated
@PutMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
public Response<SourceEntity> update(@RequestBody @Validated(ValidationGroup.Crud.Update.class) SourceEntity configure)
{

22
server/src/main/java/io/edurt/datacap/server/controller/user/v2/SourceV2Controller.java
View File

@ -2,11 +2,15 @@ package io.edurt.datacap.server.controller.user.v2;
import io.edurt.datacap.server.body.SourceBody;
import io.edurt.datacap.server.common.Response;
import io.edurt.datacap.server.entity.SourceEntity;
import io.edurt.datacap.server.service.SourceService;
import io.edurt.datacap.server.validation.ValidationGroup;
import org.springframework.http.MediaType;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@ -27,4 +31,22 @@ public class SourceV2Controller
{
return this.sourceService.testConnectionV2(configure);
}
@PostMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
public Response<SourceEntity> save(@RequestBody @Validated(ValidationGroup.Crud.Create.class) SourceBody configure)
{
return this.sourceService.saveOrUpdateV2(configure);
}
@PutMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
public Response<SourceEntity> update(@RequestBody @Validated(ValidationGroup.Crud.Update.class) SourceBody configure)
{
return this.sourceService.saveOrUpdateV2(configure);
}
@GetMapping(value = "{id}")
public Response<SourceEntity> getInfo(@PathVariable(value = "id") Long id)
{
return this.sourceService.getByIdV2(id);
}
}

4
server/src/main/java/io/edurt/datacap/server/entity/SourceEntity.java
View File

@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonIncludeProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.server.common.JSON;
import io.edurt.datacap.server.plugin.configure.IConfigure;
import io.edurt.datacap.server.validation.ValidationGroup;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -104,6 +105,9 @@ public class SourceEntity
@Transient
private Map<String, Object> configures;
@Transient
private IConfigure schema;
@ManyToOne
@JoinColumn(name = "user_id")
@JsonIncludeProperties(value = {"id", "username"})

5
server/src/main/java/io/edurt/datacap/server/service/SourceService.java
View File

@ -12,6 +12,7 @@ import java.util.Map;
public interface SourceService
{
@Deprecated
Response<SourceEntity> saveOrUpdate(SourceEntity configure);
Response<PageEntity<SourceEntity>> getAll(int offset, int limit);
@ -30,4 +31,8 @@ public interface SourceService
Response<Object> shared(SharedSourceBody configure);
Response<Object> testConnectionV2(SourceBody configure);
Response<SourceEntity> saveOrUpdateV2(SourceBody configure);
Response<SourceEntity> getByIdV2(Long id);
}

88
server/src/main/java/io/edurt/datacap/server/service/impl/SourceServiceImpl.java
View File

@ -55,6 +55,7 @@ public class SourceServiceImpl
this.environment = environment;
}
@Deprecated
@Override
public Response<SourceEntity> saveOrUpdate(SourceEntity configure)
{
@ -117,21 +118,19 @@ public class SourceServiceImpl
public Response<Map<String, List<PluginEntity>>> getPlugins()
{
Map<String, List<PluginEntity>> pluginMap = new ConcurrentHashMap<>();
this.injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}))
.stream()
.forEach(plugin -> {
PluginEntity entity = new PluginEntity();
entity.setName(plugin.name());
entity.setDescription(plugin.description());
entity.setType(plugin.type().name());
entity.setConfigure(PluginCommon.loadConfigure(plugin.type().name(), plugin.name(), plugin.name(), environment));
List<PluginEntity> plugins = pluginMap.get(plugin.type().name());
if (ObjectUtils.isEmpty(plugins)) {
plugins = new ArrayList<>();
}
plugins.add(entity);
pluginMap.put(plugin.type().name(), plugins);
});
this.injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {})).stream().forEach(plugin -> {
PluginEntity entity = new PluginEntity();
entity.setName(plugin.name());
entity.setDescription(plugin.description());
entity.setType(plugin.type().name());
entity.setConfigure(PluginCommon.loadConfigure(plugin.type().name(), plugin.name(), plugin.name(), environment));
List<PluginEntity> plugins = pluginMap.get(plugin.type().name());
if (ObjectUtils.isEmpty(plugins)) {
plugins = new ArrayList<>();
}
plugins.add(entity);
pluginMap.put(plugin.type().name(), plugins);
});
return Response.success(pluginMap);
}
@ -175,15 +174,15 @@ public class SourceServiceImpl
}
// Filter required
List<IConfigureField> requiredMismatchConfigures = configure.getConfigure().getConfigures().stream().filter(v -> v.isRequired())
.filter(v -> ObjectUtils.isEmpty(v.getValue()))
.collect(Collectors.toList());
List<IConfigureField> requiredMismatchConfigures = configure.getConfigure().getConfigures().stream().filter(v -> v.isRequired()).filter(v -> ObjectUtils.isEmpty(v.getValue())).collect(Collectors.toList());
if (requiredMismatchConfigures.size() > 0) {
return Response.failure(ServiceState.PLUGIN_CONFIGURE_REQUIRED, IConfigureCommon.preparedMessage(requiredMismatchConfigures));
}
Plugin plugin = pluginOptional.get();
Configure _configure = IConfigureCommon.preparedConfigure(configure.getConfigure().getConfigures());
// The filter parameter value is null data
List<IConfigureField> applyConfigures = IConfigureCommon.filterNotEmpty(configure.getConfigure().getConfigures());
Configure _configure = IConfigureCommon.preparedConfigure(applyConfigures);
plugin.connect(_configure);
io.edurt.datacap.spi.model.Response response = plugin.execute(plugin.validator());
plugin.destroy();
@ -192,4 +191,55 @@ public class SourceServiceImpl
}
return Response.failure(ServiceState.PLUGIN_EXECUTE_FAILED, response.getMessage());
}
@Override
public Response<SourceEntity> saveOrUpdateV2(SourceBody configure)
{
Optional<Plugin> pluginOptional = PluginCommon.getPluginByNameAndType(this.injector, configure.getName(), configure.getType());
if (!pluginOptional.isPresent()) {
return Response.failure(ServiceState.PLUGIN_NOT_FOUND);
}
// Check configure
IConfigure iConfigure = PluginCommon.loadConfigure(configure.getType(), configure.getName(), configure.getName(), environment);
if (ObjectUtils.isEmpty(iConfigure) || iConfigure.getConfigures().size() != configure.getConfigure().getConfigures().size()) {
return Response.failure(ServiceState.PLUGIN_CONFIGURE_MISMATCH);
}
// Filter required
List<IConfigureField> requiredMismatchConfigures = configure.getConfigure().getConfigures().stream().filter(v -> v.isRequired()).filter(v -> ObjectUtils.isEmpty(v.getValue())).collect(Collectors.toList());
if (requiredMismatchConfigures.size() > 0) {
return Response.failure(ServiceState.PLUGIN_CONFIGURE_REQUIRED, IConfigureCommon.preparedMessage(requiredMismatchConfigures));
}
// The filter parameter value is null data
List<IConfigureField> applyConfigures = IConfigureCommon.filterNotEmpty(configure.getConfigure().getConfigures());
SourceEntity source = IConfigureCommon.preparedSourceEntity(applyConfigures);
source.setProtocol(configure.getType());
source.setType(configure.getName());
source.setUser(UserDetailsService.getUser());
if (ObjectUtils.isNotEmpty(configure.getId())) {
source.setId(configure.getId());
}
return Response.success(this.sourceRepository.save(source));
}
@Override
public Response<SourceEntity> getByIdV2(Long id)
{
Optional<SourceEntity> optionalSource = this.sourceRepository.findById(id);
if (!optionalSource.isPresent()) {
return Response.failure(ServiceState.SOURCE_NOT_FOUND);
}
SourceEntity entity = optionalSource.get();
SourceBody configure = new SourceBody();
configure.setId(id);
configure.setName(entity.getType());
configure.setType(entity.getProtocol());
// Load default configure
IConfigure iConfigure = PluginCommon.loadConfigure(configure.getType(), configure.getName(), configure.getName(), environment);
configure.setConfigure(IConfigureCommon.preparedConfigure(iConfigure, entity));
entity.setSchema(iConfigure);
return Response.success(entity);
}
}

26
web/console-fe/src/services/SourceV2Service.ts Normal file
View File

@ -0,0 +1,26 @@
import {ResponseModel} from "@/model/ResponseModel";
import {isEmpty} from "lodash";
import {HttpCommon} from "@/common/HttpCommon";
const baseUrl = "/api/v2/source";
class SourceV2Service
{
saveAndUpdate(configure, isUpdate: boolean): Promise<ResponseModel>
{
configure.protocol = isEmpty(configure.protocol) ? 'HTTP' : configure.protocol;
if (isUpdate) {
return new HttpCommon().put(baseUrl, JSON.stringify(configure));
}
else {
return new HttpCommon().post(baseUrl, JSON.stringify(configure));
}
}
getById(id: number): Promise<ResponseModel>
{
return new HttpCommon().get(baseUrl + "/" + id);
}
}
export default new SourceV2Service();

35
web/console-fe/src/views/pages/admin/source/SourceDetail.vue
View File

@ -32,7 +32,7 @@
</Row>
<Form :model="formState" :label-width="80">
<Tabs v-model="activeKey" :animated="false" @update:modelValue="handlerFilterConfigure($event)">
<TabPane :label="$t('common.' + type)" v-for="type in pluginTabs" :name="type" :disabled="!formState.type" icon="md-apps">
<TabPane :label="$t('common.' + type)" v-for="type in pluginTabs" :name="type" v-bind:key="type" :disabled="!formState.type" icon="md-apps">
<div v-if="type === 'source'">
<RadioGroup v-if="plugins" v-model="formState.type" type="button" @on-change="handlerChangePlugin($event)">
<div v-for="key in Object.keys(plugins)" v-bind:key="key">
@ -52,7 +52,7 @@
<Row>
<Col :span="5"/>
<Col :span="14">
<FormItem v-for="configure in pluginTabConfigure" :required="configure.required" :prop="configure.field">
<FormItem v-for="configure in pluginTabConfigure" :required="configure.required" v-bind:key="configure.field" :prop="configure.field">
<template #label>
<span v-if="configure.field !== 'configures'">{{ $t('common.' + configure.field) }}</span>
</template>
@ -119,8 +119,8 @@ import {SourceService} from "@/services/SourceService";
import {emptySource} from "@/views/pages/admin/source/SourceGenerate";
import {defineComponent, reactive, ref} from "vue";
import {Configure} from "@/model/Configure";
import {Arrays} from "@/common/Arrays";
import {clone} from 'lodash'
import SourceV2Service from "@/services/SourceV2Service";
interface TestInfo
{
@ -183,20 +183,16 @@ export default defineComponent({
handlerInitialize()
{
if (this.id > 0) {
new SourceService().getById(this.id)
SourceV2Service.getById(this.id)
.then(response => {
if (response.status) {
this.formState = reactive(response.data);
this.formState.type = this.formState.type + ' ' + this.formState.protocol;
if (response.data.configures) {
Object.keys(response.data.configures).forEach((value) => {
const configure: Configure = {
field: value,
value: response.data.configures[value]
};
this.configure.push(configure);
});
}
this.applyPlugin = response.data['schema'];
this.pluginConfigure = response.data['schema']['configures'];
// Clear
this.pluginTabs = ['source'];
this.pluginTabs = [...this.pluginTabs, ...Array.from(new Set(this.pluginConfigure.map(v => v.group)))];
}
});
}
@ -213,13 +209,14 @@ export default defineComponent({
},
handlerSave()
{
this.formState.configures = Arrays.arrayToObject(this.configure);
const applyConfigure = clone(this.formState);
const temp = clone(this.formState.type).split(' ');
applyConfigure.type = temp[0];
applyConfigure.protocol = temp[1];
new SourceService()
.saveAndUpdate(applyConfigure, this.isUpdate)
const configure = {
id: this.id,
type: temp[1],
name: temp[0],
configure: this.applyPlugin
};
SourceV2Service.saveAndUpdate(configure, this.isUpdate)
.then((response) => {
if (response.status) {
this.$Message.success("Create successful");