[Refactor] [Core] Refactoring plug-in configuration extraction mode (all part)

common/src/main/java/io/edurt/datacap/common/OptionalUtils.java

@@ -1,27 +1,27 @@
 package io.edurt.datacap.common;
 
-import org.apache.commons.lang3.ObjectUtils;
-import org.apache.commons.lang3.StringUtils;
-
 import java.util.Optional;
 
 public class OptionalUtils
 {
     private OptionalUtils() {}
 
-    public static boolean isEmpty(Optional<String> optional)
+    public static boolean checkEmpty(Optional<String> optional)
     {
         boolean flag = false;
         if (optional.isPresent()) {
-            if (ObjectUtils.isEmpty(optional.get()) && StringUtils.isEmpty(optional.get())) {
+            flag = true;
-                flag = true;
-            }
         }
         return flag;
     }
 
+    public static boolean isEmpty(Optional<String> optional)
+    {
+        return !checkEmpty(optional);
+    }
+
     public static boolean isNotEmpty(Optional<String> optional)
     {
-        return !isEmpty(optional);
+        return checkEmpty(optional);
     }
 }

server/src/main/etc/conf/plugins/native/redis.json

@@ -0,0 +1,33 @@
+{
+  "name": "Redis",
+  "supportTime": "2022-12-01",
+  "configures": [
+    {
+      "field": "name",
+      "type": "String",
+      "required": true,
+      "message": "name is a required field, please be sure to enter"
+    },
+    {
+      "field": "host",
+      "type": "String",
+      "required": true,
+      "value": "127.0.0.1",
+      "message": "host is a required field, please be sure to enter"
+    },
+    {
+      "field": "port",
+      "type": "Number",
+      "required": true,
+      "min": 1,
+      "max": 65535,
+      "value": 6379,
+      "message": "port is a required field, please be sure to enter"
+    },
+    {
+      "field": "password",
+      "type": "String",
+      "group": "authorization"
+    }
+  ]
+}

server/src/main/java/io/edurt/datacap/server/common/IConfigureCommon.java

@@ -1,18 +1,23 @@
 package io.edurt.datacap.server.common;
 
 import com.google.common.base.Preconditions;
+import io.edurt.datacap.server.entity.SourceEntity;
+import io.edurt.datacap.server.plugin.configure.IConfigure;
 import io.edurt.datacap.server.plugin.configure.IConfigureField;
 import io.edurt.datacap.server.plugin.configure.IConfigureFieldName;
 import io.edurt.datacap.server.plugin.configure.IConfigureFieldType;
 import io.edurt.datacap.spi.FormatType;
 import io.edurt.datacap.spi.model.Configure;
+import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 public class IConfigureCommon
 {
@@ -64,6 +69,109 @@ public class IConfigureCommon
         return configure;
     }
 
+    public static IConfigure preparedConfigure(IConfigure configure, SourceEntity source)
+    {
+        configure.getConfigures().forEach(v -> {
+            switch (v.getField()) {
+                case name:
+                    v.setValue(source.getName());
+                    break;
+                case host:
+                    v.setValue(source.getHost());
+                    break;
+                case port:
+                    v.setValue(source.getPort());
+                    break;
+                case username:
+                    v.setValue(source.getUsername());
+                    break;
+                case password:
+                    v.setValue(source.getPassword());
+                    break;
+                case database:
+                    v.setValue(source.getDatabase());
+                    break;
+                case ssl:
+                    v.setValue(source.getSsl());
+                    break;
+                case catalog:
+                    v.setValue(source.getCatalog());
+                    break;
+                case configures:
+                    List<Map<String, Object>> fields = new ArrayList<>();
+                    if (ObjectUtils.isNotEmpty(source.getConfigures())) {
+                        source.getConfigures().entrySet().forEach(entry -> {
+                            Map<String, Object> map = new LinkedHashMap<>();
+                            map.put(IConfigureFieldName.field.name(), entry.getKey());
+                            map.put(IConfigureFieldName.value.name(), entry.getValue());
+                            fields.add(map);
+                        });
+                    }
+                    v.setValue(fields);
+                    break;
+            }
+        });
+        return configure;
+    }
+
+    public static SourceEntity preparedSourceEntity(List<IConfigureField> configures)
+    {
+        SourceEntity configure = new SourceEntity();
+        configures.forEach(v -> {
+            switch (v.getField()) {
+                case name:
+                    configure.setName(IConfigureCommon.getStringValue(configures, IConfigureFieldName.name));
+                case host:
+                    configure.setHost(IConfigureCommon.getStringValue(configures, IConfigureFieldName.host));
+                    break;
+                case port:
+                    configure.setPort(IConfigureCommon.getIntegerValue(configures, IConfigureFieldName.port));
+                    break;
+                case username:
+                    configure.setUsername(IConfigureCommon.getStringValue(configures, IConfigureFieldName.username));
+                    break;
+                case password:
+                    configure.setPassword(IConfigureCommon.getStringValue(configures, IConfigureFieldName.password));
+                    break;
+                case database:
+                    String database = IConfigureCommon.getStringValue(configures, IConfigureFieldName.database);
+                    configure.setDatabase(database);
+                    break;
+                case catalog:
+                    String catalog = IConfigureCommon.getStringValue(configures, IConfigureFieldName.catalog);
+                    configure.setCatalog(catalog);
+                    break;
+                case ssl:
+                    configure.setSsl(IConfigureCommon.getBooleanValue(configures, IConfigureFieldName.ssl));
+                    break;
+                case configures:
+                    configure.setConfigure(JSON.toJSON(IConfigureCommon.getMapValue(configures, IConfigureFieldName.configures)));
+                    break;
+            }
+        });
+        return configure;
+    }
+
+    public static List<IConfigureField> filterNotEmpty(List<IConfigureField> configures)
+    {
+        return configures.stream().filter(v -> !isEmpty(v)).collect(Collectors.toList());
+    }
+
+    public static boolean isEmpty(IConfigureField field)
+    {
+        boolean flag = true;
+        switch (field.getType()) {
+            case String:
+                if (ObjectUtils.isNotEmpty(field.getValue())) {
+                    flag = StringUtils.isEmpty(String.valueOf(field.getValue()));
+                }
+                break;
+            default:
+                flag = false;
+        }
+        return flag;
+    }
+
     public static IConfigureField getConfigure(List<IConfigureField> configures, IConfigureFieldName key)
     {
         Optional<IConfigureField> configureFieldOptional = configures.stream().filter(v -> v.getField().equals(key)).findFirst();

server/src/main/java/io/edurt/datacap/server/controller/user/SourceController.java

@@ -34,12 +34,14 @@ public class SourceController
         this.sourceService = sourceService;
     }
 
+    @Deprecated
     @PostMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
     public Response<SourceEntity> save(@RequestBody @Validated(ValidationGroup.Crud.Create.class) SourceEntity configure)
     {
         return this.sourceService.saveOrUpdate(configure);
     }
 
+    @Deprecated
     @PutMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
     public Response<SourceEntity> update(@RequestBody @Validated(ValidationGroup.Crud.Update.class) SourceEntity configure)
     {

server/src/main/java/io/edurt/datacap/server/controller/user/v2/SourceV2Controller.java

@@ -2,11 +2,15 @@ package io.edurt.datacap.server.controller.user.v2;
 
 import io.edurt.datacap.server.body.SourceBody;
 import io.edurt.datacap.server.common.Response;
+import io.edurt.datacap.server.entity.SourceEntity;
 import io.edurt.datacap.server.service.SourceService;
 import io.edurt.datacap.server.validation.ValidationGroup;
 import org.springframework.http.MediaType;
 import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
@@ -27,4 +31,22 @@ public class SourceV2Controller
     {
         return this.sourceService.testConnectionV2(configure);
     }
+
+    @PostMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
+    public Response<SourceEntity> save(@RequestBody @Validated(ValidationGroup.Crud.Create.class) SourceBody configure)
+    {
+        return this.sourceService.saveOrUpdateV2(configure);
+    }
+
+    @PutMapping(produces = {MediaType.APPLICATION_JSON_VALUE})
+    public Response<SourceEntity> update(@RequestBody @Validated(ValidationGroup.Crud.Update.class) SourceBody configure)
+    {
+        return this.sourceService.saveOrUpdateV2(configure);
+    }
+
+    @GetMapping(value = "{id}")
+    public Response<SourceEntity> getInfo(@PathVariable(value = "id") Long id)
+    {
+        return this.sourceService.getByIdV2(id);
+    }
 }

server/src/main/java/io/edurt/datacap/server/entity/SourceEntity.java

@@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonIncludeProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import io.edurt.datacap.server.common.JSON;
+import io.edurt.datacap.server.plugin.configure.IConfigure;
 import io.edurt.datacap.server.validation.ValidationGroup;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -104,6 +105,9 @@ public class SourceEntity
     @Transient
     private Map<String, Object> configures;
 
+    @Transient
+    private IConfigure schema;
+
     @ManyToOne
     @JoinColumn(name = "user_id")
     @JsonIncludeProperties(value = {"id", "username"})

server/src/main/java/io/edurt/datacap/server/service/SourceService.java

@@ -12,6 +12,7 @@ import java.util.Map;
 
 public interface SourceService
 {
+    @Deprecated
     Response<SourceEntity> saveOrUpdate(SourceEntity configure);
 
     Response<PageEntity<SourceEntity>> getAll(int offset, int limit);
@@ -30,4 +31,8 @@ public interface SourceService
     Response<Object> shared(SharedSourceBody configure);
 
     Response<Object> testConnectionV2(SourceBody configure);
+
+    Response<SourceEntity> saveOrUpdateV2(SourceBody configure);
+
+    Response<SourceEntity> getByIdV2(Long id);
 }

server/src/main/java/io/edurt/datacap/server/service/impl/SourceServiceImpl.java

@@ -55,6 +55,7 @@ public class SourceServiceImpl
         this.environment = environment;
     }
 
+    @Deprecated
     @Override
     public Response<SourceEntity> saveOrUpdate(SourceEntity configure)
     {
@@ -117,21 +118,19 @@ public class SourceServiceImpl
     public Response<Map<String, List<PluginEntity>>> getPlugins()
     {
         Map<String, List<PluginEntity>> pluginMap = new ConcurrentHashMap<>();
-        this.injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}))
+        this.injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {})).stream().forEach(plugin -> {
-                .stream()
+            PluginEntity entity = new PluginEntity();
-                .forEach(plugin -> {
+            entity.setName(plugin.name());
-                    PluginEntity entity = new PluginEntity();
+            entity.setDescription(plugin.description());
-                    entity.setName(plugin.name());
+            entity.setType(plugin.type().name());
-                    entity.setDescription(plugin.description());
+            entity.setConfigure(PluginCommon.loadConfigure(plugin.type().name(), plugin.name(), plugin.name(), environment));
-                    entity.setType(plugin.type().name());
+            List<PluginEntity> plugins = pluginMap.get(plugin.type().name());
-                    entity.setConfigure(PluginCommon.loadConfigure(plugin.type().name(), plugin.name(), plugin.name(), environment));
+            if (ObjectUtils.isEmpty(plugins)) {
-                    List<PluginEntity> plugins = pluginMap.get(plugin.type().name());
+                plugins = new ArrayList<>();
-                    if (ObjectUtils.isEmpty(plugins)) {
+            }
-                        plugins = new ArrayList<>();
+            plugins.add(entity);
-                    }
+            pluginMap.put(plugin.type().name(), plugins);
-                    plugins.add(entity);
+        });
-                    pluginMap.put(plugin.type().name(), plugins);
-                });
         return Response.success(pluginMap);
     }
 
@@ -175,15 +174,15 @@ public class SourceServiceImpl
         }
 
         // Filter required
-        List<IConfigureField> requiredMismatchConfigures = configure.getConfigure().getConfigures().stream().filter(v -> v.isRequired())
+        List<IConfigureField> requiredMismatchConfigures = configure.getConfigure().getConfigures().stream().filter(v -> v.isRequired()).filter(v -> ObjectUtils.isEmpty(v.getValue())).collect(Collectors.toList());
-                .filter(v -> ObjectUtils.isEmpty(v.getValue()))
-                .collect(Collectors.toList());
         if (requiredMismatchConfigures.size() > 0) {
             return Response.failure(ServiceState.PLUGIN_CONFIGURE_REQUIRED, IConfigureCommon.preparedMessage(requiredMismatchConfigures));
         }
 
         Plugin plugin = pluginOptional.get();
-        Configure _configure = IConfigureCommon.preparedConfigure(configure.getConfigure().getConfigures());
+        // The filter parameter value is null data
+        List<IConfigureField> applyConfigures = IConfigureCommon.filterNotEmpty(configure.getConfigure().getConfigures());
+        Configure _configure = IConfigureCommon.preparedConfigure(applyConfigures);
         plugin.connect(_configure);
         io.edurt.datacap.spi.model.Response response = plugin.execute(plugin.validator());
         plugin.destroy();
@@ -192,4 +191,55 @@ public class SourceServiceImpl
         }
         return Response.failure(ServiceState.PLUGIN_EXECUTE_FAILED, response.getMessage());
     }
+
+    @Override
+    public Response<SourceEntity> saveOrUpdateV2(SourceBody configure)
+    {
+        Optional<Plugin> pluginOptional = PluginCommon.getPluginByNameAndType(this.injector, configure.getName(), configure.getType());
+        if (!pluginOptional.isPresent()) {
+            return Response.failure(ServiceState.PLUGIN_NOT_FOUND);
+        }
+
+        // Check configure
+        IConfigure iConfigure = PluginCommon.loadConfigure(configure.getType(), configure.getName(), configure.getName(), environment);
+        if (ObjectUtils.isEmpty(iConfigure) || iConfigure.getConfigures().size() != configure.getConfigure().getConfigures().size()) {
+            return Response.failure(ServiceState.PLUGIN_CONFIGURE_MISMATCH);
+        }
+
+        // Filter required
+        List<IConfigureField> requiredMismatchConfigures = configure.getConfigure().getConfigures().stream().filter(v -> v.isRequired()).filter(v -> ObjectUtils.isEmpty(v.getValue())).collect(Collectors.toList());
+        if (requiredMismatchConfigures.size() > 0) {
+            return Response.failure(ServiceState.PLUGIN_CONFIGURE_REQUIRED, IConfigureCommon.preparedMessage(requiredMismatchConfigures));
+        }
+
+        // The filter parameter value is null data
+        List<IConfigureField> applyConfigures = IConfigureCommon.filterNotEmpty(configure.getConfigure().getConfigures());
+        SourceEntity source = IConfigureCommon.preparedSourceEntity(applyConfigures);
+        source.setProtocol(configure.getType());
+        source.setType(configure.getName());
+        source.setUser(UserDetailsService.getUser());
+        if (ObjectUtils.isNotEmpty(configure.getId())) {
+            source.setId(configure.getId());
+        }
+        return Response.success(this.sourceRepository.save(source));
+    }
+
+    @Override
+    public Response<SourceEntity> getByIdV2(Long id)
+    {
+        Optional<SourceEntity> optionalSource = this.sourceRepository.findById(id);
+        if (!optionalSource.isPresent()) {
+            return Response.failure(ServiceState.SOURCE_NOT_FOUND);
+        }
+        SourceEntity entity = optionalSource.get();
+        SourceBody configure = new SourceBody();
+        configure.setId(id);
+        configure.setName(entity.getType());
+        configure.setType(entity.getProtocol());
+        // Load default configure
+        IConfigure iConfigure = PluginCommon.loadConfigure(configure.getType(), configure.getName(), configure.getName(), environment);
+        configure.setConfigure(IConfigureCommon.preparedConfigure(iConfigure, entity));
+        entity.setSchema(iConfigure);
+        return Response.success(entity);
+    }
 }

web/console-fe/src/services/SourceV2Service.ts

@@ -0,0 +1,26 @@
+import {ResponseModel} from "@/model/ResponseModel";
+import {isEmpty} from "lodash";
+import {HttpCommon} from "@/common/HttpCommon";
+
+const baseUrl = "/api/v2/source";
+
+class SourceV2Service
+{
+  saveAndUpdate(configure, isUpdate: boolean): Promise<ResponseModel>
+  {
+    configure.protocol = isEmpty(configure.protocol) ? 'HTTP' : configure.protocol;
+    if (isUpdate) {
+      return new HttpCommon().put(baseUrl, JSON.stringify(configure));
+    }
+    else {
+      return new HttpCommon().post(baseUrl, JSON.stringify(configure));
+    }
+  }
+
+  getById(id: number): Promise<ResponseModel>
+  {
+    return new HttpCommon().get(baseUrl + "/" + id);
+  }
+}
+
+export default new SourceV2Service();

web/console-fe/src/views/pages/admin/source/SourceDetail.vue

@@ -32,7 +32,7 @@
       </Row>
       <Form :model="formState" :label-width="80">
         <Tabs v-model="activeKey" :animated="false" @update:modelValue="handlerFilterConfigure($event)">
-          <TabPane :label="$t('common.' + type)" v-for="type in pluginTabs" :name="type" :disabled="!formState.type" icon="md-apps">
+          <TabPane :label="$t('common.' + type)" v-for="type in pluginTabs" :name="type" v-bind:key="type" :disabled="!formState.type" icon="md-apps">
             <div v-if="type === 'source'">
               <RadioGroup v-if="plugins" v-model="formState.type" type="button" @on-change="handlerChangePlugin($event)">
                 <div v-for="key in Object.keys(plugins)" v-bind:key="key">
@@ -52,7 +52,7 @@
               <Row>
                 <Col :span="5"/>
                 <Col :span="14">
-                  <FormItem v-for="configure in pluginTabConfigure" :required="configure.required" :prop="configure.field">
+                  <FormItem v-for="configure in pluginTabConfigure" :required="configure.required" v-bind:key="configure.field" :prop="configure.field">
                     <template #label>
                       <span v-if="configure.field !== 'configures'">{{ $t('common.' + configure.field) }}</span>
                     </template>
@@ -119,8 +119,8 @@ import {SourceService} from "@/services/SourceService";
 import {emptySource} from "@/views/pages/admin/source/SourceGenerate";
 import {defineComponent, reactive, ref} from "vue";
 import {Configure} from "@/model/Configure";
-import {Arrays} from "@/common/Arrays";
 import {clone} from 'lodash'
+import SourceV2Service from "@/services/SourceV2Service";
 
 interface TestInfo
 {
@@ -183,20 +183,16 @@ export default defineComponent({
     handlerInitialize()
     {
       if (this.id > 0) {
-        new SourceService().getById(this.id)
+        SourceV2Service.getById(this.id)
           .then(response => {
             if (response.status) {
               this.formState = reactive(response.data);
               this.formState.type = this.formState.type + ' ' + this.formState.protocol;
-              if (response.data.configures) {
+              this.applyPlugin = response.data['schema'];
-                Object.keys(response.data.configures).forEach((value) => {
+              this.pluginConfigure = response.data['schema']['configures'];
-                  const configure: Configure = {
+              // Clear
-                    field: value,
+              this.pluginTabs = ['source'];
-                    value: response.data.configures[value]
+              this.pluginTabs = [...this.pluginTabs, ...Array.from(new Set(this.pluginConfigure.map(v => v.group)))];
-                  };
-                  this.configure.push(configure);
-                });
-              }
             }
           });
       }
@@ -213,13 +209,14 @@ export default defineComponent({
     },
     handlerSave()
     {
-      this.formState.configures = Arrays.arrayToObject(this.configure);
-      const applyConfigure = clone(this.formState);
       const temp = clone(this.formState.type).split(' ');
-      applyConfigure.type = temp[0];
+      const configure = {
-      applyConfigure.protocol = temp[1];
+        id: this.id,
-      new SourceService()
+        type: temp[1],
-        .saveAndUpdate(applyConfigure, this.isUpdate)
+        name: temp[0],
+        configure: this.applyPlugin
+      };
+      SourceV2Service.saveAndUpdate(configure, this.isUpdate)
         .then((response) => {
           if (response.status) {
             this.$Message.success("Create successful");