Feature integration seatunnel (#286)

2024-12-02 20:17:45 +08:00 · 2023-03-19 20:14:14 +08:00 · 2023-03-19 20:14:14 +08:00 · 791c4b3e8a
commit 791c4b3e8a
parent d9e10edbb0 dcb7ef1c63
21 changed files with 176 additions and 65 deletions
--- a/README.md
+++ b/README.md
@ -6,7 +6,6 @@

 ---

-![](https://visitor-badge.glitch.me/badge?page_id=datacap)
 ![Visitors](https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fgithub.com%2FEdurtIO%2Fdatacap.git&countColor=%23263759&style=flat&labelStyle=none)
 [![](https://tokei.rs/b1/github/EdurtIO/datacap)](https://github.com/EdurtIO/datacap)
 ![version](https://img.shields.io/github/v/release/EdurtIO/datacap.svg)
--- a/core/datacap-server/src/main/etc/conf/application.properties
+++ b/core/datacap-server/src/main/etc/conf/application.properties
@ -12,4 +12,4 @@ datacap.security.expiration=86400000
 # Executor
 ### If this directory is not set, the system will get the project root directory to build the data subdirectory
 datacap.executor.data=
-datacap.executor.seatunnel.home=/Users/shicheng/Tools/apache-seatunnel-incubating-2.3.0
+datacap.executor.seatunnel.home=/opt/lib/seatunnel
--- a/core/datacap-server/src/main/etc/conf/plugins/http/clickhouse.yaml
+++ b/core/datacap-server/src/main/etc/conf/plugins/http/clickhouse.yaml
@ -24,7 +24,7 @@ pipelines:
    type: SOURCE
    fields:
      - field: host
-        origin: host
+        origin: host|port
        required: true
      - field: database
        origin: database
@ -42,7 +42,7 @@ pipelines:
    type: SINK
    fields:
      - field: host
-        origin: host
+        origin: host|port
        required: true
      - field: database
        origin: database
--- a/core/datacap-server/src/main/etc/conf/plugins/jdbc/clickhouse.yaml
+++ b/core/datacap-server/src/main/etc/conf/plugins/jdbc/clickhouse.yaml
@ -37,6 +37,7 @@ pipelines:
    type: SOURCE
    fields:
      - field: host
+        origin: host|port
        required: true
      - field: database
        required: true
@ -55,6 +56,7 @@ pipelines:
    type: SINK
    fields:
      - field: host
+        origin: host|port
        required: true
      - field: database
        override: true
--- a/core/datacap-server/src/main/etc/conf/plugins/native/alioss.yaml
+++ b/core/datacap-server/src/main/etc/conf/plugins/native/alioss.yaml
@ -1,5 +1,6 @@
 name: Alioss
 supportTime: '2023-02-23'
+
 configures:
  - field: name
    type: String
--- a/core/datacap-server/src/main/etc/conf/plugins/native/kafka.yaml
+++ b/core/datacap-server/src/main/etc/conf/plugins/native/kafka.yaml
@ -1,5 +1,6 @@
 name: Kafka
 supportTime: '2023-03-06'
+
 configures:
  - field: name
    type: String
@ -10,3 +11,27 @@ configures:
    required: true
    value: 127.0.0.1:9092
    message: host is a required field, please be sure to enter
+
+pipelines:
+  - executor: Seatunnel
+    type: SOURCE
+    fields:
+      - field: bootstrap.servers
+        origin: host
+        required: true
+      - field: topic
+        origin: database
+        required: true
+        override: true
+        input: true
+  - executor: Seatunnel
+    type: SINK
+    fields:
+      - field: bootstrap.servers
+        origin: host
+        required: true
+      - field: topic
+        origin: database
+        required: true
+        override: true
+        input: true
--- a/core/datacap-server/src/main/java/io/edurt/datacap/server/body/UserQuestionBody.java
+++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/body/UserQuestionBody.java
@ -11,6 +11,10 @@ import lombok.ToString;
@AllArgsConstructor
 public class UserQuestionBody
 {
-    private String question;
+    private String locale;
+    private String content;
    private String type;
+    private String engine;
+    private String error;
+    private String transType;
 }
--- a/core/datacap-server/src/main/java/io/edurt/datacap/server/common/AiSupportCommon.java
+++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/common/AiSupportCommon.java
@ -0,0 +1,42 @@
+package io.edurt.datacap.server.common;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+
+public class AiSupportCommon
+{
+    private AiSupportCommon()
+    {}
+
+    public static String getValue(String locale, AiSupportEnum supportType)
+    {
+        if (StringUtils.isEmpty(locale)) {
+            return supportType.enValue;
+        }
+        if (locale.equalsIgnoreCase("zh")) {
+            return supportType.zhValue;
+        }
+        else {
+            return supportType.enValue;
+        }
+    }
+
+    @AllArgsConstructor
+    public enum AiSupportEnum
+    {
+        OPTIMIZE("优化以下SQL\n${sql}\n输出为markdown\n引擎是${engine}", "Help me optimize the following SQL\n${sql}\noutput is markdown\nThe engine is ${engine}"),
+        ANALYSIS("解析以下SQL\n${sql}\n输出为markdown\n引擎是${engine}", "Help me analyze the following SQL\n${sql}\noutput is markdown\nThe engine is ${engine}"),
+        TRANSLATOR("将此SQL查询翻译成自然语言\n${sql}\n输出为markdown", "Translate this SQL query into natural language\n${sql}\noutput is markdown"),
+        FINDBUGS("查找以下SQL出现的错误信息\n${sql}\n输出为markdown\n引擎是${engine}", "Find the error message that appears in the following SQL\n${sql}\nThe output is markdown\nThe engine is ${engine}"),
+        FIXEDBUGS("修复一下SQL出现的错误\n${sql}\n输出返回结果内容为markdown\n引擎是${engine}\n出现的错误是\n${error}", "Fix the error in SQL\n${sql}\nThe output returns the result content as markdown\nThe engine is ${engine}\nThe error that occurred is\n${error}");
+
+        @Getter
+        @Setter
+        private String zhValue;
+        @Getter
+        @Setter
+        private String enValue;
+    }
+}
--- a/core/datacap-server/src/main/java/io/edurt/datacap/server/service/impl/PipelineServiceImpl.java
+++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/service/impl/PipelineServiceImpl.java
@ -208,8 +208,14 @@ public class PipelineServiceImpl
    {
        Object value = "";
        if (ObjectUtils.isNotEmpty(field.getOrigin())) {
-            if (ObjectUtils.isNotEmpty(configure.get(field.getOrigin()))) {
-                value = configure.get(field.getOrigin());
+            String[] split = String.valueOf(field.getOrigin()).split("\\|");
+            if (split.length > 1) {
+                value = String.join(":", String.valueOf(configure.get(split[0])), String.valueOf(configure.get(split[1])));
+            }
+            else {
+                if (ObjectUtils.isNotEmpty(configure.get(field.getOrigin()))) {
+                    value = configure.get(field.getOrigin());
+                }
            }
        }
        else {
--- a/core/datacap-server/src/main/java/io/edurt/datacap/server/service/impl/UserServiceImpl.java
+++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/service/impl/UserServiceImpl.java
@ -8,6 +8,7 @@ import io.edurt.datacap.server.audit.AuditUserLog;
 import io.edurt.datacap.server.body.UserNameBody;
 import io.edurt.datacap.server.body.UserPasswordBody;
 import io.edurt.datacap.server.body.UserQuestionBody;
+import io.edurt.datacap.server.common.AiSupportCommon;
 import io.edurt.datacap.server.common.JSON;
 import io.edurt.datacap.server.common.JwtResponse;
 import io.edurt.datacap.server.common.Response;
@ -19,6 +20,7 @@ import io.edurt.datacap.server.repository.UserRepository;
 import io.edurt.datacap.server.security.JwtService;
 import io.edurt.datacap.server.security.UserDetailsService;
 import io.edurt.datacap.server.service.UserService;
+import org.apache.commons.lang.text.StrSubstitutor;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.security.authentication.AuthenticationManager;
@ -34,6 +36,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Properties;
 import java.util.Set;
 import java.util.stream.Collectors;

@ -181,9 +184,27 @@ public class UserServiceImpl
                        OpenAiClient openAiClient = OpenAiClient.builder()
                                .apiKey(token)
                                .build();
+                        String forwardContent = configure.getContent();
+                        try {
+                            AiSupportCommon.AiSupportEnum type = AiSupportCommon.AiSupportEnum.valueOf(configure.getTransType());
+                            String replaceContent = AiSupportCommon.getValue(configure.getLocale(), type);
+                            Properties properties = new Properties();
+                            properties.put("sql", configure.getContent());
+                            if (ObjectUtils.isNotEmpty(configure.getEngine())) {
+                                properties.put("engine", configure.getEngine());
+                            }
+                            if (ObjectUtils.isNotEmpty(configure.getError())) {
+                                properties.put("error", configure.getError());
+                            }
+                            StrSubstitutor sub = new StrSubstitutor(properties);
+                            forwardContent = sub.replace(replaceContent);
+                        }
+                        catch (Exception exception) {
+                            // Ignore it
+                        }
                        Message message = Message.builder()
                                .role(Message.Role.USER)
-                                .content(configure.getQuestion())
+                                .content(forwardContent)
                                .build();
                        ChatCompletion chatCompletion = ChatCompletion.builder().messages(Arrays.asList(message)).build();
                        ChatCompletionResponse chatCompletionResponse = openAiClient.chatCompletion(chatCompletion);
--- a/core/datacap-web/console-fe/package.json
+++ b/core/datacap-web/console-fe/package.json
@ -10,7 +10,6 @@
    "lint": "vue-cli-service lint"
  },
  "dependencies": {
-    "@kangc/v-md-editor": "^2.3.15",
    "@types/nprogress": "^0.2.0",
    "@types/watermark-dom": "^2.3.1",
    "ag-grid-community": "^28.2.1",
--- a/core/datacap-web/console-fe/src/i18n/langs/en/common.ts
+++ b/core/datacap-web/console-fe/src/i18n/langs/en/common.ts
@ -88,4 +88,7 @@ export default {
  work: 'Work',
  from: 'From',
  to: 'To',
+  analysis: 'Analysis',
+  optimize: 'Optimize',
+  fixedbugs: 'Fixed Bugs'
 }
--- a/core/datacap-web/console-fe/src/i18n/langs/zhCn/common.ts
+++ b/core/datacap-web/console-fe/src/i18n/langs/zhCn/common.ts
@ -87,5 +87,8 @@ export default {
  pipeline: '流水线',
  work: '工作目录',
  from: '来源',
-  to: '目标'
+  to: '目标',
+  analysis: '解析',
+  optimize: '优化',
+  fixedbugs: '修复问题'
 }
--- a/core/datacap-web/console-fe/src/main.ts
+++ b/core/datacap-web/console-fe/src/main.ts
@ -5,20 +5,10 @@ import router from "./router";
 import ViewUIPlus from 'view-ui-plus';
 import 'view-ui-plus/dist/styles/viewuiplus.css';

-import VMdEditor from '@kangc/v-md-editor';
-import VMdPreview from '@kangc/v-md-editor/lib/preview';
-import '@kangc/v-md-editor/lib/style/preview.css';
-import githubTheme from '@kangc/v-md-editor/lib/theme/github';
-import '@kangc/v-md-editor/lib/theme/style/github.css';
-
-VMdPreview.use(githubTheme);
-
 import i18n from "@/i18n/I18n";

 const app = createApp(App);
 app.use(router);
 app.use(ViewUIPlus);
 app.use(i18n);
-app.use(VMdEditor);
-app.use(VMdPreview);
 app.mount("#app");
--- a/core/datacap-web/console-fe/src/model/User.ts
+++ b/core/datacap-web/console-fe/src/model/User.ts
@ -7,8 +7,12 @@ export interface User

 export class UserQuestion
 {
-  question: string;
+  content: string;
  type: string;
+  locale?: string;
+  engine?: string;
+  error?: string;
+  transType?: string;
 }

 export class UserQuestionItem
--- a/core/datacap-web/console-fe/src/views/layout/components/LayoutHeader.vue
+++ b/core/datacap-web/console-fe/src/views/layout/components/LayoutHeader.vue
@ -44,6 +44,10 @@
                <Icon type="ios-book"/>
                {{ $t('common.history') }}
              </MenuItem>
+              <MenuItem name="admin_pipeline" to="/admin/pipeline">
+                <Icon type="md-list-box"/>
+                {{ $t('common.pipeline') }}
+              </MenuItem>
            </MenuGroup>
          </Submenu>
          <Submenu name="settings">
--- a/core/datacap-web/console-fe/src/views/pages/profile/ProfileChatGPT.vue
+++ b/core/datacap-web/console-fe/src/views/pages/profile/ProfileChatGPT.vue
@ -10,9 +10,9 @@
        </Tooltip>
      </template>
      <div>
-        <Layout>
-          <Content style="padding: 0px 0px 0px 10px">
-            <div ref="scrollDiv" style="height: 300px; max-height: 300px; overflow: auto; background-color: #f5f7f9">
+        <Layout style="background-color: #FFFFFF;">
+          <Content style="padding: 0px 0px 0px 10px;">
+            <div ref="scrollDiv" style="height: 300px; max-height: 300px; overflow: auto;">
              <List item-layout="vertical">
                <ListItem v-for="item in userQuestionItems" :key="item">
                  <ListItemMeta style="margin-bottom: 0px;">
@ -24,7 +24,9 @@
                      <Avatar v-else icon="md-ionitron" style="background-color: #87d068;"></Avatar>
                    </template>
                  </ListItemMeta>
-                  <v-md-preview :text="item.content"/>
+                  <div style="margin: 0px 10px;">
+                    <VMarkdownView v-if="item.content" :mode="'light'" :content="item.content"></VMarkdownView>
+                  </div>
                </ListItem>
              </List>
            </div>
@ -68,6 +70,8 @@ import UserService from "@/services/UserService";
 import {ThirdConfigure, User, UserQuestion, UserQuestionItem} from '@/model/User';
 import Common from "@/common/Common";
 import {AuthResponse} from "@/model/AuthResponse";
+import {VMarkdownView} from 'vue3-markdown'
+import 'vue3-markdown/dist/style.css'

 export default defineComponent({
  setup()
@ -81,6 +85,7 @@ export default defineComponent({
      username
    }
  },
+  components: {VMarkdownView},
  created()
  {
    this.handlerInitialize()
@ -142,7 +147,7 @@ export default defineComponent({
    {
      const userQuestion = new UserQuestion();
      userQuestion.type = 'ChatGPT';
-      userQuestion.question = this.userQuestionContext;
+      userQuestion.content = this.userQuestionContext;
      const question = new UserQuestionItem();
      question.content = this.userQuestionContext;
      question.isSelf = true;
--- a/core/datacap-web/console-fe/src/views/pages/query/QueryAiHelp.vue
+++ b/core/datacap-web/console-fe/src/views/pages/query/QueryAiHelp.vue
@ -2,15 +2,8 @@
  <div>
    <Modal title="AI" width="80%" :closable="false" v-model="visible" :maskClosable="false" :z-index="9">
      <div style="height: 350px; max-height: 350px;">
-        <Tabs v-if="!loading" :animated="false" @on-click="handlerTab($event)">
-          <TabPane :label="$t('ai.optimizeSQL')" name="OPTIMIZE" :disabled="actionLoading">
-            <div style="height: 300px; max-height: 350px; overflow: auto;">
-              <Skeleton v-if="actionLoading" loading :title="false" :animated="true"
-                        :paragraph="{ rows: 5, width: [100, 200, '300px', '50%', '62%'] }"/>
-              <VMarkdownView v-else :mode="'light'" :content="finalContent"></VMarkdownView>
-            </div>
-          </TabPane>
-          <TabPane :label="$t('ai.analysisSQL')" name="ANALYSIS" :disabled="actionLoading">
+        <Tabs v-if="!loading" :model-value="tabValue" :animated="false" @on-click="handlerTab($event)">
+          <TabPane :label="$t('common.' + support.toLowerCase())" v-for="support of aiSupports" :name="support" :key="support" :disabled="actionLoading">
            <div style="height: 300px; max-height: 350px; overflow: auto;">
              <Skeleton v-if="actionLoading" loading :title="false" :animated="true"
                        :paragraph="{ rows: 5, width: [100, 200, '300px', '50%', '62%'] }"/>
@ -34,12 +27,6 @@ import {VMarkdownView} from 'vue3-markdown'
 import 'vue3-markdown/dist/style.css'
 import {Skeleton} from "view-ui-plus";

-export enum TabType
-{
-  OPTIMIZE = ('OPTIMIZE'),
-  ANALYSIS = ('ANALYSIS')
-}
-
 export default defineComponent({
  name: 'QueryAiHelp',
  props: {
@ -50,6 +37,18 @@ export default defineComponent({
    content: {
      type: String,
      default: () => ''
+    },
+    engine: {
+      type: String,
+      default: () => ''
+    },
+    error: {
+      type: String,
+      default: () => ''
+    },
+    aiSupports: {
+      type: Array,
+      default: () => []
    }
  },
  components: {Skeleton, VMarkdownView},
@ -59,7 +58,8 @@ export default defineComponent({
      loading: false,
      actionLoading: false,
      userInfo: null as User,
-      finalContent: null
+      finalContent: null,
+      tabValue: 'ANALYSIS'
    }
  },
  created()
@ -74,32 +74,24 @@ export default defineComponent({
        .then(response => {
          if (response.status) {
            this.userInfo = response.data;
-            this.handlerTab(TabType.OPTIMIZE);
+            this.handlerTab(this.tabValue);
          }
        })
        .finally(() => {
          this.loading = false;
        });
    },
-    handlerTab(value: TabType)
+    handlerTab(value: string)
    {
      this.finalContent = null;
-      let message = '';
-      if (value === TabType.OPTIMIZE) {
-        message += this.$t('ai.optimizeSQLContent');
-        message += '\n';
-        message += this.content;
-      }
-      else if (value === TabType.ANALYSIS) {
-        message += this.$t('ai.analysisSQLContent');
-        message += '\n';
-        message += this.content;
-      }
-
      this.actionLoading = true;
      const userQuestion = new UserQuestion();
      userQuestion.type = 'ChatGPT';
-      userQuestion.question = message;
+      userQuestion.content = this.content;
+      userQuestion.transType = value;
+      userQuestion.engine = this.engine;
+      userQuestion.error = this.error;
+      userQuestion.locale = this.$i18n.locale;
      UserService.startChat(userQuestion)
        .then(response => {
          if (response.status) {
--- a/core/datacap-web/console-fe/src/views/pages/query/QueryHome.vue
+++ b/core/datacap-web/console-fe/src/views/pages/query/QueryHome.vue
@ -54,11 +54,8 @@
                    </Space>
                  </template>
                </Poptip>
-                <Badge v-if="applySource && activeEditorValue">
+                <Badge v-if="applySource && activeEditorValue" :count="aiSupportType.length">
                  <Button type="primary" size="small" icon="md-ionitron" @click="handlerVisibleHelp(true)"></Button>
-                  <template #count>
-                    <Icon class="go" type="md-help-circle" color="#ed4014" size="20"/>
-                  </template>
                </Badge>
              </Space>
            </template>
@ -85,7 +82,9 @@
    <SnippetDetails v-if="snippetDetails" :isVisible="snippetDetails"
                    :codeSnippet="activeEditorValue" @close="handlerCloseSnippetDetails($event)">
    </SnippetDetails>
-    <QueryAiHelp v-if="visibleAiHelp" :isVisible="visibleAiHelp" :content="activeEditorValue" @close="handlerVisibleHelp($event)"></QueryAiHelp>
+    <QueryAiHelp v-if="visibleAiHelp" :isVisible="visibleAiHelp" :content="activeEditorValue"
+                 :aiSupports="aiSupportType" :error="error"
+                 :engine="engine" @close="handlerVisibleHelp($event)"></QueryAiHelp>
  </div>
 </template>

@ -149,7 +148,10 @@ export default defineComponent({
      editors,
      activeKey,
      editorValueMap,
-      visibleAiHelp: false
+      visibleAiHelp: false,
+      engine: null,
+      aiSupportType: ['ANALYSIS', 'OPTIMIZE'],
+      error: null
    }
  },
  created()
@ -253,6 +255,8 @@ export default defineComponent({
    handlerRun()
    {
      this.tableConfigure = null;
+      this.error = null;
+      this.aiSupportType = ['ANALYSIS', 'OPTIMIZE'];
      this.response = {};
      this.cancelToken = axios.CancelToken.source();
      this.tableLoading = true;
@ -278,6 +282,8 @@ export default defineComponent({
          }
          else {
            this.$Message.error(response.message);
+            this.error = response.message;
+            this.aiSupportType.push('FIXEDBUGS');
            this.tableConfigure = null;
          }
        })
@ -290,6 +296,7 @@ export default defineComponent({
      const idAndType = value.split(':');
      this.applySource = idAndType[0];
      this.applySourceType = idAndType[1];
+      this.engine = idAndType[1];
      this.editorCompletionProvider.dispose();
      this.handlerEditorDidMount(editorMap.get(activeKey.value), idAndType[1]);
    },
@ -324,6 +331,8 @@ export default defineComponent({
    },
    handlerPlusEditor()
    {
+      this.error = null;
+      this.aiSupportType = ['ANALYSIS', 'OPTIMIZE'];
      activeKey.value = 'newTab' + activeKey.value + Date.parse(new Date().toString());
      editors.value.push({title: 'New Tab', key: activeKey.value, closable: true});
      editorValueMap.set(activeKey.value, '');
@ -353,6 +362,8 @@ export default defineComponent({
    },
    handlerChangeEditor(targetKey: string)
    {
+      this.error = null;
+      this.aiSupportType = ['ANALYSIS', 'OPTIMIZE'];
      this.activeEditorValue = editorValueMap.get(targetKey) as string;
    },
    handlerChangeEditorValue(value: string)
--- a/executor/executor-example/pom.xml
+++ b/executor/executor-example/pom.xml
@ -10,7 +10,7 @@
        <relativePath>../../pom.xml</relativePath>
    </parent>

-    <artifactId>executor-example</artifactId>
+    <artifactId>datacap-executor-example</artifactId>
    <description>DataCap - Executor example</description>

    <dependencies>
--- a/executor/executor-seatunnel/pom.xml
+++ b/executor/executor-seatunnel/pom.xml
@ -10,7 +10,7 @@
        <relativePath>../../pom.xml</relativePath>
    </parent>

-    <artifactId>executor-seatunnel</artifactId>
+    <artifactId>datacap-executor-seatunnel</artifactId>
    <description>DataCap - Executor seatunnel</description>

    <dependencies>