Feature integration seatunnel (#286)

README.md

@@ -6,7 +6,6 @@
 
 ---
 
-![](https://visitor-badge.glitch.me/badge?page_id=datacap)
 ![Visitors](https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fgithub.com%2FEdurtIO%2Fdatacap.git&countColor=%23263759&style=flat&labelStyle=none)
 [![](https://tokei.rs/b1/github/EdurtIO/datacap)](https://github.com/EdurtIO/datacap)
 ![version](https://img.shields.io/github/v/release/EdurtIO/datacap.svg)

core/datacap-server/src/main/etc/conf/application.properties

@@ -12,4 +12,4 @@ datacap.security.expiration=86400000
 # Executor
 ### If this directory is not set, the system will get the project root directory to build the data subdirectory
 datacap.executor.data=
-datacap.executor.seatunnel.home=/Users/shicheng/Tools/apache-seatunnel-incubating-2.3.0
+datacap.executor.seatunnel.home=/opt/lib/seatunnel

core/datacap-server/src/main/etc/conf/plugins/http/clickhouse.yaml

@@ -24,7 +24,7 @@ pipelines:
     type: SOURCE
     fields:
       - field: host
-        origin: host
+        origin: host|port
         required: true
       - field: database
         origin: database
@@ -42,7 +42,7 @@ pipelines:
     type: SINK
     fields:
       - field: host
-        origin: host
+        origin: host|port
         required: true
       - field: database
         origin: database

core/datacap-server/src/main/etc/conf/plugins/jdbc/clickhouse.yaml

@@ -37,6 +37,7 @@ pipelines:
     type: SOURCE
     fields:
       - field: host
+        origin: host|port
         required: true
       - field: database
         required: true
@@ -55,6 +56,7 @@ pipelines:
     type: SINK
     fields:
       - field: host
+        origin: host|port
         required: true
       - field: database
         override: true

core/datacap-server/src/main/etc/conf/plugins/native/alioss.yaml

@@ -1,5 +1,6 @@
 name: Alioss
 supportTime: '2023-02-23'
+
 configures:
   - field: name
     type: String

core/datacap-server/src/main/etc/conf/plugins/native/kafka.yaml

@@ -1,5 +1,6 @@
 name: Kafka
 supportTime: '2023-03-06'
+
 configures:
   - field: name
     type: String
@@ -10,3 +11,27 @@ configures:
     required: true
     value: 127.0.0.1:9092
     message: host is a required field, please be sure to enter
+
+pipelines:
+  - executor: Seatunnel
+    type: SOURCE
+    fields:
+      - field: bootstrap.servers
+        origin: host
+        required: true
+      - field: topic
+        origin: database
+        required: true
+        override: true
+        input: true
+  - executor: Seatunnel
+    type: SINK
+    fields:
+      - field: bootstrap.servers
+        origin: host
+        required: true
+      - field: topic
+        origin: database
+        required: true
+        override: true
+        input: true

core/datacap-server/src/main/java/io/edurt/datacap/server/body/UserQuestionBody.java

@@ -11,6 +11,10 @@ import lombok.ToString;
 @AllArgsConstructor
 public class UserQuestionBody
 {
-    private String question;
+    private String locale;
+    private String content;
     private String type;
+    private String engine;
+    private String error;
+    private String transType;
 }

core/datacap-server/src/main/java/io/edurt/datacap/server/common/AiSupportCommon.java

@@ -0,0 +1,42 @@
+package io.edurt.datacap.server.common;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+
+public class AiSupportCommon
+{
+    private AiSupportCommon()
+    {}
+
+    public static String getValue(String locale, AiSupportEnum supportType)
+    {
+        if (StringUtils.isEmpty(locale)) {
+            return supportType.enValue;
+        }
+        if (locale.equalsIgnoreCase("zh")) {
+            return supportType.zhValue;
+        }
+        else {
+            return supportType.enValue;
+        }
+    }
+
+    @AllArgsConstructor
+    public enum AiSupportEnum
+    {
+        OPTIMIZE("优化以下SQL\n${sql}\n输出为markdown\n引擎是${engine}", "Help me optimize the following SQL\n${sql}\noutput is markdown\nThe engine is ${engine}"),
+        ANALYSIS("解析以下SQL\n${sql}\n输出为markdown\n引擎是${engine}", "Help me analyze the following SQL\n${sql}\noutput is markdown\nThe engine is ${engine}"),
+        TRANSLATOR("将此SQL查询翻译成自然语言\n${sql}\n输出为markdown", "Translate this SQL query into natural language\n${sql}\noutput is markdown"),
+        FINDBUGS("查找以下SQL出现的错误信息\n${sql}\n输出为markdown\n引擎是${engine}", "Find the error message that appears in the following SQL\n${sql}\nThe output is markdown\nThe engine is ${engine}"),
+        FIXEDBUGS("修复一下SQL出现的错误\n${sql}\n输出返回结果内容为markdown\n引擎是${engine}\n出现的错误是\n${error}", "Fix the error in SQL\n${sql}\nThe output returns the result content as markdown\nThe engine is ${engine}\nThe error that occurred is\n${error}");
+
+        @Getter
+        @Setter
+        private String zhValue;
+        @Getter
+        @Setter
+        private String enValue;
+    }
+}

core/datacap-server/src/main/java/io/edurt/datacap/server/service/impl/PipelineServiceImpl.java

@@ -208,10 +208,16 @@ public class PipelineServiceImpl
     {
         Object value = "";
         if (ObjectUtils.isNotEmpty(field.getOrigin())) {
+            String[] split = String.valueOf(field.getOrigin()).split("\\|");
+            if (split.length > 1) {
+                value = String.join(":", String.valueOf(configure.get(split[0])), String.valueOf(configure.get(split[1])));
+            }
+            else {
                 if (ObjectUtils.isNotEmpty(configure.get(field.getOrigin()))) {
                     value = configure.get(field.getOrigin());
                 }
             }
+        }
         else {
             if (ObjectUtils.isNotEmpty(configure.get(field.getField()))) {
                 value = configure.get(field.getField());

core/datacap-server/src/main/java/io/edurt/datacap/server/service/impl/UserServiceImpl.java

@@ -8,6 +8,7 @@ import io.edurt.datacap.server.audit.AuditUserLog;
 import io.edurt.datacap.server.body.UserNameBody;
 import io.edurt.datacap.server.body.UserPasswordBody;
 import io.edurt.datacap.server.body.UserQuestionBody;
+import io.edurt.datacap.server.common.AiSupportCommon;
 import io.edurt.datacap.server.common.JSON;
 import io.edurt.datacap.server.common.JwtResponse;
 import io.edurt.datacap.server.common.Response;
@@ -19,6 +20,7 @@ import io.edurt.datacap.server.repository.UserRepository;
 import io.edurt.datacap.server.security.JwtService;
 import io.edurt.datacap.server.security.UserDetailsService;
 import io.edurt.datacap.server.service.UserService;
+import org.apache.commons.lang.text.StrSubstitutor;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.security.authentication.AuthenticationManager;
@@ -34,6 +36,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Properties;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -181,9 +184,27 @@ public class UserServiceImpl
                         OpenAiClient openAiClient = OpenAiClient.builder()
                                 .apiKey(token)
                                 .build();
+                        String forwardContent = configure.getContent();
+                        try {
+                            AiSupportCommon.AiSupportEnum type = AiSupportCommon.AiSupportEnum.valueOf(configure.getTransType());
+                            String replaceContent = AiSupportCommon.getValue(configure.getLocale(), type);
+                            Properties properties = new Properties();
+                            properties.put("sql", configure.getContent());
+                            if (ObjectUtils.isNotEmpty(configure.getEngine())) {
+                                properties.put("engine", configure.getEngine());
+                            }
+                            if (ObjectUtils.isNotEmpty(configure.getError())) {
+                                properties.put("error", configure.getError());
+                            }
+                            StrSubstitutor sub = new StrSubstitutor(properties);
+                            forwardContent = sub.replace(replaceContent);
+                        }
+                        catch (Exception exception) {
+                            // Ignore it
+                        }
                         Message message = Message.builder()
                                 .role(Message.Role.USER)
-                                .content(configure.getQuestion())
+                                .content(forwardContent)
                                 .build();
                         ChatCompletion chatCompletion = ChatCompletion.builder().messages(Arrays.asList(message)).build();
                         ChatCompletionResponse chatCompletionResponse = openAiClient.chatCompletion(chatCompletion);

core/datacap-web/console-fe/package.json

@@ -10,7 +10,6 @@
     "lint": "vue-cli-service lint"
   },
   "dependencies": {
-    "@kangc/v-md-editor": "^2.3.15",
     "@types/nprogress": "^0.2.0",
     "@types/watermark-dom": "^2.3.1",
     "ag-grid-community": "^28.2.1",

core/datacap-web/console-fe/src/i18n/langs/en/common.ts

@@ -88,4 +88,7 @@ export default {
   work: 'Work',
   from: 'From',
   to: 'To',
+  analysis: 'Analysis',
+  optimize: 'Optimize',
+  fixedbugs: 'Fixed Bugs'
 }

core/datacap-web/console-fe/src/i18n/langs/zhCn/common.ts

@@ -87,5 +87,8 @@ export default {
   pipeline: '流水线',
   work: '工作目录',
   from: '来源',
-  to: '目标'
+  to: '目标',
+  analysis: '解析',
+  optimize: '优化',
+  fixedbugs: '修复问题'
 }

core/datacap-web/console-fe/src/main.ts

@@ -5,20 +5,10 @@ import router from "./router";
 import ViewUIPlus from 'view-ui-plus';
 import 'view-ui-plus/dist/styles/viewuiplus.css';
 
-import VMdEditor from '@kangc/v-md-editor';
-import VMdPreview from '@kangc/v-md-editor/lib/preview';
-import '@kangc/v-md-editor/lib/style/preview.css';
-import githubTheme from '@kangc/v-md-editor/lib/theme/github';
-import '@kangc/v-md-editor/lib/theme/style/github.css';
-
-VMdPreview.use(githubTheme);
-
 import i18n from "@/i18n/I18n";
 
 const app = createApp(App);
 app.use(router);
 app.use(ViewUIPlus);
 app.use(i18n);
-app.use(VMdEditor);
-app.use(VMdPreview);
 app.mount("#app");

core/datacap-web/console-fe/src/model/User.ts

@@ -7,8 +7,12 @@ export interface User
 
 export class UserQuestion
 {
-  question: string;
+  content: string;
   type: string;
+  locale?: string;
+  engine?: string;
+  error?: string;
+  transType?: string;
 }
 
 export class UserQuestionItem

core/datacap-web/console-fe/src/views/layout/components/LayoutHeader.vue

@@ -44,6 +44,10 @@
                 <Icon type="ios-book"/>
                 {{ $t('common.history') }}
               </MenuItem>
+              <MenuItem name="admin_pipeline" to="/admin/pipeline">
+                <Icon type="md-list-box"/>
+                {{ $t('common.pipeline') }}
+              </MenuItem>
             </MenuGroup>
           </Submenu>
           <Submenu name="settings">

core/datacap-web/console-fe/src/views/pages/profile/ProfileChatGPT.vue

@@ -10,9 +10,9 @@
         </Tooltip>
       </template>
       <div>
-        <Layout>
+        <Layout style="background-color: #FFFFFF;">
-          <Content style="padding: 0px 0px 0px 10px">
+          <Content style="padding: 0px 0px 0px 10px;">
-            <div ref="scrollDiv" style="height: 300px; max-height: 300px; overflow: auto; background-color: #f5f7f9">
+            <div ref="scrollDiv" style="height: 300px; max-height: 300px; overflow: auto;">
               <List item-layout="vertical">
                 <ListItem v-for="item in userQuestionItems" :key="item">
                   <ListItemMeta style="margin-bottom: 0px;">
@@ -24,7 +24,9 @@
                       <Avatar v-else icon="md-ionitron" style="background-color: #87d068;"></Avatar>
                     </template>
                   </ListItemMeta>
-                  <v-md-preview :text="item.content"/>
+                  <div style="margin: 0px 10px;">
+                    <VMarkdownView v-if="item.content" :mode="'light'" :content="item.content"></VMarkdownView>
+                  </div>
                 </ListItem>
               </List>
             </div>
@@ -68,6 +70,8 @@ import UserService from "@/services/UserService";
 import {ThirdConfigure, User, UserQuestion, UserQuestionItem} from '@/model/User';
 import Common from "@/common/Common";
 import {AuthResponse} from "@/model/AuthResponse";
+import {VMarkdownView} from 'vue3-markdown'
+import 'vue3-markdown/dist/style.css'
 
 export default defineComponent({
   setup()
@@ -81,6 +85,7 @@ export default defineComponent({
       username
     }
   },
+  components: {VMarkdownView},
   created()
   {
     this.handlerInitialize()
@@ -142,7 +147,7 @@ export default defineComponent({
     {
       const userQuestion = new UserQuestion();
       userQuestion.type = 'ChatGPT';
-      userQuestion.question = this.userQuestionContext;
+      userQuestion.content = this.userQuestionContext;
       const question = new UserQuestionItem();
       question.content = this.userQuestionContext;
       question.isSelf = true;

core/datacap-web/console-fe/src/views/pages/query/QueryAiHelp.vue

@@ -2,15 +2,8 @@
   <div>
     <Modal title="AI" width="80%" :closable="false" v-model="visible" :maskClosable="false" :z-index="9">
       <div style="height: 350px; max-height: 350px;">
-        <Tabs v-if="!loading" :animated="false" @on-click="handlerTab($event)">
+        <Tabs v-if="!loading" :model-value="tabValue" :animated="false" @on-click="handlerTab($event)">
-          <TabPane :label="$t('ai.optimizeSQL')" name="OPTIMIZE" :disabled="actionLoading">
+          <TabPane :label="$t('common.' + support.toLowerCase())" v-for="support of aiSupports" :name="support" :key="support" :disabled="actionLoading">
-            <div style="height: 300px; max-height: 350px; overflow: auto;">
-              <Skeleton v-if="actionLoading" loading :title="false" :animated="true"
-                        :paragraph="{ rows: 5, width: [100, 200, '300px', '50%', '62%'] }"/>
-              <VMarkdownView v-else :mode="'light'" :content="finalContent"></VMarkdownView>
-            </div>
-          </TabPane>
-          <TabPane :label="$t('ai.analysisSQL')" name="ANALYSIS" :disabled="actionLoading">
             <div style="height: 300px; max-height: 350px; overflow: auto;">
               <Skeleton v-if="actionLoading" loading :title="false" :animated="true"
                         :paragraph="{ rows: 5, width: [100, 200, '300px', '50%', '62%'] }"/>
@@ -34,12 +27,6 @@ import {VMarkdownView} from 'vue3-markdown'
 import 'vue3-markdown/dist/style.css'
 import {Skeleton} from "view-ui-plus";
 
-export enum TabType
-{
-  OPTIMIZE = ('OPTIMIZE'),
-  ANALYSIS = ('ANALYSIS')
-}
-
 export default defineComponent({
   name: 'QueryAiHelp',
   props: {
@@ -50,6 +37,18 @@ export default defineComponent({
     content: {
       type: String,
       default: () => ''
+    },
+    engine: {
+      type: String,
+      default: () => ''
+    },
+    error: {
+      type: String,
+      default: () => ''
+    },
+    aiSupports: {
+      type: Array,
+      default: () => []
     }
   },
   components: {Skeleton, VMarkdownView},
@@ -59,7 +58,8 @@ export default defineComponent({
       loading: false,
       actionLoading: false,
       userInfo: null as User,
-      finalContent: null
+      finalContent: null,
+      tabValue: 'ANALYSIS'
     }
   },
   created()
@@ -74,32 +74,24 @@ export default defineComponent({
         .then(response => {
           if (response.status) {
             this.userInfo = response.data;
-            this.handlerTab(TabType.OPTIMIZE);
+            this.handlerTab(this.tabValue);
           }
         })
         .finally(() => {
           this.loading = false;
         });
     },
-    handlerTab(value: TabType)
+    handlerTab(value: string)
     {
       this.finalContent = null;
-      let message = '';
-      if (value === TabType.OPTIMIZE) {
-        message += this.$t('ai.optimizeSQLContent');
-        message += '\n';
-        message += this.content;
-      }
-      else if (value === TabType.ANALYSIS) {
-        message += this.$t('ai.analysisSQLContent');
-        message += '\n';
-        message += this.content;
-      }
-
       this.actionLoading = true;
       const userQuestion = new UserQuestion();
       userQuestion.type = 'ChatGPT';
-      userQuestion.question = message;
+      userQuestion.content = this.content;
+      userQuestion.transType = value;
+      userQuestion.engine = this.engine;
+      userQuestion.error = this.error;
+      userQuestion.locale = this.$i18n.locale;
       UserService.startChat(userQuestion)
         .then(response => {
           if (response.status) {

core/datacap-web/console-fe/src/views/pages/query/QueryHome.vue

@@ -54,11 +54,8 @@
                     </Space>
                   </template>
                 </Poptip>
-                <Badge v-if="applySource && activeEditorValue">
+                <Badge v-if="applySource && activeEditorValue" :count="aiSupportType.length">
                   <Button type="primary" size="small" icon="md-ionitron" @click="handlerVisibleHelp(true)"></Button>
-                  <template #count>
-                    <Icon class="go" type="md-help-circle" color="#ed4014" size="20"/>
-                  </template>
                 </Badge>
               </Space>
             </template>
@@ -85,7 +82,9 @@
     <SnippetDetails v-if="snippetDetails" :isVisible="snippetDetails"
                     :codeSnippet="activeEditorValue" @close="handlerCloseSnippetDetails($event)">
     </SnippetDetails>
-    <QueryAiHelp v-if="visibleAiHelp" :isVisible="visibleAiHelp" :content="activeEditorValue" @close="handlerVisibleHelp($event)"></QueryAiHelp>
+    <QueryAiHelp v-if="visibleAiHelp" :isVisible="visibleAiHelp" :content="activeEditorValue"
+                 :aiSupports="aiSupportType" :error="error"
+                 :engine="engine" @close="handlerVisibleHelp($event)"></QueryAiHelp>
   </div>
 </template>
 
@@ -149,7 +148,10 @@ export default defineComponent({
       editors,
       activeKey,
       editorValueMap,
-      visibleAiHelp: false
+      visibleAiHelp: false,
+      engine: null,
+      aiSupportType: ['ANALYSIS', 'OPTIMIZE'],
+      error: null
     }
   },
   created()
@@ -253,6 +255,8 @@ export default defineComponent({
     handlerRun()
     {
       this.tableConfigure = null;
+      this.error = null;
+      this.aiSupportType = ['ANALYSIS', 'OPTIMIZE'];
       this.response = {};
       this.cancelToken = axios.CancelToken.source();
       this.tableLoading = true;
@@ -278,6 +282,8 @@ export default defineComponent({
           }
           else {
             this.$Message.error(response.message);
+            this.error = response.message;
+            this.aiSupportType.push('FIXEDBUGS');
             this.tableConfigure = null;
           }
         })
@@ -290,6 +296,7 @@ export default defineComponent({
       const idAndType = value.split(':');
       this.applySource = idAndType[0];
       this.applySourceType = idAndType[1];
+      this.engine = idAndType[1];
       this.editorCompletionProvider.dispose();
       this.handlerEditorDidMount(editorMap.get(activeKey.value), idAndType[1]);
     },
@@ -324,6 +331,8 @@ export default defineComponent({
     },
     handlerPlusEditor()
     {
+      this.error = null;
+      this.aiSupportType = ['ANALYSIS', 'OPTIMIZE'];
       activeKey.value = 'newTab' + activeKey.value + Date.parse(new Date().toString());
       editors.value.push({title: 'New Tab', key: activeKey.value, closable: true});
       editorValueMap.set(activeKey.value, '');
@@ -353,6 +362,8 @@ export default defineComponent({
     },
     handlerChangeEditor(targetKey: string)
     {
+      this.error = null;
+      this.aiSupportType = ['ANALYSIS', 'OPTIMIZE'];
       this.activeEditorValue = editorValueMap.get(targetKey) as string;
     },
     handlerChangeEditorValue(value: string)

executor/executor-example/pom.xml

@@ -10,7 +10,7 @@
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
-    <artifactId>executor-example</artifactId>
+    <artifactId>datacap-executor-example</artifactId>
     <description>DataCap - Executor example</description>
 
     <dependencies>

executor/executor-seatunnel/pom.xml

@@ -10,7 +10,7 @@
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
-    <artifactId>executor-seatunnel</artifactId>
+    <artifactId>datacap-executor-seatunnel</artifactId>
     <description>DataCap - Executor seatunnel</description>
 
     <dependencies>