[Core] [Query] Add limit to query data (close #487) (#527)

configure/etc/conf/application.properties

@@ -81,3 +81,8 @@ datacap.pipeline.maxRunning=100
 datacap.pipeline.maxQueue=200
 # When the service is restarted, the status of the pipeline with status RUNNING is reset.
 datacap.pipeline.reset=STOPPED
+
+################################# Experimental features #################################
+# This configuration is used to dynamically increase the total number of rows of returned data in SQL during query, and currently only takes effect for user-directed queries
+# If the total number of rows returned is included in the SQL, it will not be automatically incremented
+datacap.experimental.autoLimit=true

core/datacap-common/src/main/java/io/edurt/datacap/common/utils/SqlCheckerUtils.java

@@ -0,0 +1,33 @@
+package io.edurt.datacap.common.utils;
+
+public class SqlCheckerUtils
+{
+    private SqlCheckerUtils()
+    {
+    }
+
+    /**
+     * Check whether the input SQL statement contains paging configurations
+     * <p>
+     * SQL input @param sql
+     *
+     * @return true indicates that paging configuration is included, and false indicates that paging configuration is not included
+     */
+    public static boolean isPagedSql(String sql)
+    {
+        // Check whether the SQL statement contains the limit keyword or the rownum pseudo column
+        if (sql.toLowerCase().contains("limit") || sql.toLowerCase().contains("rownum")) {
+            // If a pagination configuration is included, you need to further determine whether it is in the comments
+            String[] lines = sql.split("\\n");
+            for (String line : lines) {
+                // Remove the space before the comment symbol "--".
+                line = line.replaceAll("\\s*--.*", "");
+                if (line.toLowerCase().contains("limit") || line.toLowerCase().contains("rownum")) {
+                    // If the paging configuration is still included, the description is not in the comments
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+}

core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ExecuteEntity.java

@@ -14,4 +14,5 @@ public class ExecuteEntity
     private String content;
     private Map<String, Object> env;
     private FormatType format = FormatType.NONE;
+    private int limit;
 }

core/datacap-service/src/main/java/io/edurt/datacap/service/initializer/InitializerConfigure.java

@@ -46,6 +46,10 @@ public class InitializerConfigure
     @Value(value = "${datacap.pipeline.maxQueue}")
     private Integer maxQueue;
 
+    @Getter
+    @Value(value = "${datacap.experimental.autoLimit}")
+    private Boolean autoLimit;
+
     @Getter
     private LoadingCache<Long, ResultEntity> cache;
 
@@ -95,6 +99,11 @@ public class InitializerConfigure
         }
         log.info("Datacap pipeline max queue: {}", this.maxQueue);
 
+        if (ObjectUtils.isEmpty(autoLimit)) {
+            autoLimit = Boolean.TRUE;
+        }
+        log.info("Datacap experimental auto limit: [ {} ]", this.autoLimit);
+
         this.taskQueue = new LinkedBlockingQueue<>(this.maxQueue);
         this.taskExecutors = Maps.newConcurrentMap();
     }

core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ExecuteServiceImpl.java

@@ -6,11 +6,13 @@ import io.edurt.datacap.common.response.CommonResponse;
 import io.edurt.datacap.common.sql.SqlBuilder;
 import io.edurt.datacap.common.sql.configure.SqlBody;
 import io.edurt.datacap.common.sql.configure.SqlType;
+import io.edurt.datacap.common.utils.SqlCheckerUtils;
 import io.edurt.datacap.service.audit.AuditPlugin;
 import io.edurt.datacap.service.body.ExecuteDslBody;
 import io.edurt.datacap.service.common.PluginUtils;
 import io.edurt.datacap.service.entity.ExecuteEntity;
 import io.edurt.datacap.service.entity.SourceEntity;
+import io.edurt.datacap.service.initializer.InitializerConfigure;
 import io.edurt.datacap.service.repository.SourceRepository;
 import io.edurt.datacap.service.service.ExecuteService;
 import io.edurt.datacap.spi.Plugin;
@@ -29,12 +31,14 @@ public class ExecuteServiceImpl
     private final Injector injector;
     private final SourceRepository sourceRepository;
     private final Environment environment;
+    private final InitializerConfigure initializerConfigure;
 
-    public ExecuteServiceImpl(Injector injector, SourceRepository sourceRepository, Environment environment)
+    public ExecuteServiceImpl(Injector injector, SourceRepository sourceRepository, Environment environment, InitializerConfigure initializerConfigure)
     {
         this.injector = injector;
         this.sourceRepository = sourceRepository;
         this.environment = environment;
+        this.initializerConfigure = initializerConfigure;
     }
 
     @AuditPlugin
@@ -74,6 +78,13 @@ public class ExecuteServiceImpl
             _configure.setId(String.valueOf(entity.getId()));
         }
         plugin.connect(_configure);
+
+        if (initializerConfigure.getAutoLimit()) {
+            if (!SqlCheckerUtils.isPagedSql(configure.getContent())) {
+                configure.setContent(String.format("%s%nLIMIT %s", configure.getContent(), configure.getLimit()));
+            }
+        }
+
         io.edurt.datacap.spi.model.Response response = plugin.execute(configure.getContent());
         response.setContent(configure.getContent());
         plugin.destroy();

core/datacap-web/src/i18n/langs/en/source/manager.ts

@@ -10,4 +10,5 @@ export default {
   filter: 'Filter',
   addFilter: 'Add Filter',
   addRows: 'Add Rows',
+  copyRows: 'Copy Rows',
 }

core/datacap-web/src/i18n/langs/zhCn/source/manager.ts

@@ -10,4 +10,5 @@ export default {
   filter: '筛选器',
   addFilter: '添加筛选器',
   addRows: '添加行',
+  copyRows: '复制行',
 }

core/datacap-web/src/model/ExecuteModel.ts

@@ -4,4 +4,5 @@ export interface ExecuteModel
   content: string;
   env?: object;
   format?: string;
+  limit?: number;
 }

core/datacap-web/src/views/admin/source/components/TableData.vue

@@ -85,7 +85,7 @@
               <FontAwesomeIcon icon="plus"/>
             </Button>
           </Tooltip>
-          <Tooltip :content="$t('source.manager.addRows')"
+          <Tooltip :content="$t('source.manager.copyRows')"
                    transfer>
             <Button size="small"
                     shape="circle"

core/datacap-web/src/views/user/console/ConsoleIndex.vue

@@ -90,6 +90,11 @@
                           @click="handlerVisibleHelp(true)">
                   </Button>
                 </Badge>
+                <InputNumber v-model="queryLimit"
+                             size="small"
+                             :step="10"
+                             :min="1">
+                </InputNumber>
               </Space>
             </template>
             <div ref="editorContainer">
@@ -221,7 +226,8 @@ export default defineComponent({
       aiSupportType: ['ANALYSIS', 'OPTIMIZE'],
       error: null,
       buttonRunText: null,
-      isSelection: false
+      isSelection: false,
+      queryLimit: 100
     }
   },
   created()
@@ -359,7 +365,8 @@ export default defineComponent({
       const configure: ExecuteModel = {
         name: this.applySource,
         content: this.isSelection ? editorInstance.instance.getSelectedText() : editorInstance.instance.getValue(),
-        format: "JSON"
+        format: "JSON",
+        limit: this.queryLimit
       };
       new ExecuteService()
         .execute(configure, this.cancelToken.token)
@@ -374,6 +381,7 @@ export default defineComponent({
               showSeriesNumber: false
             };
             this.tableConfigure = tConfigure;
+            editorInstance.instance.setValue(response.data.content);
           }
           else {
             this.$Message.error({