test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-12-02 20:17:45 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Feature execute (#21)

Browse Source
This commit is contained in:
qianmoQ 2022-09-21 22:09:15 +08:00 committed by GitHub
commit 230080d9de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 330 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
plugin/jdbc/mysql/pom.xml
View File

@ -38,9 +38,5 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>findbugs</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>findbugs</artifactId>
</dependency>
</dependencies>
</project>

10
plugin/jdbc/mysql/src/main/java/io/edurt/datacap/plugin/jdbc/mysql/MySQLAdapter.java Normal file
View File

@ -0,0 +1,10 @@
package io.edurt.datacap.plugin.jdbc.mysql;
import io.edurt.datacap.spi.adapter.Adapter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MySQLAdapter
extends Adapter
{
}

18
plugin/jdbc/mysql/src/main/java/io/edurt/datacap/plugin/jdbc/mysql/MySQLConnection.java
View File

@ -1,6 +1,7 @@
package io.edurt.datacap.plugin.jdbc.mysql;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
@ -17,11 +18,13 @@ public class MySQLConnection
private static String DRIVER = "com.mysql.jdbc.Driver";
private final Configure configure;
private final Response response;
private Connection connection;
public MySQLConnection(Configure configure)
public MySQLConnection(Configure configure, Response response)
{
this.configure = configure;
this.response = response;
}
private String appendURL()
@ -61,9 +64,12 @@ public class MySQLConnection
log.info("Connection username and password not present");
this.connection = DriverManager.getConnection(url);
}
response.setIsConnected(Boolean.TRUE);
}
catch (SQLException | ClassNotFoundException ex) {
log.error("Connection failed ", ex);
response.setIsConnected(Boolean.FALSE);
response.setMessage(ex.getMessage());
}
return this.connection;
}
@ -73,6 +79,16 @@ public class MySQLConnection
return this.connection;
}
public Response getResponse()
{
return this.response;
}
public Configure getConfigure()
{
return this.configure;
}
public void destroy()
{
if (ObjectUtils.isNotEmpty(this.connection)) {

15
plugin/jdbc/mysql/src/main/java/io/edurt/datacap/plugin/jdbc/mysql/MySQLPlugin.java
View File

@ -2,13 +2,17 @@ package io.edurt.datacap.plugin.jdbc.mysql;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginType;
import io.edurt.datacap.spi.adapter.Adapter;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MySQLPlugin
implements Plugin
{
private MySQLConnection connection;
private Response response;
@Override
public String getName()
@ -25,15 +29,20 @@ public class MySQLPlugin
@Override
public void connect(Configure configure)
{
this.connection = new MySQLConnection(configure);
this.response = new Response();
this.connection = new MySQLConnection(configure, response);
this.connection.openConnection();
}
@Override
public Response execute(String content)
{
MySQLProcessor processor = new MySQLProcessor(this.connection);
return processor.handlerExecute(content);
log.info("Execute plugin logic started");
response = this.connection.getResponse();
Adapter processor = new MySQLAdapter();
processor.handlerJDBCExecute(this.connection.getConfigure().getFormat(), content, this.connection.getConnection(), response);
log.info("Execute plugin logic end");
return response;
}
@Override

58
plugin/jdbc/mysql/src/main/java/io/edurt/datacap/plugin/jdbc/mysql/MySQLProcessor.java
View File

@ -1,58 +0,0 @@
package io.edurt.datacap.plugin.jdbc.mysql;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"},
justification = "I prefer to suppress these FindBugs warnings")
public class MySQLProcessor
{
private final MySQLConnection connection;
public MySQLProcessor(MySQLConnection connection)
{
this.connection = connection;
}
public Response handlerExecute(String content)
{
Response response = new Response();
try (Statement statement = this.connection.getConnection().createStatement();
ResultSet resultSet = statement.executeQuery(content)) {
List<String> headers = new ArrayList<>();
List<String> types = new ArrayList<>();
List<List<Object>> columns = new ArrayList<>();
boolean isPresent = true;
while (resultSet.next()) {
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
List<Object> _columns = new ArrayList<>();
for (int i = 1; i <= columnCount; i++) {
if (isPresent) {
headers.add(metaData.getColumnName(i));
types.add(metaData.getColumnTypeName(i));
}
_columns.add(resultSet.getString(i));
}
isPresent = false;
columns.add(_columns);
}
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
}
catch (SQLException ex) {
log.error("Execute content failed content {} exception {}", content);
}
return response;
}
}

11
pom.xml
View File

@ -24,6 +24,7 @@
<commons-lang3.version>3.12.0</commons-lang3.version>
<slf4j.version>1.7.36</slf4j.version>
<findbugs.version>3.0.1</findbugs.version>
<jackson.version>2.13.3</jackson.version>
<plugin.maven.checkstyle.version>3.0.0</plugin.maven.checkstyle.version>
<plugin.maven.findbugs.version>3.0.5</plugin.maven.findbugs.version>
<plugin.maven.compiler.version>3.3</plugin.maven.compiler.version>
@ -87,6 +88,16 @@
<artifactId>findbugs</artifactId>
<version>${findbugs.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

9
server/src/main/java/io/edurt/datacap/server/common/Response.java
View File

@ -43,4 +43,13 @@ public class Response<T>
response.status = false;
return response;
}
public static Response failure(ServiceState state, String message)
{
Response response = new Response();
response.code = state.getCode();
response.message = message;
response.status = false;
return response;
}
}

3
server/src/main/java/io/edurt/datacap/server/common/ServiceState.java
View File

@ -3,7 +3,8 @@ package io.edurt.datacap.server.common;
public enum ServiceState
{
SOURCE_NOT_FOUND(1001, "Source does not exist"),
PLUGIN_NOT_FOUND(2001, "Plugin dose not exists");
PLUGIN_NOT_FOUND(2001, "Plugin dose not exists"),
PLUGIN_EXECUTE_FAILED(2002, "Plugin execute failed");
private Integer code;
private String value;

2
server/src/main/java/io/edurt/datacap/server/entity/ExecuteEntity.java
View File

@ -1,5 +1,6 @@
package io.edurt.datacap.server.entity;
import io.edurt.datacap.spi.FormatType;
import lombok.Data;
import lombok.ToString;
@ -12,4 +13,5 @@ public class ExecuteEntity
private String name;
private String content;
private Map<String, Object> env;
private FormatType format = FormatType.NONE;
}

6
server/src/main/java/io/edurt/datacap/server/service/impl/ExecuteServiceImpl.java
View File

@ -53,9 +53,13 @@ public class ExecuteServiceImpl
_configure.setUsername(Optional.ofNullable(entity.getUsername()));
_configure.setPassword(Optional.ofNullable(entity.getPassword()));
_configure.setEnv(Optional.ofNullable(configure.getEnv()));
_configure.setFormat(configure.getFormat());
plugin.connect(_configure);
io.edurt.datacap.spi.model.Response response = plugin.execute(configure.getContent());
plugin.destroy();
return Response.success(response);
if (response.getIsSuccessful()) {
return Response.success(response);
}
return Response.failure(ServiceState.PLUGIN_EXECUTE_FAILED, response.getMessage());
}
}

2
server/src/test/java/io/edurt/datacap/server/BaseParamTest.java
View File

@ -26,7 +26,7 @@ public class BaseParamTest
{
ExecuteEntity configure = new ExecuteEntity();
configure.setName("MySQL");
configure.setContent("SHOW DATABASES");
configure.setContent("SELECT * FROM information_schema.TABLES LIMIT 100");
return configure;
}
}

24
server/src/test/java/io/edurt/datacap/server/controller/ExecuteControllerTest.java
View File

@ -2,6 +2,8 @@ package io.edurt.datacap.server.controller;
import io.edurt.datacap.server.BaseParamTest;
import io.edurt.datacap.server.common.JSON;
import io.edurt.datacap.server.entity.ExecuteEntity;
import io.edurt.datacap.spi.FormatType;
import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
@ -49,7 +51,27 @@ public class ExecuteControllerTest
.contentType(MediaType.APPLICATION_JSON)
.content(JSON.objectmapper.writeValueAsString(BaseParamTest.builderExecute())))
.andExpect(MockMvcResultMatchers.status().isOk())
.andExpect(MockMvcResultMatchers.jsonPath("$.code").value(200))
.andExpect(MockMvcResultMatchers.jsonPath("$.code").exists())
.andDo(MockMvcResultHandlers.print())
.andReturn();
log.info(mvcResult.getResponse().getContentAsString());
}
@Test
@SqlGroup(value = {
@Sql(value = "classpath:schema/source.sql"),
@Sql(value = "classpath:data/source.sql")
})
public void executeFormatJson()
throws Exception
{
ExecuteEntity entity = BaseParamTest.builderExecute();
entity.setFormat(FormatType.JSON);
MvcResult mvcResult = mockMvc.perform(MockMvcRequestBuilders.post("/api/v1/execute")
.contentType(MediaType.APPLICATION_JSON)
.content(JSON.objectmapper.writeValueAsString(entity)))
.andExpect(MockMvcResultMatchers.status().isOk())
.andExpect(MockMvcResultMatchers.jsonPath("$.code").exists())
.andDo(MockMvcResultHandlers.print())
.andReturn();
log.info(mvcResult.getResponse().getContentAsString());

6
server/src/test/java/io/edurt/datacap/server/service/ExecuteServiceTest.java
View File

@ -1,6 +1,8 @@
package io.edurt.datacap.server.service;
import io.edurt.datacap.server.entity.ExecuteEntity;
import org.apache.commons.lang3.ObjectUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -25,7 +27,7 @@ public class ExecuteServiceTest
{
ExecuteEntity configure = new ExecuteEntity();
configure.setName("MySQL");
configure.setContent("SHOW TABLES");
executeService.execute(configure);
configure.setContent("SHOW DATABASES");
Assert.assertTrue(ObjectUtils.isNotEmpty(executeService.execute(configure)));
}
}

12
spi/pom.xml
View File

@ -26,5 +26,17 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>findbugs</artifactId>
</dependency>
</dependencies>
</project>

7
spi/src/main/java/io/edurt/datacap/spi/FormatType.java Normal file
View File

@ -0,0 +1,7 @@
package io.edurt.datacap.spi;
public enum FormatType
{
NONE,
JSON
}

62
spi/src/main/java/io/edurt/datacap/spi/adapter/Adapter.java Normal file
View File

@ -0,0 +1,62 @@
package io.edurt.datacap.spi.adapter;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.model.Response;
import io.edurt.datacap.spi.record.RecordFactory;
import lombok.extern.slf4j.Slf4j;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"},
justification = "I prefer to suppress these FindBugs warnings")
public abstract class Adapter
{
private Object handlerFormatAdapterRecord(FormatType format, List<String> headers, List<Object> columns)
{
return RecordFactory.createRecord(format, headers, columns).convert();
}
public Response handlerJDBCExecute(FormatType format, String content, Connection connection, Response response)
{
if (response.getIsConnected()) {
try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(content)) {
List<String> headers = new ArrayList<>();
List<String> types = new ArrayList<>();
List<Object> columns = new ArrayList<>();
boolean isPresent = true;
while (resultSet.next()) {
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
List<Object> _columns = new ArrayList<>();
for (int i = 1; i <= columnCount; i++) {
if (isPresent) {
headers.add(metaData.getColumnName(i));
types.add(metaData.getColumnTypeName(i));
}
_columns.add(resultSet.getString(i));
}
isPresent = false;
columns.add(handlerFormatAdapterRecord(format, headers, _columns));
}
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setIsSuccessful(Boolean.TRUE);
}
catch (SQLException ex) {
log.error("Execute content failed content {} exception ", content, ex);
response.setIsSuccessful(Boolean.FALSE);
response.setMessage(ex.getMessage());
}
}
return response;
}
}

2
spi/src/main/java/io/edurt/datacap/spi/model/Configure.java
View File

@ -1,5 +1,6 @@
package io.edurt.datacap.spi.model;
import io.edurt.datacap.spi.FormatType;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -20,4 +21,5 @@ public class Configure
private Optional<String> password = Optional.empty();
private Optional<String> database = Optional.empty();
private Optional<Map<String, Object>> env = Optional.empty();
private FormatType format = FormatType.NONE;
}

5
spi/src/main/java/io/edurt/datacap/spi/model/Response.java
View File

@ -15,5 +15,8 @@ public class Response
{
private List<String> headers;
private List<String> types;
private List<List<Object>> columns;
private List<Object> columns;
private Boolean isConnected = Boolean.FALSE;
private Boolean isSuccessful = Boolean.FALSE;
private String message;
}

26
spi/src/main/java/io/edurt/datacap/spi/record/JsonRecord.java Normal file
View File

@ -0,0 +1,26 @@
package io.edurt.datacap.spi.record;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.List;
public class JsonRecord
extends Record
{
protected JsonRecord(List<String> headers, List<Object> columns)
{
super(headers, columns);
}
@Override
public Object convert()
{
ObjectMapper mapper = new ObjectMapper();
ObjectNode node = mapper.createObjectNode();
for (int i = 0; i < headers.size(); i++) {
node.put(headers.get(i), String.valueOf(columns.get(i)));
}
return node;
}
}

23
spi/src/main/java/io/edurt/datacap/spi/record/NoneRecord.java Normal file
View File

@ -0,0 +1,23 @@
package io.edurt.datacap.spi.record;
import java.util.ArrayList;
import java.util.List;
public class NoneRecord
extends Record
{
protected NoneRecord(List<String> headers, List<Object> columns)
{
super(headers, columns);
}
@Override
public Object convert()
{
List<Object> values = new ArrayList<>();
for (int i = 0; i < headers.size(); i++) {
values.add(columns.get(i));
}
return values;
}
}

17
spi/src/main/java/io/edurt/datacap/spi/record/Record.java Normal file
View File

@ -0,0 +1,17 @@
package io.edurt.datacap.spi.record;
import java.util.List;
public abstract class Record
{
protected final List<String> headers;
protected final List<Object> columns;
protected Record(List<String> headers, List<Object> columns)
{
this.headers = headers;
this.columns = columns;
}
public abstract Object convert();
}

23
spi/src/main/java/io/edurt/datacap/spi/record/RecordFactory.java Normal file
View File

@ -0,0 +1,23 @@
package io.edurt.datacap.spi.record;
import io.edurt.datacap.spi.FormatType;
import java.util.List;
public class RecordFactory
{
private RecordFactory()
{}
public static Record createRecord(FormatType format, List<String> headers, List<Object> columns)
{
Record instance;
if (format.equals(FormatType.JSON)) {
instance = new JsonRecord(headers, columns);
}
else {
instance = new NoneRecord(headers, columns);
}
return instance;
}
}

2
web/console-fe/package.json
View File

@ -9,6 +9,8 @@
"lint": "vue-cli-service lint"
},
"dependencies": {
"@antv/s2": "^1.30.0",
"@antv/s2-vue": "^1.3.0",
"ant-design-vue": "^3.2.12",
"axios": "^0.27.2",
"core-js": "^3.8.3",

6
web/console-fe/src/common/HttpCommon.ts
View File

@ -30,7 +30,7 @@ export class HttpCommon
code: data.code,
data: data.data,
message: data.message,
status: true
status: data.status
};
resolve(response);
}, error => {
@ -55,7 +55,7 @@ export class HttpCommon
code: data.code,
data: data.data,
message: data.message,
status: true
status: data.status
};
resolve(response);
}, error => {
@ -80,7 +80,7 @@ export class HttpCommon
code: data.code,
data: data.data,
message: data.message,
status: true
status: data.status
};
resolve(response);
}, error => {

1
web/console-fe/src/model/ExecuteModel.ts
View File

@ -3,4 +3,5 @@ export interface ExecuteModel
name: string;
content: string;
env?: object;
format?: string;
}

84
web/console-fe/src/views/pages/console/ConsoleOnline.vue
View File

@ -1,39 +1,45 @@
<template>
<div class="home">
<a-card size="small">
<template #title>
<a-space :size="8">
<a-select v-model:value="applySource" size="small" style="width: 120px" :options="options2">
<template #suffixIcon>
<meh-outlined class="ant-select-suffix"/>
</template>
</a-select>
<a-button type="primary" size="small" @click="handlerRun()">Run</a-button>
</a-space>
</template>
<div id="editor-section" style="height: 300px"></div>
</a-card>
<a-card title="Result" :style="{ marginTop: '10px' }">
<a-table :dataSource="columns" :columns="headers" />
</a-card>
<div ref="editorContainer">
<a-card size="small">
<template #title>
<a-space :size="8">
<a-select v-model:value="applySource" size="small" style="width: 120px" :options="options2">
<template #suffixIcon>
<meh-outlined class="ant-select-suffix"/>
</template>
</a-select>
<a-button type="primary" size="small" @click="handlerRun()">Run</a-button>
</a-space>
</template>
<div id="editor-section" style="height: 300px"></div>
</a-card>
</div>
<div style="margin-top: 5px;">
<a-card :loading="tableLoading" :body-style="{padding: '2px'}">
<SheetComponent :dataCfg="tableConfigure" :options="tableOptions" :showPagination="true" sheetType="table"/>
</a-card>
</div>
</div>
</template>
<script lang="ts">
import {defineComponent, onMounted, ref} from "vue";
import * as monaco from "monaco-editor";
import {SelectProps} from "ant-design-vue";
import {message, SelectProps} from "ant-design-vue";
import {ExecuteModel} from "@/model/ExecuteModel";
import {ExecuteService} from "@/services/ExecuteService";
import "@antv/s2-vue/dist/style.min.css";
import "ant-design-vue/dist/antd.css";
import {SheetComponent} from "@antv/s2-vue";
let codeEditor!: monaco.editor.IStandaloneCodeEditor;
export default defineComponent({
name: "DashboardConsoleView",
components: {},
components: {SheetComponent},
setup()
{
function initEditor()
{
const container: HTMLElement = document.getElementById("editor-section") as HTMLElement;
@ -65,33 +71,47 @@ export default defineComponent({
{
return {
applySource: '',
headers: [] as any[],
columns: []
tableConfigure: {},
tableOptions: {},
tableLoading: false
}
},
methods: {
handlerRun()
{
this.tableLoading = true;
const editorContainer: HTMLElement = this.$refs.editorContainer as HTMLElement;
const configure: ExecuteModel = {
name: this.applySource,
content: codeEditor.getValue()
content: codeEditor.getValue(),
format: "JSON"
};
new ExecuteService()
.execute(configure)
.then((response) => {
if (response.status) {
const _headers = response.data.headers;
const _types = response.data.types;
for (let i = 0; i < _headers.length; i++) {
this.headers.push({
title: _headers[i] + '(' + _types[i] + ')',
name: _headers[i],
dataIndex: _headers[i],
key: _headers[i],
});
this.tableConfigure = {
fields: {
columns: response.data.headers
},
data: response.data.columns
};
this.tableOptions = {
width: editorContainer.offsetWidth - 8,
height: 340,
pagination: {
pageSize: 10,
current: 1,
},
showSeriesNumber: true,
}
}
})
else {
message.error(response.message);
}
}).finally(() => {
this.tableLoading = false;
})
}
}
});