test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-12-02 03:57:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Core] [SPI] Fixed file spi

Browse Source
This commit is contained in:
qianmoQ 2024-06-10 00:04:58 +08:00
parent 45eb6f405e
commit a0c0171d2c
45 changed files with 159 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/datacap-common/src/main/java/io/edurt/datacap/common/response/JwtResponse.java
View File

@ -7,21 +7,16 @@ import lombok.ToString;
import java.util.Collections;
import java.util.List;
@Getter
@ToString
@NoArgsConstructor
public class JwtResponse
{
@Getter
private String type = "Bearer";
@Getter
private String token;
@Getter
private Long id;
@Getter
private String username;
@Getter
private List<String> roles;
@Getter
private String avatar;
public JwtResponse(String accessToken, Long id, String username, List<String> roles, String avatar)

4
core/datacap-server/pom.xml
View File

@ -410,6 +410,10 @@
<artifactId>datacap-notify-dingtalk</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-file-spi</artifactId>
</dependency>
</dependencies>
<build>

4
core/datacap-server/src/main/java/io/edurt/datacap/server/configure/PluginConfigure.java
View File

@ -3,6 +3,7 @@ package io.edurt.datacap.server.configure;
import com.google.inject.Guice;
import com.google.inject.Injector;
import io.edurt.datacap.executor.ExecutorManager;
import io.edurt.datacap.file.FileManager;
import io.edurt.datacap.fs.FsManager;
import io.edurt.datacap.notify.NotifyManager;
import io.edurt.datacap.parser.ParserManager;
@ -22,6 +23,7 @@ public class PluginConfigure
new ParserManager(),
new ScheduleManager(),
new ExecutorManager(),
new NotifyManager());
new NotifyManager(),
new FileManager());
}
}

3
core/datacap-server/src/test/java/io/edurt/datacap/server/controller/ExecuteControllerTest.java
View File

@ -4,7 +4,6 @@ import io.edurt.datacap.common.utils.JsonUtils;
import io.edurt.datacap.server.BaseParamTest;
import io.edurt.datacap.server.controller.user.ExecuteController;
import io.edurt.datacap.service.entity.ExecuteEntity;
import io.edurt.datacap.spi.FormatType;
import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
@ -53,7 +52,7 @@ public class ExecuteControllerTest
throws Exception
{
ExecuteEntity entity = BaseParamTest.builderExecute();
entity.setFormat(FormatType.JSON);
entity.setFormat("Json");
entity.setName("MySQL1");
MvcResult mvcResult = mockMvc.perform(MockMvcRequestBuilders.post("/api/v1/execute").contentType(MediaType.APPLICATION_JSON).content(JsonUtils.objectmapper.writeValueAsString(entity))).andExpect(MockMvcResultMatchers.status().isOk()).andExpect(MockMvcResultMatchers.jsonPath("$.code").exists()).andDo(MockMvcResultHandlers.print()).andReturn();
log.info(mvcResult.getResponse().getContentAsString());

3
core/datacap-service/src/main/java/io/edurt/datacap/service/body/ExecuteDslBody.java
View File

@ -2,7 +2,6 @@ package io.edurt.datacap.service.body;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.common.sql.configure.SqlBody;
import io.edurt.datacap.spi.FormatType;
import lombok.Data;
import lombok.ToString;
@ -16,5 +15,5 @@ public class ExecuteDslBody
private String name;
private SqlBody configure;
private Map<String, Object> env;
private FormatType format = FormatType.NONE;
private String format = "";
}

3
core/datacap-service/src/main/java/io/edurt/datacap/service/body/ExportBody.java
View File

@ -1,6 +1,5 @@
package io.edurt.datacap.service.body;
import io.edurt.datacap.spi.FormatType;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -12,6 +11,6 @@ import lombok.ToString;
@AllArgsConstructor
public class ExportBody
{
private FormatType format;
private String format;
private Long count;
}

5
core/datacap-service/src/main/java/io/edurt/datacap/service/common/ConfigureUtils.java
View File

@ -18,7 +18,6 @@ import io.edurt.datacap.service.configure.IConfigureFieldName;
import io.edurt.datacap.service.configure.IConfigureFieldType;
import io.edurt.datacap.service.configure.IConfigurePipelineType;
import io.edurt.datacap.service.entity.SourceEntity;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.model.Configure;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -86,7 +85,7 @@ public class ConfigureUtils
break;
}
});
configure.setFormat(FormatType.JSON);
configure.setFormat("Json");
return configure;
}
@ -241,8 +240,6 @@ public class ConfigureUtils
* Converts the given properties based on the source entity, fields, configure, environment, pipeline type, executor, query, and field body.
*
* @param entity the source entity
* @param fields the list of fields
* @param configure the properties to configure
* @param environment the environment
* @param pipelineType the pipeline type
* @param executor the executor

3
core/datacap-service/src/main/java/io/edurt/datacap/service/entity/ExecuteEntity.java
View File

@ -2,7 +2,6 @@ package io.edurt.datacap.service.entity;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.service.enums.QueryMode;
import io.edurt.datacap.spi.FormatType;
import lombok.Data;
import lombok.ToString;
@ -16,7 +15,7 @@ public class ExecuteEntity
private String name;
private String content;
private Map<String, Object> env;
private FormatType format = FormatType.NONE;
private String format = "Json";
private int limit;
private QueryMode mode;
}

3
core/datacap-service/src/main/java/io/edurt/datacap/service/entity/SourceEntity.java
View File

@ -8,7 +8,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.common.utils.JsonUtils;
import io.edurt.datacap.service.configure.IConfigure;
import io.edurt.datacap.service.configure.IConfigureExecutor;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.model.Configure;
import lombok.Getter;
import lombok.NoArgsConstructor;
@ -160,7 +159,7 @@ public class SourceEntity
configure.setDatabase(database);
configure.setSsl(Optional.ofNullable(this.getSsl()));
configure.setEnv(Optional.ofNullable(this.getConfigures()));
configure.setFormat(FormatType.JSON);
configure.setFormat("Json");
return configure;
}
}

4
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java
View File

@ -57,7 +57,6 @@ import io.edurt.datacap.service.repository.DataSetRepository;
import io.edurt.datacap.service.repository.DatasetHistoryRepository;
import io.edurt.datacap.service.security.UserDetailsService;
import io.edurt.datacap.service.service.DataSetService;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginType;
import io.edurt.datacap.spi.model.Configure;
@ -258,7 +257,6 @@ public class DataSetServiceImpl
targetConfigure.setUsername(Optional.ofNullable(initializerConfigure.getDataSetConfigure().getUsername()));
targetConfigure.setPassword(Optional.ofNullable(initializerConfigure.getDataSetConfigure().getPassword()));
targetConfigure.setDatabase(Optional.ofNullable(database));
targetConfigure.setFormat(FormatType.JSON);
plugin.connect(targetConfigure);
Response response = plugin.execute(sql);
response.setContent(sql);
@ -730,7 +728,7 @@ public class DataSetServiceImpl
SqlColumn.builder().column("active").operator(SqlOperator.EQ).value("1").build()))
.condition(" AND ")
.build());
configure.setFormat(FormatType.NONE);
configure.setFormat("");
plugin.connect(configure);
Response outputResponse = plugin.execute(builder.getSql());
if (outputResponse.getIsSuccessful()) {

1
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/ExecuteServiceImpl.java
View File

@ -76,6 +76,7 @@ public class ExecuteServiceImpl
_configure.setEnv(Optional.ofNullable(entity.getConfigures()));
_configure.setFormat(configure.getFormat());
_configure.setUsedConfig(entity.isUsedConfig());
_configure.setInjector(injector);
if (entity.isUsedConfig()) {
_configure.setUsername(Optional.of(entity.getUser().getUsername()));
String configHome = environment.getProperty("datacap.config.data");

2
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/SourceServiceImpl.java
View File

@ -44,7 +44,6 @@ import io.edurt.datacap.service.repository.metadata.DatabaseRepository;
import io.edurt.datacap.service.repository.metadata.TableRepository;
import io.edurt.datacap.service.security.UserDetailsService;
import io.edurt.datacap.service.service.SourceService;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
@ -168,7 +167,6 @@ public class SourceServiceImpl
_configure.setDatabase(_database);
_configure.setEnv(Optional.ofNullable(configure.getConfigures()));
_configure.setSsl(Optional.ofNullable(configure.getSsl()));
_configure.setFormat(FormatType.JSON);
plugin.connect(_configure);
io.edurt.datacap.spi.model.Response response = plugin.execute(plugin.validator());
plugin.destroy();

43
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/TableServiceImpl.java
View File

@ -19,6 +19,7 @@ import io.edurt.datacap.service.body.ExportBody;
import io.edurt.datacap.service.body.TableBody;
import io.edurt.datacap.service.body.TableFilter;
import io.edurt.datacap.service.common.PluginUtils;
import io.edurt.datacap.service.entity.BaseEntity;
import io.edurt.datacap.service.entity.ColumnEntity;
import io.edurt.datacap.service.entity.DatabaseEntity;
import io.edurt.datacap.service.entity.SourceEntity;
@ -30,7 +31,6 @@ import io.edurt.datacap.service.repository.metadata.DatabaseRepository;
import io.edurt.datacap.service.repository.metadata.TableRepository;
import io.edurt.datacap.service.security.UserDetailsService;
import io.edurt.datacap.service.service.TableService;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Pagination;
@ -56,6 +56,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
@ -248,7 +249,9 @@ public class TableServiceImpl
TableBuilder.Companion.COLUMNS(configure.getColumns().stream().map(item -> item.toColumnVar()).collect(Collectors.toList()));
String sql = TableBuilder.Companion.SQL();
log.info("Create table sql \n {} \n on database [ {} ]", sql, database.getName());
plugin.connect(source.toConfigure());
Configure pConfigure = source.toConfigure();
pConfigure.setInjector(injector);
plugin.connect(pConfigure);
Response response = plugin.execute(sql);
response.setContent(sql);
plugin.destroy();
@ -299,7 +302,9 @@ public class TableServiceImpl
.build();
}
else {
plugin.connect(source.toConfigure());
Configure pConfigure = source.toConfigure();
pConfigure.setInjector(injector);
plugin.connect(pConfigure);
response = plugin.execute(atomicReference.get());
response.setContent(atomicReference.get());
plugin.destroy();
@ -324,13 +329,14 @@ public class TableServiceImpl
List<SqlColumn> columns = Lists.newArrayList();
int totalRows = Integer.parseInt(table.getRows());
Configure countConfigure = source.toConfigure();
countConfigure.setFormat(FormatType.NONE);
countConfigure.setFormat("");
countConfigure.setInjector(injector);
plugin.connect(countConfigure);
SqlBody countBody = SqlBody.builder()
.type(SqlType.SELECT)
.database(table.getDatabase().getName())
.table(table.getName())
.columns(Arrays.asList(SqlColumn.builder()
.columns(Collections.singletonList(SqlColumn.builder()
.column("COUNT(1)")
.build()))
.build();
@ -396,7 +402,9 @@ public class TableServiceImpl
SqlBuilder builder = new SqlBuilder(body);
String sql = builder.getSql();
plugin.connect(source.toConfigure());
Configure pConfigure = source.toConfigure();
pConfigure.setInjector(injector);
plugin.connect(pConfigure);
Response response = plugin.execute(sql);
response.setContent(sql);
plugin.destroy();
@ -422,14 +430,15 @@ public class TableServiceImpl
{
try {
Configure updateConfigure = source.toConfigure();
updateConfigure.setFormat(FormatType.NONE);
updateConfigure.setFormat("");
updateConfigure.setInjector(injector);
plugin.connect(updateConfigure);
List<String> allSql = Lists.newArrayList();
// Gets the auto-increment column for the current row
List<String> autoIncrementColumns = table.getColumns()
.stream()
.filter(v -> v.getExtra().toLowerCase().contains("auto_increment"))
.map(v -> v.getName())
.map(BaseEntity::getName)
.collect(Collectors.toList());
configure.getNewColumns().forEach(v -> {
List<SqlColumn> columns = Lists.newArrayList();
@ -479,7 +488,8 @@ public class TableServiceImpl
{
try {
Configure updateConfigure = source.toConfigure();
updateConfigure.setFormat(FormatType.NONE);
updateConfigure.setFormat("");
updateConfigure.setInjector(injector);
plugin.connect(updateConfigure);
List<String> allSql = Lists.newArrayList();
configure.getColumns().forEach(v -> {
@ -516,7 +526,8 @@ public class TableServiceImpl
{
try {
Configure updateConfigure = source.toConfigure();
updateConfigure.setFormat(FormatType.NONE);
updateConfigure.setFormat("");
updateConfigure.setInjector(injector);
plugin.connect(updateConfigure);
List<String> allSql = Lists.newArrayList();
configure.getColumns().forEach(v -> {
@ -549,7 +560,8 @@ public class TableServiceImpl
{
try {
Configure alterConfigure = source.toConfigure();
alterConfigure.setFormat(FormatType.NONE);
alterConfigure.setFormat("");
alterConfigure.setInjector(injector);
plugin.connect(alterConfigure);
SqlBody body = SqlBody.builder()
.type(SqlType.ALTER)
@ -577,7 +589,8 @@ public class TableServiceImpl
{
try {
Configure alterConfigure = source.toConfigure();
alterConfigure.setFormat(FormatType.NONE);
alterConfigure.setFormat("");
alterConfigure.setInjector(injector);
plugin.connect(alterConfigure);
SqlBody body = SqlBody.builder()
.type(SqlType.SHOW)
@ -604,7 +617,8 @@ public class TableServiceImpl
{
try {
Configure alterConfigure = source.toConfigure();
alterConfigure.setFormat(FormatType.NONE);
alterConfigure.setFormat("");
alterConfigure.setInjector(injector);
plugin.connect(alterConfigure);
SqlBody body = SqlBody.builder()
.type(SqlType.TRUNCATE)
@ -631,7 +645,8 @@ public class TableServiceImpl
{
try {
Configure alterConfigure = source.toConfigure();
alterConfigure.setFormat(FormatType.NONE);
alterConfigure.setFormat("");
alterConfigure.setInjector(injector);
plugin.connect(alterConfigure);
SqlBody body = SqlBody.builder()
.type(SqlType.DROP)

3
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/TemplateSqlServiceImpl.java
View File

@ -17,7 +17,6 @@ import io.edurt.datacap.service.repository.SourceRepository;
import io.edurt.datacap.service.repository.TemplateSqlRepository;
import io.edurt.datacap.service.service.ExecuteService;
import io.edurt.datacap.service.service.TemplateSqlService;
import io.edurt.datacap.spi.FormatType;
import org.apache.commons.lang3.EnumUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -164,7 +163,7 @@ public class TemplateSqlServiceImpl
});
}
executeConfigure.setContent(content[0]);
executeConfigure.setFormat(FormatType.JSON);
executeConfigure.setFormat("Json");
executeConfigure.setName(String.valueOf(configure.getSourceId()));
}
catch (JsonProcessingException e) {

11
core/datacap-spi/pom.xml
View File

@ -69,6 +69,17 @@
<artifactId>mysql-connector-java</artifactId>
<scope>test</scope>
</dependency>
<!-- Support file convert -->
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-file-txt</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-file-json</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>

8
core/datacap-spi/src/main/java/io/edurt/datacap/spi/FormatType.java
View File

@ -1,8 +0,0 @@
package io.edurt.datacap.spi;
public enum FormatType
{
NONE,
JSON,
CSV
}

21
core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/Adapter.java
View File

@ -1,7 +1,10 @@
package io.edurt.datacap.spi.adapter;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.formatter.FormatterFactory;
import com.google.common.base.Preconditions;
import com.google.inject.Injector;
import io.edurt.datacap.file.FileFilter;
import io.edurt.datacap.file.model.FileRequest;
import io.edurt.datacap.file.model.FileResponse;
import io.edurt.datacap.spi.model.Response;
import java.util.List;
@ -10,8 +13,18 @@ public interface Adapter
{
Response handlerExecute(String content);
default Object handlerFormatter(FormatType format, List<String> headers, List<Object> columns)
default List<Object> handlerFormatter(Injector injector, String format, List<String> headers, List<Object> columns)
{
return FormatterFactory.createFormatter(format, headers, columns).formatter();
return FileFilter.findNotify(injector, format)
.map(file -> {
FileRequest request = new FileRequest();
request.setHeaders(headers);
request.setColumns(columns);
FileResponse response = file.format(request);
Preconditions.checkArgument(Boolean.TRUE.equals(response.getSuccessful()), "Not support format: %s", format);
return response.getColumns();
})
.orElseThrow(() -> new IllegalArgumentException("Unsupported format: " + format));
}
}

9
core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/HttpAdapter.java
View File

@ -1,14 +1,10 @@
package io.edurt.datacap.spi.adapter;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.connection.Connection;
import io.edurt.datacap.spi.formatter.FormatterFactory;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@Slf4j
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"},
justification = "I prefer to suppress these FindBugs warnings")
@ -22,11 +18,6 @@ public class HttpAdapter
this.connection = connection;
}
public Object handlerFormatter(FormatType format, List<String> headers, List<Object> columns)
{
return FormatterFactory.createFormatter(format, headers, columns).formatter();
}
@Override
public Response handlerExecute(String content)
{

16
core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/JdbcAdapter.java
View File

@ -1,12 +1,10 @@
package io.edurt.datacap.spi.adapter;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.column.Column;
import io.edurt.datacap.spi.column.JdbcColumn;
import io.edurt.datacap.spi.connection.JdbcConfigure;
import io.edurt.datacap.spi.connection.JdbcConnection;
import io.edurt.datacap.spi.formatter.FormatterFactory;
import io.edurt.datacap.spi.model.Response;
import io.edurt.datacap.spi.model.Time;
import lombok.extern.slf4j.Slf4j;
@ -22,8 +20,7 @@ import java.util.List;
import java.util.Objects;
@Slf4j
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"},
justification = "I prefer to suppress these FindBugs warnings")
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"})
public class JdbcAdapter
implements Adapter
{
@ -34,11 +31,6 @@ public class JdbcAdapter
this.connection = connection;
}
public Object handlerFormatter(FormatType format, List<String> headers, List<Object> columns)
{
return FormatterFactory.createFormatter(format, headers, columns).formatter();
}
@Override
public Response handlerExecute(String content)
{
@ -66,7 +58,7 @@ public class JdbcAdapter
for (int i = 1; i <= columnCount; i++) {
_columns.add(jdbcColumn.mappingColumnData(metaData.getColumnTypeName(i), i));
}
columns.add(handlerFormatter(configure.getFormat(), headers, _columns));
columns.add(_columns);
}
}
catch (SQLException tryUpdateEx) {
@ -86,7 +78,7 @@ public class JdbcAdapter
}
_columns.add(count);
connection.commit();
columns.add(handlerFormatter(configure.getFormat(), headers, _columns));
columns.add(_columns);
}
catch (SQLException updateEx) {
try {
@ -106,7 +98,7 @@ public class JdbcAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
response.setIsSuccessful(Boolean.TRUE);
}
}

9
core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/NativeAdapter.java
View File

@ -1,16 +1,12 @@
package io.edurt.datacap.spi.adapter;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.connection.Connection;
import io.edurt.datacap.spi.formatter.FormatterFactory;
import io.edurt.datacap.spi.model.Response;
import io.edurt.datacap.spi.parser.SqlParser;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@Slf4j
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"},
justification = "I prefer to suppress these FindBugs warnings")
@ -33,11 +29,6 @@ public class NativeAdapter
this.parser = parser;
}
public Object handlerFormatter(FormatType format, List<String> headers, List<Object> columns)
{
return FormatterFactory.createFormatter(format, headers, columns).formatter();
}
@Override
public Response handlerExecute(String content)
{

21
core/datacap-spi/src/main/java/io/edurt/datacap/spi/formatter/CsvFormatter.java
View File

@ -1,21 +0,0 @@
package io.edurt.datacap.spi.formatter;
import java.util.List;
import java.util.stream.Collectors;
public class CsvFormatter
extends Formatter
{
private CharSequence sequence = ",";
protected CsvFormatter(List<String> headers, List<Object> columns)
{
super(headers, columns);
}
@Override
public Object formatter()
{
return String.join(sequence, this.columns.stream().map(String::valueOf).collect(Collectors.toList()));
}
}

17
core/datacap-spi/src/main/java/io/edurt/datacap/spi/formatter/Formatter.java
View File

@ -1,17 +0,0 @@
package io.edurt.datacap.spi.formatter;
import java.util.List;
public abstract class Formatter
{
protected final List<String> headers;
protected final List<Object> columns;
protected Formatter(List<String> headers, List<Object> columns)
{
this.headers = headers;
this.columns = columns;
}
public abstract Object formatter();
}

26
core/datacap-spi/src/main/java/io/edurt/datacap/spi/formatter/FormatterFactory.java
View File

@ -1,26 +0,0 @@
package io.edurt.datacap.spi.formatter;
import io.edurt.datacap.spi.FormatType;
import java.util.List;
public class FormatterFactory
{
private FormatterFactory()
{}
public static Formatter createFormatter(FormatType format, List<String> headers, List<Object> columns)
{
Formatter instance;
if (format.equals(FormatType.JSON)) {
instance = new JsonFormatter(headers, columns);
}
else if (format.equals(FormatType.CSV)) {
instance = new CsvFormatter(headers, columns);
}
else {
instance = new NoneFormatter(headers, columns);
}
return instance;
}
}

26
core/datacap-spi/src/main/java/io/edurt/datacap/spi/formatter/JsonFormatter.java
View File

@ -1,26 +0,0 @@
package io.edurt.datacap.spi.formatter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.List;
public class JsonFormatter
extends Formatter
{
protected JsonFormatter(List<String> headers, List<Object> columns)
{
super(headers, columns);
}
@Override
public Object formatter()
{
ObjectMapper mapper = new ObjectMapper();
ObjectNode node = mapper.createObjectNode();
for (int i = 0; i < headers.size(); i++) {
node.put(headers.get(i), String.valueOf(columns.get(i)));
}
return node;
}
}

23
core/datacap-spi/src/main/java/io/edurt/datacap/spi/formatter/NoneFormatter.java
View File

@ -1,23 +0,0 @@
package io.edurt.datacap.spi.formatter;
import java.util.ArrayList;
import java.util.List;
public class NoneFormatter
extends Formatter
{
protected NoneFormatter(List<String> headers, List<Object> columns)
{
super(headers, columns);
}
@Override
public Object formatter()
{
List<Object> values = new ArrayList<>();
for (int i = 0; i < headers.size(); i++) {
values.add(columns.get(i));
}
return values;
}
}

7
core/datacap-spi/src/main/java/io/edurt/datacap/spi/model/Configure.java
View File

@ -1,6 +1,7 @@
package io.edurt.datacap.spi.model;
import io.edurt.datacap.spi.FormatType;
import com.google.inject.Injector;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -13,8 +14,10 @@ import java.util.Optional;
@ToString
@NoArgsConstructor
@AllArgsConstructor
@SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"})
public class Configure
{
private Injector injector;
private String host;
private Integer port;
private Optional<String> username = Optional.empty();
@ -23,7 +26,7 @@ public class Configure
private Optional<String> version = Optional.empty();
private Optional<Map<String, Object>> env = Optional.empty();
private Optional<Boolean> ssl = Optional.empty();
private FormatType format = FormatType.NONE;
private String format = "Json";
// if `to`: skip
private Optional<String> query = Optional.empty();
// Support for custom upload configuration plugins

32
core/datacap-spi/src/test/java/io/edurt/datacap/spi/formatter/CsvFormatterTest.java
View File

@ -1,32 +0,0 @@
package io.edurt.datacap.spi.formatter;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
public class CsvFormatterTest
{
private List<String> headers;
private List<Object> columns;
@Before
public void before()
{
this.headers = new ArrayList<>();
this.headers.add("name");
this.headers.add("age");
this.columns = new ArrayList<>();
this.columns.add("datacap");
this.columns.add("1");
}
@Test
public void formatter()
{
Formatter formatter = new CsvFormatter(this.headers, this.columns);
Assert.assertEquals(formatter.formatter(), "datacap,1");
}
}

2
core/datacap-ui/src/views/pages/admin/query/QueryHome.vue
View File

@ -216,7 +216,7 @@ export default defineComponent({
handlerInitialize()
{
this.createEditor()
this.queryConfigure.configure = { name: this.selectSource.id as string, content: '', mode: 'ADHOC', format: 'JSON' }
this.queryConfigure.configure = { name: this.selectSource.id as string, content: '', mode: 'ADHOC', format: 'Json' }
const params = this.$route.params
if (params) {
const code = params.code

2
file/datacap-file-json/src/main/kotlin/io/edurt/datacap/file/json/JsonFile.kt
View File

@ -38,7 +38,7 @@ class JsonFile : File
when (column)
{
is List<*> -> jsonNode.putPOJO(header, column[headerIndex])
else -> jsonNode.putPOJO(header, column.toString())
else -> jsonNode.putPOJO(header, column)
}
}
columns.add(jsonNode)

6
plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBAdapter.java
View File

@ -1,5 +1,6 @@
package io.edurt.datacap.plugin.http.ceresdb;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.adapter.HttpAdapter;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
@ -21,6 +22,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@Slf4j
@SuppressFBWarnings(value = {"REC_CATCH_EXCEPTION"})
public class CeresDBAdapter
extends HttpAdapter
{
@ -66,7 +68,7 @@ public class CeresDBAdapter
.stream()
.map(Map.Entry::getValue)
.collect(Collectors.toList());
columns.add(handlerFormatter(configure.getFormat(), headers, _columns));
columns.add(_columns);
}
}
else {
@ -82,7 +84,7 @@ public class CeresDBAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

4
plugin/datacap-http-clickhouse/src/main/java/io/edurt/datacap/plugin/http/clickhouse/ClickHouseAdapter.java
View File

@ -58,7 +58,7 @@ public class ClickHouseAdapter
types.addAll(Arrays.asList(data[i].split("\t")));
}
else {
columns.add(handlerFormatter(httpConfigure.getFormat(), headers, Arrays.asList(data[i].split("\t"))));
columns.add(Arrays.asList(data[i].split("\t")));
}
}
response.setIsSuccessful(Boolean.TRUE);
@ -71,7 +71,7 @@ public class ClickHouseAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(httpConfigure.getInjector(), httpConfigure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

2
plugin/datacap-http-clickhouse/src/test/java/io/edurt/datacap/plugin/http/clickhouse/ClickHousePluginTest.java
View File

@ -4,7 +4,6 @@ import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.model.Configure;
import org.junit.Assert;
@ -26,7 +25,6 @@ public class ClickHousePluginTest
configure = new Configure();
configure.setHost("127.0.0.1");
configure.setPort(8123);
configure.setFormat(FormatType.JSON);
}
@Test

4
plugin/datacap-http-cratedb/src/main/java/io/edurt/datacap/plugin/http/cratedb/CrateDBAdapter.java
View File

@ -59,7 +59,7 @@ public class CrateDBAdapter
types.addAll(applyResponse.getTypes());
for (Object column : applyResponse.getColumns()) {
List<Object> _columns = (List<Object>) column;
columns.add(handlerFormatter(httpConfigure.getFormat(), headers, _columns));
columns.add(_columns);
}
}
response.setIsSuccessful(Boolean.TRUE);
@ -72,7 +72,7 @@ public class CrateDBAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(httpConfigure.getInjector(), httpConfigure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

2
plugin/datacap-http-cratedb/src/test/java/io/edurt/datacap/plugin/http/cratedb/CrateDBPluginTest.java
View File

@ -4,7 +4,6 @@ import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.model.Configure;
import org.junit.Assert;
@ -28,7 +27,6 @@ public class CrateDBPluginTest
configure.setPort(4200);
configure.setUsername(Optional.of("crate"));
configure.setPassword(Optional.of(""));
configure.setFormat(FormatType.JSON);
}
@Test

4
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/GreptimeDBAdapter.java
View File

@ -66,7 +66,7 @@ public class GreptimeDBAdapter
});
}
records.getRows()
.forEach(record -> columns.add(handlerFormatter(configure.getFormat(), headers, record)));
.forEach(record -> columns.add(record));
}
else {
response.setIsSuccessful(Boolean.FALSE);
@ -81,7 +81,7 @@ public class GreptimeDBAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

4
plugin/datacap-http-questdb/src/main/java/io/edurt/datacap/plugin/http/questdb/QuestDBAdapter.java
View File

@ -63,7 +63,7 @@ public class QuestDBAdapter
});
}
requestResponse.getDataset()
.forEach(record -> columns.add(handlerFormatter(configure.getFormat(), headers, record)));
.forEach(columns::add);
}
else {
response.setIsSuccessful(Boolean.FALSE);
@ -78,7 +78,7 @@ public class QuestDBAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

6
plugin/datacap-jdbc-kyuubi/src/main/java/io/edurt/datacap/plugin/jdbc/kyuubi/KyuubiAdapter.java
View File

@ -78,7 +78,7 @@ public class KyuubiAdapter
_columns.add(jdbcColumn.mappingColumnData(metaData.getColumnTypeName(i), i));
}
isPresent = false;
columns.add(handlerFormatter(configure.getFormat(), headers, _columns));
columns.add(_columns);
}
}
catch (SQLException tryUpdateEx) {
@ -88,7 +88,7 @@ public class KyuubiAdapter
types.add(Integer.class.getSimpleName());
List<Object> _columns = new ArrayList<>();
_columns.add(statement.executeUpdate());
columns.add(handlerFormatter(configure.getFormat(), headers, _columns));
columns.add(_columns);
}
catch (SQLException updateEx) {
throw new SQLException(updateEx);
@ -101,7 +101,7 @@ public class KyuubiAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
response.setIsSuccessful(Boolean.TRUE);
}
}

4
plugin/datacap-jdbc-redis/src/main/java/io/edurt/datacap/plugin/jdbc/redis/RedisAdapter.java
View File

@ -57,11 +57,11 @@ public class RedisAdapter
_columns.add(jdbcColumn.mappingColumnData(metaData.getColumnTypeName(i), i));
}
isPresent = false;
columns.add(handlerFormatter(configure.getFormat(), headers, _columns));
columns.add(_columns);
}
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
response.setIsSuccessful(Boolean.TRUE);
}
catch (SQLException ex) {

6
plugin/datacap-native-alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossAdapter.java
View File

@ -57,11 +57,11 @@ public class AliossAdapter
types.add("String");
ObjectListing objectListing = client.listObjects(configure.getDatabase().get(), sqlBase.getTable());
objectListing.getObjectSummaries()
.forEach(column -> columns.add(handlerFormatter(configure.getFormat(), headers, Collections.singletonList(column.getKey()))));
.forEach(column -> columns.add(Collections.singletonList(column.getKey())));
response.setIsSuccessful(Boolean.TRUE);
}
else {
Preconditions.checkArgument(!sqlBase.isSuccessful(), sqlBase.getMessage());
Preconditions.checkArgument(true, sqlBase.getMessage());
}
}
catch (Exception ex) {
@ -72,7 +72,7 @@ public class AliossAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

4
plugin/datacap-native-hdfs/src/main/kotlin/io/edurt/datacap/natived/hdfs/HdfsAdapter.kt
View File

@ -64,7 +64,7 @@ class HdfsAdapter : NativeAdapter
}
types.add("String")
this.adapter(configuration, sqlBase)
.forEach { column -> columns.add(handlerFormatter(configure.format, headers, Collections.singletonList(column) as List<Any>?)) }
.forEach { column -> columns.add(Collections.singletonList(column)) }
response.isSuccessful = Boolean.TRUE
}
else
@ -89,7 +89,7 @@ class HdfsAdapter : NativeAdapter
{
response.headers = headers
response.types = types
response.columns = columns
response.columns = handlerFormatter(configure.injector, configure.format, headers, columns)
}
}
processorTime.end = Date().time

4
plugin/datacap-native-kafka/src/main/java/io/edurt/datacap/plugin/natived/kafka/KafkaAdapter.java
View File

@ -60,7 +60,7 @@ public class KafkaAdapter
}
types.add("String");
this.adapter(client, sqlBase)
.forEach(column -> columns.add(handlerFormatter(configure.getFormat(), headers, Collections.singletonList(column))));
.forEach(column -> columns.add(Collections.singletonList(column)));
response.setIsSuccessful(Boolean.TRUE);
}
else {
@ -76,7 +76,7 @@ public class KafkaAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

11
plugin/datacap-native-redis/src/main/java/io/edurt/datacap/plugin/natived/redis/RedisAdapter.java
View File

@ -11,9 +11,8 @@ import redis.clients.jedis.Connection;
import redis.clients.jedis.Protocol;
import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
@ -55,7 +54,7 @@ public class RedisAdapter
}
cmdParam.add(commands[i]);
}
String[] cmdParamArr = cmdParam.toArray(new String[]{});
String[] cmdParamArr = cmdParam.toArray(new String[] {});
method.invoke(client, cmd, cmdParamArr);
Object body = client.getOne();
headers.add(commands[1]);
@ -63,11 +62,11 @@ public class RedisAdapter
if (body instanceof List) {
List<Object> bodySplit = ((List) body);
for (Object obj : bodySplit) {
columns.add(handlerFormatter(configure.getFormat(), headers, Arrays.asList(new String((byte[]) obj, Charset.forName("UTF-8")))));
columns.add(List.of(new String((byte[]) obj, StandardCharsets.UTF_8)));
}
}
else {
columns.add(handlerFormatter(configure.getFormat(), headers, Arrays.asList(new String((byte[]) body, Charset.forName("UTF-8")))));
columns.add(List.of(new String((byte[]) body, StandardCharsets.UTF_8)));
}
response.setIsSuccessful(Boolean.TRUE);
}
@ -79,7 +78,7 @@ public class RedisAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

6
plugin/datacap-native-zookeeper/src/main/java/io/edurt/datacap/plugin/natived/zookeeper/ZookeeperAdapter.java
View File

@ -55,11 +55,11 @@ public class ZookeeperAdapter
}
types.add("String");
client.getChildren(this.parser.getExecuteContext())
.forEach(column -> columns.add(handlerFormatter(configure.getFormat(), headers, Collections.singletonList(column))));
.forEach(column -> columns.add(Collections.singletonList(column)));
response.setIsSuccessful(Boolean.TRUE);
}
else {
Preconditions.checkArgument(!sqlBase.isSuccessful(), sqlBase.getMessage());
Preconditions.checkArgument(true, sqlBase.getMessage());
}
}
catch (Exception ex) {
@ -70,7 +70,7 @@ public class ZookeeperAdapter
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
response.setColumns(handlerFormatter(configure.getInjector(), configure.getFormat(), headers, columns));
}
}
processorTime.setEnd(new Date().getTime());

36
plugin/datacap-plugin-cassandra/src/main/kotlin/io/edurt/datacap/plugin/cassandra/CassandraAdapter.kt
View File

@ -18,29 +18,35 @@ import kotlin.toString
@SuppressFBWarnings(
value = ["RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"], justification = "I prefer to suppress these FindBugs warnings"
)
class CassandraAdapter : Adapter {
class CassandraAdapter : Adapter
{
private val log: Logger = getLogger(CassandraAdapter::class.java)
private var connection: CassandraConnection? = null
constructor(connection: CassandraConnection?) : super() {
constructor(connection: CassandraConnection?) : super()
{
this.connection = connection
}
override fun handlerExecute(content: String?): Response {
override fun handlerExecute(content: String?): Response
{
val processorTime = Time()
processorTime.start = Date().time
val response: Response = this.connection!!.response
val configure: Configure = this.connection!!.configure
if (response.isConnected) {
val response: Response = this.connection !!.response
val configure: Configure = this.connection !!.configure
if (response.isConnected)
{
val headers: MutableList<String> = ArrayList()
val types: MutableList<String> = ArrayList()
val columns: MutableList<Any> = ArrayList()
try {
val resultSet: ResultSet = connection?.getSession()!!.execute(content!!)
try
{
val resultSet: ResultSet = connection?.getSession() !!.execute(content !!)
var isPresent = true
resultSet.forEach { row ->
if (isPresent) {
if (isPresent)
{
row.columnDefinitions.forEach {
types.add(it.type.asCql(true, true))
headers.add(it.name.asCql(true))
@ -51,17 +57,21 @@ class CassandraAdapter : Adapter {
headers.forEach {
_columns.add(row.getObject(it).toString())
}
columns.add(handlerFormatter(configure.format, headers, _columns))
columns.add(_columns)
}
response.isSuccessful = Boolean.TRUE
} catch (ex: Exception) {
}
catch (ex: Exception)
{
log.error("Execute content failed content {} exception ", content, ex)
response.isSuccessful = Boolean.FALSE
response.message = ex.message
} finally {
}
finally
{
response.headers = headers
response.types = types
response.columns = columns
response.columns = handlerFormatter(configure.injector, configure.format, headers, columns)
}
}
processorTime.end = Date().time

28
plugin/datacap-plugin-scylladb/src/main/kotlin/io/edurt/datacap/scylladb/ScyllaDBAdapter.kt
View File

@ -18,29 +18,35 @@ import kotlin.toString
@SuppressFBWarnings(
value = ["RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"], justification = "I prefer to suppress these FindBugs warnings"
)
class ScyllaDBAdapter : Adapter {
class ScyllaDBAdapter : Adapter
{
private val log: Logger = getLogger(this.javaClass)
private var connection: ScyllaDBConnection? = null
constructor(connection: ScyllaDBConnection?) : super() {
constructor(connection: ScyllaDBConnection?) : super()
{
this.connection = connection
}
override fun handlerExecute(content: String?): Response {
override fun handlerExecute(content: String?): Response
{
val processorTime = Time()
processorTime.start = Date().time
val response: Response = this.connection !!.response
val configure: Configure = this.connection !!.configure
if (response.isConnected) {
if (response.isConnected)
{
val headers: MutableList<String> = ArrayList()
val types: MutableList<String> = ArrayList()
val columns: MutableList<Any> = ArrayList()
try {
try
{
val resultSet: ResultSet = connection?.getSession() !!.execute(content !!)
var isPresent = true
resultSet.forEach { row ->
if (isPresent) {
if (isPresent)
{
row.columnDefinitions.forEach {
types.add(it.type.asCql(true, true))
headers.add(it.name.asCql(true))
@ -51,19 +57,21 @@ class ScyllaDBAdapter : Adapter {
headers.forEach {
_columns.add(row.getObject(it).toString())
}
columns.add(handlerFormatter(configure.format, headers, _columns))
columns.add(_columns)
}
response.isSuccessful = Boolean.TRUE
}
catch (ex: Exception) {
catch (ex: Exception)
{
log.error("Execute content failed content {} exception ", content, ex)
response.isSuccessful = Boolean.FALSE
response.message = ex.message
}
finally {
finally
{
response.headers = headers
response.types = types
response.columns = columns
response.columns = handlerFormatter(configure.injector, configure.format, headers, columns)
}
}
processorTime.end = Date().time