test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-11-30 02:57:37 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Plugin] Support alioss for native

Browse Source
This commit is contained in:
qianmoQ 2023-02-23 23:23:14 +08:00
parent 5477d332a4
commit ff910a708f
14 changed files with 472 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
assets/plugin/alioss.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.9 KiB

5
core/server/pom.xml
View File

@ -265,6 +265,11 @@
<artifactId>datacap-plugin-jdbc-duckdb</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.edurt.datacap.plugin.natived</groupId>
<artifactId>datacap-plugin-native-alioss</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>

39
core/server/src/main/etc/conf/plugins/native/alioss.json Normal file
View File

@ -0,0 +1,39 @@
{
"name": "Alioss",
"supportTime": "2023-02-23",
"configures": [
{
"field": "name",
"type": "String",
"required": true,
"message": "name is a required field, please be sure to enter"
},
{
"field": "host",
"type": "String",
"required": true,
"value": "https://oss-cn-regison.aliyuncs.com",
"message": "host is a required field, please be sure to enter"
},
{
"field": "username",
"type": "String",
"required": true,
"group": "authorization"
},
{
"field": "password",
"type": "String",
"required": true,
"group": "authorization"
},
{
"field": "database",
"type": "String",
"required": true,
"value": "default",
"message": "database is a required field, please be sure to enter",
"group": "advanced"
}
]
}

BIN
core/web/console-fe/public/static/images/plugin/Alioss.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.9 KiB

66
plugin/native/alioss/pom.xml Normal file
View File

@ -0,0 +1,66 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.edurt.datacap.plugin.natived</groupId>
<artifactId>datacap-plugin-native</artifactId>
<version>1.6.0-SNAPSHOT</version>
</parent>
<artifactId>datacap-plugin-native-alioss</artifactId>
<name>DataCap plugin for native (Ali oss)</name>
<properties>
<alioss.version>3.16.1</alioss.version>
<plugin.name>native-alioss</plugin.name>
</properties>
<dependencies>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap.common</groupId>
<artifactId>datacap-common</artifactId>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>${alioss.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>${assembly-plugin.version}</version>
<configuration>
<finalName>${plugin.name}</finalName>
<descriptors>
<descriptor>../../../configure/assembly/assembly-plugin.xml</descriptor>
</descriptors>
<outputDirectory>../../../dist/plugins/${plugin.name}</outputDirectory>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

82
plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossAdapter.java Normal file
View File

@ -0,0 +1,82 @@
package io.edurt.datacap.plugin.natived.alioss;
import com.aliyun.oss.OSS;
import com.aliyun.oss.model.ObjectListing;
import com.google.common.base.Preconditions;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.adapter.NativeAdapter;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import io.edurt.datacap.spi.model.Time;
import io.edurt.datacap.sql.SqlBase;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
@Slf4j
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE", "REC_CATCH_EXCEPTION"},
justification = "I prefer to suppress these FindBugs warnings")
public class AliossAdapter
extends NativeAdapter
{
protected AliossConnection aliossConnection;
private final AliossParser parser;
public AliossAdapter(AliossConnection aliossConnection, AliossParser parser)
{
super(aliossConnection, parser);
this.aliossConnection = aliossConnection;
this.parser = parser;
}
@Override
public Response handlerExecute(String content)
{
Time processorTime = new Time();
processorTime.setStart(new Date().getTime());
Response response = this.aliossConnection.getResponse();
Configure configure = this.aliossConnection.getConfigure();
if (response.getIsConnected()) {
List<String> headers = new ArrayList<>();
List<String> types = new ArrayList<>();
List<Object> columns = new ArrayList<>();
try {
SqlBase sqlBase = this.parser.getSqlBase();
if (sqlBase.isSuccessful()) {
OSS client = this.aliossConnection.getOssClient();
if (ObjectUtils.isNotEmpty(this.parser.getSqlBase().getColumns())) {
headers.addAll(this.parser.getSqlBase().getColumns());
}
else {
headers.add("*");
}
types.add("String");
ObjectListing objectListing = client.listObjects(configure.getDatabase().get(), sqlBase.getTable());
objectListing.getObjectSummaries()
.forEach(column -> columns.add(handlerFormatter(configure.getFormat(), headers, Collections.singletonList(column.getKey()))));
response.setIsSuccessful(Boolean.TRUE);
}
else {
Preconditions.checkArgument(!sqlBase.isSuccessful(), sqlBase.getMessage());
}
}
catch (Exception ex) {
log.error("Execute content failed content {} exception ", content, ex);
response.setIsSuccessful(Boolean.FALSE);
response.setMessage(ex.getMessage());
}
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
}
}
processorTime.setEnd(new Date().getTime());
response.setProcessor(processorTime);
return response;
}
}

57
plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossConnection.java Normal file
View File

@ -0,0 +1,57 @@
package io.edurt.datacap.plugin.natived.alioss;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import io.edurt.datacap.spi.connection.Connection;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class AliossConnection
extends Connection
{
private Configure configure;
private Response response;
@Getter
private OSS ossClient;
public AliossConnection(Configure configure, Response response)
{
super(configure, response);
}
@Override
protected String formatJdbcUrl()
{
return null;
}
@Override
protected java.sql.Connection openConnection()
{
try {
this.configure = getConfigure();
this.response = getResponse();
log.info("Connection url {}", formatJdbcUrl());
this.ossClient = new OSSClientBuilder()
.build(configure.getHost(), configure.getUsername().get(), configure.getPassword().get());
response.setIsConnected(Boolean.TRUE);
}
catch (Exception ex) {
log.error("Connection failed ", ex);
response.setIsConnected(Boolean.FALSE);
response.setMessage(ex.getMessage());
}
return null;
}
@Override
public void destroy()
{
this.ossClient.shutdown();
log.info("Connection close successful");
}
}

27
plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossParser.java Normal file
View File

@ -0,0 +1,27 @@
package io.edurt.datacap.plugin.natived.alioss;
import io.edurt.datacap.spi.parser.SqlParser;
import io.edurt.datacap.sql.SqlBase;
import io.edurt.datacap.sql.SqlBaseToken;
public class AliossParser
extends SqlParser
{
public AliossParser(String content)
{
super(content);
}
@Override
public String getExecuteContext()
{
SqlBase sqlBase = this.getSqlBase();
if (sqlBase.getToken() == SqlBaseToken.SHOW) {
return sqlBase.getTable();
}
else if (sqlBase.getToken() == SqlBaseToken.SELECT) {
return sqlBase.getTable();
}
return null;
}
}

80
plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPlugin.java Normal file
View File

@ -0,0 +1,80 @@
package io.edurt.datacap.plugin.natived.alioss;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginType;
import io.edurt.datacap.spi.adapter.Adapter;
import io.edurt.datacap.spi.connection.JdbcConfigure;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang3.ObjectUtils;
@Slf4j
public class AliossPlugin
implements Plugin
{
private Configure configure;
private AliossConnection connection;
private Response response;
@Override
public String validator()
{
return "SELECT * FROM default";
}
@Override
public String name()
{
return "Alioss";
}
@Override
public String description()
{
return "Integrate Alioss data sources";
}
@Override
public PluginType type()
{
return PluginType.NATIVE;
}
@Override
public void connect(Configure configure)
{
try {
this.response = new Response();
this.configure = new JdbcConfigure();
BeanUtils.copyProperties(this.configure, configure);
this.connection = new AliossConnection(this.configure, this.response);
}
catch (Exception ex) {
this.response.setIsConnected(Boolean.FALSE);
this.response.setMessage(ex.getMessage());
}
}
@Override
public Response execute(String content)
{
if (ObjectUtils.isNotEmpty(this.connection)) {
log.info("Execute alioss plugin logic started");
this.response = this.connection.getResponse();
Adapter processor = new AliossAdapter(this.connection, new AliossParser(content));
this.response = processor.handlerExecute(content);
log.info("Execute alioss plugin logic end");
}
return this.response;
}
@Override
public void destroy()
{
if (ObjectUtils.isNotEmpty(this.connection)) {
this.connection.destroy();
}
}
}

38
plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModule.java Normal file
View File

@ -0,0 +1,38 @@
package io.edurt.datacap.plugin.natived.alioss;
import com.google.inject.multibindings.Multibinder;
import io.edurt.datacap.spi.AbstractPluginModule;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginModule;
import io.edurt.datacap.spi.PluginType;
public class AliossPluginModule
extends AbstractPluginModule
implements PluginModule
{
@Override
public String getName()
{
return "Alioss";
}
@Override
public PluginType getType()
{
return PluginType.NATIVE;
}
@Override
public AbstractPluginModule get()
{
return this;
}
protected void configure()
{
Multibinder<String> module = Multibinder.newSetBinder(this.binder(), String.class);
module.addBinding().toInstance(this.getClass().getSimpleName());
Multibinder<Plugin> plugin = Multibinder.newSetBinder(this.binder(), Plugin.class);
plugin.addBinding().to(AliossPlugin.class);
}
}

1
plugin/native/alioss/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule Normal file
View File

@ -0,0 +1 @@
io.edurt.datacap.plugin.natived.alioss.AliossPluginModule

30
plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModuleTest.java Normal file
View File

@ -0,0 +1,30 @@
package io.edurt.datacap.plugin.natived.alioss;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.Plugin;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Set;
public class AliossPluginModuleTest
{
private Injector injector;
@Before
public void before()
{
this.injector = Guice.createInjector(new AliossPluginModule());
}
@Test
public void test()
{
Set<Plugin> plugins = injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}));
Assert.assertTrue(plugins.size() > 0);
}
}

46
plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginTest.java Normal file
View File

@ -0,0 +1,46 @@
package io.edurt.datacap.plugin.natived.alioss;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.model.Configure;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Optional;
import java.util.Set;
public class AliossPluginTest
{
private Injector injector;
private Configure configure;
@Before
public void before()
{
injector = Guice.createInjector(new AliossPluginModule());
configure = new Configure();
configure.setHost("https://oss-cn-regison.aliyuncs.com");
configure.setUsername(Optional.of("yourAccessKeyId"));
configure.setPassword(Optional.of("yourAccessKeySecret"));
configure.setDatabase(Optional.of("exampleBucket"));
}
@Test
public void test()
{
Set<Plugin> plugins = injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}));
Optional<Plugin> pluginOptional = plugins.stream()
.filter(v -> v.name().equalsIgnoreCase("Alioss"))
.findFirst();
if (pluginOptional.isPresent()) {
Plugin plugin = pluginOptional.get();
plugin.connect(configure);
Assert.assertNotNull(plugin.execute(plugin.validator()).getConnection());
plugin.destroy();
}
}
}

1
plugin/native/pom.xml
View File

@ -16,5 +16,6 @@
<modules>
<module>redis</module>
<module>zookeeper</module>
<module>alioss</module>
</modules>
</project>