diff --git a/assets/plugin/alioss.png b/assets/plugin/alioss.png new file mode 100644 index 00000000..09a1b7e9 Binary files /dev/null and b/assets/plugin/alioss.png differ diff --git a/core/server/pom.xml b/core/server/pom.xml index f18480be..69e6dfa8 100644 --- a/core/server/pom.xml +++ b/core/server/pom.xml @@ -265,6 +265,11 @@ datacap-plugin-jdbc-duckdb ${project.version} + + io.edurt.datacap.plugin.natived + datacap-plugin-native-alioss + ${project.version} + diff --git a/core/server/src/main/etc/conf/plugins/native/alioss.json b/core/server/src/main/etc/conf/plugins/native/alioss.json new file mode 100644 index 00000000..e6adbd72 --- /dev/null +++ b/core/server/src/main/etc/conf/plugins/native/alioss.json @@ -0,0 +1,39 @@ +{ + "name": "Alioss", + "supportTime": "2023-02-23", + "configures": [ + { + "field": "name", + "type": "String", + "required": true, + "message": "name is a required field, please be sure to enter" + }, + { + "field": "host", + "type": "String", + "required": true, + "value": "https://oss-cn-regison.aliyuncs.com", + "message": "host is a required field, please be sure to enter" + }, + { + "field": "username", + "type": "String", + "required": true, + "group": "authorization" + }, + { + "field": "password", + "type": "String", + "required": true, + "group": "authorization" + }, + { + "field": "database", + "type": "String", + "required": true, + "value": "default", + "message": "database is a required field, please be sure to enter", + "group": "advanced" + } + ] +} diff --git a/core/web/console-fe/public/static/images/plugin/Alioss.png b/core/web/console-fe/public/static/images/plugin/Alioss.png new file mode 100644 index 00000000..09a1b7e9 Binary files /dev/null and b/core/web/console-fe/public/static/images/plugin/Alioss.png differ diff --git a/plugin/native/alioss/pom.xml b/plugin/native/alioss/pom.xml new file mode 100644 index 00000000..08f2fb0e --- /dev/null +++ b/plugin/native/alioss/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + + io.edurt.datacap.plugin.natived + datacap-plugin-native + 1.6.0-SNAPSHOT + + + datacap-plugin-native-alioss + DataCap plugin for native (Ali oss) + + + 3.16.1 + native-alioss + + + + + io.edurt.datacap + datacap-spi + provided + + + io.edurt.datacap.common + datacap-common + + + commons-beanutils + commons-beanutils + + + com.aliyun.oss + aliyun-sdk-oss + ${alioss.version} + + + + + + + maven-assembly-plugin + ${assembly-plugin.version} + + ${plugin.name} + + ../../../configure/assembly/assembly-plugin.xml + + ../../../dist/plugins/${plugin.name} + + + + make-assembly + package + + single + + + + + + + + diff --git a/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossAdapter.java b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossAdapter.java new file mode 100644 index 00000000..23a5a2ff --- /dev/null +++ b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossAdapter.java @@ -0,0 +1,82 @@ +package io.edurt.datacap.plugin.natived.alioss; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.ObjectListing; +import com.google.common.base.Preconditions; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.edurt.datacap.spi.adapter.NativeAdapter; +import io.edurt.datacap.spi.model.Configure; +import io.edurt.datacap.spi.model.Response; +import io.edurt.datacap.spi.model.Time; +import io.edurt.datacap.sql.SqlBase; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ObjectUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +@Slf4j +@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE", "REC_CATCH_EXCEPTION"}, + justification = "I prefer to suppress these FindBugs warnings") +public class AliossAdapter + extends NativeAdapter +{ + protected AliossConnection aliossConnection; + private final AliossParser parser; + + public AliossAdapter(AliossConnection aliossConnection, AliossParser parser) + { + super(aliossConnection, parser); + this.aliossConnection = aliossConnection; + this.parser = parser; + } + + @Override + public Response handlerExecute(String content) + { + Time processorTime = new Time(); + processorTime.setStart(new Date().getTime()); + Response response = this.aliossConnection.getResponse(); + Configure configure = this.aliossConnection.getConfigure(); + if (response.getIsConnected()) { + List headers = new ArrayList<>(); + List types = new ArrayList<>(); + List columns = new ArrayList<>(); + try { + SqlBase sqlBase = this.parser.getSqlBase(); + if (sqlBase.isSuccessful()) { + OSS client = this.aliossConnection.getOssClient(); + if (ObjectUtils.isNotEmpty(this.parser.getSqlBase().getColumns())) { + headers.addAll(this.parser.getSqlBase().getColumns()); + } + else { + headers.add("*"); + } + types.add("String"); + ObjectListing objectListing = client.listObjects(configure.getDatabase().get(), sqlBase.getTable()); + objectListing.getObjectSummaries() + .forEach(column -> columns.add(handlerFormatter(configure.getFormat(), headers, Collections.singletonList(column.getKey())))); + response.setIsSuccessful(Boolean.TRUE); + } + else { + Preconditions.checkArgument(!sqlBase.isSuccessful(), sqlBase.getMessage()); + } + } + catch (Exception ex) { + log.error("Execute content failed content {} exception ", content, ex); + response.setIsSuccessful(Boolean.FALSE); + response.setMessage(ex.getMessage()); + } + finally { + response.setHeaders(headers); + response.setTypes(types); + response.setColumns(columns); + } + } + processorTime.setEnd(new Date().getTime()); + response.setProcessor(processorTime); + return response; + } +} diff --git a/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossConnection.java b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossConnection.java new file mode 100644 index 00000000..1446378b --- /dev/null +++ b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossConnection.java @@ -0,0 +1,57 @@ +package io.edurt.datacap.plugin.natived.alioss; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import io.edurt.datacap.spi.connection.Connection; +import io.edurt.datacap.spi.model.Configure; +import io.edurt.datacap.spi.model.Response; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class AliossConnection + extends Connection +{ + private Configure configure; + private Response response; + + @Getter + private OSS ossClient; + + public AliossConnection(Configure configure, Response response) + { + super(configure, response); + } + + @Override + protected String formatJdbcUrl() + { + return null; + } + + @Override + protected java.sql.Connection openConnection() + { + try { + this.configure = getConfigure(); + this.response = getResponse(); + log.info("Connection url {}", formatJdbcUrl()); + this.ossClient = new OSSClientBuilder() + .build(configure.getHost(), configure.getUsername().get(), configure.getPassword().get()); + response.setIsConnected(Boolean.TRUE); + } + catch (Exception ex) { + log.error("Connection failed ", ex); + response.setIsConnected(Boolean.FALSE); + response.setMessage(ex.getMessage()); + } + return null; + } + + @Override + public void destroy() + { + this.ossClient.shutdown(); + log.info("Connection close successful"); + } +} diff --git a/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossParser.java b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossParser.java new file mode 100644 index 00000000..97d3a225 --- /dev/null +++ b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossParser.java @@ -0,0 +1,27 @@ +package io.edurt.datacap.plugin.natived.alioss; + +import io.edurt.datacap.spi.parser.SqlParser; +import io.edurt.datacap.sql.SqlBase; +import io.edurt.datacap.sql.SqlBaseToken; + +public class AliossParser + extends SqlParser +{ + public AliossParser(String content) + { + super(content); + } + + @Override + public String getExecuteContext() + { + SqlBase sqlBase = this.getSqlBase(); + if (sqlBase.getToken() == SqlBaseToken.SHOW) { + return sqlBase.getTable(); + } + else if (sqlBase.getToken() == SqlBaseToken.SELECT) { + return sqlBase.getTable(); + } + return null; + } +} diff --git a/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPlugin.java b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPlugin.java new file mode 100644 index 00000000..614d2662 --- /dev/null +++ b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPlugin.java @@ -0,0 +1,80 @@ +package io.edurt.datacap.plugin.natived.alioss; + +import io.edurt.datacap.spi.Plugin; +import io.edurt.datacap.spi.PluginType; +import io.edurt.datacap.spi.adapter.Adapter; +import io.edurt.datacap.spi.connection.JdbcConfigure; +import io.edurt.datacap.spi.model.Configure; +import io.edurt.datacap.spi.model.Response; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.beanutils.BeanUtils; +import org.apache.commons.lang3.ObjectUtils; + +@Slf4j +public class AliossPlugin + implements Plugin +{ + private Configure configure; + private AliossConnection connection; + private Response response; + + @Override + public String validator() + { + return "SELECT * FROM default"; + } + + @Override + public String name() + { + return "Alioss"; + } + + @Override + public String description() + { + return "Integrate Alioss data sources"; + } + + @Override + public PluginType type() + { + return PluginType.NATIVE; + } + + @Override + public void connect(Configure configure) + { + try { + this.response = new Response(); + this.configure = new JdbcConfigure(); + BeanUtils.copyProperties(this.configure, configure); + this.connection = new AliossConnection(this.configure, this.response); + } + catch (Exception ex) { + this.response.setIsConnected(Boolean.FALSE); + this.response.setMessage(ex.getMessage()); + } + } + + @Override + public Response execute(String content) + { + if (ObjectUtils.isNotEmpty(this.connection)) { + log.info("Execute alioss plugin logic started"); + this.response = this.connection.getResponse(); + Adapter processor = new AliossAdapter(this.connection, new AliossParser(content)); + this.response = processor.handlerExecute(content); + log.info("Execute alioss plugin logic end"); + } + return this.response; + } + + @Override + public void destroy() + { + if (ObjectUtils.isNotEmpty(this.connection)) { + this.connection.destroy(); + } + } +} diff --git a/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModule.java b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModule.java new file mode 100644 index 00000000..818696ae --- /dev/null +++ b/plugin/native/alioss/src/main/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModule.java @@ -0,0 +1,38 @@ +package io.edurt.datacap.plugin.natived.alioss; + +import com.google.inject.multibindings.Multibinder; +import io.edurt.datacap.spi.AbstractPluginModule; +import io.edurt.datacap.spi.Plugin; +import io.edurt.datacap.spi.PluginModule; +import io.edurt.datacap.spi.PluginType; + +public class AliossPluginModule + extends AbstractPluginModule + implements PluginModule +{ + @Override + public String getName() + { + return "Alioss"; + } + + @Override + public PluginType getType() + { + return PluginType.NATIVE; + } + + @Override + public AbstractPluginModule get() + { + return this; + } + + protected void configure() + { + Multibinder module = Multibinder.newSetBinder(this.binder(), String.class); + module.addBinding().toInstance(this.getClass().getSimpleName()); + Multibinder plugin = Multibinder.newSetBinder(this.binder(), Plugin.class); + plugin.addBinding().to(AliossPlugin.class); + } +} diff --git a/plugin/native/alioss/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule b/plugin/native/alioss/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule new file mode 100644 index 00000000..1730c9f2 --- /dev/null +++ b/plugin/native/alioss/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule @@ -0,0 +1 @@ +io.edurt.datacap.plugin.natived.alioss.AliossPluginModule diff --git a/plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModuleTest.java b/plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModuleTest.java new file mode 100644 index 00000000..0f39bc6a --- /dev/null +++ b/plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginModuleTest.java @@ -0,0 +1,30 @@ +package io.edurt.datacap.plugin.natived.alioss; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.TypeLiteral; +import io.edurt.datacap.spi.Plugin; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Set; + +public class AliossPluginModuleTest +{ + private Injector injector; + + @Before + public void before() + { + this.injector = Guice.createInjector(new AliossPluginModule()); + } + + @Test + public void test() + { + Set plugins = injector.getInstance(Key.get(new TypeLiteral>() {})); + Assert.assertTrue(plugins.size() > 0); + } +} \ No newline at end of file diff --git a/plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginTest.java b/plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginTest.java new file mode 100644 index 00000000..861e7c4d --- /dev/null +++ b/plugin/native/alioss/src/test/java/io/edurt/datacap/plugin/natived/alioss/AliossPluginTest.java @@ -0,0 +1,46 @@ +package io.edurt.datacap.plugin.natived.alioss; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.TypeLiteral; +import io.edurt.datacap.spi.Plugin; +import io.edurt.datacap.spi.model.Configure; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; +import java.util.Set; + +public class AliossPluginTest +{ + private Injector injector; + private Configure configure; + + @Before + public void before() + { + injector = Guice.createInjector(new AliossPluginModule()); + configure = new Configure(); + configure.setHost("https://oss-cn-regison.aliyuncs.com"); + configure.setUsername(Optional.of("yourAccessKeyId")); + configure.setPassword(Optional.of("yourAccessKeySecret")); + configure.setDatabase(Optional.of("exampleBucket")); + } + + @Test + public void test() + { + Set plugins = injector.getInstance(Key.get(new TypeLiteral>() {})); + Optional pluginOptional = plugins.stream() + .filter(v -> v.name().equalsIgnoreCase("Alioss")) + .findFirst(); + if (pluginOptional.isPresent()) { + Plugin plugin = pluginOptional.get(); + plugin.connect(configure); + Assert.assertNotNull(plugin.execute(plugin.validator()).getConnection()); + plugin.destroy(); + } + } +} diff --git a/plugin/native/pom.xml b/plugin/native/pom.xml index cab6ea0e..0d456ec2 100644 --- a/plugin/native/pom.xml +++ b/plugin/native/pom.xml @@ -16,5 +16,6 @@ redis zookeeper + alioss \ No newline at end of file