test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-12-03 20:48:44 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

[Plugin] Support cratedb for http (#143)

Browse Source
This commit is contained in:
qianmoQ 2022-11-18 01:29:32 +08:00 committed by GitHub
commit a0f998b80e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
67 changed files with 800 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugin/example/src/main/java/io/edurt/datacap/plugin/example/ExamplePluginModule.java
View File

@ -19,7 +19,7 @@ public class ExamplePluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

56
plugin/http/cratedb/pom.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>datacap-plugin-http</artifactId>
<groupId>io.edurt.datacap.plugin.http</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>datacap-plugin-http-cratedb</artifactId>
<name>DataCap plugin for http (CrateDB)</name>
<properties>
<cratedb.version>2.6.0</cratedb.version>
<plugin.name>http-cratedb</plugin.name>
</properties>
<dependencies>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>${assembly-plugin.version}</version>
<configuration>
<finalName>${plugin.name}</finalName>
<descriptors>
<descriptor>../../../configure/assembly/assembly-plugin.xml</descriptor>
</descriptors>
<outputDirectory>../../../dist/plugins/${plugin.name}</outputDirectory>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

80
plugin/http/cratedb/src/main/java/io/edurt/datacap/plugin/http/cratedb/CrateDBAdapter.java Normal file
View File

@ -0,0 +1,80 @@
package io.edurt.datacap.plugin.http.cratedb;
import io.edurt.datacap.spi.adapter.HttpAdapter;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import io.edurt.datacap.spi.connection.http.HttpClient;
import io.edurt.datacap.spi.connection.http.HttpMethod;
import io.edurt.datacap.spi.json.JSON;
import io.edurt.datacap.spi.model.Response;
import io.edurt.datacap.spi.model.Time;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang3.ObjectUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
public class CrateDBAdapter
extends HttpAdapter
{
public CrateDBAdapter(HttpConnection httpConnection)
{
super(httpConnection);
}
@Override
public Response handlerExecute(String content)
{
Time processorTime = new Time();
processorTime.setStart(new Date().getTime());
Response response = this.httpConnection.getResponse();
HttpConfigure httpConfigure = new HttpConfigure();
if (response.getIsConnected()) {
List<String> headers = new ArrayList<>();
List<String> types = new ArrayList<>();
List<Object> columns = new ArrayList<>();
try {
BeanUtils.copyProperties(httpConfigure, this.httpConnection.getConfigure());
httpConfigure.setAutoConnected(Boolean.FALSE);
httpConfigure.setRetry(0);
httpConfigure.setPath("_sql?types");
httpConfigure.setMethod(HttpMethod.POST);
Map<String, String> body = new HashMap<>();
body.put("stmt", content);
httpConfigure.setJsonBody(JSON.toJSON(body));
HttpConnection httpConnection = new HttpConnection(httpConfigure, new Response());
HttpClient httpClient = HttpClient.getInstance(httpConfigure, httpConnection);
String responseBody = httpClient.execute();
CrateDBResponse applyResponse = JSON.objectmapper.readValue(responseBody, CrateDBResponse.class);
if (ObjectUtils.isNotEmpty(applyResponse.getError())) {
throw new Exception(String.join(":", applyResponse.getError().getCode(), applyResponse.getError().getMessage()));
}
else {
headers.addAll(applyResponse.getHeaders());
types.addAll(applyResponse.getTypes());
List<Object> _columns = (List<Object>) applyResponse.getColumns().get(0);
columns.add(handlerFormatter(httpConfigure.getFormat(), headers, _columns));
}
response.setIsSuccessful(Boolean.TRUE);
}
catch (Exception ex) {
log.error("Execute content failed content {} exception ", content, ex);
response.setIsSuccessful(Boolean.FALSE);
response.setMessage(ex.getMessage());
}
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
}
}
processorTime.setEnd(new Date().getTime());
response.setProcessor(processorTime);
return response;
}
}

14
plugin/http/cratedb/src/main/java/io/edurt/datacap/plugin/http/cratedb/CrateDBConnection.java Normal file
View File

@ -0,0 +1,14 @@
package io.edurt.datacap.plugin.http.cratedb;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import io.edurt.datacap.spi.model.Response;
public class CrateDBConnection
extends HttpConnection
{
public CrateDBConnection(HttpConfigure httpConfigure, Response response)
{
super(httpConfigure, response);
}
}

75
plugin/http/cratedb/src/main/java/io/edurt/datacap/plugin/http/cratedb/CrateDBPlugin.java Normal file
View File

@ -0,0 +1,75 @@
package io.edurt.datacap.plugin.http.cratedb;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginType;
import io.edurt.datacap.spi.adapter.Adapter;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang3.ObjectUtils;
@Slf4j
public class CrateDBPlugin
implements Plugin
{
private HttpConfigure httpConfigure;
private HttpConnection connection;
private Response response;
@Override
public String name()
{
return "CrateDB";
}
@Override
public String description()
{
return "Integrate CrateDB data sources";
}
@Override
public PluginType type()
{
return PluginType.HTTP;
}
@Override
public void connect(Configure configure)
{
try {
this.response = new Response();
this.httpConfigure = new HttpConfigure();
BeanUtils.copyProperties(this.httpConfigure, configure);
this.connection = new CrateDBConnection(this.httpConfigure, this.response);
}
catch (Exception ex) {
this.response.setIsConnected(Boolean.FALSE);
this.response.setMessage(ex.getMessage());
}
}
@Override
public Response execute(String content)
{
if (ObjectUtils.isNotEmpty(this.connection)) {
log.info("Execute cratedb plugin logic started");
this.response = this.connection.getResponse();
Adapter processor = new CrateDBAdapter(this.connection);
this.response = processor.handlerExecute(content);
log.info("Execute cratedb plugin logic end");
}
return this.response;
}
@Override
public void destroy()
{
if (ObjectUtils.isNotEmpty(this.connection)) {
this.connection.destroy();
}
}
}

38
plugin/http/cratedb/src/main/java/io/edurt/datacap/plugin/http/cratedb/CrateDBPluginModule.java Normal file
View File

@ -0,0 +1,38 @@
package io.edurt.datacap.plugin.http.cratedb;
import com.google.inject.multibindings.Multibinder;
import io.edurt.datacap.spi.AbstractPluginModule;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginModule;
import io.edurt.datacap.spi.PluginType;
public class CrateDBPluginModule
extends AbstractPluginModule
implements PluginModule
{
@Override
public String getName()
{
return "CrateDB";
}
@Override
public PluginType getType()
{
return PluginType.JDBC;
}
@Override
public AbstractPluginModule get()
{
return this;
}
protected void configure()
{
Multibinder<String> module = Multibinder.newSetBinder(this.binder(), String.class);
module.addBinding().toInstance(this.getClass().getSimpleName());
Multibinder<Plugin> plugin = Multibinder.newSetBinder(this.binder(), Plugin.class);
plugin.addBinding().to(CrateDBPlugin.class);
}
}

30
plugin/http/cratedb/src/main/java/io/edurt/datacap/plugin/http/cratedb/CrateDBResponse.java Normal file
View File

@ -0,0 +1,30 @@
package io.edurt.datacap.plugin.http.cratedb;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.List;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class CrateDBResponse
{
@JsonProperty(value = "cols")
private List<String> headers;
@JsonProperty(value = "col_types")
private List<String> types;
@JsonProperty(value = "rows")
private List<Object> columns;
@JsonProperty(value = "error")
private CrateDBResponseError error;
}

22
plugin/http/cratedb/src/main/java/io/edurt/datacap/plugin/http/cratedb/CrateDBResponseError.java Normal file
View File

@ -0,0 +1,22 @@
package io.edurt.datacap.plugin.http.cratedb;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class CrateDBResponseError
{
@JsonProperty(value = "message")
private String message;
@JsonProperty(value = "code")
private String code;
}

30
plugin/http/cratedb/src/test/java/io/edurt/datacap/plugin/http/cratedb/CrateDBPluginModuleTest.java Normal file
View File

@ -0,0 +1,30 @@
package io.edurt.datacap.plugin.http.cratedb;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.Plugin;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Set;
public class CrateDBPluginModuleTest
{
private Injector injector;
@Before
public void before()
{
this.injector = Guice.createInjector(new CrateDBPluginModule());
}
@Test
public void test()
{
Set<Plugin> plugins = injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}));
Assert.assertTrue(plugins.size() > 0);
}
}

48
plugin/http/cratedb/src/test/java/io/edurt/datacap/plugin/http/cratedb/CrateDBPluginTest.java Normal file
View File

@ -0,0 +1,48 @@
package io.edurt.datacap.plugin.http.cratedb;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.model.Configure;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Optional;
import java.util.Set;
public class CrateDBPluginTest
{
private Injector injector;
private Configure configure;
@Before
public void before()
{
injector = Guice.createInjector(new CrateDBPluginModule());
configure = new Configure();
configure.setHost("127.0.0.1");
configure.setPort(4200);
configure.setUsername(Optional.of("crate"));
configure.setPassword(Optional.of(""));
configure.setFormat(FormatType.JSON);
}
@Test
public void test()
{
Set<Plugin> plugins = injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}));
Optional<Plugin> pluginOptional = plugins.stream()
.filter(v -> v.name().equalsIgnoreCase("CrateDB"))
.findFirst();
if (pluginOptional.isPresent()) {
Plugin plugin = pluginOptional.get();
plugin.connect(configure);
Assert.assertNotNull(plugin.execute(plugin.validator()).getConnection());
plugin.destroy();
}
}
}

19
plugin/http/pom.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>datacap-plugin</artifactId>
<groupId>io.edurt.datacap.plugin</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<artifactId>datacap-plugin-http</artifactId>
<groupId>io.edurt.datacap.plugin.http</groupId>
<name>DataCap plugin for (Http)</name>
<modules>
<module>cratedb</module>
</modules>
</project>

2
plugin/jdbc/clickhouse/src/main/java/io/edurt/datacap/plugin/jdbc/clickhouse/ClickHousePlugin.java
View File

@ -33,7 +33,7 @@ public class ClickHousePlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/clickhouse/src/main/java/io/edurt/datacap/plugin/jdbc/clickhouse/ClickHousePluginModule.java
View File

@ -19,7 +19,7 @@ public class ClickHousePluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

1
plugin/jdbc/cratedb/src/main/java/io/edurt/datacap/plugin/jdbc/cratedb/CrateDBConnection.java
View File

@ -21,7 +21,6 @@ public class CrateDBConnection
{
JdbcConfigure jdbcConfigure = (JdbcConfigure) this.getConfigure();
StringBuffer buffer = new StringBuffer();
// buffer.append("jdbc:");
buffer.append(jdbcConfigure.getJdbcType());
buffer.append("://");
buffer.append(jdbcConfigure.getHost());

2
plugin/jdbc/cratedb/src/main/java/io/edurt/datacap/plugin/jdbc/cratedb/CrateDBPlugin.java
View File

@ -34,7 +34,7 @@ public class CrateDBPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/cratedb/src/main/java/io/edurt/datacap/plugin/jdbc/cratedb/CrateDBPluginModule.java
View File

@ -19,7 +19,7 @@ public class CrateDBPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/db2/src/main/java/io/edurt/datacap/plugin/jdbc/db2/Db2Plugin.java
View File

@ -43,7 +43,7 @@ public class Db2Plugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/db2/src/main/java/io/edurt/datacap/plugin/jdbc/db2/Db2PluginModule.java
View File

@ -19,7 +19,7 @@ public class Db2PluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/dremio/src/main/java/io/edurt/datacap/plugin/jdbc/dremio/DremioPlugin.java
View File

@ -34,7 +34,7 @@ public class DremioPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/dremio/src/main/java/io/edurt/datacap/plugin/jdbc/dremio/DremioPluginModule.java
View File

@ -19,7 +19,7 @@ public class DremioPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/druid/src/main/java/io/edurt/datacap/plugin/jdbc/druid/DruidPlugin.java
View File

@ -39,7 +39,7 @@ public class DruidPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/druid/src/main/java/io/edurt/datacap/plugin/jdbc/druid/DruidPluginModule.java
View File

@ -19,7 +19,7 @@ public class DruidPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/elasticsearch/src/main/java/io/edurt/datacap/plugin/jdbc/elasticsearch/ElasticSearchPlugin.java
View File

@ -33,7 +33,7 @@ public class ElasticSearchPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/elasticsearch/src/main/java/io/edurt/datacap/plugin/jdbc/elasticsearch/ElasticSearchPluginModule.java
View File

@ -19,7 +19,7 @@ public class ElasticSearchPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/h2/src/main/java/io/edurt/datacap/plugin/jdbc/h2/H2Plugin.java
View File

@ -34,7 +34,7 @@ public class H2Plugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/h2/src/main/java/io/edurt/datacap/plugin/jdbc/h2/H2PluginModule.java
View File

@ -19,7 +19,7 @@ public class H2PluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/hive/src/main/java/io/edurt/datacap/plugin/jdbc/hive/HivePlugin.java
View File

@ -34,7 +34,7 @@ public class HivePlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/hive/src/main/java/io/edurt/datacap/plugin/jdbc/hive/HivePluginModule.java
View File

@ -19,7 +19,7 @@ public class HivePluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/ignite/src/main/java/io/edurt/datacap/plugin/jdbc/ignite/IgnitePlugin.java
View File

@ -40,7 +40,7 @@ public class IgnitePlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/ignite/src/main/java/io/edurt/datacap/plugin/jdbc/ignite/IgnitePluginModule.java
View File

@ -19,7 +19,7 @@ public class IgnitePluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/kylin/src/main/java/io/edurt/datacap/plugin/jdbc/kylin/KylinPlugin.java
View File

@ -34,7 +34,7 @@ public class KylinPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/kylin/src/main/java/io/edurt/datacap/plugin/jdbc/kylin/KylinPluginModule.java
View File

@ -19,7 +19,7 @@ public class KylinPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/kyuubi/src/main/java/io/edurt/datacap/plugin/jdbc/kyuubi/KyuubiPlugin.java
View File

@ -34,7 +34,7 @@ public class KyuubiPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/kyuubi/src/main/java/io/edurt/datacap/plugin/jdbc/kyuubi/KyuubiPluginModule.java
View File

@ -19,7 +19,7 @@ public class KyuubiPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/monetdb/src/main/java/io/edurt/datacap/plugin/jdbc/monetdb/MonetDBPlugin.java
View File

@ -34,7 +34,7 @@ public class MonetDBPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/monetdb/src/main/java/io/edurt/datacap/plugin/jdbc/monetdb/MonetDBPluginModule.java
View File

@ -19,7 +19,7 @@ public class MonetDBPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/mongo/src/main/java/io/edurt/datacap/plugin/jdbc/mongo/MongoPlugin.java
View File

@ -34,7 +34,7 @@ public class MongoPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/mongo/src/main/java/io/edurt/datacap/plugin/jdbc/mongo/MongoPluginModule.java
View File

@ -19,7 +19,7 @@ public class MongoPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/mysql/src/main/java/io/edurt/datacap/plugin/jdbc/mysql/MySQLPlugin.java
View File

@ -33,7 +33,7 @@ public class MySQLPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/mysql/src/main/java/io/edurt/datacap/plugin/jdbc/mysql/MySQLPluginModule.java
View File

@ -19,7 +19,7 @@ public class MySQLPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/oracle/src/main/java/io/edurt/datacap/plugin/jdbc/oracle/OraclePlugin.java
View File

@ -34,7 +34,7 @@ public class OraclePlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/oracle/src/main/java/io/edurt/datacap/plugin/jdbc/oracle/OraclePluginModule.java
View File

@ -19,7 +19,7 @@ public class OraclePluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/phoenix/src/main/java/io/edurt/datacap/plugin/jdbc/phoenix/PhoenixPlugin.java
View File

@ -34,7 +34,7 @@ public class PhoenixPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/phoenix/src/main/java/io/edurt/datacap/plugin/jdbc/phoenix/PhoenixPluginModule.java
View File

@ -19,7 +19,7 @@ public class PhoenixPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/pom.xml
View File

@ -11,7 +11,7 @@
<packaging>pom</packaging>
<artifactId>datacap-plugin-jdbc</artifactId>
<groupId>io.edurt.datacap.plugin.jdbc</groupId>
<name>DataCap plugin for jdbc</name>
<name>DataCap plugin for (JDBC)</name>
<modules>
<module>mysql</module>

2
plugin/jdbc/postgresql/src/main/java/io/edurt/datacap/plugin/jdbc/postgresql/PostgreSQLPlugin.java
View File

@ -33,7 +33,7 @@ public class PostgreSQLPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/postgresql/src/main/java/io/edurt/datacap/plugin/jdbc/postgresql/PostgreSQLPluginModule.java
View File

@ -19,7 +19,7 @@ public class PostgreSQLPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/presto/src/main/java/io/edurt/datacap/plugin/jdbc/presto/PrestoPlugin.java
View File

@ -42,7 +42,7 @@ public class PrestoPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/presto/src/main/java/io/edurt/datacap/plugin/jdbc/presto/PrestoPluginModule.java
View File

@ -19,7 +19,7 @@ public class PrestoPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/redis/src/main/java/io/edurt/datacap/plugin/jdbc/redis/RedisPlugin.java
View File

@ -40,7 +40,7 @@ public class RedisPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/redis/src/main/java/io/edurt/datacap/plugin/jdbc/redis/RedisPluginModule.java
View File

@ -19,7 +19,7 @@ public class RedisPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/sqlserver/src/main/java/io/edurt/datacap/plugin/jdbc/sqlserver/SqlServerPlugin.java
View File

@ -34,7 +34,7 @@ public class SqlServerPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/sqlserver/src/main/java/io/edurt/datacap/plugin/jdbc/sqlserver/SqlServerPluginModule.java
View File

@ -19,7 +19,7 @@ public class SqlServerPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/trino/src/main/java/io/edurt/datacap/plugin/jdbc/trino/TrinoPlugin.java
View File

@ -33,7 +33,7 @@ public class TrinoPlugin
@Override
public PluginType type()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

2
plugin/jdbc/trino/src/main/java/io/edurt/datacap/plugin/jdbc/trino/TrinoPluginModule.java
View File

@ -19,7 +19,7 @@ public class TrinoPluginModule
@Override
public PluginType getType()
{
return PluginType.SOURCE;
return PluginType.JDBC;
}
@Override

1
plugin/pom.xml
View File

@ -17,5 +17,6 @@
<modules>
<module>example</module>
<module>jdbc</module>
<module>http</module>
</modules>
</project>

2
server/src/main/java/io/edurt/datacap/server/service/impl/SourceServiceImpl.java
View File

@ -107,7 +107,7 @@ public class SourceServiceImpl
{
List<PluginEntity> plugins = this.injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}))
.stream()
.filter(plugin -> plugin.type().equals(PluginType.SOURCE))
.filter(plugin -> plugin.type().equals(PluginType.JDBC))
.map(plugin -> {
PluginEntity entity = new PluginEntity();
entity.setName(plugin.name());

6
spi/pom.xml
View File

@ -14,6 +14,7 @@
<properties>
<guice.version>5.1.0</guice.version>
<okhttp.version>4.9.2</okhttp.version>
</properties>
<dependencies>
@ -46,5 +47,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
</dependency>
</dependencies>
</project>

4
spi/src/main/java/io/edurt/datacap/spi/PluginType.java
View File

@ -2,6 +2,6 @@ package io.edurt.datacap.spi;
public enum PluginType
{
SOURCE,
SINK
JDBC,
HTTP
}

35
spi/src/main/java/io/edurt/datacap/spi/adapter/HttpAdapter.java Normal file
View File

@ -0,0 +1,35 @@
package io.edurt.datacap.spi.adapter;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.FormatType;
import io.edurt.datacap.spi.connection.HttpConnection;
import io.edurt.datacap.spi.formatter.FormatterFactory;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@Slf4j
@SuppressFBWarnings(value = {"RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"},
justification = "I prefer to suppress these FindBugs warnings")
public class HttpAdapter
implements Adapter
{
protected HttpConnection httpConnection;
public HttpAdapter(HttpConnection httpConnection)
{
this.httpConnection = httpConnection;
}
protected Object handlerFormatter(FormatType format, List<String> headers, List<Object> columns)
{
return FormatterFactory.createFormatter(format, headers, columns).formatter();
}
@Override
public Response handlerExecute(String content)
{
return null;
}
}

27
spi/src/main/java/io/edurt/datacap/spi/connection/HttpConfigure.java Normal file
View File

@ -0,0 +1,27 @@
package io.edurt.datacap.spi.connection;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.connection.http.HttpMethod;
import io.edurt.datacap.spi.model.Configure;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import java.util.Map;
@Data
@ToString
@EqualsAndHashCode
@SuppressFBWarnings(value = {"EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC"},
justification = "I prefer to suppress these FindBugs warnings")
public class HttpConfigure
extends Configure
{
private String protocol = "http";
private String path;
private HttpMethod method;
private Integer retry = 0;
private Boolean autoConnected = Boolean.FALSE;
private Map<String, String> params;
private String jsonBody;
}

57
spi/src/main/java/io/edurt/datacap/spi/connection/HttpConnection.java Normal file
View File

@ -0,0 +1,57 @@
package io.edurt.datacap.spi.connection;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
@Slf4j
public class HttpConnection
extends io.edurt.datacap.spi.connection.Connection
{
private HttpConfigure httpConfigure;
private Response response;
public HttpConnection(HttpConfigure httpConfigure, Response response)
{
super(httpConfigure, response);
}
public String formatJdbcUrl()
{
StringBuffer buffer = new StringBuffer();
buffer.append(this.httpConfigure.getProtocol());
buffer.append("://");
buffer.append(this.httpConfigure.getHost());
buffer.append(":");
buffer.append(this.httpConfigure.getPort());
if (StringUtils.isNotEmpty(this.httpConfigure.getPath())) {
buffer.append("/");
buffer.append(this.httpConfigure.getPath());
}
return buffer.toString();
}
protected Connection openConnection()
{
try {
this.httpConfigure = (HttpConfigure) getConfigure();
this.response = getResponse();
log.info("Connection protocol {}", this.httpConfigure.getProtocol());
log.info("Connection url {}", formatJdbcUrl());
response.setIsConnected(Boolean.TRUE);
}
catch (Exception ex) {
log.error("Connection failed ", ex);
response.setIsConnected(Boolean.FALSE);
response.setMessage(ex.getMessage());
}
return null;
}
public void destroy()
{
log.info("Connection close skip");
}
}

110
spi/src/main/java/io/edurt/datacap/spi/connection/http/HttpClient.java Normal file
View File

@ -0,0 +1,110 @@
package io.edurt.datacap.spi.connection.http;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import okhttp3.ConnectionPool;
import okhttp3.HttpUrl;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.apache.commons.lang3.ObjectUtils;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
@SuppressFBWarnings(value = {"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"},
justification = "I prefer to suppress these FindBugs warnings")
public class HttpClient
{
private static final int CONNECTION_TIME_OUT = 200000;
private static final int SOCKET_TIME_OUT = 20000;
private static final int MAX_IDLE_CONNECTIONS = 30;
private static final long KEEP_ALLIVE_TIME = 60000L;
private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8");
private static volatile HttpClient httpClient;
private final OkHttpClient okHttpClient;
private final HttpConfigure configure;
private final HttpConnection httpConnection;
public HttpClient(HttpConfigure configure, HttpConnection httpConnection)
{
this.configure = configure;
this.httpConnection = httpConnection;
ConnectionPool connectionPool = new ConnectionPool(MAX_IDLE_CONNECTIONS, KEEP_ALLIVE_TIME, TimeUnit.MILLISECONDS);
this.okHttpClient = new OkHttpClient().newBuilder()
.readTimeout(SOCKET_TIME_OUT, TimeUnit.MILLISECONDS)
.writeTimeout(SOCKET_TIME_OUT, TimeUnit.MILLISECONDS)
.connectionPool(connectionPool)
.retryOnConnectionFailure(configure.getAutoConnected())
.connectTimeout(CONNECTION_TIME_OUT, TimeUnit.MILLISECONDS)
.addInterceptor(new HttpRetryInterceptor(configure))
.addNetworkInterceptor(new HttpRetryInterceptor(configure))
.build();
}
public static HttpClient getInstance(HttpConfigure configure, HttpConnection httpConnection)
{
httpClient = new HttpClient(configure, httpConnection);
return httpClient;
}
public String execute()
{
switch (configure.getMethod()) {
case GET:
return this.get();
case POST:
return this.post();
default:
return null;
}
}
private String get()
{
HttpUrl.Builder builder = HttpUrl.parse(httpConnection.formatJdbcUrl()).newBuilder();
if (ObjectUtils.isNotEmpty(configure.getParams())) {
for (String key : configure.getParams().keySet()) {
builder.addQueryParameter(key, configure.getParams().get(key));
}
}
Request request = new Request.Builder()
.addHeader("Accept-Encoding", "identity")
.url(builder.build().toString()).build();
return execute(request);
}
private String post()
{
RequestBody requestBody = RequestBody.create(JSON, configure.getJsonBody());
HttpUrl.Builder builder = HttpUrl.parse(httpConnection.formatJdbcUrl()).newBuilder();
// Adding Path Parameters
if (ObjectUtils.isNotEmpty(configure.getParams())) {
for (String key : configure.getParams().keySet()) {
builder.addQueryParameter(key, configure.getParams().get(key));
}
}
Request request = new Request.Builder().post(requestBody)
.addHeader("Accept-Encoding", "identity")
.url(builder.build().toString()).build();
return execute(request);
}
private String execute(Request request)
{
String responseBody = null;
try {
Response response = okHttpClient.newCall(request).execute();
responseBody = response.body().string();
}
catch (IOException | NullPointerException e) {
e.printStackTrace();
}
return responseBody;
}
}

9
spi/src/main/java/io/edurt/datacap/spi/connection/http/HttpMethod.java Normal file
View File

@ -0,0 +1,9 @@
package io.edurt.datacap.spi.connection.http;
public enum HttpMethod
{
POST,
PUT,
DELETE,
GET
}

45
spi/src/main/java/io/edurt/datacap/spi/connection/http/HttpRetryInterceptor.java Normal file
View File

@ -0,0 +1,45 @@
package io.edurt.datacap.spi.connection.http;
import io.edurt.datacap.spi.connection.HttpConfigure;
import okhttp3.Interceptor;
import okhttp3.Request;
import okhttp3.Response;
public class HttpRetryInterceptor
implements Interceptor
{
private final HttpConfigure configure;
private int count = 1;
public HttpRetryInterceptor(HttpConfigure configure)
{
this.configure = configure;
}
@Override
public Response intercept(Chain chain)
{
return retry(chain);
}
public Response retry(Chain chain)
{
Response response = null;
Request request = chain.request();
try {
Thread.sleep(2000);
response = chain.proceed(request);
while (!response.isSuccessful() && count < configure.getRetry()) {
count++;
response = retry(chain);
}
}
catch (Exception e) {
while (count < configure.getRetry()) {
count++;
response = retry(chain);
}
}
return response;
}
}

23
spi/src/main/java/io/edurt/datacap/spi/json/JSON.java Normal file
View File

@ -0,0 +1,23 @@
package io.edurt.datacap.spi.json;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class JSON
{
public static final ObjectMapper objectmapper = new ObjectMapper();
private JSON() {}
public static String toJSON(Object object)
{
String json;
try {
json = objectmapper.writeValueAsString(object);
}
catch (JsonProcessingException e) {
json = null;
}
return json;
}
}

28
spi/src/test/java/io/edurt/datacap/spi/connection/http/HttpClientTest.java Normal file
View File

@ -0,0 +1,28 @@
package io.edurt.datacap.spi.connection.http;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import io.edurt.datacap.spi.model.Response;
import org.junit.Assert;
import org.junit.Test;
public class HttpClientTest
{
@Test
public void test()
{
HttpConfigure httpConfigure = new HttpConfigure();
httpConfigure.setAutoConnected(Boolean.FALSE);
httpConfigure.setRetry(0);
httpConfigure.setParams(null);
httpConfigure.setProtocol("https");
httpConfigure.setHost("datacap.incubator.edurt.io");
httpConfigure.setPort(443);
httpConfigure.setMethod(HttpMethod.GET);
httpConfigure.setPath("blog/index.html");
HttpConnection httpConnection = new HttpConnection(httpConfigure, new Response());
HttpClient httpClient = HttpClient.getInstance(httpConfigure, httpConnection);
String response = httpClient.execute();
Assert.assertNotNull(response);
}
}