test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-11-29 18:48:23 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Plugin] [InfluxDB] Add plugin test

Browse Source
This commit is contained in:
qianmoQ 2024-06-26 18:35:30 +08:00
parent 8e45c301e7
commit 9a5dbbc396
16 changed files with 220 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -66,7 +66,7 @@ Here are some of the major database solutions that are supported:
| <a href="https://duckdb.org/" target="_blank"><img src="docs/docs/assets/plugin/duckdb.png" alt="DuckDB" height="50" /></a> | <a href="https://www.alibabacloud.com/zh/product/object-storage-service" target="_blank"><img src="docs/docs/assets/plugin/alioss.png" alt="Aliyun OSS" height="50" /></a> | <a href="https://kafka.apache.org" target="_blank"><img src="docs/docs/assets/plugin/kafka.png" alt="Apache Kafka" height="50" /></a> | <a href="https://docs.ceresdb.io/" target="_blank"><img src="docs/docs/assets/plugin/ceresdb.png" alt="CeresDB" height="50" /></a> | <a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" /></a> |
| <a href="https://questdb.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/questdb.png" alt="QuestDB" height="50" /></a> | <a href="https://doris.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/doris.png" alt="Apache Doris" height="50" /></a> | <a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" /></a> | <a href="https://www.alibabacloud.com/product/hologres" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hologres.png" alt="Hologres" height="60" /></a> | <a href="https://hadoop.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hdfs.png" alt="Apache Hdfs" height="60"/></a> |
| <a href="https://docs.pinot.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/pinot.png" alt="Apache Pinot" height="60" /></a> | <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height="60" /></a> | <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height="60" /></a> | <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height="60" /></a> | <a href="https://www.paradedb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/paradedb.png" alt="ParadeDB" height="80" /></a> |
| <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a> | <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a> | | | |
| <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a> | <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a> | <a href="https://www.influxdata.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/influxdb.png" alt="InfluxDB" height="60" /></a> | | |
## System architecture

2
README.ru.md
View File

@ -66,7 +66,7 @@ DataCap может запрашивать данные из любого хра
| <a href="https://duckdb.org/" target="_blank"><img src="docs/docs/assets/plugin/duckdb.png" alt="DuckDB" height="50" /></a> | <a href="https://www.alibabacloud.com/zh/product/object-storage-service" target="_blank"><img src="docs/docs/assets/plugin/alioss.png" alt="Aliyun OSS" height="50" /></a> | <a href="https://kafka.apache.org" target="_blank"><img src="docs/docs/assets/plugin/kafka.png" alt="Apache Kafka" height="50" /></a> | <a href="https://docs.ceresdb.io/" target="_blank"><img src="docs/docs/assets/plugin/ceresdb.png" alt="CeresDB" height="50" /></a> | <a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" /></a> |
| <a href="https://questdb.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/questdb.png" alt="QuestDB" height="50" /></a> | <a href="https://doris.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/doris.png" alt="Apache Doris" height="50" /></a> | <a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" /></a> | <a href="https://www.alibabacloud.com/product/hologres" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hologres.png" alt="Hologres" height="60" /></a> | <a href="https://hadoop.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hdfs.png" alt="Apache Hdfs" height="60"/></a> |
| <a href="https://docs.pinot.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/pinot.png" alt="Apache Pinot" height="60" /></a> | <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height="60" /></a> | <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height="60" /></a> | <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height="60" /></a> | <a href="https://www.paradedb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/paradedb.png" alt="ParadeDB" height="80" /></a> |
| <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a> | <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a> | | | |
| <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a> | <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a> | <a href="https://www.influxdata.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/influxdb.png" alt="InfluxDB" height="60" /></a> | | |
## Архитектура системы

2
README.zh_CN.md
View File

@ -66,7 +66,7 @@ DataCap 可以从任何使用 SQL 的数据存储或数据引擎（ClickHouse、
| <a href="https://duckdb.org/" target="_blank"><img src="docs/docs/assets/plugin/duckdb.png" alt="DuckDB" height="50" /></a> | <a href="https://www.alibabacloud.com/zh/product/object-storage-service" target="_blank"><img src="docs/docs/assets/plugin/alioss.png" alt="Aliyun OSS" height="50" /></a> | <a href="https://kafka.apache.org" target="_blank"><img src="docs/docs/assets/plugin/kafka.png" alt="Apache Kafka" height="50" /></a> | <a href="https://docs.ceresdb.io/" target="_blank"><img src="docs/docs/assets/plugin/ceresdb.png" alt="CeresDB" height="50" /></a> | <a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" /></a> |
| <a href="https://questdb.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/questdb.png" alt="QuestDB" height="50" /></a> | <a href="https://doris.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/doris.png" alt="Apache Doris" height="50" /></a> | <a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" /></a> | <a href="https://www.alibabacloud.com/product/hologres" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hologres.png" alt="Hologres" height="60" /></a> | <a href="https://hadoop.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hdfs.png" alt="Apache Hdfs" height="60"/></a> |
| <a href="https://docs.pinot.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/pinot.png" alt="Apache Pinot" height="60" /></a> | <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height="60" /></a> | <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height="60" /></a> | <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height="60" /></a> | <a href="https://www.paradedb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/paradedb.png" alt="ParadeDB" height="80" /></a> |
| <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a> | <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a> | | | |
| <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a> | <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a> | <a href="https://www.influxdata.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/influxdb.png" alt="InfluxDB" height="60" /></a> | | |
## 系统架构

31
configure/etc/conf/plugins/jdbc/influxdb.yaml Normal file
View File

@ -0,0 +1,31 @@
name: InfluxDB
supportTime: '2024-06-26'
configures:
- field: name
type: String
required: true
message: name is a required field, please be sure to enter
- field: host
type: String
required: true
value: "127.0.0.1"
message: host is a required field, please be sure to enter
- field: port
type: Number
required: true
min: 1
max: 65535
value: "8086"
message: port is a required field, please be sure to enter
- field: username
type: String
group: authorization
- field: password
type: String
group: authorization
- field: configures
type: Array
value:
- field: database
value: default
group: custom

6
core/datacap-server/pom.xml
View File

@ -383,6 +383,12 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-plugin-influxdb</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-captcha</artifactId>

1
core/datacap-service/src/main/java/io/edurt/datacap/service/common/PluginUtils.java
View File

@ -74,6 +74,7 @@ public class PluginUtils
resource = resource + ".yaml";
}
String path = root + String.format("plugins/%s/%s", type.toLowerCase(), resource.toLowerCase());
log.info("Load plugin {} type {} resource {} configure file path {}", plugin, type, resource, path);
File file = new File(path);
if (!file.exists()) {
log.warn("Plugin {} type {} configuration file {} not found, load default configuration file", plugin, type, resource);

1
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/SourceServiceImpl.java
View File

@ -281,6 +281,7 @@ public class SourceServiceImpl
_configure.setHome(cacheHome);
_configure.setUsername(Optional.of(UserDetailsService.getUser().getUsername()));
}
_configure.setInjector(injector);
plugin.connect(_configure);
io.edurt.datacap.spi.model.Response response = plugin.execute(plugin.validator());
if (response.getIsSuccessful()) {

12
core/datacap-spi/pom.xml
View File

@ -69,6 +69,18 @@
<artifactId>mysql-connector-java</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-shaded-pinot</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>net.suteren.jdbc.influxdb</groupId>
<artifactId>influxdb-jdbc</artifactId>
<version>${datacap.influxdb.version}</version>
<scope>provided</scope>
</dependency>
<!-- Support file convert -->
<dependency>
<groupId>io.edurt.datacap</groupId>

28
core/datacap-spi/src/main/java/io/edurt/datacap/spi/connection/JdbcConnection.java
View File

@ -5,8 +5,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@ -30,7 +32,12 @@ public class JdbcConnection
StringBuffer buffer = new StringBuffer();
buffer.append("jdbc:");
buffer.append(this.jdbcConfigure.getJdbcType());
buffer.append("://");
if (jdbcConfigure.getJdbcType().equals("influxdb")) {
buffer.append(":");
}
else {
buffer.append("://");
}
buffer.append(this.jdbcConfigure.getHost());
buffer.append(":");
buffer.append(this.jdbcConfigure.getPort());
@ -70,7 +77,24 @@ public class JdbcConnection
try {
this.jdbcConfigure = (JdbcConfigure) getConfigure();
this.response = getResponse();
Class.forName(this.jdbcConfigure.getJdbcDriver());
// Remove org.apache.pinot.client.PinotDriver and net.suteren.jdbc.influxdb.InfluxDbDriver
Enumeration<Driver> drivers = DriverManager.getDrivers();
while (drivers.hasMoreElements()) {
Driver driver = drivers.nextElement();
if (driver instanceof org.apache.pinot.client.PinotDriver
|| driver instanceof net.suteren.jdbc.influxdb.InfluxDbDriver) {
DriverManager.deregisterDriver(driver);
log.info("Deregistered driver {}", driver);
}
}
// Manually loading and registering the driver
Driver driver = (Driver) Class.forName(this.jdbcConfigure.getJdbcDriver())
.getDeclaredConstructor()
.newInstance();
DriverManager.registerDriver(driver);
String url = formatJdbcUrl();
log.info("Connection driver {}", this.jdbcConfigure.getJdbcDriver());
log.info("Connection url {}", url);

BIN
docs/docs/assets/plugin/influxdb.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 24 KiB

6
plugin/datacap-plugin-influxdb/pom.xml
View File

@ -38,9 +38,9 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-sql-jdbc-driver</artifactId>
<version>${datacap.arrow.version}</version>
<groupId>net.suteren.jdbc.influxdb</groupId>
<artifactId>influxdb-jdbc</artifactId>
<version>${datacap.influxdb.version}</version>
</dependency>
</dependencies>

6
plugin/datacap-plugin-influxdb/src/main/kotlin/io/edurt/datacap/plugin/influxdb/InfluxDBPlugin.kt
View File

@ -6,16 +6,16 @@ class InfluxDBPlugin : Plugin
{
override fun validator(): String
{
return "SELECT '-' AS version"
return "SELECT 1"
}
override fun driver(): String
{
return "org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver"
return "net.suteren.jdbc.influxdb.InfluxDbDriver"
}
override fun connectType(): String
{
return "arrow-flight-sql"
return "influxdb"
}
}

41
plugin/datacap-plugin-influxdb/src/test/kotlin/io/edurt/datacap/plugin/influxdb/InfluxDBContainer.kt Normal file
View File

@ -0,0 +1,41 @@
package io.edurt.datacap.plugin.influxdb
import org.testcontainers.containers.GenericContainer
import org.testcontainers.utility.DockerImageName
import org.testcontainers.utility.DockerImageName.parse
class InfluxDBContainer : GenericContainer<InfluxDBContainer>
{
constructor() : super(parse(DEFAULT_IMAGE_NAME))
{
setupContainer()
}
constructor(dockerImageName: DockerImageName) : super(dockerImageName)
{
dockerImageName.assertCompatibleWith(dockerImageName)
setupContainer()
}
private fun setupContainer()
{
withExposedPorts(PORT, DOCKER_PORT)
withEnv("DOCKER_INFLUXDB_INIT_MODE", "setup")
withEnv("DOCKER_INFLUXDB_INIT_USERNAME", USERNAME)
withEnv("DOCKER_INFLUXDB_INIT_PASSWORD", PASSWORD)
withEnv("DOCKER_INFLUXDB_INIT_ORG", ORGANIZATION)
withEnv("DOCKER_INFLUXDB_INIT_BUCKET", BUCKET)
}
companion object
{
private const val DEFAULT_IMAGE_NAME = "influxdb:2"
const val PORT = 8086
const val DOCKER_PORT = 8086
const val USERNAME = "root"
const val PASSWORD = "root12345678"
const val ORGANIZATION = "TestOrg"
const val BUCKET = "TestBucket"
}
}

80
plugin/datacap-plugin-influxdb/src/test/kotlin/io/edurt/datacap/plugin/influxdb/InfluxDBPluginTest.kt Normal file
View File

@ -0,0 +1,80 @@
package io.edurt.datacap.plugin.influxdb
import com.google.common.collect.Lists
import com.google.inject.Guice.createInjector
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.file.FileManager
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.slf4j.LoggerFactory.getLogger
import org.testcontainers.containers.Network
import org.testcontainers.lifecycle.Startables
import org.testcontainers.shaded.org.awaitility.Awaitility.given
import java.util.*
import java.util.concurrent.TimeUnit
class InfluxDBPluginTest
{
private val log = getLogger(this.javaClass)
private val host = "TestInfluxDBContainer"
private var container: InfluxDBContainer? = null
private var injector: Injector? = null
private var configure: Configure? = null
@Before
fun before()
{
val network = Network.newNetwork()
container = InfluxDBContainer()
.withNetwork(network)
.withNetworkAliases(host)
container?.portBindings = Lists.newArrayList(String.format("%s:%s", InfluxDBContainer.PORT, InfluxDBContainer.DOCKER_PORT))
Startables.deepStart(java.util.stream.Stream.of(container))
.join()
log.info("InfluxDB container started")
given().ignoreExceptions()
.await()
.atMost(1, TimeUnit.MINUTES)
injector = createInjector(
InfluxDBModule(),
FileManager()
)
configure = Configure()
configure !!.injector = injector
configure !!.host = container?.host
configure !!.port = InfluxDBContainer.PORT
configure !!.username = Optional.of(InfluxDBContainer.USERNAME)
configure !!.password = Optional.of(InfluxDBContainer.PASSWORD)
configure !!.env = Optional.of(mapOf("useEncryption" to false, "useHTTP2" to false, "db" to "test"))
}
@Test
fun test()
{
injector !!.getInstance(Key.get(object : TypeLiteral<Set<Plugin>>()
{}))
.stream()
.findFirst()
.ifPresent {
it.connect(configure)
val response: Response = it.execute(it.validator())
log.info("================ Plugin executed information =================")
if (! response.isSuccessful)
{
log.error("Message: {}", response.message)
}
else
{
response.columns.forEach { column -> log.info(column.toString()) }
}
assertTrue(response.isSuccessful)
}
}
}

12
plugin/datacap-plugin-influxdb/src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<appender name="CONSOLE_LOG" class="ch.qos.logback.core.ConsoleAppender">
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>%date %level [%thread] %logger [%file:%line] %msg%n</pattern>
</layout>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE_LOG" />
</root>
</configuration>

2
pom.xml
View File

@ -180,7 +180,7 @@
<datacap.alioss.version>3.16.1</datacap.alioss.version>
<datacap.commons-io.version>2.16.1</datacap.commons-io.version>
<datacap.solr.version>6.0.0</datacap.solr.version>
<datacap.arrow.version>16.0.0</datacap.arrow.version>
<datacap.influxdb.version>0.2.6</datacap.influxdb.version>
<!-- maven plugin -->
<plugin.maven.checkstyle.version>3.0.0</plugin.maven.checkstyle.version>
<plugin.maven.findbugs.version>4.8.5.0</plugin.maven.findbugs.version>