[Plugin] [InfluxDB] Add plugin test

README.md

@@ -66,7 +66,7 @@ Here are some of the major database solutions that are supported:
 |                       <a href="https://duckdb.org/" target="_blank"><img src="docs/docs/assets/plugin/duckdb.png" alt="DuckDB" height="50" /></a>                        |   <a href="https://www.alibabacloud.com/zh/product/object-storage-service" target="_blank"><img src="docs/docs/assets/plugin/alioss.png" alt="Aliyun OSS" height="50" /></a>    |                <a href="https://kafka.apache.org" target="_blank"><img src="docs/docs/assets/plugin/kafka.png" alt="Apache Kafka" height="50" /></a>                 |                           <a href="https://docs.ceresdb.io/" target="_blank"><img src="docs/docs/assets/plugin/ceresdb.png" alt="CeresDB" height="50" /></a>                           | <a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" /></a> |
 |        <a href="https://questdb.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/questdb.png" alt="QuestDB" height="50" /></a>        |       <a href="https://doris.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/doris.png" alt="Apache Doris" height="50" /></a>       | <a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" /></a> | <a href="https://www.alibabacloud.com/product/hologres" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hologres.png" alt="Hologres" height="60" /></a> |    <a href="https://hadoop.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hdfs.png" alt="Apache Hdfs" height="60"/></a>    |
 | <a href="https://docs.pinot.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/pinot.png" alt="Apache Pinot" height="60" /></a> | <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height="60" /></a> | <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height="60" /></a>  |           <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height="60" /></a>           |   <a href="https://www.paradedb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/paradedb.png" alt="ParadeDB" height="80" /></a>    |
-|  <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a>   |        <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a>         |                                                                                                                                                                      |                                                                                                                                                                                        |                                                                                                                                                                         |
+|  <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a>   |        <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a>         | <a href="https://www.influxdata.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/influxdb.png" alt="InfluxDB" height="60" /></a> |                                                                                                                                                                                        |                                                                                                                                                                         |
 
 ## System architecture
 

README.ru.md

@@ -66,7 +66,7 @@ DataCap может запрашивать данные из любого хра
 |                       <a href="https://duckdb.org/" target="_blank"><img src="docs/docs/assets/plugin/duckdb.png" alt="DuckDB" height="50" /></a>                        |   <a href="https://www.alibabacloud.com/zh/product/object-storage-service" target="_blank"><img src="docs/docs/assets/plugin/alioss.png" alt="Aliyun OSS" height="50" /></a>    |                <a href="https://kafka.apache.org" target="_blank"><img src="docs/docs/assets/plugin/kafka.png" alt="Apache Kafka" height="50" /></a>                 |                           <a href="https://docs.ceresdb.io/" target="_blank"><img src="docs/docs/assets/plugin/ceresdb.png" alt="CeresDB" height="50" /></a>                           | <a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" /></a> |
 |        <a href="https://questdb.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/questdb.png" alt="QuestDB" height="50" /></a>        |       <a href="https://doris.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/doris.png" alt="Apache Doris" height="50" /></a>       | <a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" /></a> | <a href="https://www.alibabacloud.com/product/hologres" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hologres.png" alt="Hologres" height="60" /></a> |    <a href="https://hadoop.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hdfs.png" alt="Apache Hdfs" height="60"/></a>    |
 | <a href="https://docs.pinot.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/pinot.png" alt="Apache Pinot" height="60" /></a> | <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height="60" /></a> | <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height="60" /></a>  |           <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height="60" /></a>           |   <a href="https://www.paradedb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/paradedb.png" alt="ParadeDB" height="80" /></a>    |
-|  <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a>   |        <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a>         |                                                                                                                                                                      |                                                                                                                                                                                        |                                                                                                                                                                         |
+|  <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a>   |        <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a>         | <a href="https://www.influxdata.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/influxdb.png" alt="InfluxDB" height="60" /></a> |                                                                                                                                                                                        |                                                                                                                                                                         |
 
 ## Архитектура системы
 

README.zh_CN.md

@@ -66,7 +66,7 @@ DataCap 可以从任何使用 SQL 的数据存储或数据引擎（ClickHouse、
 |                       <a href="https://duckdb.org/" target="_blank"><img src="docs/docs/assets/plugin/duckdb.png" alt="DuckDB" height="50" /></a>                        |   <a href="https://www.alibabacloud.com/zh/product/object-storage-service" target="_blank"><img src="docs/docs/assets/plugin/alioss.png" alt="Aliyun OSS" height="50" /></a>    |                <a href="https://kafka.apache.org" target="_blank"><img src="docs/docs/assets/plugin/kafka.png" alt="Apache Kafka" height="50" /></a>                 |                           <a href="https://docs.ceresdb.io/" target="_blank"><img src="docs/docs/assets/plugin/ceresdb.png" alt="CeresDB" height="50" /></a>                           | <a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" /></a> |
 |        <a href="https://questdb.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/questdb.png" alt="QuestDB" height="50" /></a>        |       <a href="https://doris.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/doris.png" alt="Apache Doris" height="50" /></a>       | <a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" /></a> | <a href="https://www.alibabacloud.com/product/hologres" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hologres.png" alt="Hologres" height="60" /></a> |    <a href="https://hadoop.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/hdfs.png" alt="Apache Hdfs" height="60"/></a>    |
 | <a href="https://docs.pinot.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/pinot.png" alt="Apache Pinot" height="60" /></a> | <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height="60" /></a> | <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height="60" /></a>  |           <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height="60" /></a>           |   <a href="https://www.paradedb.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/paradedb.png" alt="ParadeDB" height="80" /></a>    |
-|  <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a>   |        <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a>         |                                                                                                                                                                      |                                                                                                                                                                                        |                                                                                                                                                                         |
+|  <a href="https://www.timescale.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/timescale.png" alt="Timescale" height="60" /></a>   |        <a href="https://solr.apache.org/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/solr.png" alt="Apache Solr" height="60" /></a>         | <a href="https://www.influxdata.com/" target="_blank" class="connector-logo-index"><img src="docs/docs/assets/plugin/influxdb.png" alt="InfluxDB" height="60" /></a> |                                                                                                                                                                                        |                                                                                                                                                                         |
 
 ## 系统架构
 

configure/etc/conf/plugins/jdbc/influxdb.yaml

@@ -0,0 +1,31 @@
+name: InfluxDB
+supportTime: '2024-06-26'
+configures:
+  - field: name
+    type: String
+    required: true
+    message: name is a required field, please be sure to enter
+  - field: host
+    type: String
+    required: true
+    value: "127.0.0.1"
+    message: host is a required field, please be sure to enter
+  - field: port
+    type: Number
+    required: true
+    min: 1
+    max: 65535
+    value: "8086"
+    message: port is a required field, please be sure to enter
+  - field: username
+    type: String
+    group: authorization
+  - field: password
+    type: String
+    group: authorization
+  - field: configures
+    type: Array
+    value:
+      - field: database
+        value: default
+    group: custom

core/datacap-server/pom.xml

@@ -383,6 +383,12 @@
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>io.edurt.datacap</groupId>
+            <artifactId>datacap-plugin-influxdb</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>io.edurt.datacap</groupId>
             <artifactId>datacap-captcha</artifactId>

core/datacap-service/src/main/java/io/edurt/datacap/service/common/PluginUtils.java

@@ -74,6 +74,7 @@ public class PluginUtils
             resource = resource + ".yaml";
         }
         String path = root + String.format("plugins/%s/%s", type.toLowerCase(), resource.toLowerCase());
+        log.info("Load plugin {} type {} resource {} configure file path {}", plugin, type, resource, path);
         File file = new File(path);
         if (!file.exists()) {
             log.warn("Plugin {} type {} configuration file {} not found, load default configuration file", plugin, type, resource);

core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/SourceServiceImpl.java

@@ -281,6 +281,7 @@ public class SourceServiceImpl
             _configure.setHome(cacheHome);
             _configure.setUsername(Optional.of(UserDetailsService.getUser().getUsername()));
         }
+        _configure.setInjector(injector);
         plugin.connect(_configure);
         io.edurt.datacap.spi.model.Response response = plugin.execute(plugin.validator());
         if (response.getIsSuccessful()) {

core/datacap-spi/pom.xml

@@ -69,6 +69,18 @@
             <artifactId>mysql-connector-java</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>io.edurt.datacap</groupId>
+            <artifactId>datacap-shaded-pinot</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.suteren.jdbc.influxdb</groupId>
+            <artifactId>influxdb-jdbc</artifactId>
+            <version>${datacap.influxdb.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <!-- Support file convert -->
         <dependency>
             <groupId>io.edurt.datacap</groupId>

core/datacap-spi/src/main/java/io/edurt/datacap/spi/connection/JdbcConnection.java

@@ -5,8 +5,10 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.ObjectUtils;
 
 import java.sql.Connection;
+import java.sql.Driver;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.Enumeration;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -30,7 +32,12 @@ public class JdbcConnection
         StringBuffer buffer = new StringBuffer();
         buffer.append("jdbc:");
         buffer.append(this.jdbcConfigure.getJdbcType());
-        buffer.append("://");
+        if (jdbcConfigure.getJdbcType().equals("influxdb")) {
+            buffer.append(":");
+        }
+        else {
+            buffer.append("://");
+        }
         buffer.append(this.jdbcConfigure.getHost());
         buffer.append(":");
         buffer.append(this.jdbcConfigure.getPort());
@@ -70,7 +77,24 @@ public class JdbcConnection
         try {
             this.jdbcConfigure = (JdbcConfigure) getConfigure();
             this.response = getResponse();
-            Class.forName(this.jdbcConfigure.getJdbcDriver());
+
+            // Remove org.apache.pinot.client.PinotDriver and net.suteren.jdbc.influxdb.InfluxDbDriver
+            Enumeration<Driver> drivers = DriverManager.getDrivers();
+            while (drivers.hasMoreElements()) {
+                Driver driver = drivers.nextElement();
+                if (driver instanceof org.apache.pinot.client.PinotDriver
+                        || driver instanceof net.suteren.jdbc.influxdb.InfluxDbDriver) {
+                    DriverManager.deregisterDriver(driver);
+                    log.info("Deregistered driver {}", driver);
+                }
+            }
+
+            // Manually loading and registering the driver
+            Driver driver = (Driver) Class.forName(this.jdbcConfigure.getJdbcDriver())
+                    .getDeclaredConstructor()
+                    .newInstance();
+            DriverManager.registerDriver(driver);
+
             String url = formatJdbcUrl();
             log.info("Connection driver {}", this.jdbcConfigure.getJdbcDriver());
             log.info("Connection url {}", url);

plugin/datacap-plugin-influxdb/pom.xml

@@ -38,9 +38,9 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.apache.arrow</groupId>
+            <groupId>net.suteren.jdbc.influxdb</groupId>
-            <artifactId>flight-sql-jdbc-driver</artifactId>
+            <artifactId>influxdb-jdbc</artifactId>
-            <version>${datacap.arrow.version}</version>
+            <version>${datacap.influxdb.version}</version>
         </dependency>
     </dependencies>
 

plugin/datacap-plugin-influxdb/src/main/kotlin/io/edurt/datacap/plugin/influxdb/InfluxDBPlugin.kt

@@ -6,16 +6,16 @@ class InfluxDBPlugin : Plugin
 {
     override fun validator(): String
     {
-        return "SELECT '-' AS version"
+        return "SELECT 1"
     }
 
     override fun driver(): String
     {
-        return "org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver"
+        return "net.suteren.jdbc.influxdb.InfluxDbDriver"
     }
 
     override fun connectType(): String
     {
-        return "arrow-flight-sql"
+        return "influxdb"
     }
 }

plugin/datacap-plugin-influxdb/src/test/kotlin/io/edurt/datacap/plugin/influxdb/InfluxDBContainer.kt

@@ -0,0 +1,41 @@
+package io.edurt.datacap.plugin.influxdb
+
+import org.testcontainers.containers.GenericContainer
+import org.testcontainers.utility.DockerImageName
+import org.testcontainers.utility.DockerImageName.parse
+
+class InfluxDBContainer : GenericContainer<InfluxDBContainer>
+{
+    constructor() : super(parse(DEFAULT_IMAGE_NAME))
+    {
+        setupContainer()
+    }
+
+    constructor(dockerImageName: DockerImageName) : super(dockerImageName)
+    {
+        dockerImageName.assertCompatibleWith(dockerImageName)
+        setupContainer()
+    }
+
+    private fun setupContainer()
+    {
+        withExposedPorts(PORT, DOCKER_PORT)
+        withEnv("DOCKER_INFLUXDB_INIT_MODE", "setup")
+        withEnv("DOCKER_INFLUXDB_INIT_USERNAME", USERNAME)
+        withEnv("DOCKER_INFLUXDB_INIT_PASSWORD", PASSWORD)
+        withEnv("DOCKER_INFLUXDB_INIT_ORG", ORGANIZATION)
+        withEnv("DOCKER_INFLUXDB_INIT_BUCKET", BUCKET)
+    }
+
+    companion object
+    {
+        private const val DEFAULT_IMAGE_NAME = "influxdb:2"
+
+        const val PORT = 8086
+        const val DOCKER_PORT = 8086
+        const val USERNAME = "root"
+        const val PASSWORD = "root12345678"
+        const val ORGANIZATION = "TestOrg"
+        const val BUCKET = "TestBucket"
+    }
+}

plugin/datacap-plugin-influxdb/src/test/kotlin/io/edurt/datacap/plugin/influxdb/InfluxDBPluginTest.kt

@@ -0,0 +1,80 @@
+package io.edurt.datacap.plugin.influxdb
+
+import com.google.common.collect.Lists
+import com.google.inject.Guice.createInjector
+import com.google.inject.Injector
+import com.google.inject.Key
+import com.google.inject.TypeLiteral
+import io.edurt.datacap.file.FileManager
+import io.edurt.datacap.spi.Plugin
+import io.edurt.datacap.spi.model.Configure
+import io.edurt.datacap.spi.model.Response
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.slf4j.LoggerFactory.getLogger
+import org.testcontainers.containers.Network
+import org.testcontainers.lifecycle.Startables
+import org.testcontainers.shaded.org.awaitility.Awaitility.given
+import java.util.*
+import java.util.concurrent.TimeUnit
+
+class InfluxDBPluginTest
+{
+    private val log = getLogger(this.javaClass)
+    private val host = "TestInfluxDBContainer"
+    private var container: InfluxDBContainer? = null
+    private var injector: Injector? = null
+    private var configure: Configure? = null
+
+    @Before
+    fun before()
+    {
+        val network = Network.newNetwork()
+        container = InfluxDBContainer()
+            .withNetwork(network)
+            .withNetworkAliases(host)
+        container?.portBindings = Lists.newArrayList(String.format("%s:%s", InfluxDBContainer.PORT, InfluxDBContainer.DOCKER_PORT))
+        Startables.deepStart(java.util.stream.Stream.of(container))
+            .join()
+        log.info("InfluxDB container started")
+        given().ignoreExceptions()
+            .await()
+            .atMost(1, TimeUnit.MINUTES)
+
+        injector = createInjector(
+            InfluxDBModule(),
+            FileManager()
+        )
+        configure = Configure()
+        configure !!.injector = injector
+        configure !!.host = container?.host
+        configure !!.port = InfluxDBContainer.PORT
+        configure !!.username = Optional.of(InfluxDBContainer.USERNAME)
+        configure !!.password = Optional.of(InfluxDBContainer.PASSWORD)
+        configure !!.env = Optional.of(mapOf("useEncryption" to false, "useHTTP2" to false, "db" to "test"))
+    }
+
+    @Test
+    fun test()
+    {
+        injector !!.getInstance(Key.get(object : TypeLiteral<Set<Plugin>>()
+        {}))
+            .stream()
+            .findFirst()
+            .ifPresent {
+                it.connect(configure)
+                val response: Response = it.execute(it.validator())
+                log.info("================ Plugin executed information =================")
+                if (! response.isSuccessful)
+                {
+                    log.error("Message: {}", response.message)
+                }
+                else
+                {
+                    response.columns.forEach { column -> log.info(column.toString()) }
+                }
+                assertTrue(response.isSuccessful)
+            }
+    }
+}

plugin/datacap-plugin-influxdb/src/test/resources/logback.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration scan="true" scanPeriod="60 seconds" debug="false">
+    <appender name="CONSOLE_LOG" class="ch.qos.logback.core.ConsoleAppender">
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <pattern>%date %level [%thread] %logger [%file:%line] %msg%n</pattern>
+        </layout>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="CONSOLE_LOG" />
+    </root>
+</configuration>

pom.xml

@@ -180,7 +180,7 @@
         <datacap.alioss.version>3.16.1</datacap.alioss.version>
         <datacap.commons-io.version>2.16.1</datacap.commons-io.version>
         <datacap.solr.version>6.0.0</datacap.solr.version>
-        <datacap.arrow.version>16.0.0</datacap.arrow.version>
+        <datacap.influxdb.version>0.2.6</datacap.influxdb.version>
         <!-- maven plugin -->
         <plugin.maven.checkstyle.version>3.0.0</plugin.maven.checkstyle.version>
         <plugin.maven.findbugs.version>4.8.5.0</plugin.maven.findbugs.version>