[Plugin] [JDBC] Support hologres

2024-12-02 03:57:35 +08:00 · 2023-04-25 11:52:09 +08:00 · 2023-04-25 11:52:09 +08:00 · b98caa58c9
commit b98caa58c9
parent b521010afc
17 changed files with 456 additions and 2 deletions
--- a/README.md
+++ b/README.md
@ -172,6 +172,9 @@ Here are some of the major database solutions that are supported:
    </a>&nbsp;
    <a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index">
        <img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" />
+    </a>&nbsp;
+    <a href="https://www.alibabacloud.com/product/hologres" target="_blank" class="connector-logo-index">
+        <img src="docs/docs/assets/plugin/hologres.png" alt="Hologres" height=60" />
    </a>
 </p>

--- a/core/datacap-server/pom.xml
+++ b/core/datacap-server/pom.xml
@ -340,6 +340,11 @@
            <artifactId>datacap-jdbc-starrocks</artifactId>
            <version>${project.version}</version>
        </dependency>
+        <dependency>
+            <groupId>io.edurt.datacap</groupId>
+            <artifactId>datacap-jdbc-hologres</artifactId>
+            <version>${project.version}</version>
+        </dependency>
        <!-- Executor -->
        <dependency>
            <groupId>io.edurt.datacap</groupId>
--- a/core/datacap-server/src/main/etc/conf/plugins/jdbc/hologres.yaml
+++ b/core/datacap-server/src/main/etc/conf/plugins/jdbc/hologres.yaml
@ -0,0 +1,31 @@
+name: Hologres
+supportTime: '2023-04-25'
+configures:
+  - field: name
+    type: String
+    required: true
+    message: name is a required field, please be sure to enter
+  - field: host
+    type: String
+    required: true
+    value: hologres-cn-regison.aliyuncs.com
+    message: host is a required field, please be sure to enter
+  - field: port
+    type: Number
+    required: true
+    min: 1
+    max: 65535
+    value: 80
+    message: port is a required field, please be sure to enter
+  - field: username
+    type: String
+    group: authorization
+  - field: password
+    type: String
+    group: authorization
+  - field: database
+    type: String
+    required: true
+    value: default
+    message: database is a required field, please be sure to enter
+    group: advanced
--- a/core/datacap-spi/src/main/java/io/edurt/datacap/spi/Plugin.java
+++ b/core/datacap-spi/src/main/java/io/edurt/datacap/spi/Plugin.java
@ -10,12 +10,15 @@ public interface Plugin
        return "SELECT version() AS version";
    }

+    default PluginType type()
+    {
+        return PluginType.JDBC;
+    }
+
    String name();

    String description();

-    PluginType type();
-
    void connect(Configure configure);

    Response execute(String content);
--- a/core/datacap-web/console-fe/public/static/images/plugin/Hologres.png
+++ b/core/datacap-web/console-fe/public/static/images/plugin/Hologres.png
--- a/docs/docs/assets/plugin/hologres.png
+++ b/docs/docs/assets/plugin/hologres.png
--- a/docs/docs/reference/connectors/jdbc/hologres.md
+++ b/docs/docs/reference/connectors/jdbc/hologres.md
@ -0,0 +1,57 @@
+---
+title: Hologres
+status: new
+---
+
+<img src="/assets/plugin/hologres.png" class="connector-logo" />
+
+#### What is Hologres ?
+
+Hologres is an all-in-one real-time data warehouse engine that is compatible with PostgreSQL. It supports online analytical processing (OLAP) and ad hoc analysis of PB-scale data. Hologres supports online data serving at high concurrency and low latency. It is deeply integrated with MaxCompute, Flink and DataWorks, provides a full-stack data warehouse solution that integrates online and offline processing.
+
+#### Environment
+
+!!! note
+
+    If you need to use this data source, you need to upgrade the DataCap service to >= `1.9.x`
+
+Support Time: `2023-04-25`
+
+#### Configure
+
+---
+
+!!! note
+
+    If your Hologres service version requires other special configurations, please refer to modifying the configuration file and restarting the DataCap service.
+
+=== "Configure"
+
+    | Field  |             Required              | Default Value |
+    |:------:|:---------------------------------:|:-------------:|
+    | `Name` | :material-check-circle: { .red }  |       -       |
+    | `Host` | :material-check-circle:  { .red } |  `hologres-cn-regison.aliyuncs.com`  |
+    | `Port` | :material-check-circle:  { .red } |     `80`    |
+
+=== "Authorization"
+
+    |   Field    |        Required         | Default Value |
+    |:----------:|:-----------------------:|:-------------:|
+    | `Username` | :material-close-circle: |       -       |
+    | `Password` | :material-close-circle: |       -       |
+
+=== "Advanced"
+
+    |   Field    |        Required         | Default Value |
+    |:----------:|:-----------------------:|:-------------:|
+    | `Database` | :material-close-circle: |       -       |
+
+#### Version (Validation)
+
+---
+
+!!! warning
+
+    The online service has not been tested yet, if you have detailed test results, please submit [issues](https://github.com/EdurtIO/datacap/issues/new/choose) to us
+
+- [x] all
--- a/docs/docs/reference/connectors/jdbc/hologres.zh.md
+++ b/docs/docs/reference/connectors/jdbc/hologres.zh.md
@ -0,0 +1,57 @@
+---
+title: Hologres
+status: new
+---
+
+<img src="/assets/plugin/hologres.png" class="connector-logo" />
+
+#### 什么是 Hologres ?
+
+Hologres是兼容PostgreSQL的一站式实时数据仓库引擎，支持PB级数据多维分析（OLAP）与即席分析（Ad Hoc），支持高并发低延迟的在线数据服务（Serving）。与MaxCompute、Flink、DataWorks深度融合，提供离在线一体化全栈数仓解决方案。
+
+#### 环境
+
+!!! note
+
+    如果你需要使用这个数据源, 您需要将 DataCap 服务升级到 >= `1.9.x`
+
+支持时间: `2023-04-25`
+
+#### 配置
+
+---
+
+!!! note
+
+    如果您的 Hologres 服务版本需要其他特殊配置，请参考修改配置文件并重启 DataCap 服务。
+
+=== "配置"
+
+    | Field  |             Required              | Default Value |
+    |:------:|:---------------------------------:|:-------------:|
+    | `Name` | :material-check-circle: { .red }  |       -       |
+    | `Host` | :material-check-circle:  { .red } |  `hologres-cn-regison.aliyuncs.com`  |
+    | `Port` | :material-check-circle:  { .red } |     `80`    |
+
+=== "授权"
+
+    |   Field    |        Required         | Default Value |
+    |:----------:|:-----------------------:|:-------------:|
+    | `Username` | :material-close-circle: |       -       |
+    | `Password` | :material-close-circle: |       -       |
+
+=== "高级"
+
+    |   Field    |        Required         | Default Value |
+    |:----------:|:-----------------------:|:-------------:|
+    | `Database` | :material-close-circle: |       -       |
+
+#### 版本（验证）
+
+---
+
+!!! warning
+
+    服务版本尚未测试，如果您有详细的测试并发现错误，请提交 [issues](https://github.com/EdurtIO/datacap/issues/new/choose)
+
+- [x] all
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@ -149,6 +149,7 @@ nav:
      - Connecting to connectors:
        - reference/connectors/index.md
        - JDBC:
+          - Hologres: reference/connectors/jdbc/hologres.md
          - StarRocks: reference/connectors/jdbc/starrocks.md
          - Apache Doris: reference/connectors/jdbc/doris.md
          - DuckDB: reference/connectors/jdbc/duckdb.md
--- a/plugin/datacap-jdbc-hologres/pom.xml
+++ b/plugin/datacap-jdbc-hologres/pom.xml
@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.edurt.datacap</groupId>
+        <artifactId>datacap</artifactId>
+        <version>1.9.0-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>datacap-jdbc-hologres</artifactId>
+    <description>DataCap - Hologres</description>
+
+    <properties>
+        <pgsql.version>42.6.0</pgsql.version>
+        <plugin.name>jdbc-hologres</plugin.name>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>io.edurt.datacap</groupId>
+            <artifactId>datacap-spi</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>${pgsql.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-beanutils</groupId>
+            <artifactId>commons-beanutils</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-reflect</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>${testcontainers.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>${assembly-plugin.version}</version>
+                <configuration>
+                    <finalName>${plugin.name}</finalName>
+                    <descriptors>
+                        <descriptor>../../configure/assembly/assembly-plugin.xml</descriptor>
+                    </descriptors>
+                    <outputDirectory>../../dist/plugins/${plugin.name}</outputDirectory>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
--- a/plugin/datacap-jdbc-hologres/src/main/kotlin/io/edurt/datacap/jdbc/hologres/HologresPlugin.kt
+++ b/plugin/datacap-jdbc-hologres/src/main/kotlin/io/edurt/datacap/jdbc/hologres/HologresPlugin.kt
@ -0,0 +1,60 @@
+package io.edurt.datacap.jdbc.hologres
+
+import io.edurt.datacap.spi.Plugin
+import io.edurt.datacap.spi.adapter.JdbcAdapter
+import io.edurt.datacap.spi.connection.JdbcConfigure
+import io.edurt.datacap.spi.connection.JdbcConnection
+import io.edurt.datacap.spi.model.Configure
+import io.edurt.datacap.spi.model.Response
+import org.apache.commons.beanutils.BeanUtils
+import org.apache.commons.lang3.ObjectUtils
+import org.slf4j.LoggerFactory.getLogger
+
+class HologresPlugin : Plugin {
+    private val log = getLogger(HologresPlugin::class.java)
+
+    private var jdbcConfigure: JdbcConfigure? = null
+    private var jdbcConnection: JdbcConnection? = null
+    private var jdbcResponse: Response? = null
+    override fun name(): String {
+        return this.javaClass.simpleName.replace("Plugin", "")
+    }
+
+    override fun description(): String {
+        return String.format("Integrate %s data sources", this.name())
+    }
+
+    override fun connect(configure: Configure?) {
+        try {
+            log.info("Connecting to Hologres")
+            jdbcResponse = Response()
+            jdbcConfigure = JdbcConfigure()
+            BeanUtils.copyProperties(jdbcConfigure, configure)
+            jdbcConfigure!!.jdbcDriver = "org.postgresql.Driver"
+            jdbcConfigure!!.jdbcType = "postgresql"
+            jdbcConnection = object : JdbcConnection(jdbcConfigure, jdbcResponse) {}
+        } catch (ex: Exception) {
+            jdbcResponse!!.isConnected = false
+            jdbcResponse!!.message = ex.message
+        }
+    }
+
+    override fun execute(content: String?): Response {
+        if (ObjectUtils.isNotEmpty(jdbcConnection)) {
+            log.info("Execute hologres plugin logic started")
+            jdbcResponse = jdbcConnection?.response
+            val processor = JdbcAdapter(jdbcConnection)
+            jdbcResponse = processor.handlerExecute(content)
+            log.info("Execute hologres plugin logic end")
+        }
+        destroy()
+        return jdbcResponse!!
+    }
+
+    override fun destroy() {
+        if (ObjectUtils.isNotEmpty(jdbcConnection)) {
+            jdbcConnection?.destroy()
+            jdbcConnection = null
+        }
+    }
+}
--- a/plugin/datacap-jdbc-hologres/src/main/kotlin/io/edurt/datacap/jdbc/hologres/HologresPluginModule.kt
+++ b/plugin/datacap-jdbc-hologres/src/main/kotlin/io/edurt/datacap/jdbc/hologres/HologresPluginModule.kt
@ -0,0 +1,26 @@
+package io.edurt.datacap.jdbc.hologres
+
+import com.google.inject.multibindings.Multibinder
+import io.edurt.datacap.spi.AbstractPluginModule
+import io.edurt.datacap.spi.Plugin
+import io.edurt.datacap.spi.PluginModule
+import io.edurt.datacap.spi.PluginType
+
+class HologresPluginModule : AbstractPluginModule(), PluginModule {
+    override fun getName(): String {
+        return this.javaClass.simpleName.replace("PluginModule", "")
+    }
+
+    override fun getType(): PluginType {
+        return PluginType.JDBC
+    }
+
+    override fun get(): AbstractPluginModule {
+        return this
+    }
+
+    override fun configure() {
+        val plugin: Multibinder<Plugin> = Multibinder.newSetBinder(binder(), Plugin::class.java)
+        plugin.addBinding().to(HologresPlugin::class.java)
+    }
+}
--- a/plugin/datacap-jdbc-hologres/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule
+++ b/plugin/datacap-jdbc-hologres/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule
@ -0,0 +1 @@
+io.edurt.datacap.jdbc.hologres.HologresPluginModule
--- a/plugin/datacap-jdbc-hologres/src/test/kotlin/io/edurt/datacap/jdbc/hologres/HologresContainer.kt
+++ b/plugin/datacap-jdbc-hologres/src/test/kotlin/io/edurt/datacap/jdbc/hologres/HologresContainer.kt
@ -0,0 +1,18 @@
+package io.edurt.datacap.jdbc.hologres
+
+import org.testcontainers.containers.GenericContainer
+import org.testcontainers.utility.DockerImageName
+
+class HologresContainer : GenericContainer<HologresContainer?> {
+    constructor() : super(DEFAULT_IMAGE_NAME)
+
+    constructor(dockerImageName: DockerImageName) : super(dockerImageName) {
+        dockerImageName.assertCompatibleWith(dockerImageName)
+        withExposedPorts(RPC_PORT)
+    }
+
+    companion object {
+        private val DEFAULT_IMAGE_NAME = DockerImageName.parse("postgres:latest")
+        const val RPC_PORT = 5432
+    }
+}
--- a/plugin/datacap-jdbc-hologres/src/test/kotlin/io/edurt/datacap/jdbc/hologres/HologresPluginModuleTest.kt
+++ b/plugin/datacap-jdbc-hologres/src/test/kotlin/io/edurt/datacap/jdbc/hologres/HologresPluginModuleTest.kt
@ -0,0 +1,29 @@
+package io.edurt.datacap.jdbc.hologres
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+import io.edurt.datacap.spi.Plugin;
+import org.apache.commons.lang3.ObjectUtils
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Set;
+
+class HologresPluginModuleTest {
+    private var injector: Injector? = null
+
+    @Before
+    fun before() {
+        injector = Guice.createInjector(HologresPluginModule())
+    }
+
+    @Test
+    fun test() {
+        val plugin: Plugin? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
+                ?.first { v -> v?.name().equals("Hologres") }
+        Assert.assertTrue(ObjectUtils.isNotEmpty(plugin))
+    }
+}
--- a/plugin/datacap-jdbc-hologres/src/test/kotlin/io/edurt/datacap/jdbc/hologres/HologresPluginTest.kt
+++ b/plugin/datacap-jdbc-hologres/src/test/kotlin/io/edurt/datacap/jdbc/hologres/HologresPluginTest.kt
@ -0,0 +1,75 @@
+package io.edurt.datacap.jdbc.hologres
+
+import com.google.inject.Guice
+import com.google.inject.Injector
+import com.google.inject.Key
+import com.google.inject.TypeLiteral
+import io.edurt.datacap.spi.Plugin
+import io.edurt.datacap.spi.model.Configure
+import io.edurt.datacap.spi.model.Response
+import org.apache.commons.lang3.ObjectUtils
+import org.junit.Assert
+import org.junit.Before
+import org.junit.Test
+import org.slf4j.LoggerFactory.getLogger
+import org.testcontainers.containers.Network
+import org.testcontainers.lifecycle.Startables
+import org.testcontainers.shaded.com.google.common.collect.Lists
+import org.testcontainers.shaded.org.awaitility.Awaitility.given
+import java.util.*
+import java.util.Set
+import java.util.concurrent.TimeUnit
+
+class HologresPluginTest {
+    private val log = getLogger(HologresPluginTest::class.java)
+
+    private val host = "HologresCluster"
+    private val username = "root"
+    private val password = "12345678"
+    private var container: HologresContainer? = null
+    private var injector: Injector? = null
+    private var configure: Configure? = null
+
+    @Before
+    fun before() {
+        val network = Network.newNetwork()
+        container = HologresContainer()
+                ?.withNetwork(network)
+                ?.withNetworkAliases(host)
+                ?.withEnv("POSTGRES_USER", username)
+                ?.withEnv("POSTGRES_PASSWORD", password)
+        container?.portBindings = Lists.newArrayList(String.format("%s:%s", HologresContainer.RPC_PORT, HologresContainer.RPC_PORT))
+        Startables.deepStart(java.util.stream.Stream.of(container)).join()
+        log.info("Hologres container started")
+
+        given().ignoreExceptions()
+                .await()
+                .atMost(400, TimeUnit.SECONDS)
+
+        injector = Guice.createInjector(HologresPluginModule())
+        configure = Configure()
+        configure!!.host = "localhost"
+        configure!!.port = HologresContainer.RPC_PORT
+        configure!!.username = Optional.of(username)
+        configure!!.password = Optional.of(password)
+        configure!!.database = Optional.of("default")
+    }
+
+    @Test
+    fun test() {
+        val plugin: Plugin? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
+                ?.first { v -> v?.name().equals("Hologres") }
+        if (ObjectUtils.isNotEmpty(plugin)) {
+            plugin?.connect(configure)
+            val sql = "SHOW DATABASES"
+            val response: Response = plugin!!.execute(sql)
+            log.info("================ plugin executed information =================")
+            if (!response.isSuccessful) {
+                log.error("Message: {}", response.message)
+            } else {
+                response.columns.forEach { column -> log.info(column.toString()) }
+            }
+            Assert.assertTrue(response.isSuccessful)
+        }
+    }
+}
--- a/pom.xml
+++ b/pom.xml
@ -59,6 +59,7 @@
        <module>plugin/datacap-jdbc-trino</module>
        <module>plugin/datacap-jdbc-doris</module>
        <module>plugin/datacap-jdbc-starrocks</module>
+        <module>plugin/datacap-jdbc-hologres</module>
        <module>executor/datacap-executor-example</module>
        <module>executor/datacap-executor-seatunnel</module>
        <module>shaded/datacap-shaded-ydb</module>
				`@ -0,0 +1 @@`
				`io.edurt.datacap.jdbc.hologres.HologresPluginModule`