[Plugin] Support starrocks

This commit is contained in:
qianmoQ 2023-04-20 13:41:43 +08:00
parent 0a4e3f7a2e
commit 67d8d768d7
20 changed files with 488 additions and 15 deletions

View File

@ -163,6 +163,9 @@ Here are some of the major database solutions that are supported:
</a>&nbsp;
<a href="https://doris.apache.org/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/doris.png" alt="Apache Doris" height="50" />
</a>&nbsp;
<a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/starrocks.png" alt="StarRocks" height="50" />
</a>
</p>

View File

@ -335,6 +335,11 @@
<artifactId>datacap-jdbc-doris</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-jdbc-starrocks</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Executor -->
<dependency>
<groupId>io.edurt.datacap</groupId>

View File

@ -15,7 +15,7 @@ configures:
required: true
min: 1
max: 65535
value: 9030
value: 9093
message: port is a required field, please be sure to enter
- field: username
type: String

View File

@ -0,0 +1,31 @@
name: StarRocks
supportTime: '2023-04-20'
configures:
- field: name
type: String
required: true
message: name is a required field, please be sure to enter
- field: host
type: String
required: true
value: 127.0.0.1
message: host is a required field, please be sure to enter
- field: port
type: Number
required: true
min: 1
max: 65535
value: 9030
message: port is a required field, please be sure to enter
- field: username
type: String
group: authorization
- field: password
type: String
group: authorization
- field: database
type: String
required: true
value: default
message: database is a required field, please be sure to enter
group: advanced

View File

@ -9,6 +9,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
@Slf4j
@ -74,7 +75,11 @@ public class JdbcConnection
}
else {
log.info("Connection username and password not present");
this.connection = DriverManager.getConnection(url);
Properties properties = new Properties();
if (jdbcConfigure.getUsername().isPresent()) {
properties.put("user", jdbcConfigure.getUsername().get());
}
this.connection = DriverManager.getConnection(url, properties);
}
response.setIsConnected(Boolean.TRUE);
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.0 KiB

View File

@ -194,6 +194,9 @@ Datacap is fast, lightweight, intuitive system.
</a>&nbsp;
<a href="https://doris.apache.org/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/doris.png" alt="Apache Doris" height="50" />
</a>&nbsp;
<a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/starrocks.png" alt="StarRocks" height="50" />
</a>
</p>

View File

@ -194,6 +194,9 @@ Datacap 是快速、轻量级、直观的系统。
</a>&nbsp;
<a href="https://doris.apache.org/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/doris.png" alt="Apache Doris" height="50" />
</a>&nbsp;
<a href="https://www.starrocks.io/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/starrocks.png" alt="StarRocks" height="50" />
</a>
</p>

View File

@ -0,0 +1,57 @@
---
title: StarRocks
status: new
---
<img src="/assets/plugin/starrocks.png" class="connector-logo" />
#### What is StarRocks ?
An Open-Source, High-Performance Analytical Database
#### Environment
!!! note
If you need to use this data source, you need to upgrade the DataCap service to >= `1.9.x`
Support Time: `2023-04-20`
#### Configure
---
!!! note
If your StarRocks service version requires other special configurations, please refer to modifying the configuration file and restarting the DataCap service.
=== "Configure"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `9030` |
=== "Authorization"
| Field | Required | Default Value |
|:----------:|:-----------------------:|:-------------:|
| `Username` | :material-close-circle: | - |
| `Password` | :material-close-circle: | - |
=== "Advanced"
| Field | Required | Default Value |
|:----------:|:-----------------------:|:-------------:|
| `Database` | :material-close-circle: | - |
#### Version (Validation)
---
!!! warning
The online service has not been tested yet, if you have detailed test results, please submit [issues](https://github.com/EdurtIO/datacap/issues/new/choose) to us
- [x] 2.2.x

View File

@ -0,0 +1,57 @@
---
title: StarRocks
status: new
---
<img src="/assets/plugin/starrocks.png" class="connector-logo" />
#### 什么是 StarRocks ?
开源、高性能的分析数据库
#### 环境
!!! note
如果你需要使用这个数据源, 您需要将 DataCap 服务升级到 >= `1.9.x`
支持时间: `2023-04-20`
#### 配置
---
!!! note
如果您的 StarRocks 服务版本需要其他特殊配置,请参考修改配置文件并重启 DataCap 服务。
=== "配置"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `9030` |
=== "授权"
| Field | Required | Default Value |
|:----------:|:-----------------------:|:-------------:|
| `Username` | :material-close-circle: | - |
| `Password` | :material-close-circle: | - |
=== "高级"
| Field | Required | Default Value |
|:----------:|:-----------------------:|:-------------:|
| `Database` | :material-close-circle: | - |
#### 版本(验证)
---
!!! warning
服务版本尚未测试,如果您有详细的测试并发现错误,请提交 [issues](https://github.com/EdurtIO/datacap/issues/new/choose)
- [x] 2.2.x

View File

@ -148,6 +148,7 @@ nav:
- Connecting to connectors:
- reference/connectors/index.md
- JDBC:
- StarRocks: reference/connectors/jdbc/starrocks.md
- Apache Doris: reference/connectors/jdbc/doris.md
- DuckDB: reference/connectors/jdbc/duckdb.md
- Yandex Database: reference/connectors/jdbc/ydb.md

View File

@ -0,0 +1,82 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap</artifactId>
<version>1.9.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>datacap-jdbc-starrocks</artifactId>
<description>DataCap - StarRocks</description>
<properties>
<mysql.version>8.0.28</mysql.version>
<plugin.name>jdbc-starrocks</plugin.name>
</properties>
<dependencies>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.version}</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${assembly-plugin.version}</version>
<configuration>
<finalName>${plugin.name}</finalName>
<descriptors>
<descriptor>../../configure/assembly/assembly-plugin.xml</descriptor>
</descriptors>
<outputDirectory>../../dist/plugins/${plugin.name}</outputDirectory>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,66 @@
package io.edurt.datacap.plugin.jdbc.starrocks
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.PluginType
import io.edurt.datacap.spi.adapter.JdbcAdapter
import io.edurt.datacap.spi.connection.JdbcConfigure
import io.edurt.datacap.spi.connection.JdbcConnection
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import org.apache.commons.beanutils.BeanUtils
import org.apache.commons.lang3.ObjectUtils
import org.slf4j.LoggerFactory.getLogger
class StarRocksPlugin : Plugin {
private val log = getLogger(StarRocksPlugin::class.java)
private var jdbcConfigure: JdbcConfigure? = null
private var jdbcConnection: JdbcConnection? = null
private var jdbcResponse: Response? = null
override fun name(): String {
return "StarRocks"
}
override fun description(): String {
return "Integrate StarRocks data sources"
}
override fun type(): PluginType {
return PluginType.JDBC
}
override fun connect(configure: Configure?) {
try {
log.info("Connecting to StarRocks")
jdbcResponse = Response()
jdbcConfigure = JdbcConfigure()
BeanUtils.copyProperties(jdbcConfigure, configure)
jdbcConfigure!!.jdbcDriver = "com.mysql.cj.jdbc.Driver"
jdbcConfigure!!.jdbcType = "mysql"
jdbcConnection = object : JdbcConnection(jdbcConfigure, jdbcResponse) {}
} catch (ex: Exception) {
jdbcResponse!!.isConnected = false
jdbcResponse!!.message = ex.message
}
}
override fun execute(content: String?): Response {
if (ObjectUtils.isNotEmpty(jdbcConnection)) {
log.info("Execute starrocks plugin logic started")
jdbcResponse = jdbcConnection?.response
val processor = JdbcAdapter(jdbcConnection)
jdbcResponse = processor.handlerExecute(content)
log.info("Execute starrocks plugin logic end")
}
destroy()
return jdbcResponse!!
}
override fun destroy() {
if (ObjectUtils.isNotEmpty(jdbcConnection)) {
jdbcConnection?.destroy()
jdbcConnection = null
}
}
}

View File

@ -0,0 +1,28 @@
package io.edurt.datacap.plugin.jdbc.starrocks
import com.google.inject.multibindings.Multibinder
import io.edurt.datacap.spi.AbstractPluginModule
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.PluginModule
import io.edurt.datacap.spi.PluginType
class StarRocksPluginModule : AbstractPluginModule(), PluginModule {
override fun getName(): String {
return "StarRocks"
}
override fun getType(): PluginType {
return PluginType.JDBC
}
override fun get(): AbstractPluginModule {
return this
}
override fun configure() {
val module = Multibinder.newSetBinder(binder(), String::class.java)
module.addBinding().toInstance(this.javaClass.simpleName)
val plugin: Multibinder<Plugin> = Multibinder.newSetBinder(binder(), Plugin::class.java)
plugin.addBinding().to(StarRocksPlugin::class.java)
}
}

View File

@ -0,0 +1 @@
io.edurt.datacap.plugin.jdbc.starrocks.StarRocksPluginModule

View File

@ -0,0 +1,26 @@
package io.edurt.datacap.plugin.jdbc.starrocks
import org.testcontainers.containers.GenericContainer
import org.testcontainers.utility.DockerImageName
class StarRocksContainer : GenericContainer<StarRocksContainer?> {
constructor() : super(DEFAULT_IMAGE_NAME)
constructor(dockerImageName: DockerImageName) : super(dockerImageName) {
dockerImageName.assertCompatibleWith(dockerImageName)
withExposedPorts(BE_PORT, WEBSERVER_PORT, HEARTBEAT_SERVICE_PORT, BRPC_PORT, HTTP_PORT, RPC_PORT, QUERY_PORT, EDIT_LOG_PORT, BROKER_IPC_PORT)
}
companion object {
private val DEFAULT_IMAGE_NAME = DockerImageName.parse("starrocks:2.2.12")
const val BE_PORT = 9060
const val WEBSERVER_PORT = 8040
const val HEARTBEAT_SERVICE_PORT = 9050
const val BRPC_PORT = 8060
const val HTTP_PORT = 8030
const val RPC_PORT = 9020
const val QUERY_PORT = 9030
const val EDIT_LOG_PORT = 9010
const val BROKER_IPC_PORT = 8000
}
}

View File

@ -0,0 +1,29 @@
package io.edurt.datacap.plugin.jdbc.starrocks
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.Plugin;
import org.apache.commons.lang3.ObjectUtils
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Set;
class StarRocksPluginModuleTest {
private var injector: Injector? = null
@Before
fun before() {
injector = Guice.createInjector(StarRocksPluginModule())
}
@Test
fun test() {
val plugin: Plugin? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
?.first { v -> v?.name().equals("StarRocks") }
Assert.assertTrue(ObjectUtils.isNotEmpty(plugin))
}
}

View File

@ -0,0 +1,71 @@
package io.edurt.datacap.plugin.jdbc.starrocks
import com.google.inject.Guice
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import org.apache.commons.lang3.ObjectUtils
import org.junit.Assert
import org.junit.Before
import org.junit.Test
import org.slf4j.LoggerFactory.getLogger
import org.testcontainers.containers.Network
import org.testcontainers.lifecycle.Startables
import org.testcontainers.shaded.com.google.common.collect.Lists
import org.testcontainers.shaded.org.awaitility.Awaitility.given
import java.util.*
import java.util.Set
import java.util.concurrent.TimeUnit
class StarRocksPluginTest {
private val log = getLogger(StarRocksPluginTest::class.java)
private val host = "StarRocksCluster"
private var container: StarRocksContainer? = null
private var injector: Injector? = null
private var configure: Configure? = null
@Before
fun before() {
val network = Network.newNetwork()
container = StarRocksContainer()
?.withNetwork(network)
?.withNetworkAliases(host)
container?.portBindings = Lists.newArrayList(String.format("%s:%s", StarRocksContainer.QUERY_PORT, StarRocksContainer.QUERY_PORT))
Startables.deepStart(java.util.stream.Stream.of(container)).join()
log.info("StarRocks container started")
given().ignoreExceptions()
.await()
.atMost(400, TimeUnit.SECONDS)
injector = Guice.createInjector(StarRocksPluginModule())
configure = Configure()
configure!!.host = "0.0.0.0"
configure!!.port = StarRocksContainer.QUERY_PORT
configure!!.username = Optional.of("root")
}
@Test
fun test() {
val plugin: Plugin? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
?.first { v -> v?.name().equals("StarRocks") }
if (ObjectUtils.isNotEmpty(plugin)) {
plugin?.connect(configure)
val sql = "SHOW DATABASES"
val response: Response = plugin!!.execute(sql)
log.info("================ plugin executed information =================")
if (!response.isSuccessful) {
log.error("Message: {}", response.message)
} else {
response.columns.forEach { column -> log.info(column.toString()) }
}
Assert.assertTrue(response.isSuccessful)
}
}
}

31
pom.xml
View File

@ -58,6 +58,7 @@
<module>plugin/datacap-jdbc-tdengine</module>
<module>plugin/datacap-jdbc-trino</module>
<module>plugin/datacap-jdbc-doris</module>
<module>plugin/datacap-jdbc-starrocks</module>
<module>executor/datacap-executor-example</module>
<module>executor/datacap-executor-seatunnel</module>
</modules>
@ -117,7 +118,7 @@
<slf4j.version>1.7.36</slf4j.version>
<sql-formatter.version>2.0.3</sql-formatter.version>
<guava.version>31.1-jre</guava.version>
<kotlin.version>1.8.10</kotlin.version>
<kotlin.version>1.8.20</kotlin.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<redis.version>3.8.0</redis.version>
<testcontainers.version>1.17.6</testcontainers.version>
@ -146,6 +147,17 @@
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-jdk8</artifactId>
<version>${kotlin.version}</version>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-test</artifactId>
<version>${kotlin.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
@ -252,32 +264,25 @@
<plugins>
<plugin>
<artifactId>kotlin-maven-plugin</artifactId>
<configuration>
<jvmTarget>1.8</jvmTarget>
</configuration>
<groupId>org.jetbrains.kotlin</groupId>
<version>${kotlin.version}</version>
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
<configuration>
<sourceDirs>
<sourceDir>${project.basedir}/src/main/kotlin</sourceDir>
<sourceDir>${project.basedir}/src/main/java</sourceDir>
</sourceDirs>
</configuration>
</execution>
<execution>
<id>test-compile</id>
<phase>test-compile</phase>
<goals>
<goal>test-compile</goal>
</goals>
<configuration>
<sourceDirs>
<sourceDir>${project.basedir}/src/test/kotlin</sourceDir>
<sourceDir>${project.basedir}/src/test/java</sourceDir>
</sourceDirs>
</configuration>
</execution>
</executions>
</plugin>