[Core] [Plugin] Support some plugin (#587)

This commit is contained in:
Devlive Community 2024-01-12 14:44:19 +08:00 committed by GitHub
commit 8cc205ec3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
48 changed files with 1170 additions and 6499 deletions

View File

@ -182,7 +182,13 @@ Here are some of the major database solutions that are supported:
</a>&nbsp;
<a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height=60" />
</a>
</a>&nbsp;
<a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height=60" />
</a>&nbsp;
<a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height=60" />
</a>&nbsp;
</p>
## System architecture

View File

@ -182,7 +182,13 @@ DataCap может запрашивать данные из любого хра
</a>&nbsp;
<a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height=60" />
</a>
</a>&nbsp;
<a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height=60" />
</a>&nbsp;
<a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height=60" />
</a>&nbsp;
</p>
## Архитектура системы

View File

@ -182,7 +182,13 @@ DataCap 可以从任何使用 SQL 的数据存储或数据引擎ClickHouse、
</a>&nbsp;
<a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/cassandra.png" alt="Apache Cassandra" height=60" />
</a>
</a>&nbsp;
<a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/matrixone.png" alt="MatrixOne" height=60" />
</a>&nbsp;
<a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/scylladb.png" alt="ScyllaDB" height=60" />
</a>&nbsp;
</p>
## 系统架构

View File

@ -0,0 +1,113 @@
name: MatrixOne
supportTime: '2024-01-12'
configures:
- field: name
type: String
required: true
message: name is a required field, please be sure to enter
- field: host
type: String
required: true
value: 127.0.0.1
message: host is a required field, please be sure to enter
- field: port
type: Number
required: true
min: 1
max: 65535
value: 6001
message: port is a required field, please be sure to enter
- field: username
type: String
group: authorization
- field: password
type: String
group: authorization
- field: ssl
type: Boolean
group: authorization
- field: database
type: String
message: database is a required field, please be sure to enter
group: advanced
- field: configures
type: Array
value: []
group: custom
pipelines:
- executor: Seatunnel
type: SOURCE
protocol: JDBC
fields:
- field: url
generated: true
override: true
- field: driver
origin: driver
required: true
override: true
input: true
width: 300
type: INPUT
value: com.microsoft.sqlserver.jdbc.SQLServerDriver
- field: user
origin: username
required: false
override: false
- field: password
origin: password
required: false
override: false
- field: query
origin: context
required: false
hidden: false
description: Query statement
override: true
input: true
width: 300
type: TEXT
tooltip: If this value is entered, the default sql statement will be replaced
- field: result_table_name
origin: result_table_name
description: |
When result_table_name is not specified, the data processed by this plugin will not be registered as a data set (dataStream/dataset) that can be directly accessed by other plugins, or called a temporary table (table) ;
When result_table_name is specified, the data processed by this plugin will be registered as a data set (dataStream/dataset) that can be directly accessed by other plugins, or called a temporary table (table) . The data set (dataStream/dataset) registered here can be directly accessed by other plugins by specifying source_table_name .
required: false
hidden: true
input: true
override: true
- executor: Seatunnel
type: SINK
protocol: JDBC
fields:
- field: url
generated: true
override: true
- field: driver
origin: driver
required: true
override: true
input: true
width: 300
type: INPUT
value: com.microsoft.sqlserver.jdbc.SQLServerDriver
- field: user
origin: username
required: false
override: false
- field: password
origin: password
required: false
override: false
- field: query
origin: query
required: true
hidden: false
description: Use this sql write upstream input datas to database. e.g INSERT ...
override: true
input: true
width: 300
type: TEXT

View File

@ -0,0 +1,26 @@
name: ScyllaDB
supportTime: '2024-01-12'
configures:
- field: name
type: String
required: true
message: name is a required field, please be sure to enter
- field: host
type: String
required: true
value: 127.0.0.1
message: host is a required field, please be sure to enter
- field: port
type: Number
required: true
min: 1
max: 65535
value: 9042
message: port is a required field, please be sure to enter
- field: database
type: String
required: true
value: datacenter
message: database is a required field, please be sure to enter
group: advanced

View File

@ -358,6 +358,18 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-plugin-matrixone</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-plugin-scylladb</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-captcha</artifactId>

View File

@ -38,18 +38,34 @@ public interface Plugin
return String.format("Integrate %s data sources", this.name());
}
default String driver()
{
return "io.edurt.datacap.JdbcDriver";
}
default String connectType()
{
return "datacap";
}
default void connect(Configure configure)
{
Response response = new Response();
try {
JdbcConfigure jdbcConfigure = new JdbcConfigure();
if (jdbcConfigure.getJdbcDriver() == null) {
jdbcConfigure.setJdbcDriver(this.driver());
}
if (jdbcConfigure.getJdbcType() == null) {
jdbcConfigure.setJdbcType(this.connectType());
}
BeanUtils.copyProperties(jdbcConfigure, configure);
local.set(new JdbcConnection(jdbcConfigure, response));
}
catch (Exception ex) {
response.setIsConnected(Boolean.FALSE);
response.setMessage(ex.getMessage());
log.error("Error connecting : {}", ex.getMessage());
log.error("Error connecting : {}", ex);
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 189 KiB

View File

@ -0,0 +1,34 @@
function shuffle(array) {
let currentIndex = array.length, temporaryValue, randomIndex;
while (0 !== currentIndex) {
randomIndex = Math.floor(Math.random() * currentIndex);
currentIndex -= 1;
temporaryValue = array[currentIndex];
array[currentIndex] = array[randomIndex];
array[randomIndex] = temporaryValue;
}
return array;
}
function showRandomAnnouncement(groupId, timeInterval) {
const announceFastAPI = document.getElementById(groupId);
if (announceFastAPI) {
let children = [].slice.call(announceFastAPI.children);
children = shuffle(children)
let index = 0
const announceRandom = () => {
children.forEach((el, i) => {el.style.display = "none"});
children[index].style.display = "block"
index = (index + 1) % children.length
}
announceRandom()
setInterval(announceRandom, timeInterval)
}
}
async function main() {
showRandomAnnouncement('announce-left', 5000)
showRandomAnnouncement('announce-right', 10000)
}
main()

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.1 KiB

View File

@ -0,0 +1,6 @@
[dir=ltr] .md-nav__item--section>.md-nav {
margin-left: 5px;
}
.md-nav--lifted>.md-nav__list>.md-nav__item--active>.md-nav__link:not(.md-nav__container) {
margin-left: -12px;
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -1,146 +0,0 @@
.termynal-comment {
color: #4a968f;
font-style: italic;
display: block;
}
.termy {
/* For right to left languages */
direction: ltr;
}
.termy [data-termynal] {
white-space: pre-wrap;
}
a.external-link {
/* For right to left languages */
direction: ltr;
display: inline-block;
}
a.external-link::after {
/* \00A0 is a non-breaking space
to make the mark be on the same line as the link
*/
content: "\00A0[↪]";
}
a.internal-link::after {
/* \00A0 is a non-breaking space
to make the mark be on the same line as the link
*/
content: "\00A0↪";
}
.shadow {
box-shadow: 5px 5px 10px #999;
}
/* Give space to lower icons so Gitter chat doesn't get on top of them */
.md-footer-meta {
padding-bottom: 2em;
}
.user-list {
display: flex;
flex-wrap: wrap;
margin-bottom: 2rem;
}
.user-list-center {
justify-content: space-evenly;
}
.user {
margin: 1em;
min-width: 7em;
}
.user .avatar-wrapper {
width: 80px;
height: 80px;
margin: 10px auto;
overflow: hidden;
border-radius: 50%;
position: relative;
}
.user .avatar-wrapper img {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
}
.user .title {
text-align: center;
}
.user .count {
font-size: 80%;
text-align: center;
}
a.announce-link:link,
a.announce-link:visited {
color: #fff;
}
a.announce-link:hover {
color: var(--md-accent-fg-color);
}
.announce-wrapper {
display: flex;
justify-content: space-between;
flex-wrap: wrap;
align-items: center;
}
.announce-wrapper div.item {
display: none;
}
.announce-wrapper .sponsor-badge {
display: block;
position: absolute;
top: -10px;
right: 0;
font-size: 0.5rem;
color: #999;
background-color: #666;
border-radius: 10px;
padding: 0 10px;
z-index: 10;
}
.announce-wrapper .sponsor-image {
display: block;
border-radius: 20px;
}
.announce-wrapper > div {
min-height: 40px;
display: flex;
align-items: center;
}
.twitter {
color: #00acee;
}
/* Right to left languages */
code {
direction: ltr;
display: inline-block;
}
.md-content__inner h1 {
direction: ltr !important;
}
.illustration {
margin-top: 2em;
margin-bottom: 2em;
}

View File

@ -223,5 +223,11 @@ Datacap is a fast, lightweight, and intuitive system. It can be used to manage m
- <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/cassandra.png" alt="Apache Cassandra" height=60" />
</a>
- <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/matrixone.png" alt="MatrixOne" height=60" />
</a>
- <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/scylladb.png" alt="ScyllaDB" height=60" />
</a>
</div>

View File

@ -223,5 +223,11 @@ Datacap 是快速、轻量级、直观的系统。它可以用来管理多数据
- <a href="https://cassandra.apache.org/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/cassandra.png" alt="Apache Cassandra" height=60" />
</a>
- <a href="https://matrixorigin.cn/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/matrixone.png" alt="MatrixOne" height=60" />
</a>
- <a href="https://www.scylladb.com/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/scylladb.png" alt="ScyllaDB" height=60" />
</a>
</div>

View File

@ -0,0 +1,75 @@
---
title: MatrixOne
---
<img src="/assets/plugin/matrixone.png" class="connector-logo" style="width: 200px; margin-top: 35px;" />
#### Introduce
MatrixOne is a future-oriented hyper-converged cloud and edge native DBMS that supports transactional, analytical, and streaming workloads with a simplified and distributed database engine, across multiple data centers, clouds, edges and other heterogeneous infrastructures.
#### Environment
---
!!! note
If you need to use this data source, you need to upgrade the DataCap service to >= `2024.01.1`
Support Time: `2024-01-12`
#### Configure
---
!!! note
If your MatrixOne service version requires other special configurations, please refer to modifying the configuration file and restarting the DataCap service.
=== "Configure"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `6001` |
=== "Authorization"
| Field | Required | Default Value |
|:----------:|:---------------------------------:|:-------------:|
| `Username` | :material-check-circle: { .red } | - |
| `Password` | :material-check-circle: { .red } | - |
| `SSL` | :material-close-circle: | `false` |
=== "Advanced"
| Field | Required | Default Value |
|:----------:|:--------------------------------:|:-------------:|
| `Database` | :material-check-circle: { .red } | ` ` |
=== "Custom"
There is no default configuration, and you can add it by user
#### Function
---
| Function | Supported |
|:-------------------:|:----------------:|
| AD Hoc | :material-check: |
| DataSet | :material-check: |
| Pipeline | :material-check: |
| Metadata management | :material-close: |
#### Version
---
!!! warning
The online service has not been tested yet, if you have detailed test results, please submit [issues](https://github.com/EdurtIO/datacap/issues/new/choose) to us
- [x] `1.1.0`

View File

@ -0,0 +1,70 @@
---
title: MatrixOne
---
<img src="/assets/plugin/matrixone.png" class="connector-logo" style="width: 200px; margin-top: 35px;" />
#### 介绍
MatrixOne 是一款面向未来的超融合云和边缘原生 DBMS通过简化的分布式数据库引擎跨多个数据中心、云、边缘和其他异构基础设施支持事务、分析和流式工作负载。
#### 环境
---
!!! note
如果需要使用此数据源,则需要将 DataCap 服务升级到 >= `2024.01.1`
支持时间: `2024-01-12`
#### 配置
---
=== "配置"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `6001` |
=== "授权"
| Field | Required | Default Value |
|:----------:|:---------------------------------:|:-------------:|
| `Username` | :material-check-circle: { .red } | - |
| `Password` | :material-check-circle: { .red } | - |
| `SSL` | :material-close-circle: | `false` |
=== "高级"
| Field | Required | Default Value |
|:----------:|:--------------------------------:|:-------------:|
| `Database` | :material-check-circle: { .red } | ` ` |
=== "自定义"
暂无默认配置,支持用户自定义添加
#### 功能
---
| 功能 | 支持 |
|:-----:|:----------------:|
| 即席查询 | :material-check: |
| 数据集 | :material-check: |
| 流水线 | :material-check: |
| 元数据管理 | :material-close: |
#### 版本
---
!!! warning
在线服务尚未测试,如果您有详细的测试结果,请提交 [issues](https://github.com/EdurtIO/datacap/issues/new/choose)
- [x] `1.1.0`

View File

@ -0,0 +1,58 @@
---
title: ScyllaDB
---
<img src="/assets/plugin/scylladb.png" class="connector-logo" style="width: 200px; margin-top: 35px;" />
#### Introduce
ScyllaDB is a distributed NoSQL wide-column database for data-intensive apps that require high performance and low latency.
#### Environment
---
!!! note
If you need to use this data source, you need to upgrade the DataCap service to >= `2024.01.1`
Support Time: `2024-01-12`
#### Configure
---
=== "Configure"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `9042` |
=== "Configure"
| Field | Required | Default Value |
|:----------:|:--------------------------------:|:-------------:|
| `Database` | :material-check-circle: { .red } | ` ` |
#### Function
---
| Function | Supported |
|:-------------------:|:----------------:|
| AD Hoc | :material-check: |
| DataSet | :material-check: |
| Pipeline | :material-close: |
| Metadata management | :material-close: |
#### 版本
---
!!! Version
The online service has not been tested yet, if you have detailed test results, please submit [issues](https://github.com/EdurtIO/datacap/issues/new/choose) to us
- [x] `all`

View File

@ -0,0 +1,58 @@
---
title: ScyllaDB
---
<img src="/assets/plugin/scylladb.png" class="connector-logo" style="width: 200px; margin-top: 35px;" />
#### 介绍
ScyllaDB 是一个分布式 NoSQL 宽列数据库,适用于需要高性能和低延迟的数据密集型应用程序。
#### 环境
---
!!! note
如果需要使用此数据源,则需要将 DataCap 服务升级到 >= `2024.01.1`
支持时间: `2024-01-12`
#### 配置
---
=== "配置"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `9042` |
=== "高级"
| Field | Required | Default Value |
|:----------:|:--------------------------------:|:-------------:|
| `Database` | :material-check-circle: { .red } | ` ` |
#### 功能
---
| 功能 | 支持 |
|:-----:|:----------------:|
| 即席查询 | :material-check: |
| 数据集 | :material-check: |
| 流水线 | :material-close: |
| 元数据管理 | :material-close: |
#### 版本
---
!!! warning
在线服务尚未测试,如果您有详细的测试结果,请提交 [issues](https://github.com/EdurtIO/datacap/issues/new/choose)
- [x] `all`

View File

@ -10,7 +10,7 @@ edit_uri: "https://github.com/devlive-community/datacap/blob/dev/docs/docs"
banners:
- title: DataCap 1.18.0 is released
link: /release-latest.html
description: <a href="https://github.com/devlive-community/datacap"> Do you ❤️ DataCap? Give us a 🌟 on GitHub </a>
description: <a href="https://github.com/devlive-community/openai-java-sdk" class="text-white"> Do you ❤️ DataCap? Give us a 🌟 on GitHub </a>
copyright: Copyright &copy; 2023 devlive-community
@ -62,10 +62,13 @@ extra:
1.11.0: 1.11.0
1.14.0: 1.14.0
extra_css:
- stylesheets/extra.css
- css/datacap.css
- assets/stylesheets/extra.css
- assets/stylesheets/main.2f34a6de.min.css
- assets/stylesheets/palette.ecc776e4.min.css
- assets/stylesheets/datacap.css
extra_javascript:
- js/datacap.js
- assets/javascript/main.2dasf4fv.js
- assets/javascript/datacap.js
markdown_extensions:
- admonition
- abbr
@ -172,6 +175,8 @@ nav:
- reference/system/sql/home.md
- reference/system/menu/home.md
- NavConnectors:
- reference/connectors/scylladb/home.md
- reference/connectors/matrixone/home.md
- reference/connectors/clickhouse/home.md
- reference/connectors/mysql/home.md
- reference/connectors/redis/home.md

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

View File

@ -4,7 +4,6 @@ This file was automatically generated - do not edit
{% extends "base.html" %}
{% block extrahead %}
<link href="https://cdn.bootcdn.net/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet"/>
<link rel="stylesheet" href="{{ 'assets/stylesheets/main.82a244a8.min.css' | url }}"/>
<script>
!function (p) {
@ -24,10 +23,11 @@ This file was automatically generated - do not edit
<div id="announce-left">
{% for banner in config.banners %}
<div class="item">
<a class="announce-link" href="{{ banner.link }}">
<span class="twemoji">
{{ banner.title }}
</span> {{ banner.description }}
<a class="announce-link" href="{{ banner.link }}" style="color: #ffffff;">
<div class="twemoji">
<div>{{ banner.title }}</div>
<div style="margin-left: 100px; color: #ffffff;">{{ banner.description }}</div>
</div>
</a>
</div>
{% endfor %}
@ -92,5 +92,4 @@ This file was automatically generated - do not edit
{% block scripts %}
{{ super() }}
<script src="{{ 'assets/javascripts/bundle.39654835.min.js' | url }}"></script>
{% endblock %}

View File

@ -10,7 +10,6 @@ import kotlin.Exception
import kotlin.String
import kotlin.TODO
class CassandraConnection : Connection {
private var session: CqlSession? = null

View File

@ -0,0 +1,70 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap</artifactId>
<version>2024.01.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>datacap-plugin-matrixone</artifactId>
<description>DataCap - MatrixOne</description>
<properties>
<plugin.name>datacap-matrixone</plugin.name>
</properties>
<dependencies>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${assembly-plugin.version}</version>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<finalName>${plugin.name}</finalName>
<descriptors>
<descriptor>../../configure/assembly/plugin.xml</descriptor>
</descriptors>
<outputDirectory>../../dist/plugins</outputDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.jetbrains.dokka</groupId>
<artifactId>dokka-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,23 @@
package io.edurt.datacap.matrixone
import com.google.inject.multibindings.Multibinder
import io.edurt.datacap.spi.AbstractPluginModule
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.PluginModule
import io.edurt.datacap.spi.PluginType
class MatrixOneModule : AbstractPluginModule(), PluginModule {
override fun getType(): PluginType {
return PluginType.JDBC
}
override fun get(): AbstractPluginModule {
return this
}
override fun configure() {
Multibinder.newSetBinder(binder(), Plugin::class.java)
.addBinding()
.to(MatrixOnePlugin::class.java)
}
}

View File

@ -0,0 +1,13 @@
package io.edurt.datacap.matrixone
import io.edurt.datacap.spi.Plugin
class MatrixOnePlugin : Plugin {
override fun driver(): String {
return "com.mysql.cj.jdbc.Driver"
}
override fun connectType(): String {
return "mysql"
}
}

View File

@ -0,0 +1 @@
io.edurt.datacap.matrixone.MatrixOneModule

View File

@ -0,0 +1,26 @@
package io.edurt.datacap.matrixone
import org.testcontainers.containers.GenericContainer
import org.testcontainers.utility.DockerImageName
class MatrixOneContainer : GenericContainer<MatrixOneContainer> {
constructor() : super(DockerImageName.parse(DEFAULT_IMAGE_NAME)) {
setupContainer()
}
constructor(dockerImageName: DockerImageName) : super(dockerImageName) {
dockerImageName.assertCompatibleWith(dockerImageName)
setupContainer()
}
private fun setupContainer() {
withExposedPorts(PORT, DOCKER_PORT)
}
companion object {
private const val DEFAULT_IMAGE_NAME = "matrixorigin/matrixone:1.1.0"
const val PORT = 6001
const val DOCKER_PORT = 6001
}
}

View File

@ -0,0 +1,27 @@
package io.edurt.datacap.matrixone
import com.google.inject.Guice.createInjector
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.spi.Plugin
import org.apache.commons.lang3.ObjectUtils.isNotEmpty
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
class MatrixOneModuleTest {
private var injector: Injector? = null
@Before
fun before() {
injector = createInjector(MatrixOneModule())
}
@Test
fun test() {
val plugin: Plugin? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
?.first { v -> v?.name().equals("MatrixOne") }
assertTrue(isNotEmpty(plugin))
}
}

View File

@ -0,0 +1,70 @@
package io.edurt.datacap.matrixone
import com.google.common.collect.Lists
import com.google.inject.Guice
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import org.apache.commons.lang3.ObjectUtils
import org.junit.Assert
import org.junit.Before
import org.junit.Test
import org.slf4j.LoggerFactory.getLogger
import org.testcontainers.containers.Network
import org.testcontainers.containers.wait.strategy.Wait
import org.testcontainers.lifecycle.Startables
import org.testcontainers.shaded.org.awaitility.Awaitility.given
import java.util.*
import java.util.concurrent.TimeUnit
class MatrixOnePluginTest {
private val log = getLogger(this.javaClass)
private val host = "TestMatrixOneContainer"
private val name = "MatrixOne"
private var container: MatrixOneContainer? = null
private var injector: Injector? = null
private var configure: Configure? = null
@Before
fun before() {
val network = Network.newNetwork()
container = MatrixOneContainer()
.withNetwork(network)
.withNetworkAliases(host)
.waitingFor(Wait.forListeningPort())
container?.portBindings = Lists.newArrayList(String.format("%s:%s", MatrixOneContainer.PORT, MatrixOneContainer.DOCKER_PORT))
Startables.deepStart(java.util.stream.Stream.of(container)).join()
log.info("MatrixOne container started")
given().ignoreExceptions()
.await()
.atMost(1, TimeUnit.MINUTES)
injector = Guice.createInjector(MatrixOneModule())
configure = Configure()
configure !!.host = "127.0.0.1"
configure !!.port = MatrixOneContainer.PORT
configure !!.username = Optional.of("root")
configure !!.password = Optional.of("111")
}
@Test
fun test() {
val plugins: Set<Plugin?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
val plugin: Plugin? = plugins?.first { v -> v?.name().equals(name) }
if (ObjectUtils.isNotEmpty(plugin)) {
plugin?.connect(configure)
val response: Response = plugin !!.execute(plugin.validator())
log.info("================ plugin executed information =================")
if (! response.isSuccessful) {
log.error("Message: {}", response.message)
}
else {
response.columns.forEach { column -> log.info(column.toString()) }
}
Assert.assertTrue(response.isSuccessful)
}
}
}

View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<appender name="CONSOLE_LOG" class="ch.qos.logback.core.ConsoleAppender">
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>%date %level [%thread] %logger [%file:%line] %msg%n</pattern>
</layout>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE_LOG" />
</root>
</configuration>

View File

@ -0,0 +1,76 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap</artifactId>
<version>2024.01.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>datacap-plugin-scylladb</artifactId>
<description>DataCap - ScyllaDB</description>
<properties>
<plugin.name>datacap-scylladb</plugin.name>
<cassandra.driver.version>4.16.0</cassandra.driver.version>
</properties>
<dependencies>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.datastax.oss</groupId>
<artifactId>java-driver-core</artifactId>
<version>${cassandra.driver.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${assembly-plugin.version}</version>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<finalName>${plugin.name}</finalName>
<descriptors>
<descriptor>../../configure/assembly/plugin.xml</descriptor>
</descriptors>
<outputDirectory>../../dist/plugins</outputDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.jetbrains.dokka</groupId>
<artifactId>dokka-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,73 @@
package io.edurt.datacap.scylladb
import com.datastax.oss.driver.api.core.cql.ResultSet
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import io.edurt.datacap.spi.adapter.Adapter
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import io.edurt.datacap.spi.model.Time
import org.slf4j.Logger
import org.slf4j.LoggerFactory.getLogger
import java.lang.Boolean
import java.util.*
import kotlin.Any
import kotlin.Exception
import kotlin.String
import kotlin.toString
@SuppressFBWarnings(
value = ["RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"], justification = "I prefer to suppress these FindBugs warnings"
)
class ScyllaDBAdapter : Adapter {
private val log: Logger = getLogger(this.javaClass)
private var connection: ScyllaDBConnection? = null
constructor(connection: ScyllaDBConnection?) : super() {
this.connection = connection
}
override fun handlerExecute(content: String?): Response {
val processorTime = Time()
processorTime.start = Date().time
val response: Response = this.connection !!.response
val configure: Configure = this.connection !!.configure
if (response.isConnected) {
val headers: MutableList<String> = ArrayList()
val types: MutableList<String> = ArrayList()
val columns: MutableList<Any> = ArrayList()
try {
val resultSet: ResultSet = connection?.getSession() !!.execute(content !!)
var isPresent = true
resultSet.forEach { row ->
if (isPresent) {
row.columnDefinitions.forEach {
types.add(it.type.asCql(true, true))
headers.add(it.name.asCql(true))
}
isPresent = false
}
val _columns: MutableList<Any> = ArrayList()
headers.forEach {
_columns.add(row.getObject(it).toString())
}
columns.add(handlerFormatter(configure.format, headers, _columns))
}
response.isSuccessful = Boolean.TRUE
}
catch (ex: Exception) {
log.error("Execute content failed content {} exception ", content, ex)
response.isSuccessful = Boolean.FALSE
response.message = ex.message
}
finally {
response.headers = headers
response.types = types
response.columns = columns
}
}
processorTime.end = Date().time
response.processor = processorTime
return response
}
}

View File

@ -0,0 +1,44 @@
package io.edurt.datacap.scylladb
import com.datastax.oss.driver.api.core.CqlSession
import io.edurt.datacap.spi.connection.Connection
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import java.lang.Boolean
import java.net.InetSocketAddress
import kotlin.Exception
import kotlin.String
import kotlin.TODO
class ScyllaDBConnection : Connection {
private var session: CqlSession? = null
constructor(configure: Configure, response: Response) : super(configure, response)
override fun formatJdbcUrl(): String {
return TODO("Provide the return value")
}
override fun openConnection(): java.sql.Connection? {
try {
this.session = CqlSession.builder()
.addContactPoint(InetSocketAddress(configure?.host, configure !!.port))
.withLocalDatacenter(configure.database.orElse("datacenter"))
.build()
response?.isConnected = Boolean.TRUE
}
catch (ex: Exception) {
response?.isConnected = Boolean.FALSE
response?.message = ex.message
}
return null
}
fun getSession(): CqlSession? {
return session
}
override fun destroy() {
this.session?.close()
}
}

View File

@ -0,0 +1,23 @@
package io.edurt.datacap.scylladb
import com.google.inject.multibindings.Multibinder
import io.edurt.datacap.spi.AbstractPluginModule
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.PluginModule
import io.edurt.datacap.spi.PluginType
class ScyllaDBModule : AbstractPluginModule(), PluginModule {
override fun getType(): PluginType {
return PluginType.JDBC
}
override fun get(): AbstractPluginModule {
return this
}
override fun configure() {
Multibinder.newSetBinder(binder(), Plugin::class.java)
.addBinding()
.to(ScyllaDBPlugin::class.java)
}
}

View File

@ -0,0 +1,54 @@
package io.edurt.datacap.scylladb
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import org.apache.commons.beanutils.BeanUtils.copyProperties
import org.apache.commons.lang3.ObjectUtils.isNotEmpty
import org.slf4j.LoggerFactory.getLogger
class ScyllaDBPlugin : Plugin {
private val log = getLogger(this.javaClass)
private var configure: Configure? = null
private var connection: ScyllaDBConnection? = null
private var response: Response? = null
override fun validator(): String {
return "SELECT release_version AS version FROM system.local"
}
override fun connect(configure: Configure?) {
try {
log.info("Connecting to ScyllaDB")
response = Response()
this.configure = Configure()
copyProperties(this.configure, configure)
connection = ScyllaDBConnection(this.configure !!, response !!)
}
catch (ex: Exception) {
response !!.isConnected = false
response !!.message = ex.message
}
}
override fun execute(content: String?): Response {
if (isNotEmpty(connection)) {
log.info("ScyllaDB connection established")
response = connection?.response
val processor = ScyllaDBAdapter(connection)
response = processor.handlerExecute(content)
log.info("ScyllaDB execution completed")
}
destroy()
return response !!
}
override fun destroy() {
if (isNotEmpty(connection)) {
log.info("ScyllaDB driver destroyed")
connection?.destroy()
configure = null
}
}
}

View File

@ -0,0 +1 @@
io.edurt.datacap.scylladb.ScyllaDBModule

View File

@ -0,0 +1,26 @@
package io.edurt.datacap.scylladb
import org.testcontainers.containers.GenericContainer
import org.testcontainers.utility.DockerImageName
class ScyllaDBContainer : GenericContainer<ScyllaDBContainer> {
constructor() : super(DockerImageName.parse(DEFAULT_IMAGE_NAME)) {
setupContainer()
}
constructor(dockerImageName: DockerImageName) : super(dockerImageName) {
dockerImageName.assertCompatibleWith(dockerImageName)
setupContainer()
}
private fun setupContainer() {
withExposedPorts(PORT, DOCKER_PORT)
}
companion object {
private const val DEFAULT_IMAGE_NAME = "scylladb/scylla"
const val PORT = 9042
const val DOCKER_PORT = 9042
}
}

View File

@ -0,0 +1,29 @@
package io.edurt.datacap.scylladb
import com.google.inject.Guice.createInjector
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.spi.Plugin
import org.apache.commons.lang3.ObjectUtils.isNotEmpty
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
class ScyllaDBModuleTest {
private val name = "ScyllaDB"
private var injector: Injector? = null
@Before
fun before() {
injector = createInjector(ScyllaDBModule())
}
@Test
fun test() {
val plugin: Plugin? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
?.first { v -> v?.name().equals(name) }
assertTrue(isNotEmpty(plugin))
}
}

View File

@ -0,0 +1,70 @@
package io.edurt.datacap.scylladb
import com.google.common.collect.Lists
import com.google.inject.Guice
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.spi.Plugin
import io.edurt.datacap.spi.model.Configure
import io.edurt.datacap.spi.model.Response
import org.apache.commons.lang3.ObjectUtils
import org.junit.Assert
import org.junit.Before
import org.junit.Test
import org.slf4j.LoggerFactory.getLogger
import org.testcontainers.containers.Network
import org.testcontainers.containers.wait.strategy.Wait
import org.testcontainers.lifecycle.Startables
import org.testcontainers.shaded.org.awaitility.Awaitility.given
import java.util.*
import java.util.concurrent.TimeUnit
class ScyllaDBPluginTest {
private val log = getLogger(this.javaClass)
private val host = "TestScyllaDBContainer"
private val name = "ScyllaDB"
private var container: ScyllaDBContainer? = null
private var injector: Injector? = null
private var configure: Configure? = null
@Before
fun before() {
val network = Network.newNetwork()
container = ScyllaDBContainer()
.withNetwork(network)
.withNetworkAliases(host)
.waitingFor(Wait.forListeningPort())
container?.portBindings = Lists.newArrayList(String.format("%s:%s", ScyllaDBContainer.PORT, ScyllaDBContainer.DOCKER_PORT))
Startables.deepStart(java.util.stream.Stream.of(container)).join()
log.info("ScyllaDB container started")
given().ignoreExceptions()
.await()
.atMost(1, TimeUnit.MINUTES)
injector = Guice.createInjector(ScyllaDBModule())
configure = Configure()
configure !!.host = container?.host ?: host
configure !!.port = ScyllaDBContainer.PORT
configure !!.database = Optional.of("datacenter1")
}
@Test
fun test() {
val plugins: Set<Plugin?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Plugin?>?>() {}))
val plugin: Plugin? = plugins?.first { v -> v?.name().equals(name) }
if (ObjectUtils.isNotEmpty(plugin)) {
plugin?.connect(configure)
val response: Response = plugin !!.execute(plugin.validator())
log.info("================ plugin executed information =================")
if (! response.isSuccessful) {
log.error("Message: {}", response.message)
}
else {
response.columns.forEach { column -> log.info(column.toString()) }
}
Assert.assertTrue(response.isSuccessful)
}
}
}

View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<appender name="CONSOLE_LOG" class="ch.qos.logback.core.ConsoleAppender">
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>%date %level [%thread] %logger [%file:%line] %msg%n</pattern>
</layout>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE_LOG" />
</root>
</configuration>

View File

@ -78,6 +78,8 @@
<module>parser/datacap-parser-spi</module>
<module>parser/datacap-parser-trino</module>
<module>parser/datacap-parser-mysql</module>
<module>plugin/datacap-plugin-matrixone</module>
<module>plugin/datacap-plugin-scylladb</module>
</modules>
<name>datacap</name>