test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-11-30 02:57:37 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Plugin] [HTTP] Support greptimedb (#320)

Browse Source
This commit is contained in:
qianmoQ 2023-04-14 15:54:18 +08:00 committed by GitHub
commit 9779e3ee95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 864 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -154,6 +154,9 @@ Here are some of the major database solutions that are supported:
</a>&nbsp;
<a href="https://docs.ceresdb.io/" target="_blank">
<img src="docs/docs/assets/plugin/ceresdb.png" alt="CeresDB" height="50" />
</a>&nbsp;
<a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index">
<img src="docs/docs/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" />
</a>
</p>

5
core/datacap-server/pom.xml
View File

@ -320,6 +320,11 @@
<artifactId>datacap-http-ceresdb</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-http-greptime</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Executor -->
<dependency>
<groupId>io.edurt.datacap</groupId>

38
core/datacap-server/src/main/etc/conf/plugins/http/ceresdb.yaml
View File

@ -18,41 +18,3 @@ configures:
max: 65535
value: 5440
message: port is a required field, please be sure to enter
pipelines:
- executor: Seatunnel
type: SOURCE
fields:
- field: host
origin: host|port
required: true
- field: database
origin: database
required: true
- field: sql
origin: context
required: true
- field: username
origin: username
required: true
- field: password
origin: password
required: true
- executor: Seatunnel
type: SINK
fields:
- field: host
origin: host|port
required: true
- field: database
origin: database
required: true
- field: sql
origin: context
required: true
- field: username
origin: username
required: true
- field: password
origin: password
required: true

20
core/datacap-server/src/main/etc/conf/plugins/http/greptimedb.yaml Normal file
View File

@ -0,0 +1,20 @@
name: GreptimeDB
supportTime: '2023-04-14'
configures:
- field: name
type: String
required: true
message: name is a required field, please be sure to enter
- field: host
type: String
required: true
value: 127.0.0.1
message: host is a required field, please be sure to enter
- field: port
type: Number
required: true
min: 1
max: 65535
value: 4000
message: port is a required field, please be sure to enter

1
core/datacap-spi/src/main/java/io/edurt/datacap/spi/connection/HttpConfigure.java
View File

@ -25,5 +25,6 @@ public class HttpConfigure
private Boolean autoConnected = Boolean.FALSE;
private Map<String, String> params;
private String jsonBody;
private boolean isDecoded = false;
private MediaType mediaType = MediaType.parse("application/json; charset=utf-8");
}

26
core/datacap-spi/src/main/java/io/edurt/datacap/spi/connection/http/HttpClient.java
View File

@ -4,6 +4,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import okhttp3.ConnectionPool;
import okhttp3.FormBody;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
@ -13,6 +14,7 @@ import org.apache.commons.lang3.ObjectUtils;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@SuppressFBWarnings(value = {"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"},
justification = "I prefer to suppress these FindBugs warnings")
@ -36,10 +38,7 @@ public class HttpClient
.readTimeout(SOCKET_TIME_OUT, TimeUnit.MILLISECONDS)
.writeTimeout(SOCKET_TIME_OUT, TimeUnit.MILLISECONDS)
.connectionPool(connectionPool)
.retryOnConnectionFailure(configure.getAutoConnected())
.connectTimeout(CONNECTION_TIME_OUT, TimeUnit.MILLISECONDS)
.addInterceptor(new HttpRetryInterceptor(configure))
.addNetworkInterceptor(new HttpRetryInterceptor(configure))
.build();
}
@ -55,6 +54,9 @@ public class HttpClient
case GET:
return this.get();
case POST:
if (configure.isDecoded()) {
return this.postEncoder();
}
return this.post();
default:
return null;
@ -93,6 +95,24 @@ public class HttpClient
return execute(request);
}
private String postEncoder()
{
AtomicReference<String> key = new AtomicReference<>();
AtomicReference<String> value = new AtomicReference<>();
configure.getParams().forEach((originKey, originValue) -> {
key.set(originKey);
value.set(originValue);
});
RequestBody body = new FormBody.Builder()
.add(key.get(), value.get())
.build();
Request request = new Request.Builder()
.url(httpConnection.formatJdbcUrl())
.method(HttpMethod.POST.name(), body)
.build();
return execute(request);
}
private String execute(Request request)
{
String responseBody = null;

BIN
core/datacap-web/console-fe/public/static/images/plugin/GreptimeDB.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 15 KiB

20
core/datacap-web/console-fe/src/components/table/BasicTable.vue
View File

@ -1,9 +1,10 @@
<template>
<div>
<Card :padding="5">
<Button size="small" type="primary" @click="columnDrawerVisible = true" icon="md-list" style="margin-right: 10px;"/>
<Space>
<Button size="small" type="primary" @click="columnDrawerVisible = true" icon="md-list"/>
<Tooltip :content="$t('tooltip.pageShow')">
<Switch v-model="isPage" size="small" style="margin-right: 10px;" @on-change="handlerChange">
<Switch v-model="isPage" size="small" @on-change="handlerChange">
<template #open>
<Icon type="md-check"></Icon>
</template>
@ -12,9 +13,9 @@
</template>
</Switch>
</Tooltip>
<Dropdown style="margin-right: 10px;">
<Dropdown>
<Button type="primary" size="small">
Export
{{ $t('common.export') }}
<Icon type="md-download"/>
</Button>
<template #list>
@ -27,6 +28,13 @@
<Button size="small" icon="md-pie" @click="handlerVisualization(true)">
</Button>
</Tooltip>
<Poptip trigger="hover" placement="bottom" word-wrap width="150">
<Button size="small" shape="circle" type="warning" icon="md-help"/>
<template #content>
{{ $t('tooltip.smallTips') }}
</template>
</Poptip>
</Space>
<ag-grid-vue :key="timestamp" :style="{width: configure.width + 'px', height: configure.height + 'px', 'margin-top': '2px'}" :pagination="isPage"
class="ag-theme-datacap" :columnDefs="columnDefs" :rowData="configure.columns" :gridOptions="gridOptions">
</ag-grid-vue>
@ -54,12 +62,12 @@ import {ExportToCsv} from "export-to-csv";
import {AgGridVue} from "ag-grid-vue3";
import "ag-grid-community/styles/ag-grid.css";
import "./ag-theme-datacap.css";
import {createDefaultOptions} from "./TableGridOptions";
import {useI18n} from "vue-i18n";
import {TableColumnDef} from "@/components/table/TableColumnDef";
import {getTimestamp} from "@/common/DateCommon";
import EchartsEditor from "@/components/editor/echarts/EchartsEditor.vue";
import {EchartsConfigure} from "@/components/editor/echarts/EchartsConfigure";
import TableGridOptions from "@/components/table/TableGridOptions";
export default defineComponent({
name: "BasicTableComponent",
@ -73,7 +81,7 @@ export default defineComponent({
setup()
{
const i18n = useI18n();
const gridOptions = createDefaultOptions(i18n);
const gridOptions = TableGridOptions.createDefaultOptions(i18n);
const timestamp = getTimestamp();
return {
gridOptions,

37
core/datacap-web/console-fe/src/components/table/TableGridOptions.ts
View File

@ -1,5 +1,11 @@
const createDefaultOptions = (i18n: any) => {
return {
import useClipboard from "vue-clipboard3";
import {join} from "lodash";
class TableGridOptions
{
createDefaultOptions(i18n: any)
{
const gridOptions = {
enableSorting: true,
enableFilter: true,
enableColResize: true,
@ -9,6 +15,24 @@ const createDefaultOptions = (i18n: any) => {
rowDragManaged: true,
animateRows: true,
allowContextMenuWithControlKey: true,
// Support copy multiple selection
rowSelection: 'multiple',
onSelectionChanged: (instance) => {
let index = 0;
const separator = '\t';
const selectedRows = instance.api.getSelectedRows();
const copyRows = [];
selectedRows.forEach(row => {
if (index === 0) {
copyRows.push(join(Object.keys(row), separator));
index += 1;
}
copyRows.push(join(Object.values(row), separator));
});
useClipboard()
.toClipboard(join(copyRows, '\n'))
.then(() => console.log('Copy'));
},
// Fixed issues: https://github.com/EdurtIO/datacap/issues/219
suppressFieldDotNotation: true,
localeText: {
@ -81,11 +105,12 @@ const createDefaultOptions = (i18n: any) => {
editable: true,
sortable: true,
filter: true,
resizable: true
resizable: true,
wrapText: true
}
}
return gridOptions;
}
}
export {
createDefaultOptions
};
export default new TableGridOptions();

3
core/datacap-web/console-fe/src/i18n/langs/en/common.ts
View File

@ -101,5 +101,6 @@ export default {
xAxis: 'xAxis',
yAxis: 'yAxis',
tag: 'Tag',
data: 'Data'
data: 'Data',
export: 'Export'
}

3
core/datacap-web/console-fe/src/i18n/langs/en/tooltip.ts
View File

@ -1,5 +1,6 @@
export default {
elapsedMillisecond: 'The value is expressed in milliseconds',
pageShow: 'Open / Close the page',
multipleLines: 'One for each line. Multiple lines are represented by newline characters'
multipleLines: 'One for each line. Multiple lines are represented by newline characters',
smallTips: 'Tip: hold down the keyboard Shift and select the number of data rows with the mouse to automatically copy the contents of the selected row'
}

3
core/datacap-web/console-fe/src/i18n/langs/zhCn/common.ts
View File

@ -101,5 +101,6 @@ export default {
xAxis: '横轴',
yAxis: '纵轴',
tag: '标签',
data: '数据'
data: '数据',
export: '导出'
}

3
core/datacap-web/console-fe/src/i18n/langs/zhCn/tooltip.ts
View File

@ -1,5 +1,6 @@
export default {
elapsedMillisecond: '单位为毫秒',
pageShow: '开启/关闭分页',
multipleLines: '每行一个，多个使用换行符表示'
multipleLines: '每行一个，多个使用换行符表示',
smallTips: '小技巧：按住键盘 Shift 鼠标选择数据行数会自动复制选中行内容'
}

10
core/datacap-web/console-fe/src/views/pages/admin/source/SourceDetail.vue
View File

@ -7,8 +7,7 @@
<Card :bordered="false" dis-hover>
<div style="text-align:center">
<Avatar :size="40"
:src="formState?.type ? '/static/images/plugin/' + formState?.type.split(' ')[0] + '.png' : ''"
:style="{'background-color': !testInfo.percent ? '#CCC' : testInfo.connected ? '#52c41a' : '#ff4d4f'}" icon="ios-person"/>
:src="formState?.type ? '/static/images/plugin/' + formState?.type.split(' ')[0] + '.png' : ''" icon="ios-person"/>
<p>{{ !formState['type'] ? '_' : formState['type'] }}</p>
</div>
</Card>
@ -16,7 +15,7 @@
<Col :span="6">
<Card :bordered="false" dis-hover>
<div style="text-align:center">
<Progress :percent="testInfo.percent" :status="testInfo.connected ? 'success' : 'wrong'">
<Progress :percent="testInfo.percent" :status="(testInfo.connected && testInfo.successful) ? 'success' : 'wrong'">
<span></span>
</Progress>
</div>
@ -127,7 +126,8 @@ import SourceV2Service from "@/services/SourceV2Service";
interface TestInfo
{
connected: boolean,
percent: number
percent: number,
successful: boolean
}
export default defineComponent({
@ -249,10 +249,12 @@ export default defineComponent({
if (response.status) {
this.$Message.success("Test successful");
this.testInfo.connected = true;
this.testInfo.successful = true;
}
else {
this.$Message.error(response.message);
this.testInfo.connected = false;
this.testInfo.successful = false;
}
})
.finally(() => {

BIN
docs/docs/assets/plugin/greptimedb.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.4 KiB

3
docs/docs/index.md
View File

@ -184,6 +184,9 @@ Datacap is fast, lightweight, intuitive system.
</a>&nbsp;
<a href="https://docs.ceresdb.io/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/ceresdb.png" alt="CeresDB" height="50" />
</a>&nbsp;
<a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" />
</a>
</p>

3
docs/docs/index.zh.md
View File

@ -184,6 +184,9 @@ Datacap 是快速、轻量级、直观的系统。
</a>&nbsp;
<a href="https://docs.ceresdb.io/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/ceresdb.png" alt="CeresDB" height="50" />
</a>&nbsp;
<a href="https://docs.greptime.com/" target="_blank" class="connector-logo-index">
<img src="/assets/plugin/greptimedb.png" alt="GreptimeDB" height="70" />
</a>
</p>

44
docs/docs/reference/connectors/http/greptimedb.md Normal file
View File

@ -0,0 +1,44 @@
---
title: GreptimeDB
status: new
---
<img src="/assets/plugin/greptimedb.png" class="connector-logo" />
#### What is GreptimeDB ?
An open-source, cloud-native, distributed time-series database with PromQL/SQL/Python supported.
#### Environment
!!! note
If you need to use this data source, you need to upgrade the DataCap service to >= `1.9.x`
Support Time: `2023-04-14`
#### Configure
---
!!! note
If your GreptimeDB service version requires other special configurations, please refer to modifying the configuration file and restarting the DataCap service.
=== "Configure"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `4000` |
#### Version (Validation)
---
!!! warning
The online service has not been tested yet, if you have detailed test results, please submit [issues](https://github.com/EdurtIO/datacap/issues/new/choose) to us
- [x] 0.x

44
docs/docs/reference/connectors/http/greptimedb.zh.md Normal file
View File

@ -0,0 +1,44 @@
---
title: GreptimeDB
status: new
---
<img src="/assets/plugin/greptimedb.png" class="connector-logo" />
#### 什么是 GreptimeDB ?
一个开源、云原生、分布式时间序列数据库，支持PromQL/SQL/Python。
#### 环境
!!! note
如果你需要使用这个数据源, 您需要将 DataCap 服务升级到 >= `1.9.x`
支持时间: `2023-04-14`
#### 配置
---
!!! note
如果您的 GreptimeDB 服务版本需要其他特殊配置，请参考修改配置文件并重启 DataCap 服务。
=== "配置"
| Field | Required | Default Value |
|:------:|:---------------------------------:|:-------------:|
| `Name` | :material-check-circle: { .red } | - |
| `Host` | :material-check-circle: { .red } | `127.0.0.1` |
| `Port` | :material-check-circle: { .red } | `5440` |
#### 版本（验证）
---
!!! warning
服务版本尚未测试，如果您有详细的测试并发现错误，请提交 [issues](https://github.com/EdurtIO/datacap/issues/new/choose)
- [x] 0.x

1
docs/mkdocs.yml
View File

@ -165,6 +165,7 @@ nav:
- Zookeeper: reference/connectors/native/zookeeper.md
- Redis: reference/connectors/native/redis.md
- Http:
- GreptimeDB: reference/connectors/http/greptimedb.md
- CeresDB: reference/connectors/http/ceresdb.md
- ClickHouse: reference/connectors/http/clickhouse.md
- Developer guide:

2
plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/natived/ceresdb/CeresDBAdapter.java → plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBAdapter.java
View File

@ -1,4 +1,4 @@
package io.edurt.datacap.plugin.natived.ceresdb;
package io.edurt.datacap.plugin.http.ceresdb;
import io.edurt.datacap.spi.adapter.HttpAdapter;
import io.edurt.datacap.spi.connection.HttpConfigure;

2
plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/natived/ceresdb/CeresDBPlugin.java → plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBPlugin.java
View File

@ -1,4 +1,4 @@
package io.edurt.datacap.plugin.natived.ceresdb;
package io.edurt.datacap.plugin.http.ceresdb;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginType;

2
plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/natived/ceresdb/CeresDBPluginModule.java → plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBPluginModule.java
View File

@ -1,4 +1,4 @@
package io.edurt.datacap.plugin.natived.ceresdb;
package io.edurt.datacap.plugin.http.ceresdb;
import com.google.inject.multibindings.Multibinder;
import io.edurt.datacap.spi.AbstractPluginModule;

2
plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/natived/ceresdb/CeresDBResponse.java → plugin/datacap-http-ceresdb/src/main/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBResponse.java
View File

@ -1,4 +1,4 @@
package io.edurt.datacap.plugin.natived.ceresdb;
package io.edurt.datacap.plugin.http.ceresdb;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.AllArgsConstructor;

2
plugin/datacap-http-ceresdb/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule
View File

@ -1 +1 @@
io.edurt.datacap.plugin.natived.ceresdb.CeresDBPluginModule
io.edurt.datacap.plugin.http.ceresdb.CeresDBPluginModule

2
plugin/datacap-http-ceresdb/src/test/java/io/edurt/datacap/plugin/natived/ceresdb/CeresDBContainer.java → plugin/datacap-http-ceresdb/src/test/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBContainer.java
View File

@ -1,4 +1,4 @@
package io.edurt.datacap.plugin.natived.ceresdb;
package io.edurt.datacap.plugin.http.ceresdb;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.DockerImageName;

2
plugin/datacap-http-ceresdb/src/test/java/io/edurt/datacap/plugin/natived/ceresdb/CeresDBPluginModuleTest.java → plugin/datacap-http-ceresdb/src/test/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBPluginModuleTest.java
View File

@ -1,4 +1,4 @@
package io.edurt.datacap.plugin.natived.ceresdb;
package io.edurt.datacap.plugin.http.ceresdb;
import com.google.inject.Guice;
import com.google.inject.Injector;

2
plugin/datacap-http-ceresdb/src/test/java/io/edurt/datacap/plugin/natived/ceresdb/CeresDBPluginTest.java → plugin/datacap-http-ceresdb/src/test/java/io/edurt/datacap/plugin/http/ceresdb/CeresDBPluginTest.java
View File

@ -1,4 +1,4 @@
package io.edurt.datacap.plugin.natived.ceresdb;
package io.edurt.datacap.plugin.http.ceresdb;
import com.google.inject.Guice;
import com.google.inject.Injector;

75
plugin/datacap-http-greptime/pom.xml Normal file
View File

@ -0,0 +1,75 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap</artifactId>
<version>1.9.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>datacap-http-greptime</artifactId>
<description>DataCap - GreptimeDB</description>
<properties>
<plugin.name>http-greptime</plugin.name>
</properties>
<dependencies>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-common</artifactId>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>${assembly-plugin.version}</version>
<configuration>
<finalName>${plugin.name}</finalName>
<descriptors>
<descriptor>../../configure/assembly/assembly-plugin.xml</descriptor>
</descriptors>
<outputDirectory>../../dist/plugins/${plugin.name}</outputDirectory>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

91
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/GreptimeDBAdapter.java Normal file
View File

@ -0,0 +1,91 @@
package io.edurt.datacap.plugin.http.greptime;
import com.google.common.collect.Maps;
import io.edurt.datacap.plugin.http.greptime.response.GreptimeDBResponse;
import io.edurt.datacap.plugin.http.greptime.response.record.Records;
import io.edurt.datacap.spi.adapter.HttpAdapter;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import io.edurt.datacap.spi.connection.http.HttpClient;
import io.edurt.datacap.spi.connection.http.HttpMethod;
import io.edurt.datacap.spi.json.JSON;
import io.edurt.datacap.spi.model.Response;
import io.edurt.datacap.spi.model.Time;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang3.ObjectUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
@Slf4j
public class GreptimeDBAdapter
extends HttpAdapter
{
public GreptimeDBAdapter(HttpConnection connection)
{
super(connection);
}
@Override
public Response handlerExecute(String content)
{
Time processorTime = new Time();
processorTime.setStart(new Date().getTime());
Response response = this.httpConnection.getResponse();
HttpConfigure configure = new HttpConfigure();
if (response.getIsConnected()) {
List<String> headers = new ArrayList<>();
List<String> types = new ArrayList<>();
List<Object> columns = new ArrayList<>();
try {
BeanUtils.copyProperties(configure, this.httpConnection.getConfigure());
configure.setAutoConnected(Boolean.FALSE);
configure.setRetry(0);
configure.setMethod(HttpMethod.POST);
configure.setPath("v1/sql");
Map<String, String> parameters = Maps.newHashMap();
parameters.put("sql", content);
configure.setParams(parameters);
configure.setDecoded(true);
HttpConnection httpConnection = new HttpConnection(configure, new Response());
HttpClient httpClient = HttpClient.getInstance(configure, httpConnection);
String body = httpClient.execute();
GreptimeDBResponse requestResponse = JSON.objectmapper.readValue(body, GreptimeDBResponse.class);
if (requestResponse.getCode() == 0 || ObjectUtils.isNotEmpty(requestResponse.getOutput())) {
response.setIsSuccessful(true);
Records records = requestResponse.getOutput().get(0).getRecords();
if (ObjectUtils.isNotEmpty(records.getSchema())) {
records.getSchema()
.getColumnSchemas()
.forEach(schema -> {
headers.add(schema.getName());
types.add(schema.getDataType());
});
}
records.getRows()
.forEach(record -> columns.add(handlerFormatter(configure.getFormat(), headers, record)));
}
else {
response.setIsSuccessful(Boolean.FALSE);
response.setMessage(requestResponse.getError());
}
}
catch (Exception ex) {
log.error("Execute content failed content {} exception ", content, ex);
response.setIsSuccessful(Boolean.FALSE);
response.setMessage(ex.getMessage());
}
finally {
response.setHeaders(headers);
response.setTypes(types);
response.setColumns(columns);
}
}
processorTime.setEnd(new Date().getTime());
response.setProcessor(processorTime);
return response;
}
}

81
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/GreptimeDBPlugin.java Normal file
View File

@ -0,0 +1,81 @@
package io.edurt.datacap.plugin.http.greptime;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginType;
import io.edurt.datacap.spi.connection.HttpConfigure;
import io.edurt.datacap.spi.connection.HttpConnection;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang3.ObjectUtils;
@Slf4j
public class GreptimeDBPlugin
implements Plugin
{
private HttpConfigure configure;
private HttpConnection connection;
private Response response;
@Override
public String validator()
{
return "SELECT 1";
}
@Override
public String name()
{
return "GreptimeDB";
}
@Override
public String description()
{
return "Integrate GreptimeDB data sources";
}
@Override
public PluginType type()
{
return PluginType.HTTP;
}
@Override
public void connect(Configure configure)
{
try {
this.response = new Response();
this.configure = new HttpConfigure();
BeanUtils.copyProperties(this.configure, configure);
this.connection = new HttpConnection(this.configure, this.response);
}
catch (Exception ex) {
this.response.setIsConnected(Boolean.FALSE);
this.response.setMessage(ex.getMessage());
}
}
@Override
public Response execute(String content)
{
if (ObjectUtils.isNotEmpty(this.connection)) {
log.info("Execute greptimedb plugin logic started");
this.response = this.connection.getResponse();
GreptimeDBAdapter processor = new GreptimeDBAdapter(this.connection);
this.response = processor.handlerExecute(content);
log.info("Execute greptimedb plugin logic end");
}
this.destroy();
return this.response;
}
@Override
public void destroy()
{
if (ObjectUtils.isNotEmpty(this.connection)) {
this.connection.destroy();
}
}
}

38
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/GreptimeDBPluginModule.java Normal file
View File

@ -0,0 +1,38 @@
package io.edurt.datacap.plugin.http.greptime;
import com.google.inject.multibindings.Multibinder;
import io.edurt.datacap.spi.AbstractPluginModule;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginModule;
import io.edurt.datacap.spi.PluginType;
public class GreptimeDBPluginModule
extends AbstractPluginModule
implements PluginModule
{
@Override
public String getName()
{
return "GreptimeDB";
}
@Override
public PluginType getType()
{
return PluginType.HTTP;
}
@Override
public AbstractPluginModule get()
{
return this;
}
protected void configure()
{
Multibinder<String> module = Multibinder.newSetBinder(this.binder(), String.class);
module.addBinding().toInstance(this.getClass().getSimpleName());
Multibinder<Plugin> plugin = Multibinder.newSetBinder(this.binder(), Plugin.class);
plugin.addBinding().to(GreptimeDBPlugin.class);
}
}

22
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/response/GreptimeDBResponse.java Normal file
View File

@ -0,0 +1,22 @@
package io.edurt.datacap.plugin.http.greptime.response;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.edurt.datacap.plugin.http.greptime.response.record.Output;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.List;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class GreptimeDBResponse
{
private int code;
private String error;
private List<Output> output;
}

20
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/response/record/ColumnSchema.java Normal file
View File

@ -0,0 +1,20 @@
package io.edurt.datacap.plugin.http.greptime.response.record;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class ColumnSchema
{
private String name;
@JsonProperty(value = "data_type")
private String dataType;
}

17
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/response/record/Output.java Normal file
View File

@ -0,0 +1,17 @@
package io.edurt.datacap.plugin.http.greptime.response.record;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class Output
{
public Records records;
}

20
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/response/record/Records.java Normal file
View File

@ -0,0 +1,20 @@
package io.edurt.datacap.plugin.http.greptime.response.record;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.List;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class Records
{
public Schema schema;
public List<List<Object>> rows;
}

21
plugin/datacap-http-greptime/src/main/java/io/edurt/datacap/plugin/http/greptime/response/record/Schema.java Normal file
View File

@ -0,0 +1,21 @@
package io.edurt.datacap.plugin.http.greptime.response.record;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.ArrayList;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class Schema
{
@JsonProperty(value = "column_schemas")
public ArrayList<ColumnSchema> columnSchemas;
}

1
plugin/datacap-http-greptime/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule Normal file
View File

@ -0,0 +1 @@
io.edurt.datacap.plugin.http.greptime.GreptimeDBPluginModule

56
plugin/datacap-http-greptime/src/test/java/io/edurt/datacap/plugin/http/greptime/GreptimeDBContainer.java Normal file
View File

@ -0,0 +1,56 @@
package io.edurt.datacap.plugin.http.greptime;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.DockerImageName;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.util.Enumeration;
public class GreptimeDBContainer
extends GenericContainer<GreptimeDBContainer>
{
private static final DockerImageName DEFAULT_IMAGE_NAME =
DockerImageName.parse("greptime/greptimedb");
public static final int PORT = 4001;
public static final int HTTP_PORT = 4000;
public GreptimeDBContainer()
{
super(DEFAULT_IMAGE_NAME);
withExposedPorts(PORT, HTTP_PORT);
}
public GreptimeDBContainer(final DockerImageName dockerImageName)
{
super(dockerImageName);
dockerImageName.assertCompatibleWith(dockerImageName);
withExposedPorts(PORT, HTTP_PORT);
}
public String getLinuxLocalIp()
{
String ip = "";
try {
Enumeration<NetworkInterface> networkInterfaces =
NetworkInterface.getNetworkInterfaces();
while (networkInterfaces.hasMoreElements()) {
NetworkInterface networkInterface = networkInterfaces.nextElement();
Enumeration<InetAddress> inetAddresses = networkInterface.getInetAddresses();
while (inetAddresses.hasMoreElements()) {
InetAddress inetAddress = inetAddresses.nextElement();
if (!inetAddress.isLoopbackAddress() && inetAddress instanceof Inet4Address) {
ip = inetAddress.getHostAddress();
}
}
}
}
catch (SocketException ex) {
ex.printStackTrace();
}
return ip;
}
}

30
plugin/datacap-http-greptime/src/test/java/io/edurt/datacap/plugin/http/greptime/GreptimeDBPluginModuleTest.java Normal file
View File

@ -0,0 +1,30 @@
package io.edurt.datacap.plugin.http.greptime;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.Plugin;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Set;
public class GreptimeDBPluginModuleTest
{
private Injector injector;
@Before
public void before()
{
this.injector = Guice.createInjector(new GreptimeDBPluginModule());
}
@Test
public void test()
{
Set<Plugin> plugins = injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}));
Assert.assertTrue(plugins.size() > 0);
}
}

76
plugin/datacap-http-greptime/src/test/java/io/edurt/datacap/plugin/http/greptime/GreptimeDBPluginTest.java Normal file
View File

@ -0,0 +1,76 @@
package io.edurt.datacap.plugin.http.greptime;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.lifecycle.Startables;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
@Slf4j
public class GreptimeDBPluginTest
{
private static final String HOST = "greptimeDBCluster";
private Network network;
private GreptimeDBContainer container;
private Injector injector;
private Configure configure;
@Before
public void before()
{
network = Network.newNetwork();
container = new GreptimeDBContainer()
.withNetwork(network)
.withNetworkAliases(HOST)
.withExposedPorts(GreptimeDBContainer.HTTP_PORT)
.withCommand("standalone", "start",
"--http-addr", "0.0.0.0:4000")
.waitingFor(Wait.forHttp("/dashboard"));
Startables.deepStart(Stream.of(container)).join();
log.info("GreptimeDB container started");
injector = Guice.createInjector(new GreptimeDBPluginModule());
configure = new Configure();
configure.setHost("localhost");
configure.setPort(container.getMappedPort(GreptimeDBContainer.HTTP_PORT));
configure.setDatabase(Optional.of("default"));
}
@Test
public void test()
{
Set<Plugin> plugins = injector.getInstance(Key.get(new TypeLiteral<Set<Plugin>>() {}));
Optional<Plugin> pluginOptional = plugins.stream()
.filter(v -> v.name().equalsIgnoreCase("GreptimeDB"))
.findFirst();
if (pluginOptional.isPresent()) {
Plugin plugin = pluginOptional.get();
plugin.connect(configure);
String sql = "SELECT * FROM numbers LIMIT 5";
Response response = plugin.execute(sql);
log.info("================ plugin executed information =================");
if (!response.getIsSuccessful()) {
log.error("Message: {}", response.getMessage());
}
else {
response.getColumns().forEach(column -> log.info(column.toString()));
}
Assert.assertTrue(response.getIsSuccessful());
plugin.destroy();
}
}
}

1
pom.xml
View File

@ -25,6 +25,7 @@
<module>plugin/datacap-http-cratedb</module>
<module>plugin/datacap-http-clickhouse</module>
<module>plugin/datacap-http-ceresdb</module>
<module>plugin/datacap-http-greptime</module>
<module>plugin/datacap-jdbc-clickhouse</module>
<module>plugin/datacap-jdbc-cratedb</module>
<module>plugin/datacap-jdbc-db2</module>