From 26518270a77d923e2daea321a053a83002d4906f Mon Sep 17 00:00:00 2001
From: qianmoQ
Date: Tue, 6 Jun 2023 22:58:42 +0800
Subject: [PATCH] [Plugin] Support apache cassandra
---
README.md | 3 +
core/datacap-server/pom.xml | 6 +
.../main/etc/conf/plugins/jdbc/cassandra.yaml | 26 +++
.../server/entity/TemplateSqlEntity.java | 15 ++
.../src/main/resources/schema.sql | 28 ++-
.../src/main/schema/1.11.0/update.sql | 22 ++
.../src/main/schema/datacap.sql | 198 ++++++++++--------
.../io/edurt/datacap/spi/adapter/Adapter.java | 9 +
.../datacap/spi/adapter/HttpAdapter.java | 2 +-
.../datacap/spi/adapter/JdbcAdapter.java | 2 +-
.../datacap/spi/adapter/NativeAdapter.java | 2 +-
.../public/static/images/plugin/cassandra.png | Bin 0 -> 154321 bytes
docs/docs/assets/plugin/cassandra.png | Bin 0 -> 154321 bytes
docs/docs/index.md | 3 +
docs/docs/index.zh.md | 3 +
docs/docs/reference/connectors/cassandra.md | 53 +++++
docs/mkdocs.yml | 2 +
plugin/datacap-plugin-cassandra/pom.xml | 87 ++++++++
.../plugin/cassandra/CassandraAdapter.kt | 66 ++++++
.../plugin/cassandra/CassandraConnection.kt | 44 ++++
.../plugin/cassandra/CassandraModule.kt | 22 ++
.../plugin/cassandra/CassandraPlugin.kt | 52 +++++
.../io.edurt.datacap.spi.PluginModule | 1 +
.../plugin/cassandra/CassandraPluginTest.kt | 47 +++++
pom.xml | 1 +
25 files changed, 600 insertions(+), 94 deletions(-)
create mode 100644 core/datacap-server/src/main/etc/conf/plugins/jdbc/cassandra.yaml
create mode 100644 core/datacap-web/console-fe/public/static/images/plugin/cassandra.png
create mode 100644 docs/docs/assets/plugin/cassandra.png
create mode 100644 docs/docs/reference/connectors/cassandra.md
create mode 100644 plugin/datacap-plugin-cassandra/pom.xml
create mode 100644 plugin/datacap-plugin-cassandra/src/main/kotlin/io/edurt/datacap/plugin/cassandra/CassandraAdapter.kt
create mode 100644 plugin/datacap-plugin-cassandra/src/main/kotlin/io/edurt/datacap/plugin/cassandra/CassandraConnection.kt
create mode 100644 plugin/datacap-plugin-cassandra/src/main/kotlin/io/edurt/datacap/plugin/cassandra/CassandraModule.kt
create mode 100644 plugin/datacap-plugin-cassandra/src/main/kotlin/io/edurt/datacap/plugin/cassandra/CassandraPlugin.kt
create mode 100644 plugin/datacap-plugin-cassandra/src/main/resources/META-INF/services/io.edurt.datacap.spi.PluginModule
create mode 100644 plugin/datacap-plugin-cassandra/src/test/kotlin/io/edurt/datacap/plugin/cassandra/CassandraPluginTest.kt
diff --git a/README.md b/README.md
index b8c37b62..61e2d0d4 100644
--- a/README.md
+++ b/README.md
@@ -181,6 +181,9 @@ Here are some of the major database solutions that are supported:
+
+
+
diff --git a/core/datacap-server/pom.xml b/core/datacap-server/pom.xml
index 6fa0c877..10fd7c31 100644
--- a/core/datacap-server/pom.xml
+++ b/core/datacap-server/pom.xml
@@ -410,6 +410,12 @@
${project.version}
provided
+
+ io.edurt.datacap
+ datacap-plugin-cassandra
+ ${project.version}
+ provided
+
io.edurt.datacap
diff --git a/core/datacap-server/src/main/etc/conf/plugins/jdbc/cassandra.yaml b/core/datacap-server/src/main/etc/conf/plugins/jdbc/cassandra.yaml
new file mode 100644
index 00000000..7021a829
--- /dev/null
+++ b/core/datacap-server/src/main/etc/conf/plugins/jdbc/cassandra.yaml
@@ -0,0 +1,26 @@
+name: Apache Cassandra
+supportTime: '2023-06-07'
+
+configures:
+ - field: name
+ type: String
+ required: true
+ message: name is a required field, please be sure to enter
+ - field: host
+ type: String
+ required: true
+ value: 127.0.0.1
+ message: host is a required field, please be sure to enter
+ - field: port
+ type: Number
+ required: true
+ min: 1
+ max: 65535
+ value: 9042
+ message: port is a required field, please be sure to enter
+ - field: database
+ type: String
+ required: true
+ value: datacenter
+ message: database is a required field, please be sure to enter
+ group: advanced
diff --git a/core/datacap-server/src/main/java/io/edurt/datacap/server/entity/TemplateSqlEntity.java b/core/datacap-server/src/main/java/io/edurt/datacap/server/entity/TemplateSqlEntity.java
index 1a3d0e7a..4573d4d8 100644
--- a/core/datacap-server/src/main/java/io/edurt/datacap/server/entity/TemplateSqlEntity.java
+++ b/core/datacap-server/src/main/java/io/edurt/datacap/server/entity/TemplateSqlEntity.java
@@ -12,11 +12,14 @@ import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
import javax.persistence.Table;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.sql.Timestamp;
+import java.time.LocalDateTime;
@Entity
@Data
@@ -72,4 +75,16 @@ public class TemplateSqlEntity
@Column(name = "update_time", columnDefinition = "timestamp not null default current_timestamp")
private Timestamp updateTime;
+
+ @PrePersist
+ void prePersist()
+ {
+ createTime = Timestamp.valueOf(LocalDateTime.now());
+ }
+
+ @PreUpdate
+ void preUpdate()
+ {
+ updateTime = Timestamp.valueOf(LocalDateTime.now());
+ }
}
diff --git a/core/datacap-server/src/main/resources/schema.sql b/core/datacap-server/src/main/resources/schema.sql
index d76d82d9..e4598641 100644
--- a/core/datacap-server/src/main/resources/schema.sql
+++ b/core/datacap-server/src/main/resources/schema.sql
@@ -136,9 +136,9 @@ CREATE TABLE IF NOT EXISTS template_sql
description text NULL,
plugin varchar(50) NULL COMMENT 'Using plug-ins',
configure text NULL COMMENT 'The template must use the configuration information in the key->value format',
- create_time timestamp DEFAULT CURRENT_TIMESTAMP,
- update_time timestamp DEFAULT CURRENT_TIMESTAMP,
- `system` boolean NULL DEFAULT 0
+ create_time timestamp DEFAULT CURRENT_TIMESTAMP,
+ update_time timestamp DEFAULT CURRENT_TIMESTAMP,
+ `system` boolean NULL DEFAULT 0
);
TRUNCATE TABLE template_sql;
ALTER TABLE template_sql
@@ -283,3 +283,25 @@ VALUES ('datacap', '$2a$10$bZ4XBRlYUjKfkBovWT9TuuXlEF7lpRxVrXS8iqyCjCHUqy4RPTL8.
-- --------------------------------
alter table `audit_plugin`
add column `count` bigint default 0;
+
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllDatabase', 'SELECT keyspace_name AS name
+FROM system_schema.keyspaces', 'Gets a list of all databases', 'Cassandra', '[]', 1);
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllTablesFromDatabase', 'SELECT
+ table_name AS name
+FROM
+ system_schema.tables
+WHERE
+ keyspace_name = ''${database:String}''
+GROUP BY
+ table_name', 'Get the data table from the database', 'Cassandra', '[{"column":"database","type":"String","expression":"${database:String}"}]', 1);
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllColumnsFromDatabaseAndTable', 'SELECT
+ column_name
+FROM
+ system_schema.columns
+WHERE
+ keyspace_name = ''${database:String}''
+ and table_name = ''${table:String}''', 'Get the data column from the database and table', 'Cassandra',
+ '[{"column":"database","type":"String","expression":"${database:String}"},{"column":"table","type":"String","expression":"${table:String}"}]', 1);
diff --git a/core/datacap-server/src/main/schema/1.11.0/update.sql b/core/datacap-server/src/main/schema/1.11.0/update.sql
index b8ca4e8f..d6070bc5 100644
--- a/core/datacap-server/src/main/schema/1.11.0/update.sql
+++ b/core/datacap-server/src/main/schema/1.11.0/update.sql
@@ -35,3 +35,25 @@ where url = '/admin/menu';
update `menus`
set url = '/system/user'
where url = '/admin/user';
+
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllDatabase', 'SELECT keyspace_name AS name
+FROM system_schema.keyspaces', 'Gets a list of all databases', 'Cassandra', '[]', 1);
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllTablesFromDatabase', 'SELECT
+ table_name AS name
+FROM
+ system_schema.tables
+WHERE
+ keyspace_name = ''${database:String}''
+GROUP BY
+ table_name', 'Get the data table from the database', 'Cassandra', '[{"column":"database","type":"String","expression":"${database:String}"}]', 1);
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllColumnsFromDatabaseAndTable', 'SELECT
+ column_name
+FROM
+ system_schema.columns
+WHERE
+ keyspace_name = ''${database:String}''
+ and table_name = ''${table:String}''', 'Get the data column from the database and table', 'Cassandra',
+ '[{"column":"database","type":"String","expression":"${database:String}"},{"column":"table","type":"String","expression":"${table:String}"}]', 1);
diff --git a/core/datacap-server/src/main/schema/datacap.sql b/core/datacap-server/src/main/schema/datacap.sql
index e4f6c8c7..32ff8554 100644
--- a/core/datacap-server/src/main/schema/datacap.sql
+++ b/core/datacap-server/src/main/schema/datacap.sql
@@ -1,5 +1,5 @@
use
-datacap;
+ datacap;
-- --------------------------------
-- Table for audit_plugin
@@ -7,18 +7,18 @@ datacap;
create table audit_plugin
(
id bigint auto_increment primary key,
- state varchar(255) null,
- create_time mediumtext null,
- end_time mediumtext null,
- plugin_id bigint not null,
- content text null,
- message text null,
+ state varchar(255) null,
+ create_time mediumtext null,
+ end_time mediumtext null,
+ plugin_id bigint not null,
+ content text null,
+ message text null,
elapsed bigint default 0 null,
- user_id bigint not null
+ user_id bigint not null
);
create
-fulltext index full_text_index_for_content
+ fulltext index full_text_index_for_content
on audit_plugin (content);
-- --------------------------------
@@ -27,14 +27,14 @@ fulltext index full_text_index_for_content
create table functions
(
id bigint auto_increment primary key,
- name varchar(255) null comment 'Function name',
- content varchar(255) null comment 'Expression of function',
- description text null comment 'Function description',
- plugin varchar(255) null comment 'Trial plug-in, multiple according to, split',
- example text null comment 'Function Usage Example',
+ name varchar(255) null comment 'Function name',
+ content varchar(255) null comment 'Expression of function',
+ description text null comment 'Function description',
+ plugin varchar(255) null comment 'Trial plug-in, multiple according to, split',
+ example text null comment 'Function Usage Example',
create_time datetime default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP,
update_time datetime default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP,
- type varchar(20) default 'KEYWORDS' null
+ type varchar(20) default 'KEYWORDS' null
) comment 'Plug-in correlation function';
-- --------------------------------
@@ -43,19 +43,19 @@ create table functions
create table pipeline
(
id int auto_increment primary key,
- name varchar(255) not null,
- content text not null,
- state varchar(100) null,
- message text null,
- work text null,
+ name varchar(255) not null,
+ content text not null,
+ state varchar(100) null,
+ message text null,
+ work text null,
start_time datetime default CURRENT_TIMESTAMP null,
- end_time datetime null on update CURRENT_TIMESTAMP,
- elapsed bigint null,
- user_id int not null,
- from_id int not null,
- from_configures text null,
- to_id int not null,
- to_configures text null
+ end_time datetime null on update CURRENT_TIMESTAMP,
+ elapsed bigint null,
+ user_id int not null,
+ from_id int not null,
+ from_configures text null,
+ to_id int not null,
+ to_configures text null
);
-- --------------------------------
@@ -64,9 +64,9 @@ create table pipeline
create table role
(
id bigint auto_increment primary key,
- name varchar(255) null comment ' ',
- description varchar(255) null comment ' ',
- create_time datetime(5) default CURRENT_TIMESTAMP (5) null
+ name varchar(255) null comment ' ',
+ description varchar(255) null comment ' ',
+ create_time datetime(5) default CURRENT_TIMESTAMP(5) null
);
INSERT INTO datacap.role (name, description)
@@ -80,13 +80,13 @@ VALUES ('User', 'User role');
create table datacap.scheduled_task
(
id int auto_increment primary key,
- name varchar(255) not null,
- description text not null,
- expression varchar(100) null,
- active tinyint(1) default 1 null,
- is_system tinyint(1) default 1 null,
- create_time datetime default CURRENT_TIMESTAMP null,
- update_time datetime null on update CURRENT_TIMESTAMP
+ name varchar(255) not null,
+ description text not null,
+ expression varchar(100) null,
+ active tinyint(1) default 1 null,
+ is_system tinyint(1) default 1 null,
+ create_time datetime default CURRENT_TIMESTAMP null,
+ update_time datetime null on update CURRENT_TIMESTAMP
);
INSERT INTO datacap.scheduled_task (name, description, expression, active, is_system)
@@ -98,16 +98,16 @@ VALUES ('Synchronize table structure', 'Synchronize the table structure of the d
create table datacap.snippet
(
id bigint auto_increment primary key,
- name varchar(255) null comment ' ',
- description varchar(255) null comment ' ',
- code text null comment ' ',
+ name varchar(255) null comment ' ',
+ description varchar(255) null comment ' ',
+ code text null comment ' ',
create_time timestamp default CURRENT_TIMESTAMP not null,
update_time timestamp default CURRENT_TIMESTAMP not null,
user_id bigint not null
);
create
-fulltext index full_text_index_for_code
+ fulltext index full_text_index_for_code
on datacap.snippet (code);
-- --------------------------------
@@ -116,24 +116,24 @@ fulltext index full_text_index_for_code
create table datacap.source
(
id bigint auto_increment primary key,
- _catalog varchar(255) null,
- create_time datetime default CURRENT_TIMESTAMP null,
- _database varchar(255) null,
- description varchar(255) null,
- host varchar(255) not null,
- name varchar(255) not null,
- password varchar(255) null,
- port bigint not null,
- protocol varchar(255) null,
- username varchar(255) null,
- _type varchar(100) not null,
- `ssl` tinyint(1) default 0 null,
- _ssl tinyint(1) default 0 null,
- publish tinyint(1) default 0 null,
- public tinyint(1) default 0 null,
- user_id bigint null,
- configure text null,
- used_config boolean default false
+ _catalog varchar(255) null,
+ create_time datetime default CURRENT_TIMESTAMP null,
+ _database varchar(255) null,
+ description varchar(255) null,
+ host varchar(255) not null,
+ name varchar(255) not null,
+ password varchar(255) null,
+ port bigint not null,
+ protocol varchar(255) null,
+ username varchar(255) null,
+ _type varchar(100) not null,
+ `ssl` tinyint(1) default 0 null,
+ _ssl tinyint(1) default 0 null,
+ publish tinyint(1) default 0 null,
+ public tinyint(1) default 0 null,
+ user_id bigint null,
+ configure text null,
+ used_config boolean default false
) comment 'The storage is used to query the data connection source';
-- --------------------------------
@@ -142,14 +142,14 @@ create table datacap.source
create table datacap.template_sql
(
id bigint auto_increment primary key,
- name varchar(255) null comment 'Name of template',
- content text null,
- description text null,
- plugin varchar(50) null comment 'Using plug-ins',
- configure text null comment 'The template must use the configuration information in the key->value format',
- create_time timestamp default CURRENT_TIMESTAMP not null,
- update_time timestamp default CURRENT_TIMESTAMP not null,
- `system` tinyint(1) default 0 null
+ name varchar(255) null comment 'Name of template',
+ content text null,
+ description text null,
+ plugin varchar(50) null comment 'Using plug-ins',
+ configure text null comment 'The template must use the configuration information in the key->value format',
+ create_time timestamp default CURRENT_TIMESTAMP not null,
+ update_time timestamp default CURRENT_TIMESTAMP not null,
+ `system` tinyint(1) default 0 null
) comment 'The system preset SQL template table';
INSERT INTO datacap.template_sql (name, content, description, plugin, configure, create_time, update_time, `system`)
@@ -223,15 +223,15 @@ VALUES ( 'getAllColumnsFromDatabaseAndTable', 'SHOW COLUMNS FROM ${table:String}
create table user_chat
(
id int auto_increment primary key,
- name varchar(255) not null,
- question text not null,
- answer text null,
- type varchar(100) null,
- create_time datetime default CURRENT_TIMESTAMP null,
- end_time datetime null on update CURRENT_TIMESTAMP,
- elapsed bigint null,
- user_id int not null,
- is_new tinyint(1) default 1 null
+ name varchar(255) not null,
+ question text not null,
+ answer text null,
+ type varchar(100) null,
+ create_time datetime default CURRENT_TIMESTAMP null,
+ end_time datetime null on update CURRENT_TIMESTAMP,
+ elapsed bigint null,
+ user_id int not null,
+ is_new tinyint(1) default 1 null
);
-- --------------------------------
@@ -241,14 +241,14 @@ create table user_log
(
id bigint auto_increment
primary key,
- device varchar(255) null comment 'Login device',
- client varchar(255) null comment 'Login client',
- ip varchar(100) null comment 'Login ip',
- message varchar(225) null comment 'Error message',
- state varchar(20) null comment 'Login state',
- ua varchar(255) null comment 'Trial plug-in, multiple according to, split',
- user_id bigint not null,
- create_time datetime(5) default CURRENT_TIMESTAMP (5) null
+ device varchar(255) null comment 'Login device',
+ client varchar(255) null comment 'Login client',
+ ip varchar(100) null comment 'Login ip',
+ message varchar(225) null comment 'Error message',
+ state varchar(20) null comment 'Login state',
+ ua varchar(255) null comment 'Trial plug-in, multiple according to, split',
+ user_id bigint not null,
+ create_time datetime(5) default CURRENT_TIMESTAMP(5) null
) comment 'User login log';
-- --------------------------------
@@ -273,10 +273,10 @@ create table datacap.users
(
id bigint auto_increment
primary key,
- username varchar(255) null comment ' ',
- password varchar(255) null comment ' ',
- create_time datetime(5) default CURRENT_TIMESTAMP (5) null,
- third_configure text null
+ username varchar(255) null comment ' ',
+ password varchar(255) null comment ' ',
+ create_time datetime(5) default CURRENT_TIMESTAMP(5) null,
+ third_configure text null
);
INSERT INTO datacap.users (username, password)
@@ -770,3 +770,25 @@ values ('2', '7'),
-- --------------------------------
alter table `audit_plugin`
add column `count` bigint(20) default 0;
+
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllDatabase', 'SELECT keyspace_name AS name
+FROM system_schema.keyspaces', 'Gets a list of all databases', 'Cassandra', '[]', 1);
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllTablesFromDatabase', 'SELECT
+ table_name AS name
+FROM
+ system_schema.tables
+WHERE
+ keyspace_name = ''${database:String}''
+GROUP BY
+ table_name', 'Get the data table from the database', 'Cassandra', '[{"column":"database","type":"String","expression":"${database:String}"}]', 1);
+INSERT INTO `template_sql` (name, content, description, plugin, configure, `system`)
+VALUES ('getAllColumnsFromDatabaseAndTable', 'SELECT
+ column_name
+FROM
+ system_schema.columns
+WHERE
+ keyspace_name = ''${database:String}''
+ and table_name = ''${table:String}''', 'Get the data column from the database and table', 'Cassandra',
+ '[{"column":"database","type":"String","expression":"${database:String}"},{"column":"table","type":"String","expression":"${table:String}"}]', 1);
diff --git a/core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/Adapter.java b/core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/Adapter.java
index d6d7039d..44a6d956 100644
--- a/core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/Adapter.java
+++ b/core/datacap-spi/src/main/java/io/edurt/datacap/spi/adapter/Adapter.java
@@ -1,8 +1,17 @@
package io.edurt.datacap.spi.adapter;
+import io.edurt.datacap.spi.FormatType;
+import io.edurt.datacap.spi.formatter.FormatterFactory;
import io.edurt.datacap.spi.model.Response;
+import java.util.List;
+
public interface Adapter
{
Response handlerExecute(String content);
+
+ default Object handlerFormatter(FormatType format, List headers, List