From 215b02d67a3f769342e89af2740c00daec0eaf28 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Thu, 29 Feb 2024 12:11:32 +0800 Subject: [PATCH] [Core] [DataSet] Support custom life cycle --- .../datacap/service/enums/CreatedMode.java | 3 +- .../service/impl/DataSetServiceImpl.java | 27 +++++++++++++++++ .../io/edurt/datacap/sql/AbstractSql.kt | 30 +++++++++++++++++++ .../io/edurt/datacap/sql/StatementType.kt | 2 +- .../edurt/datacap/sql/builder/TableBuilder.kt | 12 ++++++++ 5 files changed, 72 insertions(+), 2 deletions(-) diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/enums/CreatedMode.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/enums/CreatedMode.java index e2be5fba..f3e9ca5d 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/enums/CreatedMode.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/enums/CreatedMode.java @@ -4,5 +4,6 @@ public enum CreatedMode { CREATE_TABLE, CREATE_COLUMN, - MODIFY_COLUMN + MODIFY_COLUMN, + MODIFY_LIFECYCLE } diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java index a1f4dd91..d31ef8d2 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java @@ -437,6 +437,7 @@ public class DataSetServiceImpl TableBuilder.Companion.PARTITION_BY(columnEntities.stream().filter(DataSetColumnEntity::isPartitionKey).map(DataSetColumnEntity::getName).collect(Collectors.toList())); TableBuilder.Companion.PRIMARY_KEY(columnEntities.stream().filter(DataSetColumnEntity::isPrimaryKey).map(DataSetColumnEntity::getName).collect(Collectors.toList())); TableBuilder.Companion.SAMPLING_KEY(columnEntities.stream().filter(DataSetColumnEntity::isSamplingKey).map(DataSetColumnEntity::getName).collect(Collectors.toList())); + TableBuilder.Companion.ADD_LIFECYCLE(String.format("`%s` + INTERVAL %s %s", entity.getLifeCycleColumn(), entity.getLifeCycle(), entity.getLifeCycleType())); String sql = TableBuilder.Companion.SQL(); log.info("Create table sql \n {} \n on dataset [ {} ]", sql, entity.getName()); @@ -512,6 +513,23 @@ public class DataSetServiceImpl Response response = plugin.execute(sql); Preconditions.checkArgument(response.getIsSuccessful(), response.getMessage()); } + + createdModels.stream() + .filter(item -> item.getMode().equals(CreatedMode.MODIFY_LIFECYCLE)) + .findFirst() + .ifPresent(item -> { + TableBuilder.Companion.BEGIN(); + TableBuilder.Companion.MODIFY_LIFECYCLE(tableName); + TableBuilder.Companion.LIFECYCLE(String.format("`%s` + INTERVAL %s %s", item.getColumn().getName(), item.getColumn().getLength(), item.getColumn().getDefaultValue())); + String sql = TableBuilder.Companion.SQL(); + log.info("Modify lifecycle sql \n {} \n on dataset [ {} ] id [ {} ]", sql, entity.getName(), entity.getId()); + Plugin plugin = getOutputPlugin(); + SourceEntity source = getOutputSource(); + plugin.connect(source.toConfigure()); + Response response = plugin.execute(sql); + Preconditions.checkArgument(response.getIsSuccessful(), response.getMessage()); + }); + completeState(entity, DataSetState.TABLE_SUCCESS); } catch (Exception e) { @@ -759,6 +777,15 @@ public class DataSetServiceImpl } } } + + if (entity.getLifeCycleColumn() != null) { + DataSetColumnEntity column = DataSetColumnEntity.builder() + .name(entity.getLifeCycleColumn()) + .length(Integer.parseInt(entity.getLifeCycle())) + .defaultValue(entity.getLifeCycleType()) + .build(); + models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE)); + } return models; } diff --git a/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/AbstractSql.kt b/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/AbstractSql.kt index 686fa649..1eb09b9f 100644 --- a/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/AbstractSql.kt +++ b/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/AbstractSql.kt @@ -120,6 +120,22 @@ abstract class AbstractSql { return getSelf() } + fun MODIFY_LIFECYCLE(table: String?): T { + sql().statementType = StatementType.MODIFY_LIFECYCLE + sql().tables.add(table) + return getSelf() + } + + fun LIFECYCLE(lifecycle: String?): T { + sql().lifecycle = lifecycle + return getSelf() + } + + fun ADD_LIFECYCLE(lifecycle: String?): T { + sql().lifecycle = lifecycle + return getSelf() + } + fun CREATE_COLUMN(table: String?): T { sql().statementType = StatementType.CREATE_COLUMN sql().tables.add(table) @@ -545,6 +561,7 @@ abstract class AbstractSql { val primaryKey: MutableList = ArrayList() val samplingKey: MutableList = ArrayList() var formatEngine: EngineType? = EngineType.MYSQL + var lifecycle: String? = null init { // Prevent Synthetic Access @@ -706,6 +723,9 @@ abstract class AbstractSql { if (samplingKey.isNotEmpty()) { sqlClause(builder, "SAMPLE BY", samplingKey, "(", ")", ", ") } + if (lifecycle != null) { + sqlClause(builder, "TTL", listOf(lifecycle), "", "", ", ") + } if (end) { builder.append(";") } @@ -744,6 +764,15 @@ abstract class AbstractSql { return builder.toString() } + private fun modifyLifecycleSQL(builder: SafeAppendable): String { + sqlClause(builder, "ALTER TABLE", tables, "", "", "") + sqlClause(builder, "MODIFY TTL", listOf(lifecycle), "", "", ",\n") + if (end) { + builder.append(";") + } + return builder.toString() + } + fun sql(a: Appendable): String? { val builder = SafeAppendable(a) @@ -760,6 +789,7 @@ abstract class AbstractSql { StatementType.CREATE_COLUMN -> createColumnSQL(builder) StatementType.DROP_COLUMN -> dropColumnSQL(builder) StatementType.MODIFY_COLUMN -> modifyColumnSQL(builder) + StatementType.MODIFY_LIFECYCLE -> modifyLifecycleSQL(builder) else -> throw SqlException("Unsupported statement type: [ $statementType ]") } return answer diff --git a/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/StatementType.kt b/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/StatementType.kt index 463c84cd..0df297f1 100644 --- a/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/StatementType.kt +++ b/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/StatementType.kt @@ -1,5 +1,5 @@ package io.edurt.datacap.sql enum class StatementType { - DELETE, INSERT, SELECT, UPDATE, ALTER, SHOW, TRUNCATE, DROP, CREATE_TABLE, CREATE_COLUMN, DROP_COLUMN, MODIFY_COLUMN + DELETE, INSERT, SELECT, UPDATE, ALTER, SHOW, TRUNCATE, DROP, CREATE_TABLE, CREATE_COLUMN, DROP_COLUMN, MODIFY_COLUMN, MODIFY_LIFECYCLE } diff --git a/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/builder/TableBuilder.kt b/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/builder/TableBuilder.kt index caac64bf..80db5dd4 100644 --- a/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/builder/TableBuilder.kt +++ b/core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/builder/TableBuilder.kt @@ -26,6 +26,18 @@ class TableBuilder { sql().CREATE_TABLE(table) } + fun MODIFY_LIFECYCLE(table: String?) { + sql().MODIFY_LIFECYCLE(table) + } + + fun LIFECYCLE(lifecycle: String?) { + sql().LIFECYCLE(lifecycle) + } + + fun ADD_LIFECYCLE(lifecycle: String?) { + sql().ADD_LIFECYCLE(lifecycle) + } + fun COLUMNS(values: List) { sql().COLUMNS(values) }