test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-11-29 18:48:23 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Core] [DataSet] Support create dataset table

Browse Source
This commit is contained in:
qianmoQ 2023-12-21 20:51:45 +08:00
parent 849d6b496e
commit 773426a114
26 changed files with 635 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
configure/etc/conf/application.properties
View File

@ -83,6 +83,16 @@ datacap.pipeline.maxQueue=200
# When the service is restarted, the status of the pipeline with status RUNNING is reset.
datacap.pipeline.reset=STOPPED
################################# DataSet configure #################################
datacap.dataset.type=ClickHouse
datacap.dataset.host=localhost
datacap.dataset.port=8123
datacap.dataset.username=
datacap.dataset.password=
datacap.dataset.database=datacap
datacap.dataset.tablePrefix=datacap_
datacap.dataset.tableDefaultEngine=MergeTree
################################# Experimental features #################################
# This configuration is used to dynamically increase the total number of rows of returned data in SQL during query, and currently only takes effect for user-directed queries
# If the total number of rows returned is included in the SQL, it will not be automatically incremented

16
core/datacap-common/src/main/java/io/edurt/datacap/common/enums/DataSetState.java Normal file
View File

@ -0,0 +1,16 @@
package io.edurt.datacap.common.enums;
public enum DataSetState
{
METADATA_START,
METADATA_FAILED,
METADATA_SUCCESS,
TABLE_START,
TABLE_FAILED,
TABLE_SUCCESS,
DATA_START,
DATA_FAILED,
DATA_SUCCESS,
COMPLETE_FAILED,
COMPLETE_SUCCESS
}

8
core/datacap-server/src/main/java/io/edurt/datacap/server/controller/DataSetController.java
View File

@ -5,6 +5,8 @@ import io.edurt.datacap.service.body.DataSetBody;
import io.edurt.datacap.service.entity.DataSetEntity;
import io.edurt.datacap.service.repository.DataSetRepository;
import io.edurt.datacap.service.service.DataSetService;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@ -30,4 +32,10 @@ public class DataSetController
{
return service.saveOrUpdate(configure);
}
@PutMapping(value = "rebuild/{id}")
public CommonResponse rebuild(@PathVariable Long id)
{
return service.rebuild(id);
}
}

32
core/datacap-server/src/main/schema/1.18.0/schema.sql
View File

@ -115,7 +115,8 @@ CREATE TABLE `datacap_dataset`
`description` TEXT,
`query` LONGTEXT,
`sync_mode` VARCHAR(100),
`sync_value` VARCHAR(100)
`sync_value` VARCHAR(100),
`table_name` VARCHAR(255)
);
CREATE TABLE `datacap_dataset_user_relation`
@ -132,19 +133,22 @@ CREATE TABLE `datacap_dataset_source_relation`
CREATE TABLE `datacap_dataset_column`
(
`id` BIGINT AUTO_INCREMENT PRIMARY KEY,
`name` VARCHAR(255),
`active` BOOLEAN DEFAULT TRUE,
`create_time` DATETIME,
`update_time` DATETIME,
`description` TEXT,
`type` VARCHAR(100),
`comment` VARCHAR(255),
`original` VARCHAR(255),
`default_value` VARCHAR(255),
`position` INT,
`is_nullable` BOOLEAN DEFAULT FALSE,
`length` INT
`id` BIGINT AUTO_INCREMENT PRIMARY KEY,
`name` VARCHAR(255),
`active` BOOLEAN DEFAULT TRUE,
`create_time` DATETIME,
`update_time` DATETIME,
`description` TEXT,
`type` VARCHAR(100),
`comment` VARCHAR(255),
`original` VARCHAR(255),
`default_value` VARCHAR(255),
`position` INT,
`is_nullable` BOOLEAN DEFAULT FALSE,
`length` INT,
`state` VARCHAR(100),
`message` LONGTEXT,
`is_order_by_key` BOOLEAN DEFAULT FALSE
);
CREATE TABLE `datacap_dataset_column_relation`

7
core/datacap-service/src/main/java/io/edurt/datacap/service/body/DataSetBody.java
View File

@ -3,13 +3,20 @@ package io.edurt.datacap.service.body;
import io.edurt.datacap.service.entity.DataSetColumnEntity;
import io.edurt.datacap.service.entity.SourceEntity;
import io.edurt.datacap.service.enums.SyncMode;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Set;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DataSetBody
{
private Long id;
private String name;
private String description;
private String query;

35
core/datacap-service/src/main/java/io/edurt/datacap/service/converter/ListConverter.java Normal file
View File

@ -0,0 +1,35 @@
package io.edurt.datacap.service.converter;
import com.google.common.collect.Lists;
import io.edurt.datacap.common.enums.DataSetState;
import org.apache.commons.lang3.StringUtils;
import javax.persistence.AttributeConverter;
import java.util.Arrays;
import java.util.List;
public class ListConverter
implements AttributeConverter<List, String>
{
@Override
public String convertToDatabaseColumn(List map)
{
List<String> values = Lists.newArrayList();
for (Object state : map) {
values.add(state.toString());
}
return String.join(",", values);
}
@Override
public List convertToEntityAttribute(String s)
{
if (StringUtils.isEmpty(s)) {
return null;
}
else {
return Lists.newArrayList(Arrays.stream(s.split(",")).map(DataSetState::valueOf).toArray(DataSetState[]::new));
}
}
}

3
core/datacap-service/src/main/java/io/edurt/datacap/service/entity/DataSetColumnEntity.java
View File

@ -55,6 +55,9 @@ public class DataSetColumnEntity
@Column(name = "length")
private int length;
@Column(name = "is_order_by_key")
private boolean isOrderByKey;
@ManyToOne
@JoinTable(name = "datacap_dataset_column_relation",
joinColumns = @JoinColumn(name = "column_id"),

23
core/datacap-service/src/main/java/io/edurt/datacap/service/entity/DataSetEntity.java
View File

@ -1,8 +1,9 @@
package io.edurt.datacap.service.entity;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.common.enums.DataSetState;
import io.edurt.datacap.service.converter.ListConverter;
import io.edurt.datacap.service.enums.SyncMode;
import lombok.AllArgsConstructor;
import lombok.Data;
@ -11,20 +12,18 @@ import lombok.ToString;
import lombok.experimental.SuperBuilder;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Convert;
import javax.persistence.Entity;
import javax.persistence.EntityListeners;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.JoinTable;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import java.util.Set;
import java.util.List;
@Data
@SuperBuilder
@ -51,6 +50,16 @@ public class DataSetEntity
@Column(name = "sync_value")
private String syncValue; // only for TIMING
@Column(name = "state")
@Convert(converter = ListConverter.class)
private List<DataSetState> state;
@Column(name = "message")
private String message;
@Column(name = "table_name")
private String tableName;
@ManyToOne
@JoinTable(name = "datacap_dataset_source_relation",
joinColumns = @JoinColumn(name = "dataset_id"),
@ -64,8 +73,4 @@ public class DataSetEntity
inverseJoinColumns = @JoinColumn(name = "user_id"))
@JsonIgnoreProperties(value = {"roles", "thirdConfigure", "avatarConfigure"})
private UserEntity user;
@OneToMany(mappedBy = "dataset", cascade = {CascadeType.ALL}, fetch = FetchType.LAZY)
@JsonIgnore
private Set<DataSetColumnEntity> columns;
}

20
core/datacap-service/src/main/java/io/edurt/datacap/service/initializer/DataSetConfigure.java Normal file
View File

@ -0,0 +1,20 @@
package io.edurt.datacap.service.initializer;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "datacap.dataset")
public class DataSetConfigure
{
private String type;
private String host;
private String port;
private String username;
private String password;
private String database;
private String tableDefaultEngine;
private String tablePrefix;
}

6
core/datacap-service/src/main/java/io/edurt/datacap/service/initializer/InitializerConfigure.java
View File

@ -71,9 +71,13 @@ public class InitializerConfigure
@Getter
private final FsConfigure fsConfigure;
public InitializerConfigure(FsConfigure fsConfigure)
@Getter
private final DataSetConfigure dataSetConfigure;
public InitializerConfigure(FsConfigure fsConfigure, DataSetConfigure dataSetConfigure)
{
this.fsConfigure = fsConfigure;
this.dataSetConfigure = dataSetConfigure;
}
/**

4
core/datacap-service/src/main/java/io/edurt/datacap/service/repository/DataSetColumnRepository.java
View File

@ -1,9 +1,13 @@
package io.edurt.datacap.service.repository;
import io.edurt.datacap.service.entity.DataSetColumnEntity;
import io.edurt.datacap.service.entity.DataSetEntity;
import org.springframework.data.repository.PagingAndSortingRepository;
import java.util.List;
public interface DataSetColumnRepository
extends PagingAndSortingRepository<DataSetColumnEntity, Long>
{
List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset);
}

2
core/datacap-service/src/main/java/io/edurt/datacap/service/service/DataSetService.java
View File

@ -8,4 +8,6 @@ public interface DataSetService
extends BaseService<DataSetEntity>
{
CommonResponse<DataSetEntity> saveOrUpdate(DataSetBody configure);
CommonResponse<DataSetEntity> rebuild(Long id);
}

203
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java
View File

@ -1,56 +1,233 @@
package io.edurt.datacap.service.service.impl;
import com.google.common.collect.Lists;
import com.google.inject.Injector;
import io.edurt.datacap.common.enums.DataSetState;
import io.edurt.datacap.common.response.CommonResponse;
import io.edurt.datacap.service.adapter.PageRequestAdapter;
import io.edurt.datacap.service.body.DataSetBody;
import io.edurt.datacap.service.body.FilterBody;
import io.edurt.datacap.service.common.PluginUtils;
import io.edurt.datacap.service.entity.DataSetColumnEntity;
import io.edurt.datacap.service.entity.DataSetEntity;
import io.edurt.datacap.service.entity.PageEntity;
import io.edurt.datacap.service.entity.UserEntity;
import io.edurt.datacap.service.enums.ColumnType;
import io.edurt.datacap.service.initializer.InitializerConfigure;
import io.edurt.datacap.service.repository.DataSetColumnRepository;
import io.edurt.datacap.service.repository.DataSetRepository;
import io.edurt.datacap.service.security.UserDetailsService;
import io.edurt.datacap.service.service.DataSetService;
import io.edurt.datacap.spi.Plugin;
import io.edurt.datacap.spi.PluginType;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import io.edurt.datacap.sql.builder.TableBuilder;
import io.edurt.datacap.sql.model.Column;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Pageable;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
@Service
@Slf4j
public class DataSetServiceImpl
implements DataSetService
{
private final DataSetRepository repository;
public final DataSetColumnRepository columnRepository;
private final Injector injector;
private final InitializerConfigure initializerConfigure;
public DataSetServiceImpl(DataSetRepository repository, DataSetColumnRepository columnRepository)
public DataSetServiceImpl(DataSetRepository repository, DataSetColumnRepository columnRepository, Injector injector, InitializerConfigure initializerConfigure)
{
this.repository = repository;
this.columnRepository = columnRepository;
this.injector = injector;
this.initializerConfigure = initializerConfigure;
}
@Transactional
public CommonResponse<DataSetEntity> saveOrUpdate(DataSetBody configure)
{
DataSetEntity entity = DataSetEntity.builder()
.name(configure.getName())
.query(configure.getQuery())
.user(UserDetailsService.getUser())
.source(configure.getSource())
.description(configure.getDescription())
.syncMode(configure.getSyncMode())
.syncValue(configure.getSyncValue())
.build();
repository.save(entity);
configure.getColumns().stream().forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
columnRepository.saveAll(configure.getColumns());
UserEntity user = UserDetailsService.getUser();
ExecutorService service = Executors.newSingleThreadExecutor();
service.submit(() -> {
DataSetEntity entity = DataSetEntity.builder()
.id(configure.getId())
.name(configure.getName())
.query(configure.getQuery())
.user(user)
.source(configure.getSource())
.description(configure.getDescription())
.syncMode(configure.getSyncMode())
.syncValue(configure.getSyncValue())
.build();
completeState(entity, DataSetState.METADATA_START);
startBuild(entity, configure, true);
});
return CommonResponse.success(configure);
}
@Override
public CommonResponse<DataSetEntity> rebuild(Long id)
{
Optional<DataSetEntity> entity = repository.findById(id);
if (!entity.isPresent()) {
return CommonResponse.failure(String.format("DataSet [ %s ] not found", id));
}
ExecutorService service = Executors.newSingleThreadExecutor();
service.submit(() -> {
DataSetEntity configure = entity.get();
DataSetBody body = DataSetBody.builder()
.name(configure.getName())
.query(configure.getQuery())
.source(configure.getSource())
.description(configure.getDescription())
.syncMode(configure.getSyncMode())
.syncValue(configure.getSyncValue())
.build();
startBuild(configure, body, false);
});
return CommonResponse.success(entity);
}
@Override
public CommonResponse<PageEntity<DataSetEntity>> getAll(PagingAndSortingRepository pagingAndSortingRepository, FilterBody filter)
{
Pageable pageable = PageRequestAdapter.of(filter);
return CommonResponse.success(PageEntity.build(repository.findAllByUser(UserDetailsService.getUser(), pageable)));
}
private void completeState(DataSetEntity entity, DataSetState state)
{
List<DataSetState> sourceStates = entity.getState();
List<DataSetState> states = Lists.newArrayList();
if (sourceStates != null) {
states = sourceStates;
states = states.stream().filter(item -> !item.name().startsWith(state.name().split("_")[0])).collect(Collectors.toList());
}
states.add(state);
entity.setState(states);
}
private String getColumnType(ColumnType type)
{
switch (type) {
case NUMBER:
return "bigint";
case BOOLEAN:
return "boolean";
case STRING:
default:
return "varchar";
}
}
private void startBuild(DataSetEntity entity, DataSetBody configure, boolean rebuildColumn)
{
switch (entity.getState().get(entity.getState().size() - 1)) {
case METADATA_START:
case METADATA_FAILED:
createMetadata(entity, configure, rebuildColumn);
break;
case METADATA_SUCCESS:
case TABLE_START:
case TABLE_FAILED:
createTable(entity, configure);
break;
}
}
private void createMetadata(DataSetEntity entity, DataSetBody configure, boolean rebuildColumn)
{
try {
repository.save(entity);
if (rebuildColumn) {
configure.getColumns()
.stream()
.forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
columnRepository.saveAll(configure.getColumns());
}
completeState(entity, DataSetState.METADATA_SUCCESS);
}
catch (Exception e) {
log.warn("Create dataset [ {} ] ", entity.getName(), e);
completeState(entity, DataSetState.METADATA_FAILED);
entity.setMessage(e.getMessage());
}
finally {
repository.save(entity);
startBuild(entity, configure, rebuildColumn);
}
}
private void createTable(DataSetEntity entity, DataSetBody configure)
{
try {
Optional<Plugin> pluginOptional = PluginUtils.getPluginByNameAndType(injector, initializerConfigure.getDataSetConfigure().getType(), PluginType.JDBC.name());
if (!pluginOptional.isPresent()) {
throw new IllegalArgumentException(String.format("Plugin [ %s ] not found", initializerConfigure.getDataSetConfigure().getType()));
}
Plugin plugin = pluginOptional.get();
String database = initializerConfigure.getDataSetConfigure().getDatabase();
String tablePrefix = initializerConfigure.getDataSetConfigure().getTablePrefix();
String originTableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", ""));
entity.setTableName(originTableName);
String tableDefaultEngine = initializerConfigure.getDataSetConfigure().getTableDefaultEngine();
List<Column> columns = Lists.newArrayList();
List<DataSetColumnEntity> columnEntities = columnRepository.findAllByDataset(entity);
columnEntities.forEach(item -> {
Column column = new Column();
column.setName(item.getName());
column.setType(getColumnType(item.getType()));
column.setComment(item.getComment());
column.setLength(item.getLength());
column.setNullable(item.isNullable());
column.setDefaultValue(item.getDefaultValue());
columns.add(column);
});
TableBuilder.Companion.BEGIN();
TableBuilder.Companion.CREATE_TABLE(String.format("`%s`.`%s`", database, originTableName));
TableBuilder.Companion.COLUMNS(columns.stream().map(item -> item.toColumnVar()).collect(Collectors.toList()));
TableBuilder.Companion.ENGINE(tableDefaultEngine);
TableBuilder.Companion.ORDER_BY(columnEntities.stream().filter(DataSetColumnEntity::isOrderByKey).map(DataSetColumnEntity::getName).collect(Collectors.toList()));
String sql = TableBuilder.Companion.SQL();
log.info("Create table sql \n {} \n on dataset [ {} ]", sql, entity.getName());
Configure targetConfigure = new Configure();
targetConfigure.setHost(initializerConfigure.getDataSetConfigure().getHost());
targetConfigure.setPort(Integer.valueOf(initializerConfigure.getDataSetConfigure().getPort()));
targetConfigure.setUsername(Optional.ofNullable(initializerConfigure.getDataSetConfigure().getUsername()));
targetConfigure.setPassword(Optional.ofNullable(initializerConfigure.getDataSetConfigure().getPassword()));
targetConfigure.setDatabase(Optional.ofNullable(database));
plugin.connect(targetConfigure);
Response response = plugin.execute(sql);
if (response.getIsSuccessful()) {
completeState(entity, DataSetState.TABLE_SUCCESS);
}
else {
throw new RuntimeException(response.getMessage());
}
}
catch (Exception e) {
log.warn("Create dataset [ {} ] ", entity.getName(), e);
completeState(entity, DataSetState.TABLE_FAILED);
entity.setMessage(e.getMessage());
}
finally {
repository.save(entity);
}
}
}

18
core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/AbstractSql.kt
View File

@ -143,6 +143,16 @@ abstract class AbstractSql<T> {
return getSelf()
}
fun ENGINE(engine: String?): T {
sql().engine = engine
return getSelf()
}
fun ORDER_BY_KEY(columns: List<String>): T {
sql().orderByKey.addAll(columns.map { item -> "\t$item" })
return getSelf()
}
fun FROM(table: String?): T {
sql().tables.add(table)
return getSelf()
@ -508,6 +518,8 @@ abstract class AbstractSql<T> {
var offset: String? = null
var limit: String? = null
var limitingRowsStrategy: LimitingRowsStrategy = LimitingRowsStrategy.NOP
var engine: String? = null
var orderByKey: MutableList<String?> = ArrayList()
init {
// Prevent Synthetic Access
@ -648,6 +660,12 @@ abstract class AbstractSql<T> {
private fun createTableSQL(builder: SafeAppendable): String {
sqlClause(builder, "CREATE TABLE", tables, "", "", "")
sqlClause(builder, "", columns, "(\n", "\n)", ",\n")
if (engine != null) {
sqlClause(builder, "ENGINE", listOf(engine), " = ", "", ",\n")
}
if (orderByKey.isNotEmpty()) {
sqlClause(builder, "ORDER BY", orderByKey, "(", ")", ", ")
}
if (end) {
builder.append(";")
}

8
core/datacap-sql/src/main/kotlin/io/edurt/datacap/sql/builder/TableBuilder.kt
View File

@ -30,6 +30,14 @@ class TableBuilder {
sql().COLUMNS(values)
}
fun ENGINE(engine: String?) {
sql().ENGINE(engine)
}
fun ORDER_BY(values: List<String>) {
sql().ORDER_BY_KEY(values)
}
fun END() {
sql().END()
}

38
core/datacap-sql/src/test/kotlin/io/edurt/datacap/sql/builder/TableBuilderTest.kt
View File

@ -2,8 +2,10 @@ package io.edurt.datacap.sql.builder
import org.junit.Assert
import org.junit.Test
import org.slf4j.LoggerFactory
class TableBuilderTest {
private val log = LoggerFactory.getLogger(this.javaClass)
private val tableName: String = "TestTable"
private val columns: List<String> = listOf("id int(32) primary key auto_increment", "name varchar(32) comment 'name'", "age varchar(200) not null default 'xxx'");
@ -19,4 +21,40 @@ class TableBuilderTest {
"\tage varchar(200) not null default 'xxx'\n" +
")")
}
@Test
fun testCreateTableWithEngine() {
TableBuilder.BEGIN()
TableBuilder.CREATE_TABLE(tableName)
TableBuilder.COLUMNS(columns)
TableBuilder.ENGINE("MergeTree")
val sql = TableBuilder.SQL()
log.info(sql)
Assert.assertEquals(sql, "CREATE TABLE TestTable\n" +
"(\n" +
"\tid int(32) primary key auto_increment,\n" +
"\tname varchar(32) comment 'name',\n" +
"\tage varchar(200) not null default 'xxx'\n" +
")\n" +
"ENGINE = MergeTree")
}
@Test
fun testCreateTableWithEngineAndOrderBy() {
TableBuilder.BEGIN()
TableBuilder.CREATE_TABLE(tableName)
TableBuilder.COLUMNS(columns)
TableBuilder.ENGINE("MergeTree")
TableBuilder.ORDER_BY(listOf("id"))
val sql = TableBuilder.SQL()
log.info(sql)
Assert.assertEquals(sql, "CREATE TABLE TestTable\n" +
"(\n" +
"\tid int(32) primary key auto_increment,\n" +
"\tname varchar(32) comment 'name',\n" +
"\tage varchar(200) not null default 'xxx'\n" +
")\n" +
"ENGINE = MergeTree\n" +
"ORDER BY(\tid)")
}
}

4
core/datacap-web/src/fontawesome.ts
View File

@ -14,7 +14,9 @@ import {
faCircle,
faCircleInfo,
faCircleMinus,
faCirclePlay,
faCirclePlus,
faCircleStop,
faClock,
faClone,
faColumns,
@ -58,6 +60,8 @@ import {
const createIcons = (app: any) => {
library.add(faArrowRight,
faCirclePlus,
faCirclePlay,
faCircleStop,
faPager,
faChartLine,
faChartBar,

9
core/datacap-web/src/i18n/langs/en/dataset.ts
View File

@ -13,8 +13,17 @@ export default {
columnDefaultValue: 'Default Value',
columnIsNullable: 'Is Nullable',
columnLength: 'Length',
columnIsOrderByKey: 'Whether it is a sort key',
create: 'Create Dataset',
syncMode: 'Sync Mode',
syncModeManual: 'Manual',
syncModeOutSync: 'Out Sync',
rebuild: 'Rebuild',
rebuildProgress: 'Rebuilding will only progress unfinished',
complete: 'Complete',
failed: 'Failed',
stateOfStart: 'Start',
stateOfMetadata: 'Metadata State',
stateOfMetadataStarted: 'Metadata Started',
stateOfCreateTable: 'Create Table State',
}

9
core/datacap-web/src/i18n/langs/zhCn/dataset.ts
View File

@ -13,8 +13,17 @@ export default {
columnDefaultValue: '列默认值',
columnIsNullable: '是否允许为空',
columnLength: '列长度',
columnIsOrderByKey: '是否为排序键',
create: '创建数据集',
syncMode: '同步模式',
syncModeManual: '手动',
syncModeOutSync: '不同步',
rebuild: '重建',
rebuildProgress: '重建只会进行未完成进度',
complete: '完成',
failed: '失败',
stateOfStarted: '已启动',
stateOfMetadata: '元数据状态',
stateOfMetadataStarted: '元数据已启动',
stateOfCreateTable: '创建表状态',
}

5
core/datacap-web/src/services/admin/DatasetService.ts
View File

@ -23,6 +23,11 @@ class DatasetService
}
}
rebuild(id: number): Promise<ResponseModel>
{
return new HttpCommon().put(`${baseUrl}/rebuild/${id}`);
}
getByName<T>(name: string): Promise<ResponseModel>
{
return Promise.resolve(undefined);

44
core/datacap-web/src/views/admin/dataset/AdminDataset.vue
View File

@ -13,8 +13,26 @@
</Avatar>
</Tooltip>
</template>
<template #state="{ row }">
<Poptip trigger="hover"
placement="bottom"
transfer>
<Text>{{ getState(row.state) }}</Text>
<template #content>
<DatasetState style="margin-top: 25px;"
:states="row.state">
</DatasetState>
</template>
</Poptip>
</template>
<template #action="{ row }">
<Space>
<Button type="primary"
shape="circle"
size="small"
@click="handlerRebuild(row, true)">
<FontAwesomeIcon :icon="row.state === 'SUCCESS' ? 'circle-stop' : 'circle-play'"/>
</Button>
</Space>
</template>
</Table>
@ -31,6 +49,11 @@
</Page>
</p>
</Card>
<DatasetRebuild v-if="rebuildVisible"
:is-visible="rebuildVisible"
:data="contextData"
@close="handlerRebuild(null, false)">
</DatasetRebuild>
</div>
</template>
@ -41,10 +64,14 @@ import {ResponsePage} from "@/model/ResponsePage";
import {createHeaders} from "@/views/admin/dataset/DatasetUtils";
import DatasetService from "@/services/admin/DatasetService";
import {Filter} from "@/model/Filter";
import {FontAwesomeIcon} from "@fortawesome/vue-fontawesome";
import DatasetRebuild from "@/views/admin/dataset/DatasetRebuild.vue";
import DatasetState from "@/views/admin/dataset/components/DatasetState.vue";
const filter: Filter = new Filter();
export default defineComponent({
name: 'AdminDataset',
components: {DatasetState, DatasetRebuild, FontAwesomeIcon},
setup()
{
const i18n = useI18n()
@ -60,7 +87,9 @@ export default defineComponent({
return {
loading: false,
data: null as ResponsePage,
pagination: {total: 0, current: 1, pageSize: 10}
pagination: {total: 0, current: 1, pageSize: 10},
rebuildVisible: false,
contextData: null
}
},
created()
@ -95,6 +124,19 @@ export default defineComponent({
this.pagination.current = pagination.current;
this.pagination.pageSize = pagination.pageSize;
this.handlerInitialize(this.filter)
},
handlerRebuild(record: any, opened: boolean)
{
this.rebuildVisible = opened
this.contextData = record
},
getState(state: Array<any> | null)
{
if (state && state.length > 0) {
const s = state[state.length - 1];
return s;
}
return null;
}
}
});

19
core/datacap-web/src/views/admin/dataset/DatasetInfo.vue
View File

@ -20,9 +20,10 @@
<Col span="4">{{ $t('dataset.columnName') }}</Col>
<Col span="4">{{ $t('dataset.columnType') }}</Col>
<Col span="4">{{ $t('dataset.columnDefaultValue') }}</Col>
<Col span="4">{{ $t('dataset.columnIsNullable') }}</Col>
<Col span="4">{{ $t('dataset.columnLength') }}</Col>
<Col span="4">{{ $t('dataset.columnComment') }}</Col>
<Col span="3">{{ $t('dataset.columnIsNullable') }}</Col>
<Col span="3">{{ $t('dataset.columnIsOrderByKey') }}</Col>
<Col span="3">{{ $t('dataset.columnLength') }}</Col>
<Col span="3">{{ $t('dataset.columnComment') }}</Col>
</Row>
</FormItem>
<template v-for="(item, index) in formState.columns" :key="index">
@ -45,13 +46,16 @@
type="text">
</Input>
</Col>
<Col span="4">
<Col span="3">
<Switch v-model="item.isNullable"/>
</Col>
<Col span="4">
<Col span="3">
<Switch v-model="item.isOrderByKey"/>
</Col>
<Col span="3">
<InputNumber v-model="item.length"/>
</Col>
<Col span="4">
<Col span="3">
<Input v-model="item.comment"
type="textarea">
</Input>
@ -168,7 +172,8 @@ export default defineComponent({
position: index,
isNullable: false,
length: 0,
original: header
original: header,
isOrderByKey: false
}
this.formState.columns.push(column)
})

85
core/datacap-web/src/views/admin/dataset/DatasetRebuild.vue Normal file
View File

@ -0,0 +1,85 @@
<template>
<div>
<Modal v-model="visible"
:title="$t('dataset.rebuild') + ' [ ' + data.name + ' ]'"
:mask-closable="false"
@cancel="handlerCancel()">
<Alert type="warning"
show-icon>
{{ $t('dataset.rebuildProgress') }}
</Alert>
<!-- <Alert type="error"-->
<!-- show-icon>-->
<!-- {{ $t('report.deleteTip2') }}-->
<!-- </Alert>-->
<!-- <p>{{ $t('report.deleteTip3').replace('REPLACE_NAME', data.name) }}</p>-->
<!-- <Input v-model="inputValue"/>-->
<template #footer>
<Button type="primary"
:loading="loading"
@click="handlerRebuild">
{{ $t('dataset.rebuild') }}
</Button>
</template>
</Modal>
</div>
</template>
<script lang="ts">
import {defineComponent} from 'vue';
import DatasetService from "@/services/admin/DatasetService";
export default defineComponent({
name: 'DatasetRebuild',
props: {
isVisible: {
type: Boolean,
default: () => false
},
data: {
type: Object
}
},
data()
{
return {
loading: false
}
},
methods: {
handlerRebuild()
{
this.loading = true;
DatasetService.rebuild(this.data.id)
.then((response) => {
if (response.status) {
this.$Message.success(`${this.$t('dataset.rebuild')} [ ${this.data.name} ] ${this.$t('common.success')}`)
this.handlerCancel()
}
})
.finally(() => {
this.loading = false
})
},
handlerCancel()
{
this.visible = false;
}
},
computed: {
visible: {
get(): boolean
{
return this.isVisible;
},
set(value: boolean)
{
this.$emit('close', value);
}
}
}
});
</script>
<style scoped>
</style>

7
core/datacap-web/src/views/admin/dataset/DatasetUtils.ts
View File

@ -29,6 +29,13 @@ const createHeaders = (i18n: any) => {
ellipsis: true,
align: 'center'
},
{
title: i18n.t('common.state'),
key: 'state',
slot: 'state',
ellipsis: true,
align: 'center'
},
{
title: i18n.t('common.createTime'),
key: 'createTime',

65
core/datacap-web/src/views/admin/dataset/components/DatasetState.vue Normal file
View File

@ -0,0 +1,65 @@
<template>
<div>
<Timeline>
<TimelineItem v-for="state in states"
:key="state">
<div v-if="state === 'START'">
<p class="time">
{{ $t('dataset.stateOfStarted') }}
<span class="content">{{ $t('dataset.complete') }}</span>
</p>
</div>
<div v-else-if="state.startsWith('METADATA_')">
<p class="time">
{{ $t('dataset.stateOfMetadata') }}
<span v-if="state.endsWith('SUCCESS')"
class="content">
{{ $t('dataset.complete') }}
</span>
<span v-else-if="state.endsWith('FAILED')"
class="content">
{{ $t('dataset.failed') }}
</span>
</p>
</div>
<div v-else-if="state.startsWith('TABLE_')">
<p class="time">
{{ $t('dataset.stateOfCreateTable') }}
<span v-if="state.endsWith('SUCCESS')"
class="content">
{{ $t('dataset.complete') }}
</span>
<span v-else-if="state.endsWith('FAILED')"
class="content">
{{ $t('dataset.failed') }}
</span>
</p>
</div>
</TimelineItem>
</Timeline>
</div>
</template>
<script lang="ts">
import {defineComponent} from 'vue';
export default defineComponent({
name: 'DatasetState',
props: {
states: {
type: Array
}
}
});
</script>
<style scoped>
.time {
font-size: 14px;
}
.content {
margin-left: 30px;
font-weight: bold;
}
</style>

1
core/datacap-web/src/views/admin/report/AdminReport.vue
View File

@ -142,7 +142,6 @@ export default defineComponent({
{
this.deleteVisible = opened;
this.contextData = data;
console.log(this.contextData, opened);
if (!opened) {
this.handlerInitialize(this.filter);
}