mirror of
https://gitee.com/devlive-community/datacap.git
synced 2024-12-04 21:18:22 +08:00
[Feature] [DataSet] Support virtual columns (#693)
This commit is contained in:
qianmoQ
GitHub
commit
408e1b733b
@ -5,8 +5,10 @@
|
||||
USE `datacap`;
|
||||
|
||||
ALTER TABLE `datacap_dataset_column`
|
||||
ADD COLUMN `is_primary_key` BOOLEAN DEFAULT FALSE,
|
||||
ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE;
|
||||
ADD COLUMN `is_primary_key` BOOLEAN DEFAULT FALSE,
|
||||
ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE,
|
||||
ADD COLUMN `is_custom_column` BOOLEAN DEFAULT FALSE,
|
||||
ADD COLUMN `is_virtual_column` BOOLEAN DEFAULT FALSE;
|
||||
|
||||
ALTER TABLE `datacap_dataset`
|
||||
ADD COLUMN `total_rows` BIGINT(100) DEFAULT 0,
|
||||
|
||||
@ -75,6 +75,12 @@ public class DataSetColumnEntity
|
||||
@Column(name = "alias_name")
|
||||
private String aliasName;
|
||||
|
||||
@Column(name = "is_custom_column")
|
||||
private boolean customColumn;
|
||||
|
||||
@Column(name = "is_virtual_column")
|
||||
private boolean virtualColumn;
|
||||
|
||||
@ManyToOne
|
||||
@JoinTable(name = "datacap_dataset_column_relation",
|
||||
joinColumns = @JoinColumn(name = "column_id"),
|
||||
|
||||
@ -10,4 +10,6 @@ public interface DataSetColumnRepository
|
||||
extends PagingAndSortingRepository<DataSetColumnEntity, Long>
|
||||
{
|
||||
List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset);
|
||||
|
||||
List<DataSetColumnEntity> findAllByDatasetOrderByPositionAsc(DataSetEntity dataset);
|
||||
}
|
||||
|
||||
@ -142,17 +142,15 @@ public class DataSetServiceImpl
|
||||
public CommonResponse<Set<DataSetColumnEntity>> getColumns(Long id)
|
||||
{
|
||||
Optional<DataSetEntity> entity = repository.findById(id);
|
||||
if (!entity.isPresent()) {
|
||||
return CommonResponse.failure(String.format("DataSet [ %s ] not found", id));
|
||||
}
|
||||
return CommonResponse.success(columnRepository.findAllByDataset(entity.get()));
|
||||
return entity.map(dataSet -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(dataSet)))
|
||||
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", id)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public CommonResponse<Set<DataSetColumnEntity>> getColumnsByCode(String code)
|
||||
{
|
||||
Optional<DataSetEntity> entity = repository.findByCode(code);
|
||||
return entity.map(item -> CommonResponse.success(columnRepository.findAllByDataset(item)))
|
||||
return entity.map(item -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(item)))
|
||||
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code)));
|
||||
}
|
||||
|
||||
@ -356,14 +354,8 @@ public class DataSetServiceImpl
|
||||
String tableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", ""));
|
||||
entity.setTableName(tableName);
|
||||
}
|
||||
DataSetState state = entity.getState().get(entity.getState().size() - 1);
|
||||
if (state.equals(DataSetState.METADATA_START)
|
||||
|| state.equals(DataSetState.METADATA_FAILED)
|
||||
|| state.equals(DataSetState.TABLE_FAILED)) {
|
||||
log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId());
|
||||
createMetadata(entity, rebuildColumn);
|
||||
}
|
||||
throw new IllegalArgumentException(String.format("Invalid state [ %s ]", state));
|
||||
log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId());
|
||||
createMetadata(entity, rebuildColumn);
|
||||
}
|
||||
|
||||
private void createMetadata(DataSetEntity entity, boolean rebuildColumn)
|
||||
@ -376,6 +368,13 @@ public class DataSetServiceImpl
|
||||
entity.getColumns()
|
||||
.forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
|
||||
columnRepository.saveAll(entity.getColumns());
|
||||
|
||||
List<DataSetColumnEntity> originalColumns = columnRepository.findAllByDataset(entity);
|
||||
List<DataSetColumnEntity> columnsNotInEntity = originalColumns.stream()
|
||||
.filter(originalColumn -> entity.getColumns().stream()
|
||||
.noneMatch(addedColumn -> addedColumn.getId().equals(originalColumn.getId())))
|
||||
.collect(Collectors.toList());
|
||||
columnRepository.deleteAll(columnsNotInEntity);
|
||||
}
|
||||
completeState(entity, DataSetState.METADATA_SUCCESS);
|
||||
}
|
||||
@ -746,45 +745,55 @@ public class DataSetServiceImpl
|
||||
private Set<CreatedModel> createdModeProcess(Set<DataSetColumnEntity> targetColumns, DataSetEntity entity)
|
||||
{
|
||||
Set<CreatedModel> models = Sets.newHashSet();
|
||||
|
||||
// If the id tag data is not set to new
|
||||
if (entity.getId() == null) {
|
||||
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
|
||||
return models;
|
||||
}
|
||||
|
||||
// Check whether there is a data table bound to the current dataset
|
||||
if (!checkTableExists(entity)) {
|
||||
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
|
||||
return models;
|
||||
}
|
||||
|
||||
List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity);
|
||||
for (DataSetColumnEntity sourceColumn : sourceColumns) {
|
||||
Optional<DataSetColumnEntity> filterColumn = targetColumns.stream()
|
||||
.filter(item -> item.getId().equals(sourceColumn.getId()))
|
||||
.findFirst();
|
||||
if (!filterColumn.isPresent()) {
|
||||
models.add(new CreatedModel(sourceColumn, CreatedMode.CREATE_COLUMN));
|
||||
try {
|
||||
// If the id tag data is not set to new
|
||||
if (entity.getId() == null) {
|
||||
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
|
||||
return models;
|
||||
}
|
||||
else {
|
||||
boolean matchFound = targetColumns.stream()
|
||||
|
||||
// Check whether there is a data table bound to the current dataset
|
||||
if (!checkTableExists(entity)) {
|
||||
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
|
||||
return models;
|
||||
}
|
||||
|
||||
List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity)
|
||||
.stream()
|
||||
.filter(item -> !item.isVirtualColumn())
|
||||
.collect(Collectors.toList());
|
||||
targetColumns = targetColumns.stream()
|
||||
.filter(item -> !item.isVirtualColumn())
|
||||
.collect(Collectors.toSet());
|
||||
for (DataSetColumnEntity sourceColumn : sourceColumns) {
|
||||
Optional<DataSetColumnEntity> filterColumn = targetColumns.stream()
|
||||
.filter(item -> item.getId().equals(sourceColumn.getId()))
|
||||
.anyMatch(targetColumn ->
|
||||
targetColumn.getType().equals(sourceColumn.getType()) && targetColumn.getName().equals(sourceColumn.getName()));
|
||||
if (!matchFound) {
|
||||
models.add(new CreatedModel(filterColumn.get(), CreatedMode.MODIFY_COLUMN));
|
||||
.findFirst();
|
||||
if (!filterColumn.isPresent()) {
|
||||
models.add(new CreatedModel(sourceColumn, CreatedMode.CREATE_COLUMN));
|
||||
}
|
||||
else {
|
||||
boolean matchFound = targetColumns.stream()
|
||||
.filter(item -> item.getId().equals(sourceColumn.getId()))
|
||||
.anyMatch(targetColumn ->
|
||||
targetColumn.getType().equals(sourceColumn.getType()) && targetColumn.getName().equals(sourceColumn.getName()));
|
||||
if (!matchFound) {
|
||||
models.add(new CreatedModel(filterColumn.get(), CreatedMode.MODIFY_COLUMN));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (entity.getLifeCycleColumn() != null) {
|
||||
DataSetColumnEntity column = DataSetColumnEntity.builder()
|
||||
.name(entity.getLifeCycleColumn())
|
||||
.length(Integer.parseInt(entity.getLifeCycle()))
|
||||
.defaultValue(entity.getLifeCycleType())
|
||||
.build();
|
||||
models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE));
|
||||
if (entity.getLifeCycleColumn() != null) {
|
||||
DataSetColumnEntity column = DataSetColumnEntity.builder()
|
||||
.name(entity.getLifeCycleColumn())
|
||||
.length(Integer.parseInt(entity.getLifeCycle()))
|
||||
.defaultValue(entity.getLifeCycleType())
|
||||
.build();
|
||||
models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE));
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.warn("Get create model for dataset [ {} ] id [ {} ] failed", entity.getName(), entity.getId(), e);
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
@ -53,6 +53,7 @@
|
||||
<Col class="w100 center">{{ $t('dataset.columnIsSampling') }}</Col>
|
||||
<Col class="w100 center">{{ $t('dataset.columnLength') }}</Col>
|
||||
<Col class="w200">{{ $t('dataset.columnComment') }}</Col>
|
||||
<Col class="w100 center">{{ $t('common.action') }}</Col>
|
||||
</Row>
|
||||
</FormItem>
|
||||
<template v-for="(item, index) in formState.columns"
|
||||
@ -93,30 +94,41 @@
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<Input v-model="item.defaultValue"
|
||||
type="text">
|
||||
type="text"
|
||||
:disabled="item.virtualColumn">
|
||||
</Input>
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<Switch v-model="item.nullable"/>
|
||||
<Switch v-model="item.nullable"
|
||||
:disabled="item.virtualColumn">
|
||||
</Switch>
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<Switch v-model="item.orderByKey"/>
|
||||
<Switch v-model="item.orderByKey"
|
||||
:disabled="item.virtualColumn">
|
||||
</Switch>
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<Switch v-model="item.partitionKey"/>
|
||||
<Switch v-model="item.partitionKey"
|
||||
:disabled="item.virtualColumn">
|
||||
</Switch>
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<Switch v-model="item.primaryKey"/>
|
||||
<Switch v-model="item.primaryKey"
|
||||
:disabled="item.virtualColumn">
|
||||
</Switch>
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<Switch v-model="item.samplingKey"
|
||||
:disabled="item.virtualColumn"
|
||||
@on-change="validatorSampling">
|
||||
</Switch>
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<InputNumber v-model="item.length"
|
||||
min="0"
|
||||
max="65536">
|
||||
max="65536"
|
||||
:disabled="item.type === 'BOOLEAN' || item.type === 'DATETIME' || item.virtualColumn">
|
||||
</InputNumber>
|
||||
</Col>
|
||||
<Col class="w200">
|
||||
@ -124,6 +136,23 @@
|
||||
type="textarea">
|
||||
</Input>
|
||||
</Col>
|
||||
<Col class="w100 center">
|
||||
<Space>
|
||||
<Button type="error"
|
||||
size="small"
|
||||
shape="circle"
|
||||
icon="md-trash"
|
||||
:disabled="!item.customColumn"
|
||||
@click="handlerRemoveColumn(index)">
|
||||
</Button>
|
||||
<Button type="primary"
|
||||
size="small"
|
||||
shape="circle"
|
||||
icon="md-add"
|
||||
@click="handlerAddColumn(index)">
|
||||
</Button>
|
||||
</Space>
|
||||
</Col>
|
||||
</Row>
|
||||
</FormItem>
|
||||
</template>
|
||||
@ -358,7 +387,9 @@ export default defineComponent({
|
||||
partitionKey: false,
|
||||
primaryKey: false,
|
||||
samplingKey: false,
|
||||
mode: 'DIMENSION'
|
||||
mode: 'DIMENSION',
|
||||
virtualColumn: false,
|
||||
customColumn: false
|
||||
}
|
||||
this.formState.columns.push(column)
|
||||
})
|
||||
@ -377,6 +408,32 @@ export default defineComponent({
|
||||
})
|
||||
.finally(() => this.saving = false)
|
||||
},
|
||||
handlerAddColumn(index: number)
|
||||
{
|
||||
this.formState.columns.splice(index + 1, 0, {
|
||||
id: null,
|
||||
name: null,
|
||||
aliasName: null,
|
||||
type: 'STRING',
|
||||
comment: null,
|
||||
defaultValue: null,
|
||||
position: index + 1,
|
||||
nullable: false,
|
||||
length: 0,
|
||||
original: null,
|
||||
orderByKey: false,
|
||||
partitionKey: false,
|
||||
primaryKey: false,
|
||||
samplingKey: false,
|
||||
mode: 'DIMENSION',
|
||||
virtualColumn: true,
|
||||
customColumn: true
|
||||
})
|
||||
},
|
||||
handlerRemoveColumn(index: number)
|
||||
{
|
||||
this.formState.columns.splice(index, 1)
|
||||
},
|
||||
validatorSampling()
|
||||
{
|
||||
const samplingColumns = this.formState.columns
|
||||
|
||||
Loading…
Reference in New Issue
Block a user