[Feature] [DataSet] Support virtual columns (#693)

This commit is contained in:
qianmoQ 2024-03-01 11:39:10 +08:00 committed by GitHub
commit 408e1b733b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 132 additions and 56 deletions

View File

@ -6,7 +6,9 @@ USE `datacap`;
ALTER TABLE `datacap_dataset_column` ALTER TABLE `datacap_dataset_column`
ADD COLUMN `is_primary_key` BOOLEAN DEFAULT FALSE, ADD COLUMN `is_primary_key` BOOLEAN DEFAULT FALSE,
ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE; ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE,
ADD COLUMN `is_custom_column` BOOLEAN DEFAULT FALSE,
ADD COLUMN `is_virtual_column` BOOLEAN DEFAULT FALSE;
ALTER TABLE `datacap_dataset` ALTER TABLE `datacap_dataset`
ADD COLUMN `total_rows` BIGINT(100) DEFAULT 0, ADD COLUMN `total_rows` BIGINT(100) DEFAULT 0,

View File

@ -75,6 +75,12 @@ public class DataSetColumnEntity
@Column(name = "alias_name") @Column(name = "alias_name")
private String aliasName; private String aliasName;
@Column(name = "is_custom_column")
private boolean customColumn;
@Column(name = "is_virtual_column")
private boolean virtualColumn;
@ManyToOne @ManyToOne
@JoinTable(name = "datacap_dataset_column_relation", @JoinTable(name = "datacap_dataset_column_relation",
joinColumns = @JoinColumn(name = "column_id"), joinColumns = @JoinColumn(name = "column_id"),

View File

@ -10,4 +10,6 @@ public interface DataSetColumnRepository
extends PagingAndSortingRepository<DataSetColumnEntity, Long> extends PagingAndSortingRepository<DataSetColumnEntity, Long>
{ {
List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset); List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset);
List<DataSetColumnEntity> findAllByDatasetOrderByPositionAsc(DataSetEntity dataset);
} }

View File

@ -142,17 +142,15 @@ public class DataSetServiceImpl
public CommonResponse<Set<DataSetColumnEntity>> getColumns(Long id) public CommonResponse<Set<DataSetColumnEntity>> getColumns(Long id)
{ {
Optional<DataSetEntity> entity = repository.findById(id); Optional<DataSetEntity> entity = repository.findById(id);
if (!entity.isPresent()) { return entity.map(dataSet -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(dataSet)))
return CommonResponse.failure(String.format("DataSet [ %s ] not found", id)); .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", id)));
}
return CommonResponse.success(columnRepository.findAllByDataset(entity.get()));
} }
@Override @Override
public CommonResponse<Set<DataSetColumnEntity>> getColumnsByCode(String code) public CommonResponse<Set<DataSetColumnEntity>> getColumnsByCode(String code)
{ {
Optional<DataSetEntity> entity = repository.findByCode(code); Optional<DataSetEntity> entity = repository.findByCode(code);
return entity.map(item -> CommonResponse.success(columnRepository.findAllByDataset(item))) return entity.map(item -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(item)))
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code))); .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code)));
} }
@ -356,15 +354,9 @@ public class DataSetServiceImpl
String tableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", "")); String tableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", ""));
entity.setTableName(tableName); entity.setTableName(tableName);
} }
DataSetState state = entity.getState().get(entity.getState().size() - 1);
if (state.equals(DataSetState.METADATA_START)
|| state.equals(DataSetState.METADATA_FAILED)
|| state.equals(DataSetState.TABLE_FAILED)) {
log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId()); log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId());
createMetadata(entity, rebuildColumn); createMetadata(entity, rebuildColumn);
} }
throw new IllegalArgumentException(String.format("Invalid state [ %s ]", state));
}
private void createMetadata(DataSetEntity entity, boolean rebuildColumn) private void createMetadata(DataSetEntity entity, boolean rebuildColumn)
{ {
@ -376,6 +368,13 @@ public class DataSetServiceImpl
entity.getColumns() entity.getColumns()
.forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build())); .forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
columnRepository.saveAll(entity.getColumns()); columnRepository.saveAll(entity.getColumns());
List<DataSetColumnEntity> originalColumns = columnRepository.findAllByDataset(entity);
List<DataSetColumnEntity> columnsNotInEntity = originalColumns.stream()
.filter(originalColumn -> entity.getColumns().stream()
.noneMatch(addedColumn -> addedColumn.getId().equals(originalColumn.getId())))
.collect(Collectors.toList());
columnRepository.deleteAll(columnsNotInEntity);
} }
completeState(entity, DataSetState.METADATA_SUCCESS); completeState(entity, DataSetState.METADATA_SUCCESS);
} }
@ -746,7 +745,7 @@ public class DataSetServiceImpl
private Set<CreatedModel> createdModeProcess(Set<DataSetColumnEntity> targetColumns, DataSetEntity entity) private Set<CreatedModel> createdModeProcess(Set<DataSetColumnEntity> targetColumns, DataSetEntity entity)
{ {
Set<CreatedModel> models = Sets.newHashSet(); Set<CreatedModel> models = Sets.newHashSet();
try {
// If the id tag data is not set to new // If the id tag data is not set to new
if (entity.getId() == null) { if (entity.getId() == null) {
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE)); models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
@ -759,7 +758,13 @@ public class DataSetServiceImpl
return models; return models;
} }
List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity); List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity)
.stream()
.filter(item -> !item.isVirtualColumn())
.collect(Collectors.toList());
targetColumns = targetColumns.stream()
.filter(item -> !item.isVirtualColumn())
.collect(Collectors.toSet());
for (DataSetColumnEntity sourceColumn : sourceColumns) { for (DataSetColumnEntity sourceColumn : sourceColumns) {
Optional<DataSetColumnEntity> filterColumn = targetColumns.stream() Optional<DataSetColumnEntity> filterColumn = targetColumns.stream()
.filter(item -> item.getId().equals(sourceColumn.getId())) .filter(item -> item.getId().equals(sourceColumn.getId()))
@ -786,6 +791,10 @@ public class DataSetServiceImpl
.build(); .build();
models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE)); models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE));
} }
}
catch (Exception e) {
log.warn("Get create model for dataset [ {} ] id [ {} ] failed", entity.getName(), entity.getId(), e);
}
return models; return models;
} }

View File

@ -53,6 +53,7 @@
<Col class="w100 center">{{ $t('dataset.columnIsSampling') }}</Col> <Col class="w100 center">{{ $t('dataset.columnIsSampling') }}</Col>
<Col class="w100 center">{{ $t('dataset.columnLength') }}</Col> <Col class="w100 center">{{ $t('dataset.columnLength') }}</Col>
<Col class="w200">{{ $t('dataset.columnComment') }}</Col> <Col class="w200">{{ $t('dataset.columnComment') }}</Col>
<Col class="w100 center">{{ $t('common.action') }}</Col>
</Row> </Row>
</FormItem> </FormItem>
<template v-for="(item, index) in formState.columns" <template v-for="(item, index) in formState.columns"
@ -93,30 +94,41 @@
</Col> </Col>
<Col class="w100 center"> <Col class="w100 center">
<Input v-model="item.defaultValue" <Input v-model="item.defaultValue"
type="text"> type="text"
:disabled="item.virtualColumn">
</Input> </Input>
</Col> </Col>
<Col class="w100 center"> <Col class="w100 center">
<Switch v-model="item.nullable"/> <Switch v-model="item.nullable"
:disabled="item.virtualColumn">
</Switch>
</Col> </Col>
<Col class="w100 center"> <Col class="w100 center">
<Switch v-model="item.orderByKey"/> <Switch v-model="item.orderByKey"
:disabled="item.virtualColumn">
</Switch>
</Col> </Col>
<Col class="w100 center"> <Col class="w100 center">
<Switch v-model="item.partitionKey"/> <Switch v-model="item.partitionKey"
:disabled="item.virtualColumn">
</Switch>
</Col> </Col>
<Col class="w100 center"> <Col class="w100 center">
<Switch v-model="item.primaryKey"/> <Switch v-model="item.primaryKey"
:disabled="item.virtualColumn">
</Switch>
</Col> </Col>
<Col class="w100 center"> <Col class="w100 center">
<Switch v-model="item.samplingKey" <Switch v-model="item.samplingKey"
:disabled="item.virtualColumn"
@on-change="validatorSampling"> @on-change="validatorSampling">
</Switch> </Switch>
</Col> </Col>
<Col class="w100 center"> <Col class="w100 center">
<InputNumber v-model="item.length" <InputNumber v-model="item.length"
min="0" min="0"
max="65536"> max="65536"
:disabled="item.type === 'BOOLEAN' || item.type === 'DATETIME' || item.virtualColumn">
</InputNumber> </InputNumber>
</Col> </Col>
<Col class="w200"> <Col class="w200">
@ -124,6 +136,23 @@
type="textarea"> type="textarea">
</Input> </Input>
</Col> </Col>
<Col class="w100 center">
<Space>
<Button type="error"
size="small"
shape="circle"
icon="md-trash"
:disabled="!item.customColumn"
@click="handlerRemoveColumn(index)">
</Button>
<Button type="primary"
size="small"
shape="circle"
icon="md-add"
@click="handlerAddColumn(index)">
</Button>
</Space>
</Col>
</Row> </Row>
</FormItem> </FormItem>
</template> </template>
@ -358,7 +387,9 @@ export default defineComponent({
partitionKey: false, partitionKey: false,
primaryKey: false, primaryKey: false,
samplingKey: false, samplingKey: false,
mode: 'DIMENSION' mode: 'DIMENSION',
virtualColumn: false,
customColumn: false
} }
this.formState.columns.push(column) this.formState.columns.push(column)
}) })
@ -377,6 +408,32 @@ export default defineComponent({
}) })
.finally(() => this.saving = false) .finally(() => this.saving = false)
}, },
handlerAddColumn(index: number)
{
this.formState.columns.splice(index + 1, 0, {
id: null,
name: null,
aliasName: null,
type: 'STRING',
comment: null,
defaultValue: null,
position: index + 1,
nullable: false,
length: 0,
original: null,
orderByKey: false,
partitionKey: false,
primaryKey: false,
samplingKey: false,
mode: 'DIMENSION',
virtualColumn: true,
customColumn: true
})
},
handlerRemoveColumn(index: number)
{
this.formState.columns.splice(index, 1)
},
validatorSampling() validatorSampling()
{ {
const samplingColumns = this.formState.columns const samplingColumns = this.formState.columns