mirror of
https://gitee.com/devlive-community/datacap.git
synced 2024-12-05 13:38:07 +08:00
[Feature] [DataSet] Support virtual columns (#693)
This commit is contained in:
commit
408e1b733b
@ -6,7 +6,9 @@ USE `datacap`;
|
|||||||
|
|
||||||
ALTER TABLE `datacap_dataset_column`
|
ALTER TABLE `datacap_dataset_column`
|
||||||
ADD COLUMN `is_primary_key` BOOLEAN DEFAULT FALSE,
|
ADD COLUMN `is_primary_key` BOOLEAN DEFAULT FALSE,
|
||||||
ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE;
|
ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE,
|
||||||
|
ADD COLUMN `is_custom_column` BOOLEAN DEFAULT FALSE,
|
||||||
|
ADD COLUMN `is_virtual_column` BOOLEAN DEFAULT FALSE;
|
||||||
|
|
||||||
ALTER TABLE `datacap_dataset`
|
ALTER TABLE `datacap_dataset`
|
||||||
ADD COLUMN `total_rows` BIGINT(100) DEFAULT 0,
|
ADD COLUMN `total_rows` BIGINT(100) DEFAULT 0,
|
||||||
|
@ -75,6 +75,12 @@ public class DataSetColumnEntity
|
|||||||
@Column(name = "alias_name")
|
@Column(name = "alias_name")
|
||||||
private String aliasName;
|
private String aliasName;
|
||||||
|
|
||||||
|
@Column(name = "is_custom_column")
|
||||||
|
private boolean customColumn;
|
||||||
|
|
||||||
|
@Column(name = "is_virtual_column")
|
||||||
|
private boolean virtualColumn;
|
||||||
|
|
||||||
@ManyToOne
|
@ManyToOne
|
||||||
@JoinTable(name = "datacap_dataset_column_relation",
|
@JoinTable(name = "datacap_dataset_column_relation",
|
||||||
joinColumns = @JoinColumn(name = "column_id"),
|
joinColumns = @JoinColumn(name = "column_id"),
|
||||||
|
@ -10,4 +10,6 @@ public interface DataSetColumnRepository
|
|||||||
extends PagingAndSortingRepository<DataSetColumnEntity, Long>
|
extends PagingAndSortingRepository<DataSetColumnEntity, Long>
|
||||||
{
|
{
|
||||||
List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset);
|
List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset);
|
||||||
|
|
||||||
|
List<DataSetColumnEntity> findAllByDatasetOrderByPositionAsc(DataSetEntity dataset);
|
||||||
}
|
}
|
||||||
|
@ -142,17 +142,15 @@ public class DataSetServiceImpl
|
|||||||
public CommonResponse<Set<DataSetColumnEntity>> getColumns(Long id)
|
public CommonResponse<Set<DataSetColumnEntity>> getColumns(Long id)
|
||||||
{
|
{
|
||||||
Optional<DataSetEntity> entity = repository.findById(id);
|
Optional<DataSetEntity> entity = repository.findById(id);
|
||||||
if (!entity.isPresent()) {
|
return entity.map(dataSet -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(dataSet)))
|
||||||
return CommonResponse.failure(String.format("DataSet [ %s ] not found", id));
|
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", id)));
|
||||||
}
|
|
||||||
return CommonResponse.success(columnRepository.findAllByDataset(entity.get()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CommonResponse<Set<DataSetColumnEntity>> getColumnsByCode(String code)
|
public CommonResponse<Set<DataSetColumnEntity>> getColumnsByCode(String code)
|
||||||
{
|
{
|
||||||
Optional<DataSetEntity> entity = repository.findByCode(code);
|
Optional<DataSetEntity> entity = repository.findByCode(code);
|
||||||
return entity.map(item -> CommonResponse.success(columnRepository.findAllByDataset(item)))
|
return entity.map(item -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(item)))
|
||||||
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code)));
|
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -356,15 +354,9 @@ public class DataSetServiceImpl
|
|||||||
String tableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", ""));
|
String tableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", ""));
|
||||||
entity.setTableName(tableName);
|
entity.setTableName(tableName);
|
||||||
}
|
}
|
||||||
DataSetState state = entity.getState().get(entity.getState().size() - 1);
|
|
||||||
if (state.equals(DataSetState.METADATA_START)
|
|
||||||
|| state.equals(DataSetState.METADATA_FAILED)
|
|
||||||
|| state.equals(DataSetState.TABLE_FAILED)) {
|
|
||||||
log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId());
|
log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId());
|
||||||
createMetadata(entity, rebuildColumn);
|
createMetadata(entity, rebuildColumn);
|
||||||
}
|
}
|
||||||
throw new IllegalArgumentException(String.format("Invalid state [ %s ]", state));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void createMetadata(DataSetEntity entity, boolean rebuildColumn)
|
private void createMetadata(DataSetEntity entity, boolean rebuildColumn)
|
||||||
{
|
{
|
||||||
@ -376,6 +368,13 @@ public class DataSetServiceImpl
|
|||||||
entity.getColumns()
|
entity.getColumns()
|
||||||
.forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
|
.forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
|
||||||
columnRepository.saveAll(entity.getColumns());
|
columnRepository.saveAll(entity.getColumns());
|
||||||
|
|
||||||
|
List<DataSetColumnEntity> originalColumns = columnRepository.findAllByDataset(entity);
|
||||||
|
List<DataSetColumnEntity> columnsNotInEntity = originalColumns.stream()
|
||||||
|
.filter(originalColumn -> entity.getColumns().stream()
|
||||||
|
.noneMatch(addedColumn -> addedColumn.getId().equals(originalColumn.getId())))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
columnRepository.deleteAll(columnsNotInEntity);
|
||||||
}
|
}
|
||||||
completeState(entity, DataSetState.METADATA_SUCCESS);
|
completeState(entity, DataSetState.METADATA_SUCCESS);
|
||||||
}
|
}
|
||||||
@ -746,7 +745,7 @@ public class DataSetServiceImpl
|
|||||||
private Set<CreatedModel> createdModeProcess(Set<DataSetColumnEntity> targetColumns, DataSetEntity entity)
|
private Set<CreatedModel> createdModeProcess(Set<DataSetColumnEntity> targetColumns, DataSetEntity entity)
|
||||||
{
|
{
|
||||||
Set<CreatedModel> models = Sets.newHashSet();
|
Set<CreatedModel> models = Sets.newHashSet();
|
||||||
|
try {
|
||||||
// If the id tag data is not set to new
|
// If the id tag data is not set to new
|
||||||
if (entity.getId() == null) {
|
if (entity.getId() == null) {
|
||||||
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
|
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
|
||||||
@ -759,7 +758,13 @@ public class DataSetServiceImpl
|
|||||||
return models;
|
return models;
|
||||||
}
|
}
|
||||||
|
|
||||||
List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity);
|
List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity)
|
||||||
|
.stream()
|
||||||
|
.filter(item -> !item.isVirtualColumn())
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
targetColumns = targetColumns.stream()
|
||||||
|
.filter(item -> !item.isVirtualColumn())
|
||||||
|
.collect(Collectors.toSet());
|
||||||
for (DataSetColumnEntity sourceColumn : sourceColumns) {
|
for (DataSetColumnEntity sourceColumn : sourceColumns) {
|
||||||
Optional<DataSetColumnEntity> filterColumn = targetColumns.stream()
|
Optional<DataSetColumnEntity> filterColumn = targetColumns.stream()
|
||||||
.filter(item -> item.getId().equals(sourceColumn.getId()))
|
.filter(item -> item.getId().equals(sourceColumn.getId()))
|
||||||
@ -786,6 +791,10 @@ public class DataSetServiceImpl
|
|||||||
.build();
|
.build();
|
||||||
models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE));
|
models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
log.warn("Get create model for dataset [ {} ] id [ {} ] failed", entity.getName(), entity.getId(), e);
|
||||||
|
}
|
||||||
return models;
|
return models;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -53,6 +53,7 @@
|
|||||||
<Col class="w100 center">{{ $t('dataset.columnIsSampling') }}</Col>
|
<Col class="w100 center">{{ $t('dataset.columnIsSampling') }}</Col>
|
||||||
<Col class="w100 center">{{ $t('dataset.columnLength') }}</Col>
|
<Col class="w100 center">{{ $t('dataset.columnLength') }}</Col>
|
||||||
<Col class="w200">{{ $t('dataset.columnComment') }}</Col>
|
<Col class="w200">{{ $t('dataset.columnComment') }}</Col>
|
||||||
|
<Col class="w100 center">{{ $t('common.action') }}</Col>
|
||||||
</Row>
|
</Row>
|
||||||
</FormItem>
|
</FormItem>
|
||||||
<template v-for="(item, index) in formState.columns"
|
<template v-for="(item, index) in formState.columns"
|
||||||
@ -93,30 +94,41 @@
|
|||||||
</Col>
|
</Col>
|
||||||
<Col class="w100 center">
|
<Col class="w100 center">
|
||||||
<Input v-model="item.defaultValue"
|
<Input v-model="item.defaultValue"
|
||||||
type="text">
|
type="text"
|
||||||
|
:disabled="item.virtualColumn">
|
||||||
</Input>
|
</Input>
|
||||||
</Col>
|
</Col>
|
||||||
<Col class="w100 center">
|
<Col class="w100 center">
|
||||||
<Switch v-model="item.nullable"/>
|
<Switch v-model="item.nullable"
|
||||||
|
:disabled="item.virtualColumn">
|
||||||
|
</Switch>
|
||||||
</Col>
|
</Col>
|
||||||
<Col class="w100 center">
|
<Col class="w100 center">
|
||||||
<Switch v-model="item.orderByKey"/>
|
<Switch v-model="item.orderByKey"
|
||||||
|
:disabled="item.virtualColumn">
|
||||||
|
</Switch>
|
||||||
</Col>
|
</Col>
|
||||||
<Col class="w100 center">
|
<Col class="w100 center">
|
||||||
<Switch v-model="item.partitionKey"/>
|
<Switch v-model="item.partitionKey"
|
||||||
|
:disabled="item.virtualColumn">
|
||||||
|
</Switch>
|
||||||
</Col>
|
</Col>
|
||||||
<Col class="w100 center">
|
<Col class="w100 center">
|
||||||
<Switch v-model="item.primaryKey"/>
|
<Switch v-model="item.primaryKey"
|
||||||
|
:disabled="item.virtualColumn">
|
||||||
|
</Switch>
|
||||||
</Col>
|
</Col>
|
||||||
<Col class="w100 center">
|
<Col class="w100 center">
|
||||||
<Switch v-model="item.samplingKey"
|
<Switch v-model="item.samplingKey"
|
||||||
|
:disabled="item.virtualColumn"
|
||||||
@on-change="validatorSampling">
|
@on-change="validatorSampling">
|
||||||
</Switch>
|
</Switch>
|
||||||
</Col>
|
</Col>
|
||||||
<Col class="w100 center">
|
<Col class="w100 center">
|
||||||
<InputNumber v-model="item.length"
|
<InputNumber v-model="item.length"
|
||||||
min="0"
|
min="0"
|
||||||
max="65536">
|
max="65536"
|
||||||
|
:disabled="item.type === 'BOOLEAN' || item.type === 'DATETIME' || item.virtualColumn">
|
||||||
</InputNumber>
|
</InputNumber>
|
||||||
</Col>
|
</Col>
|
||||||
<Col class="w200">
|
<Col class="w200">
|
||||||
@ -124,6 +136,23 @@
|
|||||||
type="textarea">
|
type="textarea">
|
||||||
</Input>
|
</Input>
|
||||||
</Col>
|
</Col>
|
||||||
|
<Col class="w100 center">
|
||||||
|
<Space>
|
||||||
|
<Button type="error"
|
||||||
|
size="small"
|
||||||
|
shape="circle"
|
||||||
|
icon="md-trash"
|
||||||
|
:disabled="!item.customColumn"
|
||||||
|
@click="handlerRemoveColumn(index)">
|
||||||
|
</Button>
|
||||||
|
<Button type="primary"
|
||||||
|
size="small"
|
||||||
|
shape="circle"
|
||||||
|
icon="md-add"
|
||||||
|
@click="handlerAddColumn(index)">
|
||||||
|
</Button>
|
||||||
|
</Space>
|
||||||
|
</Col>
|
||||||
</Row>
|
</Row>
|
||||||
</FormItem>
|
</FormItem>
|
||||||
</template>
|
</template>
|
||||||
@ -358,7 +387,9 @@ export default defineComponent({
|
|||||||
partitionKey: false,
|
partitionKey: false,
|
||||||
primaryKey: false,
|
primaryKey: false,
|
||||||
samplingKey: false,
|
samplingKey: false,
|
||||||
mode: 'DIMENSION'
|
mode: 'DIMENSION',
|
||||||
|
virtualColumn: false,
|
||||||
|
customColumn: false
|
||||||
}
|
}
|
||||||
this.formState.columns.push(column)
|
this.formState.columns.push(column)
|
||||||
})
|
})
|
||||||
@ -377,6 +408,32 @@ export default defineComponent({
|
|||||||
})
|
})
|
||||||
.finally(() => this.saving = false)
|
.finally(() => this.saving = false)
|
||||||
},
|
},
|
||||||
|
handlerAddColumn(index: number)
|
||||||
|
{
|
||||||
|
this.formState.columns.splice(index + 1, 0, {
|
||||||
|
id: null,
|
||||||
|
name: null,
|
||||||
|
aliasName: null,
|
||||||
|
type: 'STRING',
|
||||||
|
comment: null,
|
||||||
|
defaultValue: null,
|
||||||
|
position: index + 1,
|
||||||
|
nullable: false,
|
||||||
|
length: 0,
|
||||||
|
original: null,
|
||||||
|
orderByKey: false,
|
||||||
|
partitionKey: false,
|
||||||
|
primaryKey: false,
|
||||||
|
samplingKey: false,
|
||||||
|
mode: 'DIMENSION',
|
||||||
|
virtualColumn: true,
|
||||||
|
customColumn: true
|
||||||
|
})
|
||||||
|
},
|
||||||
|
handlerRemoveColumn(index: number)
|
||||||
|
{
|
||||||
|
this.formState.columns.splice(index, 1)
|
||||||
|
},
|
||||||
validatorSampling()
|
validatorSampling()
|
||||||
{
|
{
|
||||||
const samplingColumns = this.formState.columns
|
const samplingColumns = this.formState.columns
|
||||||
|
Loading…
Reference in New Issue
Block a user