test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-12-05 05:27:52 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

[Feature] [DataSet] Support virtual columns (#693)

Browse Source
This commit is contained in:
qianmoQ 2024-03-01 11:39:10 +08:00 committed by GitHub
commit 408e1b733b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 132 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/datacap-server/src/main/schema/2024.02.1/schema.sql
View File

@ -6,7 +6,9 @@ USE `datacap`;
ALTER TABLE `datacap_dataset_column`
ADD COLUMN `is_primary_key` BOOLEAN DEFAULT FALSE,
ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE;
ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE,
ADD COLUMN `is_custom_column` BOOLEAN DEFAULT FALSE,
ADD COLUMN `is_virtual_column` BOOLEAN DEFAULT FALSE;
ALTER TABLE `datacap_dataset`
ADD COLUMN `total_rows` BIGINT(100) DEFAULT 0,

6
core/datacap-service/src/main/java/io/edurt/datacap/service/entity/DataSetColumnEntity.java
View File

@ -75,6 +75,12 @@ public class DataSetColumnEntity
@Column(name = "alias_name")
private String aliasName;
@Column(name = "is_custom_column")
private boolean customColumn;
@Column(name = "is_virtual_column")
private boolean virtualColumn;
@ManyToOne
@JoinTable(name = "datacap_dataset_column_relation",
joinColumns = @JoinColumn(name = "column_id"),

2
core/datacap-service/src/main/java/io/edurt/datacap/service/repository/DataSetColumnRepository.java
View File

@ -10,4 +10,6 @@ public interface DataSetColumnRepository
extends PagingAndSortingRepository<DataSetColumnEntity, Long>
{
List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset);
List<DataSetColumnEntity> findAllByDatasetOrderByPositionAsc(DataSetEntity dataset);
}

35
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java
View File

@ -142,17 +142,15 @@ public class DataSetServiceImpl
public CommonResponse<Set<DataSetColumnEntity>> getColumns(Long id)
{
Optional<DataSetEntity> entity = repository.findById(id);
if (!entity.isPresent()) {
return CommonResponse.failure(String.format("DataSet [ %s ] not found", id));
}
return CommonResponse.success(columnRepository.findAllByDataset(entity.get()));
return entity.map(dataSet -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(dataSet)))
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", id)));
}
@Override
public CommonResponse<Set<DataSetColumnEntity>> getColumnsByCode(String code)
{
Optional<DataSetEntity> entity = repository.findByCode(code);
return entity.map(item -> CommonResponse.success(columnRepository.findAllByDataset(item)))
return entity.map(item -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(item)))
.orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code)));
}
@ -356,15 +354,9 @@ public class DataSetServiceImpl
String tableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", ""));
entity.setTableName(tableName);
}
DataSetState state = entity.getState().get(entity.getState().size() - 1);
if (state.equals(DataSetState.METADATA_START)
|| state.equals(DataSetState.METADATA_FAILED)
|| state.equals(DataSetState.TABLE_FAILED)) {
log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId());
createMetadata(entity, rebuildColumn);
}
throw new IllegalArgumentException(String.format("Invalid state [ %s ]", state));
}
private void createMetadata(DataSetEntity entity, boolean rebuildColumn)
{
@ -376,6 +368,13 @@ public class DataSetServiceImpl
entity.getColumns()
.forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
columnRepository.saveAll(entity.getColumns());
List<DataSetColumnEntity> originalColumns = columnRepository.findAllByDataset(entity);
List<DataSetColumnEntity> columnsNotInEntity = originalColumns.stream()
.filter(originalColumn -> entity.getColumns().stream()
.noneMatch(addedColumn -> addedColumn.getId().equals(originalColumn.getId())))
.collect(Collectors.toList());
columnRepository.deleteAll(columnsNotInEntity);
}
completeState(entity, DataSetState.METADATA_SUCCESS);
}
@ -746,7 +745,7 @@ public class DataSetServiceImpl
private Set<CreatedModel> createdModeProcess(Set<DataSetColumnEntity> targetColumns, DataSetEntity entity)
{
Set<CreatedModel> models = Sets.newHashSet();
try {
// If the id tag data is not set to new
if (entity.getId() == null) {
models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
@ -759,7 +758,13 @@ public class DataSetServiceImpl
return models;
}
List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity);
List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity)
.stream()
.filter(item -> !item.isVirtualColumn())
.collect(Collectors.toList());
targetColumns = targetColumns.stream()
.filter(item -> !item.isVirtualColumn())
.collect(Collectors.toSet());
for (DataSetColumnEntity sourceColumn : sourceColumns) {
Optional<DataSetColumnEntity> filterColumn = targetColumns.stream()
.filter(item -> item.getId().equals(sourceColumn.getId()))
@ -786,6 +791,10 @@ public class DataSetServiceImpl
.build();
models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE));
}
}
catch (Exception e) {
log.warn("Get create model for dataset [ {} ] id [ {} ] failed", entity.getName(), entity.getId(), e);
}
return models;
}

71
core/datacap-web/src/views/admin/dataset/DatasetInfo.vue
View File

@ -53,6 +53,7 @@
<Col class="w100 center">{{ $t('dataset.columnIsSampling') }}</Col>
<Col class="w100 center">{{ $t('dataset.columnLength') }}</Col>
<Col class="w200">{{ $t('dataset.columnComment') }}</Col>
<Col class="w100 center">{{ $t('common.action') }}</Col>
</Row>
</FormItem>
<template v-for="(item, index) in formState.columns"
@ -93,30 +94,41 @@
</Col>
<Col class="w100 center">
<Input v-model="item.defaultValue"
type="text">
type="text"
:disabled="item.virtualColumn">
</Input>
</Col>
<Col class="w100 center">
<Switch v-model="item.nullable"/>
<Switch v-model="item.nullable"
:disabled="item.virtualColumn">
</Switch>
</Col>
<Col class="w100 center">
<Switch v-model="item.orderByKey"/>
<Switch v-model="item.orderByKey"
:disabled="item.virtualColumn">
</Switch>
</Col>
<Col class="w100 center">
<Switch v-model="item.partitionKey"/>
<Switch v-model="item.partitionKey"
:disabled="item.virtualColumn">
</Switch>
</Col>
<Col class="w100 center">
<Switch v-model="item.primaryKey"/>
<Switch v-model="item.primaryKey"
:disabled="item.virtualColumn">
</Switch>
</Col>
<Col class="w100 center">
<Switch v-model="item.samplingKey"
:disabled="item.virtualColumn"
@on-change="validatorSampling">
</Switch>
</Col>
<Col class="w100 center">
<InputNumber v-model="item.length"
min="0"
max="65536">
max="65536"
:disabled="item.type === 'BOOLEAN' || item.type === 'DATETIME' || item.virtualColumn">
</InputNumber>
</Col>
<Col class="w200">
@ -124,6 +136,23 @@
type="textarea">
</Input>
</Col>
<Col class="w100 center">
<Space>
<Button type="error"
size="small"
shape="circle"
icon="md-trash"
:disabled="!item.customColumn"
@click="handlerRemoveColumn(index)">
</Button>
<Button type="primary"
size="small"
shape="circle"
icon="md-add"
@click="handlerAddColumn(index)">
</Button>
</Space>
</Col>
</Row>
</FormItem>
</template>
@ -358,7 +387,9 @@ export default defineComponent({
partitionKey: false,
primaryKey: false,
samplingKey: false,
mode: 'DIMENSION'
mode: 'DIMENSION',
virtualColumn: false,
customColumn: false
}
this.formState.columns.push(column)
})
@ -377,6 +408,32 @@ export default defineComponent({
})
.finally(() => this.saving = false)
},
handlerAddColumn(index: number)
{
this.formState.columns.splice(index + 1, 0, {
id: null,
name: null,
aliasName: null,
type: 'STRING',
comment: null,
defaultValue: null,
position: index + 1,
nullable: false,
length: 0,
original: null,
orderByKey: false,
partitionKey: false,
primaryKey: false,
samplingKey: false,
mode: 'DIMENSION',
virtualColumn: true,
customColumn: true
})
},
handlerRemoveColumn(index: number)
{
this.formState.columns.splice(index, 1)
},
validatorSampling()
{
const samplingColumns = this.formState.columns