[Feature] [DataSet] Support virtual columns (#693)

core/datacap-server/src/main/schema/2024.02.1/schema.sql

@@ -6,7 +6,9 @@ USE `datacap`;
 
 ALTER TABLE `datacap_dataset_column`
     ADD COLUMN `is_primary_key`    BOOLEAN DEFAULT FALSE,
-    ADD COLUMN `is_sampling_key` BOOLEAN DEFAULT FALSE;
+    ADD COLUMN `is_sampling_key`   BOOLEAN DEFAULT FALSE,
+    ADD COLUMN `is_custom_column`  BOOLEAN DEFAULT FALSE,
+    ADD COLUMN `is_virtual_column` BOOLEAN DEFAULT FALSE;
 
 ALTER TABLE `datacap_dataset`
     ADD COLUMN `total_rows`        BIGINT(100) DEFAULT 0,

core/datacap-service/src/main/java/io/edurt/datacap/service/entity/DataSetColumnEntity.java

@@ -75,6 +75,12 @@ public class DataSetColumnEntity
     @Column(name = "alias_name")
     private String aliasName;
 
+    @Column(name = "is_custom_column")
+    private boolean customColumn;
+
+    @Column(name = "is_virtual_column")
+    private boolean virtualColumn;
+
     @ManyToOne
     @JoinTable(name = "datacap_dataset_column_relation",
             joinColumns = @JoinColumn(name = "column_id"),

core/datacap-service/src/main/java/io/edurt/datacap/service/repository/DataSetColumnRepository.java

@@ -10,4 +10,6 @@ public interface DataSetColumnRepository
         extends PagingAndSortingRepository<DataSetColumnEntity, Long>
 {
     List<DataSetColumnEntity> findAllByDataset(DataSetEntity dataset);
+
+    List<DataSetColumnEntity> findAllByDatasetOrderByPositionAsc(DataSetEntity dataset);
 }

core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java

@@ -142,17 +142,15 @@ public class DataSetServiceImpl
     public CommonResponse<Set<DataSetColumnEntity>> getColumns(Long id)
     {
         Optional<DataSetEntity> entity = repository.findById(id);
-        if (!entity.isPresent()) {
+        return entity.map(dataSet -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(dataSet)))
-            return CommonResponse.failure(String.format("DataSet [ %s ] not found", id));
+                .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", id)));
-        }
-        return CommonResponse.success(columnRepository.findAllByDataset(entity.get()));
     }
 
     @Override
     public CommonResponse<Set<DataSetColumnEntity>> getColumnsByCode(String code)
     {
         Optional<DataSetEntity> entity = repository.findByCode(code);
-        return entity.map(item -> CommonResponse.success(columnRepository.findAllByDataset(item)))
+        return entity.map(item -> CommonResponse.success(columnRepository.findAllByDatasetOrderByPositionAsc(item)))
                 .orElseGet(() -> CommonResponse.failure(String.format("DataSet [ %s ] not found", code)));
     }
 
@@ -356,15 +354,9 @@ public class DataSetServiceImpl
             String tableName = String.format("%s%s", tablePrefix, UUID.randomUUID().toString().replace("-", ""));
             entity.setTableName(tableName);
         }
-        DataSetState state = entity.getState().get(entity.getState().size() - 1);
-        if (state.equals(DataSetState.METADATA_START)
-                || state.equals(DataSetState.METADATA_FAILED)
-                || state.equals(DataSetState.TABLE_FAILED)) {
         log.info("Start build metadata for dataset [ {} ] id [ {} ]", entity.getName(), entity.getId());
         createMetadata(entity, rebuildColumn);
     }
-        throw new IllegalArgumentException(String.format("Invalid state [ %s ]", state));
-    }
 
     private void createMetadata(DataSetEntity entity, boolean rebuildColumn)
     {
@@ -376,6 +368,13 @@ public class DataSetServiceImpl
                 entity.getColumns()
                         .forEach(item -> item.setDataset(DataSetEntity.builder().id(entity.getId()).build()));
                 columnRepository.saveAll(entity.getColumns());
+
+                List<DataSetColumnEntity> originalColumns = columnRepository.findAllByDataset(entity);
+                List<DataSetColumnEntity> columnsNotInEntity = originalColumns.stream()
+                        .filter(originalColumn -> entity.getColumns().stream()
+                                .noneMatch(addedColumn -> addedColumn.getId().equals(originalColumn.getId())))
+                        .collect(Collectors.toList());
+                columnRepository.deleteAll(columnsNotInEntity);
             }
             completeState(entity, DataSetState.METADATA_SUCCESS);
         }
@@ -746,7 +745,7 @@ public class DataSetServiceImpl
     private Set<CreatedModel> createdModeProcess(Set<DataSetColumnEntity> targetColumns, DataSetEntity entity)
     {
         Set<CreatedModel> models = Sets.newHashSet();
-
+        try {
             // If the id tag data is not set to new
             if (entity.getId() == null) {
                 models.add(new CreatedModel(null, CreatedMode.CREATE_TABLE));
@@ -759,7 +758,13 @@ public class DataSetServiceImpl
                 return models;
             }
 
-        List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity);
+            List<DataSetColumnEntity> sourceColumns = columnRepository.findAllByDataset(entity)
+                    .stream()
+                    .filter(item -> !item.isVirtualColumn())
+                    .collect(Collectors.toList());
+            targetColumns = targetColumns.stream()
+                    .filter(item -> !item.isVirtualColumn())
+                    .collect(Collectors.toSet());
             for (DataSetColumnEntity sourceColumn : sourceColumns) {
                 Optional<DataSetColumnEntity> filterColumn = targetColumns.stream()
                         .filter(item -> item.getId().equals(sourceColumn.getId()))
@@ -786,6 +791,10 @@ public class DataSetServiceImpl
                         .build();
                 models.add(new CreatedModel(column, CreatedMode.MODIFY_LIFECYCLE));
             }
+        }
+        catch (Exception e) {
+            log.warn("Get create model for dataset [ {} ] id [ {} ] failed", entity.getName(), entity.getId(), e);
+        }
         return models;
     }
 

core/datacap-web/src/views/admin/dataset/DatasetInfo.vue

@@ -53,6 +53,7 @@
                   <Col class="w100 center">{{ $t('dataset.columnIsSampling') }}</Col>
                   <Col class="w100 center">{{ $t('dataset.columnLength') }}</Col>
                   <Col class="w200">{{ $t('dataset.columnComment') }}</Col>
+                  <Col class="w100 center">{{ $t('common.action') }}</Col>
                 </Row>
               </FormItem>
               <template v-for="(item, index) in formState.columns"
@@ -93,30 +94,41 @@
                     </Col>
                     <Col class="w100 center">
                       <Input v-model="item.defaultValue"
-                             type="text">
+                             type="text"
+                             :disabled="item.virtualColumn">
                       </Input>
                     </Col>
                     <Col class="w100 center">
-                      <Switch v-model="item.nullable"/>
+                      <Switch v-model="item.nullable"
+                              :disabled="item.virtualColumn">
+                      </Switch>
                     </Col>
                     <Col class="w100 center">
-                      <Switch v-model="item.orderByKey"/>
+                      <Switch v-model="item.orderByKey"
+                              :disabled="item.virtualColumn">
+                      </Switch>
                     </Col>
                     <Col class="w100 center">
-                      <Switch v-model="item.partitionKey"/>
+                      <Switch v-model="item.partitionKey"
+                              :disabled="item.virtualColumn">
+                      </Switch>
                     </Col>
                     <Col class="w100 center">
-                      <Switch v-model="item.primaryKey"/>
+                      <Switch v-model="item.primaryKey"
+                              :disabled="item.virtualColumn">
+                      </Switch>
                     </Col>
                     <Col class="w100 center">
                       <Switch v-model="item.samplingKey"
+                              :disabled="item.virtualColumn"
                               @on-change="validatorSampling">
                       </Switch>
                     </Col>
                     <Col class="w100 center">
                       <InputNumber v-model="item.length"
                                    min="0"
-                                   max="65536">
+                                   max="65536"
+                                   :disabled="item.type === 'BOOLEAN' || item.type === 'DATETIME' || item.virtualColumn">
                       </InputNumber>
                     </Col>
                     <Col class="w200">
@@ -124,6 +136,23 @@
                              type="textarea">
                       </Input>
                     </Col>
+                    <Col class="w100 center">
+                      <Space>
+                        <Button type="error"
+                                size="small"
+                                shape="circle"
+                                icon="md-trash"
+                                :disabled="!item.customColumn"
+                                @click="handlerRemoveColumn(index)">
+                        </Button>
+                        <Button type="primary"
+                                size="small"
+                                shape="circle"
+                                icon="md-add"
+                                @click="handlerAddColumn(index)">
+                        </Button>
+                      </Space>
+                    </Col>
                   </Row>
                 </FormItem>
               </template>
@@ -358,7 +387,9 @@ export default defineComponent({
               partitionKey: false,
               primaryKey: false,
               samplingKey: false,
-              mode: 'DIMENSION'
+              mode: 'DIMENSION',
+              virtualColumn: false,
+              customColumn: false
             }
             this.formState.columns.push(column)
           })
@@ -377,6 +408,32 @@ export default defineComponent({
         })
         .finally(() => this.saving = false)
     },
+    handlerAddColumn(index: number)
+    {
+      this.formState.columns.splice(index + 1, 0, {
+        id: null,
+        name: null,
+        aliasName: null,
+        type: 'STRING',
+        comment: null,
+        defaultValue: null,
+        position: index + 1,
+        nullable: false,
+        length: 0,
+        original: null,
+        orderByKey: false,
+        partitionKey: false,
+        primaryKey: false,
+        samplingKey: false,
+        mode: 'DIMENSION',
+        virtualColumn: true,
+        customColumn: true
+      })
+    },
+    handlerRemoveColumn(index: number)
+    {
+      this.formState.columns.splice(index, 1)
+    },
     validatorSampling()
     {
       const samplingColumns = this.formState.columns