[Feature][UI Next] Add spark into task. (#8442)

This commit is contained in:
Amy0104 2022-02-19 14:54:18 +08:00 committed by GitHub
parent 8d4f191ed4
commit 32161602a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 592 additions and 62 deletions

View File

@ -612,7 +612,31 @@ const project = {
prop_tips: 'prop(required)',
prop_repeat: 'prop is repeat',
value_tips: 'value(optional)',
pre_tasks: 'Pre tasks'
pre_tasks: 'Pre tasks',
program_type: 'Program Type',
spark_version: 'Spark Version',
main_class: 'Main Class',
main_class_tips: 'Please enter main class',
main_package: 'Main Package',
main_package_tips: 'Please enter main package',
deploy_mode: 'Deploy Mode',
app_name: 'App Name',
app_name_tips: 'Please enter app name(optional)',
driver_cores: 'Driver Cores',
driver_cores_tips: 'Please enter Driver cores',
driver_memory: 'Driver Memory',
driver_memory_tips: 'Please enter Driver memory',
executor_number: 'Executor Number',
executor_number_tips: 'Please enter Executor number',
executor_memory: 'Executor Memory',
executor_memory_tips: 'Please enter Executor memory',
executor_cores: 'Executor Cores',
executor_cores_tips: 'Please enter Executor cores',
main_arguments: 'Main Arguments',
main_arguments_tips: 'Please enter main arguments',
option_parameters: 'Option Parameters',
option_parameters_tips: 'Please enter option parameters',
positive_integer_tips: 'should be a positive integer'
}
}

View File

@ -605,7 +605,31 @@ const project = {
prop_tips: 'prop(必填)',
prop_repeat: 'prop中有重复',
value_tips: 'value(选填)',
pre_tasks: '前置任务'
pre_tasks: '前置任务',
program_type: '程序类型',
spark_version: 'Spark版本',
main_class: '主函数的Class',
main_class_tips: '请填写主函数的Class',
main_package: '主程序包',
main_package_tips: '请选择主程序包',
deploy_mode: '部署方式',
app_name: '任务名称',
app_name_tips: '请输入任务名称(选填)',
driver_cores: 'Driver核心数',
driver_cores_tips: '请输入Driver核心数',
driver_memory: 'Driver内存数',
driver_memory_tips: '请输入Driver内存数',
executor_number: 'Executor数量',
executor_number_tips: '请输入Executor数量',
executor_memory: 'Executor内存数',
executor_memory_tips: '请输入Executor内存数',
executor_cores: 'Executor核心数',
executor_cores_tips: '请输入Executor核心数',
main_arguments: '主程序参数',
main_arguments_tips: '请输入主程序参数',
option_parameters: '选项参数',
option_parameters_tips: '请输入选项参数',
positive_integer_tips: '应为正整数'
}
}

View File

@ -21,6 +21,7 @@ interface FileReq {
interface ResourceTypeReq {
type: 'FILE' | 'UDF'
programType?: string
}
interface UdfTypeReq {

View File

@ -31,3 +31,4 @@ export { useProcessName } from './use-process-name'
export { useChildNode } from './use-child-node'
export { useShell } from './use-shell'
export { useSpark } from './use-spark'

View File

@ -28,6 +28,6 @@ export function useDelayTime(model: { [field: string]: any }): IJsonItem {
slots: {
suffix: () => t('project.node.minute')
},
value: model.delayExecutionTime ? model.delayExecutionTime : 0
value: model.delayTime || 0
}
}

View File

@ -28,8 +28,7 @@ export function useFailed(): IJsonItem[] {
span: 12,
slots: {
suffix: () => t('project.node.times')
},
value: 0
}
},
{
type: 'input-number',
@ -38,8 +37,7 @@ export function useFailed(): IJsonItem[] {
span: 12,
slots: {
suffix: () => t('project.node.minute')
},
value: 1
}
}
]
}

View File

@ -129,7 +129,7 @@ export function useShell(model: { [field: string]: any }): IJsonItem[] {
]
}
function removeUselessChildren(list: { children?: [] }[]) {
export function removeUselessChildren(list: { children?: [] }[]) {
if (!list.length) return
list.forEach((item) => {
if (!item.children) return

View File

@ -0,0 +1,359 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ref, onMounted, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { queryResourceByProgramType } from '@/service/modules/resources'
import { removeUselessChildren } from './use-shell'
import type { IJsonItem } from '../types'
export function useSpark(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
const mainClassSpan = computed(() =>
model.programType === 'PYTHON' ? 0 : 24
)
const mainJarOptions = ref([])
const resources: { [field: string]: any } = {}
const getResourceList = async (programType: string) => {
if (resources[programType] !== void 0) {
mainJarOptions.value = resources[programType]
return
}
try {
const res = await queryResourceByProgramType({
type: 'FILE',
programType
})
removeUselessChildren(res)
mainJarOptions.value = res || []
resources[programType] = res
} catch (err) {}
}
onMounted(() => {
getResourceList(model.programType)
})
return [
{
type: 'select',
field: 'programType',
span: 12,
name: t('project.node.program_type'),
options: PROGRAM_TYPES,
props: {
'on-update:value': (value: string) => {
model.mainJar = null
model.mainClass = ''
getResourceList(value)
}
},
value: model.programType
},
{
type: 'select',
field: 'sparkVersion',
span: 12,
name: t('project.node.spark_version'),
options: SPARK_VERSIONS,
value: model.sparkVersion
},
{
type: 'input',
field: 'mainClass',
span: mainClassSpan,
name: t('project.node.main_class'),
props: {
placeholder: t('project.node.main_class_tips')
},
validate: {
trigger: ['input', 'blur'],
required: model.programType !== 'PYTHON',
validator(validate: any, value: string) {
if (model.programType !== 'PYTHON' && !value) {
return new Error(t('project.node.main_class_tips'))
}
}
}
},
{
type: 'tree-select',
field: 'mainJar',
name: t('project.node.main_package'),
props: {
cascade: true,
showPath: true,
checkStrategy: 'child',
placeholder: t('project.node.main_package_tips'),
keyField: 'id',
labelField: 'fullName'
},
validate: {
trigger: ['input', 'blur'],
required: model.programType !== 'PYTHON',
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.main_package_tips'))
}
}
},
options: mainJarOptions
},
{
type: 'radio',
field: 'deployMode',
name: t('project.node.deploy_mode'),
options: DeployModes
},
{
type: 'input',
field: 'appName',
name: t('project.node.app_name'),
props: {
placeholder: t('project.node.app_name_tips')
}
},
{
type: 'input-number',
field: 'driverCores',
name: t('project.node.driver_cores'),
span: 12,
props: {
placeholder: t('project.node.driver_cores_tips'),
min: 1
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.driver_cores_tips'))
}
}
}
},
{
type: 'input',
field: 'driverMemory',
name: t('project.node.driver_memory'),
span: 12,
props: {
placeholder: t('project.node.driver_memory_tips')
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.driver_memory_tips'))
}
if (!Number.isInteger(parseInt(value))) {
return new Error(
t('project.node.driver_memory') +
t('project.node.positive_integer_tips')
)
}
}
},
value: model.driverMemory
},
{
type: 'input-number',
field: 'numExecutors',
name: t('project.node.executor_number'),
span: 12,
props: {
placeholder: t('project.node.executor_number_tips'),
min: 1
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.executor_number_tips'))
}
}
},
value: model.numExecutors
},
{
type: 'input',
field: 'executorMemory',
name: t('project.node.executor_memory'),
span: 12,
props: {
placeholder: t('project.node.executor_memory_tips')
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.executor_memory_tips'))
}
if (!Number.isInteger(parseInt(value))) {
return new Error(
t('project.node.executor_memory_tips') +
t('project.node.positive_integer_tips')
)
}
}
},
value: model.executorMemory
},
{
type: 'input-number',
field: 'executorCores',
name: t('project.node.executor_cores'),
span: 12,
props: {
placeholder: t('project.node.executor_cores_tips'),
min: 1
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.executor_cores_tips'))
}
}
},
value: model.executorCores
},
{
type: 'input',
field: 'mainArgs',
name: t('project.node.main_arguments'),
props: {
type: 'textarea',
placeholder: t('project.node.main_arguments_tips')
}
},
{
type: 'input',
field: 'others',
name: t('project.node.option_parameters'),
props: {
type: 'textarea',
placeholder: t('project.node.option_parameters_tips')
}
},
{
type: 'tree-select',
field: 'resourceList',
name: t('project.node.resources'),
options: mainJarOptions,
props: {
multiple: true,
checkable: true,
cascade: true,
showPath: true,
checkStrategy: 'child',
placeholder: t('project.node.resources_tips'),
keyField: 'id',
labelField: 'name'
}
},
{
type: 'custom-parameters',
field: 'localParams',
name: t('project.node.custom_parameters'),
children: [
{
type: 'input',
field: 'prop',
span: 10,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'input',
field: 'value',
span: 10,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
]
}
const PROGRAM_TYPES = [
{
label: 'JAVA',
value: 'JAVA'
},
{
label: 'SCALA',
value: 'SCALA'
},
{
label: 'PYTHON',
value: 'PYTHON'
}
]
const SPARK_VERSIONS = [
{
label: 'SPARK2',
value: 'SPARK2'
},
{
label: 'SPARK1',
value: 'SPARK1'
}
]
const DeployModes = [
{
label: 'cluster',
value: 'cluster'
},
{
label: 'client',
value: 'client'
},
{
label: 'local',
value: 'local'
}
]

View File

@ -16,49 +16,62 @@
*/
import { omit } from 'lodash'
import type { INodeData, ITaskData } from './types'
import type { INodeData, ITaskData, ITaskParams } from './types'
export function formatParams(data: INodeData): {
processDefinitionCode: string
upstreamCodes: string
taskDefinitionJsonObj: object
} {
const taskParams: ITaskParams = {}
if (data.taskType === 'SPARK') {
taskParams.programType = data.programType
taskParams.sparkVersion = data.sparkVersion
taskParams.mainClass = data.mainClass
taskParams.mainJar = data.mainJar?.length
? data.mainJar.map((id: number) => ({ id }))
: []
taskParams.deployMode = data.deployMode
taskParams.appName = data.appName
taskParams.driverCores = data.driverCores
taskParams.driverMemory = data.driverMemory
taskParams.numExecutors = data.numExecutors
taskParams.executorMemory = data.executorMemory
taskParams.executorCores = data.executorCores
taskParams.mainArgs = data.mainArgs
taskParams.others = data.others
}
const params = {
processDefinitionCode: data.processName ? String(data.processName) : '',
upstreamCodes: data?.preTasks?.join(','),
taskDefinitionJsonObj: {
...omit(data, [
'delayTime',
'environmentCode',
'failRetryTimes',
'failRetryInterval',
'taskGroupId',
'localParams',
'timeoutFlag',
'timeoutNotifyStrategy',
'resourceList',
'postTaskOptions',
'preTaskOptions',
'preTasks',
'processName'
]),
code: data.code,
delayTime: data.delayTime ? String(data.delayTime) : '0',
description: data.description,
environmentCode: data.environmentCode || -1,
failRetryTimes: data.failRetryTimes ? String(data.failRetryTimes) : '0',
failRetryInterval: data.failRetryTimes
? String(data.failRetryTimes)
failRetryInterval: data.failRetryInterval
? String(data.failRetryInterval)
: '0',
failRetryTimes: data.failRetryTimes ? String(data.failRetryTimes) : '0',
flag: data.flag,
name: data.name,
taskGroupId: data.taskGroupId || 0,
taskGroupPriority: data.taskGroupPriority,
taskParams: {
localParams: data.localParams,
rawScript: data.rawScript,
resourceList: data.resourceList?.length
? data.resourceList.map((id: number) => ({ id }))
: []
: [],
...taskParams
},
taskPriority: data.taskPriority,
taskType: data.taskType,
timeout: data.timeout,
timeoutFlag: data.timeoutFlag ? 'OPEN' : 'CLOSE',
timeoutNotifyStrategy: data.timeoutNotifyStrategy?.join('')
timeoutNotifyStrategy: data.timeoutNotifyStrategy?.join(''),
workerGroup: data.workerGroup
}
} as {
processDefinitionCode: string
@ -69,36 +82,24 @@ export function formatParams(data: INodeData): {
params.taskDefinitionJsonObj.timeout = 0
params.taskDefinitionJsonObj.timeoutNotifyStrategy = ''
}
return params
}
export function formatModel(data: ITaskData) {
const params = {
name: data.name,
taskType: data.taskType,
processName: data.processName,
flag: data.flag,
description: data.description,
taskPriority: data.taskPriority,
workerGroup: data.workerGroup,
...omit(data, [
'environmentCode',
'timeoutFlag',
'timeoutNotifyStrategy',
'taskParams'
]),
...omit(data.taskParams, ['resourceList', 'mainJar', 'localParams']),
environmentCode: data.environmentCode === -1 ? null : data.environmentCode,
taskGroupId: data.taskGroupId,
taskGroupPriority: data.taskGroupPriority,
failRetryTimes: data.failRetryTimes,
failRetryInterval: data.failRetryInterval,
delayTime: data.delayTime,
timeoutFlag: data.timeoutFlag === 'OPEN',
timeoutNotifyStrategy: [data.timeoutNotifyStrategy] || [],
resourceList: data.taskParams?.resourceList || [],
timeout: data.timeout,
rawScript: data.taskParams?.rawScript,
localParams: data.taskParams?.localParams || [],
id: data.id,
code: data.code
} as {
timeoutNotifyStrategy: string[]
resourceList: number[]
}
localParams: data.taskParams?.localParams || []
} as INodeData
if (data.timeoutNotifyStrategy === 'WARNFAILED') {
params.timeoutNotifyStrategy = ['WARN', 'FAILED']
}
@ -107,5 +108,10 @@ export function formatModel(data: ITaskData) {
(item: { id: number }) => item.id
)
}
if (data.taskParams?.mainJar) {
params.mainJar = data.taskParams.mainJar.map(
(item: { id: number }) => item.id
)
}
return params
}

View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { reactive } from 'vue'
import * as Fields from '../fields/index'
import type { IJsonItem, INodeData, ITaskData } from '../types'
export function useSpark({
projectCode,
from = 0,
readonly,
data
}: {
projectCode: number
from?: number
readonly?: boolean
data?: ITaskData
}) {
const model = reactive({
taskType: 'SPARK',
name: '',
flag: 'YES',
description: '',
timeoutFlag: false,
localParams: [],
environmentCode: null,
failRetryInterval: 1,
failRetryTimes: 0,
workerGroup: 'default',
delayTime: 0,
timeout: 30,
programType: 'SCALA',
sparkVersion: 'SPARK2',
deployMode: 'cluster',
driverCores: 1,
driverMemory: '512M',
numExecutors: 2,
executorMemory: '2G',
executorCores: 2
} as INodeData)
let extra: IJsonItem[] = []
if (from === 1) {
extra = [
Fields.useTaskType(model, readonly),
Fields.useProcessName({
model,
projectCode,
isCreate: !data?.id,
from,
processName: data?.processName,
code: data?.code
})
]
}
return {
json: [
Fields.useName(),
...extra,
Fields.useRunFlag(),
Fields.useDescription(),
Fields.useTaskPriority(),
Fields.useWorkerGroup(),
Fields.useEnvironmentName(model, !data?.id),
...Fields.useTaskGroup(model, projectCode),
...Fields.useFailed(),
Fields.useDelayTime(model),
...Fields.useTimeoutAlarm(model),
...Fields.useSpark(model),
Fields.usePreTasks(model)
] as IJsonItem[],
model
}
}

View File

@ -35,9 +35,33 @@ interface ILocalParam {
type: string
value?: string
}
interface ISourceItem {
id: number
}
interface ITaskParams {
resourceList?: ISourceItem[]
mainJar?: ISourceItem[]
localParams?: ILocalParam[]
rawScript?: string
programType?: string
sparkVersion?: string
mainClass?: string
deployMode?: string
appName?: string
driverCores?: number
driverMemory?: string
numExecutors?: number
executorMemory?: string
executorCores?: number
mainArgs?: string
others?: string
}
type ITaskType = TaskType
interface INodeData {
interface INodeData extends Omit<ITaskParams, 'resourceList' | 'mainJar'> {
id?: string
taskType?: ITaskType
processName?: number
@ -49,19 +73,18 @@ interface INodeData {
flag?: 'YES' | 'NO'
taskGroupId?: number
taskGroupPriority?: number
localParams?: ILocalParam[]
rawScript?: string
taskPriority?: string
timeout?: number
timeoutFlag?: boolean
timeoutNotifyStrategy?: string[]
workerGroup?: string
resourceList?: number[]
code?: number
name?: string
preTasks?: []
preTaskOptions?: []
postTaskOptions?: []
resourceList?: number[]
mainJar?: number[]
}
interface ITaskData
@ -73,11 +96,7 @@ interface ITaskData
taskPriority?: string
timeoutFlag: 'OPEN' | 'CLOSE'
timeoutNotifyStrategy?: string | []
taskParams?: {
resourceList: []
rawScript: string
localParams: ILocalParam[]
}
taskParams?: ITaskParams
}
export {
@ -88,5 +107,6 @@ export {
ITaskData,
INodeData,
IFormItem,
IJsonItem
IJsonItem,
ITaskParams
}

View File

@ -18,6 +18,7 @@
import { useShell } from './tasks/use-shell'
import { useSubProcess } from './tasks/use-sub-process'
import { usePython } from './tasks/use-python'
import { useSpark } from './tasks/use-spark'
import { IJsonItem, INodeData, ITaskData } from './types'
export function useTask({
@ -57,5 +58,13 @@ export function useTask({
data
})
}
if (taskType === 'SPARK') {
node = useSpark({
projectCode,
from,
readonly,
data
})
}
return node
}

View File

@ -77,7 +77,6 @@ const TaskDefinition = defineComponent({
requestData()
}
const onCreate = () => {
task.taskReadonly = false
onToggleShow(true)
}
const onTaskCancel = () => {