[Feature][Resource Center] One-time migration script for unmanaged resources (#13512)

docs/docs/en/guide/upgrade/upgrade.md

@@ -38,6 +38,22 @@ jar package and add it to the `./tools/libs` directory, then export the followin
 
 Execute database upgrade script: `sh ./tools/bin/upgrade-schema.sh`
 
+### Migrate Resource
+
+After refactoring resource center in version 3.2.0, original resources become unmanaged. You can assign a target tenant and execute one-time migration script. All resources will be migrated to directory `.migrate` of target tenant.
+
+#### Example
+
+Assign an existed target tenant `abc`, the base resource path is `/dolphinscheduler/abc/`.
+
+Execute script: `sh ./tools/bin/migrate-resource.sh abc`.
+
+Execution result:
+
+- The original file resource `a/b.sh` migrates to `/dolphinscheduler/abc/resources/.migrate/a/b.sh`.
+- The original UDF resource `x/y.jar` migrates to `/dolphinscheduler/abc/udf/.migrate/x/y.jar`.
+- Update UDF function's bound resource info.
+
 ### Upgrade Service
 
 #### Change Configuration `bin/env/install_env.sh`

docs/docs/zh/guide/upgrade/upgrade.md

@@ -37,6 +37,22 @@ jar 包 并添加到 `./tools/libs` 目录下，设置以下环境变量
 
 执行数据库升级脚本：`sh ./tools/bin/upgrade-schema.sh`
 
+### 资源迁移
+
+3.2.0 版本资源中心重构，原资源中心内的资源将不受管理，您可以指定迁移到的目标租户，然后运行一次性资源迁移脚本，所有资源会迁移到目标租户的 .migrate 目录下。
+
+#### 示例：
+
+指定已存在目标租户 `abc`，其资源根目录为 `/dolphinscheduler/abc/`。
+
+执行脚本：`sh ./tools/bin/migrate-resource.sh abc`。
+
+执行结果：
+
+- 原文件资源 `a/b.sh` 迁移至 `/dolphinscheduler/abc/resources/.migrate/a/b.sh`。
+- 原 UDF 资源 `x/y.jar` 迁移至 `/dolphinscheduler/abc/udf/.migrate/x/y.jar`。
+- 更新 UDF 函数绑定资源信息。
+
 ### 服务升级
 
 #### 修改 `bin/env/install_env.sh` 配置内容

dolphinscheduler-tools/pom.xml

@@ -46,6 +46,10 @@
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-dao</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-storage-all</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-aop</artifactId>

dolphinscheduler-tools/src/main/bin/migrate-resource.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)}
+
+if [ "$DOCKER" != "true" ]; then
+  source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh"
+fi
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+$JAVA_HOME/bin/java $JAVA_OPTS \
+  -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \
+  -Dspring.profiles.active=resource,${DATABASE} \
+  org.apache.dolphinscheduler.tools.resource.MigrateResource $1

dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.tools.resource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Component;
+
+@SpringBootApplication
+@ComponentScan("org.apache.dolphinscheduler")
+public class MigrateResource {
+
+    public static void main(String[] args) {
+        SpringApplication.run(MigrateResource.class, args);
+    }
+
+    @Component
+    @Profile("resource")
+    static class MigrateResourceRunner implements CommandLineRunner {
+
+        private static final Logger logger = LoggerFactory.getLogger(MigrateResourceRunner.class);
+
+        @Autowired
+        private MigrateResourceService migrateResourceService;
+
+        @Override
+        public void run(String... args) {
+            String targetTenantCode = args[0];
+            logger.info("Moving all unmanaged resources to tenant: {}", targetTenantCode);
+            migrateResourceService.migrateResourceOnce(targetTenantCode);
+        }
+    }
+
+}

dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.tools.resource;
+
+import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
+
+import org.apache.dolphinscheduler.dao.entity.Resource;
+import org.apache.dolphinscheduler.dao.entity.UdfFunc;
+import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
+import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import org.apache.zookeeper.common.StringUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class MigrateResourceService {
+
+    private static final Logger logger = LoggerFactory.getLogger(MigrateResourceService.class);
+
+    @Autowired
+    private StorageOperate storageOperate;
+
+    @Autowired
+    private TenantMapper tenantMapper;
+
+    @Autowired
+    private ResourceMapper resourceMapper;
+
+    @Autowired
+    private UdfFuncMapper udfFuncMapper;
+
+    private static final String MIGRATE_BASE_DIR = ".migrate";
+
+    public void migrateResourceOnce(String targetTenantCode) {
+        if (true != tenantMapper.existTenant(targetTenantCode)) {
+            logger.error("Tenant not exists!");
+            return;
+        }
+
+        String resMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.FILE);
+        String udfMigrateBasePath = createMigrateDirByType(targetTenantCode, ResourceType.UDF);
+        if (StringUtils.isEmpty(resMigrateBasePath) || StringUtils.isEmpty(udfMigrateBasePath)) {
+            return;
+        }
+
+        // migrate all unmanaged resources and udfs once
+        List<Resource> resources = resourceMapper.queryResourceExceptUserId(-1);
+        resources.forEach(resource -> {
+            try {
+                String oriFullName = resource.getFullName();
+                oriFullName = oriFullName.startsWith("/") ? oriFullName.substring(1) : oriFullName;
+                if (resource.getType().equals(ResourceType.FILE)) {
+                    storageOperate.copy(oriFullName,
+                            String.format(FORMAT_S_S, resMigrateBasePath, oriFullName), true, true);
+                } else if (resource.getType().equals(ResourceType.UDF)) {
+                    String fullName = String.format(FORMAT_S_S, udfMigrateBasePath, oriFullName);
+                    storageOperate.copy(oriFullName, fullName, true, true);
+
+                    // change relative udfs resourceName
+                    List<UdfFunc> udfs = udfFuncMapper.listUdfByResourceId(new Integer[]{resource.getId()});
+                    udfs.forEach(udf -> {
+                        udf.setResourceName(fullName);
+                        udfFuncMapper.updateById(udf);
+                    });
+                }
+            } catch (IOException e) {
+                logger.error("Migrate resource failed: {}", e.getMessage());
+            }
+        });
+    }
+
+    public String createMigrateDirByType(String targetTenantCode, ResourceType type) {
+        String migrateBasePath = type.equals(ResourceType.FILE) ? storageOperate.getResDir(targetTenantCode)
+                : storageOperate.getUdfDir(targetTenantCode);
+        migrateBasePath += MIGRATE_BASE_DIR;
+        try {
+            storageOperate.mkdir(targetTenantCode, migrateBasePath);
+        } catch (IOException e) {
+            logger.error("create migrate base directory {} failed", migrateBasePath);
+            return "";
+        }
+        return migrateBasePath;
+    }
+
+}