test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-11-29 10:37:57 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(plugin): support set parent class package

Browse Source
This commit is contained in:
qianmoQ 2024-11-27 14:30:33 +08:00
parent 8625ba7b9a
commit 524178fb01
23 changed files with 4835 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
configure/etc/conf/plugins.properties
View File

@ -5,4 +5,5 @@ datacap-convert-json=convert/datacap-convert-json/pom.xml
datacap-convert-xml=convert/datacap-convert-xml/pom.xml
datacap-convert-csv=convert/datacap-convert-csv/pom.xml
datacap-convert-none=convert/datacap-convert-none/pom.xml
datacap-executor-local=executor/datacap-executor-local/pom.xml
datacap-executor-local=executor/datacap-executor-local/pom.xml
datacap-executor-seatunnel=executor/datacap-executor-seatunnel/pom.xml

15
configure/metadata.json
View File

@ -124,6 +124,21 @@
"ALL"
],
"url": "https://cdn.north.devlive.org/applications/datacap/plugins/2024.4.0-SNAPSHOT/executor/datacap-executor-local-bin.tar.gz"
},
{
"key": "datacap-executor-seatunnel",
"label": "SeatunnelExecutor",
"description": "A Seatunnel execution plugin for DataCap.",
"i18nFormat": true,
"type": "Executor",
"version": "2024.4.0-SNAPSHOT",
"author": "datacap-community",
"logo": "https://cdn.north.devlive.org/applications/datacap/resources/logo/executor/seatunnel.svg",
"released": "2024-11-21 23:11:15",
"supportVersion": [
"ALL"
],
"url": "https://cdn.north.devlive.org/applications/datacap/plugins/2024.4.0-SNAPSHOT/executor/datacap-executor-seatunnel-bin.tar.gz"
}
]
}

31
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/PluginConfigure.java
View File

@ -6,11 +6,22 @@ import lombok.Setter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
@Getter
@Builder
public class PluginConfigure
{
private static final Set<String> LOADER_PACKAGES = Set.of(
"java.",
"javax.",
"com.google.",
"org.",
"ch.qos.logback.",
"org.slf4j."
);
@Setter
private Path pluginsDir;
private boolean autoReload;
@ -23,11 +34,27 @@ public class PluginConfigure
// 自动清理, 只有卸载时才会生效
// Auto cleanup, only effective when unloading
public boolean autoCleanup;
private boolean autoCleanup;
// 同一目录下多个插件是否共享类加载器
// Whether multiple plugins in the same directory share the class loader
public boolean shareClassLoaderWhenSameDir;
private boolean shareClassLoaderWhenSameDir;
// 优先使用父类加载器包列表
// List of parent class loader packages to be prioritized
@Builder.Default
private Set<String> parentClassLoaderPackages = new HashSet<>(LOADER_PACKAGES);
/**
* 添加父类加载器包
* Add parent class loader packages
*
* @param packageNames 父类加载器包名称 Parent class loader package name
*/
public void addParentClassLoaderPackage(Set<String> packageNames)
{
this.parentClassLoaderPackages.addAll(packageNames);
}
public static PluginConfigure defaultConfig()
{

14
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/PluginManager.java
View File

@ -471,7 +471,7 @@ public class PluginManager
TarPluginLoader tarPluginLoader = new TarPluginLoader();
// 先加载插件到临时目录
// Load plugins to temporary directory
List<Plugin> plugins = tarPluginLoader.load(sourcePath, tempDir);
List<Plugin> plugins = tarPluginLoader.load(sourcePath, tempDir, config.getParentClassLoaderPackages());
if (!plugins.isEmpty()) {
// 查找解压后的子目录
// Find extracted subdirectory
@ -574,7 +574,7 @@ public class PluginManager
log.debug("Found plugin version: {}", pluginVersion);
PluginClassLoader loader;
if (config.shareClassLoaderWhenSameDir) {
if (config.isShareClassLoaderWhenSameDir()) {
log.info("Use shared ClassLoader for plugin: {} at {}", pluginBaseName, pluginDir);
// 多个插件在同一目录下，使用同一个类加载器
// Multiple plugins in the same directory, use the same class loader
@ -586,7 +586,8 @@ public class PluginManager
pluginDir,
pluginBaseName,
pluginVersion,
true
true,
config.getParentClassLoaderPackages()
);
}
catch (Exception e) {
@ -604,7 +605,8 @@ public class PluginManager
pluginDir,
pluginBaseName,
pluginVersion,
true
true,
config.getParentClassLoaderPackages()
);
}
@ -613,7 +615,7 @@ public class PluginManager
return;
}
List<Plugin> modules = PluginContextManager.runWithClassLoader(loader, () -> PluginLoaderFactory.loadPlugins(pluginDir));
List<Plugin> modules = PluginContextManager.runWithClassLoader(loader, () -> PluginLoaderFactory.loadPlugins(pluginDir, config.getParentClassLoaderPackages()));
for (Plugin module : modules) {
PluginContextManager.runWithClassLoader(loader, () -> {
@ -628,7 +630,7 @@ public class PluginManager
String pluginName = module.getName();
// 保存类加载器信息
// Save class loader information
if (config.shareClassLoaderWhenSameDir) {
if (config.isShareClassLoaderWhenSameDir()) {
pluginClassLoaders.putIfAbsent(pluginName, loader);
}
else {

3
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/CompiledPomPluginLoader.java
View File

@ -12,6 +12,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
@Slf4j
@ -25,7 +26,7 @@ public class CompiledPomPluginLoader
}
@Override
public List<Plugin> load(Path path)
public List<Plugin> load(Path path, Set<String> parentClassLoaderPackages)
{
try {
// 处理传入的是pom.xml文件的情况

6
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/DirectoryPluginLoader.java
View File

@ -11,6 +11,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -26,7 +27,7 @@ public class DirectoryPluginLoader
}
@Override
public List<Plugin> load(Path path)
public List<Plugin> load(Path path, Set<String> parentClassLoaderPackages)
{
try {
// 获取目录名作为插件名
@ -41,7 +42,8 @@ public class DirectoryPluginLoader
path,
pluginName,
version,
true
true,
parentClassLoaderPackages
);
try (Stream<Path> pathStream = Files.walk(path)) {

6
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/InjectPluginLoader.java
View File

@ -11,6 +11,7 @@ import lombok.extern.slf4j.Slf4j;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
@Slf4j
@SuppressFBWarnings(value = {"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"})
@ -50,7 +51,7 @@ public class InjectPluginLoader
}
@Override
public List<Plugin> load(Path path)
public List<Plugin> load(Path path, Set<String> parentClassLoaderPackages)
{
try {
if (isExcludedPath(path)) {
@ -76,7 +77,8 @@ public class InjectPluginLoader
path,
pluginName,
version,
true
true,
parentClassLoaderPackages
);
return PluginContextManager.runWithClassLoader(classLoader, () -> {

13
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/PluginClassLoader.java
View File

@ -7,6 +7,7 @@ import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.Set;
/**
* 插件专用类加载器
@ -27,13 +28,15 @@ public class PluginClassLoader
private final String pluginVersion;
private final boolean parentFirst;
private final Set<String> parentClassLoaders;
public PluginClassLoader(URL[] urls, ClassLoader parent, String pluginName, String pluginVersion, boolean parentFirst)
public PluginClassLoader(URL[] urls, ClassLoader parent, String pluginName, String pluginVersion, boolean parentFirst, Set<String> parentClassLoaders)
{
super(urls, parent);
this.pluginName = pluginName;
this.pluginVersion = pluginVersion;
this.parentFirst = parentFirst;
this.parentClassLoaders = parentClassLoaders;
this.name = String.join("-", "loader", pluginName.toLowerCase(), pluginVersion.toLowerCase());
log.debug("Created PluginClassLoader for {} with URLs: {}", pluginName, Arrays.toString(urls));
}
@ -57,13 +60,11 @@ public class PluginClassLoader
try {
// 系统类和框架类使用父加载器
// Use parent loader for system classes and framework classes
if (name.startsWith("java.") ||
name.startsWith("javax.") ||
name.startsWith("com.google.") ||
name.startsWith("org.") ||
if (parentClassLoaders.stream().anyMatch(name::startsWith)
// 添加 Plugin 相关的包到父优先加载列表
// Add Plugin related packages to parent-first list
(parentFirst && name.startsWith("io.edurt.datacap.plugin"))) {
|| (parentFirst && name.startsWith("io.edurt.datacap.plugin"))
) {
return super.loadClass(name, resolve);
}

2
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/PluginLoader.java
View File

@ -85,5 +85,5 @@ public interface PluginLoader
// 加载插件
// Load plugins
List<Plugin> load(Path path);
List<Plugin> load(Path path, Set<String> parentClassLoaderPackages);
}

7
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/PluginLoaderFactory.java
View File

@ -7,6 +7,7 @@ import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@Slf4j
@ -88,10 +89,12 @@ public class PluginLoaderFactory
*
* @param pluginDir 插件目录
* plugin directory
* @param parentClassLoaderPackages 父类加载器包名集合
* parent class loader package names
* @return 加载的插件模块列表
* list of loaded plugin modules
*/
public static List<Plugin> loadPlugins(Path pluginDir)
public static List<Plugin> loadPlugins(Path pluginDir, Set<String> parentClassLoaderPackages)
{
if (pluginDir == null) {
log.warn("Plugin directory is null");
@ -105,7 +108,7 @@ public class PluginLoaderFactory
PluginLoader loader = entry.getValue();
try {
List<Plugin> modules = loader.load(pluginDir);
List<Plugin> modules = loader.load(pluginDir, parentClassLoaderPackages);
if (modules != null && !modules.isEmpty()) {
log.info("Successfully loaded [ {} ] plugin(s) using loader type [ {} ]", modules.size(), type);
modules.forEach(v -> v.setClassLoader(loader.getClass().getName()));

6
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/PomPluginLoader.java
View File

@ -13,6 +13,7 @@ import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
@Slf4j
@SuppressFBWarnings(value = {"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE", "OBL_UNSATISFIED_OBLIGATION"})
@ -26,7 +27,7 @@ public class PomPluginLoader
}
@Override
public List<Plugin> load(Path path)
public List<Plugin> load(Path path, Set<String> parentClassLoaderPackages)
{
try {
Path pomFile = path.resolve("pom.xml");
@ -57,7 +58,8 @@ public class PomPluginLoader
path,
model.getArtifactId(),
version,
true
true,
parentClassLoaderPackages
);
Class<?> pluginClass = classLoader.loadClass(mainClass);

7
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/PropertiesPluginLoader.java
View File

@ -12,6 +12,7 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@Slf4j
@SuppressFBWarnings(value = {"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE", "OBL_UNSATISFIED_OBLIGATION"})
@ -25,7 +26,7 @@ public class PropertiesPluginLoader
}
@Override
public List<Plugin> load(Path path)
public List<Plugin> load(Path path, Set<String> parentClassLoaderPackages)
{
List<Plugin> plugins = new ArrayList<>();
try {
@ -69,12 +70,12 @@ public class PropertiesPluginLoader
// 使用 SpiPluginLoader 加载
// Use SpiPluginLoader to load
SpiPluginLoader compiledLoader = new SpiPluginLoader();
List<Plugin> loadedPlugins = compiledLoader.load(resolvedPath);
List<Plugin> loadedPlugins = compiledLoader.load(resolvedPath, parentClassLoaderPackages);
if (!loadedPlugins.isEmpty()) {
plugins.addAll(loadedPlugins);
}
else {
loadedPlugins = PluginLoaderFactory.loadPlugins(resolvedPath);
loadedPlugins = PluginLoaderFactory.loadPlugins(resolvedPath, parentClassLoaderPackages);
plugins.addAll(loadedPlugins);
}
}

6
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/SpiPluginLoader.java
View File

@ -12,6 +12,7 @@ import java.io.File;
import java.nio.file.Path;
import java.util.List;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@ -27,7 +28,7 @@ public class SpiPluginLoader
}
@Override
public List<Plugin> load(Path path)
public List<Plugin> load(Path path, Set<String> parentClassLoaderPackages)
{
try {
// 检查路径是否有效
@ -48,7 +49,8 @@ public class SpiPluginLoader
path,
pluginName,
version,
true
true,
parentClassLoaderPackages
);
return PluginContextManager.runWithClassLoader(classLoader, () -> {

14
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/loader/TarPluginLoader.java
View File

@ -34,6 +34,7 @@ import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -73,9 +74,9 @@ public class TarPluginLoader
* List of loaded plugins
*/
@Override
public List<Plugin> load(Path path)
public List<Plugin> load(Path path, Set<String> parentClassLoaderPackages)
{
return load(path, null);
return load(path, null, parentClassLoaderPackages);
}
/**
@ -89,7 +90,7 @@ public class TarPluginLoader
* @return 加载的插件列表
* List of loaded plugins
*/
public List<Plugin> load(Path path, Path targetDir)
public List<Plugin> load(Path path, Path targetDir, Set<String> parentClassLoaderPackages)
{
try {
// 如果是 URL 路径，先下载到本地
@ -108,7 +109,7 @@ public class TarPluginLoader
// 从解压目录加载插件
// Load plugins from extracted directory
List<Plugin> plugins = loadPluginsFromDirectory(extractDir);
List<Plugin> plugins = loadPluginsFromDirectory(extractDir, parentClassLoaderPackages);
// 如果使用的是临时目录，则清理
// Clean up if using temporary directory
@ -133,7 +134,7 @@ public class TarPluginLoader
* @return 加载的插件列表
* List of loaded plugins
*/
private List<Plugin> loadPluginsFromDirectory(Path directory)
private List<Plugin> loadPluginsFromDirectory(Path directory, Set<String> parentClassLoaderPackages)
throws Exception
{
List<Plugin> allPlugins = new ArrayList<>();
@ -156,7 +157,8 @@ public class TarPluginLoader
pluginDir,
pluginName,
version,
true
true,
parentClassLoaderPackages
);
// 在插件类加载器上下文中加载插件

12
core/datacap-plugin/src/main/java/io/edurt/datacap/plugin/utils/PluginClassLoaderUtils.java
View File

@ -8,6 +8,7 @@ import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.stream.Stream;
/**
@ -20,12 +21,6 @@ public class PluginClassLoaderUtils
{
private PluginClassLoaderUtils() {}
public static PluginClassLoader createClassLoader(Path directory, String pluginName, String pluginVersion)
throws Exception
{
return createClassLoader(directory, pluginName, pluginVersion, false);
}
/**
* 创建一个新的插件类加载器
* Create a new plugin class loader
@ -43,7 +38,7 @@ public class PluginClassLoaderUtils
* @throws Exception 创建类加载器时发生异常
* Exception occurred when creating the class loader
*/
public static PluginClassLoader createClassLoader(Path directory, String pluginName, String pluginVersion, boolean parentFirst)
public static PluginClassLoader createClassLoader(Path directory, String pluginName, String pluginVersion, boolean parentFirst, Set<String> parentClassLoaders)
throws Exception
{
log.debug("Creating new class loader for plugin: {} version: {} directory: {}",
@ -76,7 +71,8 @@ public class PluginClassLoaderUtils
systemClassLoader,
pluginName,
pluginVersion,
parentFirst
parentFirst,
parentClassLoaders
);
}

2
core/datacap-service/pom.xml
View File

@ -14,7 +14,7 @@
<description>DataCap - server</description>
<properties>
<snakeyaml.version>2.3</snakeyaml.version>
<snakeyaml.version>1.33</snakeyaml.version>
<useragent.version>1.21</useragent.version>
<openai.version>2024.01.3</openai.version>
</properties>

34
core/datacap-service/src/main/java/io/edurt/datacap/service/common/ConfigureUtils.java
View File

@ -18,6 +18,7 @@ import io.edurt.datacap.service.configure.IConfigureFieldType;
import io.edurt.datacap.service.configure.IConfigurePipelineType;
import io.edurt.datacap.service.entity.SourceEntity;
import io.edurt.datacap.spi.model.Configure;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.env.Environment;
@ -34,6 +35,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@Slf4j
@SuppressFBWarnings(value = {"SF_SWITCH_NO_DEFAULT", "SF_SWITCH_FALLTHROUGH"})
public class ConfigureUtils
{
@ -281,18 +283,28 @@ public class ConfigureUtils
public static PipelineFieldBody convertFieldBody(SourceEntity entity, String executor, IConfigurePipelineType pipelineType, Environment environment, Properties originProperties)
{
PipelineFieldBody body = new PipelineFieldBody();
body.setProtocol(entity.getProtocol());
IConfigure yamlConfigure = PluginUtils.loadYamlConfigure(entity.getProtocol(), entity.getType(), entity.getType(), environment);
yamlConfigure.getPipelines()
.stream()
.filter(v -> v.getExecutor().equals(executor) && v.getType().equals(pipelineType))
.findFirst()
.ifPresent(iConfigureExecutor -> body.setConfigures(mergeProperties(entity, iConfigureExecutor.getFields(), originProperties)));
if (body.getConfigures() == null) {
body.setConfigures(new Properties());
try {
PipelineFieldBody body = new PipelineFieldBody();
body.setProtocol(entity.getProtocol());
IConfigure yamlConfigure = PluginUtils.loadYamlConfigure(entity.getProtocol(), entity.getType(), entity.getType(), environment);
if (yamlConfigure == null) {
throw new IllegalArgumentException("Failed to load YAML configuration");
}
yamlConfigure.getPipelines()
.stream()
.filter(v -> v.getExecutor().equals(executor) && v.getType().equals(pipelineType))
.findFirst()
.ifPresent(iConfigureExecutor -> body.setConfigures(mergeProperties(entity, iConfigureExecutor.getFields(), originProperties)));
if (body.getConfigures() == null) {
body.setConfigures(new Properties());
}
return body;
}
catch (Throwable e) {
log.error("Failed to convert field body: {}", e.getMessage(), e);
throw new IllegalArgumentException("Failed to convert field body: " + e.getMessage(), e);
}
return body;
}
/**

5
core/datacap-service/src/main/java/io/edurt/datacap/service/common/PluginUtils.java
View File

@ -83,12 +83,13 @@ public class PluginUtils
else {
log.info("Load plugin {} type {} resource {} configure file path {}", plugin, type, resource, path);
}
yamlFactory.findAndRegisterModules();
IConfigure configure = null;
try {
yamlFactory.findAndRegisterModules();
configure = yamlFactory.readValue(file, IConfigure.class);
}
catch (Exception e) {
catch (Throwable e) {
log.error("Format configuration file, it may be a bug, please submit issues to solve it. plugin {} type {} resource {} configure file path {} message ", plugin, type, resource, path, e);
Preconditions.checkArgument(StringUtils.isNotEmpty(path), "Format configuration file, it may be a bug, please submit issues to solve it.");
}

18
executor/datacap-executor-seatunnel/pom.xml
View File

@ -49,6 +49,24 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
<excludeScope>provided</excludeScope>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

4
executor/datacap-executor-spi/pom.xml
View File

@ -12,10 +12,6 @@
<description>DataCap - Executor - Spi</description>
<dependencies>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>

2
lib/datacap-logger/pom.xml
View File

@ -18,10 +18,12 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>

4686
logo/executor/seatunnel.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 290 KiB

2
pom.xml
View File

@ -182,7 +182,7 @@
<jjwt.version>0.9.1</jjwt.version>
<commons-lang3.version>3.12.0</commons-lang3.version>
<findbugs.version>3.0.1</findbugs.version>
<jackson.version>2.17.2</jackson.version>
<jackson.version>2.13.4</jackson.version>
<logback.version>1.2.13</logback.version>
<slf4j.version>2.0.10</slf4j.version>
<sql-formatter.version>2.0.5</sql-formatter.version>