feat(plugin): support tar loader

This commit is contained in:
qianmoQ 2024-11-23 13:09:45 +08:00
parent c7cda15d42
commit 39465a6a67
16 changed files with 508 additions and 70 deletions

View File

@ -43,7 +43,7 @@
"type": "Converter",
"version": "2024.4.0-SNAPSHOT",
"author": "datacap-community",
"logo": "",
"logo": "https://cdn.north.devlive.org/applications/datacap/resources/logo/convert/csv.svg",
"released": "2024-11-21 23:11:15",
"supportVersion": [
"ALL"
@ -73,7 +73,7 @@
"type": "Converter",
"version": "2024.4.0-SNAPSHOT",
"author": "datacap-community",
"logo": "https://www.vectorlogo.zone/logos/json/json-icon.svg",
"logo": "https://cdn.north.devlive.org/applications/datacap/resources/logo/convert/json.svg",
"released": "2024-11-21 23:11:15",
"supportVersion": [
"ALL"
@ -88,7 +88,7 @@
"type": "Converter",
"version": "2024.4.0-SNAPSHOT",
"author": "datacap-community",
"logo": "https://www.vectorlogo.zone/logos/w3c_xml/w3c_xml-icon.svg",
"logo": "https://cdn.north.devlive.org/applications/datacap/resources/logo/convert/xml.svg",
"released": "2024-11-21 23:11:15",
"supportVersion": [
"ALL"
@ -103,7 +103,7 @@
"type": "Converter",
"version": "2024.4.0-SNAPSHOT",
"author": "datacap-community",
"logo": "",
"logo": "https://cdn.north.devlive.org/applications/datacap/resources/logo/convert/none.svg",
"released": "2024-11-21 23:11:15",
"supportVersion": [
"ALL"

View File

@ -32,7 +32,7 @@ for file in "$HOME/dist"/*bin.tar.gz; do
if [ -e "$file" ]; then
filename=$(basename "$file")
echo "Uploading binary: $filename"
qshell fput --overwrite "$BUCKET_NAME" "$DATACAP_HOME/$VERSION/$filename" "$file"
qshell fput --overwrite "$BUCKET_NAME" "$DATACAP_HOME/versions/$VERSION/$filename" "$file"
fi
done

View File

@ -13,6 +13,11 @@
<artifactId>datacap-plugin</artifactId>
<description>DataCap - Plugin Core</description>
<properties>
<common-compress.version>1.26.0</common-compress.version>
<httpclient.version>4.5.14</httpclient.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
@ -35,6 +40,16 @@
<artifactId>maven-model</artifactId>
<version>${datacap.maven.model.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>${common-compress.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpclient.version}</version>
</dependency>
</dependencies>
</project>

View File

@ -7,7 +7,8 @@ public enum SpiType
POM("Pom"),
PROPERTIES("Properties"),
SPI("Spi"),
INJECT("Inject");
INJECT("Inject"),
TAR("Tar");
private String name;

View File

@ -23,11 +23,14 @@ public class PluginClassLoader
@Getter
private final String pluginVersion;
public PluginClassLoader(URL[] urls, ClassLoader parent, String pluginName, String pluginVersion)
private final boolean parentFirst;
public PluginClassLoader(URL[] urls, ClassLoader parent, String pluginName, String pluginVersion, boolean parentFirst)
{
super(urls, parent);
this.pluginName = pluginName;
this.pluginVersion = pluginVersion;
this.parentFirst = parentFirst;
this.name = String.join("-", "loader", pluginName.toLowerCase(), pluginVersion.toLowerCase());
}
@ -53,7 +56,10 @@ public class PluginClassLoader
if (name.startsWith("java.") ||
name.startsWith("javax.") ||
name.startsWith("com.google.") ||
name.startsWith("org.")) {
name.startsWith("org.") ||
// 添加 Plugin 相关的包到父优先加载列表
// Add Plugin related packages to parent-first list
(parentFirst && name.startsWith("io.edurt.datacap.plugin"))) {
return super.loadClass(name, resolve);
}

View File

@ -24,6 +24,7 @@ public class PluginLoaderFactory
registerLoader(new DirectoryPluginLoader());
registerLoader(new PomPluginLoader());
registerLoader(new SpiPluginLoader());
registerLoader(new TarPluginLoader());
}
/**
@ -50,6 +51,18 @@ public class PluginLoaderFactory
}
}
/**
* 移除插件加载器
* Remove a plugin loader by type
*
* @param type 要移除的加载器类型
* the type of the loader to remove
*/
public static void unregisterLoader(String type)
{
loaderRegistry.remove(type);
}
/**
* 根据类型获取加载器
* Get loader by type

View File

@ -0,0 +1,308 @@
package io.edurt.datacap.plugin.loader;
import io.edurt.datacap.plugin.Plugin;
import io.edurt.datacap.plugin.PluginContextManager;
import io.edurt.datacap.plugin.SpiType;
import io.edurt.datacap.plugin.utils.PluginClassLoaderUtils;
import io.edurt.datacap.plugin.utils.VersionUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContextBuilder;
import javax.net.ssl.SSLContext;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.ServiceLoader;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import java.util.zip.GZIPInputStream;
/**
* Tar 格式插件加载器支持从本地文件系统或网络 URL 加载插件
* Tar format plugin loader, supports loading plugins from local filesystem or network URL
*/
@Slf4j
public class TarPluginLoader
implements PluginLoader
{
// 临时目录前缀
// Temporary directory prefix
private static final String TEMP_DIR_PREFIX = "plugin_";
/**
* 获取加载器类型
* Get loader type
*/
@Override
public SpiType getType()
{
return SpiType.TAR;
}
/**
* 加载插件支持本地路径和网络 URL
* Load plugins, supports local path and network URL
*
* @param path 插件路径或 URL
* Plugin path or URL
* @return 加载的插件列表
* List of loaded plugins
*/
@Override
public List<Plugin> load(Path path)
{
try {
// 如果是 URL 路径先下载到本地
// If it's a URL path, download it first
if (path.toString().startsWith("http") || path.toString().startsWith("https")) {
path = downloadTarFile(path.toString().replace(":/", "://"));
}
// 创建临时解压目录
// Create temporary directory for extraction
Path extractDir = createTempDirectory();
// 解压 tar 文件
// Extract tar file
extractTarFile(path, extractDir);
// 从解压目录加载插件
// Load plugins from extracted directory
List<Plugin> plugins = loadPluginsFromDirectory(extractDir);
// 清理临时目录
// Cleanup temporary directory
cleanupTempDirectory(extractDir);
return plugins;
}
catch (Exception e) {
log.error("Failed to load plugins from tar file: {}", path, e);
return List.of();
}
}
/**
* 从解压目录加载插件
* Load plugins from extracted directory
*
* @param directory 插件目录
* Plugin directory
* @return 加载的插件列表
* List of loaded plugins
*/
private List<Plugin> loadPluginsFromDirectory(Path directory)
throws Exception
{
List<Plugin> allPlugins = new ArrayList<>();
// 遍历目录查找插件
// Traverse directory to find plugins
try (Stream<Path> pathStream = Files.walk(directory)) {
pathStream.filter(Files::isDirectory)
.filter(this::isValidDirectory)
.forEach(pluginDir -> {
try {
// 获取插件名称和版本
// Get plugin name and version
String pluginName = pluginDir.getFileName().toString();
String version = VersionUtils.determinePluginVersion(pluginDir);
// 创建插件专用类加载器
// Create plugin-specific class loader
PluginClassLoader classLoader = PluginClassLoaderUtils.createClassLoader(
pluginDir,
pluginName,
version,
true
);
// 在插件类加载器上下文中加载插件
// Load plugins in plugin class loader context
List<Plugin> plugins = PluginContextManager.runWithClassLoader(classLoader, () -> {
ServiceLoader<Plugin> serviceLoader = ServiceLoader.load(Plugin.class, classLoader);
return StreamSupport.stream(serviceLoader.spliterator(), false)
.filter(plugin -> {
// 检查插件类是否来自排除目录
// Check if the plugin class is from an excluded directory
String className = plugin.getClass().getName();
Path classPath = Path.of(className.replace('.', File.separatorChar) + ".class");
return !isExcludedPath(classPath);
})
.peek(plugin -> {
// 设置插件的类加载器
// Set class loader for plugins
plugin.setPluginClassLoader(classLoader);
log.debug("Loaded TAR plugin: {} (version: {})", plugin.getClass().getName(), version);
})
.collect(Collectors.toList());
});
allPlugins.addAll(plugins);
}
catch (Exception e) {
log.error("Failed to load plugins from directory: {}", pluginDir, e);
}
});
}
return allPlugins;
}
/**
* URL 下载 tar 文件到临时文件
* Download tar file from URL to temporary file
*
* @param url tar 文件的 URL
* URL of the tar file
* @return 临时文件路径
* Path to temporary file
*/
private Path downloadTarFile(String url)
throws IOException
{
log.info("Downloading tar file from URL: {}", url);
Path tempFile = Files.createTempFile(TEMP_DIR_PREFIX, ".tar");
try {
// 创建信任所有证书的 SSL 上下文
// Create SSL context that trusts all certificates
SSLContext sslContext = SSLContextBuilder.create()
.loadTrustMaterial(null, (cert, authType) -> true)
.build();
// 创建允许所有主机名的验证器
// Create verifier that allows all hostnames
SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(
sslContext,
NoopHostnameVerifier.INSTANCE
);
// 创建 HttpClient
// Create HttpClient
try (CloseableHttpClient httpClient = HttpClients.custom()
.setSSLSocketFactory(sslFactory)
.build()) {
// 创建 GET 请求
// Create GET request
HttpGet request = new HttpGet(url);
request.setConfig(RequestConfig.custom()
.setConnectTimeout(30000)
.setSocketTimeout(30000)
.build());
// 执行请求并下载文件
// Execute request and download file
try (CloseableHttpResponse response = httpClient.execute(request);
InputStream in = response.getEntity().getContent();
OutputStream out = Files.newOutputStream(tempFile)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
throw new IOException("HTTP request failed with status: " + statusCode);
}
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
}
}
}
return tempFile;
}
catch (Exception e) {
try {
Files.deleteIfExists(tempFile);
}
catch (IOException deleteError) {
log.warn("Failed to delete temporary file after download failure: {}", tempFile, deleteError);
}
throw new IOException("Failed to download file from URL: " + url, e);
}
}
/**
* 创建临时目录
* Create temporary directory
*/
private Path createTempDirectory()
throws IOException
{
return Files.createTempDirectory(TEMP_DIR_PREFIX + UUID.randomUUID());
}
/**
* 解压 tar 文件到指定目录
* Extract tar file to specified directory
*
* @param tarFile tar 文件路径
* Path to tar file
* @param destDir 目标目录
* Destination directory
*/
private void extractTarFile(Path tarFile, Path destDir)
throws IOException
{
log.info("Extracting tar file to: {}", destDir);
try (InputStream fileIn = Files.newInputStream(tarFile);
BufferedInputStream buffIn = new BufferedInputStream(fileIn);
GZIPInputStream gzipIn = new GZIPInputStream(buffIn);
TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
TarArchiveEntry entry;
while ((entry = tarIn.getNextEntry()) != null) {
if (entry.isDirectory()) {
continue;
}
Path outPath = destDir.resolve(entry.getName());
Files.createDirectories(outPath.getParent());
try (OutputStream out = Files.newOutputStream(outPath)) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = tarIn.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
}
}
}
}
}
/**
* 清理临时目录
* Clean up temporary directory
*/
private void cleanupTempDirectory(Path directory)
{
try {
FileUtils.deleteDirectory(directory.toFile());
}
catch (IOException e) {
log.warn("Failed to cleanup temporary directory: {}", directory, e);
}
}
}

View File

@ -6,8 +6,8 @@ import lombok.extern.slf4j.Slf4j;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
import java.util.LinkedHashSet;
import java.util.stream.Stream;
/**
* 插件类加载器工具类
@ -16,35 +16,44 @@ import java.util.Set;
@Slf4j
public class PluginClassLoaderUtils
{
public static PluginClassLoader createClassLoader(Path directory, String pluginName, String pluginVersion)
throws Exception
{
return createClassLoader(directory, pluginName, pluginVersion, false);
}
/**
* 创建一个新的插件类加载器
* Create a new plugin class loader
*
* @param directory 插件目录
* @param directory Plugin directory
* Plugin directory
* @param pluginName 插件名称
* @param pluginName Plugin name
* Plugin name
* @param pluginVersion 插件版本
* @param pluginVersion Plugin version
* Plugin version
* @param parentFirst 是否先加载父类加载器
* Whether to load the parent class loader first
* @return 新创建的类加载器
* @return The newly created class loader
* The newly created class loader
* @throws Exception 创建类加载器时发生异常
* @throws Exception Exception occurred when creating the class loader
* Exception occurred when creating the class loader
*/
public static PluginClassLoader createClassLoader(Path directory, String pluginName, String pluginVersion)
public static PluginClassLoader createClassLoader(Path directory, String pluginName, String pluginVersion, boolean parentFirst)
throws Exception
{
log.debug("Creating new class loader for plugin: {} version: {} directory: {}",
pluginName, pluginVersion, directory);
Set<URL> urls = new HashSet<>();
LinkedHashSet<URL> urls = new LinkedHashSet<>();
if (Files.isDirectory(directory)) {
// 添加主插件JAR
// Add the main plugin JAR
Files.walk(directory)
.filter(path -> path.toString().endsWith(".jar"))
.forEach(path -> addJarAndDependencies(path, urls));
try (Stream<Path> pathStream = Files.walk(directory)) {
pathStream.filter(path -> path.toString().endsWith(".jar"))
.forEach(path -> addJarAndDependencies(path, urls));
}
// 检查常见的依赖目录
// Check common dependency directories
@ -62,7 +71,8 @@ public class PluginClassLoaderUtils
urls.toArray(new URL[0]),
systemClassLoader,
pluginName,
pluginVersion
pluginVersion,
parentFirst
);
}
@ -73,23 +83,24 @@ public class PluginClassLoaderUtils
* @param dir 依赖目录 Dependency directory
* @param urls URL集合 URL set
*/
private static void addDependenciesFromDir(Path dir, Set<URL> urls)
private static void addDependenciesFromDir(Path dir, LinkedHashSet<URL> urls)
{
if (Files.isDirectory(dir)) {
try {
log.debug("Scanning dependency directory: {}", dir);
Files.walk(dir)
.filter(path -> path.toString().endsWith(".jar"))
.forEach(path -> {
try {
urls.add(path.toUri().toURL());
log.debug("Added dependency: {}", path);
}
catch (Exception e) {
log.error("Failed to add dependency: {}", path, e);
throw new RuntimeException("Failed to add dependency: " + path, e);
}
});
try (Stream<Path> pathStream = Files.walk(dir)) {
pathStream.filter(path -> path.toString().endsWith(".jar"))
.forEach(path -> {
try {
urls.add(path.toUri().toURL());
log.debug("Added dependency: {}", path);
}
catch (Exception e) {
log.error("Failed to add dependency: {}", path, e);
throw new RuntimeException("Failed to add dependency: " + path, e);
}
});
}
}
catch (Exception e) {
log.error("Failed to scan dependency directory: {}", dir, e);
@ -105,7 +116,7 @@ public class PluginClassLoaderUtils
* @param jarPath JAR文件路径 JAR file path
* @param urls URL集合 URL set
*/
private static void addJarAndDependencies(Path jarPath, Set<URL> urls)
private static void addJarAndDependencies(Path jarPath, LinkedHashSet<URL> urls)
{
try {
urls.add(jarPath.toUri().toURL());

View File

@ -1,35 +0,0 @@
package io.edurt.datacap.plugin;
import io.edurt.datacap.plugin.utils.PluginPathUtils;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.nio.file.Path;
@Slf4j
public class PluginManagerTest
{
private PluginManager pluginManager;
@Before
public void before()
{
Path projectRoot = PluginPathUtils.findProjectRoot();
PluginConfigure config = PluginConfigure.builder()
.pluginsDir(projectRoot.resolve("test/datacap-test-plugin"))
.build();
pluginManager = new PluginManager(config);
pluginManager.start();
}
@Test
public void testLoadPlugin()
{
pluginManager.getPluginInfos().forEach(info -> log.info("{}", info));
Assert.assertFalse(pluginManager.getPlugin("Local").isEmpty());
}
}

10
logo/convert/csv.svg Normal file
View File

@ -0,0 +1,10 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<text x="50" y="60"
font-family="Arial, sans-serif"
font-size="30"
font-weight="bold"
text-anchor="middle"
fill="#000000">
CSV
</text>
</svg>

After

Width:  |  Height:  |  Size: 271 B

16
logo/convert/json.svg Normal file
View File

@ -0,0 +1,16 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64" viewBox="0 0 32 32">
<defs>
<linearGradient id="A" gradientUnits="userSpaceOnUse">
<stop offset="0" />
<stop offset="1" stop-color="#fff" />
</linearGradient>
<linearGradient x1="23.664" y1="23.664" x2="136.38" y2="136.38" id="B" xlink:href="#A" />
<linearGradient x1="136.38" y1="136.38" x2="23.664" y2="23.664" id="C" xlink:href="#A" />
</defs>
<g transform="translate(11.423 -391.912)">
<g transform="matrix(.2 0 0 .2 -11.422664 391.91163)" fill-rule="evenodd">
<path d="M79.865 119.1c35.398 48.255 70.04-13.47 69.99-50.587C149.793 24.627 105.312.1 79.836.1 38.943.1 0 33.895 0 80.135 0 131.53 44.64 160 79.836 160c-7.964-1.147-34.506-6.834-34.863-67.967-.24-41.347 13.488-57.866 34.805-50.6.477.177 23.514 9.265 23.514 38.95 0 29.56-23.427 38.715-23.427 38.715z" fill="url(#B)" />
<path d="M79.823 41.4C56.433 33.34 27.78 52.617 27.78 91.23c0 63.048 46.72 68.77 52.384 68.77C121.057 160 160 126.204 160 79.964 160 28.568 115.36.1 80.164.1c9.748-1.35 52.54 10.55 52.54 69.037 0 38.14-31.953 58.905-52.735 50.033-.477-.177-23.514-9.265-23.514-38.95 0-29.56 23.367-38.818 23.367-38.818z" fill="url(#C)" />
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

10
logo/convert/none.svg Normal file
View File

@ -0,0 +1,10 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<text x="50" y="60"
font-family="Arial, sans-serif"
font-size="30"
font-weight="bold"
text-anchor="middle"
fill="#000000">
None
</text>
</svg>

After

Width:  |  Height:  |  Size: 272 B

3
logo/convert/xml.svg Normal file
View File

@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64">
<path d="M7.208 31.416L.643 22.933H7.85l3.017 4.927 2.96-4.927h6.817l-6.398 8.533 7.124 9.6h-7.403l-3.408-5.41-3.52 5.41H0zm15.326-8.483h8.744l2.85 10.667h.056l2.85-10.667h8.744v18.134h-5.81V29.435h-.056l-3.464 11.632h-4.582L28.4 29.435h-.056v11.632h-5.81zm26.492 0h6.146V36.42H64v4.648H49.026z" />
</svg>

After

Width:  |  Height:  |  Size: 374 B

View File

@ -103,6 +103,7 @@
<module>convert/datacap-convert-xml</module>
<module>test/datacap-test-plugin</module>
<module>test/datacap-test-convert</module>
<module>test/datacap-test-core</module>
</modules>
<name>datacap</name>

View File

@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap</artifactId>
<version>2024.4.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>datacap-test-core</artifactId>
<description>DataCap - Test - Core</description>
<dependencies>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-plugin</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
<executions>
<execution>
<id>test-compile</id>
<goals>
<goal>test-compile</goal>
</goals>
<configuration>
<sourceDirs>
<sourceDir>${project.basedir}/src/test/kotlin</sourceDir>
</sourceDirs>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,21 @@
package io.edurt.datacap.test.plugin
import io.edurt.datacap.plugin.Plugin
import io.edurt.datacap.plugin.loader.TarPluginLoader
import org.junit.Test
import java.nio.file.Path
import kotlin.test.assertTrue
class TarPluginLoaderTest
{
private val tarUrl = "https://cdn.north.devlive.org/applications/datacap/plugins/2024.4.0-SNAPSHOT/convert/datacap-convert-txt-bin.tar.gz"
@Test
fun test()
{
val loader = TarPluginLoader()
val plugins: List<Plugin> = loader.load(Path.of(tarUrl))
assertTrue(plugins.isNotEmpty())
}
}