test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-12-02 20:17:45 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

[Core] [Fs] Mark fs storage experimental feature as official feature (#756)

Browse Source
This commit is contained in:
qianmoQ 2024-05-09 07:21:44 +08:00 committed by GitHub
commit 9c8a6cfe25
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 454 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
configure/etc/conf/application.properties
View File

@ -118,6 +118,29 @@ datacap.dataset.tableDefaultEngine=MergeTree
################################# Plugin configure #################################
datacap.parser.sql.defaultEngine=Trino
################################# File System configure #################################
# For the file storage type, please refer to datacap-fs-<Type>, which defaults to Local
# ------ Local File System ------ #
datacap.fs.type=Local
datacap.fs.access=
datacap.fs.secret=
datacap.fs.endpoint=
datacap.fs.bucket=
# ------ Qiniu File System ------#
#datacap.fs.type=Qiniu
#datacap.fs.access=
#datacap.fs.secret=
#datacap.fs.endpoint=
#datacap.fs.bucket=
# ------ Ali yun OSS File System ------#
#datacap.fs.type=AliOss
#datacap.fs.access=
#datacap.fs.secret=
#datacap.fs.endpoint=
#datacap.fs.bucket=
################################# Experimental features #################################
# This configuration is used to dynamically increase the total number of rows of returned data in SQL during query, and currently only takes effect for user-directed queries
# If the total number of rows returned is included in the SQL, it will not be automatically incremented
@ -127,9 +150,3 @@ datacap.experimental.data={user.dir}/data
# The path to upload the user's avatar
# `{username}` Fixed format, currently not supported to modify, this configuration will automatically get the current user name by default
datacap.experimental.avatarPath={username}/avatar/
# For the file storage type, please refer to datacap-fs-<Type>, which defaults to Local
datacap.experimental.fs.type=Local
datacap.experimental.fs.access=
datacap.experimental.fs.secret=
datacap.experimental.fs.endpoint=
datacap.experimental.fs.bucket=

5
core/datacap-server/pom.xml
View File

@ -399,6 +399,11 @@
<artifactId>datacap-fs-qiniu</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-fs-alioss</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>

6
core/datacap-service/src/main/java/io/edurt/datacap/service/audit/AuditPluginHandler.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Injector;
import io.edurt.datacap.common.enums.State;
import io.edurt.datacap.common.response.CommonResponse;
import io.edurt.datacap.common.utils.CSVUtils;
import io.edurt.datacap.common.utils.DateUtils;
import io.edurt.datacap.common.utils.SpiUtils;
import io.edurt.datacap.fs.FsRequest;
import io.edurt.datacap.service.common.FolderUtils;
@ -109,6 +110,11 @@ public class AuditPluginHandler
.stream(Files.newInputStream(tempFile.toPath()))
.fileName("result.csv")
.build();
// If it is OSS third-party storage, rebuild the default directory
if (!initializer.getFsConfigure().getType().equals("Local")) {
fsRequest.setEndpoint(initializer.getFsConfigure().getEndpoint());
fsRequest.setFileName(String.join(File.separator, user.getUsername(), DateUtils.formatYMD(), String.join(File.separator, "adhoc", uniqueId), "result.csv"));
}
SpiUtils.findFs(injector, initializer.getFsConfigure().getType())
.map(v -> v.writer(fsRequest));
log.info("Delete temp file [ {} ] on [ {} ] statue [ {} ]", tempFile, pluginAudit.getId(), tempFile.delete());

2
core/datacap-service/src/main/java/io/edurt/datacap/service/initializer/FsConfigure.java
View File

@ -6,7 +6,7 @@ import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "datacap.experimental.fs")
@ConfigurationProperties(prefix = "datacap.fs")
public class FsConfigure
{
private String type;

9
core/datacap-service/src/main/java/io/edurt/datacap/service/initializer/InitializerConfigure.java
View File

@ -6,6 +6,7 @@ import com.google.common.collect.Maps;
import io.edurt.datacap.captcha.entity.ResultEntity;
import io.edurt.datacap.service.entity.PipelineEntity;
import io.edurt.datacap.service.loader.CaptchaCacheLoader;
import io.edurt.datacap.service.security.UserDetailsService;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
@ -163,6 +164,14 @@ public class InitializerConfigure
return false;
}
public String getAvatarPath()
{
if (fsConfigure.getEndpoint() != null) {
return fsConfigure.getEndpoint();
}
return avatarPath.replace("{username}", UserDetailsService.getUser().getUsername());
}
/**
* Checks if the task is ready for submission.
*

7
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/PluginAuditServiceImpl.java
View File

@ -3,6 +3,7 @@ package io.edurt.datacap.service.service.impl;
import com.google.common.collect.Maps;
import com.google.inject.Injector;
import io.edurt.datacap.common.response.CommonResponse;
import io.edurt.datacap.common.utils.DateUtils;
import io.edurt.datacap.common.utils.SpiUtils;
import io.edurt.datacap.fs.FsRequest;
import io.edurt.datacap.fs.FsResponse;
@ -24,6 +25,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.math.RoundingMode;
@ -107,6 +109,11 @@ public class PluginAuditServiceImpl
.bucket(initializer.getFsConfigure().getBucket())
.fileName("result.csv")
.build();
// If it is OSS third-party storage, rebuild the default directory
if (!initializer.getFsConfigure().getType().equals("Local")) {
fsRequest.setEndpoint(initializer.getFsConfigure().getEndpoint());
fsRequest.setFileName(String.join(File.separator, value.getUser().getUsername(), DateUtils.formatYMD(), String.join(File.separator, "adhoc", code), "result.csv"));
}
FsResponse fsResponse = SpiUtils.findFs(injector, initializer.getFsConfigure().getType())
.map(v -> v.reader(fsRequest))
.get();

2
core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/UserServiceImpl.java
View File

@ -285,7 +285,7 @@ public class UserServiceImpl
.map(fs -> {
UserEntity user = UserDetailsService.getUser();
try {
String avatarPath = initializerConfigure.getAvatarPath().replace("{username}", user.getUsername());
String avatarPath = initializerConfigure.getAvatarPath();
log.info("Upload avatar user [ {} ] home [ {} ]", user.getUsername(), avatarPath);
FsRequest fsRequest = FsRequest.builder()

54
fs/datacap-fs-alioss/pom.xml Normal file
View File

@ -0,0 +1,54 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap</artifactId>
<version>2024.03.4-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>datacap-fs-alioss</artifactId>
<description>DataCap - File system for aliyun oss</description>
<dependencies>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-common</artifactId>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-fs-spi</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>${datacap.alioss.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jetbrains.dokka</groupId>
<artifactId>dokka-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

86
fs/datacap-fs-alioss/src/main/kotlin/io/edurt/datacap/fs/alioss/AliOssFs.kt Normal file
View File

@ -0,0 +1,86 @@
package io.edurt.datacap.fs.alioss
import io.edurt.datacap.fs.Fs
import io.edurt.datacap.fs.FsRequest
import io.edurt.datacap.fs.FsResponse
import io.edurt.datacap.fs.alioss.IOUtils.Companion.copy
import org.slf4j.LoggerFactory.getLogger
import java.io.File
import java.lang.String.join
class AliOssFs : Fs
{
private val log = getLogger(AliOssFs::class.java)
override fun writer(request: FsRequest?): FsResponse
{
requireNotNull(request) { "request must not be null" }
log.info("AliOssFs writer origin path [ {} ]", request.fileName)
val targetPath = join(File.separator, request.endpoint, request.bucket, request.fileName)
val response = FsResponse.builder()
.origin(request.fileName)
.remote(targetPath)
.successful(true)
.build()
log.info("AliOssFs writer target path [ {} ]", request.fileName)
try
{
val key = copy(request, request.stream, request.fileName)
response.remote = key
log.info("AliOssFs writer [ {} ] successfully", key)
}
catch (e: Exception)
{
log.error("AliOssFs writer error", e)
response.isSuccessful = false
response.message = e.message
}
return response
}
override fun reader(request: FsRequest?): FsResponse
{
requireNotNull(request) { "request must not be null" }
log.info("AliOssFs reader origin path [ {} ]", request.fileName)
val response = FsResponse.builder()
.remote(request.fileName)
.successful(true)
.build()
try
{
response.context = IOUtils.reader(request)
log.info("AliOssFs reader [ {} ] successfully", request.fileName)
}
catch (e: java.lang.Exception)
{
log.error("AliOssFs reader error", e)
response.isSuccessful = false
response.message = e.message
}
return response
}
override fun delete(request: FsRequest?): FsResponse
{
requireNotNull(request) { "request must not be null" }
try
{
val status = IOUtils.delete(request)
log.info("AliOssFs delete [ {} ] successfully", request.fileName)
return FsResponse.builder()
.successful(status)
.build()
}
catch (e: java.lang.Exception)
{
log.error("AliOssFs delete error", e)
return FsResponse.builder()
.successful(false)
.message(e.message)
.build()
}
}
}

15
fs/datacap-fs-alioss/src/main/kotlin/io/edurt/datacap/fs/alioss/AliOssModule.kt Normal file
View File

@ -0,0 +1,15 @@
package io.edurt.datacap.fs.alioss
import com.google.inject.multibindings.Multibinder
import io.edurt.datacap.fs.Fs
import io.edurt.datacap.fs.FsModule
class AliOssModule : FsModule()
{
override fun configure()
{
Multibinder.newSetBinder(binder(), Fs::class.java)
.addBinding()
.to(AliOssFs::class.java)
}
}

73
fs/datacap-fs-alioss/src/main/kotlin/io/edurt/datacap/fs/alioss/IOUtils.kt Normal file
View File

@ -0,0 +1,73 @@
package io.edurt.datacap.fs.alioss
import com.aliyun.oss.OSSClientBuilder
import com.aliyun.oss.model.ObjectMetadata
import io.edurt.datacap.fs.FsRequest
import java.io.ByteArrayInputStream
import java.io.InputStream
import java.util.*
class IOUtils
{
companion object
{
@JvmStatic
fun copy(request: FsRequest, stream: InputStream, fileName: String): String?
{
val client = OSSClientBuilder().build(request.endpoint, request.access, request.secret)
try
{
client.putObject(request.bucket, fileName, stream, ObjectMetadata())
val expiration = Date(Date().time + 3600L * 1000 * 24 * 365 * 10)
return client.generatePresignedUrl(request.bucket, fileName, expiration).toString()
}
catch (e: Exception)
{
throw RuntimeException(e)
}
finally
{
client?.shutdown()
}
}
@JvmStatic
fun reader(request: FsRequest): InputStream
{
val client = OSSClientBuilder().build(request.endpoint, request.access, request.secret)
try
{
val obj = client.getObject(request.bucket, request.fileName)
val byteArray = obj.objectContent.readBytes()
return ByteArrayInputStream(byteArray)
}
catch (e: Exception)
{
throw RuntimeException(e)
}
finally
{
client?.shutdown()
}
}
@JvmStatic
fun delete(request: FsRequest): Boolean
{
val client = OSSClientBuilder().build(request.endpoint, request.access, request.secret)
try
{
client.deleteObject(request.bucket, request.fileName)
return true
}
catch (e: Exception)
{
throw RuntimeException(e)
}
finally
{
client?.shutdown()
}
}
}
}

1
fs/datacap-fs-alioss/src/main/resources/META-INF/services/io.edurt.datacap.fs.FsModule Normal file
View File

@ -0,0 +1 @@
io.edurt.datacap.fs.alioss.AliOssModule

86
fs/datacap-fs-alioss/src/test/kotlin/io/edurt/datacap/fs/alioss/AliOssFsTest.kt Normal file
View File

@ -0,0 +1,86 @@
package io.edurt.datacap.fs.alioss
import com.google.inject.Guice
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.fs.Fs
import io.edurt.datacap.fs.FsManager
import io.edurt.datacap.fs.FsRequest
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.slf4j.LoggerFactory
import java.io.BufferedReader
import java.io.FileInputStream
import java.io.IOException
import java.io.InputStreamReader
import java.nio.charset.StandardCharsets
class AliOssFsTest
{
private val log = LoggerFactory.getLogger(AliOssFsTest::class.java)
private val name = "AliOss"
private var request = FsRequest()
private var injector: Injector? = null
@Before
fun before()
{
request.access = System.getProperty("access")
request.secret = System.getProperty("secret")
request.bucket = System.getProperty("bucket")
request.fileName = "IOUtilsTest.kt"
request.endpoint = System.getProperty("endpoint")
injector = Guice.createInjector(FsManager())
}
@Test
fun writer()
{
val plugins: Set<Fs?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>()
{}))
val plugin: Fs? = plugins?.first { v -> v?.name().equals(name) }
val stream = FileInputStream("src/test/kotlin/io/edurt/datacap/fs/alioss/IOUtilsTest.kt")
request.stream = stream
val response = plugin !!.writer(request)
assertTrue(response.isSuccessful)
}
@Test
fun reader()
{
val plugins: Set<Fs?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>()
{}))
val plugin: Fs? = plugins?.first { v -> v?.name().equals(name) }
val response = plugin !!.reader(request)
assertTrue(response.isSuccessful)
try
{
BufferedReader(InputStreamReader(response.context, StandardCharsets.UTF_8)).use { reader ->
var line: String?
while ((reader.readLine().also { line = it }) != null)
{
log.info(line)
}
}
}
catch (e: IOException)
{
log.error("Reader error", e)
}
}
@Test
fun testDelete()
{
val plugins: Set<Fs?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>()
{}))
val plugin: Fs? = plugins?.first { v -> v?.name().equals(name) }
val response = plugin !!.delete(request)
assertTrue(response.isSuccessful)
}
}

31
fs/datacap-fs-alioss/src/test/kotlin/io/edurt/datacap/fs/alioss/AliOssModuleTest.kt Normal file
View File

@ -0,0 +1,31 @@
package io.edurt.datacap.fs.alioss
import com.google.inject.Guice.createInjector
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.fs.Fs
import org.junit.Before
import org.junit.Test
import kotlin.test.assertTrue
class AliOssModuleTest
{
private val name = "AliOss"
private var injector: Injector? = null
@Before
fun before()
{
injector = createInjector(AliOssModule())
}
@Test
fun test()
{
val fs: Fs? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>()
{}))
?.first { v -> v?.name().equals(name) }
assertTrue(fs != null)
}
}

53
fs/datacap-fs-alioss/src/test/kotlin/io/edurt/datacap/fs/alioss/IOUtilsTest.kt Normal file
View File

@ -0,0 +1,53 @@
package io.edurt.datacap.fs.alioss
import io.edurt.datacap.fs.FsRequest
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import java.io.BufferedReader
import java.io.FileInputStream
import java.io.InputStreamReader
class IOUtilsTest
{
private var request = FsRequest()
@Before
fun before()
{
request.access = System.getProperty("access")
request.secret = System.getProperty("secret")
request.bucket = System.getProperty("bucket")
request.fileName = "IOUtilsTest.kt"
request.endpoint = System.getProperty("endpoint")
}
@Test
fun copy()
{
val stream = FileInputStream("src/test/kotlin/io/edurt/datacap/fs/alioss/IOUtilsTest.kt")
val result = IOUtils.copy(request, stream, "IOUtilsTest.kt")
assertTrue(result != null)
}
@Test
fun reader()
{
val stream = IOUtils.reader(request)
val reader = BufferedReader(InputStreamReader(stream))
while (true)
{
val line = reader.readLine() ?: break
println(line.trimIndent())
}
assertNotNull(stream)
}
@Test
fun delete()
{
assertTrue(IOUtils.delete(request))
}
}

3
plugin/datacap-native-alioss/pom.xml
View File

@ -14,7 +14,6 @@
<description>DataCap - Aliyun OSS</description>
<properties>
<alioss.version>3.16.1</alioss.version>
<plugin.name>datacap-native-alioss</plugin.name>
</properties>
@ -35,7 +34,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>${alioss.version}</version>
<version>${datacap.alioss.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.code.gson</groupId>

2
pom.xml
View File

@ -81,6 +81,7 @@
<module>fs/datacap-fs-spi</module>
<module>fs/datacap-fs-local</module>
<module>fs/datacap-fs-qiniu</module>
<module>fs/datacap-fs-alioss</module>
<module>parser/datacap-parser-spi</module>
<module>parser/datacap-parser-trino</module>
<module>parser/datacap-parser-mysql</module>
@ -167,6 +168,7 @@
<!-- datacap plugin dependency -->
<datacap.pgsql.version>42.7.3</datacap.pgsql.version>
<datacap.qiniu.version>7.15.0</datacap.qiniu.version>
<datacap.alioss.version>3.16.1</datacap.alioss.version>
<!-- maven plugin -->
<plugin.maven.checkstyle.version>3.0.0</plugin.maven.checkstyle.version>
<plugin.maven.findbugs.version>3.0.5</plugin.maven.findbugs.version>