test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-11-29 18:48:23 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(plugin): adapter fs -- qiniu

Browse Source
This commit is contained in:
qianmoQ 2024-11-28 18:39:10 +08:00
parent 924d2e2264
commit 409486e0d3
15 changed files with 191 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
.github/workflows/bofore_checker.yml vendored
View File

@ -63,21 +63,32 @@ jobs:
- name: Run SpotBugs for server
run: ./mvnw clean install spotbugs:spotbugs -Dcheckstyle.skip -Dgpg.skip -Dskip.pnpm -DskipTests=true -f core/datacap-server/pom.xml
before_checker_test:
runs-on: ubuntu-latest
needs:
- before_checker_style
- before_checker_bugs
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Maven Checker Style
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
- run: chmod 755 ./mvnw
- run: ./mvnw clean test -Dspotbugs.skip -Dgpg.skip -Dcheckstyle.skip -Dskip.pnpm -DskipAssembly=true
before_checker_test:
runs-on: ubuntu-latest
needs:
- before_checker_style
- before_checker_bugs
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Maven Checker Style
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
- run: chmod 755 ./mvnw
- name: Run Tests
env:
ACCESS: ${{ secrets.QINIU_ACCESS }}
SECRET: ${{ secrets.QINIU_SECRET }}
BUCKET: ${{ secrets.QINIU_BUCKET }}
ENDPOINT: ${{ secrets.QINIU_ENDPOINT }}
run: |
./mvnw clean test -Dspotbugs.skip -Dgpg.skip -Dcheckstyle.skip -Dskip.pnpm \
-Dqiniu.access="$ACCESS" -Dqiniu.secret="$SECRET" -Dqiniu.bucket="$BUCKET" -Dqiniu.endpoint="$ENDPOINT"
before_checker_package:
runs-on: ubuntu-latest

2
fs/datacap-fs-qiniu/pom.xml
View File

@ -9,7 +9,7 @@
</parent>
<artifactId>datacap-fs-qiniu</artifactId>
<description>DataCap - File system for qiniu</description>
<description>DataCap - File System - Qiniu</description>
<dependencies>
<dependency>

12
fs/datacap-fs-qiniu/src/main/kotlin/io/edurt/datacap/fs/qiniu/QiniuFsPlugin.kt Normal file
View File

@ -0,0 +1,12 @@
package io.edurt.datacap.fs.qiniu
import io.edurt.datacap.plugin.Plugin
import io.edurt.datacap.plugin.PluginType
class QiniuFsPlugin : Plugin()
{
override fun getType(): PluginType
{
return PluginType.FILESYSTEM
}
}

58
fs/datacap-fs-qiniu/src/main/kotlin/io/edurt/datacap/fs/qiniu/QiniuFs.kt → fs/datacap-fs-qiniu/src/main/kotlin/io/edurt/datacap/fs/qiniu/QiniuFsService.kt
View File

@ -1,33 +1,37 @@
package io.edurt.datacap.fs.qiniu
import io.edurt.datacap.fs.Fs
import io.edurt.datacap.fs.FsRequest
import io.edurt.datacap.fs.FsResponse
import io.edurt.datacap.fs.FsService
import io.edurt.datacap.fs.qiniu.IOUtils.Companion.copy
import org.slf4j.LoggerFactory.getLogger
import java.io.File
import java.lang.String.join
class QiniuFs : Fs {
private val log = getLogger(QiniuFs::class.java)
class QiniuFsService : FsService
{
private val log = getLogger(QiniuFsService::class.java)
override fun writer(request: FsRequest?): FsResponse {
override fun writer(request: FsRequest?): FsResponse
{
requireNotNull(request) { "request must not be null" }
log.info("QiniuFs writer origin path [ {} ]", request.fileName)
val targetPath = join(File.separator, request.endpoint, request.bucket, request.fileName)
val response = FsResponse.builder()
.origin(request.fileName)
.remote(targetPath)
.successful(true)
.build()
.origin(request.fileName)
.remote(targetPath)
.successful(true)
.build()
log.info("QiniuFs writer target path [ {} ]", request.fileName)
try {
try
{
val key = copy(request, request.stream, request.fileName)
response.remote = key
log.info("QiniuFs writer [ {} ] successfully", key)
}
catch (e: Exception) {
catch (e: Exception)
{
log.error("QiniuFs writer error", e)
response.isSuccessful = false
response.message = e.message
@ -35,19 +39,22 @@ class QiniuFs : Fs {
return response
}
override fun reader(request: FsRequest?): FsResponse {
override fun reader(request: FsRequest?): FsResponse
{
requireNotNull(request) { "request must not be null" }
log.info("QiniuFs reader origin path [ {} ]", request.fileName)
val response = FsResponse.builder()
.remote(request.fileName)
.successful(true)
.build()
try {
.remote(request.fileName)
.successful(true)
.build()
try
{
response.context = IOUtils.reader(request)
log.info("QiniuFs reader [ {} ] successfully", request.fileName)
}
catch (e: java.lang.Exception) {
catch (e: java.lang.Exception)
{
log.error("QiniuFs reader error", e)
response.isSuccessful = false
response.message = e.message
@ -55,22 +62,25 @@ class QiniuFs : Fs {
return response
}
override fun delete(request: FsRequest?): FsResponse {
override fun delete(request: FsRequest?): FsResponse
{
requireNotNull(request) { "request must not be null" }
try {
try
{
val status = IOUtils.delete(request)
log.info("QiniuFs delete [ {} ] successfully", request.fileName)
return FsResponse.builder()
.successful(status)
.build()
.successful(status)
.build()
}
catch (e: java.lang.Exception) {
catch (e: java.lang.Exception)
{
log.error("QiniuFs delete error", e)
return FsResponse.builder()
.successful(false)
.message(e.message)
.build()
.successful(false)
.message(e.message)
.build()
}
}
}

13
fs/datacap-fs-qiniu/src/main/kotlin/io/edurt/datacap/fs/qiniu/QiniuModule.kt
View File

@ -1,13 +0,0 @@
package io.edurt.datacap.fs.qiniu
import com.google.inject.multibindings.Multibinder
import io.edurt.datacap.fs.Fs
import io.edurt.datacap.fs.FsModule
class QiniuModule : FsModule() {
override fun configure() {
Multibinder.newSetBinder(binder(), Fs::class.java)
.addBinding()
.to(QiniuFs::class.java)
}
}

1
fs/datacap-fs-qiniu/src/main/resources/META-INF/services/io.edurt.datacap.fs.FsModule
View File

@ -1 +0,0 @@
io.edurt.datacap.fs.qiniu.QiniuModule

1
fs/datacap-fs-qiniu/src/main/resources/META-INF/services/io.edurt.datacap.plugin.Plugin Normal file
View File

@ -0,0 +1 @@
io.edurt.datacap.fs.qiniu.QiniuFsPlugin

1
fs/datacap-fs-qiniu/src/main/resources/META-INF/services/io.edurt.datacap.plugin.Service Normal file
View File

@ -0,0 +1 @@
io.edurt.datacap.fs.qiniu.QiniuFsService

38
fs/datacap-fs-qiniu/src/test/kotlin/io/edurt/datacap/fs/qiniu/IOUtilsTest.kt
View File

@ -1,38 +0,0 @@
package io.edurt.datacap.fs.qiniu
import io.edurt.datacap.fs.FsRequest
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import java.io.FileInputStream
class IOUtilsTest {
private var request = FsRequest()
@Before
fun before() {
request.access = System.getProperty("access")
request.secret = System.getProperty("secret")
request.bucket = System.getProperty("bucket")
request.fileName = "IOUtilsTest.kt"
request.endpoint = System.getProperty("endpoint")
}
@Test
fun copy() {
val stream = FileInputStream("src/test/kotlin/io/edurt/datacap/fs/qiniu/IOUtilsTest.kt")
val result = IOUtils.copy(request, stream, "IOUtilsTest.kt")
assertTrue(result != null)
}
@Test
fun reader() {
assertNotNull(IOUtils.reader(request))
}
@Test
fun delete() {
assertTrue(IOUtils.delete(request))
}
}

75
fs/datacap-fs-qiniu/src/test/kotlin/io/edurt/datacap/fs/qiniu/QiniuFsTest.kt
View File

@ -1,75 +0,0 @@
package io.edurt.datacap.fs.qiniu
import com.google.inject.Guice
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.fs.Fs
import io.edurt.datacap.fs.FsManager
import io.edurt.datacap.fs.FsRequest
import org.junit.Before
import org.junit.Test
import org.slf4j.LoggerFactory
import java.io.BufferedReader
import java.io.FileInputStream
import java.io.IOException
import java.io.InputStreamReader
import java.nio.charset.StandardCharsets
import kotlin.test.assertTrue
class QiniuFsTest {
private val log = LoggerFactory.getLogger(QiniuFsTest::class.java)
private val name = "Qiniu"
private var request = FsRequest()
private var injector: Injector? = null
@Before
fun before() {
request.access = System.getProperty("access")
request.secret = System.getProperty("secret")
request.bucket = System.getProperty("bucket")
request.fileName = "IOUtilsTest.kt"
request.endpoint = System.getProperty("endpoint")
injector = Guice.createInjector(FsManager())
}
@Test
fun writer() {
val plugins: Set<Fs?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>() {}))
val plugin: Fs? = plugins?.first { v -> v?.name().equals(name) }
val stream = FileInputStream("src/test/kotlin/io/edurt/datacap/fs/qiniu/IOUtilsTest.kt")
request.stream = stream
val response = plugin !!.writer(request)
assertTrue(response.isSuccessful)
}
@Test
fun reader() {
val plugins: Set<Fs?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>() {}))
val plugin: Fs? = plugins?.first { v -> v?.name().equals(name) }
val response = plugin !!.reader(request)
assertTrue(response.isSuccessful)
try {
BufferedReader(InputStreamReader(response.context, StandardCharsets.UTF_8)).use { reader ->
var line: String?
while ((reader.readLine().also { line = it }) != null) {
log.info(line)
}
}
}
catch (e: IOException) {
log.error("Reader error", e)
}
}
@Test
fun testDelete() {
val plugins: Set<Fs?>? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>() {}))
val plugin: Fs? = plugins?.first { v -> v?.name().equals(name) }
val response = plugin !!.delete(request)
assertTrue(response.isSuccessful)
}
}

27
fs/datacap-fs-qiniu/src/test/kotlin/io/edurt/datacap/fs/qiniu/QiniuModuleTest.kt
View File

@ -1,27 +0,0 @@
package io.edurt.datacap.fs.qiniu
import com.google.inject.Guice.createInjector
import com.google.inject.Injector
import com.google.inject.Key
import com.google.inject.TypeLiteral
import io.edurt.datacap.fs.Fs
import org.junit.Before
import org.junit.Test
import kotlin.test.assertTrue
class QiniuModuleTest {
private val name = "Qiniu"
private var injector: Injector? = null
@Before
fun before() {
injector = createInjector(QiniuModule())
}
@Test
fun test() {
val fs: Fs? = injector?.getInstance(Key.get(object : TypeLiteral<Set<Fs?>?>() {}))
?.first { v -> v?.name().equals(name) }
assertTrue(fs != null)
}
}

2
pom.xml
View File

@ -83,7 +83,7 @@
<!-- <module>shaded/datacap-shaded-pinot</module>-->
<module>fs/datacap-fs-spi</module>
<module>fs/datacap-fs-local</module>
<!-- <module>fs/datacap-fs-qiniu</module>-->
<module>fs/datacap-fs-qiniu</module>
<!-- <module>fs/datacap-fs-alioss</module>-->
<!-- <module>fs/datacap-fs-tencent-cos</module>-->
<!-- <module>fs/datacap-fs-amazon-s3</module>-->

6
test/datacap-test-fs/pom.xml
View File

@ -43,6 +43,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.edurt.datacap</groupId>
<artifactId>datacap-fs-qiniu</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>

66
test/datacap-test-fs/src/test/kotlin/io/edurt/datacap/test/local/LocalFsServiceTest.kt
View File

@ -2,13 +2,19 @@ package io.edurt.datacap.test.local
import io.edurt.datacap.fs.FsRequest
import io.edurt.datacap.fs.FsService
import io.edurt.datacap.plugin.Plugin
import io.edurt.datacap.plugin.PluginConfigure
import io.edurt.datacap.plugin.PluginManager
import io.edurt.datacap.plugin.utils.PluginPathUtils
import org.junit.Assert.assertNotNull
import org.junit.Test
import org.slf4j.LoggerFactory.getLogger
import java.io.BufferedReader
import java.io.File
import java.io.IOException
import java.io.InputStreamReader
import java.nio.charset.StandardCharsets
import java.util.*
import kotlin.test.assertTrue
class LocalFsServiceTest
@ -43,25 +49,73 @@ class LocalFsServiceTest
.build()
}
private fun getPlugin(): Optional<Plugin>
{
val plugin = pluginManager.getPlugin(pluginName)
assertNotNull(plugin.get())
return plugin
}
@Test
fun test()
fun step1_plugin()
{
val plugin = pluginManager.getPlugin(pluginName)
assertNotNull(plugin.get())
}
@Test
fun testWriter()
fun step3_writer()
{
val plugin = pluginManager.getPlugin(pluginName)
assertNotNull(plugin.get())
val plugin = getPlugin()
plugin.ifPresent {
val service = it.getService(FsService::class.java)
val response = service.writer(request)
log.info("LocalFs writer response [ {} ]", response)
assertTrue(response.isSuccessful)
log.info("LocalFs writer response [ {} ]", response)
}
}
@Test
fun step4_reader()
{
val plugin = getPlugin()
plugin.ifPresent {
val service = it.getService(FsService::class.java)
val response = service.reader(request)
assertTrue(response.isSuccessful)
log.info("====== [ {} ] ======", response.remote)
try
{
BufferedReader(InputStreamReader(response.context, StandardCharsets.UTF_8)).use { reader ->
var line: String?
while ((reader.readLine().also { line = it }) != null)
{
log.info(line)
}
}
}
catch (e: IOException)
{
log.error("Reader error", e)
}
}
}
@Test
fun step5_delete()
{
val plugin = getPlugin()
plugin.ifPresent {
val service = it.getService(FsService::class.java)
val response = service.delete(request)
assertTrue(response.isSuccessful)
log.info("LocalFs delete response [ {} ]", response)
}
}
}

49
test/datacap-test-fs/src/test/kotlin/io/edurt/datacap/test/qiniu/IOUtilsTest.kt Normal file
View File

@ -0,0 +1,49 @@
package io.edurt.datacap.fs.qiniu
import io.edurt.datacap.fs.FsRequest
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.slf4j.Logger
import org.slf4j.LoggerFactory.getLogger
import java.io.FileInputStream
class IOUtilsTest
{
private val log: Logger = getLogger(this::class.java)
private val request = FsRequest()
private val fileName = "QiniuFsTest.kt"
@Before
fun before()
{
request.access = System.getProperty("qiniu.access")
request.secret = System.getProperty("qiniu.secret")
request.bucket = System.getProperty("qiniu.bucket")
request.fileName = fileName
request.endpoint = System.getProperty("qiniu.endpoint")
}
@Test
fun step1_copy()
{
val stream = FileInputStream("src/test/kotlin/io/edurt/datacap/test/qiniu/QiniuFsTest.kt")
val response = IOUtils.copy(request, stream, fileName)
assertNotNull(response)
log.info("Copy response [ {} ]", response)
}
@Test
fun step2_reader()
{
assertNotNull(IOUtils.reader(request))
}
@Test
fun step3_delete()
{
assertTrue(IOUtils.delete(request))
}
}