test/datacap
1
0
Fork 0
mirror of https://gitee.com/devlive-community/datacap.git synced 2024-11-29 10:37:57 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(plugin): support aliyun oss

Browse Source
This commit is contained in:
qianmoQ 2024-11-26 22:23:53 +08:00
parent 8929acfe24
commit 0be43f293b
5 changed files with 94 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -76,6 +76,8 @@ core/datacap-ui/yarn.lock
# Qshell #
.qshell/*
.ossutilconfig
.ossutil_checkpoint
# Test #
test/datacap-test-convert/*convertservice

0
.qshell/account.json
View File

3
configure/cloud.conf
View File

@ -1,4 +1,5 @@
BUCKET_NAME=devlive-north
BUCKET_NAME=devlive-cdn
OSS_ENDPOINT=oss-cn-beijing.aliyuncs.com
PLUGIN_HOME=applications/datacap/plugins
DATACAP_HOME=applications/datacap
METADATA_HOME=applications/datacap/metadata

90
configure/publish/publish-oss-artifact.sh Normal file
View File

@ -0,0 +1,90 @@
#!/bin/sh
HOME=$(pwd)
VERSION=$(./mvnw -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive exec:exec -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.help=INFO | tail -1)
echo "Publish artifact, usage version is ${VERSION}"
CONF_FILE="${HOME}/configure/cloud.conf"
if [ -f "$CONF_FILE" ]; then
. "$CONF_FILE"
else
echo "The configuration file $CONF_FILE does not exist, please check the path."
exit 1
fi
PLUGIN_DIR="${HOME}/dist/plugins"
if [ ! -d "$PLUGIN_DIR" ]; then
echo "The directory $PLUGIN_DIR does not exist, please check the path."
exit 1
fi
echo "OSS Bucket Name: $BUCKET_NAME"
echo "OSS Endpoint: $OSS_ENDPOINT"
echo "Work Home: $PLUGIN_HOME"
echo "Datacap Home: $DATACAP_HOME"
echo "Metadata Home: $METADATA_HOME"
# 确保 ossutil 已配置
if ! command -v ossutil > /dev/null; then
echo "Error: ossutil is not installed. Please install it first."
echo "You can download it from: https://help.aliyun.com/document_detail/120075.html"
exit 1
fi
printf "============================================\n"
printf "Publish artifact ...\n"
# Handle bin.tar.gz files
for file in "$HOME/dist"/*bin.tar.gz; do
if [ -e "$file" ]; then
filename=$(basename "$file")
echo "Uploading binary: $filename"
ossutil cp "$file" "oss://$BUCKET_NAME/$DATACAP_HOME/versions/$VERSION/$filename" -f
fi
done
# Handle metadata file
METADATA_FILE="${HOME}/configure/metadata.json"
if [ -f "$METADATA_FILE" ]; then
# Upload new metadata file
echo "Uploading metadata file"
ossutil cp "$METADATA_FILE" "oss://$BUCKET_NAME/$METADATA_HOME/metadata.json" -f
fi
# Original plugin upload logic
for file in "$PLUGIN_DIR"/*.tar.gz; do
if [ ! -e "$file" ]; then
echo "No tar.gz files found in directory $PLUGIN_DIR."
break
fi
filename=$(basename "$file")
echo "Document found: $filename"
case "$filename" in
datacap-executor-*)
target_dir="$PLUGIN_HOME/$VERSION/executor"
;;
datacap-convert-*)
target_dir="$PLUGIN_HOME/$VERSION/convert"
;;
datacap-fs-*)
target_dir="$PLUGIN_HOME/$VERSION/fs"
;;
datacap-parser-*)
target_dir="$PLUGIN_HOME/$VERSION/parser"
;;
datacap-scheduler-*)
target_dir="$PLUGIN_HOME/$VERSION/scheduler"
;;
*)
target_dir="$PLUGIN_HOME/$VERSION/plugin"
;;
esac
echo "Uploading $filename to $target_dir"
ossutil cp "$file" "oss://$BUCKET_NAME/$target_dir/$filename" -f
done
printf "============================================\n"

4
core/datacap-service/src/main/java/io/edurt/datacap/service/common/ConfigureUtils.java
View File

@ -7,7 +7,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.common.enums.ServiceState;
import io.edurt.datacap.common.utils.BeanToPropertiesUtils;
import io.edurt.datacap.common.utils.JsonUtils;
import io.edurt.datacap.executor.common.RunProtocol;
import io.edurt.datacap.service.body.PipelineFieldBody;
import io.edurt.datacap.service.configure.FieldType;
import io.edurt.datacap.service.configure.IConfigure;
@ -277,9 +276,6 @@ public class ConfigureUtils
originProperties.setProperty("context", query);
}
}
if (fieldBody.getProtocol().equals(RunProtocol.JDBC)) {
originProperties.setProperty("url", String.format("jdbc:%s://%s:%s/%s", entity.getType().toLowerCase(), entity.getHost(), entity.getPort(), entity.getDatabase()));
}
return mergeProperties(entity, fields, originProperties);
}