fix 文件管理中心支持远程下载

This commit is contained in:
bwcx_jzy 2023-03-17 20:54:51 +08:00
parent aeaccc8408
commit 6f19df6409
No known key found for this signature in database
GPG Key ID: 5E48E9372088B9E5
9 changed files with 325 additions and 29 deletions

View File

@ -23,7 +23,6 @@
package io.jpom.controller.outgiving;
import cn.hutool.core.collection.CollStreamUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.convert.Convert;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.BooleanUtil;
@ -262,16 +261,14 @@ public class OutGivingProjectController extends BaseServerController {
public JsonMessage<String> remoteDownload(String id, String afterOpt, String clearOld, String url, String autoUnzip,
String secondaryDirectory,
String stripComponents,
String selectProject) {
String selectProject,
HttpServletRequest request) {
OutGivingModel outGivingModel = this.check(id, (status, outGivingModel1) -> Assert.state(status != OutGivingModel.Status.ING, "当前还在分发中,请等待分发结束"));
AfterOpt afterOpt1 = BaseEnum.getEnum(AfterOpt.class, Convert.toInt(afterOpt, 0));
Assert.notNull(afterOpt1, "请选择分发后的操作");
// 验证远程 地址
ServerWhitelist whitelist = outGivingWhitelistService.getServerWhitelistData(getRequest());
Set<String> allowRemoteDownloadHost = whitelist.getAllowRemoteDownloadHost();
Assert.state(CollUtil.isNotEmpty(allowRemoteDownloadHost), "还没有配置运行的远程地址");
List<String> collect = allowRemoteDownloadHost.stream().filter(s -> StrUtil.startWith(url, s)).collect(Collectors.toList());
Assert.state(CollUtil.isNotEmpty(collect), "不允许下载当前地址的文件");
ServerWhitelist whitelist = outGivingWhitelistService.getServerWhitelistData(request);
whitelist.checkAllowRemoteDownloadHost(url);
//outGivingModel = outGivingServer.getItem(id);
outGivingModel.setClearOld(Convert.toBool(clearOld, false));

View File

@ -30,8 +30,10 @@ import io.jpom.common.BaseServerController;
import io.jpom.common.JsonMessage;
import io.jpom.common.ServerConst;
import io.jpom.common.validator.ValidatorItem;
import io.jpom.controller.outgiving.OutGivingWhitelistService;
import io.jpom.func.files.model.FileStorageModel;
import io.jpom.func.files.service.FileStorageService;
import io.jpom.model.data.ServerWhitelist;
import io.jpom.model.user.UserModel;
import io.jpom.permission.ClassFeature;
import io.jpom.permission.Feature;
@ -60,11 +62,14 @@ import java.io.IOException;
public class FileStorageController extends BaseServerController {
private final ServerConfig serverConfig;
private final FileStorageService fileStorageService;
private final OutGivingWhitelistService outGivingWhitelistService;
public FileStorageController(ServerConfig serverConfig,
FileStorageService fileStorageService) {
FileStorageService fileStorageService,
OutGivingWhitelistService outGivingWhitelistService) {
this.serverConfig = serverConfig;
this.fileStorageService = fileStorageService;
this.outGivingWhitelistService = outGivingWhitelistService;
}
/**
@ -228,4 +233,20 @@ public class FileStorageController extends BaseServerController {
fileStorageService.delByKey(id);
return JsonMessage.success("删除成功");
}
@PostMapping(value = "download", produces = MediaType.APPLICATION_JSON_VALUE)
@Feature(method = MethodFeature.REMOTE_DOWNLOAD)
public JsonMessage<String> download(
@ValidatorItem String url,
Integer keepDay,
String description,
Boolean global,
HttpServletRequest request) throws IOException {
// 验证远程 地址
ServerWhitelist whitelist = outGivingWhitelistService.getServerWhitelistData(request);
whitelist.checkAllowRemoteDownloadHost(url);
String workspace = fileStorageService.getCheckUserWorkspace(request);
fileStorageService.download(url, global, workspace, keepDay, description);
return JsonMessage.success("开始异步下载");
}
}

View File

@ -63,7 +63,7 @@ public class FileStorageModel extends BaseWorkspaceModel {
*/
private String description;
/**
* 文件来源 0 上传
* 文件来源 0 上传 1 构建 2 下载
*/
private Integer source;
/**
@ -78,6 +78,16 @@ public class FileStorageModel extends BaseWorkspaceModel {
* 文件扩展名
*/
private String extName;
/**
* 只有下载的时候才使用本字段
* <p>
* 0 下载中 1 下载完成 2 下载异常
*/
private Integer status;
/**
* 进度描述
*/
private String progressDesc;
/**
* 文件是否存在
*/

View File

@ -22,36 +22,59 @@
*/
package io.jpom.func.files.service;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.SystemClock;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.StreamProgress;
import cn.hutool.core.io.unit.DataSize;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.db.Entity;
import cn.hutool.extra.servlet.ServletUtil;
import cn.hutool.http.HttpUtil;
import io.jpom.JpomApplication;
import io.jpom.common.ISystemTask;
import io.jpom.common.ServerConst;
import io.jpom.func.files.model.FileStorageModel;
import io.jpom.service.IStatusRecover;
import io.jpom.service.h2db.BaseWorkspaceService;
import io.jpom.system.ServerConfig;
import io.jpom.system.extconf.BuildExtConfig;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import top.jpom.model.PageResultDto;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author bwcx_jzy
* @since 2023/3/16
*/
@Service
public class FileStorageService extends BaseWorkspaceService<FileStorageModel> implements ISystemTask {
@Slf4j
public class FileStorageService extends BaseWorkspaceService<FileStorageModel> implements ISystemTask, IStatusRecover {
private final ServerConfig serverConfig;
private final JpomApplication configBean;
private final BuildExtConfig buildExtConfig;
public FileStorageService(ServerConfig serverConfig) {
public FileStorageService(ServerConfig serverConfig,
JpomApplication configBean,
BuildExtConfig buildExtConfig) {
this.serverConfig = serverConfig;
this.configBean = configBean;
this.buildExtConfig = buildExtConfig;
}
/**
@ -69,6 +92,142 @@ public class FileStorageService extends BaseWorkspaceService<FileStorageModel> i
return super.listPage(paramMap);
}
/**
* 远程下载
*
* @param url url
* @param workspaceId 工作空间id
* @param description 描述
* @param global 是否为全局共享
* @param keepDay 保留天数
*/
public void download(String url, Boolean global, String workspaceId, Integer keepDay, String description) {
FileStorageModel fileStorageModel = new FileStorageModel();
// 临时使用 uuid代替
String uuid = IdUtil.fastSimpleUUID();
long startTime = SystemClock.now();
{
fileStorageModel.setId(uuid);
fileStorageModel.setName("文件下载中");
String empty = StrUtil.emptyToDefault(description, StrUtil.EMPTY);
fileStorageModel.setDescription(StrUtil.format("{} 远程下载 url{}", empty, url));
String extName = "download";
String path = StrUtil.format("/{}/{}.{}", DateTime.now().toString(DatePattern.PURE_DATE_FORMAT), uuid, extName);
fileStorageModel.setExtName("download");
fileStorageModel.setPath(path);
fileStorageModel.setSize(0L);
fileStorageModel.setStatus(0);
fileStorageModel.setSource(2);
if (global != null && global) {
fileStorageModel.setWorkspaceId(ServerConst.WORKSPACE_GLOBAL);
} else {
fileStorageModel.setWorkspaceId(workspaceId);
}
fileStorageModel.validUntil(keepDay, null);
this.insert(fileStorageModel);
}
// 异步下载
ThreadUtil.execute(() -> {
try {
File tempPath = configBean.getTempPath();
File file = FileUtil.file(tempPath, "file-storage-download", uuid);
FileUtil.mkdir(file);
int logReduceProgressRatio = buildExtConfig.getLogReduceProgressRatio();
Set<Integer> progressRangeList = ConcurrentHashMap.newKeySet((int) Math.floor((float) 100 / logReduceProgressRatio));
long bytes = DataSize.ofMegabytes(1).toBytes();
File fileFromUrl = HttpUtil.downloadFileFromUrl(url, file, -1, new StreamProgress() {
@Override
public void start() {
}
@Override
public void progress(long total, long progressSize) {
if (total > 0) {
double progressPercentage = Math.floor(((float) progressSize / total) * 100);
String percent = NumberUtil.formatPercent((float) progressSize / total, 0);
int progressRange = (int) Math.floor(progressPercentage / logReduceProgressRatio);
// 存在文件总大小
if (progressRangeList.add(progressRange)) {
// total, progressSize
updateProgress(uuid, percent, total, progressSize);
}
} else {
// 不存在文件总大小
if (progressSize % bytes == 0) {
updateProgress(uuid, null, total, progressSize);
}
}
}
@Override
public void finish() {
}
});
String md5 = SecureUtil.md5(fileFromUrl);
FileStorageModel storageModel = this.getByKey(md5);
if (storageModel != null) {
this.updateError(uuid, "文件已经存在啦");
FileUtil.del(fileFromUrl);
return;
}
String extName = FileUtil.extName(fileFromUrl);
// 避免跨天数据
String path = StrUtil.format("/{}/{}.{}", new DateTime(startTime).toString(DatePattern.PURE_DATE_FORMAT), md5, extName);
File storageSavePath = serverConfig.fileStorageSavePath();
File fileStorageFile = FileUtil.file(storageSavePath, path);
FileUtil.mkParentDirs(fileStorageFile);
FileUtil.move(fileFromUrl, fileStorageFile, true);
//
FileStorageModel update = new FileStorageModel();
// 需要将 id 更新为真实 id
update.setId(md5);
update.setName(fileFromUrl.getName());
update.setExtName(extName);
update.setModifyTimeMillis(SystemClock.now());
update.setPath(path);
update.setStatus(1);
update.setSize(FileUtil.size(fileStorageFile));
Entity updateEntity = this.dataBeanToEntity(update);
Entity id = Entity.create().set("id", uuid);
this.update(updateEntity, id);
} catch (Exception e) {
log.error("下载文件失败", e);
this.updateError(uuid, e.getMessage());
}
});
}
private void updateProgress(String id, String desc, long total, long progressSize) {
FileStorageModel fileStorageModel = new FileStorageModel();
fileStorageModel.setId(id);
String fileSize = FileUtil.readableFileSize(progressSize);
desc = StrUtil.emptyToDefault(desc, fileSize);
fileStorageModel.setName("文件下载中:" + desc);
fileStorageModel.setStatus(0);
fileStorageModel.setSize(progressSize);
fileStorageModel.setProgressDesc(StrUtil.format("当前进度:{} ,文件总大小:{},已经下载:{}", desc, FileUtil.readableFileSize(total), fileSize));
this.updateById(fileStorageModel);
}
/**
* 更新进度
*
* @param id 数据id
* @param error 错误信息
*/
private void updateError(String id, String error) {
FileStorageModel fileStorageModel = new FileStorageModel();
fileStorageModel.setId(id);
fileStorageModel.setName("文件下载失败:" + StrUtil.maxLength(error, 200));
fileStorageModel.setStatus(2);
fileStorageModel.setProgressDesc(error);
this.updateById(fileStorageModel);
}
/**
* 添加文件
*
@ -107,5 +266,32 @@ public class FileStorageService extends BaseWorkspaceService<FileStorageModel> i
@Override
public void executeTask() {
// 定时删除文件
Entity entity = Entity.create();
entity.set("validUntil", " < " + SystemClock.now());
List<FileStorageModel> storageModels = this.listByEntity(entity);
if (CollUtil.isEmpty(storageModels)) {
return;
}
File storageSavePath = serverConfig.fileStorageSavePath();
for (FileStorageModel storageModel : storageModels) {
log.info("开始删除 {} 文件 {}", storageModel.getName(), storageModel.getPath());
File fileStorageFile = FileUtil.file(storageSavePath, storageModel.getPath());
FileUtil.del(fileStorageFile);
this.delByKey(storageModel.getId());
}
}
@Override
public int statusRecover() {
FileStorageModel update = new FileStorageModel();
update.setName("系统重启取消下载任务");
update.setModifyTimeMillis(SystemClock.now());
update.setStatus(2);
Entity updateEntity = this.dataBeanToEntity(update);
//
Entity where = Entity.create()
.set("source", 2)
.set("status", 0);
return this.update(updateEntity, where);
}
}

View File

@ -22,13 +22,16 @@
*/
package io.jpom.model.data;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import io.jpom.model.BaseJsonModel;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.util.Assert;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* 节点分发白名单
@ -65,4 +68,11 @@ public class ServerWhitelist extends BaseJsonModel {
public List<String> outGiving() {
return AgentWhitelist.useConvert(outGiving);
}
public void checkAllowRemoteDownloadHost(String url) {
Set<String> allowRemoteDownloadHost = this.getAllowRemoteDownloadHost();
Assert.state(CollUtil.isNotEmpty(allowRemoteDownloadHost), "还没有配置运行的远程地址");
List<String> collect = allowRemoteDownloadHost.stream().filter(s -> StrUtil.startWith(url, s)).collect(Collectors.toList());
Assert.state(CollUtil.isNotEmpty(collect), "不允许下载当前地址的文件");
}
}

View File

@ -32,3 +32,5 @@ ADD,MACHINE_SSH_INFO,osMaxOccupyDisk,Double,,,占用磁盘,false
ADD,MACHINE_SSH_INFO,osMaxOccupyDiskName,String,255,,占用磁盘 分区名,false
ADD,MACHINE_SSH_INFO,javaVersion,String,255,,java版本,false
ADD,MACHINE_SSH_INFO,jpomAgentPid,Integer,,,jpom agent进程号
ADD,FILE_STORAGE,status,TINYINT,,,true,false,0 下载中 1 下载完成 3 下载异常,
ADD,FILE_STORAGE,progressDesc,String,255,,true,false,进度描述,

1 alterType,tableName,name,type,len,defaultValue,comment,notNull
32 ADD,MACHINE_SSH_INFO,osMaxOccupyDiskName,String,255,,占用磁盘 分区名,false
33 ADD,MACHINE_SSH_INFO,javaVersion,String,255,,java版本,false
34 ADD,MACHINE_SSH_INFO,jpomAgentPid,Integer,,,jpom agent进程号
35 ADD,FILE_STORAGE,status,TINYINT,,,true,false,0 下载中 1 下载完成 3 下载异常,
36 ADD,FILE_STORAGE,progressDesc,String,255,,true,false,进度描述,

View File

@ -20,6 +20,7 @@
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DatePattern;
@ -149,4 +150,9 @@ public class TestT {
FileUtils.checkSlip("/../../../xxx/xx//aaa/../");
FileUtils.checkSlip("/../../../xxx/xx?/&&{#}:/aaa/../");
}
@Test
public void testDownload() {
// HttpUtil.download()
}
}

View File

@ -42,6 +42,15 @@ export function fileEdit(params) {
});
}
// 下载远程文件
export function remoteDownload(params) {
return axios({
url: "/file-storage/download",
method: "post",
data: params,
});
}
// 判断文件是否存在
export function hasFile(params) {
return axios({
@ -63,4 +72,11 @@ export function delFile(params) {
export const sourceMap = {
0: "上传",
1: "构建",
2: "下载",
};
export const statusMap = {
0: "下载中",
1: "下载成功",
2: "下载异常",
};

View File

@ -24,15 +24,22 @@
<a-button type="primary" :loading="loading" @click="loadData">搜索</a-button>
</a-tooltip>
<a-button type="primary" @click="handleUpload">上传文件</a-button>
<a-button type="primary" @click="handleDownload">远程下载</a-button>
</a-space>
</template>
<a-tooltip slot="tooltip" slot-scope="text" placement="topLeft" :title="text">
<span>{{ text }}</span>
</a-tooltip>
<a-tooltip slot="id" slot-scope="text, item" placement="topLeft" :title="text">
<span v-if="item.status === 0 || item.status === 2">-</span>
<span v-else>{{ text }}</span>
</a-tooltip>
<a-popover slot="name" slot-scope="text, item" title="文件信息">
<template slot="content">
<p>文件名{{ text }}</p>
<p>文件描述{{ item.description }}</p>
<p v-if="item.status !== undefined">下载状态{{ statusMap[item.status] || "未知" }}</p>
<p v-if="item.progressDesc">状态描述{{ item.progressDesc }}</p>
</template>
{{ text }}
</a-popover>
@ -48,8 +55,8 @@
</a-tooltip>
<template slot="exists" slot-scope="text">
<a-tag v-if="text">存在</a-tag>
<a-tag v-else>丢失</a-tag>
<a-tag v-if="text" color="green">存在</a-tag>
<a-tag v-else color="red">丢失</a-tag>
</template>
<template slot="global" slot-scope="text">
<a-tag v-if="text === 'GLOBAL'">全局</a-tag>
@ -63,17 +70,7 @@
</template>
</a-table>
<!-- 上传文件 -->
<a-modal
destroyOnClose
v-model="uploadVisible"
:closable="!uploading"
:footer="uploading ? null : undefined"
:keyboard="false"
width="50%"
:title="`上传文件`"
@ok="handleUploadOk"
:maskClosable="false"
>
<a-modal destroyOnClose v-model="uploadVisible" :closable="!uploading" :footer="uploading ? null : undefined" :keyboard="false" :title="`上传文件`" @ok="handleUploadOk" :maskClosable="false">
<a-form-model ref="form" :rules="rules" :model="temp" :label-col="{ span: 4 }" :wrapper-col="{ span: 20 }">
<a-form-model-item label="选择文件" prop="file">
<a-progress v-if="percentage" :percent="percentage">
@ -119,7 +116,7 @@
</a-form-model>
</a-modal>
<!-- 编辑文件 -->
<a-modal destroyOnClose v-model="editVisible" width="50%" :title="`修改文件`" @ok="handleEditOk" :maskClosable="false">
<a-modal destroyOnClose v-model="editVisible" :title="`修改文件`" @ok="handleEditOk" :maskClosable="false">
<a-form-model ref="editForm" :rules="rules" :model="temp" :label-col="{ span: 4 }" :wrapper-col="{ span: 20 }">
<a-form-model-item label="文件名">
<a-input placeholder="文件名" v-model="temp.name" />
@ -138,13 +135,33 @@
</a-form-model-item>
</a-form-model>
</a-modal>
<!--远程下载 -->
<a-modal destroyOnClose v-model="uploadRemoteFileVisible" title="远程下载文件" @ok="handleRemoteUpload" :maskClosable="false">
<a-form-model :model="temp" :label-col="{ span: 6 }" :wrapper-col="{ span: 18 }" :rules="rules" ref="remoteForm">
<a-form-model-item label="远程下载URL" prop="url">
<a-input v-model="temp.url" placeholder="远程下载地址" />
</a-form-model-item>
<a-form-model-item label="保留天数">
<a-input-number v-model="temp.keepDay" :min="1" style="width: 100%" placeholder="文件保存天数,默认 3650 天" />
</a-form-model-item>
<a-form-model-item label="文件共享">
<a-radio-group v-model="temp.global">
<a-radio :value="true"> 全局 </a-radio>
<a-radio :value="false"> 当前工作空间 </a-radio>
</a-radio-group>
</a-form-model-item>
<a-form-model-item label="文件描述">
<a-textarea v-model="temp.description" placeholder="请输入文件描述" />
</a-form-model-item>
</a-form-model>
</a-modal>
</div>
</div>
</template>
<script>
import { CHANGE_PAGE, COMPUTED_PAGINATION, PAGE_DEFAULT_LIST_QUERY, parseTime, renderSize, formatDuration } from "@/utils/const";
import { fileStorageList, uploadFile, uploadFileMerge, fileEdit, hasFile, delFile, sourceMap } from "@/api/tools/file-storage";
import { fileStorageList, uploadFile, uploadFileMerge, fileEdit, hasFile, delFile, sourceMap, remoteDownload, statusMap } from "@/api/tools/file-storage";
import { uploadPieces } from "@/utils/upload-pieces";
export default {
@ -154,9 +171,9 @@ export default {
listQuery: Object.assign({}, PAGE_DEFAULT_LIST_QUERY),
list: [],
columns: [
{ title: "文件MD5", dataIndex: "id", ellipsis: true, width: "100px", scopedSlots: { customRender: "tooltip" } },
{ title: "文件MD5", dataIndex: "id", ellipsis: true, width: "100px", scopedSlots: { customRender: "id" } },
{ title: "名称", dataIndex: "name", ellipsis: true, scopedSlots: { customRender: "name" } },
{ title: "大小", dataIndex: "size", sorter: true, ellipsis: true, scopedSlots: { customRender: "renderSize" }, width: "80px" },
{ title: "大小", dataIndex: "size", sorter: true, ellipsis: true, scopedSlots: { customRender: "renderSize" }, width: "100px" },
{ title: "后缀", dataIndex: "extName", ellipsis: true, scopedSlots: { customRender: "tooltip" }, width: "80px" },
{ title: "共享", dataIndex: "workspaceId", ellipsis: true, scopedSlots: { customRender: "global" }, width: "80px" },
{ title: "来源", dataIndex: "source", ellipsis: true, scopedSlots: { customRender: "source" }, width: "80px" },
@ -189,19 +206,22 @@ export default {
scopedSlots: { customRender: "time" },
width: "100px",
},
{ title: "操作", dataIndex: "operation", ellipsis: true, scopedSlots: { customRender: "operation" }, width: 120 },
{ title: "操作", dataIndex: "operation", ellipsis: true, scopedSlots: { customRender: "operation" }, fixed: "right", width: "120px" },
],
rules: {
name: [{ required: true, message: "请输入文件名称", trigger: "blur" }],
url: [{ required: true, message: "请输入远程地址", trigger: "blur" }],
},
temp: {},
sourceMap,
statusMap,
fileList: [],
percentage: 0,
percentageInfo: {},
uploading: false,
uploadVisible: false,
editVisible: false,
uploadRemoteFileVisible: false,
};
},
computed: {
@ -374,6 +394,34 @@ export default {
},
});
},
//
handleDownload() {
this.uploadRemoteFileVisible = true;
this.temp = {
global: false,
};
this.$refs["remoteForm"]?.resetFields();
},
//
handleRemoteUpload() {
//
this.$refs["remoteForm"].validate((valid) => {
if (!valid) {
return false;
}
remoteDownload(this.temp).then((res) => {
if (res.code === 200) {
//
this.$notification.success({
message: res.msg,
});
this.uploadRemoteFileVisible = false;
this.loadData();
}
});
});
},
},
};
</script>