优化文件下载

2024-11-30 10:58:14 +08:00 · 2023-03-18 14:25:24 +08:00 · 2023-03-18 14:25:24 +08:00 · 77609bfe2a
commit 77609bfe2a
parent 235a6b4e81
13 changed files with 164 additions and 65 deletions
--- a/modules/server/src/main/java/io/jpom/func/files/controller/FileStorageController.java
+++ b/modules/server/src/main/java/io/jpom/func/files/controller/FileStorageController.java
@ -274,8 +274,21 @@ public class FileStorageController extends BaseServerController {
    @GetMapping(value = "trigger-url", produces = MediaType.APPLICATION_JSON_VALUE)
    @Feature(method = MethodFeature.EDIT)
    public JsonMessage<Map<String, String>> getTriggerUrl(@ValidatorItem String id, String rest, HttpServletRequest request) {
-        FileStorageModel item = fileStorageService.getByKey(id, request);
        UserModel user = getUser();
+        // 查询当前工作空间
+        FileStorageModel item = fileStorageService.getByKey(id, request);
+        //
+        if (item == null) {
+            // 查询所有数据
+            item = fileStorageService.getByKey(id);
+            Assert.notNull(item, "没有对应的文件信息");
+            if (!user.isSystemUser()) {
+                // 判断创建人
+                // 不是管理员，需要验证是自己上传的文件
+                Assert.state(StrUtil.equals(item.getCreateUser(), user.getId()), "当前文件创建人不是您,不能创建下载地址");
+            }
+        }
+
        FileStorageModel updateInfo;
        if (StrUtil.isEmpty(item.getTriggerToken()) || StrUtil.isNotEmpty(rest)) {
            updateInfo = new FileStorageModel();
@ -286,12 +299,12 @@ public class FileStorageController extends BaseServerController {
        } else {
            updateInfo = item;
        }
-        Map<String, String> map = this.getBuildToken(updateInfo);
+        Map<String, String> map = this.getBuildToken(updateInfo, request);
        return JsonMessage.success(StrUtil.isEmpty(rest) ? "ok" : "重置成功", map);
    }

-    private Map<String, String> getBuildToken(FileStorageModel item) {
-        String contextPath = UrlRedirectUtil.getHeaderProxyPath(getRequest(), ServerConst.PROXY_PATH);
+    private Map<String, String> getBuildToken(FileStorageModel item, HttpServletRequest request) {
+        String contextPath = UrlRedirectUtil.getHeaderProxyPath(request, ServerConst.PROXY_PATH);
        String url = ServerOpenApi.FILE_STORAGE_DOWNLOAD.
            replace("{id}", item.getId()).
            replace("{token}", item.getTriggerToken());
--- a/modules/server/src/main/java/io/jpom/func/files/model/FileStorageModel.java
+++ b/modules/server/src/main/java/io/jpom/func/files/model/FileStorageModel.java
@ -28,6 +28,7 @@ import cn.hutool.core.date.DateTime;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.date.SystemClock;
 import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
 import io.jpom.model.BaseWorkspaceModel;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
@ -54,6 +55,11 @@ public class FileStorageModel extends BaseWorkspaceModel {
     * 文件名
     */
    private String name;
+
+    public void setName(String name) {
+        this.name = StrUtil.maxLength(name, 240);
+    }
+
    /**
     * 文件大小
     */
--- a/modules/server/src/main/java/io/jpom/func/openapi/controller/FileStorageApiController.java
+++ b/modules/server/src/main/java/io/jpom/func/openapi/controller/FileStorageApiController.java
@ -23,8 +23,11 @@
 package io.jpom.func.openapi.controller;

 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.date.DatePattern;
+import cn.hutool.core.date.DateTime;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.io.NioUtil;
 import cn.hutool.core.util.*;
 import cn.hutool.extra.servlet.ServletUtil;
 import io.jpom.common.BaseJpomController;
@ -36,6 +39,7 @@ import io.jpom.model.user.UserModel;
 import io.jpom.service.user.TriggerTokenLogServer;
 import io.jpom.system.ServerConfig;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.catalina.connector.ClientAbortException;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.util.Assert;
@ -43,11 +47,14 @@ import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RestController;

+import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.File;
-import java.io.OutputStream;
+import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.util.List;

 /**
@ -71,10 +78,54 @@ public class FileStorageApiController extends BaseJpomController {
        this.serverConfig = serverConfig;
    }

+    private long[] resolveRange(HttpServletRequest request, long fileSize, String id, String name, HttpServletResponse response) {
+        String range = ServletUtil.getHeader(request, HttpHeaders.RANGE, CharsetUtil.CHARSET_UTF_8);
+        log.debug("下载文件 {} {} {}", id, name, range);
+        long fromPos = 0, toPos, downloadSize;
+        if (StrUtil.isEmpty(range)) {
+            downloadSize = fileSize;
+        } else {
+            response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+            List<String> list = StrUtil.splitTrim(range, "=");
+            String rangeByte = CollUtil.getLast(list);
+            //  Range: bytes=0-499 表示第 0-499 字节范围的内容
+            //  Range: bytes=500-999 表示第 500-999 字节范围的内容
+            //  Range: bytes=-500 表示最后 500 字节的内容
+            //  Range: bytes=500- 表示从第 500 字节开始到文件结束部分的内容
+            //  Range: bytes=0-0,-1 表示第一个和最后一个字节
+            //  Range: bytes=500-600,601-999 同时指定几个范围
+            Assert.state(!StrUtil.contains(rangeByte, StrUtil.COMMA), "不支持分片多端下载");
+            // TODO 解析更多格式的 RANGE 请求头
+            long[] split = StrUtil.splitToLong(rangeByte, StrUtil.DASHED);
+            Assert.state(split != null, "range 传入的信息不正确");
+            if (split.length == 2) {
+                // Range: bytes=0-499 表示第 0-499 字节范围的内容
+                toPos = split[1];
+                fromPos = split[0];
+            } else if (split.length == 1) {
+                if (StrUtil.startWith(rangeByte, StrUtil.DASHED)) {
+                    // Range: bytes=-500 表示最后 500 字节的内容
+                    fromPos = Math.max(fileSize - split[0], 0);
+                    toPos = fileSize;
+                } else if (StrUtil.endWith(rangeByte, StrUtil.DASHED)) {
+                    // Range: bytes=500- 表示从第 500 字节开始到文件结束部分的内容
+                    fromPos = split[0];
+                    toPos = fileSize;
+                } else {
+                    throw new IllegalArgumentException("不支持的 range 格式 " + rangeByte);
+                }
+            } else {
+                throw new IllegalArgumentException("不支持的 range 格式 " + rangeByte);
+            }
+            downloadSize = toPos > fromPos ? (toPos - fromPos) : (fileSize - fromPos);
+        }
+        return new long[]{fromPos, downloadSize};
+    }
+
    @GetMapping(value = ServerOpenApi.FILE_STORAGE_DOWNLOAD, produces = MediaType.APPLICATION_JSON_VALUE)
    public void download(@PathVariable String id, @PathVariable String token,
                         HttpServletRequest request,
-                         HttpServletResponse response) {
+                         HttpServletResponse response) throws IOException {
        FileStorageModel storageModel = fileStorageService.getByKey(id);
        Assert.notNull(storageModel, "没有对应数据");
        Assert.state(StrUtil.equals(token, storageModel.getTriggerToken()), "token错误,或者已经失效");
@ -91,64 +142,61 @@ public class FileStorageApiController extends BaseJpomController {
        String name = ReUtil.replaceAll(storageModel.getName(), "[\\s\\\\/:\\*\\?\\\"<>\\|]", "");
        if (StrUtil.isEmpty(name)) {
            name = fileStorageFile.getName();
-        } else {
-            name += "." + storageModel.getExtName();
+        } else if (!StrUtil.endWith(name, StrUtil.DOT + storageModel.getExtName())) {
+            name += StrUtil.DOT + storageModel.getExtName();
        }
        String contentType = ObjectUtil.defaultIfNull(FileUtil.getMimeType(name), "application/octet-stream");
        String charset = ObjectUtil.defaultIfNull(response.getCharacterEncoding(), CharsetUtil.UTF_8);
        response.setHeader("Content-Disposition", StrUtil.format("attachment;filename=\"{}\"",
            URLUtil.encode(name, CharsetUtil.charset(charset))));
        response.setContentType(contentType);
-        //
        //    解析断点续传相关信息
+        long[] resolveRange = this.resolveRange(request, fileSize, storageModel.getId(), storageModel.getName(), response);
+        long fromPos = resolveRange[0];
+        long downloadSize = resolveRange[1];
+        //
+        response.setHeader(HttpHeaders.LAST_MODIFIED, DateTime.of(fileStorageFile.lastModified()).toString(DatePattern.HTTP_DATETIME_FORMAT));
        response.setHeader(HttpHeaders.ACCEPT_RANGES, "bytes");
-        String range = ServletUtil.getHeader(request, HttpHeaders.RANGE, CharsetUtil.CHARSET_UTF_8);
-        log.debug("下载文件 {} {} {}", storageModel.getId(), name, range);
-        long fromPos = 0, toPos = 0, downloadSize;
-        if (StrUtil.isEmpty(range)) {
-            downloadSize = fileSize;
-        } else {
-            response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
-            List<String> list = StrUtil.splitTrim(range, "=");
-            String rangeByte = CollUtil.getLast(list);
-            long[] split = StrUtil.splitToLong(rangeByte, "-");
-            Assert.state(split != null, "range 传入的信息不正确");
-            fromPos = split[0];
-            if (split.length == 2) {
-                toPos = split[1];
-            }
-            downloadSize = toPos > fromPos ? (int) (toPos - fromPos) : (int) (fileSize - fromPos);
-        }
+        //  Content-Range: bytes (unit first byte pos) - [last byte pos]/[entity legth]
+        response.setHeader(HttpHeaders.CONTENT_RANGE, StrUtil.format("bytes {}-{}/{}", fromPos, downloadSize, fileSize));
        response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(downloadSize));
        // Copy the stream to the response's output stream.
-        OutputStream out = null;
-        try (RandomAccessFile in = new RandomAccessFile(fileStorageFile, "r")) {
+        ServletOutputStream out = null;
+        try (RandomAccessFile in = new RandomAccessFile(fileStorageFile, "r"); FileChannel channel = in.getChannel()) {
            out = response.getOutputStream();
            // 设置下载起始位置
            if (fromPos > 0) {
-                in.seek(fromPos);
+                channel.position(fromPos);
            }
            // 缓冲区大小
            int bufLen = (int) Math.min(downloadSize, IoUtil.DEFAULT_BUFFER_SIZE);
-            byte[] buffer = new byte[bufLen];
+            ByteBuffer buffer = ByteBuffer.allocate(bufLen);
            int num;
-            int count = 0;
+            long count = 0;
            // 当前写到客户端的大小
-            while ((num = in.read(buffer)) != -1) {
-                out.write(buffer, 0, num);
+            while ((num = channel.read(buffer)) != NioUtil.EOF) {
+                buffer.flip();
+                out.write(buffer.array(), 0, num);
+                buffer.clear();
                count += num;
                //处理最后一段，计算不满缓冲区的大小
-                if (downloadSize - count < bufLen) {
-                    bufLen = (int) (downloadSize - count);
-                    if (bufLen == 0) {
+                long last = (downloadSize - count);
+                if (last == 0) {
                    break;
                }
-                    buffer = new byte[bufLen];
+                if (last < bufLen) {
+                    bufLen = (int) last;
+                    buffer = ByteBuffer.allocate(bufLen);
                }
            }
            response.flushBuffer();
+        } catch (ClientAbortException clientAbortException) {
+            log.warn("客户端终止连接：{}", clientAbortException.getMessage());
        } catch (Exception e) {
            log.error("数据下载失败", e);
+            if (out != null) {
+                out.write(StrUtil.bytes("error:" + e.getMessage()));
+            }
        } finally {
            IoUtil.close(out);
        }
--- a/modules/server/src/main/resources/application.yml
+++ b/modules/server/src/main/resources/application.yml
@ -119,6 +119,7 @@ server:
    min-response-size: 2048
  tomcat:
    uri-encoding: UTF-8
+    connection-timeout: 10M
 spring:
  profiles:
    active: mysql-1
--- a/modules/server/src/main/resources/sql-view/alter.all.v1.3.csv
+++ b/modules/server/src/main/resources/sql-view/alter.all.v1.3.csv
@ -32,6 +32,7 @@ ADD,MACHINE_SSH_INFO,osMaxOccupyDisk,Double,,,占用磁盘,false
 ADD,MACHINE_SSH_INFO,osMaxOccupyDiskName,String,255,,占用磁盘 分区名,false
 ADD,MACHINE_SSH_INFO,javaVersion,String,255,,java版本,false
 ADD,MACHINE_SSH_INFO,jpomAgentPid,Integer,,,jpom agent进程号
-ADD,FILE_STORAGE,status,TINYINT,,,0 下载中 1 下载完成 3 下载异常,false,
-ADD,FILE_STORAGE,progressDesc,String,255,,进度描述,false,
-ADD,FILE_STORAGE,triggerToken,String,200,,触发器token,false,
+ADD,FILE_STORAGE,status,TINYINT,,,0 下载中 1 下载完成 3 下载异常,false
+ADD,FILE_STORAGE,progressDesc,String,255,,进度描述,false
+ADD,FILE_STORAGE,triggerToken,String,200,,触发器token,false
+ALTER,FILE_STORAGE,name,String,255,,名称,true
--- a/modules/server/src/main/resources/sql-view/table.all.v1.1.csv
+++ b/modules/server/src/main/resources/sql-view/table.all.v1.1.csv
@ -100,7 +100,7 @@ FILE_STORAGE,modifyTimeMillis,Long,,,false,false,数据修改时间,
 FILE_STORAGE,modifyUser,String,50,,false,false,修改人,
 FILE_STORAGE,createUser,String,50,,true,false,创建人,
 FILE_STORAGE,workspaceId,String,50,,true,false,所属工作空间,
-FILE_STORAGE,name,String,50,,true,false,名称,
+FILE_STORAGE,name,String,255,,true,false,名称,
 FILE_STORAGE,description,String,255,,false,false,描述,
 FILE_STORAGE,size,Long,,,true,false,文件大小,
 FILE_STORAGE,source,TINYINT,,,true,false,文件来源,
--- a/modules/server/src/test/java/TestFile.java
+++ b/modules/server/src/test/java/TestFile.java
@ -20,7 +20,9 @@
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
+
 import cn.hutool.core.io.FileUtil;
+import cn.hutool.crypto.SecureUtil;
 import io.jpom.util.CommandUtil;
 import org.junit.Test;

@ -80,4 +82,14 @@ public class TestFile {
        System.out.println(extName);
    }

+    @Test
+    public void testMd5() {
+        File file = FileUtil.file("D:\\迅雷下载\\zh-cn_windows_11_business_editions_version_22h2_updated_sep_2022_x64_dvd_515a832b.iso");
+        File file1 = FileUtil.file("D:\\迅雷下载\\zh-cn_windows_11_business_editions_version_22h2_updated_sep_2022_x64_dvd_515a832b (1).iso");
+        File file2 = FileUtil.file("D:\\迅雷下载\\zh-cn_windows_11_business_editions_version_22h2_updated_sep_2022_x64_dvd_515a832b (2).iso");
+        System.out.println(SecureUtil.md5(file));
+        System.out.println(SecureUtil.md5(file1));
+        System.out.println(SecureUtil.md5(file2));
+    }
+
 }
--- a/modules/server/src/test/java/TestString.java
+++ b/modules/server/src/test/java/TestString.java
@ -20,6 +20,7 @@
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
+
 import cn.hutool.core.codec.Base64;
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.lang.PatternPool;
@ -177,4 +178,9 @@ public class TestString {
        ArrayList<Integer> integers = CollUtil.newArrayList(1, 2);
        System.out.println(CollUtil.sub(integers, 0 + 1, integers.size()));
    }
+
+    @Test
+    public void testLong() {
+        System.out.println(4045003435L - 5476659199L);
+    }
 }
--- a/modules/storage-module/storage-module-h2/src/main/java/top/jpom/storage/H2TableBuilderImpl.java
+++ b/modules/storage-module/storage-module-h2/src/main/java/top/jpom/storage/H2TableBuilderImpl.java
@ -79,6 +79,11 @@ public class H2TableBuilderImpl implements IStorageSqlBuilderService {
                    stringBuilder.append("ALTER TABLE ").append(viewAlterData.getTableName()).append(" ADD IF NOT EXISTS ");
                    stringBuilder.append(this.generateColumnSql(viewAlterData));
                    break;
+                case "ALTER":
+                    // alter  table table1 modify  column column1  decimal(10,1) DEFAULT NULL COMMENT '注释';
+                    stringBuilder.append("ALTER TABLE ").append(viewAlterData.getTableName()).append(" modify  column ");
+                    stringBuilder.append(this.generateColumnSql(viewAlterData));
+                    break;
                case "DROP-TABLE":
                    stringBuilder.append("drop table if exists ").append(viewAlterData.getTableName());
                    break;
--- a/modules/storage-module/storage-module-mysql/src/main/java/top/jpom/storage/MysqlTableBuilderImpl.java
+++ b/modules/storage-module/storage-module-mysql/src/main/java/top/jpom/storage/MysqlTableBuilderImpl.java
@ -82,11 +82,13 @@ public class MysqlTableBuilderImpl implements IStorageSqlBuilderService {
                case "ADD":
                    //  ALTER TABLE PROJECT_INFO ADD IF NOT EXISTS triggerToken VARCHAR (100) comment '触发器token';
                    String columnSql = this.generateColumnSql(viewAlterData);
-                    columnSql = StrUtil.replace(columnSql, "'", "\\'");
-                    int length = StrUtil.length(columnSql);
-                    Assert.state(length <= 180, "sql 语句太长啦");
                    stringBuilder.append("CALL add_column_if_not_exists('").append(viewAlterData.getTableName()).append("','").append(viewAlterData.getName()).append("','").append(columnSql).append("')");
                    break;
+                case "ALTER":
+                    // alter  table table1 modify  column column1  decimal(10,1) DEFAULT NULL COMMENT '注释';
+                    stringBuilder.append("ALTER TABLE ").append(viewAlterData.getTableName()).append(" modify  column ");
+                    stringBuilder.append(this.generateColumnSql(viewAlterData));
+                    break;
                case "DROP-TABLE":
                    stringBuilder.append("drop table if exists ").append(viewAlterData.getTableName());
                    break;
@ -177,7 +179,11 @@ public class MysqlTableBuilderImpl implements IStorageSqlBuilderService {
            stringBuilder.append("default '").append(defaultValue).append("'").append(StrUtil.SPACE);
        }
        stringBuilder.append("comment '").append(tableViewRowData.getComment()).append("'");
-        return stringBuilder.toString();
+        //
+        String columnSql = StrUtil.replace(stringBuilder.toString(), "'", "\\'");
+        int length = StrUtil.length(columnSql);
+        Assert.state(length <= 180, "sql 语句太长啦");
+        return columnSql;
    }

    @Override
--- a/web-vue/src/api/tools/file-storage.js
+++ b/web-vue/src/api/tools/file-storage.js
@ -69,7 +69,7 @@ export function delFile(params) {
  });
 }

-// 触发器
+// 下载 url
 export function triggerUrl(params) {
  return axios({
    url: "/file-storage/trigger-url",
--- a/web-vue/src/pages/node/node-layout/project/project-file.vue
+++ b/web-vue/src/pages/node/node-layout/project/project-file.vue
@ -129,7 +129,8 @@
              multiple
              :disabled="!!percentage"
            >
-              <a-icon type="loading" v-if="percentage" />
+              <template v-if="percentage"> <a-icon type="loading" v-if="this.uploadFileList.length" /><span v-else>-</span> </template>
+
              <a-button v-else icon="upload">选择文件</a-button>
            </a-upload>

--- a/web-vue/src/pages/tools/fileStorage/list.vue
+++ b/web-vue/src/pages/tools/fileStorage/list.vue
@ -24,7 +24,7 @@
              <a-button type="primary" :loading="loading" @click="loadData">搜索</a-button>
            </a-tooltip>
            <a-button type="primary" @click="handleUpload">上传文件</a-button>
-            <a-button type="primary" @click="handleDownload">远程下载</a-button>
+            <a-button type="primary" @click="handleRemoteDownload">远程下载</a-button>
          </a-space>
        </template>
        <a-tooltip slot="tooltip" slot-scope="text" placement="topLeft" :title="text">
@ -65,7 +65,7 @@
        <template slot="operation" slot-scope="text, record">
          <a-space>
            <a-button type="primary" size="small" @click="handleEdit(record)">编辑</a-button>
-            <a-button size="small" :disabled="!record.exists" type="primary" @click="handleTrigger(record)">触发器</a-button>
+            <a-button size="small" :disabled="!record.exists" type="primary" @click="handleDownloadUrl(record)">下载</a-button>
            <a-button type="danger" size="small" @click="handleDelete(record)">删除</a-button>
          </a-space>
        </template>
@ -434,7 +434,7 @@ export default {
      });
    },
    // 远程下载
-    handleDownload() {
+    handleRemoteDownload() {
      this.uploadRemoteFileVisible = true;
      this.temp = {
        global: false,
@ -461,15 +461,15 @@ export default {
        });
      });
    },
-    // 触发器
-    handleTrigger(record) {
+    // 下载地址
+    handleDownloadUrl(record) {
      this.temp = Object.assign({}, record);
      this.tempVue = Vue;
      triggerUrl({
        id: record.id,
      }).then((res) => {
        if (res.code === 200) {
-          this.fillTriggerResult(res);
+          this.fillDownloadUrlResult(res);
          this.triggerVisible = true;
        }
      });
@ -484,11 +484,11 @@ export default {
          this.$notification.success({
            message: res.msg,
          });
-          this.fillTriggerResult(res);
+          this.fillDownloadUrlResult(res);
        }
      });
    },
-    fillTriggerResult(res) {
+    fillDownloadUrlResult(res) {
      this.temp = { ...this.temp, triggerDownloadUrl: `${location.protocol}//${location.host}${res.data.triggerDownloadUrl}` };
    },
  },