fix 构建日志显示进度折叠率配置

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 1. 【all】新增 在线升级是否允许降级操作配置属性`jpom.system.allowed-downgrade`
 2. 【server】新增 分发整体状态新增`分发失败`
+3. 【server】新增 构建日志显示进度折叠率配置：`jpom.build.log-reduce-progress-ratio`
 
 ### 🐞 解决BUG、优化功能
 
@@ -22,6 +23,7 @@
 11. 【server】优化 构建日志输出各个流程耗时
 12. 【server】优化 构建发布项目文件采用分片上传、并且支持进度显示
 13. 【agent】优化 配置文件中上传文件大小限制由 1G 改为 10MB 节省插件端占用内存大小（采用分片代替）
+14. 【server】优化 手动上传的节点分发文件将自动删除，节省存储空间
 
 ### ⚠️ 注意
 

modules/server/src/main/java/io/jpom/build/BuildExecuteService.java

@@ -548,7 +548,7 @@ public class BuildExecuteService {
                     Tuple tuple = (Tuple) plugin.execute("branchAndTagList", map);
                     //GitUtil.getBranchAndTagList(repositoryModel);
                     Assert.notNull(tuple, "获取仓库分支失败");
-
+                    map.put("reduceProgressRatio", this.buildExecuteService.buildExtConfig.getLogReduceProgressRatio());
                     map.put("logWriter", logRecorder.getPrintWriter());
                     map.put("savePath", gitFile);
                     // 模糊匹配 标签
@@ -566,7 +566,7 @@ public class BuildExecuteService {
                         //author bwcx_jzy 2022.11.28 buildEnv.put("BUILD_BRANCH_NAME", newBranchName);
                         buildEnv.put("BUILD_TAG_NAME", newBranchTagName);
                         // 标签拉取模式
-                        logRecorder.info("repository tag [" + branchTagName + "] clone pull from " + newBranchTagName);
+                        logRecorder.info("repository tag [{}] clone pull from {}", branchTagName, newBranchTagName);
                         result = (String[]) plugin.execute("pullByTag", map);
                     } else {
                         String branchName = buildInfoModel.getBranchName();
@@ -704,7 +704,7 @@ public class BuildExecuteService {
                             throw Lombok.sneakyThrow(e);
                         }
                     });
-                logRecorder.info("[INFO] --- PROCESS RESULT " + waitFor);
+                logRecorder.info("[SYSTEM-INFO] --- PROCESS RESULT " + waitFor);
             } catch (Exception e) {
                 logRecorder.error("执行异常", e);
                 return false;
@@ -939,7 +939,7 @@ public class BuildExecuteService {
             }
             ScriptModel scriptModel = buildExecuteService.scriptServer.getByKey(noticeScriptId);
             if (scriptModel == null) {
-                logRecorder.info("[WARNING] noticeScript does not exist:{}", type);
+                logRecorder.info("[SYSTEM-WARNING] noticeScript does not exist:{}", type);
                 return;
             }
             // 判断是否包含需要执行的事件
@@ -947,7 +947,7 @@ public class BuildExecuteService {
                 log.warn("忽略执行事件脚本 {} {} {}", type, scriptModel.getName(), noticeScriptId);
                 return;
             }
-            logRecorder.info("[INFO] --- EXEC NOTICESCRIPT {}", type);
+            logRecorder.info("[SYSTEM-INFO] --- EXEC NOTICESCRIPT {}", type);
             // 环境变量
             Map<String, String> environment = new HashMap<>(map.size());
             for (Map.Entry<String, Object> entry : map.entrySet()) {
@@ -973,7 +973,7 @@ public class BuildExecuteService {
                         throw Lombok.sneakyThrow(e);
                     }
                 });
-                logRecorder.info("[INFO] {} --- NOTICESCRIPT PROCESS RESULT {}", type, waitFor);
+                logRecorder.info("[SYSTEM-INFO] {} --- NOTICESCRIPT PROCESS RESULT {}", type, waitFor);
             } finally {
                 if (scriptFile != null) {
                     try {

modules/server/src/main/java/io/jpom/build/ReleaseManage.java

@@ -61,6 +61,7 @@ import io.jpom.service.node.ssh.SshService;
 import io.jpom.service.system.WorkspaceEnvVarService;
 import io.jpom.system.ExtConfigBean;
 import io.jpom.system.JpomRuntimeException;
+import io.jpom.system.extconf.BuildExtConfig;
 import io.jpom.util.CommandUtil;
 import io.jpom.util.FileUtils;
 import io.jpom.util.LogRecorder;
@@ -73,10 +74,8 @@ import org.springframework.util.Assert;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.HashMap;
+import java.util.*;
-import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
-import java.util.Map;
-import java.util.Objects;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -99,6 +98,7 @@ public class ReleaseManage implements Runnable {
 
     private LogRecorder logRecorder;
     private File resultFile;
+    private BuildExtConfig buildExtConfig;
 
     private void init() {
         if (this.logRecorder == null) {
@@ -118,6 +118,7 @@ public class ReleaseManage implements Runnable {
 //        envFileMap.put("BUILD_NAME", this.buildExtraModule.getName());
         buildEnv.put("BUILD_RESULT_FILE", FileUtil.getAbsolutePath(this.resultFile));
 //        envFileMap.put("BUILD_NUMBER_ID", this.buildNumberId + StrUtil.EMPTY);
+        this.buildExtConfig = SpringUtil.getBean(BuildExtConfig.class);
     }
 
 //	/**
@@ -512,13 +513,21 @@ public class ReleaseManage implements Runnable {
             String startPath = StringUtil.delStartPath(file, resultFileParent, false);
             startPath = FileUtil.normalize(startPath + StrUtil.SLASH + directory);
             //
+            Set<Integer> progressRangeList = ConcurrentHashMap.newKeySet((int) Math.floor((float) 100 / buildExtConfig.getLogReduceProgressRatio()));
+            int finalI = i;
             JsonMessage<String> jsonMessage = OutGivingRun.fileUpload(file, startPath,
                 projectId, false, last ? afterOpt : AfterOpt.No, nodeModel, false,
                 this.buildExtraModule.getProjectUploadCloseFirst(), (total, progressSize) -> {
+                    double progressPercentage = Math.floor(((float) progressSize / total) * 100);
+                    int progressRange = (int) Math.floor(progressPercentage / buildExtConfig.getLogReduceProgressRatio());
+                    if (progressRangeList.add(progressRange)) {
                         //  total, progressSize
-                    logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", file.getName(),
+                        logRecorder.info("[SYSTEM-INFO] 上传文件进度:{}[{}/{}] {}/{} {} ", file.getName(),
+                            (finalI + 1), diffSize,
                             FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total),
-                        NumberUtil.formatPercent(((float) progressSize / total), 0));
+                            NumberUtil.formatPercent(((float) progressSize / total), 0)
+                        );
+                    }
                 });
             Assert.state(jsonMessage.success(), "同步项目文件失败：" + jsonMessage);
             if (last) {
@@ -556,15 +565,20 @@ public class ReleaseManage implements Runnable {
             unZip = false;
         }
         String name = zipFile.getName();
+        Set<Integer> progressRangeList = ConcurrentHashMap.newKeySet((int) Math.floor((float) 100 / buildExtConfig.getLogReduceProgressRatio()));
         JsonMessage<String> jsonMessage = OutGivingRun.fileUpload(zipFile,
             this.buildExtraModule.getProjectSecondaryDirectory(),
             projectId,
             unZip,
             afterOpt,
             nodeModel, clearOld, this.buildExtraModule.getProjectUploadCloseFirst(), (total, progressSize) -> {
+                double progressPercentage = Math.floor(((float) progressSize / total) * 100);
+                int progressRange = (int) Math.floor(progressPercentage / buildExtConfig.getLogReduceProgressRatio());
+                if (progressRangeList.add(progressRange)) {
                     logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", name,
                         FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total),
                         NumberUtil.formatPercent(((float) progressSize / total), 0));
+                }
             });
         if (jsonMessage.success()) {
             logRecorder.info("发布项目包成功：" + jsonMessage);

modules/server/src/main/java/io/jpom/system/extconf/BuildExtConfig.java

@@ -58,5 +58,13 @@ public class BuildExtConfig {
      * 构建任务等待数量，超过此数量将取消构建任务，值最小为 1
      */
     private int poolWaitQueue = 10;
+    /**
+     * 压缩折叠显示进度比例 范围 1-100
+     */
+    private int logReduceProgressRatio = 5;
 
+    public void setLogReduceProgressRatio(int logReduceProgressRatio) {
+        // 修正值
+        this.logReduceProgressRatio = Math.min(Math.max(logReduceProgressRatio, 1), 100);
+    }
 }

modules/server/src/main/resources/application.yml

@@ -92,7 +92,8 @@ jpom:
     pool-size: 5
     # 构建任务等待数量，超过此数量将取消构建任务，值最小为 1
     pool-wait-queue: 10
-
+    # 日志显示 压缩折叠显示进度比例 范围 1-100
+    log-reduce-progress-ratio: 5
 server:
   #运行端口号
   port: 2122

modules/server/src/main/resources/config_default/application.yml

@@ -93,7 +93,8 @@ jpom:
     pool-size: 5
     # 构建任务等待数量，超过此数量将取消构建任务，值最小为 1
     pool-wait-queue: 10
-
+    # 日志显示 压缩折叠显示进度比例 范围 1-100
+    log-reduce-progress-ratio: 5
 server:
   #运行端口号
   port: 2122

modules/server/src/test/java/TopTest.java

@@ -20,9 +20,11 @@
  * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
+
 import cn.hutool.core.io.IoUtil;
 import cn.hutool.core.thread.GlobalThreadPool;
 import cn.hutool.core.util.CharsetUtil;
+import org.junit.Test;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -40,8 +42,13 @@ public class TopTest {
             System.out.println(s);
             System.out.println("结束");
         });
+    }
 
-
+    @Test
+    public void testFlor() {
+        double floor = Math.floor(((float) 103 / 103) * 100);
+        int floor1 = (int) Math.floor(floor / 600);
+        System.out.println(floor + "  " + floor1);
     }
 
     private static String execCommand(String[] command) {

modules/storage-module/storage-module-common/src/main/java/top/jpom/db/IStorageService.java

@@ -82,7 +82,7 @@ public interface IStorageService extends AutoCloseable, IMode {
      * @throws Exception 异常
      */
     default boolean hasDbData() throws Exception {
-        throw new IllegalArgumentException("没有实现改功能");
+        throw new UnsupportedOperationException("没有实现该功能");
     }
 
     /**
@@ -92,7 +92,7 @@ public interface IStorageService extends AutoCloseable, IMode {
      * @throws Exception 异常
      */
     default File recoverDb() throws Exception {
-        throw new IllegalArgumentException("没有实现改功能");
+        throw new UnsupportedOperationException("没有实现该功能");
     }
 
     /**
@@ -102,7 +102,7 @@ public interface IStorageService extends AutoCloseable, IMode {
      * @throws Exception 异常
      */
     default String deleteDbFiles() throws Exception {
-        throw new IllegalArgumentException("没有实现改功能");
+        throw new UnsupportedOperationException("没有实现该功能");
     }
 
     /**
@@ -111,7 +111,7 @@ public interface IStorageService extends AutoCloseable, IMode {
      * @param sqlFile sql 文件
      */
     default void transformSql(File sqlFile) {
-        throw new IllegalArgumentException("没有实现改功能");
+        throw new UnsupportedOperationException("没有实现该功能");
     }
 
     /**
@@ -125,7 +125,7 @@ public interface IStorageService extends AutoCloseable, IMode {
         if (recoverSqlFile == null) {
             return;
         }
-        throw new IllegalArgumentException("没有实现改功能");
+        throw new UnsupportedOperationException("没有实现该功能");
     }
 
     /**
@@ -137,7 +137,7 @@ public interface IStorageService extends AutoCloseable, IMode {
      * @throws SQLException sql 异常
      */
     default void alterUser(String oldUes, String newUse, String newPwd) throws SQLException {
-        throw new IllegalArgumentException("没有实现改功能");
+        throw new UnsupportedOperationException("没有实现该功能");
     }
 
     /**
@@ -151,7 +151,7 @@ public interface IStorageService extends AutoCloseable, IMode {
      * @throws Exception 异常
      */
     default void backupSql(String url, String user, String pass, String backupSqlPath, List<String> tableName) throws Exception {
-        throw new IllegalArgumentException("没有实现改功能");
+        throw new UnsupportedOperationException("没有实现该功能");
     }
 
     /**

modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/GitUtil.java

@@ -41,7 +41,10 @@ import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.NoHeadException;
 import org.eclipse.jgit.api.errors.TransportException;
 import org.eclipse.jgit.internal.JGitText;
-import org.eclipse.jgit.lib.*;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.PersonIdent;
+import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.transport.*;
@@ -137,7 +140,8 @@ public class GitUtil {
         }
         CloneCommand cloneCommand = Git.cloneRepository();
         if (printWriter != null) {
-            cloneCommand.setProgressMonitor(new SmallTextProgressMonitor(printWriter));
+            Integer progressRatio = (Integer) parameter.get("reduceProgressRatio");
+            cloneCommand.setProgressMonitor(new SmallTextProgressMonitor(printWriter, progressRatio));
         }
         if (branchName != null) {
             cloneCommand.setBranch(Constants.R_HEADS + branchName);
@@ -375,7 +379,8 @@ public class GitUtil {
      * @throws Exception 异常
      */
     private static PullResult pull(Git git, Map<String, Object> parameter, String branchName, PrintWriter printWriter) throws Exception {
-        SmallTextProgressMonitor progressMonitor = new SmallTextProgressMonitor(printWriter);
+        Integer progressRatio = (Integer) parameter.get("reduceProgressRatio");
+        SmallTextProgressMonitor progressMonitor = new SmallTextProgressMonitor(printWriter, progressRatio);
         // 放弃本地修改
         git.checkout().setName(branchName).setForced(true).setProgressMonitor(progressMonitor).call();
 

modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/SmallTextProgressMonitor.java

@@ -25,6 +25,8 @@ package io.jpom.plugin;
 import org.eclipse.jgit.lib.TextProgressMonitor;
 
 import java.io.Writer;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * 少量的输出，只在进度 x%5=0 的时候输出
@@ -34,13 +36,23 @@ import java.io.Writer;
  */
 public class SmallTextProgressMonitor extends TextProgressMonitor {
 
-    public SmallTextProgressMonitor(Writer out) {
+    private final Set<Integer> progressRangeList;
+    private final int reduceProgressRatio;
+
+    /**
+     * @param out                 输出流
+     * @param reduceProgressRatio 压缩折叠显示进度比例 范围 1-100
+     */
+    public SmallTextProgressMonitor(Writer out, int reduceProgressRatio) {
         super(out);
+        this.reduceProgressRatio = reduceProgressRatio;
+        this.progressRangeList = new HashSet<>((int) Math.floor((float) 100 / reduceProgressRatio));
     }
 
     @Override
     protected void onUpdate(String taskName, int cmp, int totalWork, int pcnt) {
-        if (pcnt % 5 == 0) {
+        int progressRange = (int) Math.floor((float) pcnt / this.reduceProgressRatio);
+        if (progressRangeList.add(progressRange)) {
             super.onUpdate(taskName, cmp, totalWork, pcnt);
         }
     }