From 5a93e469b7c2eae700acb08aa4c9870909ed83b3 Mon Sep 17 00:00:00 2001 From: bwcx_jzy Date: Mon, 9 Jan 2023 09:20:19 +0800 Subject: [PATCH] =?UTF-8?q?fix=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E8=BF=9B=E5=BA=A6=E6=8A=98=E5=8F=A0=E7=8E=87?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 ++ .../io/jpom/build/BuildExecuteService.java | 12 +++---- .../java/io/jpom/build/ReleaseManage.java | 36 +++++++++++++------ .../jpom/system/extconf/BuildExtConfig.java | 8 +++++ .../server/src/main/resources/application.yml | 3 +- .../resources/config_default/application.yml | 3 +- modules/server/src/test/java/TopTest.java | 9 ++++- .../java/top/jpom/db/IStorageService.java | 14 ++++---- .../src/main/java/io/jpom/plugin/GitUtil.java | 11 ++++-- .../jpom/plugin/SmallTextProgressMonitor.java | 16 +++++++-- 10 files changed, 82 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ead449220..3cfb3baea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ 1. 【all】新增 在线升级是否允许降级操作配置属性`jpom.system.allowed-downgrade` 2. 【server】新增 分发整体状态新增`分发失败` +3. 【server】新增 构建日志显示进度折叠率配置：`jpom.build.log-reduce-progress-ratio` ### 🐞 解决BUG、优化功能 @@ -22,6 +23,7 @@ 11. 【server】优化 构建日志输出各个流程耗时 12. 【server】优化 构建发布项目文件采用分片上传、并且支持进度显示 13. 【agent】优化 配置文件中上传文件大小限制由 1G 改为 10MB 节省插件端占用内存大小（采用分片代替） +14. 【server】优化 手动上传的节点分发文件将自动删除，节省存储空间 ### ⚠️ 注意 diff --git a/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java b/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java index d3299cc32..d81317e3b 100644 --- a/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java +++ b/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java @@ -548,7 +548,7 @@ public class BuildExecuteService { Tuple tuple = (Tuple) plugin.execute("branchAndTagList", map); //GitUtil.getBranchAndTagList(repositoryModel); Assert.notNull(tuple, "获取仓库分支失败"); - + map.put("reduceProgressRatio", this.buildExecuteService.buildExtConfig.getLogReduceProgressRatio()); map.put("logWriter", logRecorder.getPrintWriter()); map.put("savePath", gitFile); // 模糊匹配 标签 @@ -566,7 +566,7 @@ public class BuildExecuteService { //author bwcx_jzy 2022.11.28 buildEnv.put("BUILD_BRANCH_NAME", newBranchName); buildEnv.put("BUILD_TAG_NAME", newBranchTagName); // 标签拉取模式 - logRecorder.info("repository tag [" + branchTagName + "] clone pull from " + newBranchTagName); + logRecorder.info("repository tag [{}] clone pull from {}", branchTagName, newBranchTagName); result = (String[]) plugin.execute("pullByTag", map); } else { String branchName = buildInfoModel.getBranchName(); @@ -704,7 +704,7 @@ public class BuildExecuteService { throw Lombok.sneakyThrow(e); } }); - logRecorder.info("[INFO] --- PROCESS RESULT " + waitFor); + logRecorder.info("[SYSTEM-INFO] --- PROCESS RESULT " + waitFor); } catch (Exception e) { logRecorder.error("执行异常", e); return false; @@ -939,7 +939,7 @@ public class BuildExecuteService { } ScriptModel scriptModel = buildExecuteService.scriptServer.getByKey(noticeScriptId); if (scriptModel == null) { - logRecorder.info("[WARNING] noticeScript does not exist:{}", type); + logRecorder.info("[SYSTEM-WARNING] noticeScript does not exist:{}", type); return; } // 判断是否包含需要执行的事件 @@ -947,7 +947,7 @@ public class BuildExecuteService { log.warn("忽略执行事件脚本 {} {} {}", type, scriptModel.getName(), noticeScriptId); return; } - logRecorder.info("[INFO] --- EXEC NOTICESCRIPT {}", type); + logRecorder.info("[SYSTEM-INFO] --- EXEC NOTICESCRIPT {}", type); // 环境变量 Map environment = new HashMap<>(map.size()); for (Map.Entry entry : map.entrySet()) { @@ -973,7 +973,7 @@ public class BuildExecuteService { throw Lombok.sneakyThrow(e); } }); - logRecorder.info("[INFO] {} --- NOTICESCRIPT PROCESS RESULT {}", type, waitFor); + logRecorder.info("[SYSTEM-INFO] {} --- NOTICESCRIPT PROCESS RESULT {}", type, waitFor); } finally { if (scriptFile != null) { try { diff --git a/modules/server/src/main/java/io/jpom/build/ReleaseManage.java b/modules/server/src/main/java/io/jpom/build/ReleaseManage.java index 61ad3f1a3..d7ece5630 100644 --- a/modules/server/src/main/java/io/jpom/build/ReleaseManage.java +++ b/modules/server/src/main/java/io/jpom/build/ReleaseManage.java @@ -61,6 +61,7 @@ import io.jpom.service.node.ssh.SshService; import io.jpom.service.system.WorkspaceEnvVarService; import io.jpom.system.ExtConfigBean; import io.jpom.system.JpomRuntimeException; +import io.jpom.system.extconf.BuildExtConfig; import io.jpom.util.CommandUtil; import io.jpom.util.FileUtils; import io.jpom.util.LogRecorder; @@ -73,10 +74,8 @@ import org.springframework.util.Assert; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -99,6 +98,7 @@ public class ReleaseManage implements Runnable { private LogRecorder logRecorder; private File resultFile; + private BuildExtConfig buildExtConfig; private void init() { if (this.logRecorder == null) { @@ -118,6 +118,7 @@ public class ReleaseManage implements Runnable { // envFileMap.put("BUILD_NAME", this.buildExtraModule.getName()); buildEnv.put("BUILD_RESULT_FILE", FileUtil.getAbsolutePath(this.resultFile)); // envFileMap.put("BUILD_NUMBER_ID", this.buildNumberId + StrUtil.EMPTY); + this.buildExtConfig = SpringUtil.getBean(BuildExtConfig.class); } // /** @@ -512,13 +513,21 @@ public class ReleaseManage implements Runnable { String startPath = StringUtil.delStartPath(file, resultFileParent, false); startPath = FileUtil.normalize(startPath + StrUtil.SLASH + directory); // + Set progressRangeList = ConcurrentHashMap.newKeySet((int) Math.floor((float) 100 / buildExtConfig.getLogReduceProgressRatio())); + int finalI = i; JsonMessage jsonMessage = OutGivingRun.fileUpload(file, startPath, projectId, false, last ? afterOpt : AfterOpt.No, nodeModel, false, this.buildExtraModule.getProjectUploadCloseFirst(), (total, progressSize) -> { - // total, progressSize - logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", file.getName(), - FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total), - NumberUtil.formatPercent(((float) progressSize / total), 0)); + double progressPercentage = Math.floor(((float) progressSize / total) * 100); + int progressRange = (int) Math.floor(progressPercentage / buildExtConfig.getLogReduceProgressRatio()); + if (progressRangeList.add(progressRange)) { + // total, progressSize + logRecorder.info("[SYSTEM-INFO] 上传文件进度:{}[{}/{}] {}/{} {} ", file.getName(), + (finalI + 1), diffSize, + FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total), + NumberUtil.formatPercent(((float) progressSize / total), 0) + ); + } }); Assert.state(jsonMessage.success(), "同步项目文件失败：" + jsonMessage); if (last) { @@ -556,15 +565,20 @@ public class ReleaseManage implements Runnable { unZip = false; } String name = zipFile.getName(); + Set progressRangeList = ConcurrentHashMap.newKeySet((int) Math.floor((float) 100 / buildExtConfig.getLogReduceProgressRatio())); JsonMessage jsonMessage = OutGivingRun.fileUpload(zipFile, this.buildExtraModule.getProjectSecondaryDirectory(), projectId, unZip, afterOpt, nodeModel, clearOld, this.buildExtraModule.getProjectUploadCloseFirst(), (total, progressSize) -> { - logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", name, - FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total), - NumberUtil.formatPercent(((float) progressSize / total), 0)); + double progressPercentage = Math.floor(((float) progressSize / total) * 100); + int progressRange = (int) Math.floor(progressPercentage / buildExtConfig.getLogReduceProgressRatio()); + if (progressRangeList.add(progressRange)) { + logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", name, + FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total), + NumberUtil.formatPercent(((float) progressSize / total), 0)); + } }); if (jsonMessage.success()) { logRecorder.info("发布项目包成功：" + jsonMessage); diff --git a/modules/server/src/main/java/io/jpom/system/extconf/BuildExtConfig.java b/modules/server/src/main/java/io/jpom/system/extconf/BuildExtConfig.java index b0f08a064..a1e0e4336 100644 --- a/modules/server/src/main/java/io/jpom/system/extconf/BuildExtConfig.java +++ b/modules/server/src/main/java/io/jpom/system/extconf/BuildExtConfig.java @@ -58,5 +58,13 @@ public class BuildExtConfig { * 构建任务等待数量，超过此数量将取消构建任务，值最小为 1 */ private int poolWaitQueue = 10; + /** + * 压缩折叠显示进度比例 范围 1-100 + */ + private int logReduceProgressRatio = 5; + public void setLogReduceProgressRatio(int logReduceProgressRatio) { + // 修正值 + this.logReduceProgressRatio = Math.min(Math.max(logReduceProgressRatio, 1), 100); + } } diff --git a/modules/server/src/main/resources/application.yml b/modules/server/src/main/resources/application.yml index 2cee35e6b..f8c39bade 100644 --- a/modules/server/src/main/resources/application.yml +++ b/modules/server/src/main/resources/application.yml @@ -92,7 +92,8 @@ jpom: pool-size: 5 # 构建任务等待数量，超过此数量将取消构建任务，值最小为 1 pool-wait-queue: 10 - + # 日志显示 压缩折叠显示进度比例 范围 1-100 + log-reduce-progress-ratio: 5 server: #运行端口号 port: 2122 diff --git a/modules/server/src/main/resources/config_default/application.yml b/modules/server/src/main/resources/config_default/application.yml index c647e5851..b705516e7 100644 --- a/modules/server/src/main/resources/config_default/application.yml +++ b/modules/server/src/main/resources/config_default/application.yml @@ -93,7 +93,8 @@ jpom: pool-size: 5 # 构建任务等待数量，超过此数量将取消构建任务，值最小为 1 pool-wait-queue: 10 - + # 日志显示 压缩折叠显示进度比例 范围 1-100 + log-reduce-progress-ratio: 5 server: #运行端口号 port: 2122 diff --git a/modules/server/src/test/java/TopTest.java b/modules/server/src/test/java/TopTest.java index 7c399b284..3cd01af50 100644 --- a/modules/server/src/test/java/TopTest.java +++ b/modules/server/src/test/java/TopTest.java @@ -20,9 +20,11 @@ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ + import cn.hutool.core.io.IoUtil; import cn.hutool.core.thread.GlobalThreadPool; import cn.hutool.core.util.CharsetUtil; +import org.junit.Test; import java.io.IOException; import java.io.InputStream; @@ -40,8 +42,13 @@ public class TopTest { System.out.println(s); System.out.println("结束"); }); + } - + @Test + public void testFlor() { + double floor = Math.floor(((float) 103 / 103) * 100); + int floor1 = (int) Math.floor(floor / 600); + System.out.println(floor + " " + floor1); } private static String execCommand(String[] command) { diff --git a/modules/storage-module/storage-module-common/src/main/java/top/jpom/db/IStorageService.java b/modules/storage-module/storage-module-common/src/main/java/top/jpom/db/IStorageService.java index 72455f7b1..4a9d16c9c 100644 --- a/modules/storage-module/storage-module-common/src/main/java/top/jpom/db/IStorageService.java +++ b/modules/storage-module/storage-module-common/src/main/java/top/jpom/db/IStorageService.java @@ -82,7 +82,7 @@ public interface IStorageService extends AutoCloseable, IMode { * @throws Exception 异常 */ default boolean hasDbData() throws Exception { - throw new IllegalArgumentException("没有实现改功能"); + throw new UnsupportedOperationException("没有实现该功能"); } /** @@ -92,7 +92,7 @@ public interface IStorageService extends AutoCloseable, IMode { * @throws Exception 异常 */ default File recoverDb() throws Exception { - throw new IllegalArgumentException("没有实现改功能"); + throw new UnsupportedOperationException("没有实现该功能"); } /** @@ -102,7 +102,7 @@ public interface IStorageService extends AutoCloseable, IMode { * @throws Exception 异常 */ default String deleteDbFiles() throws Exception { - throw new IllegalArgumentException("没有实现改功能"); + throw new UnsupportedOperationException("没有实现该功能"); } /** @@ -111,7 +111,7 @@ public interface IStorageService extends AutoCloseable, IMode { * @param sqlFile sql 文件 */ default void transformSql(File sqlFile) { - throw new IllegalArgumentException("没有实现改功能"); + throw new UnsupportedOperationException("没有实现该功能"); } /** @@ -125,7 +125,7 @@ public interface IStorageService extends AutoCloseable, IMode { if (recoverSqlFile == null) { return; } - throw new IllegalArgumentException("没有实现改功能"); + throw new UnsupportedOperationException("没有实现该功能"); } /** @@ -137,7 +137,7 @@ public interface IStorageService extends AutoCloseable, IMode { * @throws SQLException sql 异常 */ default void alterUser(String oldUes, String newUse, String newPwd) throws SQLException { - throw new IllegalArgumentException("没有实现改功能"); + throw new UnsupportedOperationException("没有实现该功能"); } /** @@ -151,7 +151,7 @@ public interface IStorageService extends AutoCloseable, IMode { * @throws Exception 异常 */ default void backupSql(String url, String user, String pass, String backupSqlPath, List tableName) throws Exception { - throw new IllegalArgumentException("没有实现改功能"); + throw new UnsupportedOperationException("没有实现该功能"); } /** diff --git a/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/GitUtil.java b/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/GitUtil.java index 2f8c4fb2f..c28a697b6 100644 --- a/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/GitUtil.java +++ b/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/GitUtil.java @@ -41,7 +41,10 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.NoHeadException; import org.eclipse.jgit.api.errors.TransportException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.*; @@ -137,7 +140,8 @@ public class GitUtil { } CloneCommand cloneCommand = Git.cloneRepository(); if (printWriter != null) { - cloneCommand.setProgressMonitor(new SmallTextProgressMonitor(printWriter)); + Integer progressRatio = (Integer) parameter.get("reduceProgressRatio"); + cloneCommand.setProgressMonitor(new SmallTextProgressMonitor(printWriter, progressRatio)); } if (branchName != null) { cloneCommand.setBranch(Constants.R_HEADS + branchName); @@ -375,7 +379,8 @@ public class GitUtil { * @throws Exception 异常 */ private static PullResult pull(Git git, Map parameter, String branchName, PrintWriter printWriter) throws Exception { - SmallTextProgressMonitor progressMonitor = new SmallTextProgressMonitor(printWriter); + Integer progressRatio = (Integer) parameter.get("reduceProgressRatio"); + SmallTextProgressMonitor progressMonitor = new SmallTextProgressMonitor(printWriter, progressRatio); // 放弃本地修改 git.checkout().setName(branchName).setForced(true).setProgressMonitor(progressMonitor).call(); diff --git a/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/SmallTextProgressMonitor.java b/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/SmallTextProgressMonitor.java index 509b5d137..8471b4f52 100644 --- a/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/SmallTextProgressMonitor.java +++ b/modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/SmallTextProgressMonitor.java @@ -25,6 +25,8 @@ package io.jpom.plugin; import org.eclipse.jgit.lib.TextProgressMonitor; import java.io.Writer; +import java.util.HashSet; +import java.util.Set; /** * 少量的输出，只在进度 x%5=0 的时候输出 @@ -34,13 +36,23 @@ import java.io.Writer; */ public class SmallTextProgressMonitor extends TextProgressMonitor { - public SmallTextProgressMonitor(Writer out) { + private final Set progressRangeList; + private final int reduceProgressRatio; + + /** + * @param out 输出流 + * @param reduceProgressRatio 压缩折叠显示进度比例 范围 1-100 + */ + public SmallTextProgressMonitor(Writer out, int reduceProgressRatio) { super(out); + this.reduceProgressRatio = reduceProgressRatio; + this.progressRangeList = new HashSet<>((int) Math.floor((float) 100 / reduceProgressRatio)); } @Override protected void onUpdate(String taskName, int cmp, int totalWork, int pcnt) { - if (pcnt % 5 == 0) { + int progressRange = (int) Math.floor((float) pcnt / this.reduceProgressRatio); + if (progressRangeList.add(progressRange)) { super.onUpdate(taskName, cmp, totalWork, pcnt); } }