test/Jpom
1
0
Fork 0
mirror of https://gitee.com/dromara/Jpom.git synced 2024-11-30 10:58:14 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

fix 构建日志显示进度折叠率配置

Browse Source
This commit is contained in:
bwcx_jzy 2023-01-09 09:20:19 +08:00
parent b119fc2350
commit 5a93e469b7
No known key found for this signature in database
GPG Key ID: 5E48E9372088B9E5
10 changed files with 82 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -6,6 +6,7 @@
1. 【all】新增 在线升级是否允许降级操作配置属性`jpom.system.allowed-downgrade`
2. 【server】新增 分发整体状态新增`分发失败`
3. 【server】新增 构建日志显示进度折叠率配置：`jpom.build.log-reduce-progress-ratio`
### 🐞 解决BUG、优化功能
@ -22,6 +23,7 @@
11. 【server】优化 构建日志输出各个流程耗时
12. 【server】优化 构建发布项目文件采用分片上传、并且支持进度显示
13. 【agent】优化 配置文件中上传文件大小限制由 1G 改为 10MB 节省插件端占用内存大小（采用分片代替）
14. 【server】优化 手动上传的节点分发文件将自动删除，节省存储空间
### ⚠️ 注意

12
modules/server/src/main/java/io/jpom/build/BuildExecuteService.java
View File

@ -548,7 +548,7 @@ public class BuildExecuteService {
Tuple tuple = (Tuple) plugin.execute("branchAndTagList", map);
//GitUtil.getBranchAndTagList(repositoryModel);
Assert.notNull(tuple, "获取仓库分支失败");
map.put("reduceProgressRatio", this.buildExecuteService.buildExtConfig.getLogReduceProgressRatio());
map.put("logWriter", logRecorder.getPrintWriter());
map.put("savePath", gitFile);
// 模糊匹配 标签
@ -566,7 +566,7 @@ public class BuildExecuteService {
//author bwcx_jzy 2022.11.28 buildEnv.put("BUILD_BRANCH_NAME", newBranchName);
buildEnv.put("BUILD_TAG_NAME", newBranchTagName);
// 标签拉取模式
logRecorder.info("repository tag [" + branchTagName + "] clone pull from " + newBranchTagName);
logRecorder.info("repository tag [{}] clone pull from {}", branchTagName, newBranchTagName);
result = (String[]) plugin.execute("pullByTag", map);
} else {
String branchName = buildInfoModel.getBranchName();
@ -704,7 +704,7 @@ public class BuildExecuteService {
throw Lombok.sneakyThrow(e);
}
});
logRecorder.info("[INFO] --- PROCESS RESULT " + waitFor);
logRecorder.info("[SYSTEM-INFO] --- PROCESS RESULT " + waitFor);
} catch (Exception e) {
logRecorder.error("执行异常", e);
return false;
@ -939,7 +939,7 @@ public class BuildExecuteService {
}
ScriptModel scriptModel = buildExecuteService.scriptServer.getByKey(noticeScriptId);
if (scriptModel == null) {
logRecorder.info("[WARNING] noticeScript does not exist:{}", type);
logRecorder.info("[SYSTEM-WARNING] noticeScript does not exist:{}", type);
return;
}
// 判断是否包含需要执行的事件
@ -947,7 +947,7 @@ public class BuildExecuteService {
log.warn("忽略执行事件脚本 {} {} {}", type, scriptModel.getName(), noticeScriptId);
return;
}
logRecorder.info("[INFO] --- EXEC NOTICESCRIPT {}", type);
logRecorder.info("[SYSTEM-INFO] --- EXEC NOTICESCRIPT {}", type);
// 环境变量
Map<String, String> environment = new HashMap<>(map.size());
for (Map.Entry<String, Object> entry : map.entrySet()) {
@ -973,7 +973,7 @@ public class BuildExecuteService {
throw Lombok.sneakyThrow(e);
}
});
logRecorder.info("[INFO] {} --- NOTICESCRIPT PROCESS RESULT {}", type, waitFor);
logRecorder.info("[SYSTEM-INFO] {} --- NOTICESCRIPT PROCESS RESULT {}", type, waitFor);
} finally {
if (scriptFile != null) {
try {

36
modules/server/src/main/java/io/jpom/build/ReleaseManage.java
View File

@ -61,6 +61,7 @@ import io.jpom.service.node.ssh.SshService;
import io.jpom.service.system.WorkspaceEnvVarService;
import io.jpom.system.ExtConfigBean;
import io.jpom.system.JpomRuntimeException;
import io.jpom.system.extconf.BuildExtConfig;
import io.jpom.util.CommandUtil;
import io.jpom.util.FileUtils;
import io.jpom.util.LogRecorder;
@ -73,10 +74,8 @@ import org.springframework.util.Assert;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -99,6 +98,7 @@ public class ReleaseManage implements Runnable {
private LogRecorder logRecorder;
private File resultFile;
private BuildExtConfig buildExtConfig;
private void init() {
if (this.logRecorder == null) {
@ -118,6 +118,7 @@ public class ReleaseManage implements Runnable {
// envFileMap.put("BUILD_NAME", this.buildExtraModule.getName());
buildEnv.put("BUILD_RESULT_FILE", FileUtil.getAbsolutePath(this.resultFile));
// envFileMap.put("BUILD_NUMBER_ID", this.buildNumberId + StrUtil.EMPTY);
this.buildExtConfig = SpringUtil.getBean(BuildExtConfig.class);
}
// /**
@ -512,13 +513,21 @@ public class ReleaseManage implements Runnable {
String startPath = StringUtil.delStartPath(file, resultFileParent, false);
startPath = FileUtil.normalize(startPath + StrUtil.SLASH + directory);
//
Set<Integer> progressRangeList = ConcurrentHashMap.newKeySet((int) Math.floor((float) 100 / buildExtConfig.getLogReduceProgressRatio()));
int finalI = i;
JsonMessage<String> jsonMessage = OutGivingRun.fileUpload(file, startPath,
projectId, false, last ? afterOpt : AfterOpt.No, nodeModel, false,
this.buildExtraModule.getProjectUploadCloseFirst(), (total, progressSize) -> {
// total, progressSize
logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", file.getName(),
FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total),
NumberUtil.formatPercent(((float) progressSize / total), 0));
double progressPercentage = Math.floor(((float) progressSize / total) * 100);
int progressRange = (int) Math.floor(progressPercentage / buildExtConfig.getLogReduceProgressRatio());
if (progressRangeList.add(progressRange)) {
// total, progressSize
logRecorder.info("[SYSTEM-INFO] 上传文件进度:{}[{}/{}] {}/{} {} ", file.getName(),
(finalI + 1), diffSize,
FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total),
NumberUtil.formatPercent(((float) progressSize / total), 0)
);
}
});
Assert.state(jsonMessage.success(), "同步项目文件失败：" + jsonMessage);
if (last) {
@ -556,15 +565,20 @@ public class ReleaseManage implements Runnable {
unZip = false;
}
String name = zipFile.getName();
Set<Integer> progressRangeList = ConcurrentHashMap.newKeySet((int) Math.floor((float) 100 / buildExtConfig.getLogReduceProgressRatio()));
JsonMessage<String> jsonMessage = OutGivingRun.fileUpload(zipFile,
this.buildExtraModule.getProjectSecondaryDirectory(),
projectId,
unZip,
afterOpt,
nodeModel, clearOld, this.buildExtraModule.getProjectUploadCloseFirst(), (total, progressSize) -> {
logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", name,
FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total),
NumberUtil.formatPercent(((float) progressSize / total), 0));
double progressPercentage = Math.floor(((float) progressSize / total) * 100);
int progressRange = (int) Math.floor(progressPercentage / buildExtConfig.getLogReduceProgressRatio());
if (progressRangeList.add(progressRange)) {
logRecorder.info("[SYSTEM-INFO] 上传文件进度:{} {}/{} {}", name,
FileUtil.readableFileSize(progressSize), FileUtil.readableFileSize(total),
NumberUtil.formatPercent(((float) progressSize / total), 0));
}
});
if (jsonMessage.success()) {
logRecorder.info("发布项目包成功：" + jsonMessage);

8
modules/server/src/main/java/io/jpom/system/extconf/BuildExtConfig.java
View File

@ -58,5 +58,13 @@ public class BuildExtConfig {
* 构建任务等待数量，超过此数量将取消构建任务，值最小为 1
*/
private int poolWaitQueue = 10;
/**
* 压缩折叠显示进度比例 范围 1-100
*/
private int logReduceProgressRatio = 5;
public void setLogReduceProgressRatio(int logReduceProgressRatio) {
// 修正值
this.logReduceProgressRatio = Math.min(Math.max(logReduceProgressRatio, 1), 100);
}
}

3
modules/server/src/main/resources/application.yml
View File

@ -92,7 +92,8 @@ jpom:
pool-size: 5
# 构建任务等待数量，超过此数量将取消构建任务，值最小为 1
pool-wait-queue: 10
# 日志显示 压缩折叠显示进度比例 范围 1-100
log-reduce-progress-ratio: 5
server:
#运行端口号
port: 2122

3
modules/server/src/main/resources/config_default/application.yml
View File

@ -93,7 +93,8 @@ jpom:
pool-size: 5
# 构建任务等待数量，超过此数量将取消构建任务，值最小为 1
pool-wait-queue: 10
# 日志显示 压缩折叠显示进度比例 范围 1-100
log-reduce-progress-ratio: 5
server:
#运行端口号
port: 2122

9
modules/server/src/test/java/TopTest.java
View File

@ -20,9 +20,11 @@
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.thread.GlobalThreadPool;
import cn.hutool.core.util.CharsetUtil;
import org.junit.Test;
import java.io.IOException;
import java.io.InputStream;
@ -40,8 +42,13 @@ public class TopTest {
System.out.println(s);
System.out.println("结束");
});
}
@Test
public void testFlor() {
double floor = Math.floor(((float) 103 / 103) * 100);
int floor1 = (int) Math.floor(floor / 600);
System.out.println(floor + " " + floor1);
}
private static String execCommand(String[] command) {

14
modules/storage-module/storage-module-common/src/main/java/top/jpom/db/IStorageService.java
View File

@ -82,7 +82,7 @@ public interface IStorageService extends AutoCloseable, IMode {
* @throws Exception 异常
*/
default boolean hasDbData() throws Exception {
throw new IllegalArgumentException("没有实现改功能");
throw new UnsupportedOperationException("没有实现该功能");
}
/**
@ -92,7 +92,7 @@ public interface IStorageService extends AutoCloseable, IMode {
* @throws Exception 异常
*/
default File recoverDb() throws Exception {
throw new IllegalArgumentException("没有实现改功能");
throw new UnsupportedOperationException("没有实现该功能");
}
/**
@ -102,7 +102,7 @@ public interface IStorageService extends AutoCloseable, IMode {
* @throws Exception 异常
*/
default String deleteDbFiles() throws Exception {
throw new IllegalArgumentException("没有实现改功能");
throw new UnsupportedOperationException("没有实现该功能");
}
/**
@ -111,7 +111,7 @@ public interface IStorageService extends AutoCloseable, IMode {
* @param sqlFile sql 文件
*/
default void transformSql(File sqlFile) {
throw new IllegalArgumentException("没有实现改功能");
throw new UnsupportedOperationException("没有实现该功能");
}
/**
@ -125,7 +125,7 @@ public interface IStorageService extends AutoCloseable, IMode {
if (recoverSqlFile == null) {
return;
}
throw new IllegalArgumentException("没有实现改功能");
throw new UnsupportedOperationException("没有实现该功能");
}
/**
@ -137,7 +137,7 @@ public interface IStorageService extends AutoCloseable, IMode {
* @throws SQLException sql 异常
*/
default void alterUser(String oldUes, String newUse, String newPwd) throws SQLException {
throw new IllegalArgumentException("没有实现改功能");
throw new UnsupportedOperationException("没有实现该功能");
}
/**
@ -151,7 +151,7 @@ public interface IStorageService extends AutoCloseable, IMode {
* @throws Exception 异常
*/
default void backupSql(String url, String user, String pass, String backupSqlPath, List<String> tableName) throws Exception {
throw new IllegalArgumentException("没有实现改功能");
throw new UnsupportedOperationException("没有实现该功能");
}
/**

11
modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/GitUtil.java
View File

@ -41,7 +41,10 @@ import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.NoHeadException;
import org.eclipse.jgit.api.errors.TransportException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.*;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.*;
@ -137,7 +140,8 @@ public class GitUtil {
}
CloneCommand cloneCommand = Git.cloneRepository();
if (printWriter != null) {
cloneCommand.setProgressMonitor(new SmallTextProgressMonitor(printWriter));
Integer progressRatio = (Integer) parameter.get("reduceProgressRatio");
cloneCommand.setProgressMonitor(new SmallTextProgressMonitor(printWriter, progressRatio));
}
if (branchName != null) {
cloneCommand.setBranch(Constants.R_HEADS + branchName);
@ -375,7 +379,8 @@ public class GitUtil {
* @throws Exception 异常
*/
private static PullResult pull(Git git, Map<String, Object> parameter, String branchName, PrintWriter printWriter) throws Exception {
SmallTextProgressMonitor progressMonitor = new SmallTextProgressMonitor(printWriter);
Integer progressRatio = (Integer) parameter.get("reduceProgressRatio");
SmallTextProgressMonitor progressMonitor = new SmallTextProgressMonitor(printWriter, progressRatio);
// 放弃本地修改
git.checkout().setName(branchName).setForced(true).setProgressMonitor(progressMonitor).call();

16
modules/sub-plugin/git-clone/src/main/java/io/jpom/plugin/SmallTextProgressMonitor.java
View File

@ -25,6 +25,8 @@ package io.jpom.plugin;
import org.eclipse.jgit.lib.TextProgressMonitor;
import java.io.Writer;
import java.util.HashSet;
import java.util.Set;
/**
* 少量的输出，只在进度 x%5=0 的时候输出
@ -34,13 +36,23 @@ import java.io.Writer;
*/
public class SmallTextProgressMonitor extends TextProgressMonitor {
public SmallTextProgressMonitor(Writer out) {
private final Set<Integer> progressRangeList;
private final int reduceProgressRatio;
/**
* @param out 输出流
* @param reduceProgressRatio 压缩折叠显示进度比例 范围 1-100
*/
public SmallTextProgressMonitor(Writer out, int reduceProgressRatio) {
super(out);
this.reduceProgressRatio = reduceProgressRatio;
this.progressRangeList = new HashSet<>((int) Math.floor((float) 100 / reduceProgressRatio));
}
@Override
protected void onUpdate(String taskName, int cmp, int totalWork, int pcnt) {
if (pcnt % 5 == 0) {
int progressRange = (int) Math.floor((float) pcnt / this.reduceProgressRatio);
if (progressRangeList.add(progressRange)) {
super.onUpdate(taskName, cmp, totalWork, pcnt);
}
}