From f7e051bd27e3e343901f44f94c10c2564bd8184b Mon Sep 17 00:00:00 2001 From: bwcx_jzy Date: Sun, 6 Feb 2022 23:41:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=20tag=20=E9=80=89=E6=8B=A9,n?= =?UTF-8?q?ode=20=E6=94=AF=E6=8C=81=E5=88=A4=E6=96=AD=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E6=9E=B6=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/jpom/build/BuildExecuteService.java | 20 ++++--- .../main/java/io/jpom/build/DockerYmlDsl.java | 11 +++- .../service/docker/DockerInfoService.java | 26 +++++++++ .../service/h2db/BaseDbCommonService.java | 42 +++++++++++--- .../io/jpom/service/h2db/BaseDbService.java | 2 - .../src/test/java/io/jpom/DockerInfoTest.java | 6 ++ .../java/io/jpom/DefaultDockerPluginImpl.java | 56 ++++++++++++------- .../src/main/java/io/jpom/DockerUtil.java | 13 ++++- .../main/resources/ubuntu-latest/Dockerfile | 23 +------- .../src/main/resources/uses/node/install.sh | 18 +++++- web-vue/src/pages/build/list-info.vue | 4 +- 11 files changed, 154 insertions(+), 67 deletions(-) diff --git a/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java b/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java index 538363fbd..3ffa538dc 100644 --- a/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java +++ b/modules/server/src/main/java/io/jpom/build/BuildExecuteService.java @@ -507,7 +507,17 @@ public class BuildExecuteService { BuildInfoModel buildInfoModel = taskData.buildInfoModel; String script = buildInfoModel.getScript(); DockerYmlDsl dockerYmlDsl = DockerYmlDsl.build(script); - DockerInfoModel dockerInfoModel = buildExecuteService.dockerInfoService.queryByBean(DockerInfoModel.builder().status(1).build()); + String fromTag = dockerYmlDsl.getFromTag(); + + List dockerInfoModels; + if (StrUtil.isNotEmpty(fromTag)) { + // 根据 tag 查询 + dockerInfoModels = buildExecuteService.dockerInfoService.queryByTag(fromTag, 1); + } else { + DockerInfoModel.DockerInfoModelBuilder builder = DockerInfoModel.builder(); + dockerInfoModels = buildExecuteService.dockerInfoService.queryList(builder.status(1).build(), 1); + } + DockerInfoModel dockerInfoModel = CollUtil.getFirst(dockerInfoModels); Assert.notNull(dockerInfoModel, "没有可用的 docker server"); String workingDir = "/home/jpom/"; Map map = dockerInfoModel.toParameter(); @@ -519,9 +529,9 @@ public class BuildExecuteService { map.put("logFile", FileUtil.getAbsolutePath(logRecorder.getFile())); // List copy = ObjectUtil.defaultIfNull(dockerYmlDsl.getCopy(), new ArrayList<>()); - //new ArrayList<>(); copy.add(FileUtil.getAbsolutePath(this.gitFile) + StrUtil.COLON + workingDir + StrUtil.COLON + "true"); map.put("copy", copy); + map.put("binds", ObjectUtil.defaultIfNull(dockerYmlDsl.getBinds(), new ArrayList<>())); Map env = ObjectUtil.defaultIfNull(dockerYmlDsl.getEnv(), new HashMap<>()); env.put("JPOM_BUILD_ID", buildInfoModelId); @@ -532,12 +542,8 @@ public class BuildExecuteService { String resultDirFile = buildInfoModel.getResultDirFile(); String resultFile = FileUtil.normalize(workingDir + StrUtil.SLASH + resultDirFile); map.put("resultFile", resultFile); + // 产物输出目录 File toFile = BuildUtil.getHistoryPackageFile(buildInfoModelId, buildInfoModel.getBuildId(), resultDirFile); - - // 下载文件包含一级文件名 所以需要向上切换 - //int toParent = StrUtil.equals(workingDir, resultFile) ? 0 : 1; - //FileUtil.getParent(resultFileOut, 1); -// FileUtil.getParent(toFile, toParent) map.put("resultFileOut", FileUtil.getAbsolutePath(toFile)); IPlugin plugin = PluginFactory.getPlugin("docker-cli"); try { diff --git a/modules/server/src/main/java/io/jpom/build/DockerYmlDsl.java b/modules/server/src/main/java/io/jpom/build/DockerYmlDsl.java index d01ddd910..10eebda42 100644 --- a/modules/server/src/main/java/io/jpom/build/DockerYmlDsl.java +++ b/modules/server/src/main/java/io/jpom/build/DockerYmlDsl.java @@ -58,7 +58,10 @@ public class DockerYmlDsl extends BaseJsonModel { * 基础镜像 */ private String runsOn; - + /** + * 使用对应到 docker tag 构建 + */ + private String fromTag; /** * 构建步骤 */ @@ -77,7 +80,11 @@ public class DockerYmlDsl extends BaseJsonModel { * * if root directory is ignored */ private List copy; - + /** + * bind mounts 将宿主机上的任意位置的文件或者目录挂在到容器 （--mount type=bind,src=源目录,dst=目标目录） + * /host:/container:ro + */ + private List binds; /** * 环境变量 */ diff --git a/modules/server/src/main/java/io/jpom/service/docker/DockerInfoService.java b/modules/server/src/main/java/io/jpom/service/docker/DockerInfoService.java index d8e4509ba..79330e832 100644 --- a/modules/server/src/main/java/io/jpom/service/docker/DockerInfoService.java +++ b/modules/server/src/main/java/io/jpom/service/docker/DockerInfoService.java @@ -27,6 +27,7 @@ import cn.hutool.core.date.SystemClock; import cn.hutool.core.thread.ThreadUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.cron.task.Task; +import cn.hutool.db.sql.Condition; import com.alibaba.fastjson.JSONObject; import io.jpom.cron.CronUtils; import io.jpom.cron.IAsyncLoad; @@ -111,6 +112,31 @@ public class DockerInfoService extends BaseWorkspaceService imp super.update(dockerInfoModel); } + /** + * 根据 tag 查询 容器 + * + * @param tag tag + * @param status 状态 + * @return list + */ + public List queryByTag(String tag, Integer status) { + Condition condition = new Condition(" instr(tags,'" + tag + "')", ""); + condition.setPlaceHolder(false); + condition.setOperator(""); + if (status == null) { + return super.findByCondition(condition); + } else { + Condition statusCondition = new Condition("status", status); + return super.findByCondition(condition, statusCondition); + } + } + + /** + * 根据 tag 查询 容器 + * + * @param tag tag + * @return list + */ public List queryByTag(String tag) { String sql = StrUtil.format("SELECT * FROM {} where instr(tags,?);", super.getTableName()); return super.queryList(sql, StrUtil.wrap(tag, StrUtil.COLON, StrUtil.COLON)); diff --git a/modules/server/src/main/java/io/jpom/service/h2db/BaseDbCommonService.java b/modules/server/src/main/java/io/jpom/service/h2db/BaseDbCommonService.java index fce428e25..eddf1147c 100644 --- a/modules/server/src/main/java/io/jpom/service/h2db/BaseDbCommonService.java +++ b/modules/server/src/main/java/io/jpom/service/h2db/BaseDbCommonService.java @@ -33,6 +33,7 @@ import cn.hutool.db.Db; import cn.hutool.db.Entity; import cn.hutool.db.Page; import cn.hutool.db.PageResult; +import cn.hutool.db.sql.Condition; import cn.hutool.db.sql.Order; import cn.jiangzeyin.common.DefaultSystemLog; import io.jpom.model.PageResultDto; @@ -285,7 +286,7 @@ public abstract class BaseDbCommonService { * @param 乏型 * @return data */ - protected R entityToBean(Entity entity, Class rClass) { + protected R entityToBean(Entity entity, Class rClass) { if (entity == null) { return null; } @@ -452,6 +453,28 @@ public abstract class BaseDbCommonService { } } + /** + * 查询列表 + * + * @param wheres 条件 + * @return List + */ + public List findByCondition(Condition... wheres) { + if (!DbConfig.getInstance().isInit()) { + // ignore + DefaultSystemLog.getLog().error("The database is not initialized, this execution will be ignored"); + return null; + } + Db db = Db.use(); + db.setWrapper((Character) null); + try { + List entities = db.findBy(getTableName(), wheres); + return this.entityToBeanList(entities); + } catch (Exception e) { + throw warpException(e); + } + } + /** * 查询列表 * @@ -567,14 +590,15 @@ public abstract class BaseDbCommonService { */ public List queryList(String sql, Object... params) { List query = this.query(sql, params); - if (query != null) { - return query.stream().map((entity -> { - T entityToBean = this.entityToBean(entity, this.tClass); - this.fillSelectResult(entityToBean); - return entityToBean; - })).collect(Collectors.toList()); - } - return null; + return this.entityToBeanList(query); + // if (query != null) { + // return query.stream().map((entity -> { + // T entityToBean = this.entityToBean(entity, this.tClass); + // this.fillSelectResult(entityToBean); + // return entityToBean; + // })).collect(Collectors.toList()); + // } + // return null; } /** diff --git a/modules/server/src/main/java/io/jpom/service/h2db/BaseDbService.java b/modules/server/src/main/java/io/jpom/service/h2db/BaseDbService.java index f5902fee4..a8fe9f4de 100644 --- a/modules/server/src/main/java/io/jpom/service/h2db/BaseDbService.java +++ b/modules/server/src/main/java/io/jpom/service/h2db/BaseDbService.java @@ -189,7 +189,6 @@ public abstract class BaseDbService extends BaseDbCommonS return this.updateById(t); } - public List list() { return super.listByBean(ReflectUtil.newInstance(this.tClass)); } @@ -198,7 +197,6 @@ public abstract class BaseDbService extends BaseDbCommonS return super.count(Entity.create()); } - /** * 通用的分页查询, 使用该方法查询，数据库表字段不能保护 "page", "limit", "order_field", "order", "total" *

diff --git a/modules/server/src/test/java/io/jpom/DockerInfoTest.java b/modules/server/src/test/java/io/jpom/DockerInfoTest.java index df19c14b6..cfe57ce5c 100644 --- a/modules/server/src/test/java/io/jpom/DockerInfoTest.java +++ b/modules/server/src/test/java/io/jpom/DockerInfoTest.java @@ -48,4 +48,10 @@ public class DockerInfoTest { List models = dockerInfoService.queryByTag("sdfsd"); System.out.println(models); } + + @Test + public void testQueryTag2() { + List models = dockerInfoService.queryByTag("sdfsd", 1); + System.out.println(models); + } } diff --git a/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DefaultDockerPluginImpl.java b/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DefaultDockerPluginImpl.java index d97a203b6..2ac559d84 100644 --- a/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DefaultDockerPluginImpl.java +++ b/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DefaultDockerPluginImpl.java @@ -27,6 +27,7 @@ import cn.hutool.core.convert.Convert; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.util.ArrayUtil; +import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.crypto.SecureUtil; import cn.hutool.crypto.digest.MD5; @@ -45,13 +46,8 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import java.io.File; import java.io.FileOutputStream; -import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -82,11 +78,12 @@ public class DefaultDockerPluginImpl implements IDefaultPlugin { } } - - public void build(Map parameter) { + private void build(Map parameter) { DockerClient dockerClient = DockerUtil.build(parameter, 10); String logFile = (String) parameter.get("logFile"); File tempDir = (File) parameter.get("tempDir"); + // 生成临时目录 + tempDir = FileUtil.file(tempDir, "docker-temp", IdUtil.fastSimpleUUID()); LogRecorder logRecorder = LogRecorder.builder().filePath(logFile).build(); List copy = (List) parameter.get("copy"); String resultFile = (String) parameter.get("resultFile"); @@ -105,12 +102,11 @@ public class DefaultDockerPluginImpl implements IDefaultPlugin { mounts.addAll(this.checkDependPlugin(dockerClient, image, steps, tempDir, logRecorder)); String containerId = this.buildNewContainer(dockerClient, parameter, mounts); try { - String buildShell = generateBuildShell(steps, buildId); - Path tempDirectory = Files.createTempDirectory("build"); - Path tempFile = Files.createFile(Paths.get(tempDirectory.toString() + File.separator + "build.sh")); - Files.write(tempFile, buildShell.getBytes(), StandardOpenOption.WRITE); + String buildShell = this.generateBuildShell(steps, buildId); + File tempFile = DockerUtil.createTemp("build.sh", tempDir); + FileUtil.writeUtf8String(buildShell, tempFile); dockerClient.copyArchiveToContainerCmd(containerId) - .withHostResource(tempFile.toAbsolutePath().toString()) + .withHostResource(tempFile.getAbsolutePath()) .withRemotePath("/tmp/") .exec(); this.copyArchiveToContainerCmd(dockerClient, containerId, copy); @@ -127,10 +123,10 @@ public class DefaultDockerPluginImpl implements IDefaultPlugin { this.waitContainerCmd(dockerClient, containerId, logRecorder); // 获取容器执行结果文件 this.copyArchiveFromContainerCmd(dockerClient, containerId, logRecorder, resultFile, resultFileOut); - } catch (IOException e) { - logRecorder.error("创建临时文件异常:", e); } finally { this.removeContainerCmd(dockerClient, containerId); + // 删除临时目录 + FileUtil.del(tempDir); } } @@ -231,7 +227,12 @@ public class DefaultDockerPluginImpl implements IDefaultPlugin { } dockerClient.buildImageCmd(file) .withTags(tags) - .exec(new ResultCallback.Adapter<>()).awaitCompletion(); + .exec(new ResultCallback.Adapter() { + @Override + public void onNext(BuildResponseItem object) { + logRecorder.append(object.getStream()); + } + }).awaitCompletion(); } catch (InterruptedException ex) { logRecorder.error("构建 runsOn 镜像被中断", ex); } @@ -384,22 +385,37 @@ public class DefaultDockerPluginImpl implements IDefaultPlugin { } } - + /** + * 构建一个执行 镜像 + * + * @param dockerClient docker 客户端连接 + * @param parameter 参数 + * @param mounts 挂载目录 + * @return 容器ID + */ private String buildNewContainer(DockerClient dockerClient, Map parameter, List mounts) { String image = (String) parameter.get("image"); String workingDir = (String) parameter.get("workingDir"); String dockerName = (String) parameter.get("dockerName"); + List binds = (List) parameter.get("binds"); Map env = (Map) parameter.get("env"); CreateContainerCmd containerCmd = dockerClient.createContainerCmd(image); containerCmd.withName(dockerName).withWorkingDir(workingDir); + // + List bindList = new ArrayList<>(); + if (CollUtil.isNotEmpty(binds)) { + bindList = binds.stream().map(Bind::parse).collect(Collectors.toList()); + } HostConfig hostConfig = HostConfig.newHostConfig() - .withMounts(mounts); + .withMounts(mounts).withBinds(bindList); containerCmd.withHostConfig(hostConfig); - - + // 环境变量 if (env != null) { - List envList = env.entrySet().stream().map(stringStringEntry -> StrUtil.format("{}={}", stringStringEntry.getKey(), stringStringEntry.getValue())).collect(Collectors.toList()); + List envList = env.entrySet() + .stream() + .map(entry -> StrUtil.format("{}={}", entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); containerCmd.withEnv(envList); } // 如果容器已经存在则先删除 diff --git a/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DockerUtil.java b/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DockerUtil.java index 6912d65f6..a799179e7 100644 --- a/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DockerUtil.java +++ b/modules/sub-plugin/docker-cli/src/main/java/io/jpom/DockerUtil.java @@ -76,6 +76,17 @@ public class DockerUtil { return DockerClientImpl.getInstance(config, httpClient); } + /** + * 临时文件目录 + * + * @param name 文件名 + * @param tempDir 临时文件目录 + * @return temp + */ + public static File createTemp(String name, File tempDir) { + return FileUtil.file(tempDir, name); + } + /** * 转化文件 * @@ -87,7 +98,7 @@ public class DockerUtil { try { Resource resourceObj = ResourceUtil.getResourceObj(name); InputStream stream = resourceObj.getStream(); - File tempFile = FileUtil.file(tempDir, "docker-temp", name); + File tempFile = DockerUtil.createTemp(name, tempDir); FileUtil.writeFromStream(stream, tempFile); return tempFile; } catch (Exception e) { diff --git a/modules/sub-plugin/docker-cli/src/main/resources/ubuntu-latest/Dockerfile b/modules/sub-plugin/docker-cli/src/main/resources/ubuntu-latest/Dockerfile index b00d7d802..9124fe2f1 100644 --- a/modules/sub-plugin/docker-cli/src/main/resources/ubuntu-latest/Dockerfile +++ b/modules/sub-plugin/docker-cli/src/main/resources/ubuntu-latest/Dockerfile @@ -20,31 +20,10 @@ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -# -# The MIT License (MIT) -# -# Copyright (c) 2019 Code Technology Studio -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -# the Software, and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# FROM ubuntu:latest ENV DEBIAN_FRONTEND=noninteractive +ENV LANG C.UTF-8 RUN sed -i.bak 's/archive.ubuntu.com/mirror.nju.edu.cn/' /etc/apt/sources.list \ && sed -i 's/security.ubuntu.com/mirror.nju.edu.cn/' /etc/apt/sources.list \ && apt-get update \ diff --git a/modules/sub-plugin/docker-cli/src/main/resources/uses/node/install.sh b/modules/sub-plugin/docker-cli/src/main/resources/uses/node/install.sh index 683a35185..3d00e4250 100644 --- a/modules/sub-plugin/docker-cli/src/main/resources/uses/node/install.sh +++ b/modules/sub-plugin/docker-cli/src/main/resources/uses/node/install.sh @@ -7,7 +7,19 @@ else exit 1 fi ARCH=`uname -m` +case "${ARCH}" in + aarch64|arm64) + BINARY_ARCH='arm64'; + ;; + amd64|x86_64) + BINARY_ARCH='x64'; + ;; + *) + echo "Unsupported arch: ${ARCH}"; + exit 1; + ;; +esac; cd /tmp -wget https://registry.npmmirror.com/-/binary/node/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${ARCH}.tar.gz -tar -zxf node-v${NODE_VERSION}-linux-${ARCH}.tar.gz -cp -r node-v${NODE_VERSION}-linux-${ARCH}/* /opt/node/ +wget https://registry.npmmirror.com/-/binary/node/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${BINARY_ARCH}.tar.gz +tar -zxf node-v${NODE_VERSION}-linux-${BINARY_ARCH}.tar.gz +cp -r node-v${NODE_VERSION}-linux-${BINARY_ARCH}/* /opt/node/ diff --git a/web-vue/src/pages/build/list-info.vue b/web-vue/src/pages/build/list-info.vue index 63ffa8176..71cc2d564 100644 --- a/web-vue/src/pages/build/list-info.vue +++ b/web-vue/src/pages/build/list-info.vue @@ -678,7 +678,9 @@ export default { " path: /root/.m2\n" + " - run: npm config set registry https://registry.npmmirror.com\n" + " - run: cd ${JPOM_WORKING_DIR}/web-vue && npm i && npm run build\n" + - " - run: cd ${JPOM_WORKING_DIR} && mvn package -s script/settings.xml", + " - run: cd ${JPOM_WORKING_DIR} && mvn package -s script/settings.xml\n" + + "# binds:\n" + + "# - /Users/user/.m2/settings.xml:/root/.m2/\n", }; }, computed: {