diff --git a/modules/common/src/test/java/i8n/BaidubceRpcTexttransTest.java b/modules/common/src/test/java/i8n/BaiduBceRpcTexttransTest.java similarity index 60% rename from modules/common/src/test/java/i8n/BaidubceRpcTexttransTest.java rename to modules/common/src/test/java/i8n/BaiduBceRpcTexttransTest.java index ffb014c3a..6cff920dc 100644 --- a/modules/common/src/test/java/i8n/BaidubceRpcTexttransTest.java +++ b/modules/common/src/test/java/i8n/BaiduBceRpcTexttransTest.java @@ -5,27 +5,62 @@ import cn.hutool.core.date.SystemClock; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.io.resource.ResourceUtil; +import cn.hutool.core.map.MapUtil; import cn.hutool.core.net.url.UrlBuilder; +import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.http.*; import cn.hutool.system.SystemUtil; -import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; import org.junit.Assert; import org.junit.Test; import java.io.File; import java.io.InputStream; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author bwcx_jzy1 * @since 2024/6/12 */ -public class BaidubceRpcTexttransTest { +public class BaiduBceRpcTexttransTest { @Test - public void doTranslate() { + public void testTranslate() { + ArrayList strings = CollUtil.newArrayList("请输入正确的验证码", "请传入 body 参数", "开始准备项目重启：{} {}"); + JSONObject jsonObject = this.doTranslate(strings); + System.out.println(jsonObject); + } + + private boolean checkHasI18nKey(JSONObject jsonObject) { + Set keyed = jsonObject.keySet(); + for (String s : keyed) { + if (StrUtil.startWith(s, "i18n.")) { + // 提前失败 或者翻译失败 + //System.err.println("翻译失败或者提取失败," + s + "=" + jsonObject.get(s)); + return true; + } + } + return false; + } + + public JSONObject doTranslate(Collection words) { + while (true) { + JSONObject jsonObject = this.doTranslate2(words); + if (checkHasI18nKey(jsonObject)) { + System.err.println("翻译失败或者提取失败,自动重试," + jsonObject); + } else { + return jsonObject; + } + } + } + + + private JSONObject doTranslate2(Collection words) { String token = this.getToken(); UrlBuilder urlBuilder = UrlBuilder.of("https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions"); urlBuilder.addQuery("access_token", token); @@ -40,14 +75,17 @@ public class BaidubceRpcTexttransTest { // InputStream inputStream = ResourceUtil.getStream("baidubce_translate.txt"); String string = IoUtil.readUtf8(inputStream); - HashMap map = new HashMap<>(); - JSONArray jsonArray = new JSONArray(); - jsonArray.add("测试变量"); - jsonArray.add("连接关闭 {} {}"); - jsonArray.add("清除临时文件失败,请手动清理："); - map.put("REQUEST_STR", jsonArray.toString()); - string = StrUtil.format(string, map); - System.out.println(string); + // + JSONObject from = new JSONObject(); + for (String value : words) { + String key; + do { + key = StrUtil.format("i18n.{}", RandomUtil.randomStringUpper(6)); + } while (from.containsKey(key)); + from.put(key, value); + } + string = StrUtil.format(string, MapUtil.of("REQUEST_STR", from.toString())); + //System.out.println(string); message.put("content", string); jsonObject.put("messages", CollUtil.newArrayList(message)); // @@ -55,9 +93,26 @@ public class BaidubceRpcTexttransTest { String result = httpRequest.thenFunction(httpResponse -> { String body = httpResponse.body(); JSONObject jsonObject1 = JSONObject.parseObject(body); + if (jsonObject1.getIntValue("error_code") != 0) { + Assert.fail(jsonObject1.getString("error_msg")); + } return jsonObject1.getString("result"); }); - System.out.println(result); + String patternString = "(?s)```json\\s*([^`]*?)\\s*```"; + Pattern pattern = Pattern.compile(patternString); + Matcher matcher = pattern.matcher(result); + // + JSONObject jsonObject1 = null; + while (matcher.find()) { + //System.out.println(result); + String jsonContent = matcher.group(1); + jsonObject1 = JSONObject.parseObject(jsonContent); + if (!this.checkHasI18nKey(jsonObject1)) { + return jsonObject1; + } + } + Assert.assertNotNull("翻译失败或者提取失败", jsonObject1); + return jsonObject1; } private String getToken() { @@ -77,9 +132,9 @@ public class BaidubceRpcTexttransTest { } /** - * https://cloud.baidu.com/doc/WENXINWORKSHOP/s/7lpch74jm + * https://cloud.baidu.com/doc/WENXINWORKSHOP/s/7lpch74jm * - * @return + * @return token */ private JSONObject doTokenByApi(File file) { String bceCi = SystemUtil.get("JPOM_TRANSLATE_BAIDUBCE_CI", StrUtil.EMPTY); diff --git a/modules/common/src/test/java/i8n/ExtractI18nTest.java b/modules/common/src/test/java/i8n/ExtractI18nTest.java index 747cc31b5..36182695c 100644 --- a/modules/common/src/test/java/i8n/ExtractI18nTest.java +++ b/modules/common/src/test/java/i8n/ExtractI18nTest.java @@ -3,10 +3,14 @@ package i8n; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.CharsetUtil; -import cn.hutool.core.util.RandomUtil; +import cn.hutool.core.util.PageUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.crypto.SecureUtil; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; import lombok.Lombok; -import lombok.SneakyThrows; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.io.BufferedReader; @@ -20,6 +24,7 @@ import java.util.*; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * 提交代码中的中文并生成随机 key 转存到 properties @@ -30,10 +35,6 @@ import java.util.regex.Pattern; * @since 2024/6/11 */ public class ExtractI18nTest { - /** - * 中文字符串 - */ - private Collection wordsSet = new LinkedHashSet<>(); /** * 中文对应的 key map *

@@ -49,6 +50,8 @@ public class ExtractI18nTest { * 项目根路径 */ private File rootFile; + + private final Properties zhProperties = new Properties(); /** * 匹配中文字符的正则表达式 */ @@ -81,38 +84,125 @@ public class ExtractI18nTest { "@ValidatorItem", "nameKey = \"" }; - - @Test - @SneakyThrows - public void extract() { + @Before + public void before() throws Exception { File file = new File(""); String rootPath = file.getAbsolutePath(); - rootFile = file = new File(rootPath).getParentFile(); + rootFile = new File(rootPath).getParentFile(); + // + // 中文资源文件存储路径 + File zhPropertiesFile = FileUtil.file(rootFile, "common/src/main/resources/i18n/messages_zh_CN.properties"); + Charset charset = CharsetUtil.CHARSET_UTF_8; + try (BufferedReader inputStream = FileUtil.getReader(zhPropertiesFile, charset)) { + zhProperties.load(inputStream); + } + } + + @Test + public void extract() { // 删除临时文件 FileUtil.del(FileUtil.file(rootFile, "i18n-temp")); + // 中文字符串 + Set wordsSet = new LinkedHashSet<>(); // 提取中文 - walkFile(file, file1 -> { + walkFile(rootFile, file1 -> { try { for (Pattern chinesePattern : chinesePatterns) { verifyDuplicates(file1, chinesePattern); - extractFile(file1, chinesePattern); + extractFile(file1, chinesePattern, wordsSet); } } catch (Exception e) { throw Lombok.sneakyThrow(e); } }); - // 生成 key - generateKey(file); + // 检查去除前后空格后是否重复 + Map collect = wordsSet.stream() + .map(StrUtil::trim) + .collect(Collectors.groupingBy(e -> e, Collectors.counting())); + for (Map.Entry entry : collect.entrySet()) { + long value = entry.getValue(); + Assert.assertEquals("[" + entry.getKey() + "]出现去重空格后重复", 1L, value); + } + // 语意化中文存储为 key + Collection wordsSetSort = CollUtil.sort(wordsSet, String::compareTo); + int pageSize = 50; + int total = CollUtil.size(wordsSet); + int page = PageUtil.totalPage(total, pageSize); + JSONObject allResult = new JSONObject(); + // + for (int i = PageUtil.getFirstPageNo(); i <= page; i++) { + int start = PageUtil.getStart(i, pageSize); + int end = PageUtil.getEnd(i, pageSize); + + List sub = CollUtil.sub(wordsSetSort, start, end); + while (true) out:{ + BaiduBceRpcTexttransTest bceRpcTexttrans = new BaiduBceRpcTexttransTest(); + JSONObject jsonObject = bceRpcTexttrans.doTranslate(sub); + System.out.println("翻译结果：" + jsonObject); + // 转换为可用 key + for (Map.Entry entry : jsonObject.entrySet()) { + String key = entry.getKey(); + String value = (String) entry.getValue(); + String originalValue = findOriginal(sub, value); + if (originalValue == null) { + System.err.println("翻译后的中文和翻译前的中文不一致（需要重试）：" + value); + break out; + } + String buildKey = this.buildKey(key, originalValue, allResult); + allResult.put(buildKey, originalValue); + + } + // 提前保存 + File wordsFile = FileUtil.file(rootFile, "common/src/main/resources/i18n/words.json"); + FileUtil.writeString(JSONArray.toJSONString(allResult), wordsFile, StandardCharsets.UTF_8); + break; + } + } + } + + private String buildKey(String key, String value, JSONObject jsonObject) { + int md5IdLen = 4; + while (true) { + String md5 = SecureUtil.md5(value); + Assert.assertTrue("截取中文 md5 key 超范围：" + value, md5.length() >= md5IdLen); + md5 = md5.substring(0, md5IdLen); + String newKey = StrUtil.format("i18n.{}.{}", StrUtil.toUnderlineCase(key), md5); + if (jsonObject.containsKey(newKey)) { + md5IdLen += 2; + continue; + } + return newKey; + } + } + + /** + * 找到原始的中文字符串（大模型处理后面前后空格可能不存在） + * + * @param list list + * @param value value + * @return 原始的中文字符串 ，null 不存在 + */ + private String findOriginal(List list, String value) { + for (String s : list) { + if (StrUtil.equals(s, value) || StrUtil.equals(StrUtil.trim(s), value)) { + return value; + } + } + return null; + } + + @Test + public void replace() { // 替换中文 -// walkFile(file, file1 -> { -// try { -// for (Pattern chinesePattern : chinesePatterns) { -// replaceQuotedChineseInFile(file1, chinesePattern); -// } -// } catch (Exception e) { -// throw Lombok.sneakyThrow(e); -// } -// }); + walkFile(rootFile, file1 -> { + try { + for (Pattern chinesePattern : chinesePatterns) { + replaceQuotedChineseInFile(file1, chinesePattern); + } + } catch (Exception e) { + throw Lombok.sneakyThrow(e); + } + }); } /** @@ -144,64 +234,57 @@ public class ExtractI18nTest { */ private void generateKey(File file) throws IOException { // /Users/user/IdeaProjects/Jpom/jpom-parent/modules/.DS_Store - // 中文资源文件存储路径 - File zhPropertiesFile = FileUtil.file(file, "common/src/main/resources/i18n/messages_zh_CN.properties"); - Properties zhProperties = new Properties(); - Charset charset = CharsetUtil.CHARSET_UTF_8; - try (BufferedReader inputStream = FileUtil.getReader(zhPropertiesFile, charset)) { - zhProperties.load(inputStream); - } - Collection oldKeys = zhProperties.keySet(); - Collection linkUsed = new LinkedHashSet<>(); - wordsSet = CollUtil.sort(wordsSet, String::compareTo); - wordsSet.forEach(s -> { - // 根据中文反查 key - String key = null; - for (Map.Entry entry : zhProperties.entrySet()) { - if (StrUtil.equals(StrUtil.toStringOrNull(entry.getValue()), s)) { - key = (String) entry.getKey(); - break; - } - } - if (key == null) { - do { - key = StrUtil.format("key.{}", RandomUtil.randomStringUpper(6)); - } while (zhProperties.containsKey(key)); - System.out.println("生成新的 key:" + key); - zhProperties.put(key, s); - } - linkUsed.add(key); - }); - // 删除不存在的 - int beforeSize = oldKeys.size(); - oldKeys.removeIf(next -> { - // - boolean b = !linkUsed.contains(next) && !useKeys.contains(next); - if (b) { - System.out.println("删除 key：" + next); - } - return b; - }); - int afterSize = oldKeys.size(); - if (beforeSize != afterSize) { - System.out.println(beforeSize + " " + afterSize); - } - - for (Object useKey : useKeys) { - if (zhProperties.containsKey(useKey)) { - continue; - } - System.out.println("存在未关联的key:" + useKey); - } - - try (BufferedWriter writer = FileUtil.getWriter(zhPropertiesFile, charset, false)) { - zhProperties.store(writer, "i18n zh"); - } - System.out.println(zhProperties.size()); - - for (Map.Entry entry : zhProperties.entrySet()) { - chineseMap.put(StrUtil.toStringOrNull(entry.getValue()), StrUtil.toStringOrNull(entry.getKey())); - } +// Collection oldKeys = zhProperties.keySet(); +// Collection linkUsed = new LinkedHashSet<>(); +// wordsSet = CollUtil.sort(wordsSet, String::compareTo); +// wordsSet.forEach(s -> { +// // 根据中文反查 key +// String key = null; +// for (Map.Entry entry : zhProperties.entrySet()) { +// if (StrUtil.equals(StrUtil.toStringOrNull(entry.getValue()), s)) { +// key = (String) entry.getKey(); +// break; +// } +// } +// if (key == null) { +// do { +// key = StrUtil.format("i18n.{}", RandomUtil.randomStringUpper(6)); +// } while (zhProperties.containsKey(key)); +// System.out.println("生成新的 key:" + key); +// zhProperties.put(key, s); +// } +// linkUsed.add(key); +// }); +// // 删除不存在的 +// int beforeSize = oldKeys.size(); +// oldKeys.removeIf(next -> { +// // +// boolean b = !linkUsed.contains(next) && !useKeys.contains(next); +// if (b) { +// System.out.println("删除 key：" + next); +// } +// return b; +// }); +// int afterSize = oldKeys.size(); +// if (beforeSize != afterSize) { +// System.out.println(beforeSize + " " + afterSize); +// } +// +// for (Object useKey : useKeys) { +// if (zhProperties.containsKey(useKey)) { +// continue; +// } +// System.out.println("存在未关联的key:" + useKey); +// } +// +// try (BufferedWriter writer = FileUtil.getWriter(zhPropertiesFile, charset, false)) { +// zhProperties.store(writer, "i18n zh"); +// } +// System.out.println(zhProperties.size()); +// +// for (Map.Entry entry : zhProperties.entrySet()) { +// chineseMap.put(StrUtil.toStringOrNull(entry.getValue()), StrUtil.toStringOrNull(entry.getKey())); +// } } /** @@ -311,7 +394,7 @@ public class ExtractI18nTest { * @param pattern 匹配的正则 * @throws IOException io 异常 */ - private void extractFile(File file, Pattern pattern) throws Exception { + private void extractFile(File file, Pattern pattern, Collection wordsSet) throws Exception { try (BufferedReader reader = Files.newBufferedReader(file.toPath())) { String line; while ((line = reader.readLine()) != null) { diff --git a/modules/common/src/test/resources/baidubce_translate.txt b/modules/common/src/test/resources/baidubce_translate.txt index 4fb79403f..7cde72307 100644 --- a/modules/common/src/test/resources/baidubce_translate.txt +++ b/modules/common/src/test/resources/baidubce_translate.txt @@ -1,15 +1,15 @@ You are the service that converts a user request JSON into a new (user-expected) JSON object based on the following JavaScript-defined JSON object: ``` -// 将下面json中的 将`值`的含义进行转为语义化且简短的首字母为小写的小驼峰英文变量名替换无意义字符串 +// 将下面json中的 根据`值`的含义将 `key` 转为语义化且简短的首字母为小写的小驼峰英文变量名替换无意义字符串 // 此处进行替换,禁止出现 k1 k2 k3 -const json = [ -"string", -"string2" -] +const template = { + "key1": "string1", + "key2": "string2", +} ``` The following is a user request: ``` -const json ={REQUEST_STR} +const template = {REQUEST_STR} ``` diff --git a/modules/server/src/main/java/org/dromara/jpom/controller/system/WorkspaceController.java b/modules/server/src/main/java/org/dromara/jpom/controller/system/WorkspaceController.java index 17c8bdcb4..00591d2b9 100644 --- a/modules/server/src/main/java/org/dromara/jpom/controller/system/WorkspaceController.java +++ b/modules/server/src/main/java/org/dromara/jpom/controller/system/WorkspaceController.java @@ -263,7 +263,7 @@ public class WorkspaceController extends BaseServerController { systemParametersServer.delByKey(StrUtil.format("node_config_{}", id)); // 删除信息 workspaceService.delByKey(id); - return new JsonMessage<>(200, "删除成功 " + autoDelete); + return new JsonMessage<>(200, "删除成功:" + autoDelete); } /** diff --git a/modules/server/src/main/java/org/dromara/jpom/socket/handler/NodeUpdateHandler.java b/modules/server/src/main/java/org/dromara/jpom/socket/handler/NodeUpdateHandler.java index d7daed9ca..4b456b61e 100644 --- a/modules/server/src/main/java/org/dromara/jpom/socket/handler/NodeUpdateHandler.java +++ b/modules/server/src/main/java/org/dromara/jpom/socket/handler/NodeUpdateHandler.java @@ -235,7 +235,7 @@ public class NodeUpdateHandler extends BaseProxyHandler { } } catch (Exception e) { log.error("升级失败", e); - this.onError(session, "升级失败 " + e.getMessage()); + this.onError(session, "升级失败:" + e.getMessage()); } }