From eda370e4853a464c9d718454c41b23d08d54bd8d Mon Sep 17 00:00:00 2001 From: imalasong <2879499479@qq.com> Date: Fri, 14 Jul 2023 21:52:53 +0800 Subject: [PATCH 1/5] feat: ignore file --- Dashboard/.gitignore | 29 +++++++++++++++++++++++++++++ client/.gitignore | 29 +++++++++++++++++++++++++++++ clientlog4j/.gitignore | 29 +++++++++++++++++++++++++++++ clientlog4j2/.gitignore | 29 +++++++++++++++++++++++++++++ clientlogback/.gitignore | 29 +++++++++++++++++++++++++++++ common/.gitignore | 29 +++++++++++++++++++++++++++++ config/.gitignore | 29 +++++++++++++++++++++++++++++ config/config-apollo/.gitignore | 29 +++++++++++++++++++++++++++++ config/config-core/.gitignore | 29 +++++++++++++++++++++++++++++ config/config-etcd/.gitignore | 29 +++++++++++++++++++++++++++++ config/config-nacos/.gitignore | 29 +++++++++++++++++++++++++++++ config/config-zk/.gitignore | 29 +++++++++++++++++++++++++++++ example/.gitignore | 29 +++++++++++++++++++++++++++++ worker/.gitignore | 29 +++++++++++++++++++++++++++++ 14 files changed, 406 insertions(+) create mode 100644 Dashboard/.gitignore create mode 100644 client/.gitignore create mode 100644 clientlog4j/.gitignore create mode 100644 clientlog4j2/.gitignore create mode 100644 clientlogback/.gitignore create mode 100644 common/.gitignore create mode 100644 config/.gitignore create mode 100644 config/config-apollo/.gitignore create mode 100644 config/config-core/.gitignore create mode 100644 config/config-etcd/.gitignore create mode 100644 config/config-nacos/.gitignore create mode 100644 config/config-zk/.gitignore create mode 100644 example/.gitignore create mode 100644 worker/.gitignore diff --git a/Dashboard/.gitignore b/Dashboard/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/Dashboard/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/client/.gitignore b/client/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/client/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/clientlog4j/.gitignore b/clientlog4j/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/clientlog4j/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/clientlog4j2/.gitignore b/clientlog4j2/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/clientlog4j2/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/clientlogback/.gitignore b/clientlogback/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/clientlogback/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/common/.gitignore b/common/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/common/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/config/.gitignore b/config/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/config/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/config/config-apollo/.gitignore b/config/config-apollo/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/config/config-apollo/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/config/config-core/.gitignore b/config/config-core/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/config/config-core/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/config/config-etcd/.gitignore b/config/config-etcd/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/config/config-etcd/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/config/config-nacos/.gitignore b/config/config-nacos/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/config/config-nacos/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/config/config-zk/.gitignore b/config/config-zk/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/config/config-zk/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/worker/.gitignore b/worker/.gitignore new file mode 100644 index 0000000..153c933 --- /dev/null +++ b/worker/.gitignore @@ -0,0 +1,29 @@ +HELP.md +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ From 85770c6597ce5efbda89da0e1fcb1e48e02267c9 Mon Sep 17 00:00:00 2001 From: imalasong <2879499479@qq.com> Date: Fri, 14 Jul 2023 21:53:07 +0800 Subject: [PATCH 2/5] feat: ignore file --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 153c933..7c96042 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ HELP.md ### VS Code ### .vscode/ + +/log/ \ No newline at end of file From 44dda9c74e564245b86cf51c254cd6ce4a53e0ad Mon Sep 17 00:00:00 2001 From: imalasong <2879499479@qq.com> Date: Sat, 15 Jul 2023 17:13:48 +0800 Subject: [PATCH 3/5] =?UTF-8?q?feat:=E8=AE=B0=E5=BD=95=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E4=BF=A1=E6=81=AF=EF=BC=9Brefactor:=E9=87=8D?= =?UTF-8?q?=E6=9E=84=E4=BC=A0=E8=BE=93=E5=AF=B9=E8=B1=A1(map->obj)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/dashboard/service/TracerService.java | 34 ++--- .../resources/templates/tracer/index.html | 5 +- .../jlog/client/filter/HttpFilter.java | 114 ++++++-------- .../platform/jlog/client/udp/UdpSender.java | 36 ++--- .../jlog/common/constant/LogTypeEnum.java | 15 ++ .../jlog/common/model/RunLogMessage.java | 15 ++ .../jlog/common/model/TracerBean.java | 143 ++++++++++++------ .../jlog/common/model/TracerData.java | 33 +++- .../jlog/common/utils/StringUtils.java | 42 +++++ .../jlog/clientdemo/web/TestController.java | 2 +- .../jlog/worker/disruptor/TracerConsumer.java | 80 ++++------ 11 files changed, 314 insertions(+), 205 deletions(-) create mode 100644 common/src/main/java/com/jd/platform/jlog/common/constant/LogTypeEnum.java create mode 100644 common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java diff --git a/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java b/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java index 9b8f12d..f1d713b 100644 --- a/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java +++ b/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java @@ -90,16 +90,16 @@ public class TracerService { */ public Map findOne(String tracerId, String uid, String beginTime, String endTime) { List causes = new ArrayList<>(1); - if (!StringUtils.isEmpty(uid)) { - DbUtils.addEqualWhereCause(causes, "uid", uid); - } - //查单条也要带着时间 - if (!StringUtils.isEmpty(beginTime)) { - DbUtils.addGeWhereCause(causes, "createTime", beginTime); - } - if (!StringUtils.isEmpty(endTime)) { - DbUtils.addLeWhereCause(causes, "createTime", endTime); - } +// if (!StringUtils.isEmpty(uid)) { +// DbUtils.addEqualWhereCause(causes, "uid", uid); +// } +// //查单条也要带着时间 +// if (!StringUtils.isEmpty(beginTime)) { +// DbUtils.addGeWhereCause(causes, "createTime", beginTime); +// } +// if (!StringUtils.isEmpty(endTime)) { +// DbUtils.addLeWhereCause(causes, "createTime", endTime); +// } DbUtils.addEqualWhereCause(causes, "tracerId", tracerId); @@ -123,13 +123,13 @@ public class TracerService { TracerListVO tracerListVO = new TracerListVO(); List causes = new ArrayList<>(1); - if (!StringUtils.isEmpty(createTime)) { - String beginTime = DateUtils.addAndSubtractTime(createTime, -5000L); - String endTime = DateUtils.addAndSubtractTime(createTime, 5000L); - //查单条也要带着时间 - DbUtils.addGeWhereCause(causes, "createTime", beginTime); - DbUtils.addLeWhereCause(causes, "createTime", endTime); - } +// if (!StringUtils.isEmpty(createTime)) { +// String beginTime = DateUtils.addAndSubtractTime(createTime, -5000L); +// String endTime = DateUtils.addAndSubtractTime(createTime, 5000L); +// //查单条也要带着时间 +// DbUtils.addGeWhereCause(causes, "createTime", beginTime); +// DbUtils.addLeWhereCause(causes, "createTime", endTime); +// } DbUtils.addEqualWhereCause(causes, "tracerId", tracerId); try { List> list = db.queryOne(LOG_TABLE_NAME, null, causes, pageNum); diff --git a/Dashboard/src/main/resources/templates/tracer/index.html b/Dashboard/src/main/resources/templates/tracer/index.html index 4bc00d5..939e797 100644 --- a/Dashboard/src/main/resources/templates/tracer/index.html +++ b/Dashboard/src/main/resources/templates/tracer/index.html @@ -193,7 +193,10 @@ }, { field: 'errmsg', - title: '错误信息' + title: '错误信息', + formatter: function(value, row, index) { + return $.table.tooltip(value, 100, "open"); + } }, { field: 'createTime', diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index 28f5f4f..3672c47 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -6,11 +6,11 @@ import com.jd.platform.jlog.client.percent.DefaultTracerPercentImpl; import com.jd.platform.jlog.client.percent.ITracerPercent; import com.jd.platform.jlog.client.tracerholder.TracerHolder; import com.jd.platform.jlog.client.udp.UdpSender; -import com.jd.platform.jlog.common.handler.CompressHandler.Outcome; import com.jd.platform.jlog.common.model.TracerBean; -import com.jd.platform.jlog.common.utils.CollectionUtil; +import com.jd.platform.jlog.common.handler.CompressHandler.Outcome; import com.jd.platform.jlog.common.utils.IdWorker; import com.jd.platform.jlog.common.utils.IpUtils; +import com.jd.platform.jlog.common.utils.StringUtils; import com.jd.platform.jlog.core.ClientHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,13 +18,11 @@ import org.slf4j.LoggerFactory; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.BufferedReader; import java.io.IOException; import java.io.PrintWriter; import java.util.*; -import static com.jd.platform.jlog.common.constant.Constant.REQ; -import static com.jd.platform.jlog.common.constant.Constant.RESP; - /** * HttpFilter @@ -64,74 +62,64 @@ public class HttpFilter implements Filter { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { + HttpServletResponse resp = (HttpServletResponse) servletResponse; + RequestWrapper requestWrapper = new RequestWrapper((HttpServletRequest) servletRequest); + long currentTImeMills = System.currentTimeMillis(); + String uri = requestWrapper.getRequestURI().replace("/", ""); + //设置随机数 + Random random = new Random(currentTImeMills); + //1-100之间 + int number = random.nextInt(100) + 1; + //此处要有个开关，控制百分比 + if (iTracerPercent.percent() < number) { + filterChain.doFilter(requestWrapper, servletResponse); + return; + } + //如果是要忽略的接口，就继续执行，不搜集信息 + if (iTracerPercent.ignoreUriSet() != null && iTracerPercent.ignoreUriSet().contains(uri)) { + filterChain.doFilter(requestWrapper, servletResponse); + return; + } + //链路唯一Id + long tracerId = IdWorker.nextId(); + TracerHolder.setTracerId(tracerId); + TracerBean tracerBean = new TracerBean(); + tracerBean.setTracerId(tracerId); + tracerBean.setCreateTimeLong(System.currentTimeMillis()); + tracerBean.setUri(uri); + tracerBean.setApp(Context.APP_NAME); + + //处理request的各个入参 + parseRequestMap(requestWrapper, tracerBean); try { - HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; - HttpServletResponse resp = (HttpServletResponse) servletResponse; - String uri = httpRequest.getRequestURI().replace("/", ""); - long currentTImeMills = System.currentTimeMillis(); - - //设置随机数 - Random random = new Random(currentTImeMills); - //1-100之间 - int number = random.nextInt(100) + 1; - //此处要有个开关，控制百分比 - if (iTracerPercent.percent() < number) { - filterChain.doFilter(servletRequest, servletResponse); - return; - } - - //如果是要忽略的接口，就继续执行，不搜集信息 - if (iTracerPercent.ignoreUriSet() != null && iTracerPercent.ignoreUriSet().contains(uri)) { - filterChain.doFilter(servletRequest, servletResponse); - return; - } - - //链路唯一Id - long tracerId = IdWorker.nextId(); - TracerHolder.setTracerId(tracerId); - - //传输对象基础属性设置 - TracerBean tracerBean = new TracerBean(); - tracerBean.setCreateTime(System.currentTimeMillis()); - List> tracerObject = new ArrayList<>(); - tracerBean.setTracerObject(tracerObject); - tracerBean.setTracerId(tracerId + ""); - - //处理request的各个入参 - dealRequestMap(servletRequest, tracerObject, tracerId, uri); - //处理response - dealResponseMap(servletRequest, servletResponse, resp, tracerObject, filterChain); - + tracerBean.setResponseContent(dealResponseMap(requestWrapper, servletResponse, + resp, filterChain)); + } catch (Exception e) { + //异常信息 + tracerBean.setErrmsg(StringUtils.errorInfoToString(e)); + filterChain.doFilter(requestWrapper, servletResponse); + }finally { //设置耗时 - tracerBean.setCostTime((int) (System.currentTimeMillis() - tracerBean.getCreateTime())); - + tracerBean.setCostTime((System.currentTimeMillis() - tracerBean.getCreateTimeLong())); //udp发送 UdpSender.offerBean(tracerBean); - } catch (Exception e) { - filterChain.doFilter(servletRequest, servletResponse); } } /** * 处理出参相关信息 */ - private void dealResponseMap(ServletRequest servletRequest, ServletResponse servletResponse, HttpServletResponse resp, - List> tracerObject, FilterChain filterChain) throws IOException, ServletException { + private byte[] dealResponseMap(ServletRequest servletRequest, ServletResponse servletResponse, HttpServletResponse resp, + FilterChain filterChain) throws IOException, ServletException { // 包装响应对象 resp 并缓存响应数据 ResponseWrapper mResp = new ResponseWrapper(resp); filterChain.doFilter(servletRequest, mResp); byte[] contentBytes = mResp.getContent(); String content = new String(contentBytes); - Map responseMap = new HashMap<>(8); Map map = ExtParamFactory.getRespMap(content); Outcome outcome = ClientHandler.processResp(contentBytes, map); - responseMap.put(RESP, outcome.getContent()); - if(CollectionUtil.isNotEmpty(outcome.getTagMap())){ - responseMap.putAll(outcome.getTagMap()); - } - tracerObject.add(responseMap); //此处可以对content做处理,然后再把content写回到输出流中 servletResponse.setContentLength(-1); @@ -139,29 +127,25 @@ public class HttpFilter implements Filter { out.write(content); out.flush(); out.close(); + + return (byte[]) outcome.getContent(); } /** * 处理入参相关信息 */ - private void dealRequestMap(ServletRequest servletRequest, List> tracerObject, - long tracerId, String uri) throws IllegalAccessException, InstantiationException { + private void parseRequestMap(RequestWrapper requestWrapper, TracerBean tracerBean) { //request的各个入参 - Map params = servletRequest.getParameterMap(); + Map params = requestWrapper.getParameterMap(); Map requestMap = new HashMap<>(params.size()); for (String key : params.keySet()) { requestMap.put(key, params.get(key)[0]); } - requestMap.put("appName", Context.APP_NAME); - requestMap.put("serverIp", IpUtils.getIp()); - requestMap.put("tracerId", tracerId); - requestMap.put("uri", uri); + tracerBean.setUid((String) requestMap.get("uid")); // 自定义的其他的参数对 - requestMap.putAll(ExtParamFactory.getReqMap(servletRequest)); - + requestMap.putAll(ExtParamFactory.getReqMap(requestWrapper)); Outcome out = ClientHandler.processReq(requestMap); - requestMap.put(REQ, out.getContent()); - tracerObject.add(out.getTagMap()); + tracerBean.setRequestContent((byte[]) out.getContent()); } @Override diff --git a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java index 2fa36d0..464fe1c 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java +++ b/client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java @@ -4,13 +4,13 @@ import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.client.modeholder.ModeHolder; import com.jd.platform.jlog.client.worker.WorkerInfoHolder; import com.jd.platform.jlog.common.constant.Constant; -import com.jd.platform.jlog.common.model.RunLogMessage; +import com.jd.platform.jlog.common.constant.LogTypeEnum; import com.jd.platform.jlog.common.model.TracerBean; +import com.jd.platform.jlog.common.model.RunLogMessage; import com.jd.platform.jlog.common.model.TracerData; import com.jd.platform.jlog.common.utils.AsyncPool; import com.jd.platform.jlog.common.utils.AsyncWorker; import io.netty.channel.ChannelFuture; -import io.netty.channel.socket.DatagramPacket; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,9 +57,9 @@ public class UdpSender { /** * 写入队列 */ - public static void offerBean(TracerBean tracerBean) { + public static void offerBean(TracerBean tracerModel) { //容量是否已满 - boolean success = tracerBeanQueue.offer(tracerBean); + boolean success = tracerBeanQueue.offer(tracerModel); if (!success) { long failCount = FAIL_OFFER_COUNT.incrementAndGet(); if (failCount % 10 == 0) { @@ -105,7 +105,10 @@ public class UdpSender { TracerBean tracerBean = tracerBeanQueue.take(); tempTracers.add(tracerBean); - send(tempTracers); + TracerData tracerData = new TracerData(); + tracerData.setTracerBeanList(tempTracers); + tracerData.setType(LogTypeEnum.SPAN); + send(tracerData); } catch (Exception e) { e.printStackTrace(); } @@ -121,21 +124,10 @@ public class UdpSender { if (tempLogs.size() == 0) { continue; } - - List tempTracers = new ArrayList<>(); - TracerBean tracerBean = new TracerBean(); - tracerBean.setTracerId("-1"); - List> tracerObject = new ArrayList<>(); - - Map map = new HashMap<>(); - for (RunLogMessage runLogMessage : tempLogs) { - map.put(UUID.randomUUID().toString(), runLogMessage); - } - tracerObject.add(map); - tracerBean.setTracerObject(tracerObject); - tempTracers.add(tracerBean); - - send(tempTracers); + TracerData tracerData = new TracerData(); + tracerData.setTempLogs(tempLogs); + tracerData.setType(LogTypeEnum.TRADE); + send(tracerData); } catch (Exception e) { e.printStackTrace(); } @@ -146,9 +138,7 @@ public class UdpSender { /** * 往worker发traceBean */ - private static void send(List tracerBeans) throws InterruptedException { - TracerData tracerData = new TracerData(); - tracerData.setTracerBeanList(tracerBeans); + private static void send(TracerData tracerData) throws InterruptedException { if(!ModeHolder.getSendMode().getUnicast()){ Listips= WorkerInfoHolder.selectWorkers(); for(String ip:ips){ diff --git a/common/src/main/java/com/jd/platform/jlog/common/constant/LogTypeEnum.java b/common/src/main/java/com/jd/platform/jlog/common/constant/LogTypeEnum.java new file mode 100644 index 0000000..0c71d57 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/constant/LogTypeEnum.java @@ -0,0 +1,15 @@ +package com.jd.platform.jlog.common.constant; + +import java.io.Serializable; + +/** + * @author xiaochangbai + * @date 2023-07-15 11:56 + */ +public enum LogTypeEnum implements Serializable { + + TRADE, + + SPAN; + +} diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java b/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java index 592820e..3e84198 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java +++ b/common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java @@ -107,4 +107,19 @@ public class RunLogMessage { public void setTagMap(Map tagMap) { this.tagMap = tagMap; } + + + @Override + public String toString() { + return "RunLogMessage{" + + "tracerId=" + tracerId + + ", createTime=" + createTime + + ", content=" + content + + ", logLevel='" + logLevel + '\'' + + ", className='" + className + '\'' + + ", methodName='" + methodName + '\'' + + ", threadName='" + threadName + '\'' + + ", tagMap=" + tagMap + + '}'; + } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java b/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java index ac6b03d..85e0930 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java +++ b/common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java @@ -1,75 +1,130 @@ package com.jd.platform.jlog.common.model; import java.io.Serializable; -import java.util.List; -import java.util.Map; +import java.util.Arrays; /** - * 承载传输信息的对象 - * @author wuweifeng - * @version 1.0 - * @date 2021-08-12 + * @author xiaochangbai + * @date 2023-07-15 11:25 */ public class TracerBean implements Serializable { - /** - * 时间戳 - */ - private long createTime; - /** - * 耗时（毫秒） - */ - private int costTime; - /** - * 唯一id，代表一条链路 - */ - private String tracerId; - /** - * tracer对象，里面放的是List> - * 第一个元素是request对象，key为request，value为完整request入参，从中可找到uuid - * 最后一个元素是response响应，key为response，value为响应值byte[]，可转为String入库 - */ - private List> tracerObject; + + private Long tracerId; + private byte[] requestContent; + private byte[] responseContent; + private Long costTime; + private String uid; + private String errno; + private String errmsg; + private String app; + private String uri; + private String createTime; + + private Long createTimeLong; - @Override - public String toString() { - return "TracerBean{" + - "createTime=" + createTime + - ", costTime=" + costTime + - ", tracerId='" + tracerId + '\'' + - ", tracerObject=" + tracerObject + - '}'; - } - - public String getTracerId() { + public Long getTracerId() { return tracerId; } - public void setTracerId(String tracerId) { + public void setTracerId(Long tracerId) { this.tracerId = tracerId; } - public int getCostTime() { + public byte[] getRequestContent() { + return requestContent; + } + + public void setRequestContent(byte[] requestContent) { + this.requestContent = requestContent; + } + + public byte[] getResponseContent() { + return responseContent; + } + + public void setResponseContent(byte[] responseContent) { + this.responseContent = responseContent; + } + + public Long getCostTime() { return costTime; } - public void setCostTime(int costTime) { + public void setCostTime(Long costTime) { this.costTime = costTime; } - public long getCreateTime() { + public String getUid() { + return uid; + } + + public void setUid(String uid) { + this.uid = uid; + } + + public String getErrno() { + return errno; + } + + public void setErrno(String errno) { + this.errno = errno; + } + + public String getErrmsg() { + return errmsg; + } + + public void setErrmsg(String errmsg) { + this.errmsg = errmsg; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getCreateTime() { return createTime; } - public void setCreateTime(long createTime) { + public void setCreateTime(String createTime) { this.createTime = createTime; } - public List> getTracerObject() { - return tracerObject; + public Long getCreateTimeLong() { + return createTimeLong; } - public void setTracerObject(List> tracerObject) { - this.tracerObject = tracerObject; + public void setCreateTimeLong(Long createTimeLong) { + this.createTimeLong = createTimeLong; + } + + @Override + public String toString() { + return "TracerModel{" + + "tracerId=" + tracerId + + ", requestContent=" + Arrays.toString(requestContent) + + ", responseContent=" + Arrays.toString(responseContent) + + ", costTime=" + costTime + + ", uid='" + uid + '\'' + + ", errno='" + errno + '\'' + + ", errmsg='" + errmsg + '\'' + + ", app='" + app + '\'' + + ", uri='" + uri + '\'' + + ", createTime=" + createTime + + ", createTimeLong=" + createTimeLong + + '}'; } } diff --git a/common/src/main/java/com/jd/platform/jlog/common/model/TracerData.java b/common/src/main/java/com/jd/platform/jlog/common/model/TracerData.java index a52d211..3827e5a 100644 --- a/common/src/main/java/com/jd/platform/jlog/common/model/TracerData.java +++ b/common/src/main/java/com/jd/platform/jlog/common/model/TracerData.java @@ -1,5 +1,7 @@ package com.jd.platform.jlog.common.model; +import com.jd.platform.jlog.common.constant.LogTypeEnum; + import java.io.Serializable; import java.net.InetSocketAddress; import java.util.List; @@ -11,11 +13,22 @@ import java.util.List; * @date 2021-08-17 */ public class TracerData implements Serializable { + + /** + * type + */ + private LogTypeEnum type; + /** * 多个tracer批量打包后 */ private List tracerBeanList; + /** + * span日志 + */ + List tempLogs; + //发送地址（仅多播时候使用） private transient InetSocketAddress address; @@ -31,10 +44,28 @@ public class TracerData implements Serializable { this.tracerBeanList = tracerBeanList; } + public LogTypeEnum getType() { + return type; + } + + public void setType(LogTypeEnum type) { + this.type = type; + } + + public List getTempLogs() { + return tempLogs; + } + + public void setTempLogs(List tempLogs) { + this.tempLogs = tempLogs; + } + @Override public String toString() { return "TracerData{" + - "tracerBeanList=" + tracerBeanList + + "type=" + type + + ", tracerBeanList=" + tracerBeanList + + ", tempLogs=" + tempLogs + ", address=" + address + '}'; } diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java b/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java new file mode 100644 index 0000000..5b4b551 --- /dev/null +++ b/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java @@ -0,0 +1,42 @@ +package com.jd.platform.jlog.common.utils; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * @author xiaochangbai + * @date 2023-07-14 22:55 + */ +public final class StringUtils { + + /** + * 异常堆栈信息转string + * @param e + * @return + */ + public static String errorInfoToString(Exception e) { + StringWriter sw = null; + PrintWriter pw = null; + try{ + sw = new StringWriter(); + pw = new PrintWriter(sw); + e.printStackTrace(pw); + return sw.toString(); + } catch (Exception e2) { + return "ErrorInfoFromException"; + }finally { + if(sw!=null){ + try { + sw.close(); + } catch (IOException ex) { + return "ErrorInfoFromException"; + } + } + if(pw!=null){ + pw.close(); + } + } + } + +} diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java index 66a4ae3..a5e432f 100644 --- a/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java @@ -41,7 +41,7 @@ public class TestController { @RequestMapping("/index") public Object index() { TracerBean tracerBean = new TracerBean(); - tracerBean.setTracerId("11111"); + tracerBean.setTracerId(11111L); Configurator configurator = ConfiguratorFactory.getInstance(); try{ diff --git a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java index d525574..8812d7e 100644 --- a/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java +++ b/worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java @@ -1,18 +1,17 @@ package com.jd.platform.jlog.worker.disruptor; -import com.alibaba.fastjson.JSON; -import com.jd.platform.jlog.common.model.RunLogMessage; +import com.jd.platform.jlog.common.constant.LogTypeEnum; import com.jd.platform.jlog.common.model.TracerBean; +import com.jd.platform.jlog.common.model.RunLogMessage; import com.jd.platform.jlog.common.model.TracerData; -import com.jd.platform.jlog.common.utils.FastJsonUtils; import com.jd.platform.jlog.common.utils.ProtostuffUtils; import com.jd.platform.jlog.common.utils.ZstdUtils; import com.jd.platform.jlog.worker.store.TracerLogToDbStore; import com.jd.platform.jlog.worker.store.TracerModelToDbStore; import com.lmax.disruptor.WorkHandler; -import io.netty.util.internal.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cglib.beans.BeanMap; import java.time.Instant; import java.time.LocalDateTime; @@ -22,8 +21,6 @@ import java.util.*; import java.util.concurrent.atomic.LongAdder; import static com.jd.platform.jlog.common.constant.Constant.DEFAULT_BYTE; -import static com.jd.platform.jlog.common.constant.Constant.REQ; -import static com.jd.platform.jlog.common.constant.Constant.RESP; /** * TracerConsumer @@ -69,9 +66,8 @@ public class TracerConsumer implements WorkHandler { TracerData tracerData = ProtostuffUtils.deserialize(decompressBytes, TracerData.class); - //包含了多个tracer对象 - List tracerBeanList = tracerData.getTracerBeanList(); - buildTracerModel(tracerBeanList); + //消费处理 + buildTracerModel(tracerData); //处理完毕，将数量加1 totalDealCount.increment(); @@ -84,30 +80,24 @@ public class TracerConsumer implements WorkHandler { /** * 构建要入库的对象 */ - private void buildTracerModel(List tracerBeanList) { - //遍历传过来的 - for (TracerBean tracerBean : tracerBeanList) { - //普通日志 - if ("-1".equals(tracerBean.getTracerId())) { - dealTracerLog(tracerBean); - } else { - dealFilterModel(tracerBean); - } - + private void buildTracerModel(TracerData tracerData) { + //普通日志 + if (LogTypeEnum.TRADE.equals(tracerData.getType())) { + dealTracerLog(tracerData.getTempLogs()); + } else { + dealFilterModel(tracerData.getTracerBeanList()); } } /** * 处理中途日志 */ - private void dealTracerLog(TracerBean tracerBean) { - List> mapList = tracerBean.getTracerObject(); - Map objectMap = mapList.get(0); - //遍历value集合，里面每个都是一个RunLogMessage对象 - for (Object object :objectMap.values()) { + private void dealTracerLog(List tempLogs) { + if(tempLogs==null){ + return; + } + for (RunLogMessage runLogMessage :tempLogs) { Map map = new HashMap<>(12); - - RunLogMessage runLogMessage = (RunLogMessage) object; map.put("tracerId", runLogMessage.getTracerId()); map.put("className", runLogMessage.getClassName()); map.put("threadName", runLogMessage.getThreadName()); @@ -118,41 +108,25 @@ public class TracerConsumer implements WorkHandler { map.putAll(runLogMessage.getTagMap()); tracerLogToDbStore.offer(map); } - } /** * 处理filter里处理的出入参 */ - private void dealFilterModel(TracerBean tracerBean) { - - List> mapList = tracerBean.getTracerObject(); - Map requestMap = mapList.get(0); - - Object req = requestMap.get(REQ); - if(req == null){ - req = DEFAULT_BYTE; + private void dealFilterModel(List tracerList) { + if(tracerList==null){ + return; } - requestMap.remove(REQ); - - Map map = new HashMap<>(requestMap); - long tracerId = Long.parseLong(tracerBean.getTracerId()); - //filter的出入参 - Map responseMap = mapList.get(mapList.size() - 1); - - Object resp = responseMap.get(RESP); - if(resp == null){ - resp = DEFAULT_BYTE; + for(TracerBean tracerModel:tracerList){ + if(tracerModel.getResponseContent()==null){ + tracerModel.setResponseContent(DEFAULT_BYTE); + } + tracerModel.setCreateTime(formatLongTime(tracerModel.getCreateTimeLong())); + Map map = new HashMap(BeanMap.create(tracerModel)); + map.remove("createTimeLong"); + tracerModelToDbStore.offer(map); } - map.put("requestContent", req); - map.put("responseContent", resp); - map.put("costTime", tracerBean.getCostTime()); - map.put("tracerId", tracerId); - map.put("createTime", formatLongTime(tracerBean.getCreateTime())); - responseMap.remove("response"); - map.putAll(responseMap); - tracerModelToDbStore.offer(map); } private static String formatLongTime(long time) { From 6c2d845234a2bd1c653ab16804d8206bbcb592a0 Mon Sep 17 00:00:00 2001 From: imalasong <2879499479@qq.com> Date: Sat, 15 Jul 2023 17:16:47 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/dashboard/service/TracerService.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java b/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java index f1d713b..9b8f12d 100644 --- a/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java +++ b/Dashboard/src/main/java/com/jd/platform/jlog/dashboard/service/TracerService.java @@ -90,16 +90,16 @@ public class TracerService { */ public Map findOne(String tracerId, String uid, String beginTime, String endTime) { List causes = new ArrayList<>(1); -// if (!StringUtils.isEmpty(uid)) { -// DbUtils.addEqualWhereCause(causes, "uid", uid); -// } -// //查单条也要带着时间 -// if (!StringUtils.isEmpty(beginTime)) { -// DbUtils.addGeWhereCause(causes, "createTime", beginTime); -// } -// if (!StringUtils.isEmpty(endTime)) { -// DbUtils.addLeWhereCause(causes, "createTime", endTime); -// } + if (!StringUtils.isEmpty(uid)) { + DbUtils.addEqualWhereCause(causes, "uid", uid); + } + //查单条也要带着时间 + if (!StringUtils.isEmpty(beginTime)) { + DbUtils.addGeWhereCause(causes, "createTime", beginTime); + } + if (!StringUtils.isEmpty(endTime)) { + DbUtils.addLeWhereCause(causes, "createTime", endTime); + } DbUtils.addEqualWhereCause(causes, "tracerId", tracerId); @@ -123,13 +123,13 @@ public class TracerService { TracerListVO tracerListVO = new TracerListVO(); List causes = new ArrayList<>(1); -// if (!StringUtils.isEmpty(createTime)) { -// String beginTime = DateUtils.addAndSubtractTime(createTime, -5000L); -// String endTime = DateUtils.addAndSubtractTime(createTime, 5000L); -// //查单条也要带着时间 -// DbUtils.addGeWhereCause(causes, "createTime", beginTime); -// DbUtils.addLeWhereCause(causes, "createTime", endTime); -// } + if (!StringUtils.isEmpty(createTime)) { + String beginTime = DateUtils.addAndSubtractTime(createTime, -5000L); + String endTime = DateUtils.addAndSubtractTime(createTime, 5000L); + //查单条也要带着时间 + DbUtils.addGeWhereCause(causes, "createTime", beginTime); + DbUtils.addLeWhereCause(causes, "createTime", endTime); + } DbUtils.addEqualWhereCause(causes, "tracerId", tracerId); try { List> list = db.queryOne(LOG_TABLE_NAME, null, causes, pageNum); From 97e7244b49f296f83124848b0f0c5b206a96d2a4 Mon Sep 17 00:00:00 2001 From: imalasong <2879499479@qq.com> Date: Sat, 15 Jul 2023 17:37:34 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=88=A0=E9=99=A4=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jlog/client/filter/HttpFilter.java | 4 +- .../jlog/common/utils/StringUtils.java | 42 ------------------- .../jlog/clientdemo/web/TestController.java | 3 ++ 3 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java diff --git a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java index 3672c47..04945d6 100644 --- a/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java +++ b/client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java @@ -2,6 +2,7 @@ package com.jd.platform.jlog.client.filter; import com.jd.platform.jlog.client.Context; import com.jd.platform.jlog.client.cache.ExtParamFactory; +import com.jd.platform.jlog.client.log.LogExceptionStackTrace; import com.jd.platform.jlog.client.percent.DefaultTracerPercentImpl; import com.jd.platform.jlog.client.percent.ITracerPercent; import com.jd.platform.jlog.client.tracerholder.TracerHolder; @@ -10,7 +11,6 @@ import com.jd.platform.jlog.common.model.TracerBean; import com.jd.platform.jlog.common.handler.CompressHandler.Outcome; import com.jd.platform.jlog.common.utils.IdWorker; import com.jd.platform.jlog.common.utils.IpUtils; -import com.jd.platform.jlog.common.utils.StringUtils; import com.jd.platform.jlog.core.ClientHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,7 +97,7 @@ public class HttpFilter implements Filter { resp, filterChain)); } catch (Exception e) { //异常信息 - tracerBean.setErrmsg(StringUtils.errorInfoToString(e)); + tracerBean.setErrmsg(LogExceptionStackTrace.erroStackTrace(e).toString()); filterChain.doFilter(requestWrapper, servletResponse); }finally { //设置耗时 diff --git a/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java b/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java deleted file mode 100644 index 5b4b551..0000000 --- a/common/src/main/java/com/jd/platform/jlog/common/utils/StringUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.jd.platform.jlog.common.utils; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -/** - * @author xiaochangbai - * @date 2023-07-14 22:55 - */ -public final class StringUtils { - - /** - * 异常堆栈信息转string - * @param e - * @return - */ - public static String errorInfoToString(Exception e) { - StringWriter sw = null; - PrintWriter pw = null; - try{ - sw = new StringWriter(); - pw = new PrintWriter(sw); - e.printStackTrace(pw); - return sw.toString(); - } catch (Exception e2) { - return "ErrorInfoFromException"; - }finally { - if(sw!=null){ - try { - sw.close(); - } catch (IOException ex) { - return "ErrorInfoFromException"; - } - } - if(pw!=null){ - pw.close(); - } - } - } - -} diff --git a/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java index a5e432f..8efd0c0 100644 --- a/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java +++ b/example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java @@ -70,6 +70,9 @@ public class TestController { if(newKey == 1){ return 1; } + if(uid!=null && 3==uid){ + throw new RuntimeException("发生异常了"); + } return new Response("滴滴员工tangbohu的终身代号是什么？？？是9527"); }