test/jlog
1
0
Fork 0
mirror of https://gitee.com/jd-platform-opensource/jlog.git synced 2024-11-29 18:38:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

!1 mult_config

Browse Source 
Merge pull request !1 from liyunfeng/multi_configuration
This commit is contained in:
tianyaleixiaowu 2022-03-15 01:33:03 +00:00 committed by Gitee
commit 7fae46175f
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
101 changed files with 4425 additions and 975 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Dashboard/pom.xml
View File

@ -55,11 +55,6 @@
<artifactId>zstd-jni</artifactId>
<version>1.5.0-4</version>
</dependency>
<dependency>
<groupId>com.jd.ump</groupId>
<artifactId>profiler</artifactId>
<version>6.2.13</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>

2
Dashboard/src/main/resources/logback-spring.xml
View File

@ -6,7 +6,7 @@ debug：当此属性设置为true时，将打印出logback内部日志信息，
-->
<configuration scan="false" scanPeriod="60 seconds" debug="false">
<!-- 定义日志的根目录 -->
<property name="LOG_HOME" value="/export/Logs" />
<property name="LOG_HOME" value="/home/Logs" />
<!-- 定义日志文件名称 -->
<property name="appName" value="dashboard"/>
<!-- ch.qos.logback.core.ConsoleAppender 表示控制台输出 -->

26
client/pom.xml
View File

@ -18,6 +18,32 @@
<version>1.4-SNAPSHOT</version>
</dependency>
<!-- <dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-zk</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>-->
<!-- <dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-nacos</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>-->
<!-- <dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-etcd</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>-->
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-apollo</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<!-- <dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-core</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>-->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>

62
client/src/main/java/com/jd/platform/jlog/client/TracerClientStarter.java
View File

@ -1,11 +1,16 @@
package com.jd.platform.jlog.client;
import com.jd.platform.jlog.client.etcd.EtcdConfigFactory;
import com.jd.platform.jlog.client.etcd.EtcdStarter;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.client.mdc.Mdc;
import com.jd.platform.jlog.client.udp.HttpSender;
import com.jd.platform.jlog.client.udp.UdpClient;
import com.jd.platform.jlog.client.udp.UdpSender;
import com.jd.platform.jlog.client.task.Monitor;
import com.jd.platform.jlog.common.handler.TagConfig;
import com.jd.platform.jlog.core.ClientHandlerBuilder;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* TracerClientStarter
@ -14,15 +19,22 @@ import com.jd.platform.jlog.client.udp.UdpSender;
* @date 2021-08-13
*/
public class TracerClientStarter {
/**
* etcd地址
*/
private String etcdServer;
private final static Logger LOGGER = LoggerFactory.getLogger(TracerClientStarter.class);
/**
* 机房
*/
private Mdc mdc;
/**
* 如果直接配置在app.properties/yaml等主配置文件
* 可以用ConfigurationProperties直接获取有值对象
*/
private TagConfig tagConfig;
/**
* TracerClientStarter
*/
@ -35,8 +47,8 @@ public class TracerClientStarter {
public static class Builder {
private String appName;
private String etcdServer;
private Mdc mdc;
private TagConfig tagConfig;
public Builder() {
}
@ -51,40 +63,48 @@ public class TracerClientStarter {
return this;
}
public Builder setEtcdServer(String etcdServer) {
this.etcdServer = etcdServer;
public Builder setTagConfig(TagConfig tagConfig) {
this.tagConfig = tagConfig;
return this;
}
public TracerClientStarter build() {
TracerClientStarter tracerClientStarter = new TracerClientStarter(appName);
tracerClientStarter.etcdServer = etcdServer;
tracerClientStarter.tagConfig = tagConfig;
tracerClientStarter.mdc = mdc;
return tracerClientStarter;
}
}
/**
* 启动监听etcd
* 启动监听
*/
public void startPipeline() {
//设置ConfigCenter
EtcdConfigFactory.buildConfigCenter(etcdServer);
// 初始化配置
initJLogConfig();
Context.MDC = mdc;
EtcdStarter starter = new EtcdStarter();
//与etcd相关的监听都开启
Monitor starter = new Monitor();
starter.start();
UdpClient udpClient = new UdpClient();
/* UdpClient udpClient = new UdpClient();
udpClient.start();
//开启发送
UdpSender.uploadToWorker();
//开启大对象http发送
HttpSender.uploadToWorker();
HttpSender.uploadToWorker();*/
}
/**
* 如果未赋值，从配置器获取，底层是Properties
*/
private void initJLogConfig(){
LOGGER.info("从主配置获取的tagConfig:{}", JSON.toJSONString(tagConfig));
Configurator configurator = ConfiguratorFactory.getInstance();
ClientHandlerBuilder.buildHandler(tagConfig, configurator);
}
}

24
client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdConfigFactory.java
View File

@ -1,24 +0,0 @@
package com.jd.platform.jlog.client.etcd;
import com.jd.platform.jlog.common.config.IConfigCenter;
import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder;
/**
* @author wuweifeng wrote on 2020-01-07
* @version 1.0
*/
public class EtcdConfigFactory {
private static IConfigCenter configCenter;
private EtcdConfigFactory() {}
public static IConfigCenter configCenter() {
return configCenter;
}
public static void buildConfigCenter(String etcdServer) {
//连接多个时，逗号分隔
configCenter = JdEtcdBuilder.build(etcdServer);
}
}

25
client/src/main/java/com/jd/platform/jlog/client/filter/UserFilter.java → client/src/main/java/com/jd/platform/jlog/client/filter/HttpFilter.java
View File

@ -8,7 +8,7 @@ import com.jd.platform.jlog.client.udp.UdpSender;
import com.jd.platform.jlog.common.model.TracerBean;
import com.jd.platform.jlog.common.utils.IdWorker;
import com.jd.platform.jlog.common.utils.IpUtils;
import com.jd.platform.jlog.common.utils.ZstdUtils;
import com.jd.platform.jlog.core.ClientHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -20,14 +20,14 @@ import java.io.PrintWriter;
import java.util.*;
/**
* UserFilter
* HttpFilter
* http://blog.chinaunix.net/uid-20783755-id-4729930.html
*
* @author wuweifeng
* @version 1.0
* @date 2021-08-16
*/
public class UserFilter implements Filter {
public class HttpFilter implements Filter {
/**
* 获取切量百分比的
*/
@ -38,11 +38,11 @@ public class UserFilter implements Filter {
/**
* 传入百分比实现类
*/
public UserFilter(ITracerPercent iTracerPercent) {
public HttpFilter(ITracerPercent iTracerPercent) {
this.iTracerPercent = iTracerPercent;
}
public UserFilter() {
public HttpFilter() {
iTracerPercent = new DefaultTracerPercentImpl();
}
@ -127,12 +127,14 @@ public class UserFilter implements Filter {
filterChain.doFilter(servletRequest, mResp);
byte[] contentBytes = mResp.getContent();
String content = new String(contentBytes);
//最终的要发往worker的response，经历了base64压缩
byte[] bytes = ZstdUtils.compress(contentBytes);
byte[] base64Bytes = Base64.getEncoder().encode(bytes);
Map<String, Object> responseMap = new HashMap<>(8);
responseMap.put("response", base64Bytes);
Map<String, Object> map = new HashMap<>(1);
map.put("errno", 200);
ClientHandler.Outcome outcome = ClientHandler.processResp(contentBytes, map);
responseMap.put("response", outcome.getContent());
if(!outcome.getMap().isEmpty()){
responseMap.putAll(outcome.getMap());
}
tracerObject.add(responseMap);
//此处可以对content做处理,然后再把content写回到输出流中
@ -150,7 +152,6 @@ public class UserFilter implements Filter {
long tracerId, String uri) {
//request的各个入参
Map<String, String[]> params = servletRequest.getParameterMap();
//将request信息保存
Map<String, Object> requestMap = new HashMap<>(params.size());
for (String key : params.keySet()) {
requestMap.put(key, params.get(key)[0]);
@ -159,7 +160,7 @@ public class UserFilter implements Filter {
requestMap.put("serverIp", IpUtils.getIp());
requestMap.put("tracerId", tracerId);
requestMap.put("uri", uri);
tracerObject.add(requestMap);
tracerObject.add(ClientHandler.processReq(requestMap));
}
@Override

41
client/src/main/java/com/jd/platform/jlog/client/jsf/JsfRequest.java
View File

@ -1,41 +0,0 @@
package com.jd.platform.jlog.client.jsf;
/**
* jsf请求入参包装对象
* @author wuweifeng
* @version 1.0
* @date 2021-10-26
*/
public interface JsfRequest {
/**
* 请求的完整入参
*/
String requestContent();
/**
* 接口名
*/
String uri();
/**
* 用户pin
*/
default String pin() {
return "";
}
/**
* 用户uuid
*/
default String uuid() {
return "";
}
/**
* 用户ip
*/
default String userIp() {
return "";
}
}

91
client/src/main/java/com/jd/platform/jlog/client/jsf/JsfTracer.java
View File

@ -1,91 +0,0 @@
package com.jd.platform.jlog.client.jsf;
import com.jd.platform.jlog.client.Context;
import com.jd.platform.jlog.client.tracerholder.TracerHolder;
import com.jd.platform.jlog.client.udp.UdpSender;
import com.jd.platform.jlog.common.model.TracerBean;
import com.jd.platform.jlog.common.utils.IdWorker;
import com.jd.platform.jlog.common.utils.IpUtils;
import com.jd.platform.jlog.common.utils.ZstdUtils;
import java.nio.charset.StandardCharsets;
import java.util.*;
/**
* JsfTracer工具类
*
* @author wuweifeng
* @version 1.0
* @date 2021-10-26
*/
public class JsfTracer {
/**
* 临时保存入参
*/
private static Map<Long, TracerBean> TEMP_HOLDER = new HashMap<>(128);
/**
* 跟踪开启
*/
public static void begin(JsfRequest jsfRequest) {
long tracerId = IdWorker.nextId();
TracerHolder.setTracerId(tracerId);
TracerBean tracerBean = new TracerBean();
tracerBean.setCreateTime(System.currentTimeMillis());
List<Map<String, Object>> tracerObject = new ArrayList<>();
tracerBean.setTracerObject(tracerObject);
//将request信息保存
Map<String, Object> requestMap = new HashMap<>();
requestMap.put("appName", Context.APP_NAME);
requestMap.put("serverIp", IpUtils.getIp());
requestMap.put("tracerId", tracerId);
requestMap.put("uri", jsfRequest.uri().trim());
//将用户整个request都放进去
requestMap.put("wholeRequest", jsfRequest.requestContent());
requestMap.put("pin", jsfRequest.pin());
requestMap.put("uuid", jsfRequest.uuid());
requestMap.put("userIp", jsfRequest.userIp());
tracerObject.add(requestMap);
tracerBean.setTracerId(tracerId + "");
TEMP_HOLDER.put(tracerId, tracerBean);
}
/**
* 记录请求出入参，发送到worker
*/
public static void end(String response) {
long tracerId = TracerHolder.getTracerId();
TracerBean tracerBean = TEMP_HOLDER.get(tracerId);
if (tracerBean == null) {
return;
}
if (response != null) {
//最终的要发往worker的response，经历了base64压缩
byte[] bytes = ZstdUtils.compress(response.getBytes(StandardCharsets.UTF_8));
byte[] base64Bytes = Base64.getEncoder().encode(bytes);
Map<String, Object> responseMap = new HashMap<>(8);
responseMap.put("response", base64Bytes);
List<Map<String, Object>> tracerObject = tracerBean.getTracerObject();
tracerObject.add(responseMap);
}
//设置耗时
tracerBean.setCostTime((int) (System.currentTimeMillis() - tracerBean.getCreateTime()));
UdpSender.offerBean(tracerBean);
//从缓存删除它
TEMP_HOLDER.remove(tracerId);
}
}

76
client/src/main/java/com/jd/platform/jlog/client/etcd/EtcdStarter.java → client/src/main/java/com/jd/platform/jlog/client/task/Monitor.java
View File

@ -1,41 +1,34 @@
package com.jd.platform.jlog.client.etcd;
package com.jd.platform.jlog.client.task;
import cn.hutool.core.collection.CollectionUtil;
import com.ibm.etcd.api.KeyValue;
import com.jd.platform.jlog.client.Context;
import com.jd.platform.jlog.client.mdc.Mdc;
import com.jd.platform.jlog.client.worker.WorkerInfoHolder;
import com.jd.platform.jlog.common.config.IConfigCenter;
import com.jd.platform.jlog.common.constant.Constant;
import io.grpc.StatusRuntimeException;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* etcd连接管理器
*
* @author wuweifeng
* @version 1.0
* @date 2021-08-13
* @author tangbohu
* @version 1.0.0
* @ClassName Watchdog.java
* @Description TODO
* @createTime 2022年02月12日 10:20:00
*/
public class EtcdStarter {
/**
* logger
*/
private Logger logger = LoggerFactory.getLogger(getClass());
public class Monitor {
private final static Logger LOGGER = LoggerFactory.getLogger(Monitor.class);
/**
* 开始获取workerIp地址并保存</>
* 监听workerIp地址变化
*/
public void start() {
fetchWorkerInfo();
//fetchWorkerInfo();
}
/**
@ -45,8 +38,12 @@ public class EtcdStarter {
ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
//开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取
scheduledExecutorService.scheduleAtFixedRate(() -> {
logger.info("trying to connect to etcd and fetch worker info");
fetch();
LOGGER.info("trying to connect to etcd and fetch worker info");
try {
fetch();
} catch (Exception e) {
e.printStackTrace();
}
}, 0, 30, TimeUnit.SECONDS);
}
@ -54,39 +51,22 @@ public class EtcdStarter {
/**
* 从配置中心获取worker的ip集合
*/
private void fetch() {
IConfigCenter configCenter = EtcdConfigFactory.configCenter();
private void fetch() throws Exception {
Configurator configurator = ConfiguratorFactory.getInstance();
//获取所有worker的ip
List<KeyValue> keyValues = null;
List addresses;
try {
//如果设置了机房属性，则拉取同机房的worker。如果同机房没worker，则拉取所有
if (Context.MDC != null) {
String mdc = parseMdc(Context.MDC);
keyValues = configCenter.getPrefix(Constant.WORKER_PATH + Context.APP_NAME + "/" + mdc);
}
if (CollectionUtil.isEmpty(keyValues)) {
keyValues = configCenter.getPrefix(Constant.WORKER_PATH + Context.APP_NAME);
}
addresses = configurator.getList("workers");
//全是空，给个警告
if (CollectionUtil.isEmpty(keyValues)) {
logger.warn("very important warn !!! workers ip info is null!!!");
if (addresses == null || addresses.size() == 0) {
LOGGER.warn("very important warn !!! workers ip info is null!!!");
return;
}
List<String> addresses = new ArrayList<>();
if (keyValues != null) {
for (KeyValue keyValue : keyValues) {
//value里放的是ip地址
String ipPort = keyValue.getValue().toStringUtf8();
addresses.add(ipPort);
}
}
//将对应的worker保存下来
WorkerInfoHolder.mergeAndConnectNew(addresses);
} catch (StatusRuntimeException ex) {
//etcd连不上
logger.error("etcd connected fail. Check the etcd address!!!");
} catch (Exception ex) {
LOGGER.error("config server connected fail. Check the config address!!!");
}
}

4
client/src/main/java/com/jd/platform/jlog/client/udp/UdpClient.java
View File

@ -60,7 +60,7 @@ public class UdpClient {
});
//4.bind并返回一个channel
try {
Channel channel = bootstrap.bind(8888).sync().channel();
Channel channel = bootstrap.bind(8887).sync().channel();
Context.CHANNEL = channel;
//6.等待channel的close
@ -68,6 +68,8 @@ public class UdpClient {
//7.关闭group
group.shutdownGracefully();
} catch (InterruptedException e) {
System.out.println("NioEventLoopGroup ==> "+e.toString());
e.printStackTrace();
}
}

7
client/src/main/java/com/jd/platform/jlog/client/udp/UdpSender.java
View File

@ -1,12 +1,11 @@
package com.jd.platform.jlog.client.udp;
import cn.hutool.core.collection.CollectionUtil;
import com.google.common.collect.Queues;
import com.jd.platform.jlog.client.Context;
import com.jd.platform.jlog.common.model.RunLogMessage;
import com.jd.platform.jlog.common.model.TracerBean;
import com.jd.platform.jlog.common.model.TracerData;
import com.jd.platform.jlog.common.utils.AsyncPool;
import com.jd.platform.jlog.common.utils.AsyncWorker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -113,8 +112,8 @@ public class UdpSender {
try {
//要么key达到500个，要么达到1秒，就汇总上报给worker一次
List<RunLogMessage> tempLogs = new ArrayList<>();
Queues.drain(logBeanQueue, tempLogs, 500, 1, TimeUnit.SECONDS);
if (CollectionUtil.isEmpty(tempLogs)) {
AsyncWorker.drain(logBeanQueue, tempLogs, 500, 1, TimeUnit.SECONDS);
if (tempLogs.size() == 0) {
continue;
}

3
client/src/main/java/com/jd/platform/jlog/client/worker/WorkerInfoHolder.java
View File

@ -1,6 +1,5 @@
package com.jd.platform.jlog.client.worker;
import cn.hutool.core.collection.CollectionUtil;
import java.util.Collections;
import java.util.List;
@ -57,7 +56,7 @@ public class WorkerInfoHolder {
* address例子：10.12.139.152:11111
*/
public static void mergeAndConnectNew(List<String> allAddresses) {
if (CollectionUtil.isEmpty(allAddresses)) {
if (allAddresses.size() == 0) {
WORKER_HOLDER.clear();
return;
}

34
clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java
View File

@ -1,34 +0,0 @@
package com.jd.platform.jlog.clientdemo.web;
import com.jd.platform.jlog.common.model.TracerBean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author shenkaiwen5
* @version 1.0
* @date 2021-12-27
*/
@RestController
public class TestController {
/**
* do nothing
* just as an adapter for this project common log helper
*
*/
private static Logger RequestLog = LoggerFactory.getLogger("RequestLog");
@RequestMapping("/index")
public Object index() {
TracerBean tracerBean = new TracerBean();
tracerBean.setTracerId("11111");
RequestLog.info("哈哈哈哈哈哈");
return tracerBean;
}
}

5
clientdemo/src/main/resources/application.yml
View File

@ -1,5 +0,0 @@
#etcd的地址，如有多个用逗号分隔
config:
server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！
server:
port: 8081

54
clientdemo/src/main/resources/log4j2.xml
View File

@ -1,54 +0,0 @@
<?xml version='1.0' encoding="GBK" ?>
<Configuration status="DEBUG" monitorInterval="60">
<Appenders>
<Console name="CONSOLE" target="SYSTEM_OUT">
<PatternLayout pattern="[%7r] %6p - %30.30c - %m \n"/>
</Console>
<RollingRandomAccessFile name="ServerSide" fileName="${log-dir}/serverside.log" filePattern="${log-dir}/serverside.%d{yyyy-MM-dd}.log">
<PatternLayout pattern="%d [%7r] %6p - %30.30c - %m \n"/>
<Policies>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<DefaultRolloverStrategy>
<Delete basePath="${log-dir}" maxDepth="1">
<IfFileName glob="serverside.*.log" />
<IfLastModified age="5d" />
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<RollingRandomAccessFile name="requestLog" fileName="${log-dir}/request.log" filePattern="${log-dir}/request.%d{yyyy-MM-dd}.log">
<PatternLayout pattern="%d [%7r] %6p - %30.30c - %m \n"/>
<Policies>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<DefaultRolloverStrategy>
<Delete basePath="${log-dir}" maxDepth="1">
<IfFileName glob="request.*.log" />
<IfLastModified age="5d" />
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<tracer name="tracerAppender"></tracer>
</Appenders>
<Loggers>
<Logger name="exceptionLog" level="ERROR" additivity="false">
<AppenderRef ref="exceptionLog"/>
<AppenderRef ref="tracerAppender" />
</Logger>
<Logger name="RequestLog" level="INFO" additivity="false">
<AppenderRef ref="requestLog"/>
<AppenderRef ref="tracerAppender" />
</Logger>
<Root level="INFO" includeLocation="false">
</Root>
</Loggers>
</Configuration>

38
common/pom.xml
View File

@ -17,10 +17,12 @@
<snappy.version>1.1.8.4</snappy.version>
<zstd.version>1.5.0-4</zstd.version>
<fastjson.version>1.2.70</fastjson.version>
<protostuff.version>1.7.2</protostuff.version>
<!-- <zookeeper.version>3.4.14</zookeeper.version>
<caffeine.version>2.8.0</caffeine.version>
<hutool.version>5.1.0</hutool.version>
<hp-etcd.version>0.0.16</hp-etcd.version>
<protostuff.version>1.7.2</protostuff.version>
<hp-etcd.version>0.0.16</hp-etcd.version>-->
</properties>
<dependencies>
@ -29,6 +31,7 @@
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.32</version>
</dependency>
<dependency>
<groupId>io.protostuff</groupId>
<artifactId>protostuff-core</artifactId>
@ -41,40 +44,29 @@
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>${hutool.version}</version>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>${zstd.version}</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>
<dependency>
<groupId>com.ibm.etcd</groupId>
<artifactId>etcd-java</artifactId>
<version>${hp-etcd.version}</version>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>guava</artifactId>
<version>${caffeine.version}</version>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty4.version}</version>
</dependency>
</dependencies>
</project>

89
common/src/main/java/com/jd/platform/jlog/common/config/IConfigCenter.java
View File

@ -1,89 +0,0 @@
package com.jd.platform.jlog.common.config;
import com.ibm.etcd.api.KeyValue;
import com.ibm.etcd.client.kv.KvClient;
import java.util.List;
/**
* 配合中心接口
*
* @author wuweifeng wrote on 2019-12-09
* @version 1.0
*/
public interface IConfigCenter {
/**
* 存入key，value
*/
void put(String key, String value);
/**
* 存入key、value，和租约id
*/
void put(String key, String value, long leaseId);
/**
* 删除一个lease
*/
void revoke(long leaseId);
/**
* 存入key、value，和过期时间，单位是秒
*/
long putAndGrant(String key, String value, long ttl);
/**
* 给key设置新的leaseId
*/
long setLease(String key, long leaseId);
void delete(String key);
/**
* 根据key，获取value
*/
String get(String key);
/**
* 获取指定前缀的所有key-value
*/
List<KeyValue> getPrefix(String key);
/**
* 监听key
*/
KvClient.WatchIterator watch(String key);
/**
* 监听前缀为key的
*/
KvClient.WatchIterator watchPrefix(String key);
/**
* 自动续约
* @param frequencySecs 续约频率，最小是4秒，默认是5秒
* @param minTtl 最小存活时间，最小是2秒，默认是10秒
* @return 返回leaseId
*/
long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception;
/**
* 构建一个可自动续约的lease
*/
long buildAliveLease(int frequencySecs, int minTtl) throws Exception;
/**
* 构建一个普通lease
*/
long buildNormalLease(long ttl);
/**
* 判断剩余的过期时间
*/
long timeToLive(long leaseId);
/**
* 根据key，获取value
*/
KeyValue getKv(String key);
}

18
common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdBuilder.java
View File

@ -1,18 +0,0 @@
package com.jd.platform.jlog.common.config.etcd;
import com.ibm.etcd.client.EtcdClient;
/**
* etcd连接器
* @author wuweifeng wrote on 2019-12-10
* @version 1.0
*/
public class JdEtcdBuilder {
/**
* @param endPoints 如https://127.0.0.1:2379 有多个时逗号分隔
*/
public static JdEtcdClient build(String endPoints) {
return new JdEtcdClient(EtcdClient.forEndpoints(endPoints).withPlainText().build());
}
}

165
common/src/main/java/com/jd/platform/jlog/common/config/etcd/JdEtcdClient.java
View File

@ -1,165 +0,0 @@
package com.jd.platform.jlog.common.config.etcd;
import cn.hutool.core.collection.CollectionUtil;
import com.google.protobuf.ByteString;
import com.ibm.etcd.api.KeyValue;
import com.ibm.etcd.api.LeaseGrantResponse;
import com.ibm.etcd.api.RangeResponse;
import com.ibm.etcd.client.KvStoreClient;
import com.ibm.etcd.client.kv.KvClient;
import com.ibm.etcd.client.lease.LeaseClient;
import com.ibm.etcd.client.lease.PersistentLease;
import com.ibm.etcd.client.lock.LockClient;
import com.jd.platform.jlog.common.config.IConfigCenter;
import java.util.List;
import java.util.concurrent.ExecutionException;
import static java.util.concurrent.TimeUnit.SECONDS;
/**
* etcd客户端
*
* @author wuweifeng wrote on 2019-12-06
* @version 1.0
*/
public class JdEtcdClient implements IConfigCenter {
private KvClient kvClient;
private LeaseClient leaseClient;
private LockClient lockClient;
public JdEtcdClient(KvStoreClient kvStoreClient) {
this.kvClient = kvStoreClient.getKvClient();
this.leaseClient = kvStoreClient.getLeaseClient();
this.lockClient = kvStoreClient.getLockClient();
}
public LeaseClient getLeaseClient() {
return leaseClient;
}
public void setLeaseClient(LeaseClient leaseClient) {
this.leaseClient = leaseClient;
}
public KvClient getKvClient() {
return kvClient;
}
public void setKvClient(KvClient kvClient) {
this.kvClient = kvClient;
}
public LockClient getLockClient() {
return lockClient;
}
public void setLockClient(LockClient lockClient) {
this.lockClient = lockClient;
}
@Override
public void put(String key, String value) {
kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value)).sync();
}
@Override
public void put(String key, String value, long leaseId) {
kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value), leaseId).sync();
}
@Override
public void revoke(long leaseId) {
leaseClient.revoke(leaseId);
}
@Override
public long putAndGrant(String key, String value, long ttl) {
LeaseGrantResponse lease = leaseClient.grant(ttl).sync();
put(key, value, lease.getID());
return lease.getID();
}
@Override
public long setLease(String key, long leaseId) {
kvClient.setLease(ByteString.copyFromUtf8(key), leaseId);
return leaseId;
}
@Override
public void delete(String key) {
kvClient.delete(ByteString.copyFromUtf8(key)).sync();
}
@Override
public String get(String key) {
RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).sync();
List<KeyValue> keyValues = rangeResponse.getKvsList();
if (CollectionUtil.isEmpty(keyValues)) {
return null;
}
return keyValues.get(0).getValue().toStringUtf8();
}
@Override
public KeyValue getKv(String key) {
RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).sync();
List<KeyValue> keyValues = rangeResponse.getKvsList();
if (CollectionUtil.isEmpty(keyValues)) {
return null;
}
return keyValues.get(0);
}
@Override
public List<KeyValue> getPrefix(String key) {
RangeResponse rangeResponse = kvClient.get(ByteString.copyFromUtf8(key)).asPrefix().sync();
return rangeResponse.getKvsList();
}
@Override
public KvClient.WatchIterator watch(String key) {
return kvClient.watch(ByteString.copyFromUtf8(key)).start();
}
@Override
public KvClient.WatchIterator watchPrefix(String key) {
return kvClient.watch(ByteString.copyFromUtf8(key)).asPrefix().start();
}
@Override
public long keepAlive(String key, String value, int frequencySecs, int minTtl) throws Exception {
//minTtl秒租期，每frequencySecs秒续约一下
PersistentLease lease = leaseClient.maintain().leaseId(System.currentTimeMillis()).keepAliveFreq(frequencySecs).minTtl(minTtl).start();
long newId = lease.get(3L, SECONDS);
put(key, value, newId);
return newId;
}
@Override
public long buildAliveLease(int frequencySecs, int minTtl) throws Exception {
PersistentLease lease = leaseClient.maintain().leaseId(System.currentTimeMillis()).keepAliveFreq(frequencySecs).minTtl(minTtl).start();
return lease.get(3L, SECONDS);
}
@Override
public long buildNormalLease(long ttl) {
LeaseGrantResponse lease = leaseClient.grant(ttl).sync();
return lease.getID();
}
@Override
public long timeToLive(long leaseId) {
try {
return leaseClient.ttl(leaseId).get().getTTL();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
return 0L;
}
}
}

53
common/src/main/java/com/jd/platform/jlog/common/constant/Constant.java
View File

@ -1,5 +1,8 @@
package com.jd.platform.jlog.common.constant;
import java.util.HashSet;
import java.util.Set;
/**
* 常量工具类Constant
*
@ -24,4 +27,54 @@ public class Constant {
* 当客户端要删除某个key时，就往etcd里赋值这个value，设置1秒过期，就算删除了
*/
public static String DEFAULT_DELETE_VALUE = "#[DELETE]#";
/**
* 普通tag最大长度 超过则不会单独存储，不超过则会存入tag：{"normal" : "logContent"}
*/
public static int TAG_NORMAL_KEY_MAX_LEN = 20;
/**
* 可供提取的日志最小长度
*/
public static int EXTRACT_MIN_LEN = 5;
/**
* 符合正则但不构成kv结构的普通日志，如「xx查询为空」
*/
public static final String TAG_NORMAL_KEY = "normal";
/**
* 小于1000不考虑压缩
*/
public static final long MIN = 1000;
/**
* 默认大于10000才考虑压缩
*/
public static final long THRESHOLD = 10000;
/**
* 正则里需要特殊处理的字符串
*/
public static final Set<String> SPECIAL_CHAR = new HashSet<String>() {
{
add("*");
add(".");
add("?");
add("+");
add("$");
add("^");
add("[");
add("]");
add("(");
add(")");
add("{");
add("}");
add("|");
}
};
}

100
common/src/main/java/com/jd/platform/jlog/common/handler/CollectMode.java Normal file
View File

@ -0,0 +1,100 @@
package com.jd.platform.jlog.common.handler;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName CollectMode.java
* @Description 定义采集模式的组合 用于以后method级别的tag注解
* @createTime 2022年02月24日 23:10:00
*/
public class CollectMode {
/**
* 挂起 不提取不压缩
*/
public static final long SUSPEND = 0L;
/**
* 提取req
*/
public static final long E_REQ = 1L;
/**
* 压缩req
*/
public static final long C_REQ = 1L << 2;
/**
* 提取log
*/
public static final long E_LOG = 1L << 3;
/**
* 压缩log
*/
public static final long C_LOG = 1L << 4;
/**
* 提取resp
*/
public static final long E_RESP = 1L << 5;
/**
* 压缩resp
*/
public static final long C_RESP = 1L << 6;
/** ======================================= 下面是组合 ======================================= */
/**
* 提取req+log
*/
public static final long EXTRACT_REQ_LOG = E_REQ | E_LOG;
/**
* 提取req+resp
*/
public static final long EXTRACT_REQ_REQP = E_REQ | E_RESP;
/**
* 提取req+resp
*/
public static final long EXTRACT_REQ_RESP = E_REQ | E_RESP;
/**
* 提取req+log+resp
*/
public static final long EXTRACT_ALL = E_REQ | E_LOG | E_RESP;
/**
* 压缩req+log
*/
public static final long COMPRESS_REQ_LOG = C_REQ | C_LOG;
/**
* 压缩req+resp
*/
public static final long COMPRESS_REQ_REQP = C_REQ | C_RESP;
/**
* 压缩log+resp
*/
public static final long COMPRESS_LOG_RESP = C_REQ | C_RESP;
/**
* 压缩req+log+resp
*/
public static final long COMPRESS_ALL = C_REQ | C_LOG | C_RESP;
public static boolean isMatched(long indicator, long position) {
return (indicator & position) == position;
}
}

97
common/src/main/java/com/jd/platform/jlog/common/handler/CompressHandler.java Normal file
View File

@ -0,0 +1,97 @@
package com.jd.platform.jlog.common.handler;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.utils.ZstdUtils;
import com.sun.istack.internal.NotNull;
import java.util.Base64;
import java.util.Map;
import static com.jd.platform.jlog.common.constant.Constant.MIN;
import static com.jd.platform.jlog.common.constant.Constant.THRESHOLD;
import static com.jd.platform.jlog.common.handler.CollectMode.*;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName CompressHandler.java
* @Description TODO
* @createTime 2022年03月10日 20:52:00
*/
public class CompressHandler {
/**
* 压缩策略
*/
private long compress;
/**
* 超过limit的才压缩
*/
private long threshold;
private CompressHandler(long compress, long threshold){
this.compress = compress;
this.threshold = threshold;
}
private static volatile CompressHandler instance = null;
public static void buildCompressHandler(Long compress, Long threshold) {
if(compress == null || compress < 1){
compress = COMPRESS_LOG_RESP;
}
if( threshold == null || threshold < MIN){
threshold = THRESHOLD;
}
instance = new CompressHandler(compress, threshold);
}
public static void refresh(Long compress, Long threshold){
instance = null;
buildCompressHandler(compress, threshold);
}
public static Map<String, Object> compressReq(Map<String, Object> map){
if(instance == null || !isMatched(instance.compress, E_REQ)){
return map;
}
for (Map.Entry<String, Object> entry : map.entrySet()) {
map.put(entry.getKey(), doCompress(entry.getValue().toString().getBytes()));
}
return map;
}
public static byte[] compressLog(byte[] contentBytes){
if(instance == null || !isMatched(instance.compress, E_LOG)){ return contentBytes; }
return doCompress(contentBytes);
}
public static byte[] compressResp(byte[] contentBytes){
if(instance == null || !isMatched(instance.compress, E_RESP)){ return contentBytes; }
return doCompress(contentBytes);
}
private static byte[] doCompress(byte[] contentBytes){
if(contentBytes.length < instance.threshold){
return contentBytes;
}
//最终的要发往worker的response，经历了base64压缩
byte[] bytes = ZstdUtils.compress(contentBytes);
return Base64.getEncoder().encode(bytes);
}
public long getCompress() {
return compress;
}
public void setCompress(long compress) {
this.compress = compress;
}
}

225
common/src/main/java/com/jd/platform/jlog/common/handler/ExtractHandler.java Normal file
View File

@ -0,0 +1,225 @@
package com.jd.platform.jlog.common.handler;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.common.utils.ConfigUtil;
import com.jd.platform.jlog.common.utils.StringUtil;
import com.sun.istack.internal.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.jd.platform.jlog.common.constant.Constant.EXTRACT_MIN_LEN;
import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY;
import static com.jd.platform.jlog.common.constant.Constant.TAG_NORMAL_KEY_MAX_LEN;
import static com.jd.platform.jlog.common.handler.CollectMode.*;
import static com.jd.platform.jlog.common.handler.CollectMode.E_LOG;
import static com.jd.platform.jlog.common.handler.CollectMode.E_REQ;
import static com.jd.platform.jlog.common.utils.ConfigUtil.RANDOM;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ExtractHandler.java
* @createTime 2022年02月12日 21:28:00
*/
public class ExtractHandler {
private static final Logger LOGGER = LoggerFactory.getLogger(ExtractHandler.class);
private List<String> reqTags;
private List<String> logTags;
private List<String> respTags;
private String delimiter = "|";
private int delimiterLen = delimiter.length();
private String join = "=";
private Pattern pattern;
private long extract;
private static volatile ExtractHandler instance = null;
/**
* 构建标签处理器
* @param tagConfig 配置类
*/
public static void buildExtractHandler(TagConfig tagConfig) {
if(tagConfig.getExtract() == SUSPEND){
return;
}
ExtractHandler handler = new ExtractHandler();
handler.extract = tagConfig.getExtract();
handler.reqTags = tagConfig.getReqTags();
handler.logTags = tagConfig.getLogTags();
handler.respTags = tagConfig.getRespTags();
String regex = tagConfig.getRegex();
if(StringUtil.isNotBlank(regex)){
handler.pattern = Pattern.compile(regex);
}else{
String escapeDelimiter = ConfigUtil.escapeExprSpecialWord(tagConfig.getDelimiter());
String str = String.format("(%s[\\w\\W]*?%s)", escapeDelimiter, escapeDelimiter);
handler.pattern = Pattern.compile(str);
}
handler.delimiter = tagConfig.getDelimiter();
handler.delimiterLen = tagConfig.getDelimiter().length();
handler.join = tagConfig.getJoin();
instance = handler;
LOGGER.info("构建标签处理器单例完成:{}",instance.toString());
}
/**
* 提取请求参数里的标签
* @param reqMap 额外附加的,如ip等
* @return tags
*/
public static Map<String, Object> extractReqTag(Map<String, Object> reqMap) {
if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; }
System.out.println("### REQ INSTANCE :"+instance.toString());
Map<String, Object> tagMap = new HashMap<>(instance.reqTags.size());
for (String tag : instance.reqTags) {
Object val = reqMap.get(tag);
if(val != null){
tagMap.put(tag, val);
}
}
System.out.println("提取到了请求入参日志标签："+JSON.toJSONString(tagMap));
return tagMap;
}
/**
* 提取普通log里标签
* @param content 内容
* @return tags
*/
public static Map<String, Object> extractLogTag(String content) {
if(instance == null || !isMatched(instance.extract, E_LOG) || content.length() < EXTRACT_MIN_LEN){
return null;
}
Map<String,Object> tagMap = new HashMap<>(3);
Matcher m = instance.pattern.matcher(content);
while (m.find()) {
String str = m.group().substring(instance.delimiterLen, m.group().length() - instance.delimiterLen);
if(str.contains(instance.join)){
String[] arr = str.split(instance.join);
if(instance.logTags.contains(arr[0])){
tagMap.put(arr[0], arr[1]);
}
}else if(str.length() < TAG_NORMAL_KEY_MAX_LEN){
tagMap.put(TAG_NORMAL_KEY, str);
}else{
if (RANDOM.nextInt(50) == 1) {
LOGGER.info("Some logs lack tags and are larger than 20 in length. Therefore, they are simply stored");
}
}
}
System.out.println("提取到了请求log日志标签："+JSON.toJSONString(tagMap));
return tagMap;
}
/**
* 提取返回结果的tag
* @param resp 参数
* @return map
*/
public static Map<String, Object> extractRespTag(Map<String, Object> resp) {
if(instance == null || !isMatched(instance.extract, E_REQ)){ return null; }
Map<String, Object> requestMap = new HashMap<>(instance.respTags.size());
for (String tag : instance.respTags) {
Object val = resp.get(tag);
if(val != null){
requestMap.put(tag, val);
}
}
System.out.println("提取到了请求出参日志标签："+JSON.toJSONString(requestMap));
return requestMap;
}
/**
* 刷新标签处理器 加锁是为了防止极端情况下, 先到的config1覆盖后到的config2
* @param tagConfig 新的配置
*/
public synchronized static void refresh(TagConfig tagConfig) {
instance = null;
buildExtractHandler(tagConfig);
}
@Override
public String toString() {
return "TagHandler{" +
"reqTags=" + reqTags +
", logTags=" + logTags +
", delimiter='" + delimiter + '\'' +
", delimiterLen=" + delimiterLen +
", join='" + join + '\'' +
", pattern=" + pattern +
", extract=" + extract +
'}';
}
// static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)");
public static List<String> extractTest(String content) {
TagConfig cfg = new TagConfig();
cfg.setDelimiter("|");
cfg.setJoin("=");
buildExtractHandler(cfg);
List<String> list = new ArrayList<>();
// Matcher m = BRACKET_PATTERN.matcher(content);
Matcher m = instance.pattern.matcher(content);
while (m.find()) {
list.add(m.group().substring(1, m.group().length() - 1));
// list.add(m.group().substring(2, m.group().length() - 2));
}
return list;
}
//static Pattern BRACKET_PATTERN = Pattern.compile("(\\{\\{[\\w\\W]*?\\}\\})");
// static Pattern BRACKET_PATTERN = Pattern.compile("(\\|\\|[\\w\\W]*?\\|\\|)");
static Pattern BRACKET_PATTERN = Pattern.compile("(\\|[\\w\\W]*?\\|)");
static String str1 = "|errno=val3||node=val4||这是随便的log|";
static String str2 = "||a=1||b=2||qwewe||";
static String str3 = "||a=1||eee||b=2";
static String str4 = "||a=1||eee||b=2||";
public static void main(String[] args) {
//### INSTANCE:TagHandler{reqTags=[uid, url], logTags=[node, bizType],
// delimiter='|', delimiterLen=1, join='"="', pattern="", extract=41}
//### .content:|errno=val3||node=val4||这是随便的log|
System.out.println("msgByRegular1==> "+JSON.toJSONString(extractTest(str1)));
/* System.out.println("msgByRegular2==> "+JSON.toJSONString(extractTest(str2)));
System.out.println("msgByRegular3==> "+JSON.toJSONString(extractTest(str3)));
System.out.println("msgByRegular4==> "+JSON.toJSONString(extractTest(str4)));*/
}
}

93
common/src/main/java/com/jd/platform/jlog/common/handler/JcProperties.java Normal file
View File

@ -0,0 +1,93 @@
package com.jd.platform.jlog.common.handler;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.utils.ConfigUtil;
import com.jd.platform.jlog.common.utils.FastJsonUtils;
import com.jd.platform.jlog.common.utils.StringUtil;
import java.lang.reflect.Field;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import static com.jd.platform.jlog.common.utils.ConfigUtil.invoke;
import static com.jd.platform.jlog.common.utils.ConfigUtil.lowerFirst;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName JcProperties.java
* @Description TODO
* @createTime 2022年03月07日 12:43:00
*/
public class JcProperties extends Properties {
public JcProperties() {}
public String getString(String key) {
Object val = get(key);
if(val == null){
return null;
}
return String.valueOf(val);
}
public Long getLong(String key) {
String val = getString(key);
if(val == null){
return null;
}
return Long.parseLong(val);
}
public List<String> getStrList(String key) {
if(StringUtil.isEmpty(key)){
return null;
}
String val = getString(key);
if(StringUtil.isNotEmpty(val)){
return FastJsonUtils.toList(val,String.class);
}
return null;
}
/* public List<String> getStrList(String key) {
if(StringUtil.isEmpty(key)){
return null;
}
int index = 0;
String suffix;
String fastSuffix;
List<String> list = new ArrayList<>();
do{
suffix = "["+index+"]";
fastSuffix = "["+(index+1)+"]";
list.add(getString(key+suffix));
index ++;
}while (getString(key + fastSuffix) != null);
return list;
}*/
public <T> T getBean(String key, Class<T> clz) {
T bean = FastJsonUtils.toBean(JSON.toJSONString(get(key)), clz);
if(bean != null){
return bean;
}
try {
T instance = clz.newInstance();
invoke(instance, this, "");
return instance;
}catch (Exception e){
e.printStackTrace();
}
return null;
}
}

25
common/src/main/java/com/jd/platform/jlog/common/handler/Tag.java Normal file
View File

@ -0,0 +1,25 @@
package com.jd.platform.jlog.common.handler;
import java.lang.annotation.*;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName Tag.java
* @Description TODO 后续完善使用
* @createTime 2022年02月24日 20:21:00
*/
@Documented
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface Tag {
boolean extractReq();
boolean compressReq();
boolean extractResp();
boolean compressResp();
}

123
common/src/main/java/com/jd/platform/jlog/common/handler/TagConfig.java Normal file
View File

@ -0,0 +1,123 @@
package com.jd.platform.jlog.common.handler;
import java.io.Serializable;
import java.util.List;
import static com.jd.platform.jlog.common.handler.CollectMode.COMPRESS_ALL;
import static com.jd.platform.jlog.common.handler.CollectMode.EXTRACT_ALL;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName TagConfig.java
* @createTime 2022年02月13日 22:35:00
*/
public class TagConfig implements Serializable {
/**
* 入参的tag
*/
private List<String> reqTags;
/**
* 普通日志的tag
*/
private List<String> logTags;
/**
* 返参的tag
*/
private List<String> respTags;
/**
* 自定义正则 可为空
*/
private String regex;
/**
* 分隔符 默认｜
*/
private String delimiter = "|";
/**
* 连接符 默认=
*/
private String join = "=";
/**
* 提取策略
*/
private long extract = EXTRACT_ALL;
public List<String> getReqTags() {
return reqTags;
}
public void setReqTags(List<String> reqTags) {
this.reqTags = reqTags;
}
public List<String> getLogTags() {
return logTags;
}
public void setLogTags(List<String> logTags) {
this.logTags = logTags;
}
public List<String> getRespTags() {
return respTags;
}
public void setRespTags(List<String> respTags) {
this.respTags = respTags;
}
public String getRegex() {
return regex;
}
public void setRegex(String regex) {
this.regex = regex;
}
public String getDelimiter() {
return delimiter;
}
public void setDelimiter(String delimiter) {
this.delimiter = delimiter;
}
public String getJoin() {
return join;
}
public void setJoin(String join) {
this.join = join;
}
public long getExtract() {
return extract;
}
public void setExtract(long extract) {
this.extract = extract;
}
@Override
public String toString() {
return "TagConfig{" +
"reqTags=" + reqTags +
", logTags=" + logTags +
", respTags=" + respTags +
", regex='" + regex + '\'' +
", delimiter='" + delimiter + '\'' +
", join='" + join + '\'' +
", extract='" + extract + '\'' +
'}';
}
}

15
common/src/main/java/com/jd/platform/jlog/common/model/RunLogMessage.java
View File

@ -1,5 +1,7 @@
package com.jd.platform.jlog.common.model;
import java.util.Map;
/**
* className：RunLogMessage
* description：
@ -37,6 +39,11 @@ public class RunLogMessage {
*/
private String threadName;
/**
* 标签map
*/
private Map<String,Object> tagMap;
public long getTracerId() {
return tracerId;
}
@ -92,4 +99,12 @@ public class RunLogMessage {
public void setThreadName(String threadName) {
this.threadName = threadName;
}
public Map<String, Object> getTagMap() {
return tagMap;
}
public void setTagMap(Map<String, Object> tagMap) {
this.tagMap = tagMap;
}
}

2
common/src/main/java/com/jd/platform/jlog/common/model/TracerBean.java
View File

@ -25,7 +25,7 @@ public class TracerBean implements Serializable {
private String tracerId;
/**
* tracer对象，里面放的是List<Map<String, Object>>
* 第一个元素是request对象，key为request，value为完整request入参，从中可找到pin、uuid
* 第一个元素是request对象，key为request，value为完整request入参，从中可找到uuid
* 最后一个元素是response响应，key为response，value为响应值byte[]，可转为String入库
*/
private List<Map<String, Object>> tracerObject;

5
common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java
View File

@ -1,7 +1,10 @@
package com.jd.platform.jlog.common.utils;
import java.util.Collection;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* 异步线程池AsyncPool
@ -19,4 +22,6 @@ public class AsyncPool {
public static void shutDown() {
threadPoolExecutor.shutdown();
}
}

37
common/src/main/java/com/jd/platform/jlog/common/utils/AsyncWorker.java Normal file
View File

@ -0,0 +1,37 @@
package com.jd.platform.jlog.common.utils;
import java.util.Collection;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName AsyncWorker.java
* @Description TODO
* @createTime 2022年02月21日 17:10:00
*/
public class AsyncWorker {
public static <E> int drain(BlockingQueue<E> q, Collection<? super E> buffer, int numElements, long timeout, TimeUnit unit) throws Exception {
if(buffer == null){
throw new Exception("[Assertion failed] - the buffer argument cannot be null");
}
long deadline = System.nanoTime() + unit.toNanos(timeout);
int added = 0;
while(added < numElements) {
added += q.drainTo(buffer, numElements - added);
if (added < numElements) {
E e = q.poll(deadline - System.nanoTime(), TimeUnit.NANOSECONDS);
if (e == null) {
break;
}
buffer.add(e);
++added;
}
}
return added;
}
}

219
common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java Normal file
View File

@ -0,0 +1,219 @@
package com.jd.platform.jlog.common.utils;
import com.alibaba.fastjson.JSON;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName CollectionUtil.java
* @Description TODO
* @createTime 2022年02月21日 17:14:00
*/
public class CollectionUtil {
public static boolean isEmpty(Collection<?> collection) {
return collection == null || collection.isEmpty();
}
public static boolean isNotEmpty(Collection<?> collection) {
return collection != null && collection.isEmpty();
}
public static boolean isEmpty(Map<?, ?> map) {
return !isNotEmpty(map);
}
public static boolean isNotEmpty(Map<?, ?> map) {
return map != null && !map.isEmpty();
}
public static <T> boolean equals(List<T> c1, List<T> c2) {
if (c1 == null && c2 == null) {
return true;
} else if (c1 != null && c2 != null) {
if (c1.size() != c2.size()) {
return false;
} else {
for(int i = 0; i < c1.size(); ++i) {
if (!Objects.equals(c1.get(i), c2.get(i))) {
return false;
}
}
return true;
}
} else {
return false;
}
}
public static <K, V> boolean equals(Map<K, V> c1, Map<K, V> c2) {
if (c1 == null && c2 == null) {
return true;
} else if (c1 != null && c2 != null) {
if (c1.size() != c2.size()) {
return false;
} else {
Iterator var2 = c1.entrySet().iterator();
Object v1;
Object v2;
do {
if (!var2.hasNext()) {
return true;
}
Map.Entry<K, V> entry = (Map.Entry)var2.next();
K k = entry.getKey();
v1 = entry.getValue();
v2 = c2.get(k);
} while(Objects.equals(v1, v2));
return false;
}
} else {
return false;
}
}
public static <K, V> Set<String> diffKeys(Map<K, V> m1, Map<K, V> m2){
Set<String> diff = new HashSet<>(1);
for (Map.Entry<K,V> kvEntry : m1.entrySet()) {
V val = m2.get(kvEntry.getKey());
if(!kvEntry.getValue().equals(val)){
diff.add(kvEntry.getKey().toString());
}
}
for (Map.Entry<K,V> kvEntry : m2.entrySet()) {
V val = m1.get(kvEntry.getKey());
if(!kvEntry.getValue().equals(val)){
diff.add(kvEntry.getKey().toString());
}
}
return diff;
}
public static <K, V> HashMap<K,V> diffMap(Map<K, V> m1, Map<K, V> m2){
HashMap<K, V> diff = new HashMap<>(1);
for (Map.Entry<K,V> kvEntry : m1.entrySet()) {
V val = m2.get(kvEntry.getKey());
if(!kvEntry.getValue().equals(val)){
diff.put(kvEntry.getKey(), kvEntry.getValue());
}
}
for (Map.Entry<K,V> kvEntry : m2.entrySet()) {
V val = m1.get(kvEntry.getKey());
if(!kvEntry.getValue().equals(val)){
diff.put(kvEntry.getKey(), kvEntry.getValue());
}
}
return diff;
}
public static void main(String[] args) {
int fail = 0;
int count = 10000;
Set<Integer> set = new HashSet<>();
for (int i = 0; i < count; i++) {
int h = randomMac4Qemu().hashCode();
int val = indexFor(h);
boolean r = set.add(val);
// System.out.println(val);
if(!r){
fail++;
}
}
int fail2 = 0;
Set<Integer> set2 = new HashSet<>();
for (int i = 0; i < count; i++) {
int h = randomMac4Qemu().hashCode();
Random rd = new Random(h);
int val = rd.nextInt(16385);
boolean r = set.add(val);
System.out.println(val);
if(!r){
fail2++;
}
}
System.out.println("11--》 "+fail);
System.out.println("22-》 "+ fail2);
}
static int indexFor(int h) {
return h & (16384-1);
}
public static String randomMac4Qemu() {
/* if(1==1){
return getRandomIp();
}*/
Random random = new Random();
String[] mac = {
/* String.format("%02x", 0x52),
String.format("%02x", 0x54),
String.format("%02x", 0x00),*/
String.format("%02x", random.nextInt(0xff)),
String.format("%02x", random.nextInt(0xff)),
String.format("%02x", random.nextInt(0xff))
};
return String.join(":", mac);
}
public static String getRandomIp(){
//ip范围
int[][] range = {{607649792,608174079},//36.56.0.0-36.63.255.255
{1038614528,1039007743},//61.232.0.0-61.237.255.255
{1783627776,1784676351},//106.80.0.0-106.95.255.255
{2035023872,2035154943},//121.76.0.0-121.77.255.255
{2078801920,2079064063},//123.232.0.0-123.235.255.255
{-1950089216,-1948778497},//139.196.0.0-139.215.255.255
{-1425539072,-1425014785},//171.8.0.0-171.15.255.255
{-1236271104,-1235419137},//182.80.0.0-182.92.255.255
{-770113536,-768606209},//210.25.0.0-210.47.255.255
{-569376768,-564133889}, //222.16.0.0-222.95.255.255
};
Random rdint = new Random();
int index = rdint.nextInt(10);
return num2ip(range[index][0]+new Random().nextInt(range[index][1]-range[index][0]));
}
public static String num2ip(int ip) {
int [] b=new int[4] ;
String x = "";
b[0] = (int)((ip >> 24) & 0xff);
b[1] = (int)((ip >> 16) & 0xff);
b[2] = (int)((ip >> 8) & 0xff);
b[3] = (int)(ip & 0xff);
x=Integer.toString(b[0])+"."+Integer.toString(b[1])+"."+Integer.toString(b[2])+"."+Integer.toString(b[3]);
return x;
}
}

236
common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java Normal file
View File

@ -0,0 +1,236 @@
package com.jd.platform.jlog.common.utils;
import com.jd.platform.jlog.common.constant.Constant;
import com.jd.platform.jlog.common.handler.JcProperties;
import java.beans.IntrospectionException;
import java.beans.PropertyDescriptor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ConfigUtil.java
* @Description TODO
* @createTime 2022年02月13日 21:40:00
*/
public class ConfigUtil {
private static final String SERVER_SUFFIX = "Server";
public static final char MID_LINE = '-';
public static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current();
public static String escapeExprSpecialWord(String str) {
if (str != null && str.length() > 0) {
for (String s : Constant.SPECIAL_CHAR) {
if (str.contains(s)) {
str = str.replace(s, "\\" + s);
}
}
}
return str;
}
public static String formatConfigStr(Properties properties) {
StringBuilder sb = new StringBuilder();
Enumeration<?> enumeration = properties.propertyNames();
while (enumeration.hasMoreElements()) {
String key = (String) enumeration.nextElement();
Object property = properties.get(key);
if (property != null) {
property = String.valueOf(property);
}
sb.append(key).append("=").append(property).append("\n");
}
return sb.toString();
}
public static byte[] formatConfigByte(Properties properties) {
return formatConfigStr(properties).getBytes();
}
//递归解析map对象
public static void loadRecursion(Map<String, Object> map, String key, Properties props) {
map.forEach((k, v) -> {
System.out.println("k => " + k + " v.class => " + v.getClass() + " val => " + v.toString() + " 是不是爸爸 => " + isParent(v));
if (isParent(v)) {
if (v instanceof Boolean || v instanceof List) {
props.put(key + "." + k, v);
} else {
Map<String, Object> nextValue = (Map<String, Object>) v;
loadRecursion(nextValue, (("".equals(key) ? "" : key + ".") + k), props);
}
} else {
props.put(key + "." + k, v);
}
});
}
//判断是否还有子节点
public static boolean isParent(Object o) {
if (!(o instanceof String || o instanceof Character || o instanceof Byte)) {
try {
Number n = (Number) o;
// System.out.println("isParent ==> "+n);
} catch (Exception e) {
return true;
}
}
return false;
}
/*
public static ConfigCenterEnum getCenter(CenterConfig config) throws Exception {
Class<?> clz = config.getClass();
Field[] fields = clz.getDeclaredFields();
for (Field field : fields) {
Method m = clz.getMethod("get" + getMethodName(field.getName()));
String val = (String)m.invoke(config);
if (val != null) {
for (ConfigCenterEnum center : ConfigCenterEnum.values()) {
String fd = field.getName().replace(SERVER_SUFFIX, "");
if(center.name().equals(fd.toUpperCase())){
return center;
}
}
}
}
throw new Exception("Configuration center cannot be found");
}
*/
private static String getMethodName(String fieldName) {
byte[] items = fieldName.getBytes();
items[0] = (byte) ((char) items[0] - 'a' + 'A');
return new String(items);
}
public static String camelToMidline(String param) {
if (param == null || "".equals(param.trim())) {
return "";
}
int len = param.length();
StringBuilder sb = new StringBuilder(len);
for (int i = 0; i < len; i++) {
char c = param.charAt(i);
if (Character.isUpperCase(c)) {
sb.append("-");
sb.append(Character.toLowerCase(c));
} else {
sb.append(c);
}
}
return sb.toString();
}
public static String lowerFirst(String fromStr){
char[] chars = fromStr.toCharArray();
chars[0] += 32;
return String.valueOf(chars);
}
/**
* 只支持简单的对象形配置
* @param model bean
* @param properties 配置
* @param prefix 前缀
*/
@Deprecated
public static void invoke(Object model, JcProperties properties, String prefix) throws
IllegalAccessException, ClassNotFoundException, InstantiationException, ParseException {
Class<?> clz = model.getClass();
Field[] fields = model.getClass().getDeclaredFields();
for (Field field : fields) {
String type = field.getGenericType().toString();
field.setAccessible(true);
String curObjName = ConfigUtil.camelToMidline(lowerFirst(clz.getSimpleName()));
prefix = StringUtil.isEmpty(prefix) ? curObjName : prefix;
String fillName = !curObjName.equals(prefix) ? prefix +"."+ curObjName + "." + field.getName() : curObjName + "." + field.getName();
switch (type){
case "class java.lang.String":
field.set(model, properties.getString(fillName)) ;
break;
case "byte":
field.setByte(model, Byte.valueOf(properties.getString(fillName)));
break;
case "short":
field.setShort(model, Short.valueOf(properties.getString(fillName)));
break;
case "int":
field.setInt(model, Integer.parseInt(properties.getString(fillName))) ;
break;
case "long":
field.setLong(model, properties.getLong(fillName));
break;
case "double":
field.setDouble(model, Double.valueOf(properties.getString(fillName)));
break;
case "float":
field.setFloat(model, Float.valueOf(properties.getString(fillName)));
break;
case "boolean":
field.setBoolean(model, Boolean.parseBoolean(properties.getString(fillName)));
break;
case "class java.util.Date":
Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(properties.getString(fillName));
field.set(model,date) ;
break;
default:
String tn = field.getType().getTypeName();
if("java.util.List".equals(tn)){
String[] arr = fillName.split("\\[");
int index = 0;
String suffix;
String fastSuffix;
List<String> list = new ArrayList<>();
do{
suffix = "["+index+"]";
fastSuffix = "["+(index+1)+"]";
list.add(properties.getString(arr[0]+suffix));
index ++;
}while (properties.getString(arr[0]+fastSuffix) != null);
field.set(model, list);
}else if("java.util.Map".equals(tn)){
String val = properties.getString(fillName);
field.set(model,FastJsonUtils.toMap(val));
}else if(field.getType().isArray()){
String val = properties.getString(fillName);
field.set(model,FastJsonUtils.toArray(val));
}else{
String[] ar = type.split(" ");
Object tinyObj = Class.forName(ar[1]).newInstance();
invoke(tinyObj, properties, prefix);
field.set(model,tinyObj);
}
}
}
}
}

2
common/src/main/java/com/jd/platform/jlog/common/utils/FastJsonUtils.java
View File

@ -72,7 +72,7 @@ public class FastJsonUtils {
/**
* json字符串转化为map
*/
public static <K, V> Map<K, V> stringToCollect(String s) {
public static <K, V> Map<K, V> toMap(String s) {
return (Map<K, V>) JSONObject.parseObject(s);
}

6
common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java
View File

@ -1,6 +1,6 @@
package com.jd.platform.jlog.common.utils;
import com.google.common.base.Preconditions;
import com.sun.tools.javac.util.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -86,14 +86,14 @@ public class IdWorker {
* @param workerId 工作进程Id
*/
private static void setWorkerId(final Long workerId) {
Preconditions.checkArgument(workerId >= 0L && workerId < WORKER_ID_MAX_VALUE);
Assert.check(workerId >= 0L && workerId < WORKER_ID_MAX_VALUE);
IdWorker.workerId = workerId;
}
//下一个ID生成算法
public static long nextId() {
long time = System.currentTimeMillis();
Preconditions.checkState(lastTime <= time, "Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds", lastTime, time);
Assert.check(lastTime <= time, "Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds"+lastTime);
if (lastTime == time) {
if (0L == (sequence = ++sequence & SEQUENCE_MASK)) {
time = waitUntilNextTime(time);

45
common/src/main/java/com/jd/platform/jlog/common/utils/StringUtil.java Normal file
View File

@ -0,0 +1,45 @@
package com.jd.platform.jlog.common.utils;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName StringUtil.java
* @Description TODO
* @createTime 2022年02月21日 17:13:00
*/
public class StringUtil {
public StringUtil() {
}
public static boolean isEmpty(CharSequence cs) {
return cs == null || cs.length() == 0;
}
public static boolean isNotEmpty(CharSequence cs) {
return !isEmpty(cs);
}
public static boolean isBlank(CharSequence cs) {
int strLen;
if (cs != null && (strLen = cs.length()) != 0) {
for(int i = 0; i < strLen; ++i) {
if (!Character.isWhitespace(cs.charAt(i))) {
return false;
}
}
return true;
} else {
return true;
}
}
public static boolean isNotBlank(CharSequence cs) {
return !isBlank(cs);
}
}

117
common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java
View File

@ -2,6 +2,11 @@ package com.jd.platform.jlog.common.utils;
import com.github.luben.zstd.Zstd;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* zstd压缩工具类
*
@ -39,4 +44,116 @@ public class ZstdUtils {
return ob;
}
/**
* 解压
*/
public static byte[] decompressBytes2(byte[] bytes) {
int size = (int) Zstd.decompressedSize(bytes);
System.out.println("size-> "+size);
byte[] ob = new byte[size];
Zstd.decompress(ob, bytes);
return ob;
}
public static void main(String[] args) throws InterruptedException, ClassNotFoundException, IllegalAccessException, InstantiationException {
String s = "-75";
char firstChar = s.charAt(0);
System.out.println(firstChar);
if (firstChar < '0') { // Possible leading "+" or "-"
if (firstChar == '-') {
System.out.println("=========");
} else if (firstChar != '+')
System.out.println("eeee");
}
// System.out.println(Integer.parseInt("-75"));
String str1 = "1a";
String str2 = "0a";
String dbStr = Arrays.toString(compress(str1.getBytes()));
System.out.println(dbStr);
String[] arr0 = dbStr.split(",");
byte[] bt = new byte[arr0.length];
for (int i = 0; i < arr0.length; i++) {
if(i == 0){
String first = arr0[0].replace("[", "");
bt[i] = Byte.valueOf(first);
continue;
}
if(i == arr0.length-1){
String last = arr0[arr0.length-1].replace("]", "");
bt[i] = Byte.valueOf(last.trim());
continue;
}
System.out.println(arr0[i]+" len => "+arr0[i].length());
bt[i] = Byte.parseByte(arr0[i].trim());
}
System.out.println(Arrays.toString(bt));
// System.out.println(Arrays.toString(compress(str2.getBytes())));
AtomicInteger fail = new AtomicInteger(0);
AtomicInteger ok = new AtomicInteger(0);
for (int j = 0; j < 1; j++) {
new Thread(() -> {
for (int i = 0; i < 100; i++) {
int id = new Random().nextInt(99000000);
/* String ml = String.valueOf(getRandomChar()) +id+getRandomChar();
String ml2 = LocalDateTime.now().toString()+ getRandomChar();
ml = ml2 + ml;*/
// 压缩后的
String ml = ""+i;
byte[] mlc1= compress(ml.getBytes());
//存进去new String(mlc1)
byte[] resul = new String(mlc1).getBytes();
System.out.println(Arrays.toString(new String(mlc1).getBytes()));
byte[] arr = new byte[9];
arr[0] = 40;
arr[1] = -17;
arr[2] = -65;
arr[3] = -67;
arr[4] = 47;
arr[5] = -17;
arr[6] = -65;
arr[7] = -67;
arr[8] = 32;
// System.out.println(new String(arr));
if(resul[0]==40
&& resul[1]==-17
&& resul[2]==-65
&& resul[3]==-67
&& resul[4]==47
&& resul[5]==-17
&& resul[6]==-65
&& resul[7]==-67
&& resul[8]==32 ){
ok.incrementAndGet();
}else{
fail.incrementAndGet();
}
}
}).start();
}
Thread.sleep(30000);
System.out.println("ok == > "+ok.get());
System.out.println("fail == > "+fail.get());
}
public static char getRandomChar() {
return (char) (0x4e00 + (int) (Math.random() * (0x9fa5 - 0x4e00 + 1)));
}
}

36
config/config-apollo/pom.xml Normal file
View File

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config</artifactId>
<version>1.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>config-apollo</artifactId>
<name>config-apollo</name>
<dependencies>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-core</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.ctrip.framework.apollo</groupId>
<artifactId>apollo-client</artifactId>
<version>1.9.2</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<!-- <exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>-->
</exclusions>
</dependency>
</dependencies>
</project>

165
config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfigurator.java Normal file
View File

@ -0,0 +1,165 @@
package com.jd.platform.jlog.config.apollo;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.*;
import com.alibaba.fastjson.JSON;
import com.ctrip.framework.apollo.ConfigFile;
import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
import com.ctrip.framework.apollo.core.utils.StringUtils;
import com.ctrip.framework.apollo.enums.PropertyChangeType;
import com.ctrip.framework.apollo.model.ConfigChange;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.handler.TagConfig;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.common.utils.FastJsonUtils;
import com.jd.platform.jlog.core.*;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.ctrip.framework.apollo.core.ApolloClientSystemConsts.APP_ID;
import static com.jd.platform.jlog.common.utils.ConfigUtil.invoke;
import static com.jd.platform.jlog.config.apollo.ApolloConstant.*;
import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE;
/**
* @author tangbohu
* @version 1.0.0
* @Description todo env and cluster
* @ClassName ApolloConfigurator.java
* @createTime 2022年02月21日 21:21:00
*/
public class ApolloConfigurator implements Configurator {
private static final Logger LOGGER = LoggerFactory.getLogger(ApolloConfigurator.class);
private static final Configurator FILE_CONFIG = ConfiguratorFactory.base;
/**
* 里面有resourceProperties 和 configProperties
*/
private static volatile Config config;
private static volatile ApolloConfigurator instance;
private ApolloConfigurator() {
loadApolloServerConfig();
if (config == null) {
synchronized (ApolloConfigurator.class) {
if (config == null) {
// apollo的监听是按照namespace维度
config = ConfigService.getConfig(DEFAULT_NAMESPACE);
ApolloListener apolloListener = new ApolloListener();
config.addChangeListener(changeEvent -> {
LOGGER.info("Apollo收到事件变更, keys={}", changeEvent.changedKeys());
for (String key : changeEvent.changedKeys()) {
ConfigChange change = changeEvent.getChange(key);
ConfigChangeEvent event = new ConfigChangeEvent(key, change.getNamespace(), change.getOldValue(), change.getNewValue(), getChangeType(change.getChangeType()));
apolloListener.onProcessEvent(event);
}
});
}
}
}
LOGGER.info("Apollo配置器构建完成");
}
public static ApolloConfigurator getInstance() {
if (instance == null) {
synchronized (ApolloConfigurator.class) {
if (instance == null) {
instance = new ApolloConfigurator();
}
}
}
return instance;
}
@Override
public String getString(String key) {
return config.getProperty(key, null);
}
@Override
public Long getLong(String key) {
return config.getLongProperty(key,null);
}
@Override
public List<String> getList(String key) {
return FastJsonUtils.toList(config.getProperty(key,""), String.class) ;
}
@Override
public <T> T getObject(String key, Class<T> clz) {
return FastJsonUtils.toBean(config.getProperty(key,""), clz);
}
@Override
public boolean putConfig(String key, String content) {
return false;
}
@Override
public boolean putConfig(String dataId, String content, long timeoutMills) {
return false;
}
@Override
public String getType() {
return "apollo";
}
private ConfigChangeType getChangeType(PropertyChangeType changeType) {
switch (changeType) {
case ADDED:
return ConfigChangeType.ADD;
case DELETED:
return ConfigChangeType.DELETE;
default:
return ConfigChangeType.MODIFY;
}
}
private void loadApolloServerConfig() {
Properties properties = System.getProperties();
if (!properties.containsKey(PROP_APP_ID)) {
System.setProperty(PROP_APP_ID, FILE_CONFIG.getString(APP_ID));
}
if (!properties.containsKey(PROP_APOLLO_META)) {
System.setProperty(PROP_APOLLO_META, FILE_CONFIG.getString(APOLLO_META));
}
if (!properties.containsKey(PROP_APOLLO_SECRET)) {
String secretKey = FILE_CONFIG.getString(APOLLO_SECRET);
if (!StringUtils.isBlank(secretKey)) {
System.setProperty(PROP_APOLLO_SECRET, secretKey);
}
}
if (!properties.containsKey(APOLLO_CLUSTER)) {
if (!StringUtils.isBlank(FILE_CONFIG.getString(APOLLO_CLUSTER))) {
System.setProperty(PROP_APOLLO_CLUSTER, FILE_CONFIG.getString(APOLLO_CLUSTER));
}
}
if (!properties.containsKey(APOLLO_CONFIG_SERVICE)) {
System.setProperty(PROP_APOLLO_CONFIG_SERVICE, FILE_CONFIG.getString(APOLLO_CONFIG_SERVICE));
}
}
}

18
config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConfiguratorProvider.java Normal file
View File

@ -0,0 +1,18 @@
package com.jd.platform.jlog.config.apollo;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorProvider;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ApolloConfiguratorProvider.java
* @createTime 2022年02月21日 21:26:00
*/
public class ApolloConfiguratorProvider implements ConfiguratorProvider {
@Override
public Configurator build() {
return ApolloConfigurator.getInstance();
}
}

20
config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloConstant.java Normal file
View File

@ -0,0 +1,20 @@
package com.jd.platform.jlog.config.apollo;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ApolloConstant.java
* @createTime 2022年02月22日 11:06:00
*/
public class ApolloConstant {
static final String APOLLO_META = "apolloMeta";
static final String APOLLO_SECRET = "apolloAccesskeySecret";
static final String APOLLO_CLUSTER = "jLog";
static final String APOLLO_CONFIG_SERVICE = "apollo.config-service";
static final String PROP_APP_ID = "app.id";
static final String PROP_APOLLO_META = "apollo.meta";
static final String PROP_APOLLO_CONFIG_SERVICE = "apollo.config-service";
static final String PROP_APOLLO_SECRET = "apollo.accesskey.secret";
static final String PROP_APOLLO_CLUSTER = "apollo.cluster";
}

19
config/config-apollo/src/main/java/com/jd/platform/jlog/config/apollo/ApolloListener.java Normal file
View File

@ -0,0 +1,19 @@
package com.jd.platform.jlog.config.apollo;
import com.jd.platform.jlog.core.ClientHandlerBuilder;
import com.jd.platform.jlog.core.ConfigChangeEvent;
import com.jd.platform.jlog.core.ConfigChangeListener;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ApolloListener.java
* @createTime 2022年02月22日 19:18:00
*/
public class ApolloListener implements ConfigChangeListener {
@Override
public void onChangeEvent(ConfigChangeEvent event) {
ClientHandlerBuilder.refresh();
}
}

1
config/config-apollo/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider Normal file
View File

@ -0,0 +1 @@
com.jd.platform.jlog.config.apollo.ApolloConfiguratorProvider

26
config/config-core/pom.xml Normal file
View File

@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config</artifactId>
<version>1.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>config-core</artifactId>
<name>config-core</name>
<dependencies>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>common</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.28</version>
</dependency>
</dependencies>
</project>

63
config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandler.java Normal file
View File

@ -0,0 +1,63 @@
package com.jd.platform.jlog.core;
import com.jd.platform.jlog.common.handler.CompressHandler;
import com.jd.platform.jlog.common.handler.ExtractHandler;
import java.util.Map;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ClientHandler.java
* @createTime 2022年03月13日 16:53:00
*/
public class ClientHandler {
public static Map<String, Object> processReq(Map<String, Object> reqMap){
Map<String, Object> map = ExtractHandler.extractReqTag(reqMap);
return CompressHandler.compressReq(map);
}
public static Outcome processLog(String content){
Map<String, Object> tagMap = ExtractHandler.extractLogTag(content);
return new Outcome(tagMap, null);
}
public static Outcome processResp(byte[] resp, Map<String, Object> respMap){
Map<String, Object> tagMap = ExtractHandler.extractRespTag(respMap);
byte[] newContent = CompressHandler.compressResp(resp);
return new Outcome(tagMap, newContent);
}
public static class Outcome{
Map<String, Object> map;
byte[] content;
public Outcome(Map<String, Object> map, byte[] content) {
this.map = map;
this.content = content;
}
public Map<String, Object> getMap() {
return map;
}
public void setMap(Map<String, Object> map) {
this.map = map;
}
public byte[] getContent() {
return content;
}
public void setContent(byte[] content) {
this.content = content;
}
}
}

44
config/config-core/src/main/java/com/jd/platform/jlog/core/ClientHandlerBuilder.java Normal file
View File

@ -0,0 +1,44 @@
package com.jd.platform.jlog.core;
import com.jd.platform.jlog.common.handler.CompressHandler;
import com.jd.platform.jlog.common.handler.ExtractHandler;
import com.jd.platform.jlog.common.handler.TagConfig;
import static com.jd.platform.jlog.common.constant.Constant.THRESHOLD;
import static com.jd.platform.jlog.common.handler.CollectMode.COMPRESS_LOG_RESP;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ClientHandlerBuilder.java
* @Description TODO
* @createTime 2022年03月05日 22:07:00
*/
public class ClientHandlerBuilder {
public static void buildHandler(TagConfig tagConfig, Configurator configurator){
if(tagConfig == null){
tagConfig = buildTagConfigByConfigurator(configurator);
}
ExtractHandler.buildExtractHandler(tagConfig);
CompressHandler.buildCompressHandler(configurator.getLong("compress"), configurator.getLong("threshold"));
}
public static void refresh(){
Configurator configurator = ConfiguratorFactory.getInstance();
if(configurator == null){
throw new RuntimeException("configurator is null");
}
ExtractHandler.refresh(buildTagConfigByConfigurator(configurator));
CompressHandler.refresh(configurator.getLong("compress"), configurator.getLong("threshold") );
}
private static TagConfig buildTagConfigByConfigurator(Configurator configurator){
return configurator.getObject("tag-config", TagConfig.class);
}
}

107
config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeEvent.java Normal file
View File

@ -0,0 +1,107 @@
package com.jd.platform.jlog.core;
import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ConfigChangeEvent.java
* @createTime 2022年02月16日 22:13:00
*/
public class ConfigChangeEvent {
private String key;
private String oldValue;
private String newValue;
private String namespace;
private ConfigChangeType changeType;
public ConfigChangeEvent(){
}
public ConfigChangeEvent(String key, String newValue) {
this(key, DEFAULT_NAMESPACE, null, newValue, ConfigChangeType.MODIFY);
}
public ConfigChangeEvent(String key, String oldValue, String newValue){
this(key, DEFAULT_NAMESPACE, oldValue, newValue, ConfigChangeType.MODIFY);
}
public ConfigChangeEvent(String key, String namespace, String oldValue, String newValue,
ConfigChangeType type) {
this.key = key;
this.namespace = namespace;
this.oldValue = oldValue;
this.newValue = newValue;
this.changeType = type;
}
public String getKey() {
return key;
}
public ConfigChangeEvent setKey(String key) {
this.key = key;
return this;
}
public String getOldValue() {
return oldValue;
}
public ConfigChangeEvent setOldValue(String oldValue) {
this.oldValue = oldValue;
return this;
}
public String getNewValue() {
return newValue;
}
public ConfigChangeEvent setNewValue(String newValue) {
this.newValue = newValue;
return this;
}
public ConfigChangeType getChangeType() {
return changeType;
}
public ConfigChangeEvent setChangeType(ConfigChangeType changeType) {
this.changeType = changeType;
return this;
}
public String getNamespace() {
return namespace;
}
public ConfigChangeEvent setNamespace(String namespace) {
this.namespace = namespace;
return this;
}
@Override
public String toString() {
return "ConfigChangeEvent{" +
"key='" + key + '\'' +
", oldValue='" + oldValue + '\'' +
", newValue='" + newValue + '\'' +
", namespace='" + namespace + '\'' +
", changeType=" + changeType +
'}';
}
}

82
config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeListener.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright 1999-2019 Seata.io Group.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jd.platform.jlog.core;
import io.netty.util.concurrent.DefaultThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.*;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ConfigChangeListener.java
* @createTime 2022年02月16日 12:06:00
*/
public interface ConfigChangeListener {
Logger LOGGER = LoggerFactory.getLogger(ConfigChangeListener.class);
ExecutorService EXECUTOR_SERVICE = new ThreadPoolExecutor(1, 1,
Integer.MAX_VALUE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DefaultThreadFactory("configListener", 1));
/**
* 事件具体处理
* @param event event
*/
void onChangeEvent(ConfigChangeEvent event);
/**
* 处理变更事件
* @param event 事件
*/
default void onProcessEvent(ConfigChangeEvent event) {
LOGGER.info("通用监听器配置处理器", event.toString());
getExecutorService().submit(() -> {
beforeEvent();
try {
onChangeEvent(event);
} catch (Exception e) {
e.printStackTrace();
}
afterEvent();
});
}
default void onShutDown() {
getExecutorService().shutdown();
}
default ExecutorService getExecutorService() {
return EXECUTOR_SERVICE;
}
default void beforeEvent() {
}
default void afterEvent() {
}
}

14
config/config-core/src/main/java/com/jd/platform/jlog/core/ConfigChangeType.java Normal file
View File

@ -0,0 +1,14 @@
package com.jd.platform.jlog.core;
/**
* @author tangbohu
*/
public enum ConfigChangeType {
ADD,
MODIFY,
DELETE
}

69
config/config-core/src/main/java/com/jd/platform/jlog/core/Configurator.java Normal file
View File

@ -0,0 +1,69 @@
package com.jd.platform.jlog.core;
import java.util.List;
/**
* @author tangbohu
* @version 1.0.0
* @desc 参考log4J
* @ClassName Configurator.java
* @createTime 2022年02月15日 17:06:00
*/
public interface Configurator {
/**
* 获取string配置
* @param key key
* @return val
*/
String getString(String key);
/**
* 获取LONG配置
* @param key key
* @return val
*/
Long getLong(String key);
/**
* 获取LIST类型配置
* @param key key
* @return val
*/
List<String> getList(String key);
/**
* 获取实体类型配置
* @param key key
* @return val
*/
<T> T getObject(String key, Class<T> clz);
/**
* 设置配置
* @param key key
* @param content val
* @return content val
*/
boolean putConfig(String key, String content);
/**
* 设置配置
* @param key key
* @return content val
* @param timeoutMills timeoutMills
* @param content content
*/
boolean putConfig(String key, String content, long timeoutMills);
/**
* 获取配置器类型
* @return string example:apollo
*/
String getType();
}

77
config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorFactory.java Normal file
View File

@ -0,0 +1,77 @@
package com.jd.platform.jlog.core;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ServiceLoader;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* @author tangbohu
* @version 1.0.0
* @desc 参考log4J
* @ClassName ConfiguratorFactory.java
* @createTime 2022年02月15日 21:54:00
*/
public class ConfiguratorFactory {
private static final Logger LOGGER = LoggerFactory.getLogger(ConfiguratorFactory.class);
private static volatile Configurator instance = null;
public static volatile Configurator base = null;
public static AtomicBoolean useFileConfig = new AtomicBoolean(true);
public static Configurator getInstance(){
if (instance == null) {
synchronized (Configurator.class) {
if (instance == null) {
instance = buildConfiguration();
LOGGER.info("构建总配置器单例完成 instance 获取类型结果:{}", instance.getType());
LOGGER.info("构建总配置器单例完成 Base 获取serverAddr结果:{}", base.getString("serverAddr"));
}
}
}
return instance;
}
/**
* SPI实现装载不同的配置器
* @return 配置器
*/
private static Configurator buildConfiguration() {
try {
/*
配置文件的配置器
*/
base = new FileConfigurator();
} catch (IOException e) {
LOGGER.info("文件配置器构建失败", e);
throw new RuntimeException("build file buildConfiguration fail", e);
}
/*
配置中心的配置器，如果没有就用文件配置器
*/
ServiceLoader<ConfiguratorProvider> builders = ServiceLoader.load(ConfiguratorProvider.class);
//noinspection LoopStatementThatDoesntLoop
for (ConfiguratorProvider provider : builders) {
LOGGER.info("配置中心的配置器获取成功, 类型为:{}", provider.build().getType());
useFileConfig.set(false);
return provider.build();
}
return base;
}
}

14
config/config-core/src/main/java/com/jd/platform/jlog/core/ConfiguratorProvider.java Normal file
View File

@ -0,0 +1,14 @@
package com.jd.platform.jlog.core;
/**
* @author tangbohu
*/
public interface ConfiguratorProvider {
/**
* 根据实现类构建配置器
* @return 配置器
*/
Configurator build();
}

52
config/config-core/src/main/java/com/jd/platform/jlog/core/Constant.java Normal file
View File

@ -0,0 +1,52 @@
package com.jd.platform.jlog.core;
import java.util.HashSet;
import java.util.Set;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName Constant.java
* @Description TODO
* @createTime 2022年02月26日 10:19:00
*/
public class Constant {
/**
* 监听重读配置文件间隔 单位ms
*/
static final long LISTENER_CONFIG_INTERVAL = 10000;
static final String CONFIG_FILE_PROPERTIES = "/application.properties";
static final String CONFIG_FILE_YML = "/application.yml";
static final String JLOG_CONFIG_FILE = "/jLog.properties";
static final String ENV = "env";
static final String YML = "yml";
public static final long DEFAULT_TIMEOUT = 2 * 1000;
static final long AWAIT_TIME = 3 * 1000;
public static final String SERVER_ADDR_KEY = "serverAddr";
public static final String NAMESPACE_KEY = "namespace";
public static final String DEFAULT_NAMESPACE = "jLog";
/**
* 配置文件集合
*/
public static final Set<String> CONFIG_FILES = new HashSet<String>() {
{
add(CONFIG_FILE_PROPERTIES);
add(CONFIG_FILE_YML);
add(JLOG_CONFIG_FILE);
}
};
}

246
config/config-core/src/main/java/com/jd/platform/jlog/core/FileConfigurator.java Normal file
View File

@ -0,0 +1,246 @@
package com.jd.platform.jlog.core;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.common.utils.StringUtil;
import io.netty.util.concurrent.DefaultThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import java.io.*;
import java.net.URL;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import static com.jd.platform.jlog.core.ConfigChangeListener.EXECUTOR_SERVICE;
import static com.jd.platform.jlog.core.ConfiguratorFactory.useFileConfig;
import static com.jd.platform.jlog.core.Constant.*;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName FileConfigurator.java
* @createTime 2022年02月17日 23:22:00
*/
public class FileConfigurator implements Configurator {
private static final Logger LOGGER = LoggerFactory.getLogger(FileConfigurator.class);
private static JcProperties PROPERTIES = new JcProperties();
private static volatile FileListener FILELISTENER = null;
private static final Set<String> LISTENED_FILES = new CopyOnWriteArraySet<>();
private final Map<String, FileWrapper> FILE_MODIFY_MAP = new ConcurrentHashMap<>();
FileConfigurator() throws IOException{
String env = System.getenv(ENV);
for (String file : CONFIG_FILES) {
String fileName = StringUtil.isEmpty(env) ? file : file + "_" + env;
URL url = this.getClass().getResource(file);
if(url == null){
continue;
}
try (InputStream is = url.openStream()) {
JcProperties props = new JcProperties();
if (fileName.contains(YML)) {
props.putAll(new Yaml().loadAs(is, Map.class));
} else {
props.load(is);
}
FILE_MODIFY_MAP.put(fileName, new FileWrapper(new File(url.getFile()).lastModified(), props));
PROPERTIES.putAll(props);
LOGGER.info("{}配置文件配置:{}", file, props.toString());
}
}
LOGGER.info("合并后的配置:{}",PROPERTIES.toString());
for (String file : CONFIG_FILES) {
file = StringUtil.isEmpty(env) ? file : file + "_" + env;
LISTENED_FILES.add(file);
if(FILELISTENER == null){
synchronized (FileConfigurator.class){
FILELISTENER = new FileListener();
FILELISTENER.addListener();
}
}
}
}
@Override
public String getString(String key) {
return PROPERTIES.getString(key);
}
@Override
public Long getLong(String key) {
return PROPERTIES.getLong(key);
}
@Override
public List<String> getList(String key) {
return PROPERTIES.getStrList(key);
}
@Override
public <T> T getObject(String key, Class<T> clz) {
return PROPERTIES.getBean(key, clz);
}
@Override
public boolean putConfig(String key, String content) { return false; }
@Override
public boolean putConfig(String key, String content, long timeoutMills) {
return false;
}
@Override
public String getType() {
return "file";
}
class FileListener implements ConfigChangeListener {
private final ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(),
new DefaultThreadFactory("fileListener", 1));
FileListener() {}
synchronized void addListener() {
FILELISTENER.onProcessEvent(new ConfigChangeEvent());
}
@Override
public void onChangeEvent(ConfigChangeEvent event) {
for(; !getExecutorService().isShutdown() && !getExecutorService().isShutdown(); checkAndConfigure()) {
try {
if(!useFileConfig.get()){
LOGGER.info("装配了配置中心, 文件配置器关闭");
executor.shutdown();
if(!executor.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)){
// 超时的时候向线程池中所有的线程发出中断(interrupted)。
executor.shutdownNow();
return;
}
}
Thread.sleep(LISTENER_CONFIG_INTERVAL);
} catch (InterruptedException ignored) {
}
}
}
@Override
public ExecutorService getExecutorService() {
return executor;
}
}
/**
* 检测文件最后修改时间 和重载文件
*/
private void checkAndConfigure(){
AtomicBoolean change = new AtomicBoolean(false);
Map<String, FileWrapper> newModifyMap = checkAndReload();
FILE_MODIFY_MAP.forEach((k, v)->{
FileWrapper newFile = newModifyMap.get(k);
if(newFile != null && newFile.change){
Set<String> diffKeys = CollectionUtil.diffKeys(newFile.props, v.props);
if(!diffKeys.isEmpty()){
change.set(true);
for (String diffKey : diffKeys) {
LOGGER.warn("文件 {} 配置变更 key={}变更事件:{}", k, diffKey, new ConfigChangeEvent(diffKey, String.valueOf(v.props.get(diffKey)), String.valueOf(newFile.props.get(diffKey))));
}
v.props = newFile.props;
v.lastModify= newFile.lastModify;
}
}
});
if(change.get()){
LOGGER.debug("变更之前的总配置：{}", JSON.toJSONString(PROPERTIES));
PROPERTIES.clear();
FILE_MODIFY_MAP.forEach((k,v)-> PROPERTIES.putAll(v.props));
LOGGER.info("变更之后的总配置：{}", JSON.toJSONString(PROPERTIES));
ClientHandlerBuilder.refresh();
}
}
/**
* 检测文件最后修改时间 和重载文件
*/
private Map<String, FileWrapper> checkAndReload() {
Map<String, FileWrapper> fileWrapperMap = new ConcurrentHashMap<>(3);
for (String fileName : LISTENED_FILES) {
URL url = this.getClass().getResource(fileName);
if(url == null){
continue;
}
File file = new File(url.getFile());
long curLastMod = file.lastModified();
FileWrapper wrapper = FILE_MODIFY_MAP.get(fileName);
Long cacheLastMod = wrapper.lastModify;
if(curLastMod <= cacheLastMod){
continue;
}
JcProperties props = new JcProperties();
try (InputStream is = url.openStream()) {
if (fileName.contains(YML)) {
props.putAll(new Yaml().loadAs(is, Map.class));
} else {
props.load(is);
}
fileWrapperMap.put(fileName, new FileWrapper(curLastMod, true, props));
LOGGER.info("定时任务监测到文件更新：{}最新配置:{}", fileName, JSON.toJSONString(props));
} catch (Exception e) {
LOGGER.error("定时任务load失败：{}配置文件配置重载失败", fileName, e);
}
}
return fileWrapperMap;
}
static class FileWrapper{
private long lastModify;
private boolean change;
private JcProperties props;
FileWrapper(long lastModify, JcProperties props) {
this.lastModify = lastModify;
this.change = false;
this.props = props;
}
FileWrapper(long lastModify, boolean change, JcProperties props) {
this.lastModify = lastModify;
this.change = change;
this.props = props;
}
}
}

81
config/config-core/src/main/java/com/jd/platform/jlog/core/FileNode.java Normal file
View File

@ -0,0 +1,81 @@
package com.jd.platform.jlog.core;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.handler.CollectMode;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.handler.TagConfig;
import com.jd.platform.jlog.common.utils.ZstdUtils;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Base64;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName FileNode.java
* @createTime 2022年03月08日 16:32:00
*/
public final class FileNode {
private String fillPath;
private long lastModity;
public static void main(String[] args) throws IOException {
String a = "汉字";
System.out.println("a byte length："+a.getBytes().length);
byte[] bt1 = ZstdUtils.compress(a.getBytes());
String ckStr = new String(bt1);
String newStr1 = ZstdUtils.decompress(bt1);
System.out.println("老 字符串 bt："+ Arrays.toString(a.getBytes()));
System.out.println("新 字符串 bt："+ Arrays.toString(ckStr.getBytes()));
System.out.println("新字符串："+newStr1);
System.out.println("新字符串的byte长度："+newStr1.getBytes().length);
String str = "这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是"+"{\"@level\":\"info\",\"@message\":\"response json: \",\"@module\":\"testing-platform.cfeature_plugin\",\"@timestamp\":\"2022-03-10T19:37:55.181928+08:00\",这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是这是实打实分身乏术的故事大哥大哥大哥算法算法发生大发DVD放v都是" +
"\"EXTRA_VALUE_AT_END\":{\"data\":{\"accountAddr\":\"bg\",\"accountArea\":\"\",\"accountBank\":\"012\",\"accountBankBranch\":\"\",\"accountBankBranchNo\":\"\",\"accountBankName\":\"BBVA BANCOMER SA\",\"accountBranch\":\"\",\"accountCardDigit\":\"\",\"accountCardType\":null,\"accountCardno\":\"012420015684144165\",\"accountCity\":\"\",\"accountCompany\":\"\",\"accountFirstName\":\"\",\"accountGrantPhoto\":{\"keys\":[\"do1_tQxmThhy446Wf8JLHK5E\"],\"urls\":[\"http://img1.didiglobal.com/static/sailing_private_b/do1_tQxmThhy446Wf8JLHK5E?expire=1647517074\\u0026signiture=iTo-tiI21Ysdn7n7Y6y2YGs36avRi0cfLUpRl0rlE-g=\"]},\"accountHolder\":\"test_accountHolder\",\"accountHolderId\":\"\",\"accountLastName\":null,\"accountType\":0,\"addr\":\"test_addr\",\"addrAlph\":\"\",\"applyId\":\"5764607808352618589\",\"applyStatus\":0,\"applyType\":1,\"areaName\":\"Area0\",\"arrBusinessAttr\":[\"140\"],\"arrTags\":[],\"authorizationInfo\":{\"payment\":{\"authorizedPics\":null," +
"\"extraFiles\":null,\"pics\":null},\"sign\":{\"authorizedPics\":null,\"extraFiles\":null,\"pics\":null}},\"autoSwitch\":\"0\",\"avgPrice\":{\"max\":0,\"min\":0},\"avgProduceTime\":600,\"bLicenseAddr\":\"\",\"bLicenseBrand\":\"\",\"bLicenseExpireFlag\":61,\"bLicenseExpireTime\":\"1970-01-01 08:00:00\",\"bLicenseId\":\"\",\"bLicenseLpName\":\"\",\"bLicensePic\":\"\",\"bLicensePicUrl\":\"\",\"bLicenseStartTime\":\"1970-01-01 08:00:00\",\"bLicenseStatus\":\"3\",\"bLicenseValidType\":0,\"bankAgentDigit\":\"\",\"bankAgentno\":\"\",\"bdName\":\"sim_data_01(simdata01_test_v)\",\"bdPhone\":\"\",\"bizDayTime\":null,\"bizDayTimeNonAffiliate\":\"\",\"bizStatus\":1,\"bizTime\":\"{\\\"day\\\":[1,1,1,1,1,1,1],\\\"time\\\":[{\\\"begin\\\":\\\"10:00\\\",\\\"end\\\":\\\"18:00\\\"}]}\",\"bizoppDocInfo\":{\"city\":{\"files\":null,\"licensePics\":null},\"corp\":{\"files\":null,\"licensePics\":null},\"health\":{\"files\":null,\"licensePics\":null},\"land\":{\"files\":null,\"" +
"licensePics\":null}},\"bizoppInfo\":{\"bussinessHours\":\"\",\"deliveryDistanceTime\":\"\",\"deliveryPrice\":0,\"itemCount\":null,\"minPrice\":0,\"priceRange\":null,\"ratingCount\":\"\",\"ratingValue\":\"\"},\"bizoppStatus\":5,\"blicenseShow\":1,\"brand\":\"\",\"brandId\":0,\"businessType\":1,\"button\":{\"addVisitButton\":1,\"applyOpenButton\":0,\"applySignButton\":0,\"approveButton\":0,\"batchAllotButton\":1,\"businessInformationViewButton\":1,\"cancelApplyButton\":0,\"conSignPadButton\":0,\"confirmButton\":0,\"controlOperationButton\":1,\"copyMsgGatherUrlButton\":0,\"deliveryPadButton\":1,\"deliverySettingButton\":1,\"editButton\":0,\"financialOperationButton\":1,\"giveUp\":0,\"msgGatherButton\":0,\"msgGatherEditButton\":0,\"msgGatherPlusButton\":0,\"orderInfoButton\":1,\"pickIn\":0,\"sendMsgGatherButton\":0,\"switchToNonAffiliateButton\":0,\"tagsButton\":1,\"topEatsButton\":0,\"updateAbnormalTagsButton\":1,\"updateAddressButton\":1,\"updateBusinessInfoButton\":1," +
"\"updateCategoryButton\":1,\"updateNormalTag\":1,\"updateShopBaseInfo\":1,\"updateShopInfo\":1,\"uploadPictureButton\":1,\"viewOpenButton\":1,\"visitListButton\":1},\"cLicenseAddr\":\"\",\"cLicenseBrand\":\"\",\"cLicenseExpireFlag\":61,\"cLicenseExpireTime\":\"1970-01-01 08:00:00\",\"cLicenseId\":\"\",\"cLicenseLpName\":\"\",\"cLicensePic\":\"\",\"cLicensePicUrl\":\"\",\"cLicenseStatus\":\"3\",\"cLicenseValidType\":0,\"category\":[],\"categoryIds\":[],\"channel\":1,\"checkStatus\":0,\"city\":52080200,\"cityId\":52080200,\"clicenseShow\":1,\"clusterId\":5764607713452294000,\"clusterName\":\"\",\"commissionJson\":[],\"competeBizTime\":[],\"competeItemMenuUrl\":[],\"completeStatus\":1,\"contactCallingCode\":\"\",\"contactName\":\"test_partyaContact\",\"contactPhone\":\"15512300000\",\"contractorId\":\"5764607748000712588\",\"contractorStatus\":3,\"cosTrans\":0,\"country\":\"MX\",\"countryCode\":\"\",\"createTime\":\"2022-02-16 19:39:42\",\"creatorUserName\":\"simdata01_test_v\"," +
"\"currency\":{\"display\":\"MX$0.00\",\"number\":\"0.00\",\"position\":0,\"sign\":\"\",\"symbol\":\"MX$\"},\"decorateId\":0,\"deliverDistance\":0,\"deliverType\":2,\"deviceFlagbin\":3,\"deviceFlagbinNeedPad\":1,\"displayable\":0,\"displayableNonAffiliate\":0,\"docFlagbin\":0,\"docId\":\"5764607687590152047\",\"docInfo\":{\"city\":{\"files\":null,\"licensePics\":null},\"corp\":{\"files\":null,\"licensePics\":null},\"health\":{\"files\":null,\"licensePics\":null},\"land\":{\"files\":null,\"licensePics\":null}},\"editTime\":\"2022-02-16 19:40:14\",\"environmentalPhoto\":[\"https://img0.didiglobal.com/static/soda_public/do1_QSl1NDnsiCYaB2A5gmis\"],\"extendMaterial\":\"\",\"extraInfo\":{\"bankFlow\":{\"files\":null,\"pics\":null},\"other\":{\"files\":null,\"pics\":null}},\"filterTags\":[{\"text\":\"正常门店\",\"type\":1}],\"filterTagsValue\":0,\"flagbin\":0,\"foodSafe\":\"\",\"foodSafePic\":\"\",\"foodSafePicUrl\":\"\",\"foodSafeShow\":1," +
"\"frontPhoto\":[\"https://img0.didiglobal.com/static/soda_public/do1_AMgzb2wxVwL4usmf4qHu\"],\"grade\":\"\",\"headImg\":\"\",\"isDeliverCash\":1,\"isEnableCash\":1,\"isEnablePaypay\":1,\"isEnablePos\":0,\"isFoodCity\":0,\"isIllegal\":0,\"isNeedPad\":0,\"isRate\":2,\"isScopeDelivery\":1,\"itemFiles\":null,\"itemPics\":[\"https://img0.didiglobal.com/static/soda_public/do1_iad8ykReSCUcGh7sUJye\"],\"ka\":1,\"kpContactTime\":null,\"kpDepartment\":\"\",\"kpEmail\":\"test_data_platform@didiglobal.com\",\"kpLastName\":\"\",\"kpLastNameAlph\":\"\",\"kpName\":\"test_kpName\",\"kpNameAlph\":\"\",\"kpOutId\":\"\",\"kpPhone\":\"15512300001\",\"kpPhoneExtra\":null,\"kpPics\":{\"keys\":[\"https://img0.didiglobal.com/static/soda_public/do1_BXtt9g8f8dfiecSHC1dZ\"],\"urls\":[false]},\"kpSource\":\"\",\"lat\":\"28.7370616\",\"lng\":\"-106.1306375\",\"localTaxId\":\"\",\"localTaxPayer\":\"\",\"localTaxPayerAddr\":\"\",\"localTaxPayerIdType\":0,\"localTaxPaymentMethod\":\"\"," +
"\"localTaxPics\":\"\",\"localTaxVatPayment\":0,\"logo\":\"https://img0.didiglobal.com/static/soda_public/do1_gLdUuwSCysHlKsPzYD9F\",\"logoImg\":\"https://img0.didiglobal.com/static/soda_public/do1_gLdUuwSCysHlKsPzYD9F\",\"lpLicenseId\":\"\",\"lpLicensePicBack\":\"\",\"lpLicensePicBackUrl\":\"\",\"lpLicensePicFront\":\"\",\"lpLicensePicFrontUrl\":\"\",\"lpLicensePics\":\"\",\"lpLicensePicsUrl\":\"\",\"lpLicenseStatus\":\"3\",\"lpLicenseType\":\"0\",\"lpName\":\"\",\"mainCategory\":\"\",\"manualCategory\":[],\"minPrice\":0,\"monthSales\":0,\"monthSold\":0,\"msgGatherUrl\":\"\",\"name\":\"test_1645011556_kFvjwPijhdxkmmgRXKC\",\"nameAlph\":\"\",\"notifyEmail\":\"\",\"notifyPhone\":\"\",\"objBusinessAttr\":[],\"objTags\":[],\"officialUrl\":\"\",\"open\":1,\"openCheck\":[{\"button\":\"已添加\",\"text\":\"联系电话\",\"type\":1},{\"button\":\"已填写\",\"text\":\"配送信息\",\"type\":1},{\"button\":\"已添加\",\"text\":\"门店头图\",\"type\":1},{\"button\":\"未完成\",\"text\":\"可售菜品数量\"," +
"\"type\":0},{\"button\":\"已通过\",\"text\":\"营业时间段\",\"type\":1},{\"button\":\"已添加\",\"text\":\"门店主营品类\",\"type\":1},{\"button\":\"已添加\",\"text\":\"出餐时长\",\"type\":1}],\"openCheckNonAffiliate\":null,\"orderConfirmMethod\":\"0\",\"padInfo\":{\"isNeedPad\":0,\"isPadOnlineSign\":1},\"padStatus\":0,\"payVerifyReason\":\"\",\"payVerifyStatus\":0,\"permAddr\":\"\",\"permNo\":\"\",\"phone\":\"164501155695\",\"poi\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiId\":\"ChIJOdOau5VB6oYR-LfkU_bfr_g\",\"poiName\":\"Arcos de Terragona 16934, Villa del Nte, 31137 Chihuahua, Chih., México\",\"postCode\":\"\",\"produceTimeMax\":\"0\",\"rank\":5,\"repeatShopId\":\"\",\"reprePhone\":\"\",\"riderAreaName\":\"\",\"riderClusterId\":52080200,\"score\":0,\"secondCategory\":\"\",\"selectCityId\":52080200,\"settledStatus\":0,\"settledTags\":0,\"shopId\":\"5764607747732277132\",\"shopLayerLevel\":\"\",\"shopLevel\":\"\",\"shopOnlineStatus\":3,\"shopOnlineStatusDesc\":\"需手工开启营业\"," +
"\"shopOnlineStatusMsg\":\"歇业中\",\"shopPhone\":\"\",\"shopPics\":{\"environmentalPhoto\":[\"/static/soda_public/do1_QSl1NDnsiCYaB2A5gmis\"],\"frontPhoto\":[\"/static/soda_public/do1_AMgzb2wxVwL4usmf4qHu\"]},\"shopStatus\":3,\"shopTag\":0,\"signPic\":null,\"signType\":1,\"signedTime\":1645011613,\"source\":2,\"startupCost\":500000,\"startupCostLimit\":50000,\"startupCostNum\":0,\"startupCostProportion\":1000,\"startupExecWeek\":1,\"tags\":0,\"taxId\":\"234234123112\",\"taxIdType\":1,\"taxPayer\":\"\",\"taxPayerAddr\":\"\",\"taxPayerIdType\":0,\"taxPaymentMethod\":\"\",\"taxPics\":[\"https://img0.didiglobal.com/static/soda_public/do1_Gip4umImni45IUHPTvQQ\"],\"taxVatPayment\":0,\"thirdCategory\":\"\",\"timezone\":\"\",\"todayBizTime\":[],\"type\":1,\"updateTime\":\"2022-03-10 15:46:20\",\"visible\":false,\"visitStatus\":0},\"errmsg\":\"ok\",\"errno\":0,\"machine\":\"\",\"reqParams\":{\"locale\":\"zh-CN\",\"shopId\":\"5764607747732277132\"},\"time\":1646912275," +
"\"traceId\":\"s060310x1646912272\"},\"timestamp\":\"2022-03-10T19:37:55.180+0800\"}";
System.out.println("字符串长度： "+str.length());
System.out.println("字符串byte长度："+str.getBytes().length);
byte[] bt = ZstdUtils.compress(str.getBytes());
String newStr = ZstdUtils.decompress(bt);
System.out.println("新字符串："+newStr);
System.out.println("新字符串的byte长度："+newStr.getBytes().length);
System.out.println("Zstd压缩后的byte长度："+bt.length);
byte[] bs = GzipCompress.compress(str.getBytes());
System.out.println("Gzip压缩后的byte长度："+bs.length);
System.out.println("Zstd压缩后的byte长度 + Gzip压缩后的byte长度："+GzipCompress.compress(bt).length);
System.out.println("Base64 后的byte长度: "+Base64.getEncoder().encode(bt).length);
System.out.println("=======");
System.out.println(CollectMode.COMPRESS_LOG_RESP);
System.out.println(CollectMode.EXTRACT_ALL);
System.out.println();
JcProperties properties = new JcProperties();
String path = "/Users/didi/Desktop/jlog/example/target/classes/application.properties";
properties.load(new FileInputStream(path));
System.out.println(JSON.toJSONString(properties));
TagConfig bean = properties.getBean("tag-config", TagConfig.class);
System.out.println(JSON.toJSONString(bean));
}
}

49
config/config-core/src/main/java/com/jd/platform/jlog/core/GzipCompress.java Normal file
View File

@ -0,0 +1,49 @@
package com.jd.platform.jlog.core;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/**
* The Data Compression Based on gzip.
*
*/
public class GzipCompress {
public static byte[] compress(byte[] data) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream gzip;
try {
gzip = new GZIPOutputStream(out);
gzip.write(data);
gzip.close();
} catch (IOException e) {
e.printStackTrace();
}
return out.toByteArray();
}
public static byte[] uncompress(byte[] data) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayInputStream in = new ByteArrayInputStream(data);
try {
GZIPInputStream ungzip = new GZIPInputStream(in);
byte[] buffer = new byte[2048];
int n;
while ((n = ungzip.read(buffer)) >= 0) {
out.write(buffer, 0, n);
}
} catch (IOException e) {
e.printStackTrace();
}
return out.toByteArray();
}
}

26
config/config-etcd/pom.xml Normal file
View File

@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config</artifactId>
<version>1.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>config-etcd</artifactId>
<name>config-etcd</name>
<dependencies>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-core</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.ibm.etcd</groupId>
<artifactId>etcd-java</artifactId>
<version>0.0.16</version>
</dependency>
</dependencies>
</project>

15
config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurationProvider.java Normal file
View File

@ -0,0 +1,15 @@
package com.jd.platform.jlog.etcd;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorProvider;
/**
* @author tangbohu
*/
public class EtcdConfigurationProvider implements ConfiguratorProvider {
@Override
public Configurator build() {
return EtcdConfigurator.getInstance();
}
}

123
config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdConfigurator.java Normal file
View File

@ -0,0 +1,123 @@
package com.jd.platform.jlog.etcd;
import java.io.IOException;
import java.io.StringReader;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import com.alibaba.fastjson.JSON;
import com.google.protobuf.ByteString;
import com.ibm.etcd.api.KeyValue;
import com.ibm.etcd.api.RangeResponse;
import com.ibm.etcd.client.EtcdClient;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.common.utils.StringUtil;
import com.jd.platform.jlog.core.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigByte;
import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr;
import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT;
import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY;
/**
* @author tangbohu
* @version 1.0.0
* @desc 参考log4J
* @ClassName EtcdConfigurator.java
* @createTime 2022年02月21日 21:46:00
*/
public class EtcdConfigurator implements Configurator {
private static final Logger LOGGER = LoggerFactory.getLogger(EtcdConfigurator.class);
private static volatile EtcdConfigurator instance;
static volatile EtcdClient client;
private static final Configurator FILE_CONFIG = ConfiguratorFactory.base;
static final String ROOT = "/jLog";
static final String PROPERTIES_PATH = "/jLog/jLog.properties";
static JcProperties PROPERTIES = new JcProperties();
private EtcdConfigurator() {
LOGGER.info("开始构建etcd客户端, serverAddr:{}",FILE_CONFIG.getString(SERVER_ADDR_KEY));
client = EtcdClient.forEndpoints(FILE_CONFIG.getString(SERVER_ADDR_KEY)).withPlainText().build();
RangeResponse rangeResponse = client.getKvClient().get(ByteString.copyFromUtf8(PROPERTIES_PATH)).sync();
List<KeyValue> keyValues = rangeResponse.getKvsList();
if (CollectionUtil.isEmpty(keyValues)) {
return;
}
String val = keyValues.get(0).getValue().toStringUtf8();
if(StringUtil.isNotBlank(val)){
try {
PROPERTIES.load(new StringReader(val));
} catch (IOException e) {
e.printStackTrace();
}
}
LOGGER.info("初始化etcd配置", JSON.toJSONString(PROPERTIES));
new EtcdListener().onProcessEvent(new ConfigChangeEvent());
}
public static EtcdConfigurator getInstance() {
if (instance == null) {
synchronized (EtcdConfigurator.class) {
if (instance == null) {
instance = new EtcdConfigurator();
}
}
}
return instance;
}
@Override
public String getString(String key) {
return PROPERTIES.getString(key);
}
@Override
public Long getLong(String key) {
return PROPERTIES.getLong(key);
}
@Override
public List<String> getList(String key) {
return PROPERTIES.getStrList(key);
}
@Override
public <T> T getObject(String key, Class<T> clz) {
return PROPERTIES.getBean(key, clz);
}
@Override
public boolean putConfig(String key, String content) {
return putConfig(key, content, DEFAULT_TIMEOUT);
}
@Override
public boolean putConfig(String key, String content, long timeoutMills) {
if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){
return false;
}
PROPERTIES.setProperty(key, content);
client.getKvClient().put(ByteString.copyFromUtf8(PROPERTIES_PATH), ByteString.copyFromUtf8(formatConfigStr(PROPERTIES))).sync();
return true;
}
@Override
public String getType() {
return "etcd";
}
}

102
config/config-etcd/src/main/java/com/jd/platform/jlog/etcd/EtcdListener.java Normal file
View File

@ -0,0 +1,102 @@
package com.jd.platform.jlog.etcd;
import com.google.protobuf.ByteString;
import com.ibm.etcd.api.Event;
import com.ibm.etcd.api.KeyValue;
import com.ibm.etcd.api.RangeResponse;
import com.ibm.etcd.client.EtcdClient;
import com.ibm.etcd.client.kv.KvClient;
import com.ibm.etcd.client.kv.WatchUpdate;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.common.utils.StringUtil;
import com.jd.platform.jlog.core.ClientHandlerBuilder;
import com.jd.platform.jlog.core.ConfigChangeEvent;
import com.jd.platform.jlog.core.ConfigChangeListener;
import com.jd.platform.jlog.core.ConfigChangeType;
import io.netty.util.concurrent.DefaultThreadFactory;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import static com.jd.platform.jlog.etcd.EtcdConfigurator.PROPERTIES;
import static com.jd.platform.jlog.etcd.EtcdConfigurator.PROPERTIES_PATH;
import static com.jd.platform.jlog.etcd.EtcdConfigurator.ROOT;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName EtcdListener.java
* @Description TODO
* @createTime 2022年02月21日 23:34:00
*/
public class EtcdListener implements ConfigChangeListener {
private KvClient.WatchIterator iterator;
public EtcdListener() {
iterator = EtcdConfigurator.client.getKvClient().watch(ByteString.copyFromUtf8(ROOT)).asPrefix().start();
getExecutorService().submit(() -> {
while (iterator.hasNext()){
try {
WatchUpdate update = iterator.next();
Event eve = update.getEvents().get(0);
KeyValue kv = eve.getKv();
Event.EventType eveType = eve.getType();
ConfigChangeType changeType = eveType.equals(Event.EventType.DELETE) ? ConfigChangeType.MODIFY : ConfigChangeType.DELETE;
ConfigChangeEvent event = new ConfigChangeEvent();
event.setKey(kv.getKey().toStringUtf8()).setNewValue(kv.getValue().toStringUtf8()).setChangeType(changeType);
onChangeEvent(event);
}catch (RuntimeException e){
e.printStackTrace();
}
}
});
}
@Override
public void onShutDown() {
this.iterator.close();
getExecutorService().shutdownNow();
}
@Override
public void onChangeEvent(ConfigChangeEvent event) {
RangeResponse rangeResponse = EtcdConfigurator.client.getKvClient().get(ByteString.copyFromUtf8(PROPERTIES_PATH)).sync();
List<KeyValue> keyValues = rangeResponse.getKvsList();
if (CollectionUtil.isEmpty(keyValues)) {
return;
}
String val = keyValues.get(0).getValue().toStringUtf8();
JcProperties props = new JcProperties();
if(StringUtil.isNotBlank(val)){
try {
props.load(new StringReader(val));
} catch (IOException e) {
e.printStackTrace();
}
Set<String> diffKeys = CollectionUtil.diffKeys(props, PROPERTIES);
if(!diffKeys.isEmpty()){
PROPERTIES = props;
for (String diffKey : diffKeys) {
LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", event.getKey(), diffKey,
String.valueOf(props.get(diffKey)),
String.valueOf(PROPERTIES.get(diffKey)));
}
ClientHandlerBuilder.refresh();
}
}
}
}

1
config/config-etcd/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider Normal file
View File

@ -0,0 +1 @@
com.jd.platform.jlog.etcd.EtcdConfigurationProvider

27
config/config-nacos/pom.xml Normal file
View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config</artifactId>
<version>1.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>config-nacos</artifactId>
<name>config-nacos</name>
<dependencies>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-core</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.alibaba.nacos</groupId>
<artifactId>nacos-client</artifactId>
<version>1.4.2</version>
</dependency>
</dependencies>
</project>

145
config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfigurator.java Normal file
View File

@ -0,0 +1,145 @@
package com.jd.platform.jlog.nacos;
import java.io.*;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import com.alibaba.fastjson.JSON;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.exception.NacosException;
import com.alibaba.nacos.common.utils.StringUtils;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.utils.StringUtil;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigStr;
import static com.jd.platform.jlog.core.Constant.*;
/**
* @author tangbohu
*/
public class NacosConfigurator implements Configurator {
private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurator.class);
private static final Configurator FILE_CONFIG = ConfiguratorFactory.base;
private static volatile ConfigService configService;
static volatile JcProperties PROPERTIES = new JcProperties();
static final String JLOG_GROUP = "JLOG_GROUP";
static final String DEFAULT_DATA_ID = "jLog.properties";
static NacosListener NACOSLISTENER = new NacosListener();
private static volatile NacosConfigurator instance;
public static NacosConfigurator getInstance() {
if (instance == null) {
synchronized (NacosConfigurator.class) {
if (instance == null) {
instance = new NacosConfigurator();
}
}
}
return instance;
}
private NacosConfigurator() {
if (configService == null) {
try {
configService = NacosFactory.createConfigService(getConfigProperties());
String config = configService.getConfig(DEFAULT_DATA_ID, JLOG_GROUP, DEFAULT_TIMEOUT);
LOGGER.info("从NaCos获取配置进行初始化 config = {}", config);
if (StringUtils.isNotBlank(config)) {
PROPERTIES.load(new StringReader(config));
LOGGER.info("初始化本地缓存 props:{} ", JSON.toJSONString(PROPERTIES));
configService.addListener(DEFAULT_DATA_ID, JLOG_GROUP, NACOSLISTENER);
}
} catch (NacosException | IOException e) {
throw new RuntimeException(e);
}
}
}
@Override
public String getString(String key) {
return PROPERTIES.getString(key);
}
@Override
public Long getLong(String key) {
return PROPERTIES.getLong(key);
}
@Override
public List<String> getList(String key) {
return PROPERTIES.getStrList(key);
}
@Override
public <T> T getObject(String key, Class<T> clz) {
return PROPERTIES.getBean(key, clz);
}
@Override
public boolean putConfig(String key, String content) {
return putConfig(key, content, DEFAULT_TIMEOUT);
}
@Override
public boolean putConfig(String key, String content, long timeoutMills) {
boolean result = false;
if(StringUtil.isEmpty(key)){
return false;
}
try {
if (!PROPERTIES.isEmpty()) {
PROPERTIES.setProperty(key, content);
result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, formatConfigStr(PROPERTIES));
} else {
result = configService.publishConfig(DEFAULT_DATA_ID, JLOG_GROUP, content);
}
} catch (NacosException ex) {
LOGGER.error(ex.getErrMsg());
}
return result;
}
private static Properties getConfigProperties() {
Properties properties = new Properties();
String address = FILE_CONFIG.getString(SERVER_ADDR_KEY);
if (address != null) {
properties.setProperty(SERVER_ADDR_KEY, address);
}
properties.setProperty(NAMESPACE_KEY, DEFAULT_NAMESPACE);
return properties;
}
@Override
public String getType() {
return "nacos";
}
}

12
config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosConfiguratorProvider.java Normal file
View File

@ -0,0 +1,12 @@
package com.jd.platform.jlog.nacos;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorProvider;
public class NacosConfiguratorProvider implements ConfiguratorProvider {
@Override
public Configurator build() {
return NacosConfigurator.getInstance();
}
}

76
config/config-nacos/src/main/java/com/jd/platform/jlog/nacos/NacosListener.java Normal file
View File

@ -0,0 +1,76 @@
package com.jd.platform.jlog.nacos;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.config.listener.AbstractSharedListener;
import com.alibaba.nacos.api.exception.NacosException;
import com.alibaba.nacos.api.naming.listener.Event;
import com.alibaba.nacos.api.naming.listener.EventListener;
import com.alibaba.nacos.common.utils.StringUtils;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.core.ClientHandlerBuilder;
import com.jd.platform.jlog.core.ConfigChangeEvent;
import com.jd.platform.jlog.core.ConfigChangeListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.Properties;
import java.util.Set;
import static com.jd.platform.jlog.core.Constant.*;
import static com.jd.platform.jlog.nacos.NacosConfigurator.*;
/**
* @author tangbohu
*/
public class NacosListener extends AbstractSharedListener implements ConfigChangeListener, EventListener {
private static final Logger LOGGER = LoggerFactory.getLogger(NacosListener.class);
public NacosListener() {}
@Override
public void innerReceive(String dataId, String group, String configInfo) {
LOGGER.debug("configInfo:{}", configInfo);
ConfigChangeEvent event = new ConfigChangeEvent().setKey(dataId).setNewValue(configInfo).setNamespace(group);
this.onProcessEvent(event);
}
@Override
public void onChangeEvent(ConfigChangeEvent event) {
LOGGER.debug("通用[配置]变更事件 event:{}",event);
JcProperties props = new JcProperties();
if (StringUtils.isNotBlank(event.getNewValue())) {
try {
props.load(new StringReader(event.getNewValue()));
} catch (IOException e) {
return;
}
}
Set<String> diffKeys = CollectionUtil.diffKeys(props, PROPERTIES);
if(!diffKeys.isEmpty()){
PROPERTIES = props;
for (String diffKey : diffKeys) {
LOGGER.warn("NACOS {} 配置变更 key={} 变更事件:{}", DEFAULT_DATA_ID, diffKey,
String.valueOf(props.get(diffKey)),
String.valueOf(PROPERTIES.get(diffKey)));
}
ClientHandlerBuilder.refresh();
}
}
@Override
public void onEvent(Event event) {
LOGGER.info("通用[服务]事件 event:{}",event);
}
}

1
config/config-nacos/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider Normal file
View File

@ -0,0 +1 @@
com.jd.platform.jlog.nacos.NacosConfiguratorProvider

68
config/config-nacos/src/test/NacosTest.java Normal file
View File

@ -0,0 +1,68 @@
package com.jd.platform.jlog.nacos;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.exception.NacosException;
import com.alibaba.nacos.api.naming.NamingFactory;
import com.alibaba.nacos.api.naming.NamingService;
import com.alibaba.nacos.api.naming.listener.NamingEvent;
import static com.jd.platform.jlog.nacos.NacosConstant.DEFAULT_DATA_ID;
import static com.jd.platform.jlog.nacos.NacosConstant.JLOG_GROUP;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName NacosTest.java
* @Description TODO
* @createTime 2022年02月22日 22:04:00
*/
public class NacosTest {
public static void main(String[] args) throws NacosException, InterruptedException {
String serverAddr = "101.42.242.201:8848";
String dataId = DEFAULT_DATA_ID;
String group = JLOG_GROUP;
ConfigService configService = NacosFactory.createConfigService(serverAddr);
NamingService naming = NamingFactory.createNamingService(serverAddr);
naming.registerInstance("nacos.test1", JLOG_GROUP,"172.22.216.105", 8888, "TEST1");
Thread.sleep(3000);
NacosListener server = new NacosListener();
naming.subscribe("nacos.test1", JLOG_GROUP,server);
String content = configService.getConfig(dataId, group, 2000L);
System.out.println("content: "+content);
NacosListener nL = new NacosListener();
configService.addListener(dataId, group, nL);
System.out.println("新增完成");
Thread.sleep(20000);
System.out.println("移除开始");
configService.removeListener(dataId, group, nL);
System.out.println("移除结束");
naming.deregisterInstance("nacos.test1", JLOG_GROUP,"172.22.216.105", 8888, "TEST1");
Thread.sleep(5000);
System.out.println("睡眠结束");
/* System.out.println("移除开始222");
configService.removeListener(dataId, group, new Listener() {
@Override
public Executor getExecutor() {
return Executors.newSingleThreadExecutor();
}
@Override
public void receiveConfigInfo(String s) {
System.out.println("#### 移除开始222");
}
});*/
Thread.sleep(90000);
System.out.println("---------------------------睡眠结束");
}
}

41
config/config-zk/pom.xml Normal file
View File

@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 1999-2019 Seata.io Group.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config</artifactId>
<version>1.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>config-zk</artifactId>
<name>config-zk</name>
<dependencies>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-core</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>4.2.0</version>
</dependency>
</dependencies>
</project>

119
config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfigurator.java Normal file
View File

@ -0,0 +1,119 @@
package com.jd.platform.jlog.zk;
import java.io.ByteArrayInputStream;
import java.util.List;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.handler.JcProperties;
import com.jd.platform.jlog.common.utils.StringUtil;
import com.jd.platform.jlog.core.*;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.jd.platform.jlog.common.utils.ConfigUtil.formatConfigByte;
import static com.jd.platform.jlog.core.Constant.DEFAULT_TIMEOUT;
import static com.jd.platform.jlog.core.Constant.DEFAULT_NAMESPACE;
import static com.jd.platform.jlog.core.Constant.SERVER_ADDR_KEY;
/**
* @author tangbohu
*/
public class ZkConfigurator implements Configurator {
private final static Logger LOGGER = LoggerFactory.getLogger(ZkConfigurator.class);
private static final Configurator FILE_CONFIG = ConfiguratorFactory.base;
static volatile CuratorFramework zkClient;
static volatile JcProperties PROPERTIES = new JcProperties();
private static final String DEFAULT_CONFIG_PATH = "/jLog.properties";
public ZkConfigurator() throws Exception {
if (zkClient == null) {
synchronized (ZkConfigurator.class) {
zkClient = CuratorFrameworkFactory.builder().connectString(FILE_CONFIG.getString(SERVER_ADDR_KEY))
// 连接超时时间
.sessionTimeoutMs(6000)
// 会话超时时间
.connectionTimeoutMs(2000)
.namespace(DEFAULT_NAMESPACE)
// 刚开始重试间隔为1秒，之后重试间隔逐渐增加，最多重试不超过三次
.retryPolicy(new ExponentialBackoffRetry(1000, 3))
.build();
zkClient.start();
}
loadZkData();
new ZkListener(DEFAULT_CONFIG_PATH);
LOGGER.info("初始化ZK,载入ZK数据完成 props:{}", JSON.toJSONString(PROPERTIES));
}
}
@Override
public String getString(String key) {
return PROPERTIES.getString(key);
}
@Override
public Long getLong(String key) {
return PROPERTIES.getLong(key);
}
@Override
public List<String> getList(String key) {
return PROPERTIES.getStrList(key);
}
@Override
public <T> T getObject(String key, Class<T> clz) {
return PROPERTIES.getBean(key, clz);
}
@Override
public boolean putConfig(String key, String content) {
return putConfig(key, content, DEFAULT_TIMEOUT);
}
@Override
public boolean putConfig(String key, String content, long timeoutMills) {
if(StringUtil.isEmpty(key) || StringUtil.isEmpty(content)){
return false;
}
PROPERTIES.setProperty(key, content);
try {
zkClient.setData().forPath(DEFAULT_CONFIG_PATH, formatConfigByte(PROPERTIES));
} catch (Exception e) {
return false;
}
return true;
}
@Override
public String getType() {
return "zk";
}
static void loadZkData() throws Exception {
byte[] bt = zkClient.getData().forPath(DEFAULT_CONFIG_PATH);
if (bt != null && bt.length > 0){
ByteArrayInputStream bArray = new ByteArrayInputStream(bt);
PROPERTIES.load(bArray);
}
LOGGER.info("# loadZkData # PROPERTIES:{}",JSON.toJSONString(PROPERTIES));
}
}

20
config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkConfiguratorProvider.java Normal file
View File

@ -0,0 +1,20 @@
package com.jd.platform.jlog.zk;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorProvider;
/**
* @author tangbohu
*/
public class ZkConfiguratorProvider implements ConfiguratorProvider {
@Override
public Configurator build() {
try {
return new ZkConfigurator();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

70
config/config-zk/src/main/java/com/jd/platform/jlog/zk/ZkListener.java Normal file
View File

@ -0,0 +1,70 @@
package com.jd.platform.jlog.zk;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.core.ClientHandlerBuilder;
import com.jd.platform.jlog.core.ConfigChangeEvent;
import com.jd.platform.jlog.core.ConfigChangeListener;
import com.jd.platform.jlog.core.ConfigChangeType;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.NodeCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import static com.jd.platform.jlog.zk.ZkConfigurator.*;
/**
* @author didi
*/
public class ZkListener implements ConfigChangeListener {
private final static Logger LOGGER = LoggerFactory.getLogger(ZkListener.class);
private NodeCache cache;
private String path;
public ZkListener(String path) {
this.path = path;
cache = new NodeCache(zkClient, path);
try {
cache.start(true);
cache.getListenable().addListener(() -> onChangeEvent(null));
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void onChangeEvent(ConfigChangeEvent event) {
LOGGER.info("ZK数据变更-当前监听器关注的path:{} PROPERTIES:{}", path, JSON.toJSONString(PROPERTIES));
Properties props = new Properties();
props.putAll(PROPERTIES);
try {
LOGGER.debug("ZK数据变更,旧Properties:{}", JSON.toJSONString(props));
loadZkData();
LOGGER.debug("ZK数据变更,新Properties:{}", JSON.toJSONString(PROPERTIES));
} catch (Exception e) {
e.printStackTrace();
}
Set<String> diffKeys = CollectionUtil.diffKeys(props, PROPERTIES);
if(!diffKeys.isEmpty()){
for (String diffKey : diffKeys) {
LOGGER.warn("ZK {} 配置变更 key={} 变更事件:{}", path, diffKey,
new ConfigChangeEvent(diffKey,
String.valueOf(props.get(diffKey)),
String.valueOf(PROPERTIES.get(diffKey))));
}
ClientHandlerBuilder.refresh();
}
}
}

1
config/config-zk/src/main/resources/META-INF/services/com.jd.platform.jlog.core.ConfiguratorProvider Normal file
View File

@ -0,0 +1 @@
com.jd.platform.jlog.zk.ZkConfiguratorProvider

36
config/pom.xml Normal file
View File

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 1999-2019 Seata.io Group.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>JLog</artifactId>
<groupId>com.jd.platfrom.jlog</groupId>
<version>1.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>config</artifactId>
<packaging>pom</packaging>
<name>config</name>
<modules>
<module>config-core</module>
<module>config-etcd</module>
<module>config-apollo</module>
<module>config-nacos</module>
<module>config-zk</module>
</modules>
</project>

79
clientdemo/pom.xml → example/pom.xml
View File

@ -10,10 +10,10 @@
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>clientdemo</artifactId>
<artifactId>example</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>clientdemo</name>
<description>Demo project for using JLog</description>
<name>example</name>
<description>example for using JLog</description>
<properties>
<java.version>1.8</java.version>
</properties>
@ -23,42 +23,71 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions><!-- 去掉springboot默认配置 -->
<exclusion>
<!--<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusion>-->
</exclusions>
</dependency>
<dependency>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>client</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-test</artifactId>
<version>2.5.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.3.10</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.20</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>clientlog4j2</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<!-- <dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>clientlog4j2</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>-->
</dependencies>
<build>

8
clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/ClientDemoApplication.java → example/src/main/java/com/jd/platform/jlog/clientdemo/ExampleApplication.java
View File

@ -11,9 +11,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
* @date 2021-12-27
*/
@SpringBootApplication
public class ClientDemoApplication {
public class ExampleApplication {
public static void main(String[] args) {
SpringApplication.run(ClientDemoApplication.class, args);
try {
SpringApplication.run(ExampleApplication.class, args);
}catch (Exception e){
e.printStackTrace();
}
}
}

115
example/src/main/java/com/jd/platform/jlog/clientdemo/appender/TracerLogbackAppender.java Normal file
View File

@ -0,0 +1,115 @@
package com.jd.platform.jlog.clientdemo.appender;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.ThrowableProxy;
import ch.qos.logback.core.AppenderBase;
import com.jd.platform.jlog.client.log.LogExceptionStackTrace;
import com.jd.platform.jlog.client.tracerholder.TracerHolder;
import com.jd.platform.jlog.client.udp.UdpSender;
import com.jd.platform.jlog.common.model.RunLogMessage;
import com.jd.platform.jlog.core.ClientHandler;
import org.slf4j.helpers.MessageFormatter;
/**
* className：TracerLog4JAppender
*
* @author wuweifeng
* @version 1.0.0
*/
public class TracerLogbackAppender extends AppenderBase<ILoggingEvent> {
/**
* 日志的堆栈级别，如直接在类里使用logger.info，则该值为0. 如调用另一个日志类如CommonLogger打印，则该值为1
*/
private int loggerStage;
@Override
protected void append(ILoggingEvent iLoggingEvent) {
try {
long tracerId = TracerHolder.getTracerId();
if (0L == tracerId) {
return;
}
RunLogMessage logMessage = getLogMessage(iLoggingEvent);
UdpSender.offerLogger(logMessage);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 转化为对象
*/
private RunLogMessage getLogMessage(ILoggingEvent loggingEvent) {
RunLogMessage logMessage = new RunLogMessage();
//设置链路唯一id
logMessage.setTracerId(TracerHolder.getTracerId());
logMessage.setClassName(loggingEvent.getLoggerName());
logMessage.setThreadName(loggingEvent.getThreadName());
StackTraceElement stackTraceElement = loggingEvent.getCallerData()[loggerStage];
String method = stackTraceElement.getMethodName();
String line = String.valueOf(stackTraceElement.getLineNumber());
logMessage.setMethodName(method + "(" + stackTraceElement.getFileName() + ":" + line + ")");
logMessage.setLogLevel(loggingEvent.getLevel().toString());
logMessage.setCreateTime(loggingEvent.getTimeStamp());
String formattedMessage = getMessage(loggingEvent);
logMessage.setContent(formattedMessage);
ClientHandler.Outcome out = ClientHandler.processLog(formattedMessage);
logMessage.setTagMap(out.getMap());
return logMessage;
}
/**
* 日志正文信息
*/
private String getMessage(ILoggingEvent logEvent) {
if (logEvent.getLevel() == Level.ERROR) {
if (logEvent.getThrowableProxy() != null) {
ThrowableProxy throwableProxy = (ThrowableProxy) logEvent.getThrowableProxy();
String[] args = new String[]{logEvent.getFormattedMessage() + "\n" + LogExceptionStackTrace.erroStackTrace(throwableProxy.getThrowable()).toString()};
return packageMessage("{}", args);
} else {
Object[] args = logEvent.getArgumentArray();
if (args != null) {
for (int i = 0; i < args.length; i++) {
if (args[i] instanceof Throwable) {
args[i] = LogExceptionStackTrace.erroStackTrace(args[i]);
}
}
return packageMessage(logEvent.getMessage(), args);
}
}
}
return logEvent.getFormattedMessage();
}
private String packageMessage(String message, Object[] args) {
if (message != null && message.contains("{}")) {
return MessageFormatter.arrayFormat(message, args).getMessage();
}
return packageMsg(message, args);
}
private String packageMsg(String message, Object[] args) {
StringBuilder builder = new StringBuilder(128);
builder.append(message);
for (Object arg : args) {
builder.append("\n").append(arg);
}
return builder.toString();
}
public int getLoggerStage() {
return loggerStage;
}
public void setLoggerStage(int loggerStage) {
this.loggerStage = loggerStage;
}
}

35
clientdemo/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java → example/src/main/java/com/jd/platform/jlog/clientdemo/config/DemoConfig.java
View File

@ -1,12 +1,14 @@
package com.jd.platform.jlog.clientdemo.config;
import com.jd.platform.jlog.client.TracerClientStarter;
import com.jd.platform.jlog.client.filter.UserFilter;
import org.springframework.beans.factory.annotation.Value;
import com.jd.platform.jlog.client.filter.HttpFilter;
import com.jd.platform.jlog.common.handler.TagConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
/**
@ -16,24 +18,37 @@ import javax.annotation.PostConstruct;
* @version 1.0
* @date 2021-12-27
*/
@Configuration
@Component
@ConfigurationProperties()
public class DemoConfig {
@Value("${config.server}")
private String etcdServer;
private Logger logger = LoggerFactory.getLogger(getClass());
private TagConfig tagConfig ;
public TagConfig getTagConfig() {
return tagConfig;
}
public void setTagConfig(TagConfig tagConfig) {
this.tagConfig = tagConfig;
}
@PostConstruct
public void begin() {
public void begin() throws Exception {
TracerClientStarter tracerClientStarter = new TracerClientStarter.Builder()
.setAppName("demo")
.setEtcdServer(etcdServer).build();
.setTagConfig(tagConfig)
.build();
logger.info("初始化tagConfig: {}",tagConfig);
tracerClientStarter.startPipeline();
}
@Bean
public FilterRegistrationBean urlFilter() {
FilterRegistrationBean registration = new FilterRegistrationBean();
UserFilter userFilter = new UserFilter();
HttpFilter userFilter = new HttpFilter();
registration.setFilter(userFilter);
registration.addUrlPatterns("/*");

78
example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestController.java Normal file
View File

@ -0,0 +1,78 @@
package com.jd.platform.jlog.clientdemo.web;
import com.jd.platform.jlog.common.model.TracerBean;
import com.jd.platform.jlog.common.handler.TagConfig;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.*;
/**
* @author shenkaiwen5
* @version 1.0
* @date 2021-12-27
*/
@Component
//@ConfigurationProperties()
@RestController
public class TestController {
private TagConfig tagConfig ;
public TagConfig getTagConfig() {
return tagConfig;
}
public void setTagConfig(TagConfig tagConfig) {
this.tagConfig = tagConfig;
}
/**
* do nothing
* just as an adapter for this project common log helper
*
*/
private static Logger RequestLog = LoggerFactory.getLogger("RequestLog");
@RequestMapping("/index")
public Object index() {
TracerBean tracerBean = new TracerBean();
tracerBean.setTracerId("11111");
Configurator configurator = ConfiguratorFactory.getInstance();
try{
configurator.putConfig("/test","val1");
}catch (Exception e){
e.printStackTrace();
}
String val = configurator.getString("/test");
System.out.println("val ===> "+val);
RequestLog.info("哈哈哈哈哈哈");
return tracerBean;
}
@RequestMapping("/log")
public Object log() {
RequestLog.info("|errno=val3||node=val4||这是随便的log|");
return 1;
}
@PostMapping(value = "/test", consumes = MediaType.APPLICATION_JSON_VALUE)
public Object test(@RequestParam Integer uid, @RequestParam Integer newKey,@RequestBody TestReq req) {
String config = ConfiguratorFactory.getInstance().getString("reqTags");
// System.out.println("tagConfig ===> " + tagConfig.toString());
RequestLog.info("|errno=val3||node=val4||这是随便的log|");
return 1;
}
}

15
example/src/main/java/com/jd/platform/jlog/clientdemo/web/TestReq.java Normal file
View File

@ -0,0 +1,15 @@
package com.jd.platform.jlog.clientdemo.web;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName TestReq.java
* @Description TODO
* @createTime 2022年02月17日 16:40:00
*/
public class TestReq {
private Integer id;
private String name;
}

6
example/src/main/resources/application.properties Normal file
View File

@ -0,0 +1,6 @@
server.port=8085
serverAddr=101.42.242.201:2379
apollo.meta=http://127.0.0.1:8080
apollo.config-service=http://127.0.0.1:8080
app.id=order

24
example/src/main/resources/bakapplication.yml Normal file
View File

@ -0,0 +1,24 @@
workers: [1,2,3]
compress: 68
threshold: 10000
serverAddr: 101.42.242.201:2181
server:
port: 8085
test: true
tag-config:
reqTags:
- uid
- url
- newK
logTags:
- node
- bizType
respTags:
- errno
delimiter: "|"
join: "="

18
example/src/main/resources/bakjLog.properties Normal file
View File

@ -0,0 +1,18 @@
server.port=8085
tag-config.reqTags[0]=uid
tag-config.reqTags[1]=url
tag-config.logTags[0]=node
tag-config.logTags[1]=bizType
tag-config.respTags[0]=errno
tag-config.respTags[1]=msg
tag-config.delimiter=|
tag-config.join==
tag-config.extract=41
compress=68
threshold=10000
workers=[1,2,3]

62
example/src/main/resources/logback-spring.xml Normal file
View File

@ -0,0 +1,62 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<contextName>logs</contextName>
<!-- 日志位置 -->
<property name="log.path" value="log" />
<!-- 日志保留时长 -->
<property name="log.maxHistory" value="15" />
<!-- 控制台格式化及颜色 -->
<property name="log.colorPattern" value="%magenta(%d{yyyy-MM-dd HH:mm:ss}) %highlight(%-5level) %yellow(%thread) %green(%logger) %msg%n"/>
<property name="log.pattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5level %thread %logger %msg%n"/>
<!--输出到控制台-->
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>${log.colorPattern}</pattern>
</encoder>
</appender>
<appender name="tracerLogAppender" class="com.jd.platform.jlog.clientdemo.appender.TracerLogbackAppender">
</appender>
<!--输出到文件-->
<appender name="file_info" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${log.path}/info/info.%d{yyyy-MM-dd}.log</fileNamePattern>
<MaxHistory>${log.maxHistory}</MaxHistory>
</rollingPolicy>
<encoder>
<pattern>${log.pattern}</pattern>
</encoder>
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>INFO</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="file_error" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${log.path}/error/error.%d{yyyy-MM-dd}.log</fileNamePattern>
</rollingPolicy>
<encoder>
<pattern>${log.pattern}</pattern>
</encoder>
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>ERROR</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!-- 日志类型为debug时，输出到控制台 -->
<root level="debug">
<appender-ref ref="console" />
</root>
<!-- 日志类型为info时，输出到配置好的文件 -->
<root level="info">
<appender-ref ref="file_info" />
<appender-ref ref="file_error" />
<appender-ref ref="tracerLogAppender" />
</root>
</configuration>

67
example/src/test/java/com/jd/platform/jlog/test/Common.java Normal file
View File

@ -0,0 +1,67 @@
package com.jd.platform.jlog.test;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.handler.TagConfig;
import com.jd.platform.jlog.core.Configurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.List;
import java.util.Random;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName Common.java
* @Description TODO
* @createTime 2022年03月01日 07:36:00
*/
public class Common {
private final static Logger LOGGER = LoggerFactory.getLogger(Common.class);
public static void getTest(Configurator configurator){
LOGGER.info("配置器类型：{}", configurator.getType());
String addr = configurator.getString("serverAddr");
LOGGER.info("配置器get addr：{}", addr);
TagConfig tagConfig = configurator.getObject("tag-config", TagConfig.class);
LOGGER.info("配置器get tagConfig：{}", tagConfig.toString());
List workers = configurator.getList("workers");
LOGGER.info("配置器get workers：{}", JSON.toJSONString(workers));
}
public static void modifyFile(String path)throws Exception{
String temp;
File file = new File(path);
FileInputStream fis = new FileInputStream(file);
InputStreamReader isr = new InputStreamReader(fis);
BufferedReader br = new BufferedReader(isr);
StringBuffer buf = new StringBuffer();
int id = new Random().nextInt(1000);
int num = 0;
// 保存该行前面的内容
while ((temp = br.readLine()) != null) {
if(num == 0){
buf = buf.append("testKey: ").append(id);
}else{
buf = buf.append(temp);
}
num++;
buf = buf.append(System.getProperty("line.separator"));
}
br.close();
FileOutputStream fos = new FileOutputStream(file);
PrintWriter pw = new PrintWriter(fos);
pw.write(buf.toString().toCharArray());
pw.flush();
pw.close();
}
}

85
example/src/test/java/com/jd/platform/jlog/test/EtcdConfiguratorTest.java Normal file
View File

@ -0,0 +1,85 @@
package com.jd.platform.jlog.test;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.clientdemo.ExampleApplication;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.List;
import java.util.Random;
import static com.jd.platform.jlog.test.Common.getTest;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName EtcdConfiguratorTest.java
* @Description TODO
* @createTime 2022年03月03日 07:35:00
*/
@SpringBootTest(classes = ExampleApplication.class)
@RunWith(SpringRunner.class)
public class EtcdConfiguratorTest {
private final static Logger LOGGER = LoggerFactory.getLogger(EtcdConfiguratorTest.class);
private Configurator configurator = null;
@Before
public void init() {
configurator = ConfiguratorFactory.getInstance();
getTest(configurator);
}
// @Test
public void testUpdateCFG() throws Exception {
List<String> workers = configurator.getList("workers");
LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers));
String myIp = "121.1.1.0";
if(workers.contains(myIp)){
// do nothing
LOGGER.info("自己的IP还在配置list里 什么也不做");
return;
}else{
LOGGER.info("自己的IP不在配置list里 添加进去并发布");
workers.add(myIp);
}
configurator.putConfig("workers",JSON.toJSONString(workers));
List<String> workers2 = configurator.getList("workers");
LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2));
}
@Test
public void testAddConfigListener() throws Exception {
int i1 = new Random().nextInt(2000);
int i2 = new Random().nextInt(2000);
String val1 = configurator.getString("testKey");
LOGGER.info("初始化的testKey的val:{}", val1);
LOGGER.info("添加监听器后， 修改配置testKey = {}", i1);
configurator.putConfig("testKey",i1 + "");
LOGGER.info("修改完毕 准备触发监听器");
Thread.sleep(5000);
String val2 = configurator.getString("testKey");
LOGGER.info("第一次修改后的的val:{}", val2);
Thread.sleep(5000);
// LOGGER.info("移除监听器后:修改配置testKey = {}",i2);
// configurator.putConfig("testKey",i2 + "");
LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey"));
LOGGER.info("第二次添加监听器");
Thread.sleep(22000);
}
}

69
example/src/test/java/com/jd/platform/jlog/test/FileConfiguratorTest.java Normal file
View File

@ -0,0 +1,69 @@
package com.jd.platform.jlog.test;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.clientdemo.ExampleApplication;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import org.yaml.snakeyaml.Yaml;
import java.io.*;
import java.util.Map;
import java.util.Properties;
import static com.jd.platform.jlog.test.Common.getTest;
import static com.jd.platform.jlog.test.Common.modifyFile;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName FileConfiguratorTest.java
* @Description TODO
* @createTime 2022年02月28日 19:45:00
*/
@SpringBootTest(classes = ExampleApplication.class)
@RunWith(SpringRunner.class)
public class FileConfiguratorTest {
private final static Logger LOGGER = LoggerFactory.getLogger(FileConfiguratorTest.class);
private Configurator configurator = null;
@Before
public void init() {
configurator = ConfiguratorFactory.getInstance();
getTest(configurator);
}
@Test
public void testAddConfigListener() throws Exception {
String path = "/Users/didi/Desktop/jlog/example/target/classes/bakapplication.yml";
Properties props = new Properties();
FileInputStream fis = new FileInputStream(new File(path));
if (path.contains("yml")) {
props.putAll(new Yaml().loadAs(fis, Map.class));
} else {
props.load(fis);
}
LOGGER.info("读取文件：{} 最新配置:{}", path, JSON.toJSONString(props));
modifyFile(path);
LOGGER.info("修改文件完毕 准备触发监听器");
Thread.sleep(10000);
}
}

81
example/src/test/java/com/jd/platform/jlog/test/NacosConfiguratorTest.java Normal file
View File

@ -0,0 +1,81 @@
package com.jd.platform.jlog.test;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.clientdemo.ExampleApplication;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.List;
import java.util.Random;
import static com.jd.platform.jlog.test.Common.getTest;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName NacosConfiguratorTest.java
* @Description TODO
* @createTime 2022年03月01日 07:35:00
*/
@SpringBootTest(classes = ExampleApplication.class)
@RunWith(SpringRunner.class)
public class NacosConfiguratorTest {
private final static Logger LOGGER = LoggerFactory.getLogger(NacosConfiguratorTest.class);
private Configurator configurator = null;
@Before
public void init() {
configurator = ConfiguratorFactory.getInstance();
getTest(configurator);
}
@Test
public void testUpdateCFG() throws Exception {
List<String> workers = configurator.getList("workers");
LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers));
String myIp = "121.1.1.0";
if(workers.contains(myIp)){
// do nothing
LOGGER.info("自己的IP还在配置list里 什么也不做");
return;
}else{
LOGGER.info("自己的IP不在配置list里 添加进去并发布");
workers.add(myIp);
}
configurator.putConfig("workers",JSON.toJSONString(workers));
List<String> workers2 = configurator.getList("workers");
LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2));
}
@Test
public void testAddConfigListener() throws Exception {
int i1 = new Random().nextInt(2000);
int i2 = new Random().nextInt(2000);
String val1 = configurator.getString("testKey");
LOGGER.info("初始化的testKey的val:{}", val1);
// configurator.putConfig("testKey",i1 + "");
LOGGER.info("修改完毕 准备触发监听器");
Thread.sleep(1000);
String val2 = configurator.getString("testKey");
LOGGER.info("第一次修改后的的val:{}", val2);
configurator.putConfig("testKey",i2 + "");
LOGGER.info("准备验证监听器是否停止 最新testKey={}", configurator.getString("testKey"));
Thread.sleep(35000);
}
}

12
clientdemo/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java → example/src/test/java/com/jd/platform/jlog/test/TracerPacketTest.java
View File

@ -2,15 +2,11 @@ package com.jd.platform.jlog.test;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.client.udp.UdpSender;
import com.jd.platform.jlog.clientdemo.ClientDemoApplication;
import com.jd.platform.jlog.clientdemo.ExampleApplication;
import com.jd.platform.jlog.common.model.TracerBean;
import com.jd.platform.jlog.common.utils.IpUtils;
import com.jd.platform.jlog.common.utils.ZstdUtils;
import org.apache.tomcat.util.codec.binary.Base64;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.nio.charset.StandardCharsets;
import java.util.*;
@ -22,11 +18,11 @@ import java.util.*;
* @version 1.0
* @date 2021-12-27
*/
@SpringBootTest(classes = ClientDemoApplication.class)
@RunWith(SpringRunner.class)
//@SpringBootTest(classes = ExampleApplication.class)
//@RunWith(SpringRunner.class)
public class TracerPacketTest {
@Test
//@Test
public void testSendUdp() {
TracerBean tracerBean = new TracerBean();
List<Map<String, Object>> tracerObject = new ArrayList<>();

79
example/src/test/java/com/jd/platform/jlog/test/ZKConfiguratorTest.java Normal file
View File

@ -0,0 +1,79 @@
package com.jd.platform.jlog.test;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.clientdemo.ExampleApplication;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.List;
import java.util.Random;
import static com.jd.platform.jlog.test.Common.getTest;
/**
* @author tangbohu
* @version 1.0.0
* @ClassName ZKConfiguratorTest.java
* @Description TODO
* @createTime 2022年03月01日 07:35:00
*/
@SpringBootTest(classes = ExampleApplication.class)
@RunWith(SpringRunner.class)
public class ZKConfiguratorTest {
private final static Logger LOGGER = LoggerFactory.getLogger(ZKConfiguratorTest.class);
private Configurator configurator = null;
@Before
public void init() {
configurator = ConfiguratorFactory.getInstance();
getTest(configurator);
}
@Test
public void testUpdateCFG() throws Exception {
List<String> workers = configurator.getList("workers");
LOGGER.info("初始化的workers：{}", JSON.toJSONString(workers));
String myIp = "121.0";
if(workers.contains(myIp)){
LOGGER.info("自己的IP还在配置list里 什么也不做");
return;
}else{
LOGGER.info("自己的IP不在配置list里 添加进去并发布");
workers.add(myIp);
}
configurator.putConfig("workers",JSON.toJSONString(workers));
List<String> workers2 = configurator.getList("workers");
LOGGER.info("最新的workers：{}", JSON.toJSONString(workers2));
}
@Test
public void testAddConfigListener() throws Exception {
int i1 = new Random().nextInt(2000);
int i2 = new Random().nextInt(2000);
String val1 = configurator.getString("testKey");
LOGGER.info("初始化的testKey的val:{}", val1);
LOGGER.info("添加监听器后， 修改配置testKey = {}", i1);
// configurator.putConfig("testKey",i1 + "");
LOGGER.info("修改完毕 准备触发监听器");
Thread.sleep(1000);
String val2 = configurator.getString("testKey");
LOGGER.info("第一次修改后的的val:{}", val2);
Thread.sleep(1000);
}
}

3
pom.xml
View File

@ -16,7 +16,8 @@
<module>clientlog4j2</module>
<module>clientlogback</module>
<module>Dashboard</module>
<module>clientdemo</module>
<module>example</module>
<module>config</module>
</modules>
<properties>

11
worker/pom.xml
View File

@ -22,6 +22,11 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.jd.platfrom.jlog</groupId>
<artifactId>config-zk</artifactId>
<version>1.4-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@ -43,6 +48,12 @@
<artifactId>disruptor</artifactId>
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
<build>

6
worker/src/main/java/com/jd/platform/jlog/worker/Starter.java
View File

@ -1,6 +1,6 @@
package com.jd.platform.jlog.worker;
import com.jd.platform.jlog.worker.config.EtcdStarter;
import com.jd.platform.jlog.worker.config.CenterStarter;
import com.jd.platform.jlog.worker.store.TracerLogToDbStore;
import com.jd.platform.jlog.worker.store.TracerModelToDbStore;
import com.jd.platform.jlog.worker.udp.UdpServer;
@ -42,7 +42,7 @@ public class Starter {
* etcd启动器
*/
@Resource
private EtcdStarter etcdStarter;
private CenterStarter centerStarter;
@PostConstruct
public void start() {
@ -58,7 +58,7 @@ public class Starter {
tracerLogToDbStore.beginIntoDb();
//上报自己ip到配置中心
etcdStarter.uploadSelfInfo();
centerStarter.uploadSelfInfo();
}
}

30
worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdStarter.java → worker/src/main/java/com/jd/platform/jlog/worker/config/CenterStarter.java
View File

@ -1,12 +1,16 @@
package com.jd.platform.jlog.worker.config;
import com.jd.platform.jlog.common.config.IConfigCenter;
import com.alibaba.fastjson.JSON;
import com.jd.platform.jlog.common.constant.Constant;
import com.jd.platform.jlog.common.utils.IpUtils;
import com.jd.platform.jlog.core.Configurator;
import com.jd.platform.jlog.core.ConfiguratorFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ -19,7 +23,7 @@ import java.util.concurrent.TimeUnit;
* @date 2021-08-12
*/
@Component
public class EtcdStarter {
public class CenterStarter {
/**
* 该worker为哪个app服务
*/
@ -35,23 +39,23 @@ public class EtcdStarter {
*/
@Value("${config.mdc}")
private String mdc;
/**
* configCenter
*/
@Resource
private IConfigCenter configCenter;
/**
* 上报自己的ip到配置中心
*/
public void uploadSelfInfo() {
//开启上传worker信息
Configurator config = ConfiguratorFactory.getInstance();
ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
scheduledExecutorService.scheduleAtFixedRate(() -> {
try {
configCenter.putAndGrant(buildKey(), buildValue(), 8);
configCenter.putAndGrant(buildSecondKey(), buildValue(), 8);
List<String> list = config.getList("workers");
if(!list.contains(buildKey())){
list.add(buildValue());
config.putConfig("workers", JSON.toJSONString(list));
}
} catch (Exception e) {
//do nothing
e.printStackTrace();
@ -68,14 +72,6 @@ public class EtcdStarter {
return Constant.WORKER_PATH + workerPath + "/" + hostName;
}
/**
* 在配置中心对应机房存放的key
*/
private String buildSecondKey() {
String hostName = IpUtils.getHostName();
return Constant.WORKER_PATH + workerPath + "/" + mdc + "/" + hostName;
}
/**
* 在配置中心存放的value
*/

31
worker/src/main/java/com/jd/platform/jlog/worker/config/EtcdConfig.java
View File

@ -1,31 +0,0 @@
package com.jd.platform.jlog.worker.config;
import com.jd.platform.jlog.common.config.IConfigCenter;
import com.jd.platform.jlog.common.config.etcd.JdEtcdBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* EtcdConfig
* @author wuweifeng wrote on 2019-12-06
* @version 1.0
*/
@Configuration
public class EtcdConfig {
@Value("${config.server}")
private String etcdServer;
private Logger logger = LoggerFactory.getLogger(getClass());
@Bean
public IConfigCenter client() {
logger.info("etcd address : " + etcdServer);
//连接多个时，逗号分隔
return JdEtcdBuilder.build(etcdServer);
}
}

10
worker/src/main/java/com/jd/platform/jlog/worker/config/IConfig.java
View File

@ -1,10 +0,0 @@
package com.jd.platform.jlog.worker.config;
/**
* 配置中心接口
* @author wuweifeng
* @version 1.0
* @date 2021-08-13
*/
public interface IConfig {
}

5
worker/src/main/java/com/jd/platform/jlog/worker/controller/UserController.java
View File

@ -1,6 +1,5 @@
package com.jd.platform.jlog.worker.controller;
import cn.hutool.core.date.DateUtil;
import com.jd.platform.jlog.worker.db.Db;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@ -32,12 +31,12 @@ public class UserController {
Map<String, Object> map0 = new HashMap<>();
map0.put("clientType", 1);
map0.put("pin", "abcd");
map0.put("createTime", DateUtil.formatDateTime(new Date()));
// map0.put("createTime", DateUtil.formatDateTime(new Date()));
Map<String, Object> map1 = new HashMap<>();
map1.put("clientType", 1);
map1.put("pin", "abcd");
map1.put("createTime", DateUtil.formatDateTime(new Date()));
// map1.put("createTime", DateUtil.formatDateTime(new Date()));
datas.add(map0);
datas.add(map1);

58
worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java
View File

@ -1,6 +1,5 @@
package com.jd.platform.jlog.worker.disruptor;
import cn.hutool.core.date.DateUtil;
import com.jd.platform.jlog.common.model.RunLogMessage;
import com.jd.platform.jlog.common.model.TracerBean;
import com.jd.platform.jlog.common.model.TracerData;
@ -105,8 +104,8 @@ public class TracerConsumer implements WorkHandler<OneTracer> {
map.put("threadName", runLogMessage.getThreadName());
map.put("methodName", runLogMessage.getMethodName());
map.put("logLevel", runLogMessage.getLogLevel());
map.put("createTime", DateUtil.formatDateTime(new Date(runLogMessage.getCreateTime())));
map.put("content", runLogMessage.getContent());
map.putAll(runLogMessage.getTagMap());
tracerLogToDbStore.offer(map);
}
@ -116,9 +115,12 @@ public class TracerConsumer implements WorkHandler<OneTracer> {
* 处理filter里处理的出入参
*/
private void dealFilterModel(TracerBean tracerBean) {
List<Map<String, Object>> mapList = tracerBean.getTracerObject();
Map<String, Object> requestMap = mapList.get(0);
Map<String, Object> map = new HashMap<>(requestMap);
long tracerId = requestMap.get("tracerId") == null ? 0 : Long.valueOf(requestMap.get("tracerId").toString());
//filter的出入参
Map<String, Object> responseMap = mapList.get(mapList.size() - 1);
@ -128,59 +130,11 @@ public class TracerConsumer implements WorkHandler<OneTracer> {
responseBytes = (byte[]) responseMap.get("response");
}
Map<String, Object> map = new HashMap<>();
//jsf的是用户自己设置的request入参，http的是从httpRequest读取的
if (requestMap.get("wholeRequest") == null) {
map.put("requestContent", FastJsonUtils.collectToString(requestMap));
} else {
map.put("requestContent", requestMap.get("wholeRequest"));
}
//此处做了一个base64编码，否则原编码直接进去，取出来后是String，直接getBytes后无法用Zstd解压
map.put("responseContent", responseBytes);
map.put("createTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime())));
map.put("costTime", tracerBean.getCostTime());
map.put("tracerId", tracerId);
String pin = requestMap.get("pin") == null ? "" : requestMap.get("pin").toString();
map.put("pin", pin);
String uri = requestMap.get("uri") == null ? "" : requestMap.get("uri").toString();
map.put("uri", uri);
//appName
String appName = requestMap.get("appName") == null ? "" : requestMap.get("appName").toString();
map.put("appName", appName);
String openudid = requestMap.get("openudid") == null ? "" : requestMap.get("openudid").toString();
if (StringUtil.isNullOrEmpty(openudid)) {
String uuid = requestMap.get("uuid") == null ? "" : requestMap.get("uuid").toString();
map.put("uuid", uuid);
} else {
map.put("uuid", openudid);
}
String client = requestMap.get("client") == null ? "" : requestMap.get("client").toString();
int clientType = 0;
if ("apple".equals(client)) {
clientType = 2;
} else if ("android".equals(client)) {
clientType = 1;
}
map.put("clientType", clientType);
String clientVersion = requestMap.get("clientVersion") == null ? "" : requestMap.get("clientVersion").toString();
map.put("clientVersion", clientVersion);
String userIp = requestMap.get("ip") == null ? "" : requestMap.get("ip").toString();
map.put("userIp", userIp);
String serverIp = requestMap.get("serverIp") == null ? "" : requestMap.get("serverIp").toString();
map.put("serverIp", serverIp);
map.put("intoDbTime", DateUtil.formatDateTime(new Date(tracerBean.getCreateTime())));
responseMap.remove("response");
map.putAll(responseMap);
tracerModelToDbStore.offer(map);
}

160
worker/src/main/java/com/jd/platform/jlog/worker/entity/TracerModel.java
View File

@ -1,160 +0,0 @@
package com.jd.platform.jlog.worker.entity;
import java.io.Serializable;
/**
* 入库、检索时用的对象
*
* @author wuweifeng
* @version 1.0
* @date 2021-08-20
*/
public class TracerModel implements Serializable {
/**
* tracerId
*/
private long tracerId;
/**
* 请求的入参
*/
private String requestContent;
/**
* 响应的出参
*/
private String responseContent;
/**
* 日志请求时间（数据库里存的是DateTime，2021-08-24 19:47:30.0）
*/
private long createTime;
/**
* 请求耗时（即响应时间戳减去请求时间戳）
*/
private int costTime;
/**
* 用户pin
*/
private String pin;
/**
* 用户uuid
*/
private String uuid;
/**
* Android=1、ios=2
*/
private int clientType;
/**
* 客户端版本号，9.3.6
*/
private String clientVersion;
/**
* 用户ip
*/
private String userIp;
/**
* 服务端ip
*/
private String serverIp;
/**
* 入库时间
*/
private long intoDbTime;
public long getIntoDbTime() {
return intoDbTime;
}
public void setIntoDbTime(long intoDbTime) {
this.intoDbTime = intoDbTime;
}
public String getRequestContent() {
return requestContent;
}
public void setRequestContent(String requestContent) {
this.requestContent = requestContent;
}
public String getResponseContent() {
return responseContent;
}
public void setResponseContent(String responseContent) {
this.responseContent = responseContent;
}
public long getTracerId() {
return tracerId;
}
public void setTracerId(long tracerId) {
this.tracerId = tracerId;
}
public long getCreateTime() {
return createTime;
}
public void setCreateTime(long createTime) {
this.createTime = createTime;
}
public int getCostTime() {
return costTime;
}
public void setCostTime(int costTime) {
this.costTime = costTime;
}
public String getPin() {
return pin;
}
public void setPin(String pin) {
this.pin = pin;
}
public String getUuid() {
return uuid;
}
public void setUuid(String uuid) {
this.uuid = uuid;
}
public int getClientType() {
return clientType;
}
public void setClientType(int clientType) {
this.clientType = clientType;
}
public String getClientVersion() {
return clientVersion;
}
public void setClientVersion(String clientVersion) {
this.clientVersion = clientVersion;
}
public String getUserIp() {
return userIp;
}
public void setUserIp(String userIp) {
this.userIp = userIp;
}
public String getServerIp() {
return serverIp;
}
public void setServerIp(String serverIp) {
this.serverIp = serverIp;
}
}

8
worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java
View File

@ -1,8 +1,8 @@
package com.jd.platform.jlog.worker.store;
import cn.hutool.core.collection.CollectionUtil;
import com.google.common.collect.Queues;
import com.jd.platform.jlog.common.utils.AsyncPool;
import com.jd.platform.jlog.common.utils.AsyncWorker;
import com.jd.platform.jlog.worker.db.Db;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -94,8 +94,8 @@ public class TracerLogToDbStore {
try {
List<Map<String, Object>> tempModels = new ArrayList<>();
//每1s入库一次
Queues.drain(logQueue, tempModels, Integer.valueOf(batchSize), interval, TimeUnit.SECONDS);
if (CollectionUtil.isEmpty(tempModels)) {
AsyncWorker.drain(logQueue, tempModels, Integer.valueOf(batchSize), interval, TimeUnit.SECONDS);
if (tempModels.size() == 0) {
continue;
}

2
worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java
View File

@ -1,8 +1,8 @@
package com.jd.platform.jlog.worker.store;
import cn.hutool.core.collection.CollectionUtil;
import com.google.common.collect.Queues;
import com.jd.platform.jlog.common.utils.AsyncPool;
import com.jd.platform.jlog.common.utils.CollectionUtil;
import com.jd.platform.jlog.worker.db.Db;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

6
worker/src/main/resources/application.yml
View File

@ -6,10 +6,8 @@ queue:
maxSize: ${queueSize:16384}
preDbSize: ${preDbSize:10000}
#etcd的地址，如有多个用逗号分隔
config:
mdc: ${mdc:default}
server: ${etcdServer:http://127.0.0.1:2379} #etcd的地址，重要！！！
workerPath: ${workerPath:default} #该worker放到哪个path下，譬如放/app1下，则该worker只能被app1使用，不会为其他client提供服务
serverAddr: 101.42.242.201:2181
server:
port: 8080
#ck信息，自行修改

Some files were not shown because too many files have changed in this diff Show More