test/jlog
1
0
Fork 0
mirror of https://gitee.com/jd-platform-opensource/jlog.git synced 2024-11-29 18:38:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

作者回来了

Browse Source
This commit is contained in:
weifengwu 2023-09-26 17:26:32 +08:00
parent d7aa8833f7
commit 9e1f8f1ae0
11 changed files with 38 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
common/pom.xml
View File

@ -13,19 +13,12 @@
<properties>
<java.version>1.8</java.version>
<netty4.version>4.1.42.Final</netty4.version>
<snappy.version>1.1.8.4</snappy.version>
<zstd.version>1.5.0-4</zstd.version>
<fastjson.version>1.2.83</fastjson.version>
<fastjson.version>2.0.31</fastjson.version>
<protostuff.version>1.7.2</protostuff.version>
</properties>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.32</version>
</dependency>
<dependency>
<groupId>io.protostuff</groupId>
<artifactId>protostuff-core</artifactId>
@ -36,13 +29,6 @@
<artifactId>protostuff-runtime</artifactId>
<version>${protostuff.version}</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>

3
common/src/main/java/com/jd/platform/jlog/common/utils/AsyncPool.java
View File

@ -1,10 +1,7 @@
package com.jd.platform.jlog.common.utils;
import java.util.Collection;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* 异步线程池AsyncPool

4
common/src/main/java/com/jd/platform/jlog/common/utils/CollectionUtil.java
View File

@ -1,11 +1,7 @@
package com.jd.platform.jlog.common.utils;
import com.alibaba.fastjson.JSON;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* @author tangbohu

44
common/src/main/java/com/jd/platform/jlog/common/utils/ConfigUtil.java
View File

@ -30,8 +30,7 @@ public class ConfigUtil {
public static String escapeExprSpecialWord(String str) {
if (str != null && str.length() > 0) {
if (str != null && !str.isEmpty()) {
for (String s : Constant.SPECIAL_CHAR) {
if (str.contains(s)) {
str = str.replace(s, "\\" + s);
@ -123,8 +122,6 @@ public class ConfigUtil {
}
public static String camelToMidline(String param) {
if (param == null || "".equals(param.trim())) {
return "";
@ -143,7 +140,7 @@ public class ConfigUtil {
return sb.toString();
}
public static String lowerFirst(String fromStr){
public static String lowerFirst(String fromStr) {
char[] chars = fromStr.toCharArray();
chars[0] += 32;
return String.valueOf(chars);
@ -152,6 +149,7 @@ public class ConfigUtil {
/**
* 只支持简单的对象形配置
*
* @param model bean
* @param properties 配置
* @param prefix 前缀
@ -169,11 +167,11 @@ public class ConfigUtil {
String curObjName = ConfigUtil.camelToMidline(lowerFirst(clz.getSimpleName()));
prefix = StringUtil.isEmpty(prefix) ? curObjName : prefix;
String fillName = !curObjName.equals(prefix) ? prefix +"."+ curObjName + "." + field.getName() : curObjName + "." + field.getName();
String fillName = !curObjName.equals(prefix) ? prefix + "." + curObjName + "." + field.getName() : curObjName + "." + field.getName();
switch (type){
switch (type) {
case "class java.lang.String":
field.set(model, properties.getString(fillName)) ;
field.set(model, properties.getString(fillName));
break;
case "byte":
field.setByte(model, Byte.valueOf(properties.getString(fillName)));
@ -182,7 +180,7 @@ public class ConfigUtil {
field.setShort(model, Short.valueOf(properties.getString(fillName)));
break;
case "int":
field.setInt(model, Integer.parseInt(properties.getString(fillName))) ;
field.setInt(model, Integer.parseInt(properties.getString(fillName)));
break;
case "long":
field.setLong(model, properties.getLong(fillName));
@ -198,34 +196,34 @@ public class ConfigUtil {
break;
case "class java.util.Date":
Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(properties.getString(fillName));
field.set(model,date) ;
field.set(model, date);
break;
default:
String tn = field.getType().getTypeName();
if("java.util.List".equals(tn)){
if ("java.util.List".equals(tn)) {
String[] arr = fillName.split("\\[");
int index = 0;
String suffix;
String fastSuffix;
List<String> list = new ArrayList<>();
do{
suffix = "["+index+"]";
fastSuffix = "["+(index+1)+"]";
list.add(properties.getString(arr[0]+suffix));
index ++;
}while (properties.getString(arr[0]+fastSuffix) != null);
do {
suffix = "[" + index + "]";
fastSuffix = "[" + (index + 1) + "]";
list.add(properties.getString(arr[0] + suffix));
index++;
} while (properties.getString(arr[0] + fastSuffix) != null);
field.set(model, list);
}else if("java.util.Map".equals(tn)){
} else if ("java.util.Map".equals(tn)) {
String val = properties.getString(fillName);
field.set(model,FastJsonUtils.toMap(val));
}else if(field.getType().isArray()){
field.set(model, FastJsonUtils.toMap(val));
} else if (field.getType().isArray()) {
String val = properties.getString(fillName);
field.set(model,FastJsonUtils.toArray(val));
}else{
field.set(model, FastJsonUtils.toArray(val));
} else {
String[] ar = type.split(" ");
Object tinyObj = Class.forName(ar[1]).newInstance();
invoke(tinyObj, properties, prefix);
field.set(model,tinyObj);
field.set(model, tinyObj);
}
}
}

16
common/src/main/java/com/jd/platform/jlog/common/utils/IdWorker.java
View File

@ -1,7 +1,5 @@
package com.jd.platform.jlog.common.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.net.NetworkInterface;
@ -18,8 +16,6 @@ import java.util.Enumeration;
* @date 2021-08-23
*/
public class IdWorker {
private static final Logger log = LoggerFactory.getLogger(IdWorker.class);
private static final long EPOCH;
private static final long SEQUENCE_BITS = 6L;
@ -73,7 +69,6 @@ public class IdWorker {
}
}
} catch (Exception e) {
log.debug("Error when getting host ip address: <{}>.", e.getMessage());
throw new IllegalStateException("Cannot get LocalHost InetAddress, please check your network!");
}
return null;
@ -85,9 +80,9 @@ public class IdWorker {
* @param workerId 工作进程Id
*/
private static void setWorkerId(final Long workerId) {
if(workerId >= 0L && workerId < WORKER_ID_MAX_VALUE){
if (workerId >= 0L && workerId < WORKER_ID_MAX_VALUE) {
IdWorker.workerId = workerId;
}else{
} else {
throw new RuntimeException("workerId is illegal");
}
}
@ -95,8 +90,8 @@ public class IdWorker {
//下一个ID生成算法
public static long nextId() {
long time = System.currentTimeMillis();
if(lastTime > time){
throw new RuntimeException("Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds"+lastTime);
if (lastTime > time) {
throw new RuntimeException("Clock is moving backwards, last time is %d milliseconds, current time is %d milliseconds" + lastTime);
}
if (lastTime == time) {
if (0L == (sequence = ++sequence & SEQUENCE_MASK)) {
@ -106,9 +101,6 @@ public class IdWorker {
sequence = 0;
}
lastTime = time;
if (log.isDebugEnabled()) {
log.debug("{}-{}-{}", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date(lastTime)), workerId, sequence);
}
return ((time - EPOCH) << TIMESTAMP_LEFT_SHIFT_BITS) | (workerId << WORKER_ID_LEFT_SHIFT_BITS) | sequence;
}

46
common/src/main/java/com/jd/platform/jlog/common/utils/SnappyUtils.java
View File

@ -1,46 +0,0 @@
package com.jd.platform.jlog.common.utils;
import org.xerial.snappy.Snappy;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
/**
* snappy压缩工具类
*
* @author wuweifeng
* @version 1.0
* @date 2021-08-16
*/
public class SnappyUtils {
/**
* 压缩
*/
public static byte[] compress(String html) {
return compress(html.getBytes(StandardCharsets.UTF_8));
}
/**
* 压缩
*/
public static byte[] compress(byte[] bytes) {
try {
return Snappy.compress(bytes);
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
/**
* 解压
*/
public static String decompress(byte[] bytes) {
try {
return new String(Snappy.uncompress(bytes));
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
}

101
common/src/main/java/com/jd/platform/jlog/common/utils/ZstdUtils.java
View File

@ -44,105 +44,4 @@ public class ZstdUtils {
return ob;
}
public static void main(String[] args) throws InterruptedException, ClassNotFoundException, IllegalAccessException, InstantiationException {
String s = "-75";
char firstChar = s.charAt(0);
System.out.println(firstChar);
if (firstChar < '0') { // Possible leading "+" or "-"
if (firstChar == '-') {
System.out.println("=========");
} else if (firstChar != '+')
System.out.println("eeee");
}
// System.out.println(Integer.parseInt("-75"));
String str1 = "1a";
String str2 = "0a";
String dbStr = Arrays.toString(compress(str1.getBytes()));
System.out.println(dbStr);
String[] arr0 = dbStr.split(",");
byte[] bt = new byte[arr0.length];
for (int i = 0; i < arr0.length; i++) {
if(i == 0){
String first = arr0[0].replace("[", "");
bt[i] = Byte.valueOf(first);
continue;
}
if(i == arr0.length-1){
String last = arr0[arr0.length-1].replace("]", "");
bt[i] = Byte.valueOf(last.trim());
continue;
}
System.out.println(arr0[i]+" len => "+arr0[i].length());
bt[i] = Byte.parseByte(arr0[i].trim());
}
System.out.println(Arrays.toString(bt));
// System.out.println(Arrays.toString(compress(str2.getBytes())));
AtomicInteger fail = new AtomicInteger(0);
AtomicInteger ok = new AtomicInteger(0);
for (int j = 0; j < 1; j++) {
new Thread(() -> {
for (int i = 0; i < 100; i++) {
int id = new Random().nextInt(99000000);
/* String ml = String.valueOf(getRandomChar()) +id+getRandomChar();
String ml2 = LocalDateTime.now().toString()+ getRandomChar();
ml = ml2 + ml;*/
// 压缩后的
String ml = ""+i;
byte[] mlc1= compress(ml.getBytes());
//存进去new String(mlc1)
byte[] resul = new String(mlc1).getBytes();
System.out.println(Arrays.toString(new String(mlc1).getBytes()));
byte[] arr = new byte[9];
arr[0] = 40;
arr[1] = -17;
arr[2] = -65;
arr[3] = -67;
arr[4] = 47;
arr[5] = -17;
arr[6] = -65;
arr[7] = -67;
arr[8] = 32;
// System.out.println(new String(arr));
if(resul[0]==40
&& resul[1]==-17
&& resul[2]==-65
&& resul[3]==-67
&& resul[4]==47
&& resul[5]==-17
&& resul[6]==-65
&& resul[7]==-67
&& resul[8]==32 ){
ok.incrementAndGet();
}else{
fail.incrementAndGet();
}
}
}).start();
}
Thread.sleep(30000);
System.out.println("ok == > "+ok.get());
System.out.println("fail == > "+fail.get());
}
public static char getRandomChar() {
return (char) (0x4e00 + (int) (Math.random() * (0x9fa5 - 0x4e00 + 1)));
}
}

12
worker/src/main/java/com/jd/platform/jlog/worker/disruptor/TracerConsumer.java
View File

@ -93,10 +93,10 @@ public class TracerConsumer implements WorkHandler<OneTracer> {
* 处理中途日志
*/
private void dealTracerLog(List<RunLogMessage> tempLogs) {
if(tempLogs==null){
if (tempLogs == null) {
return;
}
for (RunLogMessage runLogMessage :tempLogs) {
for (RunLogMessage runLogMessage : tempLogs) {
Map<String, Object> map = new HashMap<>(12);
map.put("tracerId", runLogMessage.getTracerId());
map.put("className", runLogMessage.getClassName());
@ -114,11 +114,11 @@ public class TracerConsumer implements WorkHandler<OneTracer> {
* 处理filter里处理的出入参
*/
private void dealFilterModel(List<TracerBean> tracerList) {
if(tracerList==null){
if (tracerList == null) {
return;
}
for(TracerBean tracerModel:tracerList){
if(tracerModel.getResponseContent()==null){
for (TracerBean tracerModel : tracerList) {
if (tracerModel.getResponseContent() == null) {
tracerModel.setResponseContent(DEFAULT_BYTE);
}
tracerModel.setCreateTime(formatLongTime(tracerModel.getCreateTimeLong()));
@ -130,7 +130,7 @@ public class TracerConsumer implements WorkHandler<OneTracer> {
}
private static String formatLongTime(long time) {
return DEFAULT_FORMATTER.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(time),ZoneId.systemDefault()));
return DEFAULT_FORMATTER.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(time), ZoneId.systemDefault()));
}
}

2
worker/src/main/java/com/jd/platform/jlog/worker/store/TracerLogToDbStore.java
View File

@ -81,7 +81,7 @@ public class TracerLogToDbStore {
logger.info("logQueue init success");
//初始化队列长度
logQueue = new LinkedBlockingQueue<>(preDbSize);
int pool = Integer.valueOf(poolSize);
int pool = Integer.parseInt(poolSize);
for (int i = 0; i < pool; i++) {
AsyncPool.asyncDo(() -> {

4
worker/src/main/java/com/jd/platform/jlog/worker/store/TracerModelToDbStore.java
View File

@ -80,7 +80,7 @@ public class TracerModelToDbStore {
public void beginIntoDb() {
//初始化队列长度
modelQueue = new LinkedBlockingQueue<>(preDbSize);
int pool = Integer.valueOf(poolSize);
int pool = Integer.parseInt(poolSize);
for (int i = 0; i < pool; i++) {
AsyncPool.asyncDo(() -> {
@ -93,7 +93,7 @@ public class TracerModelToDbStore {
try {
List<Map<String, Object>> tempModels = new ArrayList<>();
//每1s入库一次
AsyncWorker.drain(modelQueue, tempModels, Integer.valueOf(batchSize), interval, TimeUnit.SECONDS);
AsyncWorker.drain(modelQueue, tempModels, Integer.parseInt(batchSize), interval, TimeUnit.SECONDS);
if (CollectionUtil.isEmpty(tempModels)) {
continue;
}

7
worker/src/main/java/com/jd/platform/jlog/worker/udp/UdpServer.java
View File

@ -1,7 +1,6 @@
package com.jd.platform.jlog.worker.udp;
import com.jd.platform.jlog.worker.disruptor.Producer;
import com.jd.platform.jlog.worker.store.TracerBeanStore;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.*;
@ -12,7 +11,6 @@ import io.netty.handler.codec.MessageToMessageDecoder;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.Arrays;
import java.util.List;
/**
@ -25,11 +23,6 @@ import java.util.List;
*/
@Component
public class UdpServer {
/**
* 信息存储器
*/
@Resource
private TracerBeanStore tracerBeanStore;
/**
* producer
*/