test/radar
1
0
Fork 0
mirror of https://gitee.com/freshday/radar.git synced 2024-11-30 02:47:59 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor:围绕风控模型的缓存重构,删除MongodbUtil

Browse Source 
feat:规则引擎判断 机器学习是否启用
This commit is contained in:
wangcheng 2020-04-20 17:42:03 +08:00
parent 7fdedebbc9
commit eaea439ef7
20 changed files with 245 additions and 379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
radar-admin/src/main/java/com/pgmmers/AdminApplication.java
View File

@ -12,7 +12,6 @@ import tk.mybatis.spring.annotation.MapperScan;
public class AdminApplication {
public static void main(String[] args) {
System.setProperty("es.set.netty.runtime.available.processors", "false");
SpringApplication.run(AdminApplication.class, args);
}
}

8
radar-admin/src/main/java/com/pgmmers/radar/controller/ModelApiController.java
View File

@ -36,13 +36,13 @@ public class ModelApiController {
@GetMapping("/{id}")
public CommonResult get(@PathVariable Long id) {
CommonResult result = new CommonResult();
ModelVO modelVO = modelService.get(id);
ModelVO modelVO = modelService.getModelById(id);
if (modelVO != null) {
result.setSuccess(true);
result.getData().put("model", modelVO);
}
return result;
}
}
@GetMapping("/list")
public CommonResult list(HttpServletRequest request) {
@ -119,14 +119,14 @@ public class ModelApiController {
@PostMapping("/enable/{id}")
public CommonResult enable(@PathVariable Long id) {
ModelVO modelVO = modelService.get(id);
ModelVO modelVO = modelService.getModelById(id);
modelVO.setStatus(StatusType.ACTIVE.getKey());
return modelService.save(modelVO);
}
@PostMapping("/disable/{id}")
public CommonResult disable(@PathVariable Long id) {
ModelVO modelVO = modelService.get(id);
ModelVO modelVO = modelService.getModelById(id);
modelVO.setStatus(StatusType.INACTIVE.getKey());
return modelService.save(modelVO);
}

1
radar-engine/src/main/java/com/pgmmers/radar/EngineApplication.java
View File

@ -10,7 +10,6 @@ import tk.mybatis.spring.annotation.MapperScan;
public class EngineApplication {
public static void main(String[] args) {
System.setProperty("es.set.netty.runtime.available.processors", "false");
SpringApplication.run(EngineApplication.class, args);
}

1
radar-engine/src/main/resources/application-dev.yml
View File

@ -37,5 +37,6 @@ sys:
entity-duplicate-insert: false # 事件是否允许重复插入
mongo-restore-days: 93 # 事件保存时间，默认3个月
workdir: /radar/workdir # 工作目录
machine-learning: true # 是否启用 机器学习
server:
port: 6581

5
radar-service-impl/pom.xml
View File

@ -47,6 +47,9 @@
<groupId>org.tensorflow</groupId>
<artifactId>tensorflow</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
</dependencies>
</project>

67
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/cache/CacheServiceImpl.java vendored
View File

@ -1,19 +1,23 @@
package com.pgmmers.radar.service.impl.cache;
import com.alibaba.fastjson.JSON;
import com.pgmmers.radar.service.cache.CacheService;
import com.pgmmers.radar.service.cache.RedisService;
import com.pgmmers.radar.service.cache.SubscribeHandle;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.vo.model.*;
import com.pgmmers.radar.vo.model.AbstractionVO;
import com.pgmmers.radar.vo.model.ActivationVO;
import com.pgmmers.radar.vo.model.DataListRecordVO;
import com.pgmmers.radar.vo.model.DataListsVO;
import com.pgmmers.radar.vo.model.FieldVO;
import com.pgmmers.radar.vo.model.ModelVO;
import com.pgmmers.radar.vo.model.PreItemVO;
import com.pgmmers.radar.vo.model.RuleVO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class CacheServiceImpl implements CacheService {
@ -29,59 +33,10 @@ public class CacheServiceImpl implements CacheService {
public static final String PUB_SUB_ACTIVATION_CHANNEL = "radar_channel_activation";
public static final String PUB_SUB_LISTRECORD_CHANNEL = "radar_channel_listrecord";
public static final String PUB_SUB_DATALIST_CHANNEL = "radar_channel_datalist";
@Autowired
private RedisService redisService;
@Override
public void saveModel(ModelVO model) {
redisService.set(prefix + "model_" + model.getId(), model);
}
@Override
public ModelVO getModel(Long modelId) {
return redisService.get(prefix + "model_" + modelId, ModelVO.class);
}
@Override
public void saveField(FieldVO field) {
String key = prefix + "field_" + field.getModelId();
redisService.hset(key, field.getFieldName(), field);
}
@Override
public List<FieldVO> listFields(Long modelId) {
String key = prefix + "field_" + modelId;
List<FieldVO> list = redisService.hvals(key);
return list;
}
@Override
public void saveAbstraction(AbstractionVO abstraction) {
String key = prefix + "abstraction_" + abstraction.getModelId();
redisService.hset(key, abstraction.getName(), abstraction);
}
@Override
public List<AbstractionVO> listAbstraction(Long modelId) {
String key = prefix + "abstraction_" + modelId;
List<AbstractionVO> list = redisService.hget(key, AbstractionVO.class);
return list;
}
@Override
public void saveActivation(ActivationVO activation) {
String key = prefix + "activation_" + activation.getModelId();
redisService.hset(key, activation.getActivationName(), activation);
}
@Override
public List<ActivationVO> listActivation(Long modelId) {
String key = prefix + "activation_" + modelId;
List<ActivationVO> list = redisService.hget(key, ActivationVO.class);
return list;
}
@Override
public void saveAntiFraudResult(String modelId, String sessionId, CommonResult result) {
String key = prefix + "engine_" + modelId + sessionId;
@ -169,7 +124,7 @@ public class CacheServiceImpl implements CacheService {
@Override
public void publishDataListRecord(DataListRecordVO record) {
redisService.publish(PUB_SUB_LISTRECORD_CHANNEL, JSON.toJSONString(record));
}
@Override
@ -180,7 +135,7 @@ public class CacheServiceImpl implements CacheService {
@Override
public void publishDataList(DataListsVO dataList) {
redisService.publish(PUB_SUB_DATALIST_CHANNEL, JSON.toJSONString(dataList));
}
@Override

59
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/engine/AntiFraudEngineImpl.java
View File

@ -8,27 +8,49 @@ import com.pgmmers.radar.enums.StatusType;
import com.pgmmers.radar.service.dnn.Estimator;
import com.pgmmers.radar.service.engine.AggregateCommand;
import com.pgmmers.radar.service.engine.AntiFraudEngine;
import com.pgmmers.radar.service.engine.vo.*;
import com.pgmmers.radar.service.engine.vo.AbstractionResult;
import com.pgmmers.radar.service.engine.vo.ActivationResult;
import com.pgmmers.radar.service.engine.vo.AdaptationResult;
import com.pgmmers.radar.service.engine.vo.HitObject;
import com.pgmmers.radar.service.engine.vo.RiskObject;
import com.pgmmers.radar.service.impl.dnn.EstimatorContainer;
import com.pgmmers.radar.service.model.*;
import com.pgmmers.radar.service.model.AbstractionService;
import com.pgmmers.radar.service.model.ActivationService;
import com.pgmmers.radar.service.model.DataListsService;
import com.pgmmers.radar.service.model.EntityService;
import com.pgmmers.radar.service.model.FieldService;
import com.pgmmers.radar.service.model.ModelService;
import com.pgmmers.radar.service.model.RuleService;
import com.pgmmers.radar.util.DateUtils;
import com.pgmmers.radar.util.GroovyScriptUtil;
import com.pgmmers.radar.vo.model.*;
import com.pgmmers.radar.vo.model.AbstractionVO;
import com.pgmmers.radar.vo.model.ActivationVO;
import com.pgmmers.radar.vo.model.DataListRecordVO;
import com.pgmmers.radar.vo.model.DataListsVO;
import com.pgmmers.radar.vo.model.FieldVO;
import com.pgmmers.radar.vo.model.ModelVO;
import com.pgmmers.radar.vo.model.RuleVO;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.math.BigDecimal;
import java.util.*;
@Service
public class AntiFraudEngineImpl implements AntiFraudEngine {
private static Logger logger = LoggerFactory.getLogger(AntiFraudEngineImpl.class);
private static Map<Long, Map<String, Object>> dataListCacheMap = new HashMap<Long, Map<String, Object>>();
@Value("${sys.conf.machine-learning: false}")
private boolean machineLearning;
@Autowired
private EntityService entityService;
@ -62,7 +84,7 @@ public class AntiFraudEngineImpl implements AntiFraudEngine {
// 1. 解析 参数信息
//JSONObject entity = (JSONObject) data.get("entity");
// 2. list abstraction
// 2. list abstraction
List<AbstractionVO> abstractions = abstractionService.listAbstraction(modelId);
// 排除没有的定义 abstraction 的情况。
@ -211,9 +233,9 @@ public class AntiFraudEngineImpl implements AntiFraudEngine {
}
/**
*
*
* 后续需要优化.delete from next version
*
*
* @param modelId
* @return
* @author feihu.wang
@ -248,10 +270,13 @@ public class AntiFraudEngineImpl implements AntiFraudEngine {
@Override
public AdaptationResult executeAdaptation(Long modelId, Map<String, Map<String, ?>> data) {
AdaptationResult result = new AdaptationResult();
Estimator estimator = estimatorContainer.getByModelId(modelId);
if(estimator != null) {
float score = estimator.predict(modelId, data);
result.getAdaptationMap().put("score", score);
// 启动机器学习
if (machineLearning){
Estimator estimator = estimatorContainer.getByModelId(modelId);
if(estimator != null) {
float score = estimator.predict(modelId, data);
result.getAdaptationMap().put("score", score);
}
}
result.setSuccess(true);
data.put("adaptations", result.getAdaptationMap());
@ -264,7 +289,7 @@ public class AntiFraudEngineImpl implements AntiFraudEngine {
List<ActivationVO> activations = activationService.listActivation(modelId);
// 获取预加载的黑/白名单集合
Map<String, Object> dataCollectionMap = dataListsService.getDataListMap(modelId);
for (ActivationVO act : activations) {
if (!act.getStatus().equals(StatusType.ACTIVE.getKey())) {
continue;
@ -347,9 +372,9 @@ public class AntiFraudEngineImpl implements AntiFraudEngine {
}
/**
*
*
* 检查数据是否符合条件.
*
*
* @param ruleScript string
* @param entity map of params
* @param dataCollectionMap like black list

44
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/AbstractionServiceImpl.java
View File

@ -1,7 +1,6 @@
package com.pgmmers.radar.service.impl.model;
import com.alibaba.fastjson.JSON;
import com.pgmmers.radar.dal.bean.AbstractionQuery;
import com.pgmmers.radar.dal.model.AbstractionDal;
import com.pgmmers.radar.dal.model.ModelDal;
@ -11,21 +10,25 @@ import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.model.AbstractionService;
import com.pgmmers.radar.util.GroovyScriptUtil;
import com.pgmmers.radar.vo.model.AbstractionVO;
import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class AbstractionServiceImpl implements AbstractionService, SubscribeHandle {
public class AbstractionServiceImpl extends BaseLocalCacheService implements AbstractionService,
SubscribeHandle {
public static Logger logger = LoggerFactory.getLogger(AbstractionServiceImpl.class);
@Override
public Object query(Long modelId) {
return modelDal.listAbstraction(modelId, null);
}
@Autowired
private ModelDal modelDal;
@Autowired
@ -33,16 +36,10 @@ public class AbstractionServiceImpl implements AbstractionService, SubscribeHand
@Autowired
private CacheService cacheService;
public Map<Long, List<AbstractionVO>> contextMap = new HashMap<Long, List<AbstractionVO>>();
@Override
public List<AbstractionVO> listAbstraction(Long modelId) {
List<AbstractionVO> list = contextMap.get(modelId);
if (list == null || list.size() == 0) {
list = modelDal.listAbstraction(modelId, null);
contextMap.put(modelId, list);
}
return list;
//noinspection unchecked
return (List<AbstractionVO>) getByCache(modelId);
}
@Override
@ -50,8 +47,7 @@ public class AbstractionServiceImpl implements AbstractionService, SubscribeHand
logger.info("abstraction sub:{}", message);
AbstractionVO abstraction = JSON.parseObject(message, AbstractionVO.class);
if (abstraction != null) {
List<AbstractionVO> list = modelDal.listAbstraction(abstraction.getModelId(), null);
contextMap.put(abstraction.getModelId(), list);
invalidateCache(abstraction.getModelId());
}
}
@ -65,7 +61,7 @@ public class AbstractionServiceImpl implements AbstractionService, SubscribeHand
public CommonResult list(Long modelId) {
CommonResult result = new CommonResult();
result.setSuccess(true);
result.getData().put("list", abstractionDal.list(modelId));
result.getData().put("list", listAbstraction(modelId));
return result;
}
@ -89,11 +85,11 @@ public class AbstractionServiceImpl implements AbstractionService, SubscribeHand
}
int count = abstractionDal.save(abstraction);
if (count > 0) {
if(StringUtils.isEmpty(abstraction.getName())){
abstraction.setName("abstraction_" + abstraction.getId());
abstractionDal.save(abstraction);
}
if (StringUtils.isEmpty(abstraction.getName())) {
abstraction.setName("abstraction_" + abstraction.getId());
abstractionDal.save(abstraction);
}
result.getData().put("id", abstraction.getId());
result.setSuccess(true);
// 通知更新
@ -114,7 +110,7 @@ public class AbstractionServiceImpl implements AbstractionService, SubscribeHand
}
return result;
}
@PostConstruct
public void init() {
cacheService.subscribeAbstraction(this);

32
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/ActivationServiceImpl.java
View File

@ -1,7 +1,6 @@
package com.pgmmers.radar.service.impl.model;
import com.alibaba.fastjson.JSON;
import com.pgmmers.radar.dal.bean.ActivationQuery;
import com.pgmmers.radar.dal.model.ActivationDal;
import com.pgmmers.radar.dal.model.ModelDal;
@ -10,19 +9,21 @@ import com.pgmmers.radar.service.cache.SubscribeHandle;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.model.ActivationService;
import com.pgmmers.radar.vo.model.ActivationVO;
import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class ActivationServiceImpl implements ActivationService, SubscribeHandle {
public class ActivationServiceImpl extends BaseLocalCacheService implements ActivationService, SubscribeHandle {
@Override
public Object query(Long modelId) {
return modelDal.listActivation(modelId, null);
}
public static Logger logger = LoggerFactory.getLogger(ActivationServiceImpl.class);
@ -35,16 +36,10 @@ public class ActivationServiceImpl implements ActivationService, SubscribeHandle
@Autowired
private CacheService cacheService;
public Map<Long, List<ActivationVO>> contextMap = new HashMap<Long, List<ActivationVO>>();
@Override
public List<ActivationVO> listActivation(Long modelId) {
List<ActivationVO> list = contextMap.get(modelId);
if (list == null || list.size() == 0) {
list = modelDal.listActivation(modelId, null);
contextMap.put(modelId, list);
}
return list;
//noinspection unchecked
return (List<ActivationVO>) getByCache(modelId);
}
@Override
@ -52,8 +47,7 @@ public class ActivationServiceImpl implements ActivationService, SubscribeHandle
logger.info("activation sub:{}", message);
ActivationVO act = JSON.parseObject(message, ActivationVO.class);
if (act != null) {
List<ActivationVO> list = modelDal.listActivation(act.getModelId(), null);
contextMap.put(act.getModelId(), list);
invalidateCache(act.getModelId());
}
}
@ -99,7 +93,7 @@ public class ActivationServiceImpl implements ActivationService, SubscribeHandle
}
return result;
}
@Override
public CommonResult updateOrder(Long activationId, String ruleOrder) {
CommonResult result = new CommonResult();
@ -117,6 +111,6 @@ public class ActivationServiceImpl implements ActivationService, SubscribeHandle
@PostConstruct
public void init() {
cacheService.subscribeActivation(this);
}
}
}

35
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/BaseLocalCacheService.java Normal file
View File

@ -0,0 +1,35 @@
package com.pgmmers.radar.service.impl.model;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.pgmmers.radar.service.cache.SubscribeHandle;
import java.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 围绕规则模型的缓存服务
*/
public abstract class BaseLocalCacheService implements SubscribeHandle {
private static final Logger logger = LoggerFactory.getLogger(BaseLocalCacheService.class);
protected LoadingCache<Long, Object> localCache = Caffeine
.newBuilder()
.maximumSize(64)
.expireAfterAccess(Duration.ofHours(1))
.build(this::query);
public Object query(Long key) {
logger.error("query is not implements,check!");
return null;
}
public Object getByCache(Long key) {
return localCache.get(key);
}
public void invalidateCache(Long key) {
localCache.invalidate(key);
}
}

10
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/DataListRecordServiceImpl.java
View File

@ -1,23 +1,20 @@
package com.pgmmers.radar.service.impl.model;
import com.alibaba.fastjson.JSON;
import com.pgmmers.radar.dal.model.DataListDal;
import com.pgmmers.radar.dal.model.ModelDal;
import com.pgmmers.radar.service.cache.CacheService;
import com.pgmmers.radar.service.cache.SubscribeHandle;
import com.pgmmers.radar.service.model.DataListRecordService;
import com.pgmmers.radar.service.model.DataListsService;
import com.pgmmers.radar.vo.model.DataListRecordVO;
import com.pgmmers.radar.vo.model.DataListsVO;
import java.util.Map;
import javax.annotation.PostConstruct;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.Map;
@Service
public class DataListRecordServiceImpl implements DataListRecordService,
SubscribeHandle {
@ -29,9 +26,6 @@ public class DataListRecordServiceImpl implements DataListRecordService,
private DataListDal dataListDal;
@Autowired
private CacheService cacheService;
@Autowired
private ModelDal modelDal;
@Autowired
private DataListsService dataListsService;

41
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/DataListsServiceImpl.java
View File

@ -4,38 +4,37 @@ import com.alibaba.fastjson.JSON;
import com.pgmmers.radar.dal.bean.DataListQuery;
import com.pgmmers.radar.dal.bean.DataListRecordQuery;
import com.pgmmers.radar.dal.model.DataListDal;
import com.pgmmers.radar.dal.model.ModelDal;
import com.pgmmers.radar.service.cache.CacheService;
import com.pgmmers.radar.service.cache.SubscribeHandle;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.model.DataListsService;
import com.pgmmers.radar.service.model.ModelService;
import com.pgmmers.radar.vo.model.DataListMetaVO;
import com.pgmmers.radar.vo.model.DataListRecordVO;
import com.pgmmers.radar.vo.model.DataListsVO;
import com.pgmmers.radar.vo.model.ModelVO;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
public static Logger logger = LoggerFactory.getLogger(DataListsServiceImpl.class);
@Autowired
private DataListDal dataListDal;
@Autowired
private CacheService cacheService;
@Autowired
private ModelDal modelDal;
private ModelService modelService;
private static Map<Long, Map<String, Object>> dataListRecordCacheMap = new HashMap<>();
@ -79,10 +78,10 @@ public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
dataList.setName("dataList_"+dataList.getId());
dataListDal.save(dataList);
}
result.getData().put("id", dataList.getId());
result.setSuccess(true);
// 通知更新
// 通知更新
dataList.setOpt("new");
cacheService.publishDataList(dataList);
}
@ -96,7 +95,7 @@ public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
int count = dataListDal.delete(id);
if (count > 0) {
result.setSuccess(true);
// 通知更新
// 通知更新
dataList.setOpt("delete");
cacheService.publishDataList(dataList);
}
@ -131,7 +130,7 @@ public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
metaMap.remove(dataListMeta.getId());
}
dataListDal.saveMeta(dataListMeta);
if(StringUtils.isEmpty(dataListMeta.getFieldName())){
dataListMeta.setFieldName("dataListMeta_"+dataListMeta.getId());
dataListDal.saveMeta(dataListMeta);
@ -225,14 +224,14 @@ public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
public void init() {
cacheService.subscribeDataList(this);
new Thread(() ->{
List<ModelVO> modelList = modelDal.listModel(null);
List<ModelVO> modelList = modelService.listModel(null);
// 加载系统数据名单列表
Map<String, Object> sysDataListMap = new HashMap<>();
List<DataListsVO> sysList = dataListDal.listDataLists(0L, null);
for (DataListsVO dataListVO : sysList) {
Map<String, String> dataListRecords = new HashMap<String, String>();
// record list
// record list
List<DataListRecordVO> recordVOList = dataListDal.listDataRecord(dataListVO.getId());
if (recordVOList != null) {
for (DataListRecordVO record : recordVOList) {
@ -241,8 +240,8 @@ public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
}
sysDataListMap.put(dataListVO.getName(), dataListRecords);
}
for (ModelVO model : modelList) {
Map<String, Object> dataListMap = new HashMap<>();
// datalist list
@ -250,7 +249,7 @@ public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
if (dataLists != null) {
for (DataListsVO dataListVO : dataLists) {
Map<String, String> dataListRecords = new HashMap<>();
// record list
// record list
List<DataListRecordVO> recordVOList = dataListDal.listDataRecord(dataListVO.getId());
if (recordVOList != null) {
for (DataListRecordVO record : recordVOList) {
@ -258,11 +257,11 @@ public class DataListsServiceImpl implements DataListsService, SubscribeHandle {
}
}
dataListMap.put(dataListVO.getName(), dataListRecords);
}
}
// add sys data list
dataListMap.putAll(sysDataListMap);
dataListRecordCacheMap.put(model.getId(), dataListMap);

49
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/EntityServiceImpl.java
View File

@ -1,15 +1,11 @@
package com.pgmmers.radar.service.impl.model;
import com.pgmmers.radar.dal.model.ModelDal;
import com.pgmmers.radar.service.cache.CacheService;
import com.pgmmers.radar.service.cache.SubscribeHandle;
import com.pgmmers.radar.service.data.MongoService;
import com.pgmmers.radar.service.model.EntityService;
import com.pgmmers.radar.service.model.ModelService;
import com.pgmmers.radar.vo.model.ModelVO;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.annotation.PostConstruct;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -17,43 +13,22 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class EntityServiceImpl implements EntityService, SubscribeHandle {
public class EntityServiceImpl implements EntityService {
private Logger logger = LoggerFactory.getLogger(EntityServiceImpl.class);
@Autowired
private ModelDal modelDal;
@Autowired
private CacheService cacheService;
private ModelService modelService;
@Autowired
private MongoService mongoService;
private List<ModelVO> modelList = new ArrayList<>();
@PostConstruct
public void init() {
modelList = modelDal.listModel(null);
cacheService.subscribeModel(this);
}
@Override
public int save(Long modelId, String jsonString, boolean isAllowDuplicate) {
String tmpUrl = "entity_" + modelId;
Document doc = Document.parse(jsonString);
if (!isAllowDuplicate) {
ModelVO model = null;//cacheService.getModel(modelId);
for (ModelVO vo : modelList) {
if (vo.getId().equals(modelId)) {
model = vo;
break;
}
}
if (model == null) {
model = modelDal.getModelById(modelId);
}
ModelVO model = modelService.getModelById(modelId);
Document filter = new Document();
filter.append(model.getEntryName(), doc.get(model.getEntryName()));
long qty = mongoService.count(tmpUrl, filter);
@ -72,27 +47,13 @@ public class EntityServiceImpl implements EntityService, SubscribeHandle {
return null;
}
@Override
public void onMessage(String channel, String message) {
modelList = modelDal.listModel(null);
}
@Override
public int save(Long modelId, String jsonString, String attachJson,
boolean isAllowDuplicate) {
String tmpUrl = "entity_" + modelId;
Document doc = Document.parse(jsonString);
Document atta = Document.parse(attachJson);
ModelVO model = null;//cacheService.getModel(modelId);
for (ModelVO vo : modelList) {
if (vo.getId().equals(modelId)) {
model = vo;
break;
}
}
if (model == null) {
model = modelDal.getModelById(modelId);
}
ModelVO model = modelService.getModelById(modelId);
atta.put("radar_ref_datetime", new Date(doc.getLong(model.getReferenceDate())));
doc.putAll(atta);
if (!isAllowDuplicate) {

48
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/FieldServiceImpl.java
View File

@ -1,7 +1,6 @@
package com.pgmmers.radar.service.impl.model;
import com.alibaba.fastjson.JSON;
import com.pgmmers.radar.dal.bean.FieldQuery;
import com.pgmmers.radar.dal.bean.PageResult;
import com.pgmmers.radar.dal.model.FieldDal;
@ -12,49 +11,43 @@ import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.model.FieldService;
import com.pgmmers.radar.vo.model.FieldVO;
import com.pgmmers.radar.vo.model.ModelVO;
import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class FieldServiceImpl implements FieldService, SubscribeHandle {
public class FieldServiceImpl extends BaseLocalCacheService implements FieldService, SubscribeHandle {
public static Logger logger = LoggerFactory.getLogger(FieldServiceImpl.class);
@Autowired
private ModelDal modelDal;
@Autowired
private FieldDal fieldDal;
@Autowired
private CacheService cacheService;
private final ModelDal modelDal;
private final FieldDal fieldDal;
private final CacheService cacheService;
public FieldServiceImpl(
ModelDal modelDal, FieldDal fieldDal, CacheService cacheService) {
this.modelDal = modelDal;
this.fieldDal = fieldDal;
this.cacheService = cacheService;
}
public Map<Long, List<FieldVO>> contextMap = new HashMap<Long, List<FieldVO>>();
@Override
public Object query(Long modelId) {
return modelDal.listField(modelId);
}
@Override
public List<FieldVO> listField(Long modelId) {
List<FieldVO> list = null;// cacheService.listFields(modelId);
list = contextMap.get(modelId);
if (list == null || list.size() == 0) {
list = modelDal.listField(modelId);
contextMap.put(modelId, list);
// for (FieldVO vo: list)
// cacheService.saveField(vo);
}
return list;
//noinspection unchecked
return (List<FieldVO>) getByCache(modelId);
}
@Override
public void onMessage(String channel, String message) {
logger.info("field sub: {},{}", channel, message);
FieldVO field = JSON.parseObject(message, FieldVO.class);
List<FieldVO> list = modelDal.listField(field.getModelId());
contextMap.put(field.getModelId(), list);
invalidateCache(field.getModelId());
}
@Override
@ -89,7 +82,6 @@ public class FieldServiceImpl implements FieldService, SubscribeHandle {
field.setFieldName("field_"+field.getId());
fieldDal.save(field);
}
result.getData().put("id", field.getId());
result.setSuccess(true);
// 通知更新
@ -117,7 +109,7 @@ public class FieldServiceImpl implements FieldService, SubscribeHandle {
}
return result;
}
@PostConstruct
public void init() {
cacheService.subscribeField(this);

59
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/ModelServiceImpl.java
View File

@ -16,14 +16,17 @@ import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.data.MongoService;
import com.pgmmers.radar.service.model.ModelService;
import com.pgmmers.radar.service.search.SearchEngineService;
import com.pgmmers.radar.util.JsonUtils;
import com.pgmmers.radar.vo.model.FieldVO;
import com.pgmmers.radar.vo.model.ModelVO;
import com.pgmmers.radar.vo.model.PreItemVO;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.bson.Document;
@ -35,7 +38,12 @@ import org.springframework.stereotype.Service;
@Service
public class ModelServiceImpl implements ModelService, SubscribeHandle {
public class ModelServiceImpl extends BaseLocalCacheService implements ModelService, SubscribeHandle {
@Override
public Object query(Long modelId) {
return modelDal.getModelById(modelId);
}
public static Logger logger = LoggerFactory
.getLogger(ModelServiceImpl.class);
@ -55,53 +63,48 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
@Autowired
private MongoService mongoService;
private List<ModelVO> modelList = new ArrayList<>();
// 维护GUID到modelId的映射
private Map<String, Long> guidMap;
@PostConstruct
public void init() {
modelList = modelDal.listModel(null);
guidMap = modelDal.listModel(null).stream()
.collect(Collectors.toMap(ModelVO::getGuid, ModelVO::getId));
cacheService.subscribeModel(this);
}
@Override
public List<ModelVO> listModel(String merchantCode, Integer status) {
List<ModelVO> modelList = modelDal.listModel(merchantCode, status);
return modelList;
return modelDal.listModel(merchantCode, status);
}
@Override
public List<ModelVO> listModel(Integer status) {
if (modelList == null) {
modelList = modelDal.listModel(status);
}
return modelList;
return modelDal.listModel(status);
}
@Override
public ModelVO getModelByGuid(String guid) {
for (ModelVO mod : modelList) {
if (mod.getGuid().equals(guid)) {
return mod;
}
long modelId = guidMap.get(guid);
ModelVO vo = (ModelVO) getByCache(modelId);
if (vo == null) {
vo = modelDal.getModelByGuid(guid);
guidMap.put(vo.getGuid(), vo.getId());
}
ModelVO model = modelDal.getModelByGuid(guid);
return model;
return vo;
}
@Override
public ModelVO getModelById(Long id) {
return modelDal.getModelById(id);
return (ModelVO) getByCache(id);
}
@Override
public void onMessage(String channel, String message) {
logger.info("model sub:{}", message);
modelList = modelDal.listModel(null);
}
@Override
public ModelVO get(Long id) {
return modelDal.getModelById(id);
logger.info("model update message:{}", message);
ModelVO vo = JsonUtils.fromJson(message, ModelVO.class);
// 删除本地缓存的规则模型
invalidateCache(vo.getId());
}
@Override
@ -149,16 +152,16 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
}
@Override
public CommonResult delete(Long[] id) {
public CommonResult delete(Long[] ids) {
CommonResult result = new CommonResult();
ModelVO model = modelDal.getModelById(id[0]);
ModelVO model = modelDal.getModelById(ids[0]);
if (model.getTemplate()) {
result.setCode("701");
result.setSuccess(false);
result.setMsg("系统模板禁止删除！");
return result;
}
int count = modelDal.delete(id);
int count = modelDal.delete(ids);
if (count > 0) {
result.setSuccess(true);
// 通知更新
@ -170,7 +173,7 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
@Override
public CommonResult build(Long id) throws IOException {
CommonResult result = new CommonResult();
ModelVO modelVO = modelDal.getModelById(id);
ModelVO modelVO = getModelById(id);
List<FieldVO> fields = modelDal.listField(id);
List<PreItemVO> items = modelDal.listPreItem(id, null);
String collectionName = "entity_" + id;
@ -184,7 +187,7 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
}
// 为需要索引的字段添加索引
for (FieldVO field : fields) {
if (field.getIndexed().booleanValue()) {
if (field.getIndexed()) {
Document indexKey = new Document();
indexKey.put(field.getFieldName(), 1);
IndexModel index = new IndexModel(indexKey);

33
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/PreItemServiceImpl.java
View File

@ -3,7 +3,6 @@ package com.pgmmers.radar.service.impl.model;
import com.alibaba.fastjson.JSON;
import com.pgmmers.radar.dal.bean.PageResult;
import com.pgmmers.radar.dal.bean.PreItemQuery;
import com.pgmmers.radar.dal.model.FieldDal;
import com.pgmmers.radar.dal.model.ModelDal;
import com.pgmmers.radar.dal.model.PreItemDal;
import com.pgmmers.radar.service.cache.CacheService;
@ -11,41 +10,36 @@ import com.pgmmers.radar.service.cache.SubscribeHandle;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.model.PreItemService;
import com.pgmmers.radar.vo.model.PreItemVO;
import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class PreItemServiceImpl implements PreItemService, SubscribeHandle {
public class PreItemServiceImpl extends BaseLocalCacheService implements PreItemService, SubscribeHandle {
public static Logger logger = LoggerFactory.getLogger(FieldServiceImpl.class);
public Map<Long, List<PreItemVO>> contextMap = new HashMap<Long, List<PreItemVO>>();
@Override
public Object query(Long modelId) {
return modelDal.listPreItem(modelId, null);
}
@Autowired
private ModelDal modelDal;
@Autowired
private PreItemDal preItemDal;
@Autowired
private FieldDal fieldDal;
@Autowired
private CacheService cacheService;
@Override
public List<PreItemVO> listPreItem(Long modelId) {
List<PreItemVO> list = contextMap.get(modelId);
if (list == null) {
list = modelDal.listPreItem(modelId, null);
contextMap.put(modelId, list);
}
return list;
//noinspection unchecked
return (List<PreItemVO>) getByCache(modelId);
}
@Override
@ -53,8 +47,7 @@ public class PreItemServiceImpl implements PreItemService, SubscribeHandle {
logger.info("prefield sub:{}", message);
PreItemVO preItem = JSON.parseObject(message, PreItemVO.class);
if (preItem != null) {
List<PreItemVO> list = modelDal.listPreItem(preItem.getModelId(), null);
contextMap.put(preItem.getModelId(), list);
invalidateCache(preItem.getModelId());
}
}
@ -92,7 +85,7 @@ public class PreItemServiceImpl implements PreItemService, SubscribeHandle {
preItem.setDestField("preItem_" + preItem.getId());
preItemDal.save(preItem);
}
result.getData().put("id", preItem.getId());
result.setSuccess(true);
// 通知更新
@ -113,7 +106,7 @@ public class PreItemServiceImpl implements PreItemService, SubscribeHandle {
}
return result;
}
@PostConstruct
public void init() {
cacheService.subscribePreItem(this);

29
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/RuleServiceImpl.java
View File

@ -27,11 +27,16 @@ import javax.annotation.PostConstruct;
import java.util.*;
@Service
public class RuleServiceImpl implements RuleService, SubscribeHandle {
public class RuleServiceImpl extends BaseLocalCacheService implements RuleService, SubscribeHandle {
public static Logger logger = LoggerFactory
.getLogger(RuleServiceImpl.class);
@Override
public Object query(Long activationId) {
return modelDal.listRules(null, activationId, null);
}
@Autowired
private ModelDal modelDal;
@Autowired
@ -53,12 +58,8 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle {
@Override
public List<RuleVO> listRule(Long activationId) {
List<RuleVO> list = contextMap.get(activationId);
if (list == null) {
list = modelDal.listRules(null, activationId, null);
contextMap.put(activationId, list);
}
return list;
//noinspection unchecked
return (List<RuleVO>) getByCache(activationId);
}
@Override
@ -66,9 +67,7 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle {
logger.info("rule sub:{}", message);
RuleVO rule = JSON.parseObject(message, RuleVO.class);
if (rule != null) {
List<RuleVO> list = modelDal.listRules(null,
rule.getActivationId(), null);
contextMap.put(rule.getActivationId(), list);
invalidateCache(rule.getActivationId());
}
}
@ -81,7 +80,7 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle {
public CommonResult query(RuleQuery query) {
CommonResult result = new CommonResult();
ActivationVO activation = activationDal.get(query.getActivationId());
result.setSuccess(true);
result.getData().put("ruleOrder", activation.getRuleOrder());
result.getData().put("page", ruleDal.query(query));
@ -106,7 +105,7 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle {
}
result.getData().put("id", rule.getId());
result.setSuccess(true);
// 存储History
RuleHistoryVO ruleHistoryVO=new RuleHistoryVO();
ruleHistoryVO.setRuleId(rule.getId());
@ -120,7 +119,7 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle {
ruleHistoryVO.setRuleDefinition(rule.getRuleDefinition().asText());
ruleHistoryVO.setUpdateTime(rule.getUpdateTime());
ruleDal.saveHistory(ruleHistoryVO);
// 通知更新
cacheService.publishRule(rule);
}
@ -132,7 +131,7 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle {
CommonResult result = new CommonResult();
RuleVO rule = ruleDal.get(id[0]);
int count = ruleDal.delete(id);
if (count > 0) {
if (count > 0) {
result.setSuccess(true);
// 通知更新
cacheService.publishRule(rule);
@ -194,7 +193,7 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle {
@Override
public CommonResult queryHistory(RuleHistoryQuery query) {
CommonResult result = new CommonResult();
result.setSuccess(true);
result.getData().put("page", ruleDal.queryHistory(query));
return result;

66
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/MongodbUtil.java
View File

@ -1,66 +0,0 @@
package com.pgmmers.radar.service.impl.util;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import java.util.Iterator;
import java.util.List;
import org.bson.BsonValue;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Component;
/**
* 重新封装类。
* @author feihu.wang
* @since 2020.04.15
*/
@Deprecated
//@Component
public class MongodbUtil implements InitializingBean {
public static MongoTemplate mongoTemplate;
@Override
public void afterPropertiesSet() {
mongoTemplate = (MongoTemplate) BeanUtils.getBean("mongoTemplate");
}
public static MongoCollection<Document> getCollection(String collectionName) {
return mongoTemplate.getCollection(collectionName);
}
public static void insert(String collectionName, Document doc) {
MongoCollection<Document> collection = getCollection(collectionName);
collection.insertOne(doc);
}
public static long count(String collectionName, Bson filter) {
MongoCollection<Document> collection = getCollection(collectionName);
return collection.countDocuments(filter);
}
public static long distinctCount(String collectionName, Bson filter, String fieldName) {
MongoCollection<Document> collection =getCollection(collectionName);
long count = 0;
Iterator<BsonValue> it = collection.distinct(fieldName, filter, BsonValue.class).iterator();
while (it.hasNext()) {
it.next();
count++;
}
return count;
}
public static AggregateIterable<Document> aggregate(String collectionName, List<Bson> pipeline) {
MongoCollection<Document> collection = getCollection(collectionName);
return collection.aggregate(pipeline);
}
public static FindIterable<Document> find(String collectionName, Bson filter) {
MongoCollection<Document> collection = getCollection(collectionName);
return collection.find(filter);
}
}

34
radar-service/src/main/java/com/pgmmers/radar/service/cache/CacheService.java vendored
View File

@ -1,31 +1,17 @@
package com.pgmmers.radar.service.cache;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.vo.model.*;
import java.io.Serializable;
import java.util.List;
import com.pgmmers.radar.vo.model.AbstractionVO;
import com.pgmmers.radar.vo.model.ActivationVO;
import com.pgmmers.radar.vo.model.DataListRecordVO;
import com.pgmmers.radar.vo.model.DataListsVO;
import com.pgmmers.radar.vo.model.FieldVO;
import com.pgmmers.radar.vo.model.ModelVO;
import com.pgmmers.radar.vo.model.PreItemVO;
import com.pgmmers.radar.vo.model.RuleVO;
public interface CacheService {
void saveModel(ModelVO model);
ModelVO getModel(Long modelId);
void saveField(FieldVO field);
List<FieldVO> listFields(Long modelId);
void saveAbstraction(AbstractionVO abstraction);
List<AbstractionVO> listAbstraction(Long modelId);
void saveActivation(ActivationVO activation);
List<ActivationVO> listActivation(Long modelId);
void saveAntiFraudResult(String modelId, String sessionId, CommonResult result);
CommonResult getAntiFraudResult(String modelId, String sessionId);
@ -53,10 +39,10 @@ public interface CacheService {
void publishActivation(ActivationVO activation);
void subscribeActivation(SubscribeHandle handler);
void publishDataListRecord(DataListRecordVO record);
void subscribeDataListRecord(SubscribeHandle handler);
void publishDataList(DataListsVO dataList);
void subscribeDataList(SubscribeHandle handler);

2
radar-service/src/main/java/com/pgmmers/radar/service/model/ModelService.java
View File

@ -21,8 +21,6 @@ public interface ModelService {
ModelVO getModelById(Long id);
ModelVO get(Long id);
CommonResult query(ModelQuery query);
CommonResult save(ModelVO model);