fengchunyu
/
dataCompareSy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

处警与从业人员的数据清洗逻辑修改

Browse Source
develop
hanrenchun 1 year ago
parent 42eec6b3b0
commit 067ddcefe7
7 changed files with 184 additions and 156 deletions
Show Stats Download Patch File Download Diff File

6
config/application.properties
Unescape View File

@ -17,9 +17,9 @@ JjxxSwitch=false
NewNsydwSwitch=false
metaImpUserSwitch=false
NewNalarminfoSwitch=false
metaHandleAlarmSwitch=false
metaActualUnitUserInfoSwitch=true
metaActualUserInfoSwitch=true
metaHandleAlarmSwitch=true
metaActualUnitUserInfoSwitch=false
metaActualUserInfoSwitch=false
actualTimecjxxSwitch=false
actualTimejjxxSwitch=false
RentalHouseSwitch=false

2
src/main/java/com/ssf/mysqloracletest/domain/JcjCjxxViewMysql.java
Unescape View File

@ -223,4 +223,6 @@ public class JcjCjxxViewMysql implements Serializable {
private String JQSXMC;
@TableField(value = "RYWKSJ")
private String RYWKSJ;
private String status;
}

2
src/main/java/com/ssf/mysqloracletest/domain/ZhxfZdrygkRyjbxxViewMysql.java
Unescape View File

@ -123,6 +123,6 @@ public class ZhxfZdrygkRyjbxxViewMysql implements Serializable {
@TableField(value = "status")
private String status;
private String status;
}

91
src/main/java/com/ssf/mysqloracletest/task/MetaActualUnitUserInfoTask.java
Unescape View File

@ -1,15 +1,21 @@
package com.ssf.mysqloracletest.task;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.ssf.mysqloracletest.domain.JcjCjxxViewMysql;
import com.ssf.mysqloracletest.domain.MetaActualUnitUserInfo;
import com.ssf.mysqloracletest.domain.MetaHandleAlarmInfo;
import com.ssf.mysqloracletest.domain.VSySydwCyryMysql;
import com.ssf.mysqloracletest.service.MetaActualUnitUserInfoService;
import com.ssf.mysqloracletest.service.VSySydwCyryMysqlService;
import com.ssf.mysqloracletest.utils.ConfigParam;
import com.ssf.mysqloracletest.utils.NumKit;
import com.ssf.mysqloracletest.utils.StringKit;
import lombok.RequiredArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -17,58 +23,61 @@ import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* 用户数据接入
*/
@Component
@RequiredArgsConstructor
public class MetaActualUnitUserInfoTask {
private static final Logger logger = LoggerFactory.getLogger(MetaActualUnitUserInfoTask.class);
@Resource
private VSySydwCyryMysqlService vSySydwCyryMysqlService;
private final VSySydwCyryMysqlService vSySydwCyryMysqlService;
@Resource
private MetaActualUnitUserInfoService metaActualUnitUserInfoService;
private final MetaActualUnitUserInfoService metaActualUnitUserInfoService;
final int oncesCounts = NumKit.checkInt(ConfigParam.oncesCounts) == 0 ? 1000 : NumKit.checkInt(ConfigParam.oncesCounts);
private boolean cyryTag = false;
/**
* 连云港 对接海康平台获取人员基础数据
* //
*/
@Scheduled(initialDelay = 100, fixedDelay = 3000)
public void transcyry() {
if (cyryTag) {
return;
}
if (!"true".equals(ConfigParam.metaActualUnitUserInfoSwitch)) {
return;
}
try {
int countcyry = vSySydwCyryMysqlService.count();
logger.info("countcyry:" + countcyry);
QueryWrapper<VSySydwCyryMysql> queryWrapper = new QueryWrapper<>();
queryWrapper.isNull("status");
Page<VSySydwCyryMysql> page = new Page<>(1, 1000);
List<VSySydwCyryMysql> resultList = vSySydwCyryMysqlService.page(page, queryWrapper).getRecords();
CountDownLatch latch = new CountDownLatch(resultList.size());
ExecutorService cyryPool = Executors.newFixedThreadPool(5);
for (VSySydwCyryMysql bean : resultList) {
cyryPool.submit(() -> {
cleanCyry(bean);
latch.countDown();
});
}
if (countcyry > 0) {
cyryTag = true;
} catch (
Exception e) {
logger.info(StringKit.getTrace(e));
}
}
int count = countcyry / oncesCounts;
for (int i = 0; i <= count; i++) {
Map cyryMap = new HashMap();
cyryMap.put("rowStart", i * oncesCounts);
cyryMap.put("rowEnd", (i + 1) * oncesCounts);
private void cleanCyry(VSySydwCyryMysql bean) {
VSySydwCyryMysql cyry = new VSySydwCyryMysql();
try {
List<VSySydwCyryMysql> list = vSySydwCyryMysqlService.lambdaQuery().list();
for (VSySydwCyryMysql cyry : list) {
BeanUtils.copyProperties(bean, cyry);
MetaActualUnitUserInfo metaActualUnitUserInfo = new MetaActualUnitUserInfo();
VSySydwCyryMysql vSySydwCyryMysql = vSySydwCyryMysqlService.selectCyryById(cyry.getID());
if (cyry.getDWID() != null) {
metaActualUnitUserInfo.setUnitId(Long.parseLong(cyry.getDWID()));
}
@ -87,32 +96,20 @@ public class MetaActualUnitUserInfoTask {
} else if ("N".equals(cyry.getZT())) {
metaActualUnitUserInfo.setStaffState(2);
}
metaActualUnitUserInfo.setMetaCreateTime(cyry.getYSBZK_RKSJ());
metaActualUnitUserInfo.setMetaUpdateTime(cyry.getYSBZK_GXSJ());
metaActualUnitUserInfo.setMetaCreateTime(vSySydwCyryMysql.getYSBZK_RKSJ());
metaActualUnitUserInfo.setMetaUpdateTime(vSySydwCyryMysql.getYSBZK_GXSJ());
//确定一个数据是否唯一
MetaActualUnitUserInfo one = metaActualUnitUserInfoService.lambdaQuery().eq(MetaActualUnitUserInfo::getIdentityId, cyry.getZJHM())
.eq(MetaActualUnitUserInfo::getStaffType, cyry.getRYFL()).one();
if (one == null) {
metaActualUnitUserInfoService.save(metaActualUnitUserInfo);
logger.info("cyry-" + cyry.getZJHM() + ":插入成功");
} else {
UpdateWrapper<MetaActualUnitUserInfo> metaActualUnitUserInfoUpdateWrapper = new UpdateWrapper<>();
metaActualUnitUserInfoUpdateWrapper.eq("identity_id", cyry.getZJHM());
metaActualUnitUserInfoUpdateWrapper.eq("staff_type", cyry.getRYFL());
metaActualUnitUserInfoService.update(metaActualUnitUserInfo, metaActualUnitUserInfoUpdateWrapper);
logger.info("cyry-" + cyry.getZJHM() + ":更新成功");
}
}
} catch (Exception e) {
logger.info("cyrys:" + e.getMessage());
boolean save = metaActualUnitUserInfoService.save(metaActualUnitUserInfo);
if (save){
cyry.setStatus("1");
vSySydwCyryMysqlService.updateCyryById(cyry);
}
logger.info("cyryMap数据总量:" + countcyry + "," + "当前处理下标:" + cyryMap.get("rowStart") + "/" + cyryMap.get("rowEnd"));
}
}
} catch (Exception e) {
logger.info(StringKit.getTrace(e));
cyry.setStatus("2");
vSySydwCyryMysqlService.updateCyryById(cyry);
}
}

137
src/main/java/com/ssf/mysqloracletest/task/MetaHandleAlarmTask.java
Unescape View File

@ -1,16 +1,21 @@
package com.ssf.mysqloracletest.task;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.ssf.mysqloracletest.domain.JcjCjxxViewMysql;
import com.ssf.mysqloracletest.domain.MetaHandleAlarmInfo;
import com.ssf.mysqloracletest.domain.VJwtSyfw;
import com.ssf.mysqloracletest.service.JcjCjxxViewMysqlService;
import com.ssf.mysqloracletest.service.MetaHandleAlarmInfoService;
import com.ssf.mysqloracletest.service.MetaImpUserInfoService;
import com.ssf.mysqloracletest.utils.ConfigParam;
import com.ssf.mysqloracletest.utils.NumKit;
import com.ssf.mysqloracletest.utils.StringKit;
import lombok.RequiredArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -21,35 +26,24 @@ import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 用户数据接入
*/
@Component
@RequiredArgsConstructor
public class MetaHandleAlarmTask {
private static final Logger logger = LoggerFactory.getLogger(MetaHandleAlarmTask.class);
@Resource
private MetaHandleAlarmInfoService metaHandleAlarmInfoService;
private final MetaHandleAlarmInfoService metaHandleAlarmInfoService;
private final JcjCjxxViewMysqlService jcjCjxxViewMysqlService;
@Resource
private JcjCjxxViewMysqlService jcjCjxxViewMysqlService;
final int oncesCounts = NumKit.checkInt(ConfigParam.oncesCounts) == 0 ? 1000 : NumKit.checkInt(ConfigParam.oncesCounts);
private boolean cjxxTag = false;
/**
* 连云港 对接海康平台获取人员基础数据
* //
*/
@Scheduled(initialDelay = 100, fixedDelay = 3000)
public void transcjxx() {
if (cjxxTag) {
return;
}
if (!"true".equals(ConfigParam.metaHandleAlarmSwitch)) {
return;
@ -57,68 +51,101 @@ public class MetaHandleAlarmTask {
try {
int countcjxx = jcjCjxxViewMysqlService.count();
logger.info("countcjxx:" + countcjxx);
QueryWrapper<JcjCjxxViewMysql> queryWrapper = new QueryWrapper<>();
queryWrapper.isNull("status");
Page<JcjCjxxViewMysql> page = new Page<>(1, 1000);
List<JcjCjxxViewMysql> resultList = jcjCjxxViewMysqlService.page(page, queryWrapper).getRecords();
CountDownLatch latch = new CountDownLatch(resultList.size());
ExecutorService cjxxPool = Executors.newFixedThreadPool(5);
for (JcjCjxxViewMysql bean : resultList) {
cjxxPool.submit(() -> {
cleanCjxx(bean);
latch.countDown();
});
}
if (countcjxx > 0) {
cjxxTag = true;
try {
// Wait for all threads to complete
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
// 关闭线程池
cjxxPool.shutdown();
int count = countcjxx / oncesCounts;
CountDownLatch latch = new CountDownLatch(count);
ExecutorService cjxxPool = Executors.newFixedThreadPool(20);
} catch (Exception e) {
logger.info(StringKit.getTrace(e));
}
}
for (int i = 0; i <= count; i++) {
Map cjxxMap = new HashMap();
cjxxMap.put("rowStart", i * oncesCounts);
cjxxMap.put("rowEnd", (i + 1) * oncesCounts);
cjxxPool.submit(() -> {
private void cleanCjxx(JcjCjxxViewMysql bean) {
JcjCjxxViewMysql cjxx = new JcjCjxxViewMysql();
try {
List<JcjCjxxViewMysql> list = jcjCjxxViewMysqlService.lambdaQuery().list();
for (JcjCjxxViewMysql cjxx : list) {
BeanUtils.copyProperties(bean, cjxx);
MetaHandleAlarmInfo metaHandleAlarmInfo = new MetaHandleAlarmInfo();
metaHandleAlarmInfo.setAlarmCode(cjxx.getJJBH());
metaHandleAlarmInfo.setHandleAlarmUnit(cjxx.getCJDWMC());
metaHandleAlarmInfo.setHandleAlarmTime(cjxx.getCJSJ());
metaHandleAlarmInfo.setHandleAlarmContent(cjxx.getCLJGNR());
metaHandleAlarmInfo.setHandleAlarmAddress(cjxx.getCJXXDD());
if (!containsChinese(cjxx.getCJR())) {
metaHandleAlarmInfo.setHandleAlarmUser(cjxx.getCJRHZXS());
metaHandleAlarmInfo.setHandleAlarmUserCode(cjxx.getCJR());
metaHandleAlarmInfo.setHandleAlarmResult(cjxx.getCJJGMC());
//确定一个数据是否唯一
MetaHandleAlarmInfo one = metaHandleAlarmInfoService.lambdaQuery().eq(MetaHandleAlarmInfo::getAlarmCode, cjxx.getJJBH()).one();
if (one == null) {
metaHandleAlarmInfoService.save(metaHandleAlarmInfo);
logger.info("cjxx-" + cjxx.getJJBH() + ":插入成功");
} else {
UpdateWrapper<MetaHandleAlarmInfo> metaHandleAlarmInfoUpdateWrapper = new UpdateWrapper<>();
metaHandleAlarmInfoUpdateWrapper.eq("alarm_code", cjxx.getJJBH());
metaHandleAlarmInfoService.update(metaHandleAlarmInfo,metaHandleAlarmInfoUpdateWrapper);
logger.info("cjxx-" + cjxx.getJJBH() + ":更新成功");
}else {
String s = extractChinese(cjxx.getCJR());
String s1 = extractNonChinese(cjxx.getCJR());
String substring = s1.substring(0, s1.length() - 1);
if (cjxx.getCJRHZXS()!=null) {
metaHandleAlarmInfo.setHandleAlarmUser(cjxx.getCJRHZXS() +","+ s);
}else {
metaHandleAlarmInfo.setHandleAlarmUser(s);
}
metaHandleAlarmInfo.setHandleAlarmUserCode(substring);
}
} catch (Exception e) {
logger.info("cjxxs:" + e.getMessage());
metaHandleAlarmInfo.setHandleAlarmResult(cjxx.getCJJGMC());
boolean save = metaHandleAlarmInfoService.save(metaHandleAlarmInfo);
if (save) {
cjxx.setStatus("1");
jcjCjxxViewMysqlService.updateCjxxById(cjxx);
}
latch.countDown();
});
logger.info("cjxxMap数据总量:" + countcjxx + "," + "当前处理下标:" + cjxxMap.get("rowStart") + "/" + cjxxMap.get("rowEnd"));
} catch (Exception e) {
logger.info(StringKit.getTrace(e));
cjxx.setStatus("2");
jcjCjxxViewMysqlService.updateCjxxById(cjxx);
}
try {
// Wait for all threads to complete
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
// 关闭线程池
cjxxPool.shutdown();
private static boolean containsChinese(String str) {
Pattern pattern = Pattern.compile("[\u4e00-\u9fa5]");
Matcher matcher = pattern.matcher(str);
return matcher.find();
}
} catch (Exception e) {
logger.info(StringKit.getTrace(e));
// 提取字符串中的中文
private static String extractChinese(String str) {
Pattern pattern = Pattern.compile("[\u4e00-\u9fa5]");
Matcher matcher = pattern.matcher(str);
StringBuilder chineseCharacters = new StringBuilder();
while (matcher.find()) {
chineseCharacters.append(matcher.group());
}
return chineseCharacters.toString();
}
private static String extractNonChinese(String str) {
Pattern pattern = Pattern.compile("[^\u4e00-\u9fa5]");
Matcher matcher = pattern.matcher(str);
StringBuilder nonChineseCharacters = new StringBuilder();
while (matcher.find()) {
nonChineseCharacters.append(matcher.group());
}
return nonChineseCharacters.toString();
}
}

4
src/main/resources/mapper/JcjCjxxViewMapperMysql.xml
Unescape View File

@ -109,6 +109,7 @@
<result property="WS_YXX" column="WS_YXX" jdbcType="VARCHAR"/>
<result property="JQSXMC" column="JQSXMC" jdbcType="VARCHAR"/>
<result property="RYWKSJ" column="RYWKSJ" jdbcType="VARCHAR"/>
<result property="status" column="status" jdbcType="VARCHAR"/>
</resultMap>
@ -441,8 +442,9 @@
<if test="WS_YXX != null and WS_YXX != '' ">WS_YXX=#{WS_YXX},</if>
<if test="JQSXMC != null and JQSXMC != '' ">JQSXMC=#{JQSXMC},</if>
<if test="RYWKSJ != null and RYWKSJ != '' ">RYWKSJ=#{RYWKSJ},</if>
<if test="status != null and status != '' ">status=#{status},</if>
</trim>
where CJBH = #{CJBH}
where JJBH = #{JJBH}
</update>
<select id="selectRecordOrderByRksj" parameterType="java.lang.String" resultMap="BaseResultMap">

4
src/main/resources/mapper/MetaActualUserInfoMapper.xml
Unescape View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ssf.mysqloracletest.mapper.MetaActualUserInfoMapper">
<resultMap type="com.ssf.mysqloracletest.domain.MetaActualUserInfo" id="MetaActualUserInfoResult">

Write Preview
Loading…
Cancel
Save