fengchunyu
/
dataCompareSy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

实有人口数据清洗

Browse Source
develop
fengchunyu 10 months ago
parent b3301ac65a
commit 3a4ecb065e
8 changed files with 103 additions and 131 deletions
Show Stats Download Patch File Download Diff File

7
pom.xml
Unescape View File

@ -90,6 +90,13 @@
<version>3.6</version>
</dependency>
<!--caffeine本地缓存-->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.9.0</version>
</dependency>
</dependencies>
<build>

26
src/main/java/com/ssf/mysqloracletest/domain/MetaActualUserInfo.java
Unescape View File

@ -121,31 +121,5 @@ public class MetaActualUserInfo implements Serializable {
*/
private String metaUpdateTime;
/**
* 创建时间
*/
private Date createTime;
/**
* 创建者
*/
private Integer createBy;
/**
* 更新时间
*/
private Date updateTime;
/**
* 更新者
*/
private Integer updateBy;
/**
* 0:不活跃,1:活跃
*/
private Integer isActive;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

35
src/main/java/com/ssf/mysqloracletest/domain/MetaImpUserInfo.java
Unescape View File

@ -1,18 +1,18 @@
package com.ssf.mysqloracletest.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
import java.util.Date;
import lombok.Data;
import java.io.Serializable;
/**
* 重点人员信息表
*
* @TableName meta_imp_user_info
*/
@TableName(value ="meta_imp_user_info")
@TableName(value = "meta_imp_user_info")
@Data
public class MetaImpUserInfo implements Serializable {
/**
@ -86,31 +86,4 @@ public class MetaImpUserInfo implements Serializable {
*/
private String registerUnitName;
/**
* 创建时间
*/
private Date createTime;
/**
* 创建者
*/
private Integer createBy;
/**
* 更新时间
*/
private Date updateTime;
/**
* 更新者
*/
private Integer updateBy;
/**
* 0:不活跃,1:活跃
*/
private Integer isActive;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

6
src/main/java/com/ssf/mysqloracletest/domain/VSySydwmMysql.java
Unescape View File

@ -151,5 +151,11 @@ public class VSySydwmMysql {
@TableField(value = "YSBZK_GXSJ")
private String YSBZK_GXSJ;
/**
*
*/
@TableField(value = "STATUS")
private String STATUS;
}

2
src/main/java/com/ssf/mysqloracletest/domain/ZhxfZdrygkRyjbxxViewMysql.java
Unescape View File

@ -120,5 +120,7 @@ public class ZhxfZdrygkRyjbxxViewMysql implements Serializable {
private String YWSJHM;
@TableField(value = "CXZT")
private String CXZT;
@TableField(value = "status")
private String status;
}

140
src/main/java/com/ssf/mysqloracletest/task/MetaImpUserInfoTask.java
Unescape View File

@ -1,40 +1,44 @@
package com.ssf.mysqloracletest.task;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.ssf.mysqloracletest.domain.MetaImpUserInfo;
import com.ssf.mysqloracletest.domain.ZhxfZdrygkRyjbxxViewMysql;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.ssf.mysqloracletest.domain.*;
import com.ssf.mysqloracletest.service.MetaImpUserInfoService;
import com.ssf.mysqloracletest.service.VillageActualUserInfoService;
import com.ssf.mysqloracletest.service.ZhxfZdrygkRyjbxxViewMysqlService;
import com.ssf.mysqloracletest.utils.ConfigParam;
import com.ssf.mysqloracletest.utils.NumKit;
import com.ssf.mysqloracletest.utils.StringKit;
import lombok.RequiredArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* 用户数据接入
*/
@Component
@RequiredArgsConstructor
public class MetaImpUserInfoTask {
private static final Logger logger = LoggerFactory.getLogger(MetaImpUserInfoTask.class);
@Resource
private ZhxfZdrygkRyjbxxViewMysqlService zdryMysqlService;
private final ZhxfZdrygkRyjbxxViewMysqlService zdryMysqlService;
private final MetaImpUserInfoService metaImpUserInfoService;
private final VillageActualUserInfoService actualUserInfoService;
@Resource
private MetaImpUserInfoService metaImpUserInfoService;
final int oncesCounts = NumKit.checkInt(ConfigParam.oncesCounts) == 0 ? 1000 : NumKit.checkInt(ConfigParam.oncesCounts);
private boolean zdryTag = false;
/**
* 连云港 对接海康平台获取人员基础数据
@ -46,70 +50,76 @@ public class MetaImpUserInfoTask {
if (!"true".equals(ConfigParam.metaImpUserSwitch)) {
return;
}
try {
int countzdry = zdryMysqlService.count();
logger.info("countzdry:" + countzdry);
if (countzdry > 0) {
int count = countzdry / oncesCounts;
for (int i = 0; i <= count; i++) {
Map zdryMap = new HashMap();
zdryMap.put("rowStart", i * oncesCounts);
zdryMap.put("rowEnd", (i + 1) * oncesCounts);
try {
List<ZhxfZdrygkRyjbxxViewMysql> list = zdryMysqlService.lambdaQuery().list();
for (ZhxfZdrygkRyjbxxViewMysql zdry : list) {
MetaImpUserInfo metaImpUserInfo = new MetaImpUserInfo();
metaImpUserInfo.setUserName(zdry.getXM());
metaImpUserInfo.setIdCard(zdry.getZJHM());
if (zdry.getXB() != null) {
metaImpUserInfo.setSex(Integer.parseInt(zdry.getXB()));
}
metaImpUserInfo.setPhone(zdry.getSJHM());
String s = "000000";
if (zdry.getZDRYXL() != null && !zdry.getZDRYXL().contains(",")) {
String substring = zdry.getZDRYXL().substring(0, 4);
substring = substring + s;
metaImpUserInfo.setImpUserType(substring);
} else if (zdry.getZDRYXL().contains(",")) {
String[] split = zdry.getZDRYXL().split(",");
String substring = split[0].substring(0, 4);
String substring1 = split[1].substring(0, 4);
substring = substring + s + "," + substring1 + s;
metaImpUserInfo.setImpUserType(substring);
}
metaImpUserInfo.setImpUserSubType(zdry.getZDRYXL());
metaImpUserInfo.setHouseHoldRegister(zdry.getHJDQH());
metaImpUserInfo.setHouseHoldAddress(zdry.getHJDXZ());
metaImpUserInfo.setHabitationRegister(zdry.getXZDHQ());
metaImpUserInfo.setHabitationAddress(zdry.getXZDXZ());
metaImpUserInfo.setRegisterUnitCode(zdry.getDJRDWDM());
metaImpUserInfo.setRegisterUnitName(zdry.getDJRDWMC());
//确定一个数据是否唯一
MetaImpUserInfo one = metaImpUserInfoService.lambdaQuery().eq(MetaImpUserInfo::getIdCard, zdry.getZJHM()).one();
if (one == null) {
metaImpUserInfoService.save(metaImpUserInfo);
logger.info("zdry-" + zdry.getZJHM() + ":插入成功");
} else {
UpdateWrapper<MetaImpUserInfo> metaImpUserInfoUpdateWrapper = new UpdateWrapper<>();
metaImpUserInfoUpdateWrapper.eq("id_card", zdry.getZJHM());
metaImpUserInfoService.update(metaImpUserInfo, metaImpUserInfoUpdateWrapper);
logger.info("zdry-" + zdry.getZJHM() + ":更新成功");
QueryWrapper<ZhxfZdrygkRyjbxxViewMysql> queryWrapper = new QueryWrapper<>();
queryWrapper.isNull("status");
Page<ZhxfZdrygkRyjbxxViewMysql> page = new Page<>(1, 1000);
List<ZhxfZdrygkRyjbxxViewMysql> resultList = zdryMysqlService.page(page, queryWrapper).getRecords();
CountDownLatch latch = new CountDownLatch(resultList.size());
ExecutorService impUserPool = Executors.newFixedThreadPool(5);
for (ZhxfZdrygkRyjbxxViewMysql bean : resultList) {
impUserPool.submit(() -> {
cleanImpUser(bean);
latch.countDown();
});
}
try {
// Wait for all threads to complete
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
// 关闭线程池
impUserPool.shutdown();
} catch (Exception e) {
logger.info("zdrys:" + e.getMessage());
logger.info(StringKit.getTrace(e));
}
logger.info("zdryMap数据总量:" + countzdry + "," + "当前处理下标:" + zdryMap.get("rowStart") + "/" + zdryMap.get("rowEnd"));
}
private void cleanImpUser(ZhxfZdrygkRyjbxxViewMysql bean) {
ZhxfZdrygkRyjbxxViewMysql beanCp = new ZhxfZdrygkRyjbxxViewMysql();
try {
VillageActualUserInfo userInfoPara = new VillageActualUserInfo();
userInfoPara.setIdcard(StringKit.toString(beanCp.getZJHM()));
List<VillageActualUserInfo> actualUserInfos = actualUserInfoService.selectVillageActualUserInfoListLimit1(userInfoPara);
String userPic = "";
if (actualUserInfos != null && actualUserInfos.size() > 0) {
userPic = StringKit.toString(actualUserInfos.get(0).getUserPic());
}
BeanUtils.copyProperties(bean, beanCp);
MetaImpUserInfo userInfo = new MetaImpUserInfo();
userInfo.setUserPic(userPic);
userInfo.setUserName(StringKit.toString(beanCp.getXM()));
userInfo.setIdCard(StringKit.toString(beanCp.getZJHM()));
userInfo.setSex( "".equals(StringKit.toString(beanCp.getXB()))? 0: NumKit.checkInt(beanCp.getXB()));
userInfo.setPhone(StringKit.toString(beanCp.getSJHM()));
String zdrlx = StringKit.toString(beanCp.getZDRYXL());
if("".equals(zdrlx)){
userInfo.setImpUserType(zdrlx.substring(0,4) + "000000");
userInfo.setImpUserSubType(beanCp.getZDRYXL());
}
userInfo.setHouseHoldRegister(beanCp.getHJDQH());
userInfo.setHouseHoldAddress(beanCp.getHJDXZ());
userInfo.setHabitationRegister(beanCp.getXZDHQ());
userInfo.setHabitationAddress(beanCp.getXZDXZ());
userInfo.setRegisterUnitCode(beanCp.getDJRDWDM());
userInfo.setRegisterUnitName(beanCp.getDJRDWMC());
boolean b = metaImpUserInfoService.save(userInfo);
if (b) {
beanCp.setStatus("1");
zdryMysqlService.updateZdryjbxxById(beanCp);
}
} catch (Exception e) {
logger.info(StringKit.getTrace(e));
beanCp.setStatus("2");
zdryMysqlService.updateZdryjbxxById(beanCp);
}
}

8
src/main/resources/mapper/MetaActualUserInfoMapper.xml
Unescape View File

@ -32,7 +32,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
select id, user_pic, user_name, id_card, sex, phone, landline, house_hold_register, house_hold_address, habitation_address, police_station_code, police_station_name, police_district_code, police_district_name, management_status, home_id, home_code, account_number, meta_create_time, meta_update_time, actual_user_type from meta_actual_user_info
</sql>
<select id="selectMetaActualUserInfoList" parameterType="MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<select id="selectMetaActualUserInfoList" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<include refid="selectMetaActualUserInfoVo"/>
<where>
<if test="userPic != null and userPic != ''"> and user_pic = #{userPic}</if>
@ -63,7 +63,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
where id = #{id}
</select>
<insert id="insertMetaActualUserInfo" parameterType="MetaActualUserInfo" useGeneratedKeys="true" keyProperty="id">
<insert id="insertMetaActualUserInfo" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo" useGeneratedKeys="true" keyProperty="id">
insert into meta_actual_user_info
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="userPic != null">user_pic,</if>
@ -111,7 +111,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
</trim>
</insert>
<update id="updateMetaActualUserInfo" parameterType="MetaActualUserInfo">
<update id="updateMetaActualUserInfo" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo">
update meta_actual_user_info
<trim prefix="SET" suffixOverrides=",">
<if test="userPic != null">user_pic = #{userPic},</if>
@ -151,7 +151,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<select id="selectMetaActualUserInfoListByHomeId" parameterType="MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<select id="selectMetaActualUserInfoListByHomeId" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<include refid="selectMetaActualUserInfoVo"/>
where home_id is not null and account_number is null limit 200;
</select>

2
src/main/resources/mapper/ZhxfZdrygkRyjbxxViewMysqlMapper.xml
Unescape View File

@ -239,7 +239,7 @@
<if test="SJHM != null and SJHM != '' ">SJHM =#{SJHM },</if>
<if test="YWSJHM != null and YWSJHM != '' ">YWSJHM =#{YWSJHM },</if>
<if test="CXZT != null and CXZT != '' ">CXZT =#{CXZT },</if>
<if test="status != null and status != '' ">status =#{status },</if>
</trim>
where RYJBXXID = #{RYJBXXID}
</update>

Write Preview
Loading…
Cancel
Save