fengchunyu
/
dataCompareSy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: fengchunyu:5ac18c690b1772e2e7969b14ded1a06e9c0697d5
Branches Tags
fengchunyu:develop
fengchunyu:master
...
compare: fengchunyu:42eec6b3b05f927ea70d5da229aa6860e4c70951
Branches Tags
fengchunyu:develop
fengchunyu:master

2 Commits
5ac18c690b ... 42eec6b3b0

Author SHA1 Message Date
fengchunyu 42eec6b3b0 Merge remote-tracking branch 'origin/develop' into develop
11 months ago
fengchunyu 3a4ecb065e 实有人口数据清洗
11 months ago
8 changed files with 92 additions and 110 deletions
Show Stats

7
pom.xml
Unescape View File

@ -90,6 +90,13 @@
<version>3.6</version>
</dependency>
<!--caffeine本地缓存-->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.9.0</version>
</dependency>
</dependencies>
<build>

26
src/main/java/com/ssf/mysqloracletest/domain/MetaActualUserInfo.java
Unescape View File

@ -121,31 +121,5 @@ public class MetaActualUserInfo implements Serializable {
*/
private String metaUpdateTime;
/**
* 创建时间
*/
private Date createTime;
/**
* 创建者
*/
private Integer createBy;
/**
* 更新时间
*/
private Date updateTime;
/**
* 更新者
*/
private Integer updateBy;
/**
* 0:不活跃,1:活跃
*/
private Integer isActive;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

35
src/main/java/com/ssf/mysqloracletest/domain/MetaImpUserInfo.java
Unescape View File

@ -1,18 +1,18 @@
package com.ssf.mysqloracletest.domain;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
import java.util.Date;
import lombok.Data;
import java.io.Serializable;
/**
* 重点人员信息表
*
* @TableName meta_imp_user_info
*/
@TableName(value ="meta_imp_user_info")
@TableName(value = "meta_imp_user_info")
@Data
public class MetaImpUserInfo implements Serializable {
/**
@ -86,31 +86,4 @@ public class MetaImpUserInfo implements Serializable {
*/
private String registerUnitName;
/**
* 创建时间
*/
private Date createTime;
/**
* 创建者
*/
private Integer createBy;
/**
* 更新时间
*/
private Date updateTime;
/**
* 更新者
*/
private Integer updateBy;
/**
* 0:不活跃,1:活跃
*/
private Integer isActive;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

5
src/main/java/com/ssf/mysqloracletest/domain/VSySydwmMysql.java
Unescape View File

@ -151,6 +151,11 @@ public class VSySydwmMysql {
@TableField(value = "YSBZK_GXSJ")
private String YSBZK_GXSJ;
/**
*
*/
@TableField(value = "STATUS")
private String STATUS;
}

2
src/main/java/com/ssf/mysqloracletest/domain/ZhxfZdrygkRyjbxxViewMysql.java
Unescape View File

@ -120,6 +120,8 @@ public class ZhxfZdrygkRyjbxxViewMysql implements Serializable {
private String YWSJHM;
@TableField(value = "CXZT")
private String CXZT;
@TableField(value = "status")
private String status;
private String status;

115
src/main/java/com/ssf/mysqloracletest/task/MetaImpUserInfoTask.java
Unescape View File

@ -1,11 +1,11 @@
package com.ssf.mysqloracletest.task;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.ssf.mysqloracletest.domain.MetaImpUserInfo;
import com.ssf.mysqloracletest.domain.ZhxfZdrygkRyjbxxViewMysql;
import com.ssf.mysqloracletest.domain.*;
import com.ssf.mysqloracletest.service.MetaImpUserInfoService;
import com.ssf.mysqloracletest.service.VillageActualUserInfoService;
import com.ssf.mysqloracletest.service.ZhxfZdrygkRyjbxxViewMysqlService;
import com.ssf.mysqloracletest.utils.ConfigParam;
import com.ssf.mysqloracletest.utils.NumKit;
@ -13,13 +13,12 @@ import com.ssf.mysqloracletest.utils.StringKit;
import lombok.RequiredArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -32,8 +31,13 @@ import java.util.concurrent.Executors;
public class MetaImpUserInfoTask {
private static final Logger logger = LoggerFactory.getLogger(MetaImpUserInfoTask.class);
private final ZhxfZdrygkRyjbxxViewMysqlService zdryMysqlService;
private final MetaImpUserInfoService metaImpUserInfoService;
private final VillageActualUserInfoService actualUserInfoService;
/**
@ -46,47 +50,20 @@ public class MetaImpUserInfoTask {
if (!"true".equals(ConfigParam.metaImpUserSwitch)) {
return;
}
try {
QueryWrapper<ZhxfZdrygkRyjbxxViewMysql> wrapper = new QueryWrapper<>();
wrapper.isNull("status");
QueryWrapper<ZhxfZdrygkRyjbxxViewMysql> queryWrapper = new QueryWrapper<>();
queryWrapper.isNull("status");
Page<ZhxfZdrygkRyjbxxViewMysql> page = new Page<>(1, 1000);
List<ZhxfZdrygkRyjbxxViewMysql> list = zdryMysqlService.page(page,wrapper).getRecords();
CountDownLatch latch = new CountDownLatch(list.size());
ExecutorService zdryPool = Executors.newFixedThreadPool(5);
for (ZhxfZdrygkRyjbxxViewMysql zdry : list) {
MetaImpUserInfo metaImpUserInfo = new MetaImpUserInfo();
metaImpUserInfo.setUserName(zdry.getXM());
metaImpUserInfo.setIdCard(zdry.getZJHM());
if (zdry.getXB() != null) {
metaImpUserInfo.setSex(Integer.parseInt(zdry.getXB()));
}
metaImpUserInfo.setPhone(zdry.getSJHM());
String s = "000000";
if (zdry.getZDRYXL() != null && !zdry.getZDRYXL().contains(",")) {
String substring = zdry.getZDRYXL().substring(0, 4);
substring = substring + s;
metaImpUserInfo.setImpUserType(substring);
} else if (zdry.getZDRYXL().contains(",")) {
String[] split = zdry.getZDRYXL().split(",");
String substring = split[0].substring(0, 4);
String substring1 = split[1].substring(0, 4);
substring = substring + s + "," + substring1 + s;
metaImpUserInfo.setImpUserType(substring);
}
metaImpUserInfo.setImpUserSubType(zdry.getZDRYXL());
metaImpUserInfo.setHouseHoldRegister(zdry.getHJDQH());
metaImpUserInfo.setHouseHoldAddress(zdry.getHJDXZ());
metaImpUserInfo.setHouseHoldAddress(zdry.getHJDXZ());
metaImpUserInfo.setHabitationRegister(zdry.getXZDHQ());
metaImpUserInfo.setHabitationAddress(zdry.getXZDXZ());
metaImpUserInfo.setRegisterUnitCode(zdry.getDJRDWDM());
metaImpUserInfo.setRegisterUnitName(zdry.getDJRDWMC());
boolean save = metaImpUserInfoService.save(metaImpUserInfo);
if (save){
zdry.setStatus("1");
zdryMysqlService.updateZdryjbxxById(zdry);
}
List<ZhxfZdrygkRyjbxxViewMysql> resultList = zdryMysqlService.page(page, queryWrapper).getRecords();
CountDownLatch latch = new CountDownLatch(resultList.size());
ExecutorService impUserPool = Executors.newFixedThreadPool(5);
for (ZhxfZdrygkRyjbxxViewMysql bean : resultList) {
impUserPool.submit(() -> {
cleanImpUser(bean);
latch.countDown();
});
}
try {
// Wait for all threads to complete
@ -94,9 +71,55 @@ public class MetaImpUserInfoTask {
} catch (InterruptedException e) {
e.printStackTrace();
}
zdryPool.shutdown();
// 关闭线程池
impUserPool.shutdown();
} catch (Exception e) {
logger.info(StringKit.getTrace(e));
}
}
private void cleanImpUser(ZhxfZdrygkRyjbxxViewMysql bean) {
ZhxfZdrygkRyjbxxViewMysql beanCp = new ZhxfZdrygkRyjbxxViewMysql();
try {
VillageActualUserInfo userInfoPara = new VillageActualUserInfo();
userInfoPara.setIdcard(StringKit.toString(beanCp.getZJHM()));
List<VillageActualUserInfo> actualUserInfos = actualUserInfoService.selectVillageActualUserInfoListLimit1(userInfoPara);
String userPic = "";
if (actualUserInfos != null && actualUserInfos.size() > 0) {
userPic = StringKit.toString(actualUserInfos.get(0).getUserPic());
}
BeanUtils.copyProperties(bean, beanCp);
MetaImpUserInfo userInfo = new MetaImpUserInfo();
userInfo.setUserPic(userPic);
userInfo.setUserName(StringKit.toString(beanCp.getXM()));
userInfo.setIdCard(StringKit.toString(beanCp.getZJHM()));
userInfo.setSex( "".equals(StringKit.toString(beanCp.getXB()))? 0: NumKit.checkInt(beanCp.getXB()));
userInfo.setPhone(StringKit.toString(beanCp.getSJHM()));
String zdrlx = StringKit.toString(beanCp.getZDRYXL());
if("".equals(zdrlx)){
userInfo.setImpUserType(zdrlx.substring(0,4) + "000000");
userInfo.setImpUserSubType(beanCp.getZDRYXL());
}
userInfo.setHouseHoldRegister(beanCp.getHJDQH());
userInfo.setHouseHoldAddress(beanCp.getHJDXZ());
userInfo.setHabitationRegister(beanCp.getXZDHQ());
userInfo.setHabitationAddress(beanCp.getXZDXZ());
userInfo.setRegisterUnitCode(beanCp.getDJRDWDM());
userInfo.setRegisterUnitName(beanCp.getDJRDWMC());
boolean b = metaImpUserInfoService.save(userInfo);
if (b) {
beanCp.setStatus("1");
zdryMysqlService.updateZdryjbxxById(beanCp);
}
} catch (Exception e) {
logger.info("zdrys:" + e.getMessage());
logger.info(StringKit.getTrace(e));
beanCp.setStatus("2");
zdryMysqlService.updateZdryjbxxById(beanCp);
}
}

8
src/main/resources/mapper/MetaActualUserInfoMapper.xml
Unescape View File

@ -32,7 +32,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
select id, user_pic, user_name, id_card, sex, phone, landline, house_hold_register, house_hold_address, habitation_address, police_station_code, police_station_name, police_district_code, police_district_name, management_status, home_id, home_code, account_number, meta_create_time, meta_update_time, actual_user_type from meta_actual_user_info
</sql>
<select id="selectMetaActualUserInfoList" parameterType="MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<select id="selectMetaActualUserInfoList" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<include refid="selectMetaActualUserInfoVo"/>
<where>
<if test="userPic != null and userPic != ''"> and user_pic = #{userPic}</if>
@ -63,7 +63,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
where id = #{id}
</select>
<insert id="insertMetaActualUserInfo" parameterType="MetaActualUserInfo" useGeneratedKeys="true" keyProperty="id">
<insert id="insertMetaActualUserInfo" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo" useGeneratedKeys="true" keyProperty="id">
insert into meta_actual_user_info
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="userPic != null">user_pic,</if>
@ -111,7 +111,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
</trim>
</insert>
<update id="updateMetaActualUserInfo" parameterType="MetaActualUserInfo">
<update id="updateMetaActualUserInfo" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo">
update meta_actual_user_info
<trim prefix="SET" suffixOverrides=",">
<if test="userPic != null">user_pic = #{userPic},</if>
@ -151,7 +151,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<select id="selectMetaActualUserInfoListByHomeId" parameterType="MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<select id="selectMetaActualUserInfoListByHomeId" parameterType="com.ssf.mysqloracletest.domain.MetaActualUserInfo" resultMap="MetaActualUserInfoResult">
<include refid="selectMetaActualUserInfoVo"/>
where home_id is not null and account_number is null limit 200;
</select>

4
src/main/resources/mapper/ZhxfZdrygkRyjbxxViewMysqlMapper.xml
Unescape View File

@ -56,7 +56,6 @@
<result property="SJHM" column="SJHM" jdbcType="VARCHAR"/>
<result property="YWSJHM" column="YWSJHM" jdbcType="VARCHAR"/>
<result property="CXZT" column="CXZT" jdbcType="VARCHAR"/>
<result property="status" column="status" jdbcType="VARCHAR"/>
</resultMap>
@ -240,8 +239,7 @@
<if test="SJHM != null and SJHM != '' ">SJHM =#{SJHM },</if>
<if test="YWSJHM != null and YWSJHM != '' ">YWSJHM =#{YWSJHM },</if>
<if test="CXZT != null and CXZT != '' ">CXZT =#{CXZT },</if>
<if test="status != null and status != '' ">status =#{status},</if>
<if test="status != null and status != '' ">status =#{status },</if>
</trim>
where RYJBXXID = #{RYJBXXID}
</update>

Write Preview
Loading…
Cancel
Save