人脸抓拍数据清洗

src/main/java/com/ssf/mysqloracletest/domain/TStrhEntityInfoKs.java

@@ -1,128 +1,164 @@
 package com.ssf.mysqloracletest.domain;
 
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableName;
 import lombok.Data;
 
 @Data
+@TableName(value ="t_strh_entity_info_ks")
 public class TStrhEntityInfoKs {
     /**
      * 对象ID
      */
+    @TableField(value = "objectid")
     private String objectid;
     /**
      * 设备camerIdx
      */
+    @TableField(value = "cameraidx")
     private String cameraidx;
     /**
      * 抓拍时间
      */
+    @TableField(value = "capturetime")
     private String capturetime;
     /**
      * 抓拍大图
      */
+    @TableField(value = "image")
     private String image;
     /**
      * 图片格式，如IMAGE_JPEG
      */
+    @TableField(value = "image_format")
     private String image_format;
     /**
      * 人脸在大图中的位置
      */
+    @TableField(value = "vertices")
     private String vertices;
     /**
      * 抓拍小图
      */
+    @TableField(value = "smalimage")
     private String smalimage;
     /**
      * 图片格式，如IMAGE_JPEG
      */
+    @TableField(value = "smalimage_format")
     private String smalimage_format;
     /**
      * 算法厂商原始id
      */
+    @TableField(value = "objectid_old")
     private String objectid_old;
     /**
      * 最小年龄
      */
+    @TableField(value = "age_lower_limit")
     private String age_lower_limit;
     /**
      * 最大年龄
      */
+    @TableField(value = "age_up_limit")
     private String age_up_limit;
+    @TableField(value = "black_type")
     private String black_type;
     /**
      *帽子样式
      */
+    @TableField(value = "cap_style")
     private String cap_style;
     /**
      * 性别
      */
+    @TableField(value = "gender_code")
     private String gender_code;
     /**
      * 眼镜
      */
+    @TableField(value = "glass_style")
     private String glass_style;
     /**
      * 胡型
      */
+    @TableField(value = "mustache_style")
     private String mustache_style;
     /**
      * 口罩颜色
      */
+    @TableField(value = "respirator_color")
     private String respirator_color;
     /**
      * 年龄标识
      */
+    @TableField(value = "st_age")
     private String st_age;
     /**
      * 相似年龄
      */
+    @TableField(value = "st_age_value")
     private String st_age_value;
     /**
      * 是否遮挡
      */
+    @TableField(value = "st_expression")
     private String st_expression;
     /**
      * 头盔样式
      */
+    @TableField(value = "st_helmet_style")
     private String st_helmet_style;
     /**
      * 入库时间
      */
+    @TableField(value = "rksj")
     private String rksj;
     /**
      * 更新时间
      */
+    @TableField(value = "gxsj")
     private String gxsj;
     /**
      * 档案唯一ID
      */
+    @TableField(value = "entityid")
     private String entityid;
     /**
      * 档案更新时间
      */
+    @TableField(value = "entityupdatetime")
     private String entityupdatetime;
     /**
      * 性别代码
      */
+    @TableField(value = "gender")
     private String gender;
     /**
      * 身份证号
      */
+    @TableField(value = "identityid")
     private String identityid;
     /**
      * 姓名
      */
+    @TableField(value = "name")
     private String name;
     /**
      * 籍贯
      */
+    @TableField(value = "natives")
     private String natives;
+    @TableField(value = "remark")
     private String remark;
+    @TableField(value = "yxx")
     private String yxx;
     /**
      *封面照
      */
+    @TableField(value = "storagepath")
     private String storagepath;
+    @TableField(value = "regionid")
     private String regionid;
 
 }

src/main/java/com/ssf/mysqloracletest/mapper/TStrhEntityInfoKsMapper.java

@@ -1,9 +1,12 @@
 package com.ssf.mysqloracletest.mapper;
 
+import com.baomidou.dynamic.datasource.annotation.DS;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.ssf.mysqloracletest.domain.TStrhEntityInfoKs;
+import org.apache.ibatis.annotations.Mapper;
 
-
+@Mapper
+@DS("postgreUser")
 public interface TStrhEntityInfoKsMapper extends BaseMapper<TStrhEntityInfoKs> {
 
 }

src/main/java/com/ssf/mysqloracletest/service/impl/TMotorvehicleServiceImpl.java

@@ -1,9 +1,11 @@
 package com.ssf.mysqloracletest.service.impl;
 
+import com.baomidou.dynamic.datasource.annotation.DS;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.ssf.mysqloracletest.domain.TMotorvehicle;
 import com.ssf.mysqloracletest.service.TMotorvehicleService;
 import com.ssf.mysqloracletest.mapper.TMotorvehicleMapper;
+import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;
 
 /**
@@ -11,7 +13,10 @@ import org.springframework.stereotype.Service;
 * @description 针对表【t_motorvehicle】的数据库操作Service实现
 * @createDate 2023-12-14 19:17:07
 */
+
 @Service
+@RequiredArgsConstructor
+@DS("postgreUser")
 public class TMotorvehicleServiceImpl extends ServiceImpl<TMotorvehicleMapper, TMotorvehicle>
     implements TMotorvehicleService{
 

src/main/java/com/ssf/mysqloracletest/service/impl/TStrhEntityInfoKsServiceImpl.java

@@ -13,7 +13,7 @@ import org.springframework.stereotype.Service;
 
 @Service
 @RequiredArgsConstructor
-@DS("postgre")
+@DS("postgreUser")
 public class TStrhEntityInfoKsServiceImpl extends ServiceImpl<TStrhEntityInfoKsMapper, TStrhEntityInfoKs>
     implements TStrhEntityInfoKsService {
 

src/main/java/com/ssf/mysqloracletest/service/impl/TStrhInfoServiceImpl.java

@@ -1,9 +1,11 @@
 package com.ssf.mysqloracletest.service.impl;
 
+import com.baomidou.dynamic.datasource.annotation.DS;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.ssf.mysqloracletest.domain.TStrhInfo;
 import com.ssf.mysqloracletest.service.TStrhInfoService;
 import com.ssf.mysqloracletest.mapper.TStrhInfoMapper;
+import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;
 
 /**
@@ -11,7 +13,10 @@ import org.springframework.stereotype.Service;
 * @description 针对表【t_strh_info】的数据库操作Service实现
 * @createDate 2023-12-14 20:14:20
 */
+
 @Service
+@RequiredArgsConstructor
+@DS("postgreUser")
 public class TStrhInfoServiceImpl extends ServiceImpl<TStrhInfoMapper, TStrhInfo>
     implements TStrhInfoService{
 

src/main/java/com/ssf/mysqloracletest/task/UserAndVehicleRecordTask.java

@@ -1,23 +1,27 @@
 package com.ssf.mysqloracletest.task;
 
+import cn.hutool.db.sql.Wrapper;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.ssf.mysqloracletest.domain.*;
 import com.ssf.mysqloracletest.service.*;
 import com.ssf.mysqloracletest.utils.ConfigParam;
-import com.ssf.mysqloracletest.utils.NumKit;
 import com.ssf.mysqloracletest.utils.StringKit;
 import lombok.RequiredArgsConstructor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeanUtils;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.TimeZone;
+
 
 /**
  * 用户数据接入
@@ -27,23 +31,16 @@ import java.util.concurrent.Executors;
 public class UserAndVehicleRecordTask {
 
     private static final Logger logger = LoggerFactory.getLogger(UserAndVehicleRecordTask.class);
-    String pattern = "yyyyMMddHHmmss";
-    String pattern1 = "yyyy-MM-dd HH-mm-ss";
-    SimpleDateFormat dateFormat = new SimpleDateFormat(pattern);
-    SimpleDateFormat dateFormat1 = new SimpleDateFormat(pattern1);
 
     private final TransitUserRecordService transitUserRecordService;
-    private final TransitVehicleRecordService transitVehicleRecordService;
+    private final TStrhEntityInfoKsService tStrhEntityInfoKsService;
     private final DevopsDeviceInfoService devopsDeviceInfoService;
-    private final TMotorvehicleService tMotorvehicleService;
-    private final TStrhInfoService tStrhInfoService;
 
-    /**
-     * 连云港 对接海康平台获取人员基础数据
-     * //
-     */
+    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    String maxTime = "2023-10-01 00:00:00";
+
     @Scheduled(initialDelay = 100, fixedDelay = 3000)
-    public void transNjjxx() {
+    public void setTransitUserRecordService() {
 
         if (!"true".equals(ConfigParam.UserAndVehicleSwitch)) {
             return;
@@ -51,73 +48,61 @@ public class UserAndVehicleRecordTask {
 
 
         try {
-            List<DevopsDeviceInfo> list = devopsDeviceInfoService.list();
-            for (DevopsDeviceInfo one : list){
-                if (one.getDeviceType() == 1){
-                    List<TStrhInfo> infoList = tStrhInfoService.lambdaQuery().eq(TStrhInfo::getCameraidx, one.getGbsChannelNo()).list();
-                    for (TStrhInfo info : infoList){
+            dateFormat.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai"));
+            LambdaQueryWrapper<TransitUserRecord> queryWrapper = Wrappers.lambdaQuery();
+            queryWrapper.orderByDesc(TransitUserRecord::getPartitionField);
+            queryWrapper.last("LIMIT 1");
+            TransitUserRecord one = transitUserRecordService.getOne(queryWrapper);
+            if (one != null) {
+                Date partitionField = one.getPartitionField();
+                maxTime = dateFormat.format(partitionField);
+            }
+            List<DevopsDeviceInfo> list = devopsDeviceInfoService.lambdaQuery().eq(DevopsDeviceInfo::getDeviceType, 1).list();
+            ArrayList<String> arrayList = new ArrayList<>();
+            for (DevopsDeviceInfo devopsDeviceInfo : list) {
+                arrayList.add(devopsDeviceInfo.getGbsChannelNo());
+            }
+            Integer count = tStrhEntityInfoKsService.lambdaQuery()
+                    .in(TStrhEntityInfoKs::getCameraidx, arrayList)
+                    .gt(TStrhEntityInfoKs::getCapturetime, maxTime).count();
+            if (count > 0) {
+                int num = count / 1000;
+                for (int i = 0; i <= num; i++) {
+                    Page<TStrhEntityInfoKs> page = new Page<>(i + 1, 1000);
+                    QueryWrapper<TStrhEntityInfoKs> wrapper = new QueryWrapper<>();
+                    wrapper.in("cameraidx", arrayList);
+                    wrapper.gt("capturetime", maxTime);
+                    List<TStrhEntityInfoKs> records = tStrhEntityInfoKsService.page(page, wrapper).getRecords();
+                    for (TStrhEntityInfoKs tStrhEntityInfoKs : records) {
                         TransitUserRecord transitUserRecord = new TransitUserRecord();
-                        transitUserRecord.setPlaceCode(one.getPlaceCode());
-                        transitUserRecord.setPlaceName(one.getPlaceName());
-                        transitUserRecord.setDeviceCode(one.getDeviceCode());
-                        transitUserRecord.setDeviceName(one.getDeviceName());
-                        transitUserRecord.setGbsChannelNo(one.getGbsChannelNo());
-                        transitUserRecord.setDeviceAddress(one.getDeviceAddress());
-                        transitUserRecord.setOrientation(one.getOrientation());
-                        transitUserRecord.setDeviceIp(one.getDeviceIp());
-                        transitUserRecord.setGlobalPic(info.getImage());
-                        transitUserRecord.setPersonPic(info.getSmalimage());
-                        transitUserRecord.setPassTime(dateFormat.parse(info.getCapturetime()).getTime()/1000);
-                        transitUserRecord.setPartitionField(dateFormat1.parse(info.getCapturetime()));
-                        TransitUserRecord one1 = transitUserRecordService.lambdaQuery().eq(TransitUserRecord::getGlobalPic, info.getImage()).one();
-                        if (one1 == null){
-                            boolean save = transitUserRecordService.save(transitUserRecord);
-                            if (save){
-                                logger.info("人脸-:插入成功");
-                            }
-                        }else {
-                            UpdateWrapper<TransitUserRecord> wrapper = new UpdateWrapper<>();
-                            wrapper.eq("global_pic", info.getImage());
-                            boolean update = transitUserRecordService.update(transitUserRecord, wrapper);
-                            if (update) {
-                                logger.info("人脸-:更新成功");
-                            }
+                        DevopsDeviceInfo devopsDeviceInfo = devopsDeviceInfoService.lambdaQuery().eq(DevopsDeviceInfo::getGbsChannelNo, tStrhEntityInfoKs.getCameraidx()).one();
+                        transitUserRecord.setPlaceCode(devopsDeviceInfo.getPlaceCode());
+                        transitUserRecord.setPlaceName(devopsDeviceInfo.getPlaceName());
+                        transitUserRecord.setDeviceCode(devopsDeviceInfo.getDeviceCode());
+                        transitUserRecord.setDeviceName(devopsDeviceInfo.getDeviceName());
+                        transitUserRecord.setGbsChannelNo(devopsDeviceInfo.getGbsChannelNo());
+                        transitUserRecord.setDeviceAddress(devopsDeviceInfo.getDeviceAddress());
+                        transitUserRecord.setGlobalPic(tStrhEntityInfoKs.getImage());
+                        transitUserRecord.setPersonPic(tStrhEntityInfoKs.getSmalimage());
+                        transitUserRecord.setUserStandardPic(tStrhEntityInfoKs.getStoragepath());
+                        transitUserRecord.setUserName(tStrhEntityInfoKs.getName());
+                        transitUserRecord.setIdCard(tStrhEntityInfoKs.getIdentityid());
+                        if (tStrhEntityInfoKs.getCapturetime() != null) {
+                            transitUserRecord.setPassTime(dateFormat.parse(tStrhEntityInfoKs.getCapturetime()).getTime() / 1000);
+                            transitUserRecord.setPartitionField(dateFormat.parse(tStrhEntityInfoKs.getCapturetime()));
                         }
-                    }
-                }
-                if (one.getDeviceType() == 2){
-                    List<TMotorvehicle> tMotorvehicleList = tMotorvehicleService.lambdaQuery().eq(TMotorvehicle::getTollgateid, one.getGbsChannelNo()).list();
-                    for (TMotorvehicle tMotorvehicle : tMotorvehicleList){
-                        TransitVehicleRecord transitVehicleRecord = new TransitVehicleRecord();
-                        transitVehicleRecord.setPlaceCode(one.getPlaceCode());
-                        transitVehicleRecord.setPlaceName(one.getPlaceName());
-                        transitVehicleRecord.setDeviceCode(one.getDeviceCode());
-                        transitVehicleRecord.setDeviceName(one.getDeviceName());
-                        transitVehicleRecord.setGbsChannelNo(one.getGbsChannelNo());
-                        transitVehicleRecord.setDeviceAddress(one.getDeviceAddress());
-                        transitVehicleRecord.setOrientation(one.getOrientation());
-                        transitVehicleRecord.setDeviceIp(one.getDeviceIp());
-                        transitVehicleRecord.setPlateNo(tMotorvehicle.getPlateno());
-                        transitVehicleRecord.setGlobalPic(tMotorvehicle.getStorageurl1());
-                        transitVehicleRecord.setPlatePic(tMotorvehicle.getStorageurl2());
-                        transitVehicleRecord.setSourceId(tMotorvehicle.getSourceid());
-                        transitVehicleRecord.setPassTime(dateFormat.parse(tMotorvehicle.getPasstime()).getTime()/1000);
-                        transitVehicleRecord.setPartitionField(dateFormat.parse(tMotorvehicle.getPasstime()));
-                        transitVehicleRecord.setPlateColor(tMotorvehicle.getPlatecolor());
-                        transitVehicleRecord.setVehicleColor(tMotorvehicle.getVehiclecolor());
-                        transitVehicleRecord.setVehicleBrand(tMotorvehicle.getVehiclebrand());
-                        TransitVehicleRecord one1 = transitVehicleRecordService.lambdaQuery().eq(TransitVehicleRecord::getGlobalPic, tMotorvehicle.getStorageurl1()).one();
-                        if (one1 == null){
-                            boolean save = transitVehicleRecordService.save(transitVehicleRecord);
-                            if (save){
-                                logger.info("车辆-:插入成功");
+                        TransitUserRecord one1 = transitUserRecordService.lambdaQuery().eq(TransitUserRecord::getPersonPic, tStrhEntityInfoKs.getSmalimage()).one();
+                        if (one1 == null) {
+                            boolean save = transitUserRecordService.save(transitUserRecord);
+                            if (save) {
+                                logger.info("UserRecord-" + transitUserRecord.getDeviceCode() + ":插入成功");
                             }
-                        }else {
-                            UpdateWrapper<TransitVehicleRecord> wrapper = new UpdateWrapper<>();
-                            wrapper.eq("global_pic", tMotorvehicle.getStorageurl1());
-                            boolean update = transitVehicleRecordService.update(transitVehicleRecord, wrapper);
+                        } else {
+                            QueryWrapper<TransitUserRecord> recordQueryWrapper = new QueryWrapper<>();
+                            recordQueryWrapper.eq("", tStrhEntityInfoKs.getSmalimage());
+                            boolean update = transitUserRecordService.update(transitUserRecord, recordQueryWrapper);
                             if (update) {
-                                logger.info("车辆-:更新成功");
+                                logger.info("UserRecord-" + transitUserRecord.getDeviceCode() + ":更新成功");
                             }
                         }
                     }