feat: 解决excel导入的时候出现的数据重复问题

解决excel导入的时候出现的数据重复问题 但是在导入幂等性上还无法处理

doc/sql/database.sql

@@ -33,10 +33,10 @@ CREATE TABLE `t_employee_Post`
     `current_position`      varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '现任职位',
     `current_position_time` timestamp NOT NULL                                            DEFAULT CURRENT_TIMESTAMP COMMENT '当前职位时间',
     `current_rank_time`     timestamp NOT NULL                                            DEFAULT CURRENT_TIMESTAMP COMMENT '当前职级时间',
-    `managers_type`         varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '经理人类型',
+    `managers_type`         varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '经理人类型',
-    `appointment_documents` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '任命文件',
+    `appointment_documents` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '任命文件',
-    `post_type`             varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '岗位类型',
+    `post_type`             varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '岗位类型',
-    `remark`                varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '备注',
+    `remark`                varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci  DEFAULT NULL COMMENT '备注',
     `deleted`               bit(1)    NOT NULL                                            DEFAULT b'0' COMMENT '删除状态:0: 未删除 1: 已删除 (公共字段)',
     `create_time`           timestamp NOT NULL                                            DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间 (公共字段)',
     `update_time`           timestamp NOT NULL                                            DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间 (公共字段)',

src/main/java/com/aos/fileidentify/config/AosPageReadListener.java

@@ -0,0 +1,77 @@
+package com.aos.fileidentify.config;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.read.listener.ReadListener;
+import com.alibaba.excel.util.ListUtils;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * <p>Company: 成都返空汇网络技术有限公司</p>
+ * <p>Description:  </p>
+ *
+ * @author yangjiyu
+ * @version 1.0.0
+ * @email "mailto:yangjiyu@fkhwl.com"
+ * @date 2022.07.06 10:21
+ * @since 1.0.0
+ */
+public class AosPageReadListener<T> implements ReadListener<T> {
+
+    /**
+     * Single handle the amount of data
+     */
+    public static int BATCH_COUNT = 100;
+    /**
+     * Temporary storage of data
+     */
+    private List<T> cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
+    /**
+     * consumer
+     */
+    private final Consumer<List<T>> consumer;
+
+    /**
+     * Aos page read listener
+     *
+     * @param consumer consumer
+     * @since 0.0.1
+     */
+    public AosPageReadListener(Consumer<List<T>> consumer) {
+        this.consumer = consumer;
+    }
+
+    /**
+     * Invoke
+     *
+     * @param data    data
+     * @param context context
+     * @since 0.0.1
+     */
+    @Override
+    public void invoke(T data, AnalysisContext context) {
+        cachedDataList.add(data);
+        if (cachedDataList.size() >= BATCH_COUNT) {
+            consumer.accept(cachedDataList);
+            cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
+        }
+    }
+
+    /**
+     * 在每个sheet页写完了的时候均会调用一次该方法，所以加上清理的代码，防止doReadAll的时候重复写入数据
+     *
+     * @param context context
+     * @since 0.0.1
+     */
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+        if (CollectionUtils.isNotEmpty(cachedDataList)) {
+            consumer.accept(cachedDataList);
+            cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
+        }
+    }
+
+}

src/main/java/com/aos/fileidentify/entity/dto/EmployeeApprovalDTO.java

@@ -52,7 +52,7 @@ public class EmployeeApprovalDTO{
     private String ethnic;
     /** Birth */
     @ExcelProperty("出生年月")
-    private String birth;
+    private Date birth;
     /** Highest education */
     @ExcelProperty("最高学历")
     private String highestEducation;

src/main/java/com/aos/fileidentify/service/impl/FileIdentifyServiceImpl.java

@@ -1,10 +1,7 @@
 package com.aos.fileidentify.service.impl;
 
 import com.alibaba.excel.EasyExcel;
-import com.alibaba.excel.ExcelReader;
+import com.aos.fileidentify.config.AosPageReadListener;
-import com.alibaba.excel.read.listener.ModelBuildEventListener;
-import com.alibaba.excel.read.listener.PageReadListener;
-import com.alibaba.excel.read.metadata.ReadSheet;
 import com.aos.fileidentify.entity.Employee;
 import com.aos.fileidentify.entity.Post;
 import com.aos.fileidentify.entity.dto.EmployeeApprovalDTO;
@@ -14,6 +11,7 @@ import com.aos.fileidentify.mapper.PostMapper;
 import com.aos.fileidentify.service.FileIdentifyService;
 import com.aos.fileidentify.util.CodeUtils;
 import com.aos.fileidentify.util.PdfUtils;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
@@ -97,63 +95,42 @@ public class FileIdentifyServiceImpl implements FileIdentifyService {
     public void processExcel(MultipartFile file) {
         Assert.notNull(file, "上传数据为空");
         try {
-
+            EasyExcel.read(file.getInputStream(), EmployeeApprovalDTO.class, new AosPageReadListener<EmployeeApprovalDTO>(dataList -> {
-
-            // 写法1
-            try (ExcelReader excelReader = EasyExcel.read(file.getInputStream()).build()) {
-                // 这里为了简单 所以注册了 同样的head 和Listener 自己使用功能必须不同的Listener
-                ReadSheet readSheet1 =
-                        EasyExcel.readSheet(0).head(EmployeeApprovalDTO.class).registerReadListener(new PageReadListener<EmployeeApprovalDTO>(dataList -> {
-                            this.saveEmployeeApproval(dataList);
-                        })).build();
-                ReadSheet readSheet2 =
-                        EasyExcel.readSheet(1).head(EmployeeApprovalDTO.class).registerReadListener(new PageReadListener<EmployeeApprovalDTO>(dataList -> {
-                            this.saveEmployeeApproval(dataList);
-                        })).build();
-                // 这里注意 一定要把sheet1 sheet2 一起传进去，不然有个问题就是03版的excel 会读取多次，浪费性能
-                excelReader.read(readSheet1, readSheet2);
-            }
-
-            ExcelReader excelReader = EasyExcel.read(file.getInputStream(), EmployeeApprovalDTO.class, new PageReadListener<EmployeeApprovalDTO>(dataList -> {
                 this.saveEmployeeApproval(dataList);
-            })).build();
+            })).doReadAll();
 
-            List<ReadSheet> sheets = excelReader.excelExecutor().sheetList();
-
-
-
-            excelReader.read();
-            excelReader.finish();
         } catch (IOException e) {
             e.printStackTrace();
         }
     }
 
     private void saveEmployeeApproval(List<EmployeeApprovalDTO> dataList) {
-        // todo:此处需要更新为从数据库读到所有已经存储的数据
+        List<String> employeeKeyList =
-        List<String> employeeKeyList = new ArrayList<>();
+            employeeMapper.selectList(new QueryWrapper<>()).stream().map(Employee::getEmployeeKey).collect(Collectors.toList());
         for (EmployeeApprovalDTO employeeApprovalDTO : dataList) {
-            String employKey = CodeUtils.getEmployeeKey(employeeApprovalDTO.getName(), employeeApprovalDTO.getBirth());
+            String employKey = CodeUtils.getEmployeeKey(employeeApprovalDTO.getName(), employeeApprovalDTO.getBirth().toString());
-            employeeKeyList.add(employKey);
             if (employeeKeyList.contains(employKey)) {
-                this.savePost(employeeApprovalDTO);
+                this.savePost(employeeApprovalDTO, employKey);
             } else {
-                this.saveEmployee(employeeApprovalDTO);
+                employeeKeyList.add(employKey);
-                this.savePost(employeeApprovalDTO);
+                this.saveEmployee(employeeApprovalDTO, employKey);
+                this.savePost(employeeApprovalDTO, employKey);
             }
             log.info(employeeApprovalDTO.toString());
         }
     }
 
-    private void savePost(EmployeeApprovalDTO employeeApprovalDTO) {
+    private void savePost(EmployeeApprovalDTO employeeApprovalDTO, String employKey) {
         Post post = new Post();
         BeanUtils.copyProperties(employeeApprovalDTO, post);
+        post.setEmployeeKey(employKey);
         this.postMapper.insert(post);
     }
 
-    private void saveEmployee(EmployeeApprovalDTO employeeApprovalDTO) {
+    private void saveEmployee(EmployeeApprovalDTO employeeApprovalDTO, String employKey) {
         Employee employee = new Employee();
         BeanUtils.copyProperties(employeeApprovalDTO, employee);
+        employee.setEmployeeKey(employKey);
         this.employeeMapper.insert(employee);
     }
 }

src/main/resources/application.yml

@@ -9,7 +9,7 @@ spring:
 
 logging:
   level:
-    com.app.mapper: debug
+    com.app.mapper: info
 
 mybatis-plus:
   configuration: