AOS/aos-file-personal
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: 解决excel导入的时候出现的数据重复问题

Browse Source 
解决excel导入的时候出现的数据重复问题 但是在导入幂等性上还无法处理
This commit is contained in:
yangjiyu
2022-07-06 11:35:41 +08:00
parent 69a0f76fda
commit be9120ad2a
5 changed files with 98 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/sql/database.sql
+4 -4
View File
@@ -33,10 +33,10 @@ CREATE TABLE `t_employee_Post`
`current_position` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '现任职位',
`current_position_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '当前职位时间',
`current_rank_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '当前职级时间',
`managers_type` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '经理人类型',
`appointment_documents` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '任命文件',
`post_type` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '岗位类型',
`remark` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '备注',
`managers_type` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '经理人类型',
`appointment_documents` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '任命文件',
`post_type` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '岗位类型',
`remark` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '备注',
`deleted` bit(1) NOT NULL DEFAULT b'0' COMMENT '删除状态:0: 未删除 1: 已删除 (公共字段)',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间 (公共字段)',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间 (公共字段)',
src/main/java/com/aos/fileidentify/config/AosPageReadListener.java
+77
View File
@@ -0,0 +1,77 @@
package com.aos.fileidentify.config;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.read.listener.ReadListener;
import com.alibaba.excel.util.ListUtils;
import org.apache.commons.collections4.CollectionUtils;
import java.util.List;
import java.util.function.Consumer;
/**
* <p>Company: 成都返空汇网络技术有限公司</p>
* <p>Description: </p>
*
* @author yangjiyu
* @version 1.0.0
* @email "mailto:yangjiyu@fkhwl.com"
* @date 2022.07.06 10:21
* @since 1.0.0
*/
public class AosPageReadListener<T> implements ReadListener<T> {
/**
* Single handle the amount of data
*/
public static int BATCH_COUNT = 100;
/**
* Temporary storage of data
*/
private List<T> cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
/**
* consumer
*/
private final Consumer<List<T>> consumer;
/**
* Aos page read listener
*
* @param consumer consumer
* @since 0.0.1
*/
public AosPageReadListener(Consumer<List<T>> consumer) {
this.consumer = consumer;
}
/**
* Invoke
*
* @param data data
* @param context context
* @since 0.0.1
*/
@Override
public void invoke(T data, AnalysisContext context) {
cachedDataList.add(data);
if (cachedDataList.size() >= BATCH_COUNT) {
consumer.accept(cachedDataList);
cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
}
}
/**
* 在每个sheet页写完了的时候均会调用一次该方法，所以加上清理的代码，防止doReadAll的时候重复写入数据
*
* @param context context
* @since 0.0.1
*/
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
if (CollectionUtils.isNotEmpty(cachedDataList)) {
consumer.accept(cachedDataList);
cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
}
}
}
src/main/java/com/aos/fileidentify/entity/dto/EmployeeApprovalDTO.java
+1 -1
View File
@@ -52,7 +52,7 @@ public class EmployeeApprovalDTO{
private String ethnic;
/** Birth */
@ExcelProperty("出生年月")
private String birth;
private Date birth;
/** Highest education */
@ExcelProperty("最高学历")
private String highestEducation;
src/main/java/com/aos/fileidentify/service/impl/FileIdentifyServiceImpl.java
+15 -38
View File
@@ -1,10 +1,7 @@
package com.aos.fileidentify.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.read.listener.ModelBuildEventListener;
import com.alibaba.excel.read.listener.PageReadListener;
import com.alibaba.excel.read.metadata.ReadSheet;
import com.aos.fileidentify.config.AosPageReadListener;
import com.aos.fileidentify.entity.Employee;
import com.aos.fileidentify.entity.Post;
import com.aos.fileidentify.entity.dto.EmployeeApprovalDTO;
@@ -14,6 +11,7 @@ import com.aos.fileidentify.mapper.PostMapper;
import com.aos.fileidentify.service.FileIdentifyService;
import com.aos.fileidentify.util.CodeUtils;
import com.aos.fileidentify.util.PdfUtils;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@@ -97,63 +95,42 @@ public class FileIdentifyServiceImpl implements FileIdentifyService {
public void processExcel(MultipartFile file) {
Assert.notNull(file, "上传数据为空");
try {
// 写法1
try (ExcelReader excelReader = EasyExcel.read(file.getInputStream()).build()) {
// 这里为了简单 所以注册了 同样的head 和Listener 自己使用功能必须不同的Listener
ReadSheet readSheet1 =
EasyExcel.readSheet(0).head(EmployeeApprovalDTO.class).registerReadListener(new PageReadListener<EmployeeApprovalDTO>(dataList -> {
this.saveEmployeeApproval(dataList);
})).build();
ReadSheet readSheet2 =
EasyExcel.readSheet(1).head(EmployeeApprovalDTO.class).registerReadListener(new PageReadListener<EmployeeApprovalDTO>(dataList -> {
this.saveEmployeeApproval(dataList);
})).build();
// 这里注意 一定要把sheet1 sheet2 一起传进去，不然有个问题就是03版的excel 会读取多次，浪费性能
excelReader.read(readSheet1, readSheet2);
}
ExcelReader excelReader = EasyExcel.read(file.getInputStream(), EmployeeApprovalDTO.class, new PageReadListener<EmployeeApprovalDTO>(dataList -> {
EasyExcel.read(file.getInputStream(), EmployeeApprovalDTO.class, new AosPageReadListener<EmployeeApprovalDTO>(dataList -> {
this.saveEmployeeApproval(dataList);
})).build();
})).doReadAll();
List<ReadSheet> sheets = excelReader.excelExecutor().sheetList();
excelReader.read();
excelReader.finish();
} catch (IOException e) {
e.printStackTrace();
}
}
private void saveEmployeeApproval(List<EmployeeApprovalDTO> dataList) {
// todo:此处需要更新为从数据库读到所有已经存储的数据
List<String> employeeKeyList = new ArrayList<>();
List<String> employeeKeyList =
employeeMapper.selectList(new QueryWrapper<>()).stream().map(Employee::getEmployeeKey).collect(Collectors.toList());
for (EmployeeApprovalDTO employeeApprovalDTO : dataList) {
String employKey = CodeUtils.getEmployeeKey(employeeApprovalDTO.getName(), employeeApprovalDTO.getBirth());
employeeKeyList.add(employKey);
String employKey = CodeUtils.getEmployeeKey(employeeApprovalDTO.getName(), employeeApprovalDTO.getBirth().toString());
if (employeeKeyList.contains(employKey)) {
this.savePost(employeeApprovalDTO);
this.savePost(employeeApprovalDTO, employKey);
} else {
this.saveEmployee(employeeApprovalDTO);
this.savePost(employeeApprovalDTO);
employeeKeyList.add(employKey);
this.saveEmployee(employeeApprovalDTO, employKey);
this.savePost(employeeApprovalDTO, employKey);
}
log.info(employeeApprovalDTO.toString());
}
}
private void savePost(EmployeeApprovalDTO employeeApprovalDTO) {
private void savePost(EmployeeApprovalDTO employeeApprovalDTO, String employKey) {
Post post = new Post();
BeanUtils.copyProperties(employeeApprovalDTO, post);
post.setEmployeeKey(employKey);
this.postMapper.insert(post);
}
private void saveEmployee(EmployeeApprovalDTO employeeApprovalDTO) {
private void saveEmployee(EmployeeApprovalDTO employeeApprovalDTO, String employKey) {
Employee employee = new Employee();
BeanUtils.copyProperties(employeeApprovalDTO, employee);
employee.setEmployeeKey(employKey);
this.employeeMapper.insert(employee);
}
}
src/main/resources/application.yml
+1 -1
View File
@@ -9,7 +9,7 @@ spring:
logging:
level:
com.app.mapper: debug
com.app.mapper: info
mybatis-plus:
configuration: