当前位置： 首页 > news >正文

EasyExcel多线程批量导出数据，动态表头，静态资源访问

news 2025/9/13 20:21:05

1.导入依赖

        <dependency><groupId>com.alibaba</groupId><artifactId>easyexcel</artifactId><version>3.1.1</version></dependency>

2.建立实体

@Data
public class ActResultLogVO implements Serializable {private static final long serialVersionUID = 1L;@ExcelProperty(value = "onlineseqid",index = 0)private String onlineseqid;@ExcelProperty(value = "businessid",index = 1)private String businessid;@ExcelProperty(value = "becifno",index = 2)private String becifno;@ExcelProperty(value = "ivisresult",index = 3)private String ivisresult;@ExcelProperty(value = "createdby",index = 4)private String createdby;@ExcelProperty(value = "createddate",index = 5)private LocalDate createddate;@ExcelProperty(value = "updateby",index = 6)private String updateby;@ExcelProperty(value = "updateddate",index = 7)private LocalDate updateddate;@ExcelProperty(value = "risklevel",index = 8)private String risklevel;/*** 分页开始数量*/private Integer start;/***  分页 size*/private Integer size;
}

3.多线程异步导出

	@Async@Transactionalpublic  void exportExcel(String filePath) {//filePath 文件路径ExcelWriter writer = EasyExcel.write(filePath, ActResultLogVO.class).build();// 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1int N = 2;// 大小设置为2就可以，作为缓冲BlockingQueue<List<ActResultLogVO>> queue = new ArrayBlockingQueue<>(2);AtomicInteger start = new AtomicInteger(0);AtomicInteger num = new AtomicInteger(0);AtomicInteger sheet = new AtomicInteger(1);// 分页大小可以适当调整int pageSize = 100000;//开启多个线程分页查数据for (int i = 0; i < N; i++) {executorService.submit(() -> {while (true) {//自增int startNum= start.getAndAdd(pageSize);try {long l = System.currentTimeMillis();logger.info("[多线程，分页查询] 线程：{}，开始执行查询：startNum :{},时间：{}",Thread.currentThread().getName(),startNum,l);List<ActResultLogVO> list = selectActResultLogVOPage(startNum, pageSize);logger.info("[多线程，分页查询] 线程：{}，执行查询完成 startNum :{} 用时：{}",Thread.currentThread().getName(),startNum,System.currentTimeMillis()-l);if (CollectionUtils.isEmpty(list)) {//读到没数据也要放入空集合queue.put(Collections.EMPTY_LIST);break;}queue.put(list);} catch (Exception e) {//异常情况也要放入空集合，防止写线程无法退出循环try {queue.put(Collections.EMPTY_LIST);} catch (InterruptedException ex) {throw new RuntimeException(ex);}throw new RuntimeException(e);}}});}Future<?> submit = executorService.submit(() -> {int count = 0;while (true) {List<ActResultLogVO> list = null;try {list = queue.take();} catch (InterruptedException e) {Thread.interrupted();}if (CollectionUtils.isEmpty(list)) {count++;// 当获取到两次空集合时，说明已经读完if (count == N) {break;}continue;}if(num.get()>=500000){sheet.getAndAdd(1);num.getAndSet(0);}num.getAndAdd(list.size());long l = System.currentTimeMillis();logger.info("[多线程，写入Excel] 线程：{}，开始执行写入：sheet :{},num:{}时间：{}",Thread.currentThread().getName(),sheet.get(),num.get(),l);writer.write(list, EasyExcel.writerSheet("Sheet"+sheet).build());logger.info("[多线程，写入Excel] 线程：{}，开始执行写入结束：sheet :{},num:{},时间：{}",Thread.currentThread().getName(),sheet.get(),num.get(),System.currentTimeMillis()-l);}writer.finish();});try {// 阻塞等待完成，异步处理也可以去掉这段代码submit.get();} catch (Exception e) {}}

4.动态表头

 	 @Autowired@Qualifier("excelThreadPool")private ExecutorService executorService;@Async@Transactionalpublic void exportExcel（String filePath,LinkedHashMap<String, DynamicExcelData> nameMap) {// 指定写入的文件ExcelWriter writer = EasyExcel.write(filePath).head(ExcelUtils.getHead(nameMap)).build();// 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1int N = 2;// 大小设置为2就可以，作为缓冲BlockingQueue<List<Map<String,Object>>> queue = new ArrayBlockingQueue<>(2);AtomicInteger num = new AtomicInteger(0);AtomicInteger sheet = new AtomicInteger(1);// 分页大小可以适当调整int pageSize = ExcelConstants.SELECT_TO_DB_ROWS_MYBATIS;//开启多个线程分页查数据getData(lvnengAllCardBillVo,queue,pageSize);Future<?> submit = executorService.submit(() -> {int count = 0;while (true) {List<Map<String,Object>> list = null;try {list = queue.take();} catch (InterruptedException e) {Thread.interrupted();}if (CollectionUtils.isEmpty(list)) {count++;// 当获取到两次空集合时，说明已经读完if (count == N) {break;}continue;}if(num.get()>=600000){sheet.getAndAdd(1);num.getAndSet(0);}num.getAndAdd(list.size());List<List<String>> dataList = ExcelUtils.getDataList(nameMap, list);writer.write(dataList, EasyExcel.writerSheet("Sheet"+sheet).build());}writer.finish();});try {// 阻塞等待完成，异步处理也可以去掉这段代码submit.get();} catch (Exception e) {}}

public void getData(ActResultLogVO actResultLogVO,BlockingQueue<List<Map<String,Object>>> queue,int pageSize) {// 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1int N = 2;AtomicInteger start = new AtomicInteger(0);for (int i = 0; i < N; i++) {executorService.submit(() -> {while (true) {//自增int startNum = start.getAndAdd(pageSize);List<Map<String, Object>> list = new ArrayList<>();try {ActResultLogVO actResultLogVO1 = DeepCopyUtil.deepCopy(actResultLogVO);actResultLogVO1.setStart(startNum);List<Map<String, Object>> list = selectactResultLogVOPage(actResultLogVO1);if (CollectionUtils.isEmpty(list)) {//读到没数据也要放入空集合queue.put(Collections.EMPTY_LIST);break;}queue.put(list);} catch (Exception e) {//异常情况也要放入空集合，防止写线程无法退出循环try {queue.put(Collections.EMPTY_LIST);} catch (InterruptedException ex) {throw new RuntimeException(ex);}throw new RuntimeException(e);}}});}logger.info("查询数据完成");}

@Configuration
public class ThreadPoolConfig {@Bean("excelThreadPool")public  ExecutorService buildExcelThreadPool() {int cpuNum = Runtime.getRuntime().availableProcessors();BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(1000);ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("excel-pool-%d").build();return new ThreadPoolExecutor(10 * cpuNum, 30 * cpuNum,1, TimeUnit.MINUTES, workQueue, threadFactory);}
}

public class ExcelConstants {public static final Integer PER_SHEET_ROW_COUNT = 100*10000;public static final Integer PER_WRITE_ROW_COUNT = 20*10000;public static final Integer SHEET_WRITE_ROW_COUNT = 50*10000;public static final Integer SELECT_TO_DB_ROWS = 10*10000;public static final Integer SELECT_TO_DB_ROWS_MYBATIS = 5*10000;
}

@Data
public class DynamicExcelData {//列名private String name;//默认值private String defaultValue;public DynamicExcelData(String name, String defaultValue) {this.name = name;this.defaultValue = defaultValue;}
}

5,用到的util

5.1 EasyUtils

public class ExcelUtils {public static void dynamicExportByURL(String filePath,LinkedHashMap<String, DynamicExcelData> nameMap,List<Map<String, Object>> list,String sheetName) throws IOException {//首先判断是否有数据，没有就返回if(CollUtil.isEmpty(list)){return;}//这里的map使用LinkedHashMap，实现字段的顺序功能if(nameMap==null){throw new RuntimeException("请填写好映射表数据");}File file = getFile(filePath);dynamicExportByURL(file,nameMap,list,sheetName);}public static void dynamicExport(HttpServletResponse response,LinkedHashMap<String, DynamicExcelData> nameMap,List<Map<String, Object>> list,String sheetName) throws IOException {//首先判断是否有数据，没有就返回if(CollUtil.isEmpty(list)){return;}//这里的map使用LinkedHashMap，实现字段的顺序功能if(nameMap==null){throw new RuntimeException("请填写好映射表数据");}response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");response.setCharacterEncoding("utf-8");//使用EasyExcel write进行写出EasyExcel.write(response.getOutputStream()).head(getHead(nameMap)).sheet(sheetName).doWrite(getDataList(nameMap,list));}public static ArrayList<List<String>> getHead(LinkedHashMap<String, DynamicExcelData> nameMap) {//获取表头ArrayList<List<String>> head = new ArrayList<>();for (Map.Entry<String, DynamicExcelData> titleMap : nameMap.entrySet()) {DynamicExcelData data = titleMap.getValue();head.add(Collections.singletonList(data.getName()));}return head;}public static List<List<String>> getDataList(LinkedHashMap<String, DynamicExcelData> nameMap, List<Map<String, Object>> list) {//先初始化一下传入int size = list.size();List<List<String>> dataList = new ArrayList<>();for (int i = 0; i < size; i++) {dataList.add(new ArrayList<>());}//数据重组for (int i = 0; i < list.size(); i++) {Map<String, Object> map = list.get(i);List<String> columns = dataList.get(i);for (Map.Entry<String, DynamicExcelData> sortNameEntry : nameMap.entrySet()) {String key = sortNameEntry.getKey();Object value = map.get(key);columns.add(value != null ? String.valueOf(value) : sortNameEntry.getValue().getDefaultValue());}}return dataList;}public static File getFile(String filePath) throws IOException {File file = new File(filePath);if (!file.exists()) {file.createNewFile();}return file;}public static void dynamicExportByURL(File file,LinkedHashMap<String, DynamicExcelData> nameMap,  List date,String sheetName) {long startTime = 0;ExcelWriter excelWriter = null;try {startTime = System.currentTimeMillis();// 获取 sheet 的个数int sheetNum = getSheetNum(date.size());// 获取每个sheet 写入的次数int writeNumPerSheet = getWriteNumPerSheet();// 最后一个 sheet 写入的数量int writeNumLastSheet = date.size() - (sheetNum - 1) * ExcelConstants.PER_SHEET_ROW_COUNT;// 最后一个 sheet 写入的次数int writeNumPerLastSheet = writeNumLastSheet % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT : writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT + 1;// 指定写入的文件excelWriter = EasyExcel.write(file).head(getHead(nameMap)).build();for (int i = 0; i < sheetNum; i++) {String sheet = sheetName + i;WriteSheet writeSheet = EasyExcel.writerSheet(i, sheet).build();int writeNum = i == sheetNum - 1 ? writeNumPerLastSheet : writeNumPerSheet; // 每个sheet 写入的次数int endEndNum = i == sheetNum - 1 ? date.size() : (i + 1) * ExcelConstants.PER_SHEET_ROW_COUNT; // 每个sheet 最后一次写入的最后行数for (int j = 0; j < writeNum; j++) {long l = System.currentTimeMillis();int startNum = i * ExcelConstants.PER_SHEET_ROW_COUNT + j * ExcelConstants.PER_WRITE_ROW_COUNT;int endNum = j == writeNum - 1 ? endEndNum : i * ExcelConstants.PER_SHEET_ROW_COUNT + (j + 1) * ExcelConstants.PER_WRITE_ROW_COUNT;excelWriter.write(getDataList(nameMap,date.subList(startNum, endNum)), writeSheet);}}} catch (Exception e) {} finally {// 需要放入 finally 中if (excelWriter != null) {excelWriter.finish();}}}public static void writeExcel(String filePath, Class clazz, List date,String sheetName) throws Exception{File file = getFile(filePath);long startTime = System.currentTimeMillis();// 获取 sheet 的个数int sheetNum = getSheetNum(date.size());// 获取每个sheet 写入的次数int writeNumPerSheet = getWriteNumPerSheet();// 最后一个 sheet 写入的数量int writeNumLastSheet = date.size() - (sheetNum - 1) * ExcelConstants.PER_SHEET_ROW_COUNT;// 最后一个 sheet 写入的次数int writeNumPerLastSheet = writeNumLastSheet % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT : writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT + 1;// 指定写入的文件ExcelWriter excelWriter = EasyExcel.write(file, clazz).build();for (int i = 0; i < sheetNum; i++) {String sheet = sheetName + i;WriteSheet writeSheet = EasyExcel.writerSheet(i, sheet).build();int writeNum = i == sheetNum - 1 ? writeNumPerLastSheet : writeNumPerSheet; // 每个sheet 写入的次数int endEndNum = i == sheetNum - 1 ? date.size() : (i + 1) * ExcelConstants.PER_SHEET_ROW_COUNT; // 每个sheet 最后一次写入的最后行数for (int j = 0; j < writeNum; j++) {long l = System.currentTimeMillis();int startNum = i * ExcelConstants.PER_SHEET_ROW_COUNT + j * ExcelConstants.PER_WRITE_ROW_COUNT;int endNum = j == writeNum - 1 ? endEndNum : i * ExcelConstants.PER_SHEET_ROW_COUNT + (j + 1) * ExcelConstants.PER_WRITE_ROW_COUNT;excelWriter.write(date.subList(startNum, endNum), writeSheet);}}// 需要放入 finally 中if (excelWriter != null) {excelWriter.finish();}}public static int getSheetNum(int dateSize){return dateSize % ExcelConstants.PER_SHEET_ROW_COUNT == 0 ? (dateSize / ExcelConstants.PER_SHEET_ROW_COUNT) : (dateSize / ExcelConstants.PER_SHEET_ROW_COUNT + 1);}public static int getWriteNumPerSheet(){return ExcelConstants.PER_SHEET_ROW_COUNT % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? (ExcelConstants.PER_SHEET_ROW_COUNT / ExcelConstants.PER_WRITE_ROW_COUNT) : (ExcelConstants.PER_SHEET_ROW_COUNT / ExcelConstants.PER_WRITE_ROW_COUNT + 1);}public static  List<String> objectToList(Object obj){ArrayList<String> list = new ArrayList<>();//获取obj类中的所有字段Field[] fields = obj.getClass().getDeclaredFields();//遍历所有属性for (Field field : fields) {try {field.setAccessible(true);list.add(field.get(obj).toString());} catch (IllegalAccessException e) {throw new RuntimeException(e);}}return list;}public static Map<String,Object> objectToMap(Object obj){HashMap<String, Object> map = new HashMap<>();//获取obj类中的所有字段Field[] fields = obj.getClass().getDeclaredFields();//遍历所有属性for (Field field : fields) {try {field.setAccessible(true);map.put(field.getName(),field.get(obj));} catch (IllegalAccessException e) {throw new RuntimeException(e);}}return map;}/*** 通过反射方式将头部作为公共部分进行设置*/public static List<List<String>> setTitles(Class clazz) {List<List<String>> titles = new ArrayList<List<String>>();for (Field declaredField : clazz.getDeclaredFields()) {ExcelProperty annotation = declaredField.getAnnotation(ExcelProperty.class);if (null != annotation) {titles.add(Arrays.asList(annotation.value()));}}return titles;}
}

5.2 深拷贝

public class DeepCopyUtil {public static <T extends Serializable> T deepCopy(T object) {try {ByteArrayOutputStream outputStream = new ByteArrayOutputStream();ObjectOutputStream objectOutputStream = new ObjectOutputStream(outputStream);objectOutputStream.writeObject(object);ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());ObjectInputStream objectInputStream = new ObjectInputStream(inputStream);return (T) objectInputStream.readObject();} catch (IOException | ClassNotFoundException e) {throw new RuntimeException(e);}}
}

配置静态资源访问

@Configuration
public class MyStaticConfig extends WebMvcConfigurationSupport {@Value("${download.url}")private String fileDir;@Overridepublic void addResourceHandlers(ResourceHandlerRegistry registry) {registry.addResourceHandler("/uploads/**").addResourceLocations("classpath:/META-INF/resources/", "classpath:/resources/","classpath:/static/", "classpath:/public/","file:"+fileDir);}
}
http://www.lryc.cn/news/268913.html

相关文章：

  • 树莓派界面改成中文
  • 软件工程期末复习
  • 【linux】select实现定时器
  • Android 13 - Media框架（28）- MediaCodec（三）
  • Azure 学习总结
  • 数据库是否可以直接作为数据仓库的数据源
  • IntelliJ IDE 插件开发 | （四）开发一个时间管理大师插件
  • 【ChatGPT 默认强化学习策略】PPO 近端策略优化算法
  • 【银行测试】金融银行-理财项目面试/分析总结（二）
  • 张江智荟毁约offer
  • ubuntu 系统终端颜色设置
  • 【Vue】class与style绑定
  • 大厂前端面试题总结（百度、字节跳动、腾讯、小米.....），附上热乎面试经验！
  • EXPLORING DIFFUSION MODELS FOR UNSUPERVISED VIDEO ANOMALY DETECTION 论文阅读
  • 当 ML 遇到 DevOps：如何理解 MLOps
  • vue+element+springboot实现多张图片上传
  • react使用useState更新数组失败
  • 《LIO-SAM阅读笔记》3.后端优化
  • mac下jd-gui提示没有找到合适的jdk版本
  • FlinkSQL窗口实例分析
  • 18-网络安全框架及模型-信息系统安全保障模型
  • Android 提取(备份)apk(安装包)
  • gRPC-Go基础（4）metadata和超时设置
  • 语言模型：从n-gram到神经网络的演进
  • docker compose 部署 grafana + loki + vector 监控kafka消息
  • kubeadm创建k8s集群
  • 鸿蒙开发之android对比开发《基础知识》
  • 2702 高级打字机
  • yolov5旋转目标检测-遥感图像检测-无人机旋转目标检测-附代码和原理
  • Qt学习：Qt的意义安装Qt