4 changed files with 772 additions and 13 deletions
-
495src/main/java/com/xujie/sys/common/utils/StreamCsvExportUtil.java
-
24src/main/java/com/xujie/sys/modules/pms/mapper/MesTidEpcLogMapper.java
-
212src/main/java/com/xujie/sys/modules/pms/service/Impl/MesTidEpcLogServiceImpl.java
-
54src/main/resources/mapper/pms/MesTidEpcLogMapper.xml
@ -0,0 +1,495 @@ |
|||
package com.xujie.sys.common.utils; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import java.io.BufferedWriter; |
|||
import java.io.IOException; |
|||
import java.io.OutputStream; |
|||
import java.io.OutputStreamWriter; |
|||
import java.text.SimpleDateFormat; |
|||
import java.util.Date; |
|||
import java.util.function.Function; |
|||
import java.util.zip.ZipEntry; |
|||
import java.util.zip.ZipOutputStream; |
|||
|
|||
/** |
|||
* 流式CSV导出工具类(真正的流式导出,边查边写)- rqrq |
|||
* |
|||
* <p><b>功能特点:</b></p> |
|||
* <ul> |
|||
* <li>支持单CSV和多CSV(ZIP压缩包)两种模式</li> |
|||
* <li>使用GB18030编码,兼容Excel中文显示</li> |
|||
* <li>自动处理CSV字段转义(逗号、引号、换行)</li> |
|||
* <li>支持强制文本格式(保留前导零)</li> |
|||
* <li>内置进度日志记录</li> |
|||
* </ul> |
|||
* |
|||
* <p><b>使用方式:</b></p> |
|||
* <pre> |
|||
* // 1. 创建Writer |
|||
* StreamCsvExportUtil.CsvStreamWriter writer = StreamCsvExportUtil.createSingleCsvWriter(response.getOutputStream(), "导出文件.csv"); |
|||
* // 或创建ZIP Writer |
|||
* StreamCsvExportUtil.CsvStreamWriter writer = StreamCsvExportUtil.createZipCsvWriter(response.getOutputStream(), "导出文件.zip", "数据"); |
|||
* |
|||
* // 2. 写入表头 |
|||
* writer.writeHeader("列1", "列2", "列3"); |
|||
* |
|||
* // 3. 在ResultHandler中逐行写入 |
|||
* mapper.streamQuery(query, result -> { |
|||
* writer.writeRow( |
|||
* writer.asText(result.getField1()), // 强制文本格式 |
|||
* writer.escape(result.getField2()), // 普通转义 |
|||
* writer.formatDate(result.getDate()) // 日期格式化 |
|||
* ); |
|||
* }); |
|||
* |
|||
* // 4. 关闭Writer |
|||
* writer.close(); |
|||
* </pre> |
|||
* |
|||
* @author rqrq |
|||
* @date 2025/02/04 |
|||
*/ |
|||
@Slf4j |
|||
public class StreamCsvExportUtil { |
|||
|
|||
/** |
|||
* 默认编码(GB18030兼容Excel中文)- rqrq |
|||
*/ |
|||
private static final String DEFAULT_CHARSET = "GB18030"; |
|||
|
|||
/** |
|||
* 默认缓冲区大小(128KB)- rqrq |
|||
*/ |
|||
private static final int DEFAULT_BUFFER_SIZE = 8192 * 16; |
|||
|
|||
/** |
|||
* 每个CSV文件最大行数(默认100万)- rqrq |
|||
*/ |
|||
public static final int DEFAULT_MAX_ROWS_PER_CSV = 1000000; |
|||
|
|||
/** |
|||
* @Description 创建单CSV文件写入器 - rqrq |
|||
* @param outputStream 输出流 |
|||
* @param fileName 文件名(用于日志) |
|||
* @return CsvStreamWriter |
|||
* @author rqrq |
|||
*/ |
|||
public static CsvStreamWriter createSingleCsvWriter(OutputStream outputStream, String fileName) throws IOException { |
|||
return new SingleCsvWriter(outputStream, fileName); |
|||
} |
|||
|
|||
/** |
|||
* @Description 创建ZIP压缩包写入器(多CSV文件)- rqrq |
|||
* @param outputStream 输出流 |
|||
* @param zipFileName ZIP文件名(用于日志) |
|||
* @param csvFilePrefix CSV文件名前缀(如"数据",则生成"数据_1.csv"、"数据_2.csv") |
|||
* @return CsvStreamWriter |
|||
* @author rqrq |
|||
*/ |
|||
public static CsvStreamWriter createZipCsvWriter(OutputStream outputStream, String zipFileName, String csvFilePrefix) throws IOException { |
|||
return new ZipCsvWriter(outputStream, zipFileName, csvFilePrefix, DEFAULT_MAX_ROWS_PER_CSV); |
|||
} |
|||
|
|||
/** |
|||
* @Description 创建ZIP压缩包写入器(自定义每个CSV最大行数)- rqrq |
|||
* @param outputStream 输出流 |
|||
* @param zipFileName ZIP文件名 |
|||
* @param csvFilePrefix CSV文件名前缀 |
|||
* @param maxRowsPerCsv 每个CSV最大行数 |
|||
* @return CsvStreamWriter |
|||
* @author rqrq |
|||
*/ |
|||
public static CsvStreamWriter createZipCsvWriter(OutputStream outputStream, String zipFileName, String csvFilePrefix, int maxRowsPerCsv) throws IOException { |
|||
return new ZipCsvWriter(outputStream, zipFileName, csvFilePrefix, maxRowsPerCsv); |
|||
} |
|||
|
|||
/** |
|||
* CSV流式写入器接口 - rqrq |
|||
* |
|||
* <p><b>异常处理机制:</b></p> |
|||
* <ul> |
|||
* <li>写入失败时自动设置错误状态</li> |
|||
* <li>后续 writeRow 调用会直接跳过(避免资源浪费)</li> |
|||
* <li>调用方可通过 hasError() 检查是否发生错误</li> |
|||
* <li>调用方可通过 getLastError() 获取错误详情</li> |
|||
* </ul> |
|||
*/ |
|||
public interface CsvStreamWriter extends AutoCloseable { |
|||
|
|||
/** |
|||
* 写入表头行 - rqrq |
|||
*/ |
|||
void writeHeader(String... headers) throws IOException; |
|||
|
|||
/** |
|||
* 写入数据行(如果已发生错误则跳过)- rqrq |
|||
*/ |
|||
void writeRow(String... values) throws IOException; |
|||
|
|||
/** |
|||
* 写入数据行(安全版本,不抛异常,返回是否成功)- rqrq |
|||
* 用于 ResultHandler 中避免抛出 RuntimeException |
|||
*/ |
|||
boolean writeRowSafe(String... values); |
|||
|
|||
/** |
|||
* 获取已写入行数 - rqrq |
|||
*/ |
|||
long getWrittenRows(); |
|||
|
|||
/** |
|||
* 刷新缓冲区 - rqrq |
|||
*/ |
|||
void flush() throws IOException; |
|||
|
|||
/** |
|||
* 关闭写入器 - rqrq |
|||
*/ |
|||
@Override |
|||
void close() throws IOException; |
|||
|
|||
/** |
|||
* 检查是否已发生错误 - rqrq |
|||
*/ |
|||
boolean hasError(); |
|||
|
|||
/** |
|||
* 获取最后一次错误 - rqrq |
|||
*/ |
|||
Throwable getLastError(); |
|||
|
|||
/** |
|||
* 手动设置错误状态(用于外部捕获到异常时)- rqrq |
|||
*/ |
|||
void setError(Throwable e); |
|||
|
|||
/** |
|||
* CSV字段转义(处理逗号、引号、换行)- rqrq |
|||
*/ |
|||
default String escape(String value) { |
|||
if (value == null) { |
|||
return ""; |
|||
} |
|||
if (value.contains(",") || value.contains("\"") || value.contains("\n") || value.contains("\r")) { |
|||
return "\"" + value.replace("\"", "\"\"") + "\""; |
|||
} |
|||
return value; |
|||
} |
|||
|
|||
/** |
|||
* 强制文本格式(防止前导零丢失)- rqrq |
|||
* 使用 ="值" 格式,Excel会识别为文本 |
|||
*/ |
|||
default String asText(String value) { |
|||
if (value == null || value.isEmpty()) { |
|||
return ""; |
|||
} |
|||
return "=\"" + value.replace("\"", "\"\"") + "\""; |
|||
} |
|||
|
|||
/** |
|||
* 格式化日期 - rqrq |
|||
*/ |
|||
default String formatDate(Date date) { |
|||
if (date == null) { |
|||
return ""; |
|||
} |
|||
return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(date); |
|||
} |
|||
|
|||
/** |
|||
* 格式化日期(自定义格式)- rqrq |
|||
*/ |
|||
default String formatDate(Date date, String pattern) { |
|||
if (date == null) { |
|||
return ""; |
|||
} |
|||
return new SimpleDateFormat(pattern).format(date); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 单CSV文件写入器实现 - rqrq |
|||
*/ |
|||
private static class SingleCsvWriter implements CsvStreamWriter { |
|||
private final BufferedWriter writer; |
|||
private final String fileName; |
|||
private long writtenRows = 0; |
|||
|
|||
// 错误状态(volatile保证多线程可见性)- rqrq |
|||
private volatile boolean hasError = false; |
|||
private volatile Throwable lastError = null; |
|||
|
|||
SingleCsvWriter(OutputStream outputStream, String fileName) throws IOException { |
|||
this.fileName = fileName; |
|||
this.writer = new BufferedWriter(new OutputStreamWriter(outputStream, DEFAULT_CHARSET), DEFAULT_BUFFER_SIZE); |
|||
log.info("创建单CSV写入器: {} - rqrq", fileName); |
|||
} |
|||
|
|||
@Override |
|||
public void writeHeader(String... headers) throws IOException { |
|||
if (hasError) return; |
|||
try { |
|||
writer.write(String.join(",", headers)); |
|||
writer.newLine(); |
|||
} catch (IOException e) { |
|||
setError(e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void writeRow(String... values) throws IOException { |
|||
// 如果已发生错误,直接跳过 - rqrq |
|||
if (hasError) { |
|||
return; |
|||
} |
|||
|
|||
try { |
|||
writer.write(String.join(",", values)); |
|||
writer.newLine(); |
|||
writtenRows++; |
|||
|
|||
if (writtenRows % 100000 == 0) { |
|||
log.info("CSV已写入 {} 行 - rqrq", writtenRows); |
|||
flush(); |
|||
} |
|||
} catch (IOException e) { |
|||
setError(e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public boolean writeRowSafe(String... values) { |
|||
if (hasError) { |
|||
return false; |
|||
} |
|||
try { |
|||
writeRow(values); |
|||
return true; |
|||
} catch (IOException e) { |
|||
// 已在writeRow中设置了错误状态 |
|||
log.warn("写入行失败,后续行将跳过: {} - rqrq", e.getMessage()); |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public long getWrittenRows() { |
|||
return writtenRows; |
|||
} |
|||
|
|||
@Override |
|||
public void flush() throws IOException { |
|||
if (!hasError) { |
|||
writer.flush(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void close() throws IOException { |
|||
try { |
|||
if (!hasError) { |
|||
flush(); |
|||
} |
|||
writer.close(); |
|||
} catch (IOException e) { |
|||
log.warn("关闭写入器时发生错误: {} - rqrq", e.getMessage()); |
|||
} |
|||
log.info("单CSV写入完成: {},共 {} 行,hasError: {} - rqrq", fileName, writtenRows, hasError); |
|||
} |
|||
|
|||
@Override |
|||
public boolean hasError() { |
|||
return hasError; |
|||
} |
|||
|
|||
@Override |
|||
public Throwable getLastError() { |
|||
return lastError; |
|||
} |
|||
|
|||
@Override |
|||
public void setError(Throwable e) { |
|||
this.hasError = true; |
|||
this.lastError = e; |
|||
log.error("CSV写入器发生错误,后续写入将跳过: {} - rqrq", e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* ZIP压缩包写入器实现(多CSV文件)- rqrq |
|||
*/ |
|||
private static class ZipCsvWriter implements CsvStreamWriter { |
|||
private final ZipOutputStream zipOut; |
|||
private final String zipFileName; |
|||
private final String csvFilePrefix; |
|||
private final int maxRowsPerCsv; |
|||
|
|||
private BufferedWriter currentWriter; |
|||
private int currentCsvIndex = 0; |
|||
private long currentCsvRows = 0; |
|||
private long totalWrittenRows = 0; |
|||
private String[] headerRow; |
|||
|
|||
// 错误状态(volatile保证多线程可见性)- rqrq |
|||
private volatile boolean hasError = false; |
|||
private volatile Throwable lastError = null; |
|||
|
|||
ZipCsvWriter(OutputStream outputStream, String zipFileName, String csvFilePrefix, int maxRowsPerCsv) throws IOException { |
|||
this.zipOut = new ZipOutputStream(outputStream); |
|||
this.zipFileName = zipFileName; |
|||
this.csvFilePrefix = csvFilePrefix; |
|||
this.maxRowsPerCsv = maxRowsPerCsv; |
|||
log.info("创建ZIP写入器: {},每个CSV最多 {} 行 - rqrq", zipFileName, maxRowsPerCsv); |
|||
} |
|||
|
|||
@Override |
|||
public void writeHeader(String... headers) throws IOException { |
|||
if (hasError) return; |
|||
this.headerRow = headers; |
|||
try { |
|||
// 创建第一个CSV文件并写入表头 |
|||
createNewCsvFile(); |
|||
} catch (IOException e) { |
|||
setError(e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void writeRow(String... values) throws IOException { |
|||
// 如果已发生错误,直接跳过 - rqrq |
|||
if (hasError) { |
|||
return; |
|||
} |
|||
|
|||
try { |
|||
// 检查是否需要创建新CSV文件 |
|||
if (currentCsvRows >= maxRowsPerCsv) { |
|||
closeCurrentCsv(); |
|||
createNewCsvFile(); |
|||
} |
|||
|
|||
currentWriter.write(String.join(",", values)); |
|||
currentWriter.newLine(); |
|||
currentCsvRows++; |
|||
totalWrittenRows++; |
|||
|
|||
if (totalWrittenRows % 100000 == 0) { |
|||
log.info("ZIP总计已写入 {} 行,当前CSV {} 有 {} 行 - rqrq", |
|||
totalWrittenRows, currentCsvIndex, currentCsvRows); |
|||
flush(); |
|||
} |
|||
} catch (IOException e) { |
|||
setError(e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public boolean writeRowSafe(String... values) { |
|||
if (hasError) { |
|||
return false; |
|||
} |
|||
try { |
|||
writeRow(values); |
|||
return true; |
|||
} catch (IOException e) { |
|||
// 已在writeRow中设置了错误状态 |
|||
log.warn("写入行失败,后续行将跳过: {} - rqrq", e.getMessage()); |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public long getWrittenRows() { |
|||
return totalWrittenRows; |
|||
} |
|||
|
|||
@Override |
|||
public void flush() throws IOException { |
|||
if (!hasError && currentWriter != null) { |
|||
currentWriter.flush(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void close() throws IOException { |
|||
try { |
|||
if (!hasError) { |
|||
closeCurrentCsv(); |
|||
zipOut.finish(); |
|||
} |
|||
zipOut.close(); |
|||
} catch (IOException e) { |
|||
log.warn("关闭ZIP写入器时发生错误: {} - rqrq", e.getMessage()); |
|||
} |
|||
log.info("ZIP写入完成: {},共 {} 个CSV文件,总计 {} 行,hasError: {} - rqrq", |
|||
zipFileName, currentCsvIndex, totalWrittenRows, hasError); |
|||
} |
|||
|
|||
@Override |
|||
public boolean hasError() { |
|||
return hasError; |
|||
} |
|||
|
|||
@Override |
|||
public Throwable getLastError() { |
|||
return lastError; |
|||
} |
|||
|
|||
@Override |
|||
public void setError(Throwable e) { |
|||
this.hasError = true; |
|||
this.lastError = e; |
|||
log.error("ZIP写入器发生错误,后续写入将跳过: {} - rqrq", e.getMessage()); |
|||
} |
|||
|
|||
private void createNewCsvFile() throws IOException { |
|||
currentCsvIndex++; |
|||
// 使用ASCII文件名避免Windows解压乱码 - rqrq |
|||
String csvFileName = "TID_EPC_Log_" + currentCsvIndex + ".csv"; |
|||
ZipEntry entry = new ZipEntry(csvFileName); |
|||
zipOut.putNextEntry(entry); |
|||
|
|||
// 注意:ZIP内的Writer不能关闭底层流 |
|||
currentWriter = new BufferedWriter(new OutputStreamWriter(zipOut, DEFAULT_CHARSET), DEFAULT_BUFFER_SIZE); |
|||
|
|||
// 写入表头 |
|||
if (headerRow != null) { |
|||
currentWriter.write(String.join(",", headerRow)); |
|||
currentWriter.newLine(); |
|||
} |
|||
|
|||
currentCsvRows = 0; |
|||
log.info("创建CSV文件: {} - rqrq", csvFileName); |
|||
} |
|||
|
|||
private void closeCurrentCsv() throws IOException { |
|||
if (currentWriter != null) { |
|||
currentWriter.flush(); |
|||
zipOut.closeEntry(); |
|||
log.info("CSV文件 {} 写入完成,共 {} 行 - rqrq", currentCsvIndex, currentCsvRows); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* @Description 构建CSV数据行(通用方法)- rqrq |
|||
* @param extractors 字段提取器数组 |
|||
* @param data 数据对象 |
|||
* @return String[] CSV行数据 |
|||
* @author rqrq |
|||
*/ |
|||
@SafeVarargs |
|||
public static <T> String[] buildRow(T data, Function<T, String>... extractors) { |
|||
String[] row = new String[extractors.length]; |
|||
for (int i = 0; i < extractors.length; i++) { |
|||
row[i] = extractors[i].apply(data); |
|||
} |
|||
return row; |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue