From 1c0afff9d75a2200aae589542b2b25506e207bd8 Mon Sep 17 00:00:00 2001 From: "wenkuan.shi" Date: Fri, 14 Jun 2024 18:44:41 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9B=BE=E7=89=87=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../GetInformationForExcelController.java | 113 +------- .../reader/entity/EquipmentDataDetail.java | 3 +- .../entity/EquipmentFolderLocation.java | 3 + .../GetInformationForExcelServiceImpl.java | 273 ++++++++++++++---- .../reader/GetInformationForExcelMapper.xml | 5 +- 5 files changed, 239 insertions(+), 158 deletions(-) diff --git a/src/main/java/com/xujie/sys/modules/reader/controller/GetInformationForExcelController.java b/src/main/java/com/xujie/sys/modules/reader/controller/GetInformationForExcelController.java index f50ff7e0..380610b4 100644 --- a/src/main/java/com/xujie/sys/modules/reader/controller/GetInformationForExcelController.java +++ b/src/main/java/com/xujie/sys/modules/reader/controller/GetInformationForExcelController.java @@ -1,5 +1,8 @@ package com.xujie.sys.modules.reader.controller; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; +import com.alibaba.excel.util.CollectionUtils; import com.xujie.sys.common.utils.R; import com.xujie.sys.modules.reader.service.GetInformationForExcelService; import lombok.extern.slf4j.Slf4j; @@ -8,13 +11,10 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.nio.file.Files; import java.util.ArrayList; import java.util.List; +import java.util.Map; @RequestMapping("getInformation") @RestController @@ -31,105 +31,24 @@ public class GetInformationForExcelController { } public static void main(String[] args) throws Exception { + String filePath = "E:\\file\\2.xls"; - String filePath ="E:\\file\\cs.tff"; - int startLine = 15; // 第一次读取开始的行数 - int firstReadLines = 41; - int columnInterval = 11; - - + File excelFile = new File(filePath); + List> list = new ArrayList<>(); try { - BufferedReader reader = new BufferedReader(new FileReader(filePath)); - String line; - List list=new ArrayList<>(); - // 第一次读取从第 15 行开始的 41 行数据 - for (int i = 1; i < startLine; i++) { //跳过前面的15行 - line=reader.readLine(); - } - for (int i = 0; i < firstReadLines; i++) { - line = reader.readLine(); - if (line != null) { - // 处理当前行数据 - list.add(line); - } - } - // 移动到下一列 - for (int i = 0; i < columnInterval; i++) { - line = reader.readLine(); - } - - // 每隔 11 行读取往下 41 行的数据 - while ((line = reader.readLine()) != null) { - // 从当前行开始读取指定行数 - for (int i = 0; i < firstReadLines; i++) { - if (line != null) { - // 处理当前行数据 - list.add(line); - } - line = reader.readLine(); - } + list = EasyExcel.read(excelFile) + .excelType(ExcelTypeEnum.XLSX) + .headRowNumber(0) + .sheet() + .doReadSync(); - // 移动到下一列 - for (int i = 0; i < columnInterval-1; i++) { - line = reader.readLine(); - } - } - - for (String s : list) { - String[] data = s.split("\t"); - System.out.println(data[0]); - System.out.println(data[1]); - System.out.println(data[2]); - System.out.println(data[3]); - System.out.println(data[4]); - System.out.println(data[5]); - System.out.println(data[6]); - System.out.println(data[7]); - System.out.println(data[8]); - System.out.println(data[9]); - } - reader.close(); - } catch (IOException e) { + } catch (Exception e) { e.printStackTrace(); + log.info("读取文件失败,文件为:" + excelFile + "失败日志:" + e.getMessage()); } - // 源文件路径 - String sourcePath = "E:\\file\\cs.tff"; - // 目标文件夹路径 - String destinationFolder = "E:\\info"; - - // 创建源文件对象和目标文件夹对象 - File sourceFile = new File(sourcePath); - File destinationDir = new File(destinationFolder); - - // 确保源文件存在且是文件 - if (sourceFile.exists() && sourceFile.isFile()) { - // 确保目标文件夹存在且是目录 - if (!destinationDir.exists()) { - destinationDir.mkdirs(); // 创建目标文件夹及其父目录 - } - // 构建目标文件路径 - String destinationPath = destinationFolder + File.separator + sourceFile.getName(); - File destinationFile = new File(destinationPath); - - try { - // 复制文件到目标文件夹 - Files.copy(sourceFile.toPath(), destinationFile.toPath()); - System.out.println("文件复制成功！"); - - // 删除源文件 - if (sourceFile.delete()) { - System.out.println("源文件删除成功！"); - } else { - System.out.println("源文件删除失败！"); - } - } catch (IOException e) { - e.printStackTrace(); - System.out.println("文件操作出错：" + e.getMessage()); - } - } else { - System.out.println("源文件不存在或不是文件。"); + if (CollectionUtils.isEmpty(list)) { + log.info("读取的文件为空,或不是excel格式的"); } } - } diff --git a/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentDataDetail.java b/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentDataDetail.java index 87402ae7..f10d3c69 100644 --- a/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentDataDetail.java +++ b/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentDataDetail.java @@ -30,10 +30,9 @@ public class EquipmentDataDetail { @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private String createDate; private Date createBy; + private String photoValue1; //批次号 private int batchNo; //文件编号 private String fileNo; - - } diff --git a/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentFolderLocation.java b/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentFolderLocation.java index 08d55563..3c2291d3 100644 --- a/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentFolderLocation.java +++ b/src/main/java/com/xujie/sys/modules/reader/entity/EquipmentFolderLocation.java @@ -12,10 +12,13 @@ public class EquipmentFolderLocation { private String folderPath; //设备号 private String equipmentNo; + //图片保存位置 + private String backupPhotoPath; private String site; private String buNo; private String itemNo; private String backupFolderPath; + //文件编号 private String fileNo; diff --git a/src/main/java/com/xujie/sys/modules/reader/service/impl/GetInformationForExcelServiceImpl.java b/src/main/java/com/xujie/sys/modules/reader/service/impl/GetInformationForExcelServiceImpl.java index 56544b41..0b2d00cd 100644 --- a/src/main/java/com/xujie/sys/modules/reader/service/impl/GetInformationForExcelServiceImpl.java +++ b/src/main/java/com/xujie/sys/modules/reader/service/impl/GetInformationForExcelServiceImpl.java @@ -38,7 +38,7 @@ public class GetInformationForExcelServiceImpl extends ServiceImpl locationList = equipmentFolderLocationService.list(); if (CollectionUtils.isEmpty(locationList)) { @@ -52,7 +52,7 @@ public class GetInformationForExcelServiceImpl extends ServiceImpl files = getExcelFiles(folderFiler); - if (CollectionUtils.isEmpty(files)) { +// List files = getExcelFiles(folderFiler); + //创建本地文件夹路径 + File fileInfo = new File(folderFiler); + //获取本地文件夹下所有文件 + File[] files = fileInfo.listFiles(); + //文件 + List excelFiles = new ArrayList<>(); + //图片 + List imageFiles = new ArrayList<>(); + + List deleteFile =new ArrayList<>(); + + if (fileInfo != null && files != null) { + for (File file1 : files) { + try { + String name = file1.getName(); + String timestamp = String.valueOf(System.currentTimeMillis()); + // 构建目标文件名，使用源文件名加上时间戳后缀 + String destinationFileName = timestamp+"$"+name ; + File renamedFile = new File(file1.getParent(), destinationFileName); + String contentType = Files.probeContentType(file1.toPath()); + if (contentType != null && contentType.startsWith("image")) { + // 图片类型文件 + // 处理图片文件的逻辑 + deleteFile.add(folderFiler + "\\" + file1.getName()); + imageFiles.add(folderFiler + "\\" + renamedFile.getName()); + } else { + deleteFile.add(folderFiler + "\\" + file1.getName()); + excelFiles.add(renamedFile); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + if (CollectionUtils.isEmpty(excelFiles) && CollectionUtils.isEmpty(imageFiles) ) { return; } //判断这个批次号应该是多少 @@ -90,73 +124,71 @@ public class GetInformationForExcelServiceImpl extends ServiceImpl equipmentDataDetails = saveImage(imageFiles, excel, batchNo); + getInformationForExcelMapper.saveByExcels(equipmentDataDetails); + for (String s : deleteFile) { + deleteBypath(s, excel.getBackupPhotoPath()); + } + } - //创建一个list用来存储所有的的excel文件路径 + //创建一个list用来存储所有的的excel(csv)文件路径 List fileName = new ArrayList<>(); + //创建一个list用来存储所有的tff(txt)文件路径 + List tffName = new ArrayList<>(); //创建一个list用来存储所有的txt文件路径 List txtName = new ArrayList<>(); - for (File file : files) { + for (File file : excelFiles) { //找到.的索引 int dotIndex = file.getName().lastIndexOf("."); String s = file.getName().substring(dotIndex + 1); - if (s.equals("csv")) { + if (s.equals("csv") ||s.equals("xls")) { fileName.add(folderFiler + "\\" + file.getName()); } if (s.equals("tff")) { + tffName.add(folderFiler + "\\" + file.getName()); + } + if (s.equals("txt")) { txtName.add(folderFiler + "\\" + file.getName()); } } //excel(csv)不为空就做新增 if (CollectionUtils.isNotEmpty(fileName)) { List equipmentDataDetails = saveExcel(fileName, excel, batchNo); - if (CollectionUtils.isNotEmpty(equipmentDataDetails)){ - getInformationForExcelMapper.saveByExcels(equipmentDataDetails); + if (CollectionUtils.isNotEmpty(equipmentDataDetails)) { //批量新增 - for (String s : fileName) { + getInformationForExcelMapper.saveByExcels(equipmentDataDetails); + for (String s : deleteFile) { deleteBypath(s, excel.getBackupFolderPath()); } - }else { + } else { return; } } //txt(tff)不为空就做新增 - if (CollectionUtils.isNotEmpty(txtName)) { - List equipmentDataDetails = saveTxt(txtName, excel, batchNo); - //如果需要新增的数量小于一百就直接新增,大于一百就做线程池新增 - if (equipmentDataDetails.size()<100){ - //批量新增 - getInformationForExcelMapper.saveByExcels(equipmentDataDetails); - }else { - - // 创建一个固定大小的线程池，这里假设有10个线程 - ExecutorService executor = Executors.newFixedThreadPool(equipmentDataDetails.size()/100); - int startList =0; - int endList=100; - - for (int i = startList; i < equipmentDataDetails.size(); i= startList) { - if (endList>equipmentDataDetails.size()){ - endList = equipmentDataDetails.size(); - } - List details = equipmentDataDetails.subList(startList, endList); - startList=startList+endList; - endList=endList+endList; - executor.execute(()->{ - //批量新增 - getInformationForExcelMapper.saveByExcels(details); - }); - } - // 关闭线程池 - executor.shutdown(); + if (CollectionUtils.isNotEmpty(tffName)) { + List equipmentDataDetails = saveTxt(tffName, excel, batchNo); + //新增数据 + saveInformation(equipmentDataDetails); + for (String s : deleteFile) { + deleteBypath(s, excel.getBackupFolderPath()); } - for (String s : txtName) { - deleteBypath( s, excel.getBackupFolderPath()); + } + //txt文件不为空就做新增 + if (CollectionUtils.isNotEmpty(txtName)) { + List equipmentDataDetails1 = saveTxtFile(txtName, excel, batchNo); + saveInformation(equipmentDataDetails1); + for (String s : deleteFile) { + deleteBypath(s, excel.getBackupFolderPath()); } } } //新增完之后把原路径的文件复制到另一个文件并删除原路径的下的文件 - private static void deleteBypath(String endPath,String suffixName) throws Exception{ + private static void deleteBypath(String endPath, String suffixName) throws Exception { // 创建源文件对象和目标文件夹对象 File sourceFile = new File(endPath); File destinationDir = new File(suffixName); @@ -168,20 +200,23 @@ public class GetInformationForExcelServiceImpl extends ServiceImpl(); } //获取TEST后面的两个值 for (int i = 0; i < list.size(); i++) { - if (list.get(i).get(0)==null){ + if (list.get(i).get(0) == null) { continue; } if (list.get(i).get(0).equals("TEST")) { @@ -305,20 +339,145 @@ public class GetInformationForExcelServiceImpl extends ServiceImpl getExcelFiles(String folderFiler) { + + //对txt文件进行操作 + private static List saveTxtFile(List fileName, EquipmentFolderLocation excel, Integer batchNo) { + List detailList = new ArrayList<>(); + for (String name : fileName) { + try { + + BufferedReader bufferedReader = new BufferedReader(new FileReader(name)); + String line; + List list = new ArrayList<>(); + for (int i = 1; i <= 19; i++) { + bufferedReader.readLine(); // 跳过前面的行数 + } + while ((line = bufferedReader.readLine()) != null) { + if (line != null) { + // 处理当前行数据 + list.add(line); + } + } + for (String s : list) { + EquipmentDataDetail equipmentDataDetail = new EquipmentDataDetail(); + String[] data = s.split("\t"); + equipmentDataDetail.setEquipmentNo(excel.getEquipmentNo()); + equipmentDataDetail.setBuNo(excel.getBuNo()); + equipmentDataDetail.setSite(excel.getSite()); + equipmentDataDetail.setBatchNo(batchNo); + equipmentDataDetail.setItemNo(excel.getItemNo()); + equipmentDataDetail.setFileNo(excel.getFileNo()); + equipmentDataDetail.setValue0(data[0]); + equipmentDataDetail.setValue1(data[1]); + equipmentDataDetail.setValue2(data[2]); + equipmentDataDetail.setValue3(data[3]); + detailList.add(equipmentDataDetail); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + return detailList; + } + + //对图片进行保存 + private static List saveImage(List fileName,EquipmentFolderLocation excel,Integer batchNo){ + List detailList = new ArrayList<>(); + for (String s : fileName) { + EquipmentDataDetail equipmentDataDetail = new EquipmentDataDetail(); + equipmentDataDetail.setEquipmentNo(excel.getEquipmentNo()); + equipmentDataDetail.setBuNo(excel.getBuNo()); + equipmentDataDetail.setSite(excel.getSite()); + equipmentDataDetail.setBatchNo(batchNo); + equipmentDataDetail.setItemNo(excel.getItemNo()); + equipmentDataDetail.setFileNo(excel.getFileNo()); + equipmentDataDetail.setPhotoValue1(s); + detailList.add(equipmentDataDetail); + } + return detailList; + + } + +// //获取文件夹下的所有文件 +// private static List getExcelFiles(String folderFiler) { +// //创建本地文件夹路径 +// File file = new File(folderFiler); +// //获取本地文件夹下所有文件 +// File[] files = file.listFiles(); +// List excelFiles = new ArrayList<>(); +// if (file != null && files != null) { +// for (File file1 : files) { +// if (file1.isFile()) { +// excelFiles.add(file1); +// } +// } +// } +// return excelFiles; +// } + + //超过一百条数据做线程池新增新增 + private void saveInformation(List equipmentDataDetails) { + //如果需要新增的数量小于一百就直接新增,大于一百就做线程池新增 + if (equipmentDataDetails.size() < 100) { + //批量新增 + getInformationForExcelMapper.saveByExcels(equipmentDataDetails); + } else { + + // 创建一个固定大小的线程池，这里假设有10个线程 + ExecutorService executor = Executors.newFixedThreadPool(equipmentDataDetails.size() / 100); + int startList = 0; + int endList = 100; + + for (int i = startList; i < equipmentDataDetails.size(); i = startList) { + if (endList > equipmentDataDetails.size()) { + endList = equipmentDataDetails.size(); + } + List details = equipmentDataDetails.subList(startList, endList); + startList = startList + endList; + endList = endList + endList; + executor.execute(() -> { + //批量新增 + getInformationForExcelMapper.saveByExcels(details); + }); + } + // 关闭线程池 + executor.shutdown(); + } + } + + public static void main(String[] args) { + String folderFiler = "E:\\file"; //创建本地文件夹路径 File file = new File(folderFiler); //获取本地文件夹下所有文件 File[] files = file.listFiles(); List excelFiles = new ArrayList<>(); - if (file != null && files !=null) { + List imageFiles = new ArrayList<>(); + + List s1=new ArrayList<>(); + List s2=new ArrayList<>(); + if (file != null && files != null) { for (File file1 : files) { - if (file1.isFile()) { - excelFiles.add(file1); + try { + String contentType = Files.probeContentType(file1.toPath()); + if (contentType != null && contentType.startsWith("image")) { + // 图片类型文件 + // 处理图片文件的逻辑 + imageFiles.add(file1); + String s=folderFiler + "\\" + file1.getName(); + s1.add(s); + } else { + excelFiles.add(file1); + String s3=folderFiler + "\\" + file1.getName(); + s2.add(s3); + } + } catch (Exception e) { + e.printStackTrace(); } } } - return excelFiles; + + System.out.println(s1); + System.out.println(s2); } } diff --git a/src/main/resources/mapper/reader/GetInformationForExcelMapper.xml b/src/main/resources/mapper/reader/GetInformationForExcelMapper.xml index 51e32d11..7726f2dc 100644 --- a/src/main/resources/mapper/reader/GetInformationForExcelMapper.xml +++ b/src/main/resources/mapper/reader/GetInformationForExcelMapper.xml @@ -20,12 +20,13 @@ + - insert into Equipment_data_detail(site,bu_no,equipment_no,item_no,value0,value1,value2,value3,value4,value5,value6,value7,value8,value9,create_date,create_by,batch_no,file_no) + insert into Equipment_data_detail(site,bu_no,equipment_no,item_no,value0,value1,value2,value3,value4,value5,value6,value7,value8,value9,create_date,create_by,batch_no,file_no,photo_value1) values - (#{item.site},#{item.buNo},#{item.equipmentNo},#{item.itemNo},#{item.value0},#{item.value1},#{item.value2},#{item.value3},#{item.value4},#{item.value5},#{item.value6},#{item.value7},#{item.value8},#{item.value9},#{item.createDate},#{item.createBy},#{item.batchNo},#{item.fileNo}) + (#{item.site},#{item.buNo},#{item.equipmentNo},#{item.itemNo},#{item.value0},#{item.value1},#{item.value2},#{item.value3},#{item.value4},#{item.value5},#{item.value6},#{item.value7},#{item.value8},#{item.value9},#{item.createDate},#{item.createBy},#{item.batchNo},#{item.fileNo},#{item.photoValue1})