物资数据导入

module_kzks/src/main/java/org/jeecg/modules/projectImportList/controller/ProjectImportListController.java

@@ -119,7 +119,7 @@ public class ProjectImportListController extends JeecgController<ProjectImportLi
         } else if (projectImportList.getFileType().equals("9")) { //导入数据包至服务器
             return result.success("");
         } else if (projectImportList.getFileType().equals("10")){//物资导入
-//            return wzOutboundOrderBNewService.importExcel1(url, WzOutboundOrderBNew.class);
+            //return wzOutboundOrderBNewService.importExcel1(url, WzOutboundOrderBNew.class);
             return wzOutboundOrderBNewService.importExcelNew(url, WzOutboundOrderBNew.class);
         } else {
             result.setCode(500);

module_kzks/src/main/java/org/jeecg/modules/wzOutboundOrderBNew/monitor/WzOutboundOrderBNewListenerS.java

@@ -0,0 +1,129 @@
+package org.jeecg.modules.wzOutboundOrderBNew.monitor;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.modules.message.websocket.WebSocket;
+import org.jeecg.modules.projectChbSwf.entity.ProjectChbSwf;
+import org.jeecg.modules.projectImportList.service.impl.ProjectImportListServiceImpl;
+import org.jeecg.modules.wzOutboundOrderBNew.entity.WzOutboundOrderBNew;
+import org.jeecg.modules.wzOutboundOrderBNew.service.IWzOutboundOrderBNewService;
+import org.jeecg.modules.wzOutboundOrderBNew.service.impl.WzOutboundOrderBNewServiceImpl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * @author dzc
+ * @date 2024/4/1 9:03
+ * @package org.jeecg.modules.wzOutboundOrderBNew.monitor
+ * @project yecai_server
+ * @des
+ */
+@Slf4j
+public class WzOutboundOrderBNewListenerS extends AnalysisEventListener<WzOutboundOrderBNew> {
+
+    private static final List<WzOutboundOrderBNew> wzList = new CopyOnWriteArrayList<>(); // 存放物资数据的集合
+
+
+    private final IWzOutboundOrderBNewService wzOutboundOrderBNewService;
+
+
+    private static final int BIG_SIZE = 3000;  // 批量添加最大数量
+
+    public WzOutboundOrderBNewListenerS(IWzOutboundOrderBNewService wzOutboundOrderBNewService) {
+        this.wzOutboundOrderBNewService = wzOutboundOrderBNewService;
+    }
+
+    @Override
+    public void invoke(WzOutboundOrderBNew wzOutboundOrderBNew, AnalysisContext analysisContext) {
+        wzList.add(wzOutboundOrderBNew);
+        if (wzList.size() == BIG_SIZE){
+            saveInfoInKu(wzList);
+        }
+    }
+
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext analysisContext) {
+        saveInfoInKu(wzList);
+    }
+
+
+    private void saveInfoInKu(List<WzOutboundOrderBNew> wzList) {
+        ExecutorService threadPoolExecutors = Executors.newFixedThreadPool(8);
+        try {
+            log.info("{}条数据，开始存储数据库！", wzList.size());
+            int sonListSize = wzList.size() / 8;  // 计算将wzList分成8份之后每个子集合的大小
+
+            List<List<WzOutboundOrderBNew>> sonList = new ArrayList<>(); // 用于存放 子集合 的list
+            for (int i = 0; i < 8; i++) {
+                int startIndex = i * sonListSize;
+                int endIndex = (i + 1) * sonListSize;
+                if (i == 7){  // 避免遗漏数据，最后一个集合取全部数据
+                    endIndex = wzList.size();
+                }
+                // 根据起始索引、终止索引进行截取
+                List<WzOutboundOrderBNew> list = wzList.subList(startIndex, endIndex);
+                sonList.add(list);
+            }
+            // 八个线程同步保存数据
+            if (!sonList.isEmpty()){
+                if (!sonList.get(0).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(0));
+                    });
+                }
+                if (!sonList.get(1).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(1));
+                    });
+                }
+                if (!sonList.get(2).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(2));
+                    });
+                }
+                if (!sonList.get(3).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(3));
+                    });
+                }
+                if (!sonList.get(4).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(4));
+                    });
+                }
+                if (!sonList.get(5).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(5));
+                    });
+                }
+                if (!sonList.get(6).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(6));
+                    });
+                }
+                if (!sonList.get(7).isEmpty()){
+                    threadPoolExecutors.submit(() -> {
+                        wzOutboundOrderBNewService.saveBatch(sonList.get(7));
+                    });
+                }
+            }
+            threadPoolExecutors.shutdown();
+            boolean b = threadPoolExecutors.awaitTermination(5, TimeUnit.HOURS);
+            log.info("数据成功入库");
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        } finally {
+            threadPoolExecutors.shutdown();
+            wzList.clear();
+        }
+    }
+
+}

module_kzks/src/main/java/org/jeecg/modules/wzOutboundOrderBNew/service/IWzOutboundOrderBNewService.java

@@ -5,6 +5,7 @@ import org.jeecg.modules.wzOutboundOrderBNew.entity.WzOutboundOrderBNew;
 import com.baomidou.mybatisplus.extension.service.IService;
 
 import java.io.IOException;
+import java.util.concurrent.ExecutionException;
 
 /**
  * @Description: wz_outbound_order_b_new
@@ -14,7 +15,7 @@ import java.io.IOException;
  */
 public interface IWzOutboundOrderBNewService extends IService<WzOutboundOrderBNew> {
 
-    Result<?> importExcel1(String strUrl, Class<WzOutboundOrderBNew> clazz);
+    Result<?> importExcel1(String strUrl, Class<WzOutboundOrderBNew> clazz) throws InterruptedException, ExecutionException, IOException;
     Result<?> importExcelNew(String strUrl, Class<WzOutboundOrderBNew> clazz) throws IOException;
 
 }

module_kzks/src/main/java/org/jeecg/modules/wzOutboundOrderBNew/service/impl/WzOutboundOrderBNewServiceImpl.java

@@ -3,6 +3,7 @@ package org.jeecg.modules.wzOutboundOrderBNew.service.impl;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.EasyExcelFactory;
 import com.alibaba.excel.ExcelReader;
+import com.alibaba.excel.cache.MapCache;
 import com.alibaba.excel.read.metadata.ReadSheet;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
@@ -11,10 +12,13 @@ import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.ss.usermodel.WorkbookFactory;
 import org.jeecg.common.api.vo.Result;
 import org.jeecg.common.util.DateUtils;
+import org.jeecg.modules.dataSourceSwitch.annotation.UseSlaveDataSource;
 import org.jeecg.modules.message.websocket.WebSocket;
+import org.jeecg.modules.projectImportList.readListener.ProjectChbImportReadListener;
 import org.jeecg.modules.projectKmbh.entity.KzksProjectKmbh;
 import org.jeecg.modules.wzOutboundOrderBNew.entity.WzOutboundOrderBNew;
 import org.jeecg.modules.wzOutboundOrderBNew.mapper.WzOutboundOrderBNewMapper;
+import org.jeecg.modules.wzOutboundOrderBNew.monitor.WzOutboundOrderBNewListenerS;
 import org.jeecg.modules.wzOutboundOrderBNew.monitor.WzOutboundOrderBNewListerner;
 import org.jeecg.modules.wzOutboundOrderBNew.service.IWzOutboundOrderBNewService;
 import org.jeecgframework.poi.excel.ExcelImportUtil;
@@ -28,9 +32,7 @@ import javax.annotation.Resource;
 import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.concurrent.*;
 
 /**
  * @Description: wz_outbound_order_b_new
@@ -52,46 +54,17 @@ public class WzOutboundOrderBNewServiceImpl extends ServiceImpl<WzOutboundOrderB
     private ExecutorService executorService = Executors.newFixedThreadPool(5);
 
     @Override
-    public Result<?> importExcel1(String strUrl, Class<WzOutboundOrderBNew> clazz) {
-        InputStream inputStream = null;
-        try {
-            inputStream = new FileInputStream(strUrl);
-        } catch (FileNotFoundException e) {
-            e.printStackTrace();
-        }
-
-        // 获取上传文件对象
-        ImportParams params = new ImportParams();
-        params.setTitleRows(0);
-        params.setHeadRows(1);
-        params.setNeedSave(true);
-        try {
-            List<WzOutboundOrderBNew> list = ExcelImportUtil.importExcel(inputStream, clazz, params);
-            int year = DateUtils.getYear();
-            orderBNewMapper.deleteByDate(year);
-            //update-begin-author:taoyan date:20190528 for:批量插入数据
-            long start = System.currentTimeMillis();
-            this.saveBatch(list);
-            log.info("消耗时间" + (System.currentTimeMillis() - start) + "毫秒");
-            //update-end-author:taoyan date:20190528 for:批量插入数据
-            return Result.ok("文件导入成功！数据行数：" + list.size());
-        } catch (Exception e) {
-            //update-begin-author:taoyan date:20211124 for: 导入数据重复增加提示
-            String msg = e.getMessage();
-            log.error(msg, e);
-            if (msg != null && msg.indexOf("Duplicate entry") >= 0) {
-                return Result.error("文件导入失败:有重复数据！");
-            } else {
-                return Result.error("文件导入失败:" + e.getMessage());
-            }
-            //update-end-author:taoyan date:20211124 for: 导入数据重复增加提示
-        } finally {
-            try {
-                inputStream.close();
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-        }
+    public Result<?> importExcel1(String strUrl, Class<WzOutboundOrderBNew> clazz) throws InterruptedException, ExecutionException, IOException{
+        long startTime = System.currentTimeMillis();
+        ExcelReader excelReader = EasyExcel.read(new File(strUrl), clazz, new WzOutboundOrderBNewListenerS(this)).build();
+        ReadSheet readSheet = EasyExcel.readSheet(0).build();
+        excelReader.read(readSheet);
+        //这里千万别忘记关闭，读的时候会创建临时文件，到时磁盘会崩的
+        excelReader.finish();
+        long endTime = System.currentTimeMillis();
+        log.info("导入物资数据花费时间:{}毫秒 | {}分钟", (endTime - startTime), ((endTime - startTime) / 1000 / 60));
+
+        return Result.ok();
     }