feat: 换成json格式

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java

@@ -937,7 +937,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 			AtomicInteger totalCount=new AtomicInteger(0);
 			AtomicInteger count=new AtomicInteger(0);
 			AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
-			readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
+			readJsonFile(condition,dirName,(cells,titles,firstFlag)->{
 				if (count.get()==0) {
 					XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
 					xlsxFileInfo.filepath=SAVE_FOLDER + "/" + task.getId() + xlsxFiles.size() + ".xlsx";
@@ -1071,6 +1071,58 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 	}
 
 
+	/**
+	 * 读物文件
+	 * @param dirName
+	 * @param func
+	 */
+	private void readJsonFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
+		File file = new File(SAVE_FOLDER+"/"+dirName);
+		File[] files = file.listFiles();
+		List<FlatQueryCondition> titles=null;
+		List<FlatQueryCondition> conditions = condition.getConditions();
+		List<String> keys = conditions.stream().map(mid -> mid.getFieldMark()).collect(Collectors.toList());
+		Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
+		List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
+		for (File midFile : fileList) {
+			if (!midFile.getName().endsWith("json")) {
+				continue;
+			}
+			try {
+				BufferedReader reader=new BufferedReader(new FileReader(midFile));
+				boolean first=true;
+				Exception exception=null;
+				try {
+					String line = reader.readLine();
+					while (line != null) {
+						List<String> cellList=new ArrayList<>();
+						JSONObject jsonObject = JSONObject.parseObject(line);
+						for (String key : keys) {
+							String cellVal = jsonObject.getString(key);
+							cellList.add(cellVal);
+						}
+						String[] cells = cellList.toArray(new String[]{});
+						func.deal(cells,conditions,first);
+						first=false;
+						line = reader.readLine();
+					}
+					logger.info("读取结束：{}",midFile.getName());
+				} catch (Exception e) {
+					exception=e;
+				}finally {
+					reader.close();
+				}
+				if (exception!=null) {
+					throw exception;
+				}
+			} catch (Exception e) {
+				logger.info("读取异常：{}",e);
+				throw new RuntimeException(e);
+			}
+		}
+	}
+
+
 
 	private void takeFile(FlatQueryTaskCondition condition, Connection conn, int downloadType){
 		if (condition != null) {

gic-cloud-data-hook-service/src/test/java/com/gic/ExceTest2.java

@@ -11,6 +11,9 @@ import com.gic.cloud.data.hook.service.FileUtil;
 import com.gic.cloud.data.hook.service.entity.ColumnInfo;
 import com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl;
 import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+import com.opencsv.RFC4180Parser;
+import com.opencsv.RFC4180ParserBuilder;
 import org.apache.commons.lang.time.DateUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.poi.ss.usermodel.*;
@@ -45,9 +48,12 @@ public class ExceTest2 {
             if (!midFile.getName().endsWith("csv")) {
                 continue;
             }
+            String lastcells=null;
+            String[] cells=null;
             try {
-                CSVReader csvReader = new CSVReader(new FileReader(midFile));
-                String[] cells = csvReader.readNext();
+                RFC4180Parser rfc4180Parser = new RFC4180ParserBuilder().build();
+                CSVReader csvReader = new CSVReaderBuilder(new FileReader(midFile)).build();
+                 cells = csvReader.readNext();
                 boolean first=true;
                 do {
                     if (titles == null) {
@@ -62,10 +68,14 @@ public class ExceTest2 {
                         continue;
                     }
                     func.deal(cells,titles,first);
+                    lastcells=JSONObject.toJSONString(cells);
                 }while ((cells = csvReader.readNext())!=null);
                 logger.info("读取结束：{}",midFile.getName());
                 csvReader.close();
             } catch (Exception e) {
+                logger.info("lastcell: {}",lastcells);
+                logger.info("cells: {}",cells[0]);
+                logger.info("cells: {},{},{}",cells[1]);
                 logger.info("读取异常：{}",e);
             }
         }