Merge branch 'fix-2023-04' into 'master'

Fix 2023 04

See merge request !100

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DingWarningUtil.java

+package com.gic.cloud.data.hook.service;
+
+import com.ctrip.framework.apollo.Config;
+import com.ctrip.framework.apollo.ConfigService;
+import com.gic.commons.util.DingtalkMessageUtil;
+import com.gic.commons.util.TraceIdUtil;
+import com.gic.dubbo.entity.ProviderLocalTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DingWarningUtil {
+    private static Logger logger = LoggerFactory.getLogger(DingWarningUtil.class);
+    private static final String dingdingurl="https://oapi.dingtalk.com/robot/send?access_token=1d6a0756fcfe5f215e3ed37d7e60960635d0009f71b476f3b402e7671fc2a8ca";
+
+
+    public static void sendCountWarning(String msg) {
+        Config appConfig = ConfigService.getAppConfig();
+        String url = appConfig.getProperty("ding.url", dingdingurl);
+        ProviderLocalTag localTag = ProviderLocalTag.tag.get();
+        String traceId = localTag.traceId;
+        try {
+            msg=msg+": "+traceId;
+            DingtalkMessageUtil.sendAlertMessage(msg,url);
+        }catch (Exception e) {
+               logger.info("异常：{}",e);
+        }
+    }
+}

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/HDFSUtil.java

@@ -16,6 +16,8 @@ public class HDFSUtil {
     private static HDFSUtil hdfsUtil=null;
     private static FileSystem fileSystem=null;
 
+    public static final String HDFS_URL = "/data/hook";
+
     public static HDFSUtil getInstance(){
         if (hdfsUtil==null) {
             synchronized (HDFSUtil.class) {
@@ -56,4 +58,26 @@ public class HDFSUtil {
             return false;
         }
     }
+
+    public boolean deleteFile(String srcPath){
+        try {
+            fileSystem.delete(new Path(srcPath),true);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return true;
+    }
+
+    public String getHdfsName(String id){
+        //下载文件
+        String dirName="hdfs"+id;
+        return dirName;
+    }
+
+    public String getHdfsPath(String id){
+        //下载文件
+        String dirName=getHdfsName(id);
+        String path=HDFS_URL+"/"+dirName;
+        return path;
+    }
 }

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java

@@ -10,6 +10,7 @@ import com.gic.cloud.data.hook.api.entity.DownloadTaskStatus;
 import com.gic.cloud.data.hook.api.entity.FreeQueryTaskCondition;
 import com.gic.cloud.data.hook.api.entity.Global;
 import com.gic.cloud.data.hook.api.service.IDownloadTaskService;
+import com.gic.cloud.data.hook.service.HDFSUtil;
 import com.gic.cloud.data.hook.service.MysqlHelper;
 import com.gic.cloud.data.hook.service.dao.DownloadRecordDao;
 import com.gic.cloud.data.hook.service.dao.DownloadTaskDao;
@@ -165,6 +166,10 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
 			}
 			this.downloadTaskDao.deleteDownloadTask(task.getId());
 			CloudFileUtil.delFileByUrl(task.getFilePath());
+
+			String hdfsPath = HDFSUtil.getInstance().getHdfsPath(task.getId());
+			HDFSUtil.getInstance().deleteFile(hdfsPath);
+
 			return true;
 		} else return false;
 	}

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java

@@ -73,7 +73,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 	public static final String SAVE_FOLDER = "/usr/local/data-hook-file";
 //	public static final String SAVE_FOLDER = "D:\\testorder";
 
-	public static final String HDFS_URL = "/data/hook";
+//	public static final String HDFS_URL = "/data/hook";
 
 	public static final String LOCK_KEY="data:hook:hive";
 
@@ -919,8 +919,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 		task.setDownloadWay(-1);
 		task.setFieldSize(condition.getAllFields().size());
 		//下载文件
-		String dirName="hdfs"+task.getId();
-		String path=HDFS_URL+"/"+dirName;
+		String dirName=HDFSUtil.getInstance().getHdfsName(task.getId());
+		String path=HDFSUtil.getInstance().getHdfsPath(task.getId());
+
+		HDFSUtil.getInstance().getHdfsName(task.getId());
 		try {
 			StopWatch stopWatch = StopWatch.create("down");
 			stopWatch.start();
@@ -937,7 +939,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 			AtomicInteger totalCount=new AtomicInteger(0);
 			AtomicInteger count=new AtomicInteger(0);
 			AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
-			readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
+			readJsonFile(condition,dirName,(cells,titles,firstFlag)->{
 				if (count.get()==0) {
 					XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
 					xlsxFileInfo.filepath=SAVE_FOLDER + "/" + task.getId() + xlsxFiles.size() + ".xlsx";
@@ -953,6 +955,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 			saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType);
 			stopWatch.stop();
 			logger.info("写入本地excel耗时：{}, 数量: {}-》{}",stopWatch.getLastTaskTimeMillis(),task.getAmount(),total);
+			if (task.getAmount()!=total.intValue()) {
+				String msg="取数任务："+task.getId()+"下载数量对不上: 数量: "+task.getAmount()+"->"+total;
+				DingWarningUtil.sendCountWarning(msg);
+			}
 			stopWatch.start();
 			//是否压缩
 			boolean zipFlag = (xlsxFiles.size() > 1) ? true : false;
@@ -1071,6 +1077,58 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 	}
 
 
+	/**
+	 * 读物文件
+	 * @param dirName
+	 * @param func
+	 */
+	private void readJsonFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
+		File file = new File(SAVE_FOLDER+"/"+dirName);
+		File[] files = file.listFiles();
+		List<FlatQueryCondition> titles=null;
+		List<FlatQueryCondition> conditions = condition.getConditions();
+		List<String> keys = conditions.stream().map(mid -> mid.getFieldMark()).collect(Collectors.toList());
+		Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
+		List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
+		for (File midFile : fileList) {
+			if (!midFile.getName().endsWith("json")) {
+				continue;
+			}
+			try {
+				BufferedReader reader=new BufferedReader(new FileReader(midFile));
+				boolean first=true;
+				Exception exception=null;
+				try {
+					String line = reader.readLine();
+					while (line != null) {
+						List<String> cellList=new ArrayList<>();
+						JSONObject jsonObject = JSONObject.parseObject(line);
+						for (String key : keys) {
+							String cellVal = jsonObject.getString(key);
+							cellList.add(cellVal);
+						}
+						String[] cells = cellList.toArray(new String[]{});
+						func.deal(cells,conditions,first);
+						first=false;
+						line = reader.readLine();
+					}
+					logger.info("读取结束：{}",midFile.getName());
+				} catch (Exception e) {
+					exception=e;
+				}finally {
+					reader.close();
+				}
+				if (exception!=null) {
+					throw exception;
+				}
+			} catch (Exception e) {
+				logger.info("读取异常：{}",e);
+				throw new RuntimeException(e);
+			}
+		}
+	}
+
+
 
 	private void takeFile(FlatQueryTaskCondition condition, Connection conn, int downloadType){
 		if (condition != null) {

gic-cloud-data-hook-service/src/test/java/Test.java

@@ -42,7 +42,7 @@ public class Test {
 
     @org.junit.Test
     public void test2(){
-        String json="{\"allFields\":[\"is_mbr\",\"main_store_name\",\"main_store_code\",\"card_num\",\"open_card_time\",\"clerk_name\",\"add_clerk_code\",\"clerk_store_name\",\"clerk_store_code\",\"add_buddy_time\",\"create_time\"],\"amount\":1169963,\"authStoreIdList\":[],\"buildPermitted\":\"1\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员\",\"fieldName\":\"is_mbr\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"是\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店名称\",\"fieldName\":\"main_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店code\",\"fieldName\":\"main_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"open_card_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购姓名\",\"fieldName\":\"clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购code\",\"fieldName\":\"add_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"导购门店名称\",\"fieldName\":\"clerk_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购所属门店code\",\"fieldName\":\"clerk_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"2023-02-28 23:59:59\",\"fieldMark\":\"添加好友时间\",\"fieldName\":\"add_buddy_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"2010-03-01 00:00:00\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好办导购与客户建立好友关系时间\",\"fieldName\":\"create_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[],\"enterpriseIds\":[\"ff80808180b3c54a0180bc3df3bb4bca\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":0,\"tableId\":\"extract_entwechat_detail_d\",\"taskId\":\"1678674610424\"}";
+        String json="{\"allFields\":[\"use_time\",\"source_integral_log_id\",\"operation_integral_log_id\",\"member_name\",\"member_sxe\",\"phone_number\",\"card_num\",\"grade_name\",\"source_integral_change\",\"interval_remark\",\"remark\",\"integral_to_fee\",\"card_name\",\"card_code\",\"card_denomination\",\"cost_value\",\"status\",\"order_store_name\",\"order_store_code\",\"order_store_group_name\",\"order_store_type\",\"order_id\",\"enterprise_id\",\"integral_channel\",\"check_store_status\"],\"amount\":4509765,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗时间\",\"fieldName\":\"use_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"流水号\",\"fieldName\":\"source_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"关联流水号\",\"fieldName\":\"operation_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_sxe\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗额\",\"fieldName\":\"source_integral_change\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"事由\",\"fieldName\":\"interval_remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"备注\",\"fieldName\":\"remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"成本总额\",\"fieldName\":\"integral_to_fee\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券名称\",\"fieldName\":\"card_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券编码\",\"fieldName\":\"card_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券面额\",\"fieldName\":\"card_denomination\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券成本费用\",\"fieldName\":\"cost_value\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"优惠券状态\",\"fieldName\":\"status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店代码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店所属分组\",\"fieldName\":\"order_store_group_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"零售单号\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业id\",\"fieldName\":\"enterprise_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分来源渠道\",\"fieldName\":\"integral_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店状态\",\"fieldName\":\"check_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff8080817d9fbda8017dc20674f47fb6\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_integral_use_detail\",\"taskId\":\"1675670752773\"}";
         FlatQueryTaskCondition condition = JSON.parseObject(json, FlatQueryTaskCondition.class);
         String fullQuery = flatQueryResultService.buildFlatQuerySQL(
                 false, // 下载用途

gic-cloud-data-hook-service/src/test/java/com/gic/ExceTest2.java

@@ -8,9 +8,8 @@ import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
 import com.gic.cloud.data.hook.service.DecryptUtils;
 import com.gic.cloud.data.hook.service.DownloadFunc;
 import com.gic.cloud.data.hook.service.FileUtil;
-import com.gic.cloud.data.hook.service.entity.ColumnInfo;
-import com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl;
 import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
 import org.apache.commons.lang.time.DateUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.poi.ss.usermodel.*;
@@ -18,6 +17,7 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FileReader;
@@ -45,9 +45,11 @@ public class ExceTest2 {
             if (!midFile.getName().endsWith("csv")) {
                 continue;
             }
+            String lastcells=null;
+            String[] cells=null;
             try {
-                CSVReader csvReader = new CSVReader(new FileReader(midFile));
-                String[] cells = csvReader.readNext();
+                CSVReader csvReader = new CSVReaderBuilder(new FileReader(midFile)).build();
+                 cells = csvReader.readNext();
                 boolean first=true;
                 do {
                     if (titles == null) {
@@ -62,15 +64,70 @@ public class ExceTest2 {
                         continue;
                     }
                     func.deal(cells,titles,first);
+                    lastcells=JSONObject.toJSONString(cells);
                 }while ((cells = csvReader.readNext())!=null);
                 logger.info("读取结束：{}",midFile.getName());
                 csvReader.close();
             } catch (Exception e) {
+                logger.info("lastcell: {}",lastcells);
+                logger.info("cells: {}",cells[0]);
+                logger.info("cells: {},{},{}",cells[1]);
                 logger.info("读取异常：{}",e);
             }
         }
     }
 
+    /**
+     * 读物文件
+     * @param dirName
+     * @param func
+     */
+    private static void readJsonFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
+        File file = new File(SAVE_FOLDER+"/"+dirName);
+        File[] files = file.listFiles();
+        List<FlatQueryCondition> titles=null;
+        List<FlatQueryCondition> conditions = condition.getConditions();
+        List<String> keys = conditions.stream().map(mid -> mid.getFieldMark()).collect(Collectors.toList());
+        Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
+        List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
+        for (File midFile : fileList) {
+            if (!midFile.getName().endsWith("json")) {
+                continue;
+            }
+            try {
+                BufferedReader reader=new BufferedReader(new FileReader(midFile));
+                boolean first=true;
+                Exception exception=null;
+                try {
+                    String line = reader.readLine();
+                    while (line != null) {
+                        List<String> cellList=new ArrayList<>();
+                        JSONObject jsonObject = JSONObject.parseObject(line);
+                        for (String key : keys) {
+                            String cellVal = jsonObject.getString(key);
+                            cellList.add(cellVal);
+                        }
+                        String[] cells = cellList.toArray(new String[]{});
+                        func.deal(cells,conditions,first);
+                        first=false;
+                        line = reader.readLine();
+                    }
+                    logger.info("读取结束：{}",midFile.getName());
+                } catch (Exception e) {
+                    exception=e;
+                }finally {
+                    reader.close();
+                }
+                if (exception!=null) {
+                    throw exception;
+                }
+            } catch (Exception e) {
+                logger.info("读取异常：{}",e);
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
     public static class XlsxFileInfo{
         String filepath;
         SXSSFWorkbook workbook;
@@ -178,7 +235,7 @@ public class ExceTest2 {
         List<String> xlsxFiles=new ArrayList<>();
         AtomicInteger count=new AtomicInteger(0);
         AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
-        readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
+        readJsonFile(condition,dirName,(cells,titles,firstFlag)->{
             if (count.get()==0) {
                 XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
                 xlsxFileInfo.filepath=SAVE_FOLDER + "/" + taskId + xlsxFiles.size() + ".xlsx";