feat: 新版本的下载

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java

@@ -893,7 +893,6 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 					xlsxFiles.add(currentFile.get().filepath);
 				}
 				saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false);
-				count.incrementAndGet();
 			});
 			//结束
 			saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true);
@@ -1225,8 +1224,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 				DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
 				yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
 				xlsxFileInfo.yyyyMMdd=yyyyMMdd;
-				count.incrementAndGet();
 			}
+			count.incrementAndGet();
 
 			SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
 			if (!endFlag) {
@@ -1265,7 +1264,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 			}
 			Integer limitSize = FileUtil.getLimitSize();
 			int c = count.get();
-			if (c>limitSize||endFlag) {
+			if (c>=limitSize.intValue()||endFlag) {
 				logger.info("{}-count:{}",originalFilePath,count.get());
 				FileOutputStream fileOut = new FileOutputStream(originalFilePath);
 				wb.write(fileOut);

gic-cloud-data-hook-service/src/test/java/ExcelTest.java

@@ -21,10 +21,6 @@ public class ExcelTest {
             DataFormat format = wb.createDataFormat();
             cellStyle.setDataFormat(format.getFormat("yyyy-MM-dd HH:mm:ss"));
 
-//            CellStyle cellStyle = wb.createCellStyle();
-//            DataFormat format = wb.createDataFormat();
-//            cellStyle.setDataFormat(format.getFormat("yyyy-MM-dd HH:mm:ss"));
-
             int n=0;
             do {
                 Row row = sheet.createRow(n);

gic-cloud-data-hook-service/src/test/java/Test.java

+import com.alibaba.fastjson.JSON;
 import com.gic.cloud.data.hook.api.dto.DownloadTask;
+import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
 import com.gic.cloud.data.hook.service.impl.DownloadTaskServiceImpl;
 import com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl;
 import org.junit.runner.RunWith;
@@ -19,6 +21,7 @@ public class Test {
     @org.junit.Test
     public void test(){
         DownloadTask downloadTask = downloadTaskService.getDownloadTaskById("1680844420436");
+
         flatQueryResultService.takeFileNew(downloadTask);
         try {
             Thread.sleep(10000000000L);
@@ -26,4 +29,22 @@ public class Test {
             e.printStackTrace();
         }
     }
+
+    @org.junit.Test
+    public void test2(){
+        String json="{\"allFields\":[\"order_id\",\"is_member_order\",\"member_id\",\"order_time\",\"order_status\",\"receipts_date\",\"order_number\",\"oorder_number\",\"goods_count\",\"total_amount\",\"paid_amount\",\"pay_amount\",\"promotion_amount\",\"coupon_code\",\"using_integral\",\"get_integral\",\"order_type\",\"attention_status\",\"evaluate_status\",\"enterprise_name\",\"brand_name\",\"order_store_group\",\"order_store_code\",\"order_store_name\",\"order_store_type\",\"order_store_status\",\"order_clerk_code\",\"order_clerk_name\",\"order_grade_id\",\"order_grade_name\",\"order_age\",\"card_num\",\"member_name\",\"member_gender\",\"member_birthday\",\"phone_number\",\"grade_id\",\"grade_name\",\"card_giving_time\",\"auth_status\",\"auth_time\",\"wx_member\",\"pos_member\",\"p_open_card_channel\",\"open_store_code\",\"open_store_name\",\"open_clerk_code\",\"open_clerk_name\",\"is_new_member_order\",\"is_effective_order\",\"order_saler_group\",\"order_store_id\",\"clerk_id\",\"open_store_id\",\"order_store_tag_name\",\"open_store_tag_name\",\"open_store_group\",\"birth_month\",\"open_store_status\",\"channel_code\",\"qywx_flag\"],\"amount\":3243395,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单id\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员订单\",\"fieldName\":\"is_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员号\",\"fieldName\":\"member_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单时间（录单）\",\"fieldName\":\"order_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单类型\",\"fieldName\":\"order_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"实际消费日期（统计）\",\"fieldName\":\"receipts_date\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单编码\",\"fieldName\":\"order_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"原订单编码\",\"fieldName\":\"oorder_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单商品数量\",\"fieldName\":\"goods_count\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"吊牌价\",\"fieldName\":\"total_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售额（ERP）\",\"fieldName\":\"paid_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售实付（GIC计算）\",\"fieldName\":\"pay_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"整单活动优惠总额\",\"fieldName\":\"promotion_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡劵code\",\"fieldName\":\"coupon_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"使用积分\",\"fieldName\":\"using_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"获得积分（ERP）\",\"fieldName\":\"get_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单来源\",\"fieldName\":\"order_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否关注状态下订单\",\"fieldName\":\"attention_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"评价状态\",\"fieldName\":\"evaluate_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业名称\",\"fieldName\":\"enterprise_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"品牌名称\",\"fieldName\":\"brand_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店分组\",\"fieldName\":\"order_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店编码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店名称\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店状态\",\"fieldName\":\"order_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员编码\",\"fieldName\":\"order_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员姓名\",\"fieldName\":\"order_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级id\",\"fieldName\":\"order_grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级名称\",\"fieldName\":\"order_grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费年龄\",\"fieldName\":\"order_age\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡编码\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_gender\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员生日\",\"fieldName\":\"member_birthday\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级id\",\"fieldName\":\"grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"card_giving_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否已认证\",\"fieldName\":\"auth_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"认证时间\",\"fieldName\":\"auth_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否微信会员\",\"fieldName\":\"wx_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否POS会员\",\"fieldName\":\"pos_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡渠道\",\"fieldName\":\"p_open_card_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店编码\",\"fieldName\":\"open_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店\",\"fieldName\":\"open_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员编码\",\"fieldName\":\"open_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员\",\"fieldName\":\"open_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"新老会员订单\",\"fieldName\":\"is_new_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否有效订单\",\"fieldName\":\"is_effective_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单导购组\",\"fieldName\":\"order_saler_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店id\",\"fieldName\":\"order_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单主导购\",\"fieldName\":\"clerk_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店id\",\"fieldName\":\"open_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店标签\",\"fieldName\":\"order_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店标签\",\"fieldName\":\"open_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店分组\",\"fieldName\":\"open_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"生日月份\",\"fieldName\":\"birth_month\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店状态\",\"fieldName\":\"open_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"渠道编码\",\"fieldName\":\"channel_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否企业微信好友\",\"fieldName\":\"qywx_flag\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff80808162ad5aee0162ae7399dc002b\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_order_information\",\"taskId\":\"1659940224293\"}";
+        FlatQueryTaskCondition condition = JSON.parseObject(json, FlatQueryTaskCondition.class);
+        String fullQuery = flatQueryResultService.buildFlatQuerySQL(
+                false, // 下载用途
+                condition.getTableId(),
+                condition.getEnterpriseIds(),
+                condition.getConditions(),
+                condition.getOrderField(),
+                condition.getOrderDir(),
+                condition.getExecDistinct(),
+                0,
+                condition.getAuthStoreIdList());
+
+        System.out.println(fullQuery);
+    }
 }

gic-cloud-data-hook-service/src/test/java/hdfs.java

@@ -20,13 +20,13 @@ public class hdfs {
 //            e.printStackTrace();
 //        }
 
-//        HDFSUtil.getInstance().downloadFile("/data/emr/order-1.csv","D:\\testorder");
-        Integer[] arr=new Integer[]{1,2,3,4,5,6};
-        Arrays.asList(arr).parallelStream().forEach(mid->{
-            Thread thread = Thread.currentThread();
-            String name = thread.getName();
-            System.out.println(name);
-        });
+        HDFSUtil.getInstance().downloadFile("/data/hook/hdfs1680836264869","D:\\testorder");
+//        Integer[] arr=new Integer[]{1,2,3,4,5,6};
+//        Arrays.asList(arr).parallelStream().forEach(mid->{
+//            Thread thread = Thread.currentThread();
+//            String name = thread.getName();
+//            System.out.println(name);
+//        });
 
 //        try{
 //            CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));