Skip to content

G
gic-cloud
Commit b6641bae by fudahua
Options

feat: 取数平台过滤项

parent e7679764
Showing with 40 additions and 27 deletions
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
...@@ -934,6 +934,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -934,6 +934,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
logger.info("下载耗时：{}",stopWatch.getLastTaskTimeMillis()); logger.info("下载耗时：{}",stopWatch.getLastTaskTimeMillis());
stopWatch.start(); stopWatch.start();
List<String> xlsxFiles=new ArrayList<>(); List<String> xlsxFiles=new ArrayList<>();
AtomicInteger totalCount=new AtomicInteger(0);
AtomicInteger count=new AtomicInteger(0); AtomicInteger count=new AtomicInteger(0);
AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>(); AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{ readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
...@@ -946,9 +947,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -946,9 +947,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,queryDataType); saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,queryDataType);
}); });
//结束 //结束
Integer limitSize = FileUtil.getLimitSize();
Integer total= (xlsxFiles.size()-1)*limitSize+count.get();
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType); saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType);
stopWatch.stop(); stopWatch.stop();
logger.info("写入本地excel耗时：{}",stopWatch.getLastTaskTimeMillis()); logger.info("写入本地excel耗时：{}, 数量: {}-》{}",stopWatch.getLastTaskTimeMillis(),task.getAmount(),total);
stopWatch.start(); stopWatch.start();
//是否压缩 //是否压缩
boolean zipFlag = (xlsxFiles.size() > 1) ? true : false; boolean zipFlag = (xlsxFiles.size() > 1) ? true : false;
...@@ -974,14 +978,15 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -974,14 +978,15 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
task.setOverTime(new Date()); task.setOverTime(new Date());
task.setFilePath(cloudFileUrl); task.setFilePath(cloudFileUrl);
//删除文件
delFileOrDirByTaskId(task.getId());
}catch (Exception e) { }catch (Exception e) {
logger.info("异常：{}",e); logger.info("异常：{}",e);
task.setStatus(DownloadTaskStatus.ERROR); task.setStatus(DownloadTaskStatus.ERROR);
}finally { }finally {
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
//删除文件
delFileOrDirByTaskId(task.getId());
} }
} }
...@@ -1060,6 +1065,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -1060,6 +1065,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
csvReader.close(); csvReader.close();
} catch (Exception e) { } catch (Exception e) {
logger.info("读取异常：{}",e); logger.info("读取异常：{}",e);
throw new RuntimeException(e);
} }
} }
} }
......
gic-cloud-data-hook-service/src/test/java/com/gic/ExceTest2.java
...@@ -5,9 +5,11 @@ import com.alibaba.fastjson.JSONObject; ...@@ -5,9 +5,11 @@ import com.alibaba.fastjson.JSONObject;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition; import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.api.entity.FlatQueryFieldType; import com.gic.cloud.data.hook.api.entity.FlatQueryFieldType;
import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition; import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
import com.gic.cloud.data.hook.service.DecryptUtils;
import com.gic.cloud.data.hook.service.DownloadFunc; import com.gic.cloud.data.hook.service.DownloadFunc;
import com.gic.cloud.data.hook.service.FileUtil; import com.gic.cloud.data.hook.service.FileUtil;
import com.gic.cloud.data.hook.service.entity.ColumnInfo; import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl;
import com.opencsv.CSVReader; import com.opencsv.CSVReader;
import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
...@@ -35,9 +37,9 @@ public class ExceTest2 { ...@@ -35,9 +37,9 @@ public class ExceTest2 {
private static void readCsvFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){ private static void readCsvFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName); File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles(); File[] files = file.listFiles();
List<ColumnInfo> titles=null; List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions(); List<FlatQueryCondition> conditions = condition.getConditions();
Map<String, ColumnInfo> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid -> new ColumnInfo(mid.getFieldType(), mid.getFieldMark()))); Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList()); List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) { for (File midFile : fileList) {
if (!midFile.getName().endsWith("csv")) { if (!midFile.getName().endsWith("csv")) {
...@@ -59,7 +61,7 @@ public class ExceTest2 { ...@@ -59,7 +61,7 @@ public class ExceTest2 {
first=false; first=false;
continue; continue;
} }
// func.deal(cells,titles,first); func.deal(cells,titles,first);
}while ((cells = csvReader.readNext())!=null); }while ((cells = csvReader.readNext())!=null);
logger.info("读取结束：{}",midFile.getName()); logger.info("读取结束：{}",midFile.getName());
csvReader.close(); csvReader.close();
...@@ -69,7 +71,7 @@ public class ExceTest2 { ...@@ -69,7 +71,7 @@ public class ExceTest2 {
} }
} }
private static class XlsxFileInfo{ public static class XlsxFileInfo{
String filepath; String filepath;
SXSSFWorkbook workbook; SXSSFWorkbook workbook;
CellStyle yyyyMMddhhmmss; CellStyle yyyyMMddhhmmss;
...@@ -81,30 +83,30 @@ public class ExceTest2 { ...@@ -81,30 +83,30 @@ public class ExceTest2 {
* @param originalFilePath * @param originalFilePath
* @param cells * @param cells
*/ */
private static void saveXlsSplitNew(String originalFilePath, String[] cells, List<ColumnInfo> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count, boolean endFlag) { private static void saveXlsSplitNew(String originalFilePath, String[] cells, List<FlatQueryCondition> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count, boolean endFlag, int queryDataType){
try { try {
if (xlsxFileInfo.workbook == null) { if (xlsxFileInfo.workbook==null) {
xlsxFileInfo.workbook = new SXSSFWorkbook(100); xlsxFileInfo.workbook=new SXSSFWorkbook(100);
Sheet sheet = xlsxFileInfo.workbook.createSheet(); Sheet sheet = xlsxFileInfo.workbook.createSheet();
Row row = sheet.createRow(0); Row row = sheet.createRow(0);
Cell cell; Cell cell;
logger.info("类型:{}", JSONObject.toJSONString(titles)); logger.info("类型:{}",JSONObject.toJSONString(titles));
for (int j = 0; j < titles.size(); j++) { for(int j =0; j<titles.size(); j++){
cell = row.createCell(j); cell = row.createCell(j);
cell.setCellValue(titles.get(j).getTitle()); cell.setCellValue(titles.get(j).getFieldMark());
} }
//日期 //日期
CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle(); CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle();
DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat(); DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss")); yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss"));
xlsxFileInfo.yyyyMMddhhmmss = yyyyMMddhhmmss; xlsxFileInfo.yyyyMMddhhmmss=yyyyMMddhhmmss;
//日期 //日期
CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle(); CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle();
DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat(); DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd")); yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
xlsxFileInfo.yyyyMMdd = yyyyMMdd; xlsxFileInfo.yyyyMMdd=yyyyMMdd;
} }
count.incrementAndGet(); count.incrementAndGet();
...@@ -114,7 +116,8 @@ public class ExceTest2 { ...@@ -114,7 +116,8 @@ public class ExceTest2 {
Row row = sheet.createRow(count.get()); Row row = sheet.createRow(count.get());
for (int j = 0; j < cells.length; j++) { for (int j = 0; j < cells.length; j++) {
String type = titles.get(j).getType(); FlatQueryCondition flatQueryCondition = titles.get(j);
String type = titles.get(j).getFieldType();
Cell midCell = row.createCell(j); Cell midCell = row.createCell(j);
String columnValue = cells[j]; String columnValue = cells[j];
if (StringUtils.isBlank(columnValue)) { if (StringUtils.isBlank(columnValue)) {
...@@ -139,35 +142,38 @@ public class ExceTest2 { ...@@ -139,35 +142,38 @@ public class ExceTest2 {
} }
break; break;
default: default:
midCell.setCellValue(columnValue); midCell.setCellValue(DecryptUtils.encryptionDeal(queryDataType,columnValue,flatQueryCondition));
} }
} }
} }
Integer limitSize = FileUtil.getLimitSize(); Integer limitSize = FileUtil.getLimitSize();
int c = count.get(); int c = count.get();
if (c % 5000 == 0) { if (c%5000==0) {
logger.info("进度：{}-{}", originalFilePath, c); logger.info("进度：{}-{}",originalFilePath,c);
} }
if (c >= limitSize.intValue() || endFlag) { if (c>=limitSize.intValue()||endFlag) {
logger.info("{}-count:{}", originalFilePath, count.get()); logger.info("{}-count:{}",originalFilePath,count.get());
FileOutputStream fileOut = new FileOutputStream(originalFilePath); FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut); wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close(); fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法 wb.dispose(); // SXSSFWorkbook 没有 close 方法
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + originalFilePath + ".xlsx"); logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + originalFilePath + ".xlsx");
count.set(0); count.set(0);
} }
} catch (Exception e) { }catch (Exception e) {
logger.info("异常：{}", e); logger.info("异常：{}",e);
throw new RuntimeException(e);
} }
} }
public static void main(String[] args) { public static void main(String[] args) {
//下载文件 //下载文件
String taskId="1680836264869"; String taskId="test";
String dirName="hdfs"+taskId; String dirName="hdfs"+taskId;
String conditionStr="{\"allFields\":[\"order_id\",\"is_member_order\",\"member_id\",\"order_time\",\"order_status\",\"receipts_date\",\"order_number\",\"oorder_number\",\"goods_count\",\"total_amount\",\"paid_amount\",\"pay_amount\",\"promotion_amount\",\"coupon_code\",\"using_integral\",\"get_integral\",\"order_type\",\"attention_status\",\"evaluate_status\",\"enterprise_name\",\"brand_name\",\"order_store_group\",\"order_store_code\",\"order_store_name\",\"order_store_type\",\"order_store_status\",\"order_clerk_code\",\"order_clerk_name\",\"order_grade_id\",\"order_grade_name\",\"order_age\",\"card_num\",\"member_name\",\"member_gender\",\"member_birthday\",\"phone_number\",\"grade_id\",\"grade_name\",\"card_giving_time\",\"auth_status\",\"auth_time\",\"wx_member\",\"pos_member\",\"p_open_card_channel\",\"open_store_code\",\"open_store_name\",\"open_clerk_code\",\"open_clerk_name\",\"is_new_member_order\",\"is_effective_order\",\"order_saler_group\",\"order_store_id\",\"clerk_id\",\"open_store_id\",\"order_store_tag_name\",\"open_store_tag_name\",\"open_store_group\",\"birth_month\",\"open_store_status\",\"channel_code\",\"qywx_flag\"],\"amount\":3243395,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单id\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员订单\",\"fieldName\":\"is_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员号\",\"fieldName\":\"member_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单时间（录单）\",\"fieldName\":\"order_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单类型\",\"fieldName\":\"order_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"实际消费日期（统计）\",\"fieldName\":\"receipts_date\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单编码\",\"fieldName\":\"order_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"原订单编码\",\"fieldName\":\"oorder_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单商品数量\",\"fieldName\":\"goods_count\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"吊牌价\",\"fieldName\":\"total_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售额（ERP）\",\"fieldName\":\"paid_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售实付（GIC计算）\",\"fieldName\":\"pay_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"整单活动优惠总额\",\"fieldName\":\"promotion_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡劵code\",\"fieldName\":\"coupon_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"使用积分\",\"fieldName\":\"using_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"获得积分（ERP）\",\"fieldName\":\"get_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单来源\",\"fieldName\":\"order_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否关注状态下订单\",\"fieldName\":\"attention_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"评价状态\",\"fieldName\":\"evaluate_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业名称\",\"fieldName\":\"enterprise_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"品牌名称\",\"fieldName\":\"brand_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店分组\",\"fieldName\":\"order_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店编码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店名称\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店状态\",\"fieldName\":\"order_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员编码\",\"fieldName\":\"order_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员姓名\",\"fieldName\":\"order_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级id\",\"fieldName\":\"order_grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级名称\",\"fieldName\":\"order_grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费年龄\",\"fieldName\":\"order_age\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡编码\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_gender\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员生日\",\"fieldName\":\"member_birthday\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级id\",\"fieldName\":\"grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"card_giving_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否已认证\",\"fieldName\":\"auth_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"认证时间\",\"fieldName\":\"auth_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否微信会员\",\"fieldName\":\"wx_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否POS会员\",\"fieldName\":\"pos_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡渠道\",\"fieldName\":\"p_open_card_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店编码\",\"fieldName\":\"open_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店\",\"fieldName\":\"open_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员编码\",\"fieldName\":\"open_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员\",\"fieldName\":\"open_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"新老会员订单\",\"fieldName\":\"is_new_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否有效订单\",\"fieldName\":\"is_effective_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单导购组\",\"fieldName\":\"order_saler_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店id\",\"fieldName\":\"order_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单主导购\",\"fieldName\":\"clerk_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店id\",\"fieldName\":\"open_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店标签\",\"fieldName\":\"order_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店标签\",\"fieldName\":\"open_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店分组\",\"fieldName\":\"open_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"生日月份\",\"fieldName\":\"birth_month\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店状态\",\"fieldName\":\"open_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"渠道编码\",\"fieldName\":\"channel_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否企业微信好友\",\"fieldName\":\"qywx_flag\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff80808162ad5aee0162ae7399dc002b\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_order_information\",\"taskId\":\"1659940224293\"}"; String conditionStr="{\"allFields\":[\"wechat_nickname\",\"add_buddy_time\",\"phone_number\",\"open_card_time\",\"clerk_name\",\"clerk_store_name\",\"main_store_name\"],\"amount\":627549,\"authStoreIdList\":[],\"buildPermitted\":\"1\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"微信昵称\",\"fieldName\":\"wechat_nickname\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"添加好友时间\",\"fieldName\":\"add_buddy_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"会员手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"open_card_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购姓名\",\"fieldName\":\"clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"导购门店名称\",\"fieldName\":\"clerk_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店名称\",\"fieldName\":\"main_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"会员手机号码\"],\"enterpriseIds\":[\"ff8080815c1206fa015c1554c27701b3\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":0,\"tableId\":\"extract_entwechat_detail_d\",\"taskId\":\"1682325929774\"}";
FlatQueryTaskCondition condition = JSON.parseObject(conditionStr, FlatQueryTaskCondition.class); FlatQueryTaskCondition condition = JSON.parseObject(conditionStr, FlatQueryTaskCondition.class);
List<String> xlsxFiles=new ArrayList<>(); List<String> xlsxFiles=new ArrayList<>();
AtomicInteger count=new AtomicInteger(0); AtomicInteger count=new AtomicInteger(0);
...@@ -175,11 +181,12 @@ public class ExceTest2 { ...@@ -175,11 +181,12 @@ public class ExceTest2 {
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{ readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
if (count.get()==0) { if (count.get()==0) {
XlsxFileInfo xlsxFileInfo = new XlsxFileInfo(); XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
xlsxFileInfo.filepath=SAVE_FOLDER + "/" + taskId+ xlsxFiles.size() + ".xlsx"; xlsxFileInfo.filepath=SAVE_FOLDER + "/" + taskId + xlsxFiles.size() + ".xlsx";
currentFile.set(xlsxFileInfo); currentFile.set(xlsxFileInfo);
xlsxFiles.add(currentFile.get().filepath); xlsxFiles.add(currentFile.get().filepath);
} }
// saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false); saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,1);
}); });
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,1);
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment