Skip to content

G
gic-cloud
Commit 2388ed1f by fudahua
Options

Merge branch 'fix-2023-04' into 'developer' 

Fix 2023 04

See merge request !98
parents 1d6a105e 9a40bf8f
Showing with 189 additions and 34 deletions
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/HDFSUtil.java
...@@ -16,6 +16,8 @@ public class HDFSUtil { ...@@ -16,6 +16,8 @@ public class HDFSUtil {
private static HDFSUtil hdfsUtil=null; private static HDFSUtil hdfsUtil=null;
private static FileSystem fileSystem=null; private static FileSystem fileSystem=null;
public static final String HDFS_URL = "/data/hook";
public static HDFSUtil getInstance(){ public static HDFSUtil getInstance(){
if (hdfsUtil==null) { if (hdfsUtil==null) {
synchronized (HDFSUtil.class) { synchronized (HDFSUtil.class) {
...@@ -47,11 +49,35 @@ public class HDFSUtil { ...@@ -47,11 +49,35 @@ public class HDFSUtil {
*/ */
public boolean downloadFile(String srcPath,String toPath) { public boolean downloadFile(String srcPath,String toPath) {
try { try {
fileSystem.copyToLocalFile(true,new Path(srcPath),new Path(toPath)); Config appConfig = ConfigService.getAppConfig();
Integer delFlag = appConfig.getIntProperty("del.hive.flag", 1);
fileSystem.copyToLocalFile(delFlag.intValue()==1?true:false,new Path(srcPath),new Path(toPath));
return true; return true;
} catch (IOException e) { } catch (IOException e) {
logger.info("下载失败：{}",e); logger.info("下载失败：{}",e);
return false; return false;
} }
} }
public boolean deleteFile(String srcPath){
try {
fileSystem.delete(new Path(srcPath),true);
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
public String getHdfsName(String id){
//下载文件
String dirName="hdfs"+id;
return dirName;
}
public String getHdfsPath(String id){
//下载文件
String dirName=getHdfsName(id);
String path=HDFS_URL+"/"+dirName;
return path;
}
} }
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java
...@@ -10,6 +10,7 @@ import com.gic.cloud.data.hook.api.entity.DownloadTaskStatus; ...@@ -10,6 +10,7 @@ import com.gic.cloud.data.hook.api.entity.DownloadTaskStatus;
import com.gic.cloud.data.hook.api.entity.FreeQueryTaskCondition; import com.gic.cloud.data.hook.api.entity.FreeQueryTaskCondition;
import com.gic.cloud.data.hook.api.entity.Global; import com.gic.cloud.data.hook.api.entity.Global;
import com.gic.cloud.data.hook.api.service.IDownloadTaskService; import com.gic.cloud.data.hook.api.service.IDownloadTaskService;
import com.gic.cloud.data.hook.service.HDFSUtil;
import com.gic.cloud.data.hook.service.MysqlHelper; import com.gic.cloud.data.hook.service.MysqlHelper;
import com.gic.cloud.data.hook.service.dao.DownloadRecordDao; import com.gic.cloud.data.hook.service.dao.DownloadRecordDao;
import com.gic.cloud.data.hook.service.dao.DownloadTaskDao; import com.gic.cloud.data.hook.service.dao.DownloadTaskDao;
...@@ -165,6 +166,10 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService { ...@@ -165,6 +166,10 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
} }
this.downloadTaskDao.deleteDownloadTask(task.getId()); this.downloadTaskDao.deleteDownloadTask(task.getId());
CloudFileUtil.delFileByUrl(task.getFilePath()); CloudFileUtil.delFileByUrl(task.getFilePath());
String hdfsPath = HDFSUtil.getInstance().getHdfsPath(task.getId());
HDFSUtil.getInstance().deleteFile(hdfsPath);
return true; return true;
} else return false; } else return false;
} }
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
...@@ -78,7 +78,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -78,7 +78,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
public static final String SAVE_FOLDER = "/usr/local/data-hook-file"; public static final String SAVE_FOLDER = "/usr/local/data-hook-file";
// public static final String SAVE_FOLDER = "D:\\testorder"; // public static final String SAVE_FOLDER = "D:\\testorder";
public static final String HDFS_URL = "/data/hook"; // public static final String HDFS_URL = "/data/hook";
public static final String LOCK_KEY="data:hook:hive"; public static final String LOCK_KEY="data:hook:hive";
...@@ -930,8 +930,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -930,8 +930,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
task.setDownloadWay(-1); task.setDownloadWay(-1);
task.setFieldSize(condition.getAllFields().size()); task.setFieldSize(condition.getAllFields().size());
//下载文件 //下载文件
String dirName="hdfs"+task.getId(); String dirName=HDFSUtil.getInstance().getHdfsName(task.getId());
String path=HDFS_URL+"/"+dirName; String path=HDFSUtil.getInstance().getHdfsPath(task.getId());
HDFSUtil.getInstance().getHdfsName(task.getId());
try { try {
StopWatch stopWatch = StopWatch.create("down"); StopWatch stopWatch = StopWatch.create("down");
stopWatch.start(); stopWatch.start();
...@@ -945,9 +947,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -945,9 +947,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
logger.info("下载耗时：{}",stopWatch.getLastTaskTimeMillis()); logger.info("下载耗时：{}",stopWatch.getLastTaskTimeMillis());
stopWatch.start(); stopWatch.start();
List<String> xlsxFiles=new ArrayList<>(); List<String> xlsxFiles=new ArrayList<>();
AtomicInteger totalCount=new AtomicInteger(0);
AtomicInteger count=new AtomicInteger(0); AtomicInteger count=new AtomicInteger(0);
AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>(); AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{ readJsonFile(condition,dirName,(cells,titles,firstFlag)->{
if (count.get()==0) { if (count.get()==0) {
XlsxFileInfo xlsxFileInfo = new XlsxFileInfo(); XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
xlsxFileInfo.filepath=SAVE_FOLDER + "/" + task.getId() + xlsxFiles.size() + ".xlsx"; xlsxFileInfo.filepath=SAVE_FOLDER + "/" + task.getId() + xlsxFiles.size() + ".xlsx";
...@@ -957,9 +960,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -957,9 +960,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,queryDataType); saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,queryDataType);
}); });
//结束 //结束
Integer limitSize = FileUtil.getLimitSize();
Integer total= (xlsxFiles.size()-1)*limitSize+count.get();
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType); saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType);
stopWatch.stop(); stopWatch.stop();
logger.info("写入本地excel耗时：{}",stopWatch.getLastTaskTimeMillis()); logger.info("写入本地excel耗时：{}, 数量: {}-》{}",stopWatch.getLastTaskTimeMillis(),task.getAmount(),total);
stopWatch.start(); stopWatch.start();
//是否压缩 //是否压缩
boolean zipFlag = (xlsxFiles.size() > 1) ? true : false; boolean zipFlag = (xlsxFiles.size() > 1) ? true : false;
...@@ -986,6 +992,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -986,6 +992,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
task.setFilePath(cloudFileUrl); task.setFilePath(cloudFileUrl);
}catch (Exception e) { }catch (Exception e) {
logger.info("异常：{}",e); logger.info("异常：{}",e);
task.setStatus(DownloadTaskStatus.ERROR); task.setStatus(DownloadTaskStatus.ERROR);
...@@ -1071,6 +1078,59 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -1071,6 +1078,59 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
csvReader.close(); csvReader.close();
} catch (Exception e) { } catch (Exception e) {
logger.info("读取异常：{}",e); logger.info("读取异常：{}",e);
throw new RuntimeException(e);
}
}
}
/**
* 读物文件
* @param dirName
* @param func
*/
private void readJsonFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles();
List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions();
List<String> keys = conditions.stream().map(mid -> mid.getFieldMark()).collect(Collectors.toList());
Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) {
if (!midFile.getName().endsWith("json")) {
continue;
}
try {
BufferedReader reader=new BufferedReader(new FileReader(midFile));
boolean first=true;
Exception exception=null;
try {
String line = reader.readLine();
while (line != null) {
List<String> cellList=new ArrayList<>();
JSONObject jsonObject = JSONObject.parseObject(line);
for (String key : keys) {
String cellVal = jsonObject.getString(key);
cellList.add(cellVal);
}
String[] cells = cellList.toArray(new String[]{});
func.deal(cells,conditions,first);
first=false;
line = reader.readLine();
}
logger.info("读取结束：{}",midFile.getName());
} catch (Exception e) {
exception=e;
}finally {
reader.close();
}
if (exception!=null) {
throw exception;
}
} catch (Exception e) {
logger.info("读取异常：{}",e);
throw new RuntimeException(e);
} }
} }
} }
......
gic-cloud-data-hook-service/src/test/java/com/gic/ExceTest2.java
...@@ -5,10 +5,11 @@ import com.alibaba.fastjson.JSONObject; ...@@ -5,10 +5,11 @@ import com.alibaba.fastjson.JSONObject;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition; import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.api.entity.FlatQueryFieldType; import com.gic.cloud.data.hook.api.entity.FlatQueryFieldType;
import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition; import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
import com.gic.cloud.data.hook.service.DecryptUtils;
import com.gic.cloud.data.hook.service.DownloadFunc; import com.gic.cloud.data.hook.service.DownloadFunc;
import com.gic.cloud.data.hook.service.FileUtil; import com.gic.cloud.data.hook.service.FileUtil;
import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import com.opencsv.CSVReader; import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.*; import org.apache.poi.ss.usermodel.*;
...@@ -16,6 +17,7 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook; ...@@ -16,6 +17,7 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileReader; import java.io.FileReader;
...@@ -35,17 +37,19 @@ public class ExceTest2 { ...@@ -35,17 +37,19 @@ public class ExceTest2 {
private static void readCsvFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){ private static void readCsvFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName); File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles(); File[] files = file.listFiles();
List<ColumnInfo> titles=null; List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions(); List<FlatQueryCondition> conditions = condition.getConditions();
Map<String, ColumnInfo> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid -> new ColumnInfo(mid.getFieldType(), mid.getFieldMark()))); Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList()); List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) { for (File midFile : fileList) {
if (!midFile.getName().endsWith("csv")) { if (!midFile.getName().endsWith("csv")) {
continue; continue;
} }
String lastcells=null;
String[] cells=null;
try { try {
CSVReader csvReader = new CSVReader(new FileReader(midFile)); CSVReader csvReader = new CSVReaderBuilder(new FileReader(midFile)).build();
String[] cells = csvReader.readNext(); cells = csvReader.readNext();
boolean first=true; boolean first=true;
do { do {
if (titles == null) { if (titles == null) {
...@@ -59,17 +63,72 @@ public class ExceTest2 { ...@@ -59,17 +63,72 @@ public class ExceTest2 {
first=false; first=false;
continue; continue;
} }
// func.deal(cells,titles,first); func.deal(cells,titles,first);
lastcells=JSONObject.toJSONString(cells);
}while ((cells = csvReader.readNext())!=null); }while ((cells = csvReader.readNext())!=null);
logger.info("读取结束：{}",midFile.getName()); logger.info("读取结束：{}",midFile.getName());
csvReader.close(); csvReader.close();
} catch (Exception e) { } catch (Exception e) {
logger.info("lastcell: {}",lastcells);
logger.info("cells: {}",cells[0]);
logger.info("cells: {},{},{}",cells[1]);
logger.info("读取异常：{}",e); logger.info("读取异常：{}",e);
} }
} }
} }
private static class XlsxFileInfo{ /**
* 读物文件
* @param dirName
* @param func
*/
private static void readJsonFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles();
List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions();
List<String> keys = conditions.stream().map(mid -> mid.getFieldMark()).collect(Collectors.toList());
Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) {
if (!midFile.getName().endsWith("json")) {
continue;
}
try {
BufferedReader reader=new BufferedReader(new FileReader(midFile));
boolean first=true;
Exception exception=null;
try {
String line = reader.readLine();
while (line != null) {
List<String> cellList=new ArrayList<>();
JSONObject jsonObject = JSONObject.parseObject(line);
for (String key : keys) {
String cellVal = jsonObject.getString(key);
cellList.add(cellVal);
}
String[] cells = cellList.toArray(new String[]{});
func.deal(cells,conditions,first);
first=false;
line = reader.readLine();
}
logger.info("读取结束：{}",midFile.getName());
} catch (Exception e) {
exception=e;
}finally {
reader.close();
}
if (exception!=null) {
throw exception;
}
} catch (Exception e) {
logger.info("读取异常：{}",e);
throw new RuntimeException(e);
}
}
}
public static class XlsxFileInfo{
String filepath; String filepath;
SXSSFWorkbook workbook; SXSSFWorkbook workbook;
CellStyle yyyyMMddhhmmss; CellStyle yyyyMMddhhmmss;
...@@ -81,30 +140,30 @@ public class ExceTest2 { ...@@ -81,30 +140,30 @@ public class ExceTest2 {
* @param originalFilePath * @param originalFilePath
* @param cells * @param cells
*/ */
private static void saveXlsSplitNew(String originalFilePath, String[] cells, List<ColumnInfo> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count, boolean endFlag) { private static void saveXlsSplitNew(String originalFilePath, String[] cells, List<FlatQueryCondition> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count, boolean endFlag, int queryDataType){
try { try {
if (xlsxFileInfo.workbook == null) { if (xlsxFileInfo.workbook==null) {
xlsxFileInfo.workbook = new SXSSFWorkbook(100); xlsxFileInfo.workbook=new SXSSFWorkbook(100);
Sheet sheet = xlsxFileInfo.workbook.createSheet(); Sheet sheet = xlsxFileInfo.workbook.createSheet();
Row row = sheet.createRow(0); Row row = sheet.createRow(0);
Cell cell; Cell cell;
logger.info("类型:{}", JSONObject.toJSONString(titles)); logger.info("类型:{}",JSONObject.toJSONString(titles));
for (int j = 0; j < titles.size(); j++) { for(int j =0; j<titles.size(); j++){
cell = row.createCell(j); cell = row.createCell(j);
cell.setCellValue(titles.get(j).getTitle()); cell.setCellValue(titles.get(j).getFieldMark());
} }
//日期 //日期
CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle(); CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle();
DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat(); DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss")); yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss"));
xlsxFileInfo.yyyyMMddhhmmss = yyyyMMddhhmmss; xlsxFileInfo.yyyyMMddhhmmss=yyyyMMddhhmmss;
//日期 //日期
CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle(); CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle();
DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat(); DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd")); yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
xlsxFileInfo.yyyyMMdd = yyyyMMdd; xlsxFileInfo.yyyyMMdd=yyyyMMdd;
} }
count.incrementAndGet(); count.incrementAndGet();
...@@ -114,7 +173,8 @@ public class ExceTest2 { ...@@ -114,7 +173,8 @@ public class ExceTest2 {
Row row = sheet.createRow(count.get()); Row row = sheet.createRow(count.get());
for (int j = 0; j < cells.length; j++) { for (int j = 0; j < cells.length; j++) {
String type = titles.get(j).getType(); FlatQueryCondition flatQueryCondition = titles.get(j);
String type = titles.get(j).getFieldType();
Cell midCell = row.createCell(j); Cell midCell = row.createCell(j);
String columnValue = cells[j]; String columnValue = cells[j];
if (StringUtils.isBlank(columnValue)) { if (StringUtils.isBlank(columnValue)) {
...@@ -139,47 +199,51 @@ public class ExceTest2 { ...@@ -139,47 +199,51 @@ public class ExceTest2 {
} }
break; break;
default: default:
midCell.setCellValue(columnValue); midCell.setCellValue(DecryptUtils.encryptionDeal(queryDataType,columnValue,flatQueryCondition));
} }
} }
} }
Integer limitSize = FileUtil.getLimitSize(); Integer limitSize = FileUtil.getLimitSize();
int c = count.get(); int c = count.get();
if (c % 5000 == 0) { if (c%5000==0) {
logger.info("进度：{}-{}", originalFilePath, c); logger.info("进度：{}-{}",originalFilePath,c);
} }
if (c >= limitSize.intValue() || endFlag) { if (c>=limitSize.intValue()||endFlag) {
logger.info("{}-count:{}", originalFilePath, count.get()); logger.info("{}-count:{}",originalFilePath,count.get());
FileOutputStream fileOut = new FileOutputStream(originalFilePath); FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut); wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close(); fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法 wb.dispose(); // SXSSFWorkbook 没有 close 方法
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + originalFilePath + ".xlsx"); logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + originalFilePath + ".xlsx");
count.set(0); count.set(0);
} }
} catch (Exception e) { }catch (Exception e) {
logger.info("异常：{}", e); logger.info("异常：{}",e);
throw new RuntimeException(e);
} }
} }
public static void main(String[] args) { public static void main(String[] args) {
//下载文件 //下载文件
String taskId="1680836264869"; String taskId="test";
String dirName="hdfs"+taskId; String dirName="hdfs"+taskId;
String conditionStr="{\"allFields\":[\"order_id\",\"is_member_order\",\"member_id\",\"order_time\",\"order_status\",\"receipts_date\",\"order_number\",\"oorder_number\",\"goods_count\",\"total_amount\",\"paid_amount\",\"pay_amount\",\"promotion_amount\",\"coupon_code\",\"using_integral\",\"get_integral\",\"order_type\",\"attention_status\",\"evaluate_status\",\"enterprise_name\",\"brand_name\",\"order_store_group\",\"order_store_code\",\"order_store_name\",\"order_store_type\",\"order_store_status\",\"order_clerk_code\",\"order_clerk_name\",\"order_grade_id\",\"order_grade_name\",\"order_age\",\"card_num\",\"member_name\",\"member_gender\",\"member_birthday\",\"phone_number\",\"grade_id\",\"grade_name\",\"card_giving_time\",\"auth_status\",\"auth_time\",\"wx_member\",\"pos_member\",\"p_open_card_channel\",\"open_store_code\",\"open_store_name\",\"open_clerk_code\",\"open_clerk_name\",\"is_new_member_order\",\"is_effective_order\",\"order_saler_group\",\"order_store_id\",\"clerk_id\",\"open_store_id\",\"order_store_tag_name\",\"open_store_tag_name\",\"open_store_group\",\"birth_month\",\"open_store_status\",\"channel_code\",\"qywx_flag\"],\"amount\":3243395,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单id\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员订单\",\"fieldName\":\"is_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员号\",\"fieldName\":\"member_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单时间（录单）\",\"fieldName\":\"order_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单类型\",\"fieldName\":\"order_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"实际消费日期（统计）\",\"fieldName\":\"receipts_date\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单编码\",\"fieldName\":\"order_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"原订单编码\",\"fieldName\":\"oorder_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单商品数量\",\"fieldName\":\"goods_count\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"吊牌价\",\"fieldName\":\"total_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售额（ERP）\",\"fieldName\":\"paid_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售实付（GIC计算）\",\"fieldName\":\"pay_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"整单活动优惠总额\",\"fieldName\":\"promotion_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡劵code\",\"fieldName\":\"coupon_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"使用积分\",\"fieldName\":\"using_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"获得积分（ERP）\",\"fieldName\":\"get_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单来源\",\"fieldName\":\"order_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否关注状态下订单\",\"fieldName\":\"attention_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"评价状态\",\"fieldName\":\"evaluate_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业名称\",\"fieldName\":\"enterprise_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"品牌名称\",\"fieldName\":\"brand_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店分组\",\"fieldName\":\"order_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店编码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店名称\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店状态\",\"fieldName\":\"order_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员编码\",\"fieldName\":\"order_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员姓名\",\"fieldName\":\"order_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级id\",\"fieldName\":\"order_grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级名称\",\"fieldName\":\"order_grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费年龄\",\"fieldName\":\"order_age\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡编码\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_gender\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员生日\",\"fieldName\":\"member_birthday\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级id\",\"fieldName\":\"grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"card_giving_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否已认证\",\"fieldName\":\"auth_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"认证时间\",\"fieldName\":\"auth_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否微信会员\",\"fieldName\":\"wx_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否POS会员\",\"fieldName\":\"pos_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡渠道\",\"fieldName\":\"p_open_card_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店编码\",\"fieldName\":\"open_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店\",\"fieldName\":\"open_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员编码\",\"fieldName\":\"open_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员\",\"fieldName\":\"open_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"新老会员订单\",\"fieldName\":\"is_new_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否有效订单\",\"fieldName\":\"is_effective_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单导购组\",\"fieldName\":\"order_saler_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店id\",\"fieldName\":\"order_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单主导购\",\"fieldName\":\"clerk_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店id\",\"fieldName\":\"open_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店标签\",\"fieldName\":\"order_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店标签\",\"fieldName\":\"open_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店分组\",\"fieldName\":\"open_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"生日月份\",\"fieldName\":\"birth_month\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店状态\",\"fieldName\":\"open_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"渠道编码\",\"fieldName\":\"channel_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否企业微信好友\",\"fieldName\":\"qywx_flag\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff80808162ad5aee0162ae7399dc002b\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_order_information\",\"taskId\":\"1659940224293\"}"; String conditionStr="{\"allFields\":[\"wechat_nickname\",\"add_buddy_time\",\"phone_number\",\"open_card_time\",\"clerk_name\",\"clerk_store_name\",\"main_store_name\"],\"amount\":627549,\"authStoreIdList\":[],\"buildPermitted\":\"1\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"微信昵称\",\"fieldName\":\"wechat_nickname\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"添加好友时间\",\"fieldName\":\"add_buddy_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"会员手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"open_card_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购姓名\",\"fieldName\":\"clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"导购门店名称\",\"fieldName\":\"clerk_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店名称\",\"fieldName\":\"main_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"会员手机号码\"],\"enterpriseIds\":[\"ff8080815c1206fa015c1554c27701b3\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":0,\"tableId\":\"extract_entwechat_detail_d\",\"taskId\":\"1682325929774\"}";
FlatQueryTaskCondition condition = JSON.parseObject(conditionStr, FlatQueryTaskCondition.class); FlatQueryTaskCondition condition = JSON.parseObject(conditionStr, FlatQueryTaskCondition.class);
List<String> xlsxFiles=new ArrayList<>(); List<String> xlsxFiles=new ArrayList<>();
AtomicInteger count=new AtomicInteger(0); AtomicInteger count=new AtomicInteger(0);
AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>(); AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{ readJsonFile(condition,dirName,(cells,titles,firstFlag)->{
if (count.get()==0) { if (count.get()==0) {
XlsxFileInfo xlsxFileInfo = new XlsxFileInfo(); XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
xlsxFileInfo.filepath=SAVE_FOLDER + "/" + taskId+ xlsxFiles.size() + ".xlsx"; xlsxFileInfo.filepath=SAVE_FOLDER + "/" + taskId + xlsxFiles.size() + ".xlsx";
currentFile.set(xlsxFileInfo); currentFile.set(xlsxFileInfo);
xlsxFiles.add(currentFile.get().filepath); xlsxFiles.add(currentFile.get().filepath);
} }
// saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false); saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,1);
}); });
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,1);
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment