Skip to content

G
gic-cloud
Commit 243458e1 by fudahua
Options

下载逻辑的日志修改

parent b1b58b00
Showing with 73 additions and 22 deletions
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
......@@ -85,6 +85,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
/** 自助指标查询关联的下载条件列表 */
protected List<FlatQueryTaskCondition> taskConditions = Lists.newArrayList();
protected List<FlatQueryTaskCondition> bigTaskConditions = Lists.newArrayList();
protected List<FlatQueryTaskCondition> smallConditions = Lists.newArrayList();
private FlatQueryResultServiceImpl() {
log.debug("construct", "准备初始化 FlatQuery 查询服务");
......@@ -557,6 +558,54 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private static SimpleDateFormat timeFormatter = new SimpleDateFormat("HH:mm:ss");
/** 任务分配 */
//private Timer downloadTaskTimer = new Timer();
ScheduledExecutorService distService = new ScheduledThreadPoolExecutor(1,
new BasicThreadFactory.Builder().namingPattern("distTimer-%d").daemon(true).build());
/** 启动分配任务任务 */
private void runDistTask(Integer interval) {
downloadService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
ProviderLocalTag providerLocalTag = ProviderLocalTag.tag.get();
providerLocalTag.traceId = UUID.randomUUID().toString();
try{
if (taskConditions != null && taskConditions.size() > 0) {
FlatQueryTaskCondition condition = null;
//FlatQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件
for (int i=0; i<taskConditions.size(); i++ ) {
logger.info("[ 自助指标当前正在执行的任务为：]:{}", JSON.toJSONString(taskConditions.get(i)));
if (taskConditions.get(i).getBuildPermitted().equals(Global.YES)
&& !taskConditions.get(i).getEnterpriseIds().contains("ff8080816dd0385e016ddca436d01fe1")) {
if(taskConditions.get(i).getAllFields().size() >= maxFields) {
bigTaskConditions.add(taskConditions.get(i));
taskConditions.remove(i);
continue;
} else {
if(bigTaskRunningMap.isEmpty()){
bigTaskConditions.add(taskConditions.get(i));
taskConditions.remove(i);
continue;
}
}
if (CollectionUtils.isEmpty(smallConditions)||smallConditions.size()<10) {
smallConditions.add(taskConditions.get(i));
}
} // IF OVER
} // FOR OVER
} // 没有任务则忽略
}catch (Exception e){
logger.info("异常：{}",e);
logger.info("[ 自助指标下载异常 ]: {}", e.getMessage());
e.printStackTrace();
} finally {
}
}
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
}
/** 下载任务执行计时器 */
//private Timer downloadTaskTimer = new Timer();
......@@ -576,7 +625,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
FlatQueryTaskCondition condition = null;
//FlatQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件
for (int i=0; i<taskConditions.size(); i++ ) {
log.debug("自助指标当前正在执行的任务为：", JSON.toJSONString(taskConditions.get(i)));
logger.info("[ 自助指标当前正在执行的任务为：]:{}", JSON.toJSONString(taskConditions.get(i)));
if (taskConditions.get(i).getBuildPermitted().equals(Global.YES)
&& !taskConditions.get(i).getEnterpriseIds().contains("ff8080816dd0385e016ddca436d01fe1")) {
if(taskConditions.get(i).getAllFields().size() >= maxFields) {
......@@ -595,7 +644,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
condition = taskConditions.remove(i); // 移除并获取第一个任务条件
break;
}catch (Exception e){
log.debug("获取连接异常:", e.getMessage());
logger.info("[ 获取连接异常: ]:{}", e.getMessage());
e.printStackTrace();
continue;
}
......@@ -604,7 +653,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
takeFile(condition, connection);
} // 没有任务则忽略
}catch (Exception e){
log.debug("自助指标下载异常", e.getMessage());
logger.info("异常：{}",e);
logger.info("[ 自助指标下载异常 ]: {}", e.getMessage());
e.printStackTrace();
} finally {
if(connection != null){
......@@ -637,7 +687,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
FlatQueryTaskCondition condition = null;
//FlatQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件
for (int i=0; i<bigTaskConditions.size(); i++ ) {
log.debug("自助指标当前正在执行的任务为：", JSON.toJSONString(bigTaskConditions.get(i)));
logger.info("[ 自助指标当前正在执行的任务为：]: {}", JSON.toJSONString(bigTaskConditions.get(i)));
if (bigTaskConditions.get(i).getBuildPermitted().equals(Global.YES)) {
try{
condition = bigTaskConditions.remove(i); // 移除并获取第一个任务条件
......@@ -645,7 +695,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
connection = HiveHelper.getBigDataDownloadHiveConnection();
break;
}catch (Exception e){
log.debug("获取连接异常:", e.getMessage());
logger.info("异常：{}",e);
logger.info("[ 获取连接异常: ]: {}", e.getMessage());
e.printStackTrace();
continue;
}
......@@ -656,7 +707,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
} // 没有任务则忽略
}catch (Exception e){
logger.info("下载异常：{}",e);
log.debug("自助指标下载异常", e.getMessage());
logger.info("[ 自助指标下载异常 ]: {}", e.getMessage());
e.printStackTrace();
} finally {
if(connection != null){
......@@ -689,7 +740,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
FlatQueryTaskCondition condition = null;
//FlatQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件
for (int i=0; i<taskConditions.size(); i++ ) {
log.debug("自助指标当前正在执行的任务为：", JSON.toJSONString(taskConditions.get(i)));
logger.info("[ 自助指标当前正在执行的任务为：]: {}", JSON.toJSONString(taskConditions.get(i)));
if (taskConditions.get(i).getBuildPermitted().equals(Global.YES)
&& taskConditions.get(i).getEnterpriseIds().contains("ff8080816dd0385e016ddca436d01fe1")) {
try{
......@@ -697,7 +748,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
condition = taskConditions.remove(i); // 移除并获取第一个任务条件
break;
}catch (Exception e){
log.debug("获取连接异常:", e.getMessage());
logger.info("[ 获取连接异常: ]: {}", e.getMessage());
e.printStackTrace();
continue;
}
......@@ -708,7 +759,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
takeFile(condition, connection);
} // 没有任务则忽略
}catch (Exception e){
log.debug("自助指标下载异常", e.getMessage());
logger.info("异常：{}",e);
logger.info("[ 自助指标下载异常 ]: {}", e.getMessage());
e.printStackTrace();
} finally {
if(connection != null){
......@@ -729,8 +781,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId());
task.setStatus(DownloadTaskStatus.BUILDING);
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
logger.info("test=================");
log.debug("runDownloadTask.run", "自助指标下载任务执行：" + task.getId());
logger.info("[ runDownloadTask.run ]: {}", "自助指标下载任务执行：" + task.getId());
//初始化校验
FilterFieldUtils.timeFieldCheckInit(condition.getConditions());
......@@ -745,8 +796,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
0,
condition.getAuthStoreIdList());
log.debug("runDownloadTask.run", "获取商户连接：" + task.getId());
log.debug("sql-", task.getId() + "-" + fullQuery);
logger.info("[ runDownloadTask.run ]: {}", "获取商户连接：" + task.getId());
logger.info("[ sql- ]: {}", task.getId() + "-" + fullQuery);
if (conn != null) {
try {
Statement stat = conn.createStatement();
......@@ -759,7 +810,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
// 生成指定格式下载元文件
String originalFilePath = "";
if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式
log.debug("runDownloadTask.run", "准备生成自助指标下载文件 " + condition.getTaskId() + ".csv");
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".csv");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
File tmp = new File(originalFilePath);
if (tmp.exists()) { // 删除可能存在的文件
......@@ -772,9 +823,9 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
writer.writeAll(rs, true);
writer.close();
out.close();//记得关闭资源
log.debug("runDownloadTask.run", "已生成自助指标下载文件 " + condition.getTaskId() + ".csv");
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".csv");
} else { // 如果指定为 XLS 格式
log.debug("runDownloadTask.run", "准备生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet();
......@@ -801,14 +852,14 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
log.debug("runDownloadTask.run", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
} // IF ELSE OVER
String cloudFileUrl = "https://";
// 如果指定压缩，则使用之
//if (task.getFormat().equals("zip")) {
String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
if (task.getUseCompress().equals(Global.YES)) {
log.debug("runDownloadTask.run", "准备生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
File zipFile = new File(zipFilePath);
ZipOutputStream zos = null;
......@@ -829,21 +880,21 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
is.close();
//bos.close();
//os.close();
log.debug("runDownloadTask.run", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
} catch (Exception ex2) {
throw ex2;
} finally {
zos.closeEntry();
zos.close();
}
log.debug("开始上传压缩文件到腾讯云", task.getId());
logger.info("[ 开始上传压缩文件到腾讯云 ]: {}", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
} else {
log.debug("开始上传文件到腾讯云", task.getId());
logger.info("[ 开始上传文件到腾讯云 ]: {}", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
}
log.debug("上传腾讯云", "地址为："+cloudFileUrl);
logger.info("[ 上传腾讯云 ]: {}", "地址为："+cloudFileUrl);
task.setStatus(DownloadTaskStatus.COMPLISHED);
task.setOverTime(new Date());
task.setFilePath(cloudFileUrl);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment