Skip to content

G
gic-cloud
Commit e23d5ed7 by fudahua
Options

feat:自定义查询zip

parent 5a2cb44d
Showing with 187 additions and 85 deletions
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/entity/FreeQueryTaskCondition.java
......@@ -114,4 +114,14 @@ public class FreeQueryTaskCondition implements Serializable {
public void setDesensiType(Integer desensiType) {
this.desensiType = desensiType;
}
private int amount;
public int getAmount() {
return amount;
}
public void setAmount(int amount) {
this.amount = amount;
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
......@@ -51,6 +51,8 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
/** 日志类 */
private static LogPak log = new LogPak(FreeQueryServiceImpl.class);
private static final Integer XLS_SIZE = 1000000;
/** csv / xls 下载目录 */
public static final String SAVE_FOLDER = "/usr/local/data-hook-file";
......@@ -308,6 +310,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
// 生成条件
FreeQueryTaskCondition condition = new FreeQueryTaskCondition();
condition.setAmount(amount);
condition.setTaskId(curTask.getId());
condition.setSql(sql);
condition.setEnterpriseId(enterpriseId);
......@@ -334,6 +337,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
if(record.getDesensiType() != null){
FreeQueryTaskCondition condition = new FreeQueryTaskCondition();
condition.setTaskId(record.getId());
condition.setAmount(record.getAmount());
condition.setSql(record.getRealSql());
condition.setEnterpriseId(record.getEnterpriseId());
condition.setDesensiType(record.getDesensiType());
......@@ -400,6 +404,8 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
String fullQuery = condition.getSql();
boolean zipFlag=false;
List<File> listFile=new ArrayList<>();
Connection conn = MysqlHelper.getFreeQueryConnection(condition.getEnterpriseId());
if (conn != null) {
......@@ -432,97 +438,122 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".csv");
} else {
log.debug("runDownloadTask.run", "准备生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
String colName = rs.getMetaData().getColumnLabel(j + 1);
cell = row.createCell(j);
cell.setCellValue(colName);
int filePos=0;
if (condition.getAmount()>XLS_SIZE){
int num = (condition.getAmount() / XLS_SIZE)+(condition.getAmount()%XLS_SIZE>0?1:0);
while (num-->0) {
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId()+"-"+filePos + ".xlsx";
filePos++;
saveXlsx(condition,originalFilePath,rs,task);
File file = new File(originalFilePath);
listFile.add(file);
}
zipFlag=true;
}else{
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
saveXlsx(condition,originalFilePath,rs,task);
File file = new File(originalFilePath);
listFile.add(file);
}
// 遍历输出行
int rowCount = 0;
while (rs.next()) {
rowCount++;
row = sheet.createRow(rowCount);
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
//String c = rs.getString(j + 1);
//row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1);
if (task.getQueryDataType() == QueryDataType.SAFE
&& (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FilterFieldUtils.FILETERS_USER_NAME.contains(cName))) {
if(FilterFieldUtils.FILETERS_USER_NAME.contains(cName)){
row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(j+1)));
} else {
row.createCell(j).setCellValue("******");
}
} else {
int cType = rs.getMetaData().getColumnType(j + 1);
switch (cType) {
case Types.TIMESTAMP:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
case Types.DATE:
row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
break;
case Types.TIME:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
default:
String string = rs.getString(j + 1);
if(StringUtils.isNotBlank(string)){
if(string.contains("E0")){
string = string + "\t";
}
}
row.createCell(j).setCellValue(string);
break;
}
}
} // FOR OVER
} // WHILE OVER
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
// SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
// Sheet sheet = wb.createSheet();
// Row row = sheet.createRow(0);
// Cell cell;
// for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
// String colName = rs.getMetaData().getColumnLabel(j + 1);
// cell = row.createCell(j);
// cell.setCellValue(colName);
// }
// // 遍历输出行
// int rowCount = 0;
// while (rs.next()) {
// rowCount++;
// row = sheet.createRow(rowCount);
// for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
// //String c = rs.getString(j + 1);
// //row.createCell(j).setCellValue(c);
// String cName = rs.getMetaData().getColumnName(j+1);
// if (task.getQueryDataType() == QueryDataType.SAFE
// && (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FilterFieldUtils.FILETERS_USER_NAME.contains(cName))) {
// if(FilterFieldUtils.FILETERS_USER_NAME.contains(cName)){
// row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(j+1)));
// } else {
// row.createCell(j).setCellValue("******");
// }
// } else {
// int cType = rs.getMetaData().getColumnType(j + 1);
// switch (cType) {
// case Types.TIMESTAMP:
// row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
// break;
// case Types.DATE:
// row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
// break;
// case Types.TIME:
// row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
// break;
// default:
// String string = rs.getString(j + 1);
// if(StringUtils.isNotBlank(string)){
// if(string.contains("E0")){
// string = string + "\t";
// }
// }
// row.createCell(j).setCellValue(string);
// break;
// }
// }
// } // FOR OVER
// } // WHILE OVER
// FileOutputStream fileOut = new FileOutputStream(originalFilePath);
// wb.write(fileOut);
// //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
// fileOut.close();
// //wb.close();
// wb.dispose(); // SXSSFWorkbook 没有 close 方法
// log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
}
String cloudFileUrl = "https://";
String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
if (task.getUseCompress().equals(Global.YES)) {
String taskFileExt = task.getUseCompress().equals(Global.YES)||zipFlag ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
if (task.getUseCompress().equals(Global.YES)||zipFlag) {
log.debug("runDownloadTask.run", "准备生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
// String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
OutputStream os = new FileOutputStream(zipFilePath);
File zipFile = new File(zipFilePath);
ZipOutputStream zos = null;
byte[] buf = new byte[1024];
int length = 0;
try {
OutputStream os = new FileOutputStream(zipFilePath);
BufferedOutputStream bos = new BufferedOutputStream(os);
zos = new ZipOutputStream(bos);
zos.setLevel(6); // 压缩率选择 0-9
InputStream is = new FileInputStream(originalFilePath);
BufferedInputStream bis = new BufferedInputStream(is);
zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
while ((length = bis.read(buf)) > 0) {
zos.write(buf, 0, length);
}
bis.close();
is.close();
//bos.close();
//os.close();
log.debug("runDownloadTask.run", "已生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
} catch (Exception ex2) {
throw ex2;
} finally {
zos.closeEntry();
zos.close();
}
log.debug("开始上传压缩文件到腾讯云", task.getId());
FileUtil.toZip(listFile,os);
// File zipFile = new File(zipFilePath);
// ZipOutputStream zos = null;
// byte[] buf = new byte[1024];
// int length = 0;
// try {
// OutputStream os = new FileOutputStream(zipFilePath);
// BufferedOutputStream bos = new BufferedOutputStream(os);
// zos = new ZipOutputStream(bos);
// zos.setLevel(6); // 压缩率选择 0-9
// InputStream is = new FileInputStream(originalFilePath);
// BufferedInputStream bis = new BufferedInputStream(is);
// zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
// while ((length = bis.read(buf)) > 0) {
// zos.write(buf, 0, length);
// }
// bis.close();
// is.close();
// //bos.close();
// //os.close();
// log.debug("runDownloadTask.run", "已生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
// } catch (Exception ex2) {
// throw ex2;
// } finally {
// zos.closeEntry();
// zos.close();
// }
// log.debug("开始上传压缩文件到腾讯云", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
} else {
log.debug("开始上传文件到腾讯云", task.getId());
......@@ -559,6 +590,67 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
}
private void saveXlsx(FreeQueryTaskCondition condition,String originalFilePath,ResultSet rs,DownloadTask task) throws Exception{
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
String colName = rs.getMetaData().getColumnLabel(j + 1);
cell = row.createCell(j);
cell.setCellValue(colName);
}
// 遍历输出行
int rowCount = 0;
while (rs.next()) {
rowCount++;
row = sheet.createRow(rowCount);
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
//String c = rs.getString(j + 1);
//row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1);
if (task.getQueryDataType() == QueryDataType.SAFE
&& (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FilterFieldUtils.FILETERS_USER_NAME.contains(cName))) {
if(FilterFieldUtils.FILETERS_USER_NAME.contains(cName)){
row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(j+1)));
} else {
row.createCell(j).setCellValue("******");
}
} else {
int cType = rs.getMetaData().getColumnType(j + 1);
switch (cType) {
case Types.TIMESTAMP:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
case Types.DATE:
row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
break;
case Types.TIME:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
default:
String string = rs.getString(j + 1);
if(StringUtils.isNotBlank(string)){
if(string.contains("E0")){
string = string + "\t";
}
}
row.createCell(j).setCellValue(string);
break;
}
}
} // FOR OVER
} // WHILE OVER
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
}
/** 下载申请检查计时器 */
//private Timer applyTimer = new Timer();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment