Skip to content

G
gic-cloud
Commit ea7cb06d by fudahua
Options

zip压缩

parent f8194c53
Showing with 171 additions and 55 deletions
gic-cloud-data-hook-api/.flattened-pom.xml
......@@ -4,12 +4,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.gic</groupId>
<artifactId>gic-cloud-data-hook-api</artifactId>
<version>3.0-SNAPSHOT</version>
<version>2.38</version>
<dependencies>
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-cloud-web-service-api</artifactId>
<version>3.0-SNAPSHOT</version>
<version>3.80</version>
<scope>compile</scope>
<exclusions>
<exclusion>
......@@ -25,7 +25,7 @@
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-cloud-commons</artifactId>
<version>3.0-SNAPSHOT</version>
<version>3.64</version>
<scope>compile</scope>
<exclusions>
<exclusion>
......@@ -41,7 +41,7 @@
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-platform-config</artifactId>
<version>3.0-SNAPSHOT</version>
<version>2.25</version>
<scope>compile</scope>
</dependency>
<dependency>
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/FileUtil.java 0 → 100644
package com.gic.cloud.data.hook.service;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class FileUtil {
private static final int BUFFER_SIZE = 2 * 1024;
/**
* 压缩zip
* @param srcFiles
* @param out
* @throws RuntimeException
*/
public static void toZip(List<File> srcFiles, OutputStream out) throws RuntimeException {
long start = System.currentTimeMillis();
ZipOutputStream zos = null;
try {
zos = new ZipOutputStream(out);
for (File srcFile : srcFiles) {
byte[] buf = new byte[BUFFER_SIZE];
zos.putNextEntry(new ZipEntry(srcFile.getName()));
int len;
FileInputStream in = new FileInputStream(srcFile);
while ((len = in.read(buf)) != -1) {
zos.write(buf, 0, len);
}
zos.closeEntry();
in.close();
}
long end = System.currentTimeMillis();
System.out.println("压缩完成，耗时：" + (end - start) + " ms");
} catch (Exception e) {
throw new RuntimeException("zip error from ZipUtils", e);
} finally {
if (zos != null) {
try {
zos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
......@@ -76,6 +76,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private static final Integer SMALL_SIZE = 2;
private static final Integer XLS_SIZE = 100000;
private static final Map<String, String> bigTaskRunningMap = new ConcurrentHashMap<>();
@Autowired
......@@ -817,6 +819,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
ResultSetHelper helper = new CsvResultSetHelper(queryDataType, condition.getDecryptFilters(), condition.getConditions(), condition.getEnterpriseIds().get(0), decryptKeyService);
// 生成指定格式下载元文件
String originalFilePath = "";
boolean zipFlag=false;
List<File> listFile=new ArrayList<>();
if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".csv");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
......@@ -832,69 +836,95 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
writer.close();
out.close();//记得关闭资源
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".csv");
listFile.add(new File(originalFilePath));
} else { // 如果指定为 XLS 格式
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
String[] columnNames = helper.getColumnNames(rs);
for(int j =0; j<columnNames.length; j++){
cell = row.createCell(j);
cell.setCellValue(columnNames[j]);
int filePos=0;
if (condition.getAmount()>XLS_SIZE){
while (rs.isLast()) {
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId()+"-"+filePos + ".xlsx";
filePos++;
saveXlsSplit(originalFilePath,helper,rs,condition);
File file = new File(originalFilePath);
listFile.add(file);
}
// 遍历输出行
int rowCount = 0;
while (rs.next()) {
rowCount++;
row = sheet.createRow(rowCount);
String[] columnValues = helper.getColumnValues(rs, true, "", "");
for(int j=0; j<columnValues.length; j++){
row.createCell(j).setCellValue(columnValues[j]);
zipFlag=true;
}else{
saveXlsSplit(originalFilePath,helper,rs,condition);
File file = new File(originalFilePath);
listFile.add(file);
}
} // WHILE OVER
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
OutputStream os = new FileOutputStream(zipFilePath);
FileUtil.toZip(listFile,os);
// originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
// SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
// Sheet sheet = wb.createSheet();
// Row row = sheet.createRow(0);
// Cell cell;
// String[] columnNames = helper.getColumnNames(rs);
// for(int j =0; j<columnNames.length; j++){
// cell = row.createCell(j);
// cell.setCellValue(columnNames[j]);
// }
// // 遍历输出行
// int rowCount = 0;
// while (rs.next()) {
// rowCount++;
// row = sheet.createRow(rowCount);
// String[] columnValues = helper.getColumnValues(rs, true, "", "");
// for(int j=0; j<columnValues.length; j++){
// row.createCell(j).setCellValue(columnValues[j]);
// }
// } // WHILE OVER
// FileOutputStream fileOut = new FileOutputStream(originalFilePath);
// wb.write(fileOut);
// //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
// fileOut.close();
// //wb.close();
// wb.dispose(); // SXSSFWorkbook 没有 close 方法
// logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
} // IF ELSE OVER
String cloudFileUrl = "https://";
// 如果指定压缩，则使用之
//if (task.getFormat().equals("zip")) {
String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
if (task.getUseCompress().equals(Global.YES)) {
if (zipFlag||task.getUseCompress().equals(Global.YES)) {
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
File zipFile = new File(zipFilePath);
ZipOutputStream zos = null;
byte[] buf = new byte[1024];
int length = 0;
try {
OutputStream os = new FileOutputStream(zipFilePath);
BufferedOutputStream bos = new BufferedOutputStream(os);
zos = new ZipOutputStream(bos);
zos.setLevel(6); // 压缩率选择 0-9
InputStream is = new FileInputStream(originalFilePath);
BufferedInputStream bis = new BufferedInputStream(is);
zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
while ((length = bis.read(buf)) > 0) {
zos.write(buf, 0, length);
}
bis.close();
is.close();
//bos.close();
//os.close();
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
} catch (Exception ex2) {
throw ex2;
} finally {
zos.closeEntry();
zos.close();
}
File zipFile = new File(zipFilePath);
FileUtil.toZip(listFile,os);
//
//
// ZipOutputStream zos = null;
// byte[] buf = new byte[1024];
// int length = 0;
// try {
// OutputStream os = new FileOutputStream(zipFilePath);
// BufferedOutputStream bos = new BufferedOutputStream(os);
// zos = new ZipOutputStream(bos);
// zos.setLevel(6); // 压缩率选择 0-9
// InputStream is = new FileInputStream(originalFilePath);
// BufferedInputStream bis = new BufferedInputStream(is);
// zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
// while ((length = bis.read(buf)) > 0) {
// zos.write(buf, 0, length);
// }
// bis.close();
// is.close();
// //bos.close();
// //os.close();
// logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
// } catch (Exception ex2) {
// throw ex2;
// } finally {
// zos.closeEntry();
// zos.close();
// }
logger.info("[ 开始上传压缩文件到腾讯云 ]: {}", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
} else {
......@@ -929,6 +959,42 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
} // IF OVER
}
private void saveXlsSplit(String originalFilePath,ResultSetHelper helper,ResultSet rs,FlatQueryTaskCondition condition){
try {
// String originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
String[] columnNames = helper.getColumnNames(rs);
for(int j =0; j<columnNames.length; j++){
cell = row.createCell(j);
cell.setCellValue(columnNames[j]);
}
// 遍历输出行
int rowCount = 0;
while (rowCount<XLS_SIZE&&rs.next()) {
rowCount++;
row = sheet.createRow(rowCount);
String[] columnValues = helper.getColumnValues(rs, true, "", "");
for(int j=0; j<columnValues.length; j++){
row.createCell(j).setCellValue(columnValues[j]);
}
} // WHILE OVER
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
}catch (Exception e) {
e.printStackTrace();
logger.info("异常：{}",e);
}
}
/** 下载申请检查计时器 */
//private Timer applyTimer = new Timer();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment