Skip to content

G
gic-cloud
Commit 8330f895 by fudahua
Options

Merge branch 'developer' into 'master' 

Developer

See merge request !64
parents 54239afb bad3aeb9
Showing with 98 additions and 7 deletions
gic-cloud-data-hook-api/.flattened-pom.xml
......@@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.gic</groupId>
<artifactId>gic-cloud-data-hook-api</artifactId>
<version>2.40</version>
<version>2.42</version>
<dependencies>
<dependency>
<groupId>com.gic</groupId>
......@@ -41,7 +41,7 @@
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-platform-config</artifactId>
<version>2.26</version>
<version>2.29</version>
<scope>compile</scope>
</dependency>
<dependency>
......
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/service/IFreeQueryService.java
......@@ -84,5 +84,7 @@ public interface IFreeQueryService {
List<String> listFilterPhoneAndCardNoField();
List<String> listFilterPhoneAndCardNoCardNumField();
List<String> listFilterUserName();
}
gic-cloud-data-hook-service/pom.xml
......@@ -183,6 +183,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-enterprise-api</artifactId>
<version>${gic-enterprise-api}</version>
</dependency>
</dependencies>
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/FilterFieldUtils.java
......@@ -32,7 +32,10 @@ public class FilterFieldUtils {
/** 脱敏字段 */
public static List<String> FILTERS_PHONE_AND_CARD = Arrays.asList("card_num", "mobile", "phone", "enterprise_name", "phone_number", "receive_phone_number","receive_card_num","use_phone_number", "use_card_num","friend_card_num","from_card_num", "friend_phone_num","from_phone_num");
public static List<String> FILETERS_USER_NAME = Arrays.asList("member_name","children_name","mbr_name","receive_member_name","use_member_name","name","bb_name","friend_mbr_name","from_mbr_name");
/** 脱敏字段 */
public static List<String> FILTERS_PHONE_AND_CARD_COUPON = Arrays.asList("card_num", "mobile", "phone", "enterprise_name", "phone_number", "receive_phone_number","receive_card_num","use_phone_number", "use_card_num","friend_card_num","from_card_num", "friend_phone_num","from_phone_num","card_code");
public static List<String> FILETERS_USER_NAME = Arrays.asList("member_name","children_name","mbr_name","receive_member_name","use_member_name","name","bb_name","friend_mbr_name","from_mbr_name","card_code");
ScheduledExecutorService timer = new ScheduledThreadPoolExecutor(1,
new BasicThreadFactory.Builder().namingPattern("loadFilterFieldTimer-%d").daemon(true).build());
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java
......@@ -2,6 +2,8 @@ package com.gic.cloud.data.hook.service.impl;
import cn.medubi.client.utils.LogPak;
import com.alibaba.fastjson.JSON;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.cloud.common.api.base.Page;
import com.gic.cloud.data.hook.api.dto.*;
import com.gic.cloud.data.hook.api.entity.DownloadTaskStatus;
......@@ -99,9 +101,30 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
result.setList(preResult.getList()); // 设置数据集
result.setTotal(preResult.getTotal()); // 设置总数
result.setPageNum(preResult.getPageNum()); // 设置当前页数
if (CollectionUtils.isNotEmpty(result.getList())) {
for (DownloadTask downloadTask : result.getList()) {
if (StringUtils.isBlank(downloadTask.getFilePath())) {
continue;
}
downloadTask.setFilePath(getDownloadUrl(downloadTask.getFilePath(),downloadTask.getName()));
}
}
return result;
}
private String getDownloadUrl(String url,String filename) {
Config config = ConfigService.getConfig("COMMON.cloud.file.setting");
String hostUrl = config.getProperty("gic.proxy.url", null);
if (org.apache.commons.lang.StringUtils.isBlank(hostUrl)) {
return url;
}else{
int pos = url.lastIndexOf(".");
String ext = url.substring(pos);
return hostUrl+"/"+filename+ext+"?url="+url;
}
}
/** 创建下载任务
* @param task
* @return
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
......@@ -15,8 +15,15 @@ import com.gic.cloud.data.hook.service.dao.FlatQueryTableDao;
import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
import com.gic.cloud.data.hook.service.entity.CsvResultSetHelper;
import com.gic.dubbo.entity.ProviderLocalTag;
import com.gic.enterprise.api.dto.EnterpriseDTO;
import com.gic.enterprise.api.service.EnterpriseService;
import com.gic.qcloud.BucketNameEnum;
import com.gic.qcloud.FileRecordLogUtil;
import com.gic.qcloud.FileUploadUtil;
import com.gic.thirdparty.cloudfile.CloudFileUtil;
import com.gic.thirdparty.cloudfile.enums.CloudFileBusinessOptEnum;
import com.gic.thirdparty.cloudfile.enums.CloudFileTypeEnum;
import com.gic.thirdparty.cloudfile.pojo.CloudFileInfo;
import com.google.common.collect.Lists;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
......@@ -85,6 +92,9 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
@Autowired
DecryptKeyService decryptKeyService;
@Autowired
private EnterpriseService enterpriseService;
/** 自助指标查询关联的下载条件列表 */
protected List<FlatQueryTaskCondition> taskConditions = Lists.newArrayList();
protected List<FlatQueryTaskCondition> bigTaskConditions = Lists.newArrayList();
......@@ -99,6 +109,13 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
runApplyTask(5); // 每5秒钟进行任务状态检测
}
private String cloudFileUpload(String enterpriseId,File file,String fileExt){
EnterpriseDTO enterprise = enterpriseService.getEnterpriseById(enterpriseId);
CloudFileInfo zip = CloudFileUtil.uploadFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON);
FileRecordLogUtil.recordUseLog("FlatQueryResultServiceImpl#cloudFileUpload||"+fileExt);
return zip.getOrgFileUrl();
}
/** 生成自助指标查询最终 SQL
* @param queryOrDownload 查询或下载用途
* @param tableId
......@@ -328,6 +345,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
} else if(dataPermission == 2 && FilterFieldUtils.FILTERS_PHONE_AND_CARD.contains(fieldName)){
fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
} else if(dataPermission == 3 && FilterFieldUtils.FILTERS_PHONE_AND_CARD_COUPON.contains(fieldName)){
fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
} else if (FilterFieldUtils.FILETERS_USER_NAME.contains(fieldName)){
fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
} else fieldResult = rs.getObject(fieldColumnIndex);
......@@ -336,6 +355,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
fieldResult = "******";
} else if(dataPermission == 2 && FilterFieldUtils.FILTERS_PHONE_AND_CARD.contains(fieldName)){
fieldResult = "******";
} else if(dataPermission == 3 && FilterFieldUtils.FILTERS_PHONE_AND_CARD_COUPON.contains(fieldName)){
fieldResult = "******";
} else if(FilterFieldUtils.FILETERS_USER_NAME.contains(fieldName)){
fieldResult = DecryptUtils.dataSecurityProcessUserName(DecryptUtils.decrypt(enterpriseId,rs.getObject(fieldColumnIndex)+""));
} else fieldResult = rs.getObject(fieldColumnIndex);
......@@ -927,10 +948,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
// zos.close();
// }
logger.info("[ 开始上传压缩文件到腾讯云 ]: {}", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
// cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
cloudFileUrl = cloudFileUpload(task.getEnterpriseId(),zipFile,taskFileExt.substring(1));
} else {
logger.info("[ 开始上传文件到腾讯云 ]: {}", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
// cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
cloudFileUrl = cloudFileUpload(task.getEnterpriseId(),new File(originalFilePath),taskFileExt.substring(1));
}
logger.info("[ 上传腾讯云 ]: {}", "地址为："+cloudFileUrl);
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
......@@ -14,8 +14,15 @@ import com.gic.cloud.data.hook.service.dao.FreeQueryRecordDao;
import com.gic.cloud.data.hook.service.dao.FreeQuerySourceDao;
import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
import com.gic.cloud.data.hook.service.entity.CsvResultSetHelper;
import com.gic.enterprise.api.dto.EnterpriseDTO;
import com.gic.enterprise.api.service.EnterpriseService;
import com.gic.qcloud.BucketNameEnum;
import com.gic.qcloud.FileRecordLogUtil;
import com.gic.qcloud.FileUploadUtil;
import com.gic.thirdparty.cloudfile.CloudFileUtil;
import com.gic.thirdparty.cloudfile.enums.CloudFileBusinessOptEnum;
import com.gic.thirdparty.cloudfile.enums.CloudFileTypeEnum;
import com.gic.thirdparty.cloudfile.pojo.CloudFileInfo;
import com.google.common.collect.Lists;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
......@@ -59,6 +66,9 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
@Autowired
IDownloadTaskService downloadTaskService;
@Autowired
private EnterpriseService enterpriseService;
/** 获取指定脱敏类型的过滤字段集合
* @param desensiType
* @return
......@@ -365,6 +375,11 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
}
@Override
public List<String> listFilterPhoneAndCardNoCardNumField() {
return FilterFieldUtils.FILTERS_PHONE_AND_CARD_COUPON;
}
@Override
public List<String> listFilterUserName() {
return FilterFieldUtils.FILETERS_USER_NAME;
}
......@@ -555,10 +570,12 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
// zos.close();
// }
// log.debug("开始上传压缩文件到腾讯云", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
// cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
cloudFileUrl = cloudFileUpload(task.getEnterpriseId(), zipFile, taskFileExt.substring(1));
} else {
log.debug("开始上传文件到腾讯云", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
// cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
cloudFileUrl = cloudFileUpload(task.getEnterpriseId(), new File(originalFilePath), taskFileExt.substring(1));
}
log.debug("上传腾讯云", "地址为："+cloudFileUrl);
task.setStatus(DownloadTaskStatus.COMPLISHED);
......@@ -591,6 +608,15 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
}
private String cloudFileUpload(String enterpriseId,File file,String fileExt){
EnterpriseDTO enterprise = enterpriseService.getEnterpriseById(enterpriseId);
CloudFileInfo zip = CloudFileUtil.uploadFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON);
FileRecordLogUtil.recordUseLog("FreeQueryServiceImpl#cloudFileUpload||"+fileExt);
return zip.getOrgFileUrl();
}
private void saveXlsx(FreeQueryTaskCondition condition,String originalFilePath,ResultSet rs,DownloadTask task) throws Exception{
Integer limitSize = FileUtil.getLimitSize();
// originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
......
gic-cloud-data-hook-service/src/main/resources/data-hook-service-dubbo-config.xml
......@@ -35,6 +35,8 @@
<dubbo:service interface="com.gic.cloud.data.hook.api.service.TableSyncRecordService" ref="tableSyncRecordServiceImpl"/>
<dubbo:reference interface="com.gic.enterprise.api.service.EnterpriseService" id="enterpriseService" timeout="10000" />
<!-- 引用的 Dubbo 服务 -->
<!--<dubbo:reference interface="com.gic.dict.api.service.ManagerDictService" id="managerDictService" timeout="10000" />
<dubbo:reference interface="com.gic.cloud.communicate.api.service.member.MemberTagFieldService" id="memberTagFieldService" timeout="10000" />
......
gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/web/FlatQueryController.java
......@@ -297,6 +297,13 @@ public class FlatQueryController {
condition.setEnableEncrypt(false);
}
}
if(dataPermission == 3){
if(freeQueryService.listFilterPhoneAndCardNoCardNumField().contains(condition.getFieldName())){
condition.setEnableEncrypt(true);
} else {
condition.setEnableEncrypt(false);
}
}
}
}
return this.flatQueryResultService.buildFlatQueryDownloadTask(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment