Merge branch 'developer' into 'master'

Developer

See merge request !64

gic-cloud-data-hook-api/.flattened-pom.xml

@@ -4,7 +4,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.gic</groupId>
   <artifactId>gic-cloud-data-hook-api</artifactId>
-  <version>2.40</version>
+  <version>2.42</version>
   <dependencies>
     <dependency>
       <groupId>com.gic</groupId>
@@ -41,7 +41,7 @@
     <dependency>
       <groupId>com.gic</groupId>
       <artifactId>gic-platform-config</artifactId>
-      <version>2.26</version>
+      <version>2.29</version>
       <scope>compile</scope>
     </dependency>
     <dependency>

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/service/IFreeQueryService.java

@@ -84,5 +84,7 @@ public interface IFreeQueryService {
 
 	List<String> listFilterPhoneAndCardNoField();
 
+	List<String> listFilterPhoneAndCardNoCardNumField();
+
 	List<String> listFilterUserName();
 }

gic-cloud-data-hook-service/pom.xml

@@ -183,6 +183,11 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+		<dependency>
+			<groupId>com.gic</groupId>
+			<artifactId>gic-enterprise-api</artifactId>
+			<version>${gic-enterprise-api}</version>
+		</dependency>
 
 	</dependencies>
 

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/FilterFieldUtils.java

@@ -32,7 +32,10 @@ public class FilterFieldUtils {
     /** 脱敏字段 */
     public static List<String> FILTERS_PHONE_AND_CARD = Arrays.asList("card_num", "mobile", "phone", "enterprise_name", "phone_number", "receive_phone_number","receive_card_num","use_phone_number", "use_card_num","friend_card_num","from_card_num", "friend_phone_num","from_phone_num");
 
-    public static List<String> FILETERS_USER_NAME = Arrays.asList("member_name","children_name","mbr_name","receive_member_name","use_member_name","name","bb_name","friend_mbr_name","from_mbr_name");
+    /** 脱敏字段 */
+    public static List<String> FILTERS_PHONE_AND_CARD_COUPON = Arrays.asList("card_num", "mobile", "phone", "enterprise_name", "phone_number", "receive_phone_number","receive_card_num","use_phone_number", "use_card_num","friend_card_num","from_card_num", "friend_phone_num","from_phone_num","card_code");
+
+    public static List<String> FILETERS_USER_NAME = Arrays.asList("member_name","children_name","mbr_name","receive_member_name","use_member_name","name","bb_name","friend_mbr_name","from_mbr_name","card_code");
 
     ScheduledExecutorService timer = new ScheduledThreadPoolExecutor(1,
             new BasicThreadFactory.Builder().namingPattern("loadFilterFieldTimer-%d").daemon(true).build());

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java

@@ -2,6 +2,8 @@ package com.gic.cloud.data.hook.service.impl;
 
 import cn.medubi.client.utils.LogPak;
 import com.alibaba.fastjson.JSON;
+import com.ctrip.framework.apollo.Config;
+import com.ctrip.framework.apollo.ConfigService;
 import com.gic.cloud.common.api.base.Page;
 import com.gic.cloud.data.hook.api.dto.*;
 import com.gic.cloud.data.hook.api.entity.DownloadTaskStatus;
@@ -99,9 +101,30 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
 		result.setList(preResult.getList()); // 设置数据集
 		result.setTotal(preResult.getTotal()); // 设置总数
 		result.setPageNum(preResult.getPageNum()); // 设置当前页数
+
+		if (CollectionUtils.isNotEmpty(result.getList())) {
+			for (DownloadTask downloadTask : result.getList()) {
+				if (StringUtils.isBlank(downloadTask.getFilePath())) {
+					continue;
+				}
+				downloadTask.setFilePath(getDownloadUrl(downloadTask.getFilePath(),downloadTask.getName()));
+			}
+		}
 		return result;
 	}
 
+	private String getDownloadUrl(String url,String filename) {
+		Config config = ConfigService.getConfig("COMMON.cloud.file.setting");
+		String hostUrl = config.getProperty("gic.proxy.url", null);
+		if (org.apache.commons.lang.StringUtils.isBlank(hostUrl)) {
+			return url;
+		}else{
+			int pos = url.lastIndexOf(".");
+			String ext = url.substring(pos);
+			return hostUrl+"/"+filename+ext+"?url="+url;
+		}
+	}
+
 	/** 创建下载任务
 	 * @param task
 	 * @return

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java

@@ -15,8 +15,15 @@ import com.gic.cloud.data.hook.service.dao.FlatQueryTableDao;
 import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
 import com.gic.cloud.data.hook.service.entity.CsvResultSetHelper;
 import com.gic.dubbo.entity.ProviderLocalTag;
+import com.gic.enterprise.api.dto.EnterpriseDTO;
+import com.gic.enterprise.api.service.EnterpriseService;
 import com.gic.qcloud.BucketNameEnum;
+import com.gic.qcloud.FileRecordLogUtil;
 import com.gic.qcloud.FileUploadUtil;
+import com.gic.thirdparty.cloudfile.CloudFileUtil;
+import com.gic.thirdparty.cloudfile.enums.CloudFileBusinessOptEnum;
+import com.gic.thirdparty.cloudfile.enums.CloudFileTypeEnum;
+import com.gic.thirdparty.cloudfile.pojo.CloudFileInfo;
 import com.google.common.collect.Lists;
 import com.opencsv.CSVWriter;
 import com.opencsv.ResultSetHelper;
@@ -85,6 +92,9 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 	@Autowired
 	DecryptKeyService decryptKeyService;
 
+	@Autowired
+	private EnterpriseService enterpriseService;
+
 	/** 自助指标查询关联的下载条件列表 */
 	protected List<FlatQueryTaskCondition> taskConditions = Lists.newArrayList();
 	protected List<FlatQueryTaskCondition> bigTaskConditions = Lists.newArrayList();
@@ -99,6 +109,13 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 		runApplyTask(5); // 每5秒钟进行任务状态检测
 	}
 
+	private String cloudFileUpload(String enterpriseId,File file,String fileExt){
+		EnterpriseDTO enterprise = enterpriseService.getEnterpriseById(enterpriseId);
+		CloudFileInfo zip = CloudFileUtil.uploadFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON);
+		FileRecordLogUtil.recordUseLog("FlatQueryResultServiceImpl#cloudFileUpload||"+fileExt);
+		return zip.getOrgFileUrl();
+	}
+
 	/** 生成自助指标查询最终 SQL
 	 * @param queryOrDownload 查询或下载用途
 	 * @param tableId
@@ -328,6 +345,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 											fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
 										} else if(dataPermission == 2 && FilterFieldUtils.FILTERS_PHONE_AND_CARD.contains(fieldName)){
 											fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
+										} else if(dataPermission == 3 && FilterFieldUtils.FILTERS_PHONE_AND_CARD_COUPON.contains(fieldName)){
+											fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
 										} else if (FilterFieldUtils.FILETERS_USER_NAME.contains(fieldName)){
 											fieldResult = DecryptUtils.decrypt(enterpriseId,preResult);
 										} else fieldResult = rs.getObject(fieldColumnIndex);
@@ -336,6 +355,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 											fieldResult = "******";
 										} else if(dataPermission == 2 && FilterFieldUtils.FILTERS_PHONE_AND_CARD.contains(fieldName)){
 											fieldResult = "******";
+										} else if(dataPermission == 3 && FilterFieldUtils.FILTERS_PHONE_AND_CARD_COUPON.contains(fieldName)){
+											fieldResult = "******";
 										} else if(FilterFieldUtils.FILETERS_USER_NAME.contains(fieldName)){
 											fieldResult = DecryptUtils.dataSecurityProcessUserName(DecryptUtils.decrypt(enterpriseId,rs.getObject(fieldColumnIndex)+""));
 										} else fieldResult = rs.getObject(fieldColumnIndex);
@@ -927,10 +948,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 //							zos.close();
 //						}
 						logger.info("[ 开始上传压缩文件到腾讯云 ]: {}", task.getId());
-						cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
+//						cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
+						cloudFileUrl = cloudFileUpload(task.getEnterpriseId(),zipFile,taskFileExt.substring(1));
 					} else {
 						logger.info("[ 开始上传文件到腾讯云 ]: {}", task.getId());
-						cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
+//						cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
+						cloudFileUrl = cloudFileUpload(task.getEnterpriseId(),new File(originalFilePath),taskFileExt.substring(1));
 					}
 
 					logger.info("[ 上传腾讯云 ]: {}", "地址为："+cloudFileUrl);

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java

@@ -14,8 +14,15 @@ import com.gic.cloud.data.hook.service.dao.FreeQueryRecordDao;
 import com.gic.cloud.data.hook.service.dao.FreeQuerySourceDao;
 import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
 import com.gic.cloud.data.hook.service.entity.CsvResultSetHelper;
+import com.gic.enterprise.api.dto.EnterpriseDTO;
+import com.gic.enterprise.api.service.EnterpriseService;
 import com.gic.qcloud.BucketNameEnum;
+import com.gic.qcloud.FileRecordLogUtil;
 import com.gic.qcloud.FileUploadUtil;
+import com.gic.thirdparty.cloudfile.CloudFileUtil;
+import com.gic.thirdparty.cloudfile.enums.CloudFileBusinessOptEnum;
+import com.gic.thirdparty.cloudfile.enums.CloudFileTypeEnum;
+import com.gic.thirdparty.cloudfile.pojo.CloudFileInfo;
 import com.google.common.collect.Lists;
 import com.opencsv.CSVWriter;
 import com.opencsv.ResultSetHelper;
@@ -59,6 +66,9 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
 	@Autowired
 	IDownloadTaskService downloadTaskService;
 
+	@Autowired
+	private EnterpriseService enterpriseService;
+
 	/** 获取指定脱敏类型的过滤字段集合
 	 * @param desensiType
 	 * @return
@@ -365,6 +375,11 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
 	}
 
 	@Override
+	public List<String> listFilterPhoneAndCardNoCardNumField() {
+		return FilterFieldUtils.FILTERS_PHONE_AND_CARD_COUPON;
+	}
+
+	@Override
 	public List<String> listFilterUserName() {
 		return FilterFieldUtils.FILETERS_USER_NAME;
 	}
@@ -555,10 +570,12 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
 //											zos.close();
 //										}
 //										log.debug("开始上传压缩文件到腾讯云", task.getId());
-										cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
+//										cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
+										cloudFileUrl = cloudFileUpload(task.getEnterpriseId(), zipFile, taskFileExt.substring(1));
 									} else {
 										log.debug("开始上传文件到腾讯云", task.getId());
-										cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
+//										cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(new File(originalFilePath), task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.REPORT_50000.getName());
+										cloudFileUrl = cloudFileUpload(task.getEnterpriseId(), new File(originalFilePath), taskFileExt.substring(1));
 									}
 									log.debug("上传腾讯云", "地址为："+cloudFileUrl);
 									task.setStatus(DownloadTaskStatus.COMPLISHED);
@@ -591,6 +608,15 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
 		}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
 	}
 
+	private String cloudFileUpload(String enterpriseId,File file,String fileExt){
+		EnterpriseDTO enterprise = enterpriseService.getEnterpriseById(enterpriseId);
+		CloudFileInfo zip = CloudFileUtil.uploadFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON);
+		FileRecordLogUtil.recordUseLog("FreeQueryServiceImpl#cloudFileUpload||"+fileExt);
+		return zip.getOrgFileUrl();
+	}
+
+
+
 	private void saveXlsx(FreeQueryTaskCondition condition,String originalFilePath,ResultSet rs,DownloadTask task) throws Exception{
 		Integer limitSize = FileUtil.getLimitSize();
 //		originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";

gic-cloud-data-hook-service/src/main/resources/data-hook-service-dubbo-config.xml

@@ -35,6 +35,8 @@
 
     <dubbo:service interface="com.gic.cloud.data.hook.api.service.TableSyncRecordService" ref="tableSyncRecordServiceImpl"/>
 
+    <dubbo:reference interface="com.gic.enterprise.api.service.EnterpriseService" id="enterpriseService" timeout="10000" />
+
     <!-- 引用的 Dubbo 服务 -->
     <!--<dubbo:reference interface="com.gic.dict.api.service.ManagerDictService" id="managerDictService" timeout="10000" />
     <dubbo:reference interface="com.gic.cloud.communicate.api.service.member.MemberTagFieldService" id="memberTagFieldService" timeout="10000" />

gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/web/FlatQueryController.java

@@ -297,6 +297,13 @@ public class FlatQueryController {
 						condition.setEnableEncrypt(false);
 					}
 				}
+				if(dataPermission == 3){
+					if(freeQueryService.listFilterPhoneAndCardNoCardNumField().contains(condition.getFieldName())){
+						condition.setEnableEncrypt(true);
+					} else {
+						condition.setEnableEncrypt(false);
+					}
+				}
 			}
 		}
 		return this.flatQueryResultService.buildFlatQueryDownloadTask(