Skip to content

G
gic-cloud
Commit e7679764 by fudahua
Options

Merge branch 'feature-2023-04' into 'master' 

Feature 2023 04

See merge request !94
parents 8330f895 fced8c62
Showing with 945 additions and 76 deletions
gic-cloud-data-hook-api/.flattened-pom.xml
......@@ -4,12 +4,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.gic</groupId>
<artifactId>gic-cloud-data-hook-api</artifactId>
<version>2.42</version>
<version>3.0-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-cloud-web-service-api</artifactId>
<version>3.80</version>
<version>3.0-SNAPSHOT</version>
<scope>compile</scope>
<exclusions>
<exclusion>
......@@ -25,7 +25,7 @@
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-cloud-commons</artifactId>
<version>3.64</version>
<version>3.0-SNAPSHOT</version>
<scope>compile</scope>
<exclusions>
<exclusion>
......@@ -41,7 +41,7 @@
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-platform-config</artifactId>
<version>2.29</version>
<version>3.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
......
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/entity/DownloadTaskStatus.java
......@@ -8,6 +8,9 @@ public class DownloadTaskStatus {
/** 排队中 */
public static final String WAITING = "waiting";
/** hdfs已经处理完等待下载 */
public static final String DOWNLOAD_HDFS = "downloading";
/** 生成中 */
public static final String BUILDING = "building";
......
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/service/IDownloadTaskService.java
......@@ -69,4 +69,6 @@ public interface IDownloadTaskService {
DownloadProcessDTO getDownloadProcess();
void checkTaskStatus(String param);
public List<DownloadTask> getDownloadTaskOfBuilding(String queryDataSource);
}
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/service/IFlatQueryResultService.java
......@@ -11,6 +11,12 @@ import java.util.List;
*/
public interface IFlatQueryResultService {
/**
* mq处理
* @param params
*/
public void dealFileMq(String params) ;
/** 自助指标查询
* @param tableId 指定的表明
* @param enterpriseIds 要查询的企业编号集合
......
gic-cloud-data-hook-service/dubbo/local/gic-member/dubbo.cache
#Dubbo Registry Cache
#Thu Sep 09 18:47:36 CST 2021
com.gic.cloud.data.hook.api.service.IDownloadTaskService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IDownloadTaskService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IDownloadTaskService&methods\=createRiskModeRecord,createDownloadRecord,getRiskModeRecordPage,listUnDownloadTask,getDownloadRecordPage,deleteDownloadTask,getDownloadProcess,getDownloadTaskPage&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455577
com.gic.cloud.data.hook.api.service.SearchLogService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.SearchLogService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.SearchLogService&methods\=saveLog&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455823
com.gic.cloud.data.hook.api.service.IFlatQueryTableService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryTableService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryTableService&methods\=queryFlatQueryFavo,queryFlatQueryTableById,queryFlatQueryTablePage,createFlatQueryFavo,removeFlatQueryFavo,queryFlatQueryTableDetailByTableId&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455351
com.gic.cloud.data.hook.api.service.TableSyncRecordService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.TableSyncRecordService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.TableSyncRecordService&methods\=lastSyncDateTime&payload\=41557050&pid\=47060&side\=provider&timestamp\=1631184456001
com.gic.cloud.data.hook.api.service.IFlatQueryResultService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryResultService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryResultService&methods\=getFlatQueryFilterData,initTask,estimateFlatQueryDownloadTask,buildFlatQueryDownloadTask,getFlatQueryResult&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455489
com.gic.cloud.data.hook.api.service.IMyFlatQueryService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IMyFlatQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IMyFlatQueryService&methods\=queryMyFlatQueryPage,createMyFlatQuery,deleteMyFlatQuery,updateMyFlatQuery,queryMyFlatQueryById&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455740
com.gic.cloud.data.hook.api.service.IFreeQueryService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFreeQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFreeQueryService&methods\=getFreeQueryRecordByUserIdAndRecordId,createFreeQueryRecord,buildFreeQueryDownloadTask,deleteFreeQueryRecord,getColumnsByTable,updateFreeQueryRecord,getFreeQueryResult,getFreeQuerySource,initTask,getFreeQueryCount,getFreeQueryRecordListByUserId,getFreeQueryTableList&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184454144
#Fri Apr 07 16:28:40 CST 2023
com.gic.cloud.data.hook.api.service.SearchLogService=empty\://192.168.200.53\:20338/com.gic.cloud.data.hook.api.service.SearchLogService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.SearchLogService&methods\=saveLog&payload\=41557050&pid\=37324&side\=provider&timeout\=120000&timestamp\=1680856120390
com.gic.cloud.data.hook.api.service.IDownloadTaskService=empty\://192.168.200.53\:20338/com.gic.cloud.data.hook.api.service.IDownloadTaskService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IDownloadTaskService&methods\=createRiskModeRecord,checkTaskStatus,createDownloadRecord,listUnDownloadTask,getRiskModeRecordPage,getDownloadRecordPage,deleteDownloadTask,getDownloadProcess,getDownloadTaskPage&payload\=41557050&pid\=37324&side\=provider&timeout\=120000&timestamp\=1680856120340
com.gic.cloud.data.hook.api.service.IFlatQueryTableService=empty\://192.168.200.53\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryTableService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryTableService&methods\=queryFlatQueryFavo,queryFlatQueryTableById,queryFlatQueryTablePage,createFlatQueryFavo,queryFlatQueryTableDetailByTableId,removeFlatQueryFavo&payload\=41557050&pid\=37324&side\=provider&timeout\=120000&timestamp\=1680856120293
com.gic.enterprise.api.service.EnterpriseService=empty\://192.168.200.53/com.gic.enterprise.api.service.EnterpriseService?application\=gic-cloud-data-hook&category\=configurators&default.check\=false&default.loadbalance\=traceid&default.retries\=0&default.timeout\=10000&dubbo\=2.8.5&interface\=com.gic.enterprise.api.service.EnterpriseService&methods\=queryTelTaskLog,getEnterpriseIdBySmsSignText,isWmAppid,getEnterpriseByFactoryCode,updateHeartbeatTime,getEnterpriseAndUserByName,isExitFactoryCode,reFreshMemberSystemInfo,activityAnnounceClerkByMQ,syncToWeimob,saveAnnounceClerkLogByMQ,queryEnterpriseImplementList,findEnterpriseIdsAndNamesByCliqueId,listAllMemberErpField,getByWxappkey,sendDailyReportToAllStoreClerk,getAllEnterpriseList,listEnterprise,initEnterprise,dealSingleStoredOldData,getByWxappkeyForfwh,listCliqueBrandByEnterpriseId,saveAnnounceAndSendMsg,executeDataUpReport,getEnterpriseMemberAppSetting,listErpField,listCliqueEnterprise,isOpenClique,createTask,getEnterPriseByCorpId,getRelOpenCardList,getEnterpriseIdsByFactoryCode,repairTabGicPaidAppConfig,saveEnterpriseInfo,getAnnounceClerkExtendList,queryEnterpriseAccountStandard,updateAuthTypeByAppkey,getEnterpriseIdByQrcodeParams,deleteWaitTaskAnnounceByEnterpriseId,saveAnnounceClerkExtend,updateEnterpriseClique,getEnterPriseByAppKey,listEnterpriseAndClique,isContainInJNBY,saveEnterpriseMemberAppSetting,getEnterpriseWxaRelationList,queryStorePerformanceInfo,getAccompanyOpenidByEnterprise,activityAnnounceClerk,saveOrUpdateEnterWxaRelation,saveAnnounceClerkLog,queryAppId,getEnterpriseByIdNoStatus,saveRelOpenCardList,getUpRoleName,getDataPermissionType,getOnLiveStoreNumByEnterprise,outageEnterprise,reFreshTemplateAdvice,updateEnterPriseBydto,queryCompetitorList,getEnterpriseByEid,createStorePerformance,jpushByRegistrationIdByMQ,isExists,updateEnterpriseCustomSettingForKey,listSubEnterprise,reFreshEntetrpriseInfo,updateEnabledState,getEnterprise2OrderDTO,saveCliqueInfo,updateEnterpriseSettingInfoByEid,getEnterpriseSettingByEnterpriseId,getEnterpriseByWebankAppid,selectEnterpriseById,getEnterpriseByName,getEnterpriseCustomSettingForKey,getEnterpriseIdList,queryAuthImg,getEnterpriseCustomSetting,getEnterpriseSettingByEid,getEnterprise2OrderDTONoProcess,createStorePerformanceByMQ,getEnterpriseIdBySourceChannel,createTaskLog,listByEnterpriseId,updateTablesCliqueMemberId,updateEnterprise,getEnterpriseById,createStoreYmByStoreIdAndClerkId,getCallingPhoneByEnterpriseId,taskCount,updateTask,updateClerkTaskLog,isSycYz,getEnterPriseByMS,modifyEnterprise,getOpenClique,queryTaskRecordLog,queryAnnounceReadCount,sendWaitingTaskAnnounce,updateCliqueStatus,hasMembershipLevel,getEnterpriseIdsByIntegralFlag,updateEntDataCountFlag,getEnterpriseFromMemCacById,hasStore,queryTemplateAdvice,saveAnnouncement,getEnterprise2ERPDTOBList,getEnterpriseAppkeyList,queryTelTaskRecordLog,updateEnterpriseCustomSetting,pageClique,isExistSmsSign,queryErpType,getTemplateAdvice,enterpriseBusiness,reFreshEnterpriseSettingInfo,queryEnterpriseListByIds,listEnterpriseByIds,hasIntegralSettlement,selectEnterTransacationDTOByEnterpriseId,getMemberSystem,updateEnterpriseExpire,saveErpFieldList,updateEnterpriseSettingWeimobByEid,queryByAuthType,getEnterpriseList,deleteTask,modifyEnterpriseAppkey&pid\=37324&revision\=3.0-20230406.121146-2739&side\=consumer&timeout\=10000&timestamp\=1680856105178 empty\://192.168.200.53/com.gic.enterprise.api.service.EnterpriseService?application\=gic-cloud-data-hook&category\=routers&default.check\=false&default.loadbalance\=traceid&default.retries\=0&default.timeout\=10000&dubbo\=2.8.5&interface\=com.gic.enterprise.api.service.EnterpriseService&methods\=queryTelTaskLog,getEnterpriseIdBySmsSignText,isWmAppid,getEnterpriseByFactoryCode,updateHeartbeatTime,getEnterpriseAndUserByName,isExitFactoryCode,reFreshMemberSystemInfo,activityAnnounceClerkByMQ,syncToWeimob,saveAnnounceClerkLogByMQ,queryEnterpriseImplementList,findEnterpriseIdsAndNamesByCliqueId,listAllMemberErpField,getByWxappkey,sendDailyReportToAllStoreClerk,getAllEnterpriseList,listEnterprise,initEnterprise,dealSingleStoredOldData,getByWxappkeyForfwh,listCliqueBrandByEnterpriseId,saveAnnounceAndSendMsg,executeDataUpReport,getEnterpriseMemberAppSetting,listErpField,listCliqueEnterprise,isOpenClique,createTask,getEnterPriseByCorpId,getRelOpenCardList,getEnterpriseIdsByFactoryCode,repairTabGicPaidAppConfig,saveEnterpriseInfo,getAnnounceClerkExtendList,queryEnterpriseAccountStandard,updateAuthTypeByAppkey,getEnterpriseIdByQrcodeParams,deleteWaitTaskAnnounceByEnterpriseId,saveAnnounceClerkExtend,updateEnterpriseClique,getEnterPriseByAppKey,listEnterpriseAndClique,isContainInJNBY,saveEnterpriseMemberAppSetting,getEnterpriseWxaRelationList,queryStorePerformanceInfo,getAccompanyOpenidByEnterprise,activityAnnounceClerk,saveOrUpdateEnterWxaRelation,saveAnnounceClerkLog,queryAppId,getEnterpriseByIdNoStatus,saveRelOpenCardList,getUpRoleName,getDataPermissionType,getOnLiveStoreNumByEnterprise,outageEnterprise,reFreshTemplateAdvice,updateEnterPriseBydto,queryCompetitorList,getEnterpriseByEid,createStorePerformance,jpushByRegistrationIdByMQ,isExists,updateEnterpriseCustomSettingForKey,listSubEnterprise,reFreshEntetrpriseInfo,updateEnabledState,getEnterprise2OrderDTO,saveCliqueInfo,updateEnterpriseSettingInfoByEid,getEnterpriseSettingByEnterpriseId,getEnterpriseByWebankAppid,selectEnterpriseById,getEnterpriseByName,getEnterpriseCustomSettingForKey,getEnterpriseIdList,queryAuthImg,getEnterpriseCustomSetting,getEnterpriseSettingByEid,getEnterprise2OrderDTONoProcess,createStorePerformanceByMQ,getEnterpriseIdBySourceChannel,createTaskLog,listByEnterpriseId,updateTablesCliqueMemberId,updateEnterprise,getEnterpriseById,createStoreYmByStoreIdAndClerkId,getCallingPhoneByEnterpriseId,taskCount,updateTask,updateClerkTaskLog,isSycYz,getEnterPriseByMS,modifyEnterprise,getOpenClique,queryTaskRecordLog,queryAnnounceReadCount,sendWaitingTaskAnnounce,updateCliqueStatus,hasMembershipLevel,getEnterpriseIdsByIntegralFlag,updateEntDataCountFlag,getEnterpriseFromMemCacById,hasStore,queryTemplateAdvice,saveAnnouncement,getEnterprise2ERPDTOBList,getEnterpriseAppkeyList,queryTelTaskRecordLog,updateEnterpriseCustomSetting,pageClique,isExistSmsSign,queryErpType,getTemplateAdvice,enterpriseBusiness,reFreshEnterpriseSettingInfo,queryEnterpriseListByIds,listEnterpriseByIds,hasIntegralSettlement,selectEnterTransacationDTOByEnterpriseId,getMemberSystem,updateEnterpriseExpire,saveErpFieldList,updateEnterpriseSettingWeimobByEid,queryByAuthType,getEnterpriseList,deleteTask,modifyEnterpriseAppkey&pid\=37324&revision\=3.0-20230406.121146-2739&side\=consumer&timeout\=10000&timestamp\=1680856105178 empty\://192.168.200.53/com.gic.enterprise.api.service.EnterpriseService?application\=gic-cloud-data-hook&category\=providers&default.check\=false&default.loadbalance\=traceid&default.retries\=0&default.timeout\=10000&dubbo\=2.8.5&interface\=com.gic.enterprise.api.service.EnterpriseService&methods\=queryTelTaskLog,getEnterpriseIdBySmsSignText,isWmAppid,getEnterpriseByFactoryCode,updateHeartbeatTime,getEnterpriseAndUserByName,isExitFactoryCode,reFreshMemberSystemInfo,activityAnnounceClerkByMQ,syncToWeimob,saveAnnounceClerkLogByMQ,queryEnterpriseImplementList,findEnterpriseIdsAndNamesByCliqueId,listAllMemberErpField,getByWxappkey,sendDailyReportToAllStoreClerk,getAllEnterpriseList,listEnterprise,initEnterprise,dealSingleStoredOldData,getByWxappkeyForfwh,listCliqueBrandByEnterpriseId,saveAnnounceAndSendMsg,executeDataUpReport,getEnterpriseMemberAppSetting,listErpField,listCliqueEnterprise,isOpenClique,createTask,getEnterPriseByCorpId,getRelOpenCardList,getEnterpriseIdsByFactoryCode,repairTabGicPaidAppConfig,saveEnterpriseInfo,getAnnounceClerkExtendList,queryEnterpriseAccountStandard,updateAuthTypeByAppkey,getEnterpriseIdByQrcodeParams,deleteWaitTaskAnnounceByEnterpriseId,saveAnnounceClerkExtend,updateEnterpriseClique,getEnterPriseByAppKey,listEnterpriseAndClique,isContainInJNBY,saveEnterpriseMemberAppSetting,getEnterpriseWxaRelationList,queryStorePerformanceInfo,getAccompanyOpenidByEnterprise,activityAnnounceClerk,saveOrUpdateEnterWxaRelation,saveAnnounceClerkLog,queryAppId,getEnterpriseByIdNoStatus,saveRelOpenCardList,getUpRoleName,getDataPermissionType,getOnLiveStoreNumByEnterprise,outageEnterprise,reFreshTemplateAdvice,updateEnterPriseBydto,queryCompetitorList,getEnterpriseByEid,createStorePerformance,jpushByRegistrationIdByMQ,isExists,updateEnterpriseCustomSettingForKey,listSubEnterprise,reFreshEntetrpriseInfo,updateEnabledState,getEnterprise2OrderDTO,saveCliqueInfo,updateEnterpriseSettingInfoByEid,getEnterpriseSettingByEnterpriseId,getEnterpriseByWebankAppid,selectEnterpriseById,getEnterpriseByName,getEnterpriseCustomSettingForKey,getEnterpriseIdList,queryAuthImg,getEnterpriseCustomSetting,getEnterpriseSettingByEid,getEnterprise2OrderDTONoProcess,createStorePerformanceByMQ,getEnterpriseIdBySourceChannel,createTaskLog,listByEnterpriseId,updateTablesCliqueMemberId,updateEnterprise,getEnterpriseById,createStoreYmByStoreIdAndClerkId,getCallingPhoneByEnterpriseId,taskCount,updateTask,updateClerkTaskLog,isSycYz,getEnterPriseByMS,modifyEnterprise,getOpenClique,queryTaskRecordLog,queryAnnounceReadCount,sendWaitingTaskAnnounce,updateCliqueStatus,hasMembershipLevel,getEnterpriseIdsByIntegralFlag,updateEntDataCountFlag,getEnterpriseFromMemCacById,hasStore,queryTemplateAdvice,saveAnnouncement,getEnterprise2ERPDTOBList,getEnterpriseAppkeyList,queryTelTaskRecordLog,updateEnterpriseCustomSetting,pageClique,isExistSmsSign,queryErpType,getTemplateAdvice,enterpriseBusiness,reFreshEnterpriseSettingInfo,queryEnterpriseListByIds,listEnterpriseByIds,hasIntegralSettlement,selectEnterTransacationDTOByEnterpriseId,getMemberSystem,updateEnterpriseExpire,saveErpFieldList,updateEnterpriseSettingWeimobByEid,queryByAuthType,getEnterpriseList,deleteTask,modifyEnterpriseAppkey&pid\=37324&revision\=3.0-20230406.121146-2739&side\=consumer&timeout\=10000&timestamp\=1680856105178
com.gic.cloud.data.hook.api.service.TableSyncRecordService=empty\://192.168.200.53\:20338/com.gic.cloud.data.hook.api.service.TableSyncRecordService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.TableSyncRecordService&methods\=lastSyncDateTime&payload\=41557050&pid\=37324&side\=provider&timestamp\=1680856120415
com.gic.cloud.data.hook.api.service.IFlatQueryResultService=empty\://192.168.200.53\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryResultService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryResultService&methods\=getFlatQueryFilterData,initTask,estimateFlatQueryDownloadTask,buildFlatQueryDownloadTask,getFlatQueryResult&payload\=41557050&pid\=37324&side\=provider&timeout\=120000&timestamp\=1680856120315
com.gic.cloud.data.hook.api.service.IMyFlatQueryService=empty\://192.168.200.53\:20338/com.gic.cloud.data.hook.api.service.IMyFlatQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IMyFlatQueryService&methods\=createMyFlatQuery,queryMyFlatQueryPage,delMyFields,deleteMyFlatQuery,updateMyFlatQuery,queryMyFlatQueryById&payload\=41557050&pid\=37324&side\=provider&timeout\=120000&timestamp\=1680856120363
com.gic.cloud.data.hook.api.service.IFreeQueryService=empty\://192.168.200.53\:20338/com.gic.cloud.data.hook.api.service.IFreeQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFreeQueryService&methods\=getFreeQueryRecordByUserIdAndRecordId,listFilterUserName,createFreeQueryRecord,deleteFreeQueryRecord,getColumnsByTable,buildFreeQueryDownloadTask,updateFreeQueryRecord,getFreeQueryResult,initTask,getFreeQuerySource,listFilterPhoneAndCardNoField,getFreeQueryCount,listFilterPhoneAndCardNoCardNumField,listFilterPhoneField,getFreeQueryRecordListByUserId,getFreeQueryTableList&payload\=41557050&pid\=37324&side\=provider&timeout\=120000&timestamp\=1680856120144
gic-cloud-data-hook-service/pom.xml
......@@ -146,6 +146,16 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoopCommonVersion}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoopCommonVersion}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoopCommonVersion}</version>
<exclusions>
......@@ -188,7 +198,11 @@
<artifactId>gic-enterprise-api</artifactId>
<version>${gic-enterprise-api}</version>
</dependency>
<dependency>
<groupId>com.gic</groupId>
<artifactId>gic-redis-data</artifactId>
<version>${gic-redis-data}</version>
</dependency>
</dependencies>
<build>
......@@ -199,8 +213,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<compilerArguments>
<verbose/>
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DecryptUtils.java
......@@ -3,6 +3,7 @@ package com.gic.cloud.data.hook.service;
import com.alibaba.dubbo.common.utils.StringUtils;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.cloud.data.hook.api.dto.QueryDataType;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.service.impl.FreeQueryServiceImpl;
import com.gic.data.shield.SdkEnv;
......@@ -102,4 +103,26 @@ public class DecryptUtils {
}
}
/**
* 脱敏
* @param queryDataType
* @param columnVal
* @param condition
* @return
*/
public static String encryptionDeal(int queryDataType,String columnVal,FlatQueryCondition condition){
if(queryDataType == QueryDataType.FULL){
return columnVal;
} else {
if(condition.getEnableEncrypt()) {
return "******";
}else if (FilterFieldUtils.FILETERS_USER_NAME.contains(condition.getFieldName())) {
return DecryptUtils.dataSecurityProcessUserName(columnVal);
} else {
return columnVal;
}
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DownloadFunc.java 0 → 100644
package com.gic.cloud.data.hook.service;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import java.util.List;
public interface DownloadFunc {
public void deal(String[] cells, List<FlatQueryCondition> titles, boolean fileFirst);
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/HDFSUtil.java 0 → 100644
package com.gic.cloud.data.hook.service;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
public class HDFSUtil {
private static Logger logger = LoggerFactory.getLogger(HDFSUtil.class);
private static HDFSUtil hdfsUtil=null;
private static FileSystem fileSystem=null;
public static HDFSUtil getInstance(){
if (hdfsUtil==null) {
synchronized (HDFSUtil.class) {
if (hdfsUtil==null) {
hdfsUtil = new HDFSUtil();
}
}
}
return hdfsUtil;
}
private HDFSUtil(){
Config appConfig = ConfigService.getAppConfig();
String hdfsUrl = appConfig.getProperty("hdfs.url", null);
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS",hdfsUrl);
try {
fileSystem = FileSystem.get(new URI(hdfsUrl),configuration,"hadoop");
} catch (Exception e) {
logger.info("hdfs初始化失败-{}",e);
}
}
/**
* 下载到本地
* @param srcPath
* @param toPath
* @return
*/
public boolean downloadFile(String srcPath,String toPath) {
try {
fileSystem.copyToLocalFile(true,new Path(srcPath),new Path(toPath));
return true;
} catch (IOException e) {
logger.info("下载失败：{}",e);
return false;
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/dao/DownloadTaskDao.java
......@@ -62,8 +62,23 @@ public interface DownloadTaskDao {
*/
public List<DownloadTask> getDownloadTaskOfWaiting(@Param("queryDataSource") String queryDataSource);
/** 获取等待申请通过状态的任务
* @return
*/
public List<DownloadTask> getDownloadTaskOfHasDownload(@Param("queryDataSource") String queryDataSource, @Param("num") int num);
DownloadProcessDTO getDownloadProcess();
int updateTaskStatusError(@Param("idList") List<String> idList);
/** 获取等待申请通过状态的任务
* @return
*/
public int getCountDownloadTaskOfBuilding(@Param("queryDataSource") String queryDataSource);
/** 获取等待申请通过状态的任务
* @return
*/
public List<DownloadTask> getDownloadTaskOfBuilding(@Param("queryDataSource") String queryDataSource);
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/entity/ColumnInfo.java 0 → 100644
package com.gic.cloud.data.hook.service.entity;
import java.io.Serializable;
public class ColumnInfo implements Serializable {
private String type;
private String title;
public ColumnInfo() {
}
public ColumnInfo(String type, String title) {
this.type = type;
this.title = title;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java
......@@ -14,6 +14,7 @@ import com.gic.cloud.data.hook.service.MysqlHelper;
import com.gic.cloud.data.hook.service.dao.DownloadRecordDao;
import com.gic.cloud.data.hook.service.dao.DownloadTaskDao;
import com.gic.cloud.data.hook.service.dao.RiskModeRecordDao;
import com.gic.thirdparty.cloudfile.CloudFileUtil;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.opencsv.CSVWriter;
......@@ -104,6 +105,10 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
if (CollectionUtils.isNotEmpty(result.getList())) {
for (DownloadTask downloadTask : result.getList()) {
//下载中也是等待中 用于前端展示
if (downloadTask.getStatus().equals(DownloadTaskStatus.DOWNLOAD_HDFS)){
downloadTask.setStatus(DownloadTaskStatus.WAITING);
}
if (StringUtils.isBlank(downloadTask.getFilePath())) {
continue;
}
......@@ -159,6 +164,7 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
}
}
this.downloadTaskDao.deleteDownloadTask(task.getId());
CloudFileUtil.delFileByUrl(task.getFilePath());
return true;
} else return false;
}
......@@ -228,6 +234,27 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
return downloadTaskDao.getDownloadTaskOfWaiting(queryDataSource);
}
/** 获取在审核申请等待状态中的任务
* @return
*/
public List<DownloadTask> getDownloadTaskOfHasDownload(String queryDataSource,int num) {
return downloadTaskDao.getDownloadTaskOfHasDownload(queryDataSource, num);
}
/** 获取在审核申请等待状态中的任务
* @return
*/
public int getCountDownloadTaskOfBuilding(String queryDataSource) {
return downloadTaskDao.getCountDownloadTaskOfBuilding(queryDataSource);
}
/** 获取在审核申请等待状态中的任务
* @return
*/
public List<DownloadTask> getDownloadTaskOfBuilding(String queryDataSource) {
return downloadTaskDao.getDownloadTaskOfBuilding(queryDataSource);
}
/** 获取指定申请编号的风险模式记录
* @param applyId
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
package com.gic.cloud.data.hook.service.impl;
import cn.hutool.core.date.StopWatch;
import cn.medubi.client.utils.LogPak;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
......@@ -11,20 +12,24 @@ import com.gic.cloud.data.hook.api.service.DecryptKeyService;
import com.gic.cloud.data.hook.api.service.IDownloadTaskService;
import com.gic.cloud.data.hook.api.service.IFlatQueryResultService;
import com.gic.cloud.data.hook.service.*;
import com.gic.cloud.data.hook.service.dao.FlatQueryFieldDao;
import com.gic.cloud.data.hook.service.dao.FlatQueryTableDao;
import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
import com.gic.cloud.data.hook.service.entity.CsvResultSetHelper;
import com.gic.commons.util.GICMQClientUtil;
import com.gic.dubbo.entity.ProviderLocalTag;
import com.gic.enterprise.api.dto.EnterpriseDTO;
import com.gic.enterprise.api.service.EnterpriseService;
import com.gic.qcloud.BucketNameEnum;
import com.gic.mq.sdk.GicMQClient;
import com.gic.qcloud.FileRecordLogUtil;
import com.gic.qcloud.FileUploadUtil;
import com.gic.redis.data.util.RedisUtil;
import com.gic.thirdparty.cloudfile.CloudFileUtil;
import com.gic.thirdparty.cloudfile.enums.CloudFileBusinessOptEnum;
import com.gic.thirdparty.cloudfile.enums.CloudFileTypeEnum;
import com.gic.thirdparty.cloudfile.pojo.CloudFileInfo;
import com.google.common.collect.Lists;
import com.opencsv.CSVReader;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
import org.apache.commons.collections.CollectionUtils;
......@@ -32,11 +37,6 @@ import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFCellStyle;
import org.apache.poi.hssf.usermodel.HSSFDataFormat;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.slf4j.Logger;
......@@ -44,13 +44,9 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import javax.annotation.PostConstruct;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.*;
import java.nio.charset.Charset;
import java.sql.*;
import java.text.DecimalFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
......@@ -59,6 +55,9 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/** 自助指标查询服务实现
* @author Sam
......@@ -72,6 +71,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
/** csv / xls 下载目录 */
public static final String SAVE_FOLDER = "/usr/local/data-hook-file";
// public static final String SAVE_FOLDER = "D:\\testorder";
public static final String HDFS_URL = "/data/hook";
public static final String LOCK_KEY="data:hook:hive";
public static final List<String> PHONE = Arrays.asList("mobile", "phone", "phone_number", "receive_phone_number");
......@@ -92,6 +97,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
@Autowired
DecryptKeyService decryptKeyService;
/** 自助查询字段 DAO */
@Autowired
private FlatQueryFieldDao flatQueryFieldDao;
@Autowired
private EnterpriseService enterpriseService;
......@@ -102,20 +111,30 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private FlatQueryResultServiceImpl() {
log.debug("construct", "准备初始化 FlatQuery 查询服务");
runDistTask(3);
runDownloadTask(3);
runBalaDownloadTask(3);
runBigDataDownloadTask(3);
runDealHiveFile(3);
// runDistTask(3);
// runDownloadTask(3);
// runBalaDownloadTask(3);
// runBigDataDownloadTask(3);
runApplyTask(5); // 每5秒钟进行任务状态检测
}
private String cloudFileUpload(String enterpriseId,File file,String fileExt){
EnterpriseDTO enterprise = enterpriseService.getEnterpriseById(enterpriseId);
CloudFileInfo zip = CloudFileUtil.uploadFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON);
CloudFileInfo zip = CloudFileUtil.uploadTempFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON,2L);
FileRecordLogUtil.recordUseLog("FlatQueryResultServiceImpl#cloudFileUpload||"+fileExt);
return zip.getOrgFileUrl();
}
@Override
public void dealFileMq(String params) {
logger.info("处理文件：{}",params);
DownloadTask downloadTask = JSON.parseObject(params, DownloadTask.class);
String key=LOCK_KEY+":"+downloadTask.getId();
RedisUtil.delCache(key);
takeFileNew(downloadTask);
}
/** 生成自助指标查询最终 SQL
* @param queryOrDownload 查询或下载用途
* @param tableId
......@@ -520,7 +539,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
curTask.setApplyPermitted(Global.NO); // 完整模式默认不允许
curTask.setApplyStatus(DownloadApplyStatus.WAITING);
} // IF ELSE OVER
curTask.setRealSql(""); // 设置查询为空
// curTask.setRealSql(""); // 设置查询为空
curTask.setEnterpriseId(enterpriseIds.get(0));
Integer preReportCount = DownloadTaskServiceImpl.getInstance().getDownloadTaskCountByEnterpriseId(curTask.getEnterpriseId());
curTask.setReportId(ServiceUtils.buildReportId(curTask.getQueryDataType(), preReportCount));
......@@ -544,37 +563,74 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
condition.setAuthStoreIdList(authStoreIdList);
// 保存下载任务
curTask.setDownloadCondition(JSON.toJSONString(condition));
//sql
String fullQuery = buildFlatQuerySQL(
false, // 下载用途
condition.getTableId(),
condition.getEnterpriseIds(),
condition.getConditions(),
condition.getOrderField(),
condition.getOrderDir(),
condition.getExecDistinct(),
0,
condition.getAuthStoreIdList());
curTask.setRealSql(fullQuery);
curTask.setFieldSize(condition.getAllFields().size());
DownloadTaskServiceImpl.getInstance().createDownloadTask(curTask);
// 保存至列表
this.taskConditions.add(condition);
// this.taskConditions.add(condition);
return curTask;
}
@Override
@PostConstruct
public void initTask(){
RedisUtil.lock(LOCK_KEY,3L);
try {
List<DownloadRecord> list = this.downloadTaskService.listUnDownloadTask(QueryDataSource.FLAT_QUERY);
list = DataInitUtils.listByHost(list);
log.debug("flatInitList", JSON.toJSONString(list));
if(CollectionUtils.isNotEmpty(list)){
for(DownloadRecord record : list){
if(StringUtils.isNotBlank(record.getDownloadCondition())){
FlatQueryTaskCondition condition = JSON.parseObject(record.getDownloadCondition(), FlatQueryTaskCondition.class);
if(record.getApplyStatus().equals(DownloadApplyStatus.TIMEOUT)){
continue;
}
if(condition.getBuildPermitted().equals(Global.NO)){
if(record.getApplyPermitted().equals(Global.YES)){
condition.setBuildPermitted(Global.YES);
}
}
this.taskConditions.add(condition);
}
// List<DownloadRecord> list = this.downloadTaskService.listUnDownloadTask(QueryDataSource.FLAT_QUERY);
// list = DataInitUtils.listByHost(list);
// log.debug("flatInitList", JSON.toJSONString(list));
// if(CollectionUtils.isNotEmpty(list)){
// for(DownloadRecord record : list){
// if(StringUtils.isNotBlank(record.getDownloadCondition())){
// FlatQueryTaskCondition condition = JSON.parseObject(record.getDownloadCondition(), FlatQueryTaskCondition.class);
// if(record.getApplyStatus().equals(DownloadApplyStatus.TIMEOUT)){
// continue;
// }
// if(condition.getBuildPermitted().equals(Global.NO)){
// if(record.getApplyPermitted().equals(Global.YES)){
// condition.setBuildPermitted(Global.YES);
// }
// }
// this.taskConditions.add(condition);
//
// }
// }
// }
//之前正在处理的队列需要重新处理
// List<DownloadRecord> list = this.downloadTaskService.listUnDownloadTask(QueryDataSource.FLAT_QUERY);
List<DownloadTask> tasks = DownloadTaskServiceImpl.getInstance().getDownloadTaskOfBuilding(QueryDataSource.FLAT_QUERY);
logger.info("初始化任务：{}",JSONObject.toJSONString(tasks));
for (DownloadTask task : tasks) {
// FlatQueryTaskCondition condition = JSON.parseObject(task.getDownloadCondition(), FlatQueryTaskCondition.class);
// DownloadTask downloadTask = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId());
String key=LOCK_KEY+":"+task.getId();
Object cache = RedisUtil.getCache(key);
if (cache!=null) {
continue;
}
delFileOrDirByTaskId(task.getId());
//重新处理
task.setStatus(DownloadTaskStatus.WAITING);
task.setDownloadTime(new Date());
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
}
}catch (Exception e){
e.printStackTrace();
}finally {
RedisUtil.unlock(LOCK_KEY);
}
}
......@@ -640,7 +696,53 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
}
private
/**
* 执行 下载
* @param interval
*/
public void runDealHiveFile(Integer interval) {
distService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
String traceId = UUID.randomUUID().toString();
ProviderLocalTag providerLocalTag = ProviderLocalTag.tag.get();
providerLocalTag.traceId = traceId;
logger.info("执行hivefile");
try{
RedisUtil.lock(LOCK_KEY,3L);
int curBuildingCount = DownloadTaskServiceImpl.getInstance().getCountDownloadTaskOfBuilding(QueryDataSource.FLAT_QUERY);
Config appConfig = ConfigService.getAppConfig();
Integer maxBuildingCount = appConfig.getIntProperty("buildingCount", 5);
if (maxBuildingCount<=curBuildingCount) {
logger.info("超出限制：{}>{}",curBuildingCount,maxBuildingCount);
RedisUtil.unlock(LOCK_KEY);
return;
}
List<DownloadTask> downloadTasks = DownloadTaskServiceImpl.getInstance().getDownloadTaskOfHasDownload(QueryDataSource.FLAT_QUERY,5);
if (CollectionUtils.isNotEmpty(downloadTasks)) {
for (DownloadTask downloadTask : downloadTasks) {
downloadTask.setStatus(DownloadTaskStatus.BUILDING);
downloadTask.setDownloadTime(new Date());
DownloadTaskServiceImpl.getInstance().updateDownloadTask(downloadTask);
GicMQClient clientInstance = GICMQClientUtil.getClientInstance();
clientInstance.sendMessage("dataHookFileDeal",JSONObject.toJSONString(downloadTask));
String key=LOCK_KEY+":"+downloadTask.getId();
RedisUtil.setCache(key,1,12*60*60L);
}
}
RedisUtil.unlock(LOCK_KEY);
}catch (Exception e){
logger.info("异常：{}",e);
logger.info("[ 自助指标下载异常 ]: {}", e.getMessage());
e.printStackTrace();
} finally {
}
}
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
}
/** 下载任务执行计时器 */
......@@ -800,6 +902,170 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
}
/**
* download下载
*/
public void takeFileNew(DownloadTask task) {
if (task==null) {
logger.info("不存在");
return;
}
Integer queryDataType = task.getQueryDataType();
FlatQueryTaskCondition condition = JSON.parseObject(task.getDownloadCondition(), FlatQueryTaskCondition.class);
if (!task.getStatus().equals(DownloadTaskStatus.BUILDING)) {
logger.info("文件hdfs没有处理完成：{}",JSONObject.toJSONString(task));
return;
}
task.setDownloadWay(-1);
task.setFieldSize(condition.getAllFields().size());
//下载文件
String dirName="hdfs"+task.getId();
String path=HDFS_URL+"/"+dirName;
try {
StopWatch stopWatch = StopWatch.create("down");
stopWatch.start();
logger.info("下载开始");
boolean downloadFlag = HDFSUtil.getInstance().downloadFile(path, SAVE_FOLDER);
if (!downloadFlag) {
logger.info("下载失败：{}-{}",path,JSONObject.toJSONString(task));
throw new RuntimeException("下载失败:"+condition.getTaskId());
}
stopWatch.stop();
logger.info("下载耗时：{}",stopWatch.getLastTaskTimeMillis());
stopWatch.start();
List<String> xlsxFiles=new ArrayList<>();
AtomicInteger count=new AtomicInteger(0);
AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
if (count.get()==0) {
XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
xlsxFileInfo.filepath=SAVE_FOLDER + "/" + task.getId() + xlsxFiles.size() + ".xlsx";
currentFile.set(xlsxFileInfo);
xlsxFiles.add(currentFile.get().filepath);
}
saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,queryDataType);
});
//结束
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType);
stopWatch.stop();
logger.info("写入本地excel耗时：{}",stopWatch.getLastTaskTimeMillis());
stopWatch.start();
//是否压缩
boolean zipFlag = (xlsxFiles.size() > 1) ? true : false;
String cloudFileUrl=null;
String taskFileExt = task.getUseCompress().equals(Global.YES) || zipFlag ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
if (zipFlag || task.getUseCompress().equals(Global.YES)) {
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标压缩文件 " + task.getId() + ".zip");
String zipFilePath = SAVE_FOLDER + "/" + task.getId() + ".zip";
OutputStream os = new FileOutputStream(zipFilePath);
File zipFile = new File(zipFilePath);
List<File> files = xlsxFiles.stream().map(mid -> new File(mid)).collect(Collectors.toList());
FileUtil.toZip(files, os);
logger.info("[ 开始上传压缩文件到腾讯云 ]: {}", task.getId());
cloudFileUrl = cloudFileUpload(task.getEnterpriseId(), zipFile, taskFileExt.substring(1));
} else {
logger.info("[ 开始上传文件到腾讯云 ]: {}", task.getId());
cloudFileUrl = cloudFileUpload(task.getEnterpriseId(), new File(currentFile.get().filepath), taskFileExt.substring(1));
}
stopWatch.stop();
logger.info("上传腾讯云耗时：{}",stopWatch.getLastTaskTimeMillis());
logger.info("[ 上传腾讯云 ]: {}", "地址为：" + cloudFileUrl);
task.setStatus(DownloadTaskStatus.COMPLISHED);
task.setOverTime(new Date());
task.setFilePath(cloudFileUrl);
}catch (Exception e) {
logger.info("异常：{}",e);
task.setStatus(DownloadTaskStatus.ERROR);
}finally {
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
//删除文件
delFileOrDirByTaskId(task.getId());
}
}
private class XlsxFileInfo{
String filepath;
SXSSFWorkbook workbook;
CellStyle yyyyMMddhhmmss;
CellStyle yyyyMMdd;
}
private void delFileOrDirByTaskId(String taskId){
File file = new File(SAVE_FOLDER);
File[] files = file.listFiles();
for (File midFile : files) {
if (midFile.getName().indexOf(taskId)!=-1) {
if (midFile.isDirectory()) {
delDir(midFile);
}else{
midFile.delete();
}
}
}
}
private void delDir(File dirFile) {
if (dirFile.getPath().startsWith(SAVE_FOLDER)) {
File[] files = dirFile.listFiles();
for (File file : files) {
if (file.isDirectory()) {
delDir(file);
}else{
file.delete();
}
}
dirFile.delete();
}
}
/**
* 读物文件
* @param dirName
* @param func
*/
private void readCsvFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles();
List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions();
Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) {
if (!midFile.getName().endsWith("csv")) {
continue;
}
try {
CSVReader csvReader = new CSVReader(new FileReader(midFile));
String[] cells = csvReader.readNext();
boolean first=true;
do {
if (titles == null) {
titles=new ArrayList<>();
for (String cell : cells) {
logger.info("cell:{}",cell);
titles.add(columnInfoMap.get(cell));
}
}
if (first) {
first=false;
continue;
}
func.deal(cells,titles,first);
}while ((cells = csvReader.readNext())!=null);
logger.info("读取结束：{}",midFile.getName());
csvReader.close();
} catch (Exception e) {
logger.info("读取异常：{}",e);
}
}
}
private void takeFile(FlatQueryTaskCondition condition, Connection conn, int downloadType){
if (condition != null) {
// 更新任务状态
......@@ -983,6 +1249,97 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
} // IF OVER
}
/**
* 写入excel
* @param originalFilePath
* @param cells
*/
private void saveXlsSplitNew(String originalFilePath, String[] cells, List<FlatQueryCondition> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count,boolean endFlag,int queryDataType){
try {
if (xlsxFileInfo.workbook==null) {
xlsxFileInfo.workbook=new SXSSFWorkbook(100);
Sheet sheet = xlsxFileInfo.workbook.createSheet();
Row row = sheet.createRow(0);
Cell cell;
logger.info("类型:{}",JSONObject.toJSONString(titles));
for(int j =0; j<titles.size(); j++){
cell = row.createCell(j);
cell.setCellValue(titles.get(j).getFieldMark());
}
//日期
CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle();
DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss"));
xlsxFileInfo.yyyyMMddhhmmss=yyyyMMddhhmmss;
//日期
CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle();
DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
xlsxFileInfo.yyyyMMdd=yyyyMMdd;
}
count.incrementAndGet();
SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
if (!endFlag) {
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(count.get());
for (int j = 0; j < cells.length; j++) {
FlatQueryCondition flatQueryCondition = titles.get(j);
String type = titles.get(j).getFieldType();
Cell midCell = row.createCell(j);
String columnValue = cells[j];
if (StringUtils.isBlank(columnValue)) {
midCell.setCellValue(columnValue);
continue;
}
switch (type) {
case FlatQueryFieldType.DATE:
if (columnValue.length() == 10) {
midCell.setCellStyle(xlsxFileInfo.yyyyMMdd);
midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd"}));
} else {
midCell.setCellStyle(xlsxFileInfo.yyyyMMddhhmmss);
midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd HH:mm:ss"}));
}
break;
case FlatQueryFieldType.NUMBER:
if (columnValue.indexOf(".") > 0) {
midCell.setCellValue(Double.valueOf(columnValue));
} else {
midCell.setCellValue(Long.valueOf(columnValue));
}
break;
default:
midCell.setCellValue(DecryptUtils.encryptionDeal(queryDataType,columnValue,flatQueryCondition));
}
}
}
Integer limitSize = FileUtil.getLimitSize();
int c = count.get();
if (c%5000==0) {
logger.info("进度：{}-{}",originalFilePath,c);
}
if (c>=limitSize.intValue()||endFlag) {
logger.info("{}-count:{}",originalFilePath,count.get());
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + originalFilePath + ".xlsx");
count.set(0);
}
}catch (Exception e) {
logger.info("异常：{}",e);
throw new RuntimeException(e);
}
}
private void saveXlsSplit(String originalFilePath,ResultSetHelper helper,ResultSet rs,FlatQueryTaskCondition condition){
try {
Integer limitSize = FileUtil.getLimitSize();
......@@ -1156,6 +1513,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}, interval*1000, interval * 1000, TimeUnit.MILLISECONDS);
}
/* *//** 运行下载申请任务
* @param interval
*//*
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
......@@ -610,7 +610,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
private String cloudFileUpload(String enterpriseId,File file,String fileExt){
EnterpriseDTO enterprise = enterpriseService.getEnterpriseById(enterpriseId);
CloudFileInfo zip = CloudFileUtil.uploadFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON);
CloudFileInfo zip = CloudFileUtil.uploadTempFile(file, fileExt, CloudFileTypeEnum.FILE, enterprise.getFactoryCode(), CloudFileBusinessOptEnum.HOOK_COMMON,2L);
FileRecordLogUtil.recordUseLog("FreeQueryServiceImpl#cloudFileUpload||"+fileExt);
return zip.getOrgFileUrl();
}
......
gic-cloud-data-hook-service/src/main/resources/applicationContext-conf.xml
......@@ -21,6 +21,7 @@
<!--<import resource="classpath:data-hook-service-dubbo-config.xml"/>-->
<!--<import resource="classpath:data-hook-flat-query-source.xml"/>-->
<!--<import resource="classpath:data-hook-free-query-source.xml"/>-->
<import resource="classpath*:redis-init.xml" />
<import resource="classpath*:data-hook-service-druid-prod.xml"/>
<import resource="classpath:data-hook-init.xml"/>
<import resource="classpath*:dubbo-setting.xml"/>
......
gic-cloud-data-hook-service/src/main/resources/mapper/DownloadTaskDao.xml
......@@ -30,7 +30,8 @@
q.apply_status,
q.real_sql,
q.enterprise_id,
q.report_id
q.report_id,
q.download_condition
</sql>
......@@ -187,6 +188,19 @@
AND q.del_flag = '0'
</select>
<select id="getDownloadTaskOfHasDownload" resultType="DownloadTask">
SELECT *
FROM
<include refid="queryTables"/>
<include refid="queryJoins"/>
WHERE
q.query_data_source = #{queryDataSource}
AND q.status = "downloading"
AND q.apply_permitted = 1
AND q.del_flag = '0'
order by start_time asc limit ${num}
</select>
<select id="getDownloadProcess" resultType="DownloadProcessDTO">
SELECT sum(amount) dataAmount,count(1) taskAmount
FROM
......@@ -207,4 +221,27 @@
</foreach>
</update>
<select id="getCountDownloadTaskOfBuilding" resultType="int">
SELECT count(*)
FROM
<include refid="queryTables"/>
<include refid="queryJoins"/>
WHERE
q.query_data_source = #{queryDataSource}
AND q.status = "building"
AND q.del_flag = '0'
</select>
<select id="getDownloadTaskOfBuilding" resultType="DownloadTask">
SELECT *
FROM
<include refid="queryTables"/>
<include refid="queryJoins"/>
WHERE
q.query_data_source = #{queryDataSource}
AND q.status = "building"
AND q.del_flag = '0'
order by start_time asc
</select>
</mapper>
\ No newline at end of file
gic-cloud-data-hook-service/src/test/java/ExceTest2.java deleted 100644 → 0
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import java.io.FileOutputStream;
import java.util.Date;
public class ExceTest2 {
public static void main(String[] args) {
int n=25;
int c=100;
Double aDouble = Double.valueOf((double) n / c);
System.out.println(aDouble);
}
}
gic-cloud-data-hook-service/src/test/java/ExcelTest.java
......@@ -21,10 +21,6 @@ public class ExcelTest {
DataFormat format = wb.createDataFormat();
cellStyle.setDataFormat(format.getFormat("yyyy-MM-dd HH:mm:ss"));
// CellStyle cellStyle = wb.createCellStyle();
// DataFormat format = wb.createDataFormat();
// cellStyle.setDataFormat(format.getFormat("yyyy-MM-dd HH:mm:ss"));
int n=0;
do {
Row row = sheet.createRow(n);
......
gic-cloud-data-hook-service/src/test/java/Test.java 0 → 100644
import com.alibaba.fastjson.JSON;
import com.gic.cloud.data.hook.api.dto.DownloadTask;
import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
import com.gic.cloud.data.hook.service.impl.DownloadTaskServiceImpl;
import com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl;
import com.gic.redis.data.util.RedisUtil;
import org.junit.runner.RunWith;
import org.redisson.api.RedissonClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {"classpath*:applicationContext-conf.xml"})
public class Test {
@Autowired
private FlatQueryResultServiceImpl flatQueryResultService;
@Autowired
private DownloadTaskServiceImpl downloadTaskService;
@org.junit.Test
public void test(){
DownloadTask downloadTask = downloadTaskService.getDownloadTaskById("1680844420436");
flatQueryResultService.takeFileNew(downloadTask);
try {
Thread.sleep(10000000000L);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
@org.junit.Test
public void test3(){
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext("redis-init.xml");
RedissonClient redisClient = RedisUtil.getRedisClient();
System.out.println(redisClient);
}
@org.junit.Test
public void test2(){
String json="{\"allFields\":[\"is_mbr\",\"main_store_name\",\"main_store_code\",\"card_num\",\"open_card_time\",\"clerk_name\",\"add_clerk_code\",\"clerk_store_name\",\"clerk_store_code\",\"add_buddy_time\",\"create_time\"],\"amount\":1169963,\"authStoreIdList\":[],\"buildPermitted\":\"1\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员\",\"fieldName\":\"is_mbr\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"是\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店名称\",\"fieldName\":\"main_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店code\",\"fieldName\":\"main_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"open_card_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购姓名\",\"fieldName\":\"clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购code\",\"fieldName\":\"add_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"导购门店名称\",\"fieldName\":\"clerk_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购所属门店code\",\"fieldName\":\"clerk_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"2023-02-28 23:59:59\",\"fieldMark\":\"添加好友时间\",\"fieldName\":\"add_buddy_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"2010-03-01 00:00:00\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好办导购与客户建立好友关系时间\",\"fieldName\":\"create_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[],\"enterpriseIds\":[\"ff80808180b3c54a0180bc3df3bb4bca\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":0,\"tableId\":\"extract_entwechat_detail_d\",\"taskId\":\"1678674610424\"}";
FlatQueryTaskCondition condition = JSON.parseObject(json, FlatQueryTaskCondition.class);
String fullQuery = flatQueryResultService.buildFlatQuerySQL(
false, // 下载用途
condition.getTableId(),
condition.getEnterpriseIds(),
condition.getConditions(),
condition.getOrderField(),
condition.getOrderDir(),
condition.getExecDistinct(),
0,
condition.getAuthStoreIdList());
System.out.println(fullQuery);
}
}
gic-cloud-data-hook-service/src/test/java/com/gic/ExceTest2.java 0 → 100644
package com.gic;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.api.entity.FlatQueryFieldType;
import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
import com.gic.cloud.data.hook.service.DownloadFunc;
import com.gic.cloud.data.hook.service.FileUtil;
import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import com.opencsv.CSVReader;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
public class ExceTest2 {
private static Logger logger = LoggerFactory.getLogger(ExceTest2.class);
public static final String SAVE_FOLDER = "D:\\testorder";
/**
* 读物文件
* @param dirName
* @param func
*/
private static void readCsvFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles();
List<ColumnInfo> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions();
Map<String, ColumnInfo> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid -> new ColumnInfo(mid.getFieldType(), mid.getFieldMark())));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) {
if (!midFile.getName().endsWith("csv")) {
continue;
}
try {
CSVReader csvReader = new CSVReader(new FileReader(midFile));
String[] cells = csvReader.readNext();
boolean first=true;
do {
if (titles == null) {
titles=new ArrayList<>();
for (String cell : cells) {
logger.info("cell:{}",cell);
titles.add(columnInfoMap.get(cell));
}
}
if (first) {
first=false;
continue;
}
// func.deal(cells,titles,first);
}while ((cells = csvReader.readNext())!=null);
logger.info("读取结束：{}",midFile.getName());
csvReader.close();
} catch (Exception e) {
logger.info("读取异常：{}",e);
}
}
}
private static class XlsxFileInfo{
String filepath;
SXSSFWorkbook workbook;
CellStyle yyyyMMddhhmmss;
CellStyle yyyyMMdd;
}
/**
* 写入excel
* @param originalFilePath
* @param cells
*/
private static void saveXlsSplitNew(String originalFilePath, String[] cells, List<ColumnInfo> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count, boolean endFlag) {
try {
if (xlsxFileInfo.workbook == null) {
xlsxFileInfo.workbook = new SXSSFWorkbook(100);
Sheet sheet = xlsxFileInfo.workbook.createSheet();
Row row = sheet.createRow(0);
Cell cell;
logger.info("类型:{}", JSONObject.toJSONString(titles));
for (int j = 0; j < titles.size(); j++) {
cell = row.createCell(j);
cell.setCellValue(titles.get(j).getTitle());
}
//日期
CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle();
DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss"));
xlsxFileInfo.yyyyMMddhhmmss = yyyyMMddhhmmss;
//日期
CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle();
DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
xlsxFileInfo.yyyyMMdd = yyyyMMdd;
}
count.incrementAndGet();
SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
if (!endFlag) {
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(count.get());
for (int j = 0; j < cells.length; j++) {
String type = titles.get(j).getType();
Cell midCell = row.createCell(j);
String columnValue = cells[j];
if (StringUtils.isBlank(columnValue)) {
midCell.setCellValue(columnValue);
continue;
}
switch (type) {
case FlatQueryFieldType.DATE:
if (columnValue.length() == 10) {
midCell.setCellStyle(xlsxFileInfo.yyyyMMdd);
midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd"}));
} else {
midCell.setCellStyle(xlsxFileInfo.yyyyMMddhhmmss);
midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd HH:mm:ss"}));
}
break;
case FlatQueryFieldType.NUMBER:
if (columnValue.indexOf(".") > 0) {
midCell.setCellValue(Double.valueOf(columnValue));
} else {
midCell.setCellValue(Long.valueOf(columnValue));
}
break;
default:
midCell.setCellValue(columnValue);
}
}
}
Integer limitSize = FileUtil.getLimitSize();
int c = count.get();
if (c % 5000 == 0) {
logger.info("进度：{}-{}", originalFilePath, c);
}
if (c >= limitSize.intValue() || endFlag) {
logger.info("{}-count:{}", originalFilePath, count.get());
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
fileOut.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + originalFilePath + ".xlsx");
count.set(0);
}
} catch (Exception e) {
logger.info("异常：{}", e);
}
}
public static void main(String[] args) {
//下载文件
String taskId="1680836264869";
String dirName="hdfs"+taskId;
String conditionStr="{\"allFields\":[\"order_id\",\"is_member_order\",\"member_id\",\"order_time\",\"order_status\",\"receipts_date\",\"order_number\",\"oorder_number\",\"goods_count\",\"total_amount\",\"paid_amount\",\"pay_amount\",\"promotion_amount\",\"coupon_code\",\"using_integral\",\"get_integral\",\"order_type\",\"attention_status\",\"evaluate_status\",\"enterprise_name\",\"brand_name\",\"order_store_group\",\"order_store_code\",\"order_store_name\",\"order_store_type\",\"order_store_status\",\"order_clerk_code\",\"order_clerk_name\",\"order_grade_id\",\"order_grade_name\",\"order_age\",\"card_num\",\"member_name\",\"member_gender\",\"member_birthday\",\"phone_number\",\"grade_id\",\"grade_name\",\"card_giving_time\",\"auth_status\",\"auth_time\",\"wx_member\",\"pos_member\",\"p_open_card_channel\",\"open_store_code\",\"open_store_name\",\"open_clerk_code\",\"open_clerk_name\",\"is_new_member_order\",\"is_effective_order\",\"order_saler_group\",\"order_store_id\",\"clerk_id\",\"open_store_id\",\"order_store_tag_name\",\"open_store_tag_name\",\"open_store_group\",\"birth_month\",\"open_store_status\",\"channel_code\",\"qywx_flag\"],\"amount\":3243395,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单id\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员订单\",\"fieldName\":\"is_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员号\",\"fieldName\":\"member_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单时间（录单）\",\"fieldName\":\"order_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单类型\",\"fieldName\":\"order_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"实际消费日期（统计）\",\"fieldName\":\"receipts_date\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单编码\",\"fieldName\":\"order_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"原订单编码\",\"fieldName\":\"oorder_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单商品数量\",\"fieldName\":\"goods_count\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"吊牌价\",\"fieldName\":\"total_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售额（ERP）\",\"fieldName\":\"paid_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"销售实付（GIC计算）\",\"fieldName\":\"pay_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"整单活动优惠总额\",\"fieldName\":\"promotion_amount\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡劵code\",\"fieldName\":\"coupon_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"使用积分\",\"fieldName\":\"using_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"获得积分（ERP）\",\"fieldName\":\"get_integral\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单来源\",\"fieldName\":\"order_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否关注状态下订单\",\"fieldName\":\"attention_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"评价状态\",\"fieldName\":\"evaluate_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业名称\",\"fieldName\":\"enterprise_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"品牌名称\",\"fieldName\":\"brand_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店分组\",\"fieldName\":\"order_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店编码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店名称\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店状态\",\"fieldName\":\"order_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员编码\",\"fieldName\":\"order_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店店员姓名\",\"fieldName\":\"order_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级id\",\"fieldName\":\"order_grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费等级名称\",\"fieldName\":\"order_grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费年龄\",\"fieldName\":\"order_age\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡编码\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_gender\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员生日\",\"fieldName\":\"member_birthday\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级id\",\"fieldName\":\"grade_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"card_giving_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否已认证\",\"fieldName\":\"auth_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"认证时间\",\"fieldName\":\"auth_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否微信会员\",\"fieldName\":\"wx_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否POS会员\",\"fieldName\":\"pos_member\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡渠道\",\"fieldName\":\"p_open_card_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店编码\",\"fieldName\":\"open_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店\",\"fieldName\":\"open_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员编码\",\"fieldName\":\"open_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店店员\",\"fieldName\":\"open_clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"新老会员订单\",\"fieldName\":\"is_new_member_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否有效订单\",\"fieldName\":\"is_effective_order\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单导购组\",\"fieldName\":\"order_saler_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店id\",\"fieldName\":\"order_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"订单主导购\",\"fieldName\":\"clerk_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店id\",\"fieldName\":\"open_store_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"消费门店标签\",\"fieldName\":\"order_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店标签\",\"fieldName\":\"open_store_tag_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店分组\",\"fieldName\":\"open_store_group\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"生日月份\",\"fieldName\":\"birth_month\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡门店状态\",\"fieldName\":\"open_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"渠道编码\",\"fieldName\":\"channel_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否企业微信好友\",\"fieldName\":\"qywx_flag\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff80808162ad5aee0162ae7399dc002b\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_order_information\",\"taskId\":\"1659940224293\"}";
FlatQueryTaskCondition condition = JSON.parseObject(conditionStr, FlatQueryTaskCondition.class);
List<String> xlsxFiles=new ArrayList<>();
AtomicInteger count=new AtomicInteger(0);
AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
if (count.get()==0) {
XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
xlsxFileInfo.filepath=SAVE_FOLDER + "/" + taskId+ xlsxFiles.size() + ".xlsx";
currentFile.set(xlsxFileInfo);
xlsxFiles.add(currentFile.get().filepath);
}
// saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false);
});
}
}
gic-cloud-data-hook-service/src/test/java/com/gic/Test2.java 0 → 100644
package com.gic;
import com.gic.redis.data.util.RedisUtil;
import org.junit.Test;
import org.redisson.api.RMap;
import org.redisson.api.RedissonClient;
import org.springframework.context.support.ClassPathXmlApplicationContext;
public class Test2 {
@Test
public void test(){
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext("redis-init.xml");
RedissonClient redisClient = RedisUtil.getRedisClient();
RMap<Object, Object> aaa = redisClient.getMap("aaa123");
aaa.put("aa","2");
}
}
gic-cloud-data-hook-service/src/test/java/hdfs.java 0 → 100644
import com.gic.cloud.data.hook.service.HDFSUtil;
import com.opencsv.CSVReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.FileReader;
import java.io.IOException;
import java.util.Arrays;
public class hdfs {
public static void main(String[] args) {
// Configuration configuration = new Configuration();
// configuration.set("fs.defaultFS","hdfs://10.0.1.13:4007");
// try {
// FileSystem fileSystem = FileSystem.get(configuration);
// fileSystem.copyToLocalFile(false,new Path("/data/emr/order-1.csv"),new Path("D:\\testorder"));
// fileSystem.close();
// } catch (IOException e) {
// e.printStackTrace();
// }
HDFSUtil.getInstance().downloadFile("/data/hook/hdfs1680836264869","D:\\testorder");
// Integer[] arr=new Integer[]{1,2,3,4,5,6};
// Arrays.asList(arr).parallelStream().forEach(mid->{
// Thread thread = Thread.currentThread();
// String name = thread.getName();
// System.out.println(name);
// });
// try{
// CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));
// String[] cells = csvReader.readNext();
// System.out.println(cells);
// }catch (Exception e) {
//
// }
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment