feat: 新版本的下载

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/entity/DownloadTaskStatus.java

@@ -8,6 +8,9 @@ public class DownloadTaskStatus {
 	/** 排队中 */
 	public static final String WAITING = "waiting";
 
+	/** hdfs已经处理完等待下载 */
+	public static final String DOWNLOAD_HDFS = "downloading";
+
 	/** 生成中 */
 	public static final String BUILDING = "building";
 

gic-cloud-data-hook-service/pom.xml

@@ -146,6 +146,16 @@
 		</dependency>
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-client</artifactId>
+			<version>${hadoopCommonVersion}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-hdfs</artifactId>
+			<version>${hadoopCommonVersion}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-common</artifactId>
 			<version>${hadoopCommonVersion}</version>
 			<exclusions>
@@ -199,8 +209,8 @@
 				<artifactId>maven-compiler-plugin</artifactId>
 				<version>3.1</version>
 				<configuration>
-					<source>1.7</source>
-					<target>1.7</target>
+					<source>1.8</source>
+					<target>1.8</target>
 					<encoding>UTF-8</encoding>
 					<compilerArguments>
 						<verbose/>

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DownloadFunc.java

+package com.gic.cloud.data.hook.service;
+
+import com.gic.cloud.data.hook.service.entity.ColumnInfo;
+
+import java.util.List;
+
+public interface DownloadFunc {
+
+    public void deal(String[] cells, List<ColumnInfo> titles, boolean fileFirst);
+}

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/HDFSUtil.java

+package com.gic.cloud.data.hook.service;
+
+import com.ctrip.framework.apollo.Config;
+import com.ctrip.framework.apollo.ConfigService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class HDFSUtil {
+    private static Logger logger = LoggerFactory.getLogger(HDFSUtil.class);
+    private static HDFSUtil hdfsUtil=null;
+    private static FileSystem fileSystem=null;
+
+    public static HDFSUtil getInstance(){
+        if (hdfsUtil==null) {
+            synchronized (HDFSUtil.class) {
+                if (hdfsUtil==null) {
+                    hdfsUtil = new HDFSUtil();
+                }
+            }
+        }
+        return hdfsUtil;
+    }
+
+    private HDFSUtil(){
+        Config appConfig = ConfigService.getAppConfig();
+        String hdfsUrl = appConfig.getProperty("hdfs.url", null);
+        Configuration configuration = new Configuration();
+        configuration.set("fs.defaultFS",hdfsUrl);
+        try {
+            fileSystem = FileSystem.get(configuration);
+        } catch (IOException e) {
+            logger.info("hdfs初始化失败-{}",e);
+        }
+    }
+
+    /**
+     * 下载到本地
+     * @param srcPath
+     * @param toPath
+     * @return
+     */
+    public boolean downloadFile(String srcPath,String toPath) {
+        try {
+            fileSystem.copyToLocalFile(true,new Path(srcPath),new Path(toPath));
+            return true;
+        } catch (IOException e) {
+            logger.info("下载失败：{}",e);
+            return false;
+        }
+    }
+}

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/dao/DownloadTaskDao.java

@@ -62,6 +62,11 @@ public interface DownloadTaskDao {
 	 */
 	public List<DownloadTask> getDownloadTaskOfWaiting(@Param("queryDataSource") String queryDataSource);
 
+	/** 获取等待申请通过状态的任务
+	 * @return
+	 */
+	public List<DownloadTask> getDownloadTaskOfHasDownload(@Param("queryDataSource") String queryDataSource);
+
 	DownloadProcessDTO getDownloadProcess();
 
 	int updateTaskStatusError(@Param("idList") List<String> idList);

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/entity/ColumnInfo.java

+package com.gic.cloud.data.hook.service.entity;
+
+import java.io.Serializable;
+
+public class ColumnInfo implements Serializable {
+    private String type;
+    private String title;
+
+    public ColumnInfo() {
+    }
+
+    public ColumnInfo(String type, String title) {
+        this.type = type;
+        this.title = title;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public String getTitle() {
+        return title;
+    }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
+}

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java

@@ -228,6 +228,13 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
 		return downloadTaskDao.getDownloadTaskOfWaiting(queryDataSource);
 	}
 
+	/** 获取在审核申请等待状态中的任务
+	 * @return
+	 */
+	public List<DownloadTask> getDownloadTaskOfHasDownload(String queryDataSource) {
+		return downloadTaskDao.getDownloadTaskOfWaiting(queryDataSource);
+	}
+
 
 	/** 获取指定申请编号的风险模式记录
 	 * @param applyId

gic-cloud-data-hook-service/src/main/resources/mapper/DownloadTaskDao.xml

@@ -187,6 +187,17 @@
 			AND q.del_flag = '0'
 	</select>
 
+	<select id="getDownloadTaskOfHasDownload" resultType="DownloadTask">
+		SELECT *
+		FROM
+		<include refid="queryTables"/>
+		<include refid="queryJoins"/>
+		WHERE
+		q.query_data_source = #{queryDataSource}
+		AND q.status = "downloading"
+		AND q.del_flag = '0'
+	</select>
+
 	<select id="getDownloadProcess" resultType="DownloadProcessDTO">
 		SELECT sum(amount) dataAmount,count(1) taskAmount
 		FROM

gic-cloud-data-hook-service/src/test/java/hdfs.java

+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+public class hdfs {
+    public static void main(String[] args) {
+        Configuration configuration = new Configuration();
+        configuration.set("fs.defaultFS","hdfs://10.0.1.13:4007");
+        try {
+            FileSystem fileSystem = FileSystem.get(configuration);
+            fileSystem.copyToLocalFile(false,new Path("/data/emr/order-1.csv"),new Path("D:\\testorder"));
+            fileSystem.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}