Skip to content

G
gic-spark-tag-4.0
Commit 742d13ef by zhangyannao
Options

update

parent 292c03a3
Showing with 347 additions and 334 deletions
.gitignore 0 → 100644
/target
/gic-spark-tag-4.0.iml
src/main/java/com/gic/spark/SparkTagProcessByEnterpriseId.java
...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager; ...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager;
import com.gic.spark.config.SparkConfigManager; import com.gic.spark.config.SparkConfigManager;
import com.gic.spark.tag.TagProcessManager; import com.gic.spark.tag.TagProcessManager;
import java.util.ArrayList; import java.util.HashSet;
import java.util.List; import java.util.Set;
/** /**
* @description: * @description:
...@@ -15,7 +15,7 @@ import java.util.List; ...@@ -15,7 +15,7 @@ import java.util.List;
*/ */
public class SparkTagProcessByEnterpriseId { public class SparkTagProcessByEnterpriseId {
public static void main(String[] args){ public static void main(String[] args) {
if (args == null || args.length < 3) { if (args == null || args.length < 3) {
System.err.println("invalidate input params:"); System.err.println("invalidate input params:");
System.err.println("1.is production: true/false"); System.err.println("1.is production: true/false");
...@@ -26,9 +26,9 @@ public class SparkTagProcessByEnterpriseId { ...@@ -26,9 +26,9 @@ public class SparkTagProcessByEnterpriseId {
boolean isProd = Boolean.parseBoolean(args[0]); boolean isProd = Boolean.parseBoolean(args[0]);
boolean extractData = Boolean.parseBoolean(args[1]); boolean extractData = Boolean.parseBoolean(args[1]);
String[] arrs=args[2].split(","); String[] arrs = args[2].split(",");
List<Integer>params=new ArrayList(); Set<Integer> params = new HashSet<>();
for(String s:arrs){ for (String s : arrs) {
params.add(Integer.parseInt(s)); params.add(Integer.parseInt(s));
} }
ShardingConfigManager.initDefualt(); ShardingConfigManager.initDefualt();
......
src/main/java/com/gic/spark/SparkTagProcessByTagGroupId.java
...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager; ...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager;
import com.gic.spark.config.SparkConfigManager; import com.gic.spark.config.SparkConfigManager;
import com.gic.spark.tag.TagProcessManager; import com.gic.spark.tag.TagProcessManager;
import java.util.ArrayList; import java.util.HashSet;
import java.util.List; import java.util.Set;
/** /**
* @description: * @description:
...@@ -15,7 +15,7 @@ import java.util.List; ...@@ -15,7 +15,7 @@ import java.util.List;
*/ */
public class SparkTagProcessByTagGroupId { public class SparkTagProcessByTagGroupId {
public static void main(String[] args){ public static void main(String[] args) {
if (args == null || args.length < 3) { if (args == null || args.length < 3) {
System.err.println("invalidate input params:"); System.err.println("invalidate input params:");
System.err.println("1.is production: true/false"); System.err.println("1.is production: true/false");
...@@ -26,9 +26,9 @@ public class SparkTagProcessByTagGroupId { ...@@ -26,9 +26,9 @@ public class SparkTagProcessByTagGroupId {
boolean isProd = Boolean.parseBoolean(args[0]); boolean isProd = Boolean.parseBoolean(args[0]);
boolean extractData = Boolean.parseBoolean(args[1]); boolean extractData = Boolean.parseBoolean(args[1]);
String[] arrs=args[2].split(","); String[] arrs = args[2].split(",");
List<Long>params=new ArrayList(); Set<Long> params = new HashSet<>();
for(String s:arrs){ for (String s : arrs) {
params.add(Long.parseLong(s)); params.add(Long.parseLong(s));
} }
ShardingConfigManager.initDefualt(); ShardingConfigManager.initDefualt();
......
src/main/java/com/gic/spark/datasource/entity/DataSourceMysql.java
...@@ -43,7 +43,7 @@ public class DataSourceMysql extends DataSourceEntity { ...@@ -43,7 +43,7 @@ public class DataSourceMysql extends DataSourceEntity {
@Override @Override
public String getHiveTableName() { public String getHiveTableName() {
return getSchema()+"." + datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable; return getSchema() + "." + datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable;
} }
@Override @Override
...@@ -51,7 +51,7 @@ public class DataSourceMysql extends DataSourceEntity { ...@@ -51,7 +51,7 @@ public class DataSourceMysql extends DataSourceEntity {
SparkSession sparkSession = SparkEnvManager.getInstance().getSparkSession(); SparkSession sparkSession = SparkEnvManager.getInstance().getSparkSession();
Dataset<Row> sourceDataset = datasource.buildRddManager().getDatasetByEnterpriseIds(targetTable, enterpriseList, null, null).repartition(new Column("enterprise_id")); Dataset<Row> sourceDataset = datasource.buildRddManager().getDatasetByEnterpriseIds(targetTable, enterpriseList, null, null).repartition(new Column("enterprise_id"));
SparkHiveUtil.createHivePartitionTable(sourceDataset, "enterprise_id", getSchema(), datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable,sparkSession); SparkHiveUtil.createHivePartitionTable(sourceDataset, "enterprise_id", getSchema(), datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable, sparkSession);
HivePartitionUtil.saveDatasetToPartitionTable(sparkSession, sourceDataset, getHiveTableName()); HivePartitionUtil.saveDatasetToPartitionTable(sparkSession, sourceDataset, getHiveTableName());
......
src/main/java/com/gic/spark/filter/AbstractTagConsumRecordFilter.java
...@@ -16,82 +16,84 @@ import java.util.List; ...@@ -16,82 +16,84 @@ import java.util.List;
/** /**
* @description: * @description: 消费记录
* 消费记录
* @author: wangxk * @author: wangxk
* @date: 2020/8/3 * @date: 2020/8/3
*/ */
public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter { public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter {
DataSourceHive dataSourceHiveOrder = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_WDORDER_D); DataSourceHive dataSourceHiveOrder = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_WDORDER_D);
DataSourceHive dataSourceHiveOrderItem = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_ORDER_ITEM_D); DataSourceHive dataSourceHiveOrderItem = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_ORDER_ITEM_D);
protected static JavaRDD<TrdVirtualOrderBean> statisticsTypeHandle(JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd, AbstractFilterRequest request){ protected static JavaRDD<TrdVirtualOrderBean> statisticsTypeHandle(JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd, AbstractFilterRequest request) {
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD=orderRdd.mapPartitions(data->{ JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = orderRdd.mapPartitions(data -> {
List<TrdVirtualOrderBean> result=new ArrayList(); List<TrdVirtualOrderBean> result = new ArrayList();
while (data.hasNext()){ while (data.hasNext()) {
Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>> tp2=data.next(); Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>> tp2 = data.next();
TrdVirtualOrderBean consumeRecordBean=tp2._1(); TrdVirtualOrderBean consumeRecordBean = tp2._1();
switch (request.getStatisticsType()){ switch (request.getStatisticsType()) {
case COMMODITYBRAND: case COMMODITYBRAND:
if(tp2._2().isPresent()){ if (tp2._2().isPresent()) {
for(String ent_brand_id:tp2._2().get()){ for (String ent_brand_id : tp2._2().get()) {
if(request.getStatisticsValList().contains(ent_brand_id)){ if (request.getStatisticsValList().contains(ent_brand_id)) {
result.add(consumeRecordBean); result.add(consumeRecordBean);
break; break;
} }
} }
} }
break; break;
case CHANNEL: case CHANNEL:
if(request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getStore_info_id()))){ if (request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getStore_info_id()))) {
result.add(consumeRecordBean); result.add(consumeRecordBean);
} }
break; break;
case MCUINFO: case MCUINFO:
if(request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getArea_id()))){ if (request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getArea_id()))) {
result.add(consumeRecordBean); result.add(consumeRecordBean);
} }
break; break;
default:break; default:
} break;
} }
return result.iterator(); }
}); return result.iterator();
});
return consumeRecordRDD; return consumeRecordRDD;
} }
protected static boolean checkTime(AbstractFilterRequestTime request,long consumeTime){ protected static boolean checkTime(AbstractFilterRequestTime request, long consumeTime) {
Boolean result=false; Boolean result = false;
switch (request.getTimeRangeType()){ switch (request.getTimeRangeType()) {
case FIXATION: case FIXATION:
if(consumeTime>=request.getBeginTime().getTime() if (consumeTime >= request.getBeginTime().getTime()
&&consumeTime<=request.getEndTime().getTime()) && consumeTime <= request.getEndTime().getTime())
result=true; result = true;
break; break;
case LATELY: case LATELY:
switch (request.getYearMonthDayType()){ switch (request.getYearMonthDayType()) {
case DAY: case DAY:
if(consumeTime> DateUtil.addNumForDay(new Date(),-request.getTimeNum()).getTime()){ if (consumeTime > DateUtil.addNumForDay(new Date(), -request.getTimeNum()).getTime()) {
result=true; result = true;
} }
break; break;
case MONTH: case MONTH:
if(consumeTime>DateUtil.addNumForMonth(new Date(),-request.getTimeNum()).getTime()){ if (consumeTime > DateUtil.addNumForMonth(new Date(), -request.getTimeNum()).getTime()) {
result=true; result = true;
} }
break; break;
case YEAR: case YEAR:
if(consumeTime>DateUtil.addNumForYear(new Date(),-request.getTimeNum()).getTime()){ if (consumeTime > DateUtil.addNumForYear(new Date(), -request.getTimeNum()).getTime()) {
result=true; result = true;
} }
break; break;
default:break; default:
break;
} }
break; break;
default:break; default:
break;
} }
return result; return result;
} }
} }
src/main/java/com/gic/spark/filter/TagConsumeCommodityFilter.java
package com.gic.spark.filter; package com.gic.spark.filter;
import com.gic.spark.datasource.entity.DataSourceEntity; import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceHive;
import com.gic.spark.datasource.mysql.MysqlRddManager; import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TrdVirtualOrderBean; import com.gic.spark.entity.bean.TrdVirtualOrderBean;
import com.gic.spark.entity.bean.TrdVirtualOrderItemBean; import com.gic.spark.entity.bean.TrdVirtualOrderItemBean;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagConsumeCommodityRequest; import com.gic.spark.entity.request.TagConsumeCommodityRequest;
import com.gic.spark.util.ConstantUtil;
import com.gic.spark.util.DateUtil; import com.gic.spark.util.DateUtil;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaPairRDD;
...@@ -25,14 +23,14 @@ import java.util.List; ...@@ -25,14 +23,14 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/12 * @date: 2020/8/12
*/ */
public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{ public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter {
private static TagConsumeCommodityFilter instance; private static TagConsumeCommodityFilter instance;
public static TagConsumeCommodityFilter getInstance() { public static TagConsumeCommodityFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagConsumeCommodityFilter(); instance = new TagConsumeCommodityFilter();
} }
return instance; return instance;
} }
...@@ -47,37 +45,37 @@ public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{ ...@@ -47,37 +45,37 @@ public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeCommodityRequest commodityRequest=(TagConsumeCommodityRequest)request; TagConsumeCommodityRequest commodityRequest = (TagConsumeCommodityRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
Dataset<Row>OrderItemDS= dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId); Dataset<Row> OrderItemDS = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderAndItemRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderAndItemRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id","ent_brand_id").javaRDD() .leftOuterJoin(OrderItemDS.select("virtual_order_id", "ent_brand_id").javaRDD()
.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))) .mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1)))
.groupByKey()) .groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderAndItemRdd,commodityRequest); consumeRecordRDD = statisticsTypeHandle(orderAndItemRdd, commodityRequest);
JavaPairRDD<Long,Long>orderRdd= consumeRecordRDD.filter(data-> StringUtils.isNotEmpty(data.getReceipts_time())) JavaPairRDD<Long, Long> orderRdd = consumeRecordRDD.filter(data -> StringUtils.isNotEmpty(data.getReceipts_time()))
.filter(data->checkTime(commodityRequest,DateUtil.strToDate(data.getReceipts_time(),DateUtil.FORMAT_DATETIME_19).getTime())) .filter(data -> checkTime(commodityRequest, DateUtil.strToDate(data.getReceipts_time(), DateUtil.FORMAT_DATETIME_19).getTime()))
.mapToPair(data-> Tuple2.apply(data.getVirtual_id(),data.getEcu_id())); .mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data.getEcu_id()));
JavaPairRDD<Long,Long> orderItemRDD=MysqlRddManager.getPojoFromDataset(OrderItemDS,TrdVirtualOrderItemBean.class).javaRDD() JavaPairRDD<Long, Long> orderItemRDD = MysqlRddManager.getPojoFromDataset(OrderItemDS, TrdVirtualOrderItemBean.class).javaRDD()
.filter(data->{ .filter(data -> {
if(StringUtils.isNotEmpty(data.getSku_code()) if (StringUtils.isNotEmpty(data.getSku_code())
&&commodityRequest.getSkuCodeList().contains(data.getSku_code())){ && commodityRequest.getSkuCodeList().contains(data.getSku_code())) {
return true; return true;
} }
return false; return false;
}).mapToPair(data->Tuple2.apply(data.getVirtual_order_id(),data.getVirtual_order_id())) }).mapToPair(data -> Tuple2.apply(data.getVirtual_order_id(), data.getVirtual_order_id()))
.reduceByKey((x,y)->x); .reduceByKey((x, y) -> x);
JavaRDD<Long>ecuRdd=orderRdd.leftOuterJoin(orderItemRDD) JavaRDD<Long> ecuRdd = orderRdd.leftOuterJoin(orderItemRDD)
.filter(data->data._2()._2().isPresent()) .filter(data -> data._2()._2().isPresent())
.map(data->data._2()._1()).distinct(); .map(data -> data._2()._1()).distinct();
return ecuRdd; return ecuRdd;
} }
} }
src/main/java/com/gic/spark/filter/TagCouponFilter.java
...@@ -4,12 +4,13 @@ import com.gic.spark.datasource.entity.DataSourceEntity; ...@@ -4,12 +4,13 @@ import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceSharding; import com.gic.spark.datasource.entity.DataSourceSharding;
import com.gic.spark.datasource.mysql.MysqlRddManager; import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TagCouponBean; import com.gic.spark.entity.bean.TagCouponBean;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.util.AppEnvUtil; import com.gic.spark.util.AppEnvUtil;
import com.gic.spark.util.ConstantUtil; import com.gic.spark.util.ConstantUtil;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Column; import org.apache.spark.sql.Column;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
...@@ -22,15 +23,18 @@ import java.util.Set; ...@@ -22,15 +23,18 @@ import java.util.Set;
*/ */
public class TagCouponFilter implements BaseTagFilter { public class TagCouponFilter implements BaseTagFilter {
private DataSourceSharding dataSourceSharding=new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG); private DataSourceSharding dataSourceSharding = new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG);
private static TagCouponFilter instance; private static TagCouponFilter instance;
public static TagCouponFilter getInstance(){
if(null==instance){ public static TagCouponFilter getInstance() {
instance=new TagCouponFilter(); if (null == instance) {
instance = new TagCouponFilter();
} }
return instance; return instance;
} }
private TagCouponFilter(){}
private TagCouponFilter() {
}
@Override @Override
public List<DataSourceEntity> necessarySourceList() { public List<DataSourceEntity> necessarySourceList() {
...@@ -41,57 +45,58 @@ public class TagCouponFilter implements BaseTagFilter { ...@@ -41,57 +45,58 @@ public class TagCouponFilter implements BaseTagFilter {
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagCouponRequest couponRequest=(TagCouponRequest)request; TagCouponRequest couponRequest = (TagCouponRequest) request;
JavaRDD<TagCouponBean> couponBeanRDD= MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId) JavaRDD<TagCouponBean> couponBeanRDD = MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId)
.filter(new Column("status").isNotNull()),TagCouponBean.class).javaRDD() .filter(new Column("status").isNotNull()), TagCouponBean.class).javaRDD()
.filter(data->couponRequest.getCouponList().contains(data.getCoupon_id())); .filter(data -> couponRequest.getCouponList().contains(data.getCoupon_id()));
JavaRDD<Long>ecuRDD=couponBeanRDD.mapPartitions(data->{ JavaRDD<Long> ecuRDD = couponBeanRDD.mapPartitions(data -> {
Set<Long>result=new HashSet(); Set<Long> result = new HashSet();
while (data.hasNext()){ while (data.hasNext()) {
TagCouponBean couponBean=data.next(); TagCouponBean couponBean = data.next();
switch (couponRequest.getCouponType()){ switch (couponRequest.getCouponType()) {
case UNCLAIMED: case UNCLAIMED:
if(couponBean.getStatus()==3){ if (couponBean.getStatus() == 3) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case NO_CANCEL: case NO_CANCEL:
if(couponBean.getStatus()==4 if (couponBean.getStatus() == 4
&&couponBean.getEffect_End_Time().getTime()>System.currentTimeMillis()){ && couponBean.getEffect_End_Time().getTime() > System.currentTimeMillis()) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case NO_CANCEL_EXPIRES: case NO_CANCEL_EXPIRES:
if(couponBean.getStatus()==4 if (couponBean.getStatus() == 4
&&couponBean.getEffect_End_Time().getTime()<System.currentTimeMillis()){ && couponBean.getEffect_End_Time().getTime() < System.currentTimeMillis()) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case CANCEL: case CANCEL:
if(couponBean.getStatus()==7){ if (couponBean.getStatus() == 7) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case OCCUPY: case OCCUPY:
if(couponBean.getStatus()==8){ if (couponBean.getStatus() == 8) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case DONATION_IN: case DONATION_IN:
if(couponBean.getStatus()==9){ if (couponBean.getStatus() == 9) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case YET_DONATION: case YET_DONATION:
if(couponBean.getStatus()==10){ if (couponBean.getStatus() == 10) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
default:break; default:
} break;
} }
return result.iterator(); }
}); return result.iterator();
});
return ecuRDD; return ecuRDD;
} }
......
src/main/java/com/gic/spark/filter/TagFirstOnlineConsumptionStoreFilter.java
...@@ -20,18 +20,19 @@ import java.util.List; ...@@ -20,18 +20,19 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/11 * @date: 2020/8/11
*/ */
public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecordFilter{ public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecordFilter {
private static TagFirstOnlineConsumptionStoreFilter instance; private static TagFirstOnlineConsumptionStoreFilter instance;
public static TagFirstOnlineConsumptionStoreFilter getInstance() { public static TagFirstOnlineConsumptionStoreFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagFirstOnlineConsumptionStoreFilter(); instance = new TagFirstOnlineConsumptionStoreFilter();
} }
return instance; return instance;
} }
private TagFirstOnlineConsumptionStoreFilter(){} private TagFirstOnlineConsumptionStoreFilter() {
}
@Override @Override
public List<DataSourceEntity> necessarySourceList() { public List<DataSourceEntity> necessarySourceList() {
...@@ -43,30 +44,30 @@ public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecor ...@@ -43,30 +44,30 @@ public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecor
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeStoreRequest storeRequest=(TagConsumeStoreRequest)request; TagConsumeStoreRequest storeRequest = (TagConsumeStoreRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
JavaRDD<Row>virtualOrderItemRdd=dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id","ent_brand_id").javaRDD(); JavaRDD<Row> virtualOrderItemRdd = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id", "ent_brand_id").javaRDD();
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))).groupByKey()) .leftOuterJoin(virtualOrderItemRdd.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1))).groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderRdd,storeRequest); consumeRecordRDD = statisticsTypeHandle(orderRdd, storeRequest);
JavaRDD<Long>ecuRdd=consumeRecordRDD.filter(data->data.getOrder_channel_code()!=1 JavaRDD<Long> ecuRdd = consumeRecordRDD.filter(data -> data.getOrder_channel_code() != 1
&& StringUtils.isNotEmpty(data.getReceipts_time()) && StringUtils.isNotEmpty(data.getReceipts_time())
&& null!=data.getShop_id()) && null != data.getShop_id())
.mapToPair(data-> Tuple2.apply(data.getEcu_id(),Tuple2.apply(data.getReceipts_time(),data.getShop_id()))) .mapToPair(data -> Tuple2.apply(data.getEcu_id(), Tuple2.apply(data.getReceipts_time(), data.getShop_id())))
.reduceByKey((x,y)->{ .reduceByKey((x, y) -> {
if(DateUtil.strToDate(x._1(),DateUtil.FORMAT_DATETIME_19).getTime() if (DateUtil.strToDate(x._1(), DateUtil.FORMAT_DATETIME_19).getTime()
<DateUtil.strToDate(y._1(),DateUtil.FORMAT_DATETIME_19).getTime()){ < DateUtil.strToDate(y._1(), DateUtil.FORMAT_DATETIME_19).getTime()) {
return x; return x;
}else{ } else {
return y; return y;
} }
}).filter(data->storeRequest.getStoreList().contains(String.valueOf(data._2()._2()))) }).filter(data -> storeRequest.getStoreList().contains(String.valueOf(data._2()._2())))
.map(data->data._1()); .map(data -> data._1());
return ecuRdd; return ecuRdd;
} }
} }
src/main/java/com/gic/spark/filter/TagHistoryConsumeCommodityFilter.java
package com.gic.spark.filter; package com.gic.spark.filter;
import com.gic.spark.datasource.entity.DataSourceEntity; import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceHive;
import com.gic.spark.datasource.mysql.MysqlRddManager; import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TrdVirtualOrderBean; import com.gic.spark.entity.bean.TrdVirtualOrderBean;
import com.gic.spark.entity.bean.TrdVirtualOrderItemBean; import com.gic.spark.entity.bean.TrdVirtualOrderItemBean;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagConsumeCommodityRequest; import com.gic.spark.entity.request.TagConsumeCommodityRequest;
import com.gic.spark.util.ConstantUtil;
import com.gic.spark.util.DateUtil;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.Optional;
import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Dataset;
...@@ -25,13 +21,13 @@ import java.util.List; ...@@ -25,13 +21,13 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/12 * @date: 2020/8/12
*/ */
public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFilter{ public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFilter {
private static TagHistoryConsumeCommodityFilter instance; private static TagHistoryConsumeCommodityFilter instance;
public static TagHistoryConsumeCommodityFilter getInstance() { public static TagHistoryConsumeCommodityFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagHistoryConsumeCommodityFilter(); instance = new TagHistoryConsumeCommodityFilter();
} }
return instance; return instance;
} }
...@@ -46,27 +42,27 @@ public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFil ...@@ -46,27 +42,27 @@ public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFil
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeCommodityRequest commodityRequest=(TagConsumeCommodityRequest)request; TagConsumeCommodityRequest commodityRequest = (TagConsumeCommodityRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
Dataset<Row> OrderItemDS= dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId); Dataset<Row> OrderItemDS = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderAndItemRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderAndItemRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id","ent_brand_id").javaRDD() .leftOuterJoin(OrderItemDS.select("virtual_order_id", "ent_brand_id").javaRDD()
.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))) .mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1)))
.groupByKey()) .groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderAndItemRdd,commodityRequest); consumeRecordRDD = statisticsTypeHandle(orderAndItemRdd, commodityRequest);
JavaRDD<TrdVirtualOrderItemBean> orderItemRDD=MysqlRddManager.getPojoFromDataset(OrderItemDS,TrdVirtualOrderItemBean.class).javaRDD(); JavaRDD<TrdVirtualOrderItemBean> orderItemRDD = MysqlRddManager.getPojoFromDataset(OrderItemDS, TrdVirtualOrderItemBean.class).javaRDD();
JavaRDD<Long>ecuRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data.getEcu_id())) JavaRDD<Long> ecuRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data.getEcu_id()))
.leftOuterJoin(orderItemRDD.mapToPair(data->Tuple2.apply(data.getVirtual_order_id(),data.getSku_code())) .leftOuterJoin(orderItemRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_order_id(), data.getSku_code()))
.filter(data->StringUtils.isNotEmpty(data._2()) .filter(data -> StringUtils.isNotEmpty(data._2())
&&commodityRequest.getSkuCodeList().contains(data._2()))) && commodityRequest.getSkuCodeList().contains(data._2())))
.filter(data->data._2()._2().isPresent()) .filter(data -> data._2()._2().isPresent())
.map(data->data._2()._1()) .map(data -> data._2()._1())
.distinct(); .distinct();
return ecuRdd; return ecuRdd;
} }
} }
src/main/java/com/gic/spark/filter/TagHistoryConsumeTotalFilter.java
...@@ -19,18 +19,19 @@ import java.util.List; ...@@ -19,18 +19,19 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/6 * @date: 2020/8/6
*/ */
public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{ public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter {
private static TagHistoryConsumeTotalFilter instance; private static TagHistoryConsumeTotalFilter instance;
public static TagHistoryConsumeTotalFilter getInstance() { public static TagHistoryConsumeTotalFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagHistoryConsumeTotalFilter(); instance = new TagHistoryConsumeTotalFilter();
} }
return instance; return instance;
} }
private TagHistoryConsumeTotalFilter(){} private TagHistoryConsumeTotalFilter() {
}
@Override @Override
public List<DataSourceEntity> necessarySourceList() { public List<DataSourceEntity> necessarySourceList() {
...@@ -42,58 +43,59 @@ public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{ ...@@ -42,58 +43,59 @@ public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeAmountRequest consumeAmountRequest=(TagConsumeAmountRequest)request; TagConsumeAmountRequest consumeAmountRequest = (TagConsumeAmountRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
JavaRDD<Row>virtualOrderItemRdd=dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id","ent_brand_id").javaRDD(); JavaRDD<Row> virtualOrderItemRdd = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id", "ent_brand_id").javaRDD();
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))).groupByKey()) .leftOuterJoin(virtualOrderItemRdd.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1))).groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderRdd,consumeAmountRequest); consumeRecordRDD = statisticsTypeHandle(orderRdd, consumeAmountRequest);
int configStatus= CommonUtil.getConfigStatus(enterpriseId); int configStatus = CommonUtil.getConfigStatus(enterpriseId);
JavaRDD<Long>ecuRdd=consumeRecordRDD.mapToPair(data-> Tuple2.apply(data.getEcu_id(),configStatus==1?data.getPaid_amt():data.getPay_amt())) JavaRDD<Long> ecuRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getEcu_id(), configStatus == 1 ? data.getPaid_amt() : data.getPay_amt()))
.reduceByKey((x,y)->x+y) .reduceByKey((x, y) -> x + y)
.filter(data->{ .filter(data -> {
boolean result=false; boolean result = false;
switch (consumeAmountRequest.getNumberType()){ switch (consumeAmountRequest.getNumberType()) {
case between: case between:
if(data._2()>=consumeAmountRequest.getBeginNum() if (data._2() >= consumeAmountRequest.getBeginNum()
&&data._2()<=consumeAmountRequest.getEndNum()){ && data._2() <= consumeAmountRequest.getEndNum()) {
result=true; result = true;
} }
break; break;
case lt: case lt:
if(data._2()<consumeAmountRequest.getEndNum()){ if (data._2() < consumeAmountRequest.getEndNum()) {
result=true; result = true;
} }
break; break;
case gt: case gt:
if(data._2()>consumeAmountRequest.getBeginNum()){ if (data._2() > consumeAmountRequest.getBeginNum()) {
result=true; result = true;
} }
break; break;
case eq: case eq:
if(data._2()==consumeAmountRequest.getEqualNum()){ if (data._2() == consumeAmountRequest.getEqualNum()) {
result=true; result = true;
} }
break; break;
case lte: case lte:
if(data._2()<=consumeAmountRequest.getEndNum()){ if (data._2() <= consumeAmountRequest.getEndNum()) {
result=true; result = true;
} }
break; break;
case gte: case gte:
if(data._2()>=consumeAmountRequest.getBeginNum()){ if (data._2() >= consumeAmountRequest.getBeginNum()) {
result=true; result = true;
} }
break; break;
default:break; default:
} break;
return result; }
}).map(data->data._1()); return result;
}).map(data -> data._1());
return ecuRdd; return ecuRdd;
} }
} }
src/main/java/com/gic/spark/util/EsRequestUtil.java
package com.gic.spark.util; package com.gic.spark.util;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ClientProtocolException;
...@@ -7,7 +8,10 @@ import org.apache.http.client.methods.HttpGet; ...@@ -7,7 +8,10 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set;
/** /**
* Created by paste on 2018/7/21 16:29 * Created by paste on 2018/7/21 16:29
...@@ -16,26 +20,26 @@ import java.util.Map; ...@@ -16,26 +20,26 @@ import java.util.Map;
*/ */
public class EsRequestUtil { public class EsRequestUtil {
public static String getESIindexName(Integer enterpriseId,boolean isProduction) { public static String getESIindexName(Integer enterpriseId, boolean isProduction) {
String url = isProduction?"https://ideal.demogic.com/member-config/member/index-version/" + enterpriseId String url = isProduction ? "https://ideal.demogic.com/member-config/member/index-version/" + enterpriseId
:"https://four.gicdev.com/member-config/member/index-version/" + enterpriseId; : "https://four.gicdev.com/member-config/member/index-version/" + enterpriseId;
HttpResponse response = getHttpResponseByGet(url,enterpriseId); HttpResponse response = getHttpResponseByGet(url, enterpriseId);
int responseCode = response.getStatusLine().getStatusCode(); int responseCode = response.getStatusLine().getStatusCode();
if (responseCode == 200) { if (responseCode == 200) {
return HttpClient.getResponseString(response); return HttpClient.getResponseString(response);
} }
return null; return null;
} }
public static HttpResponse getHttpResponseByGet(String url,Integer enterpriseId) { public static HttpResponse getHttpResponseByGet(String url, Integer enterpriseId) {
org.apache.http.client.HttpClient httpClient = HttpClient.getHttpClient(); org.apache.http.client.HttpClient httpClient = HttpClient.getHttpClient();
HttpGet get = new HttpGet(url); HttpGet get = new HttpGet(url);
get.setHeader("sign",""+enterpriseId); get.setHeader("sign", "" + enterpriseId);
get.setHeader("route","/spark"); get.setHeader("route", "/spark");
get.setHeader("project","memberfour"); get.setHeader("project", "memberfour");
get.setHeader("isControl","true"); get.setHeader("isControl", "true");
get.setHeader("Refer",url); get.setHeader("Refer", url);
try { try {
HttpResponse response = httpClient.execute(get); HttpResponse response = httpClient.execute(get);
return response; return response;
...@@ -48,16 +52,16 @@ public class EsRequestUtil { ...@@ -48,16 +52,16 @@ public class EsRequestUtil {
return null; return null;
} }
public static String getIndexParam( Integer enterpriseId,Long id , boolean isProd) { public static String getIndexParam(Integer enterpriseId, Long id, boolean isProd) {
String url = isProd?"https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId="+id+"&enterpriseId="+enterpriseId String url = isProd ? "https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId=" + id + "&enterpriseId=" + enterpriseId
:"https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId="+id+"&enterpriseId="+enterpriseId; : "https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId=" + id + "&enterpriseId=" + enterpriseId;
HttpPost httpPost = new HttpPost(url); HttpPost httpPost = new HttpPost(url);
httpPost.addHeader("Content-Type", "application/json;charset=UTF-8"); httpPost.addHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.addHeader("project","member-tag"); httpPost.addHeader("project", "member-tag");
httpPost.addHeader("sign","1129"); httpPost.addHeader("sign", "1129");
httpPost.addHeader("route","get-member-size-by-condition"); httpPost.addHeader("route", "get-member-size-by-condition");
HttpResponse response = null; HttpResponse response = null;
try { try {
response = HttpClient.getHttpClient().execute(httpPost); response = HttpClient.getHttpClient().execute(httpPost);
...@@ -71,6 +75,7 @@ public class EsRequestUtil { ...@@ -71,6 +75,7 @@ public class EsRequestUtil {
} }
public static void main(String[] args) throws IOException { public static void main(String[] args) throws IOException {
System.out.println(getIndexParam(1129,183676455182233606L,true)); // System.out.println(getIndexParam(1129,183676455182233606L,true));
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment