Skip to content

G
gic-spark-tag-4.0
Commit 742d13ef by zhangyannao
Options

update

parent 292c03a3
Showing with 284 additions and 265 deletions
.gitignore 0 → 100644
/target
/gic-spark-tag-4.0.iml
src/main/java/com/gic/spark/SparkTagProcessByEnterpriseId.java
...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager; ...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager;
import com.gic.spark.config.SparkConfigManager; import com.gic.spark.config.SparkConfigManager;
import com.gic.spark.tag.TagProcessManager; import com.gic.spark.tag.TagProcessManager;
import java.util.ArrayList; import java.util.HashSet;
import java.util.List; import java.util.Set;
/** /**
* @description: * @description:
...@@ -15,7 +15,7 @@ import java.util.List; ...@@ -15,7 +15,7 @@ import java.util.List;
*/ */
public class SparkTagProcessByEnterpriseId { public class SparkTagProcessByEnterpriseId {
public static void main(String[] args){ public static void main(String[] args) {
if (args == null || args.length < 3) { if (args == null || args.length < 3) {
System.err.println("invalidate input params:"); System.err.println("invalidate input params:");
System.err.println("1.is production: true/false"); System.err.println("1.is production: true/false");
...@@ -26,9 +26,9 @@ public class SparkTagProcessByEnterpriseId { ...@@ -26,9 +26,9 @@ public class SparkTagProcessByEnterpriseId {
boolean isProd = Boolean.parseBoolean(args[0]); boolean isProd = Boolean.parseBoolean(args[0]);
boolean extractData = Boolean.parseBoolean(args[1]); boolean extractData = Boolean.parseBoolean(args[1]);
String[] arrs=args[2].split(","); String[] arrs = args[2].split(",");
List<Integer>params=new ArrayList(); Set<Integer> params = new HashSet<>();
for(String s:arrs){ for (String s : arrs) {
params.add(Integer.parseInt(s)); params.add(Integer.parseInt(s));
} }
ShardingConfigManager.initDefualt(); ShardingConfigManager.initDefualt();
......
src/main/java/com/gic/spark/SparkTagProcessByTagGroupId.java
...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager; ...@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager;
import com.gic.spark.config.SparkConfigManager; import com.gic.spark.config.SparkConfigManager;
import com.gic.spark.tag.TagProcessManager; import com.gic.spark.tag.TagProcessManager;
import java.util.ArrayList; import java.util.HashSet;
import java.util.List; import java.util.Set;
/** /**
* @description: * @description:
...@@ -15,7 +15,7 @@ import java.util.List; ...@@ -15,7 +15,7 @@ import java.util.List;
*/ */
public class SparkTagProcessByTagGroupId { public class SparkTagProcessByTagGroupId {
public static void main(String[] args){ public static void main(String[] args) {
if (args == null || args.length < 3) { if (args == null || args.length < 3) {
System.err.println("invalidate input params:"); System.err.println("invalidate input params:");
System.err.println("1.is production: true/false"); System.err.println("1.is production: true/false");
...@@ -26,9 +26,9 @@ public class SparkTagProcessByTagGroupId { ...@@ -26,9 +26,9 @@ public class SparkTagProcessByTagGroupId {
boolean isProd = Boolean.parseBoolean(args[0]); boolean isProd = Boolean.parseBoolean(args[0]);
boolean extractData = Boolean.parseBoolean(args[1]); boolean extractData = Boolean.parseBoolean(args[1]);
String[] arrs=args[2].split(","); String[] arrs = args[2].split(",");
List<Long>params=new ArrayList(); Set<Long> params = new HashSet<>();
for(String s:arrs){ for (String s : arrs) {
params.add(Long.parseLong(s)); params.add(Long.parseLong(s));
} }
ShardingConfigManager.initDefualt(); ShardingConfigManager.initDefualt();
......
src/main/java/com/gic/spark/datasource/entity/DataSourceMysql.java
...@@ -43,7 +43,7 @@ public class DataSourceMysql extends DataSourceEntity { ...@@ -43,7 +43,7 @@ public class DataSourceMysql extends DataSourceEntity {
@Override @Override
public String getHiveTableName() { public String getHiveTableName() {
return getSchema()+"." + datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable; return getSchema() + "." + datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable;
} }
@Override @Override
...@@ -51,7 +51,7 @@ public class DataSourceMysql extends DataSourceEntity { ...@@ -51,7 +51,7 @@ public class DataSourceMysql extends DataSourceEntity {
SparkSession sparkSession = SparkEnvManager.getInstance().getSparkSession(); SparkSession sparkSession = SparkEnvManager.getInstance().getSparkSession();
Dataset<Row> sourceDataset = datasource.buildRddManager().getDatasetByEnterpriseIds(targetTable, enterpriseList, null, null).repartition(new Column("enterprise_id")); Dataset<Row> sourceDataset = datasource.buildRddManager().getDatasetByEnterpriseIds(targetTable, enterpriseList, null, null).repartition(new Column("enterprise_id"));
SparkHiveUtil.createHivePartitionTable(sourceDataset, "enterprise_id", getSchema(), datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable,sparkSession); SparkHiveUtil.createHivePartitionTable(sourceDataset, "enterprise_id", getSchema(), datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable, sparkSession);
HivePartitionUtil.saveDatasetToPartitionTable(sparkSession, sourceDataset, getHiveTableName()); HivePartitionUtil.saveDatasetToPartitionTable(sparkSession, sourceDataset, getHiveTableName());
......
src/main/java/com/gic/spark/filter/AbstractTagConsumRecordFilter.java
...@@ -16,8 +16,7 @@ import java.util.List; ...@@ -16,8 +16,7 @@ import java.util.List;
/** /**
* @description: * @description: 消费记录
* 消费记录
* @author: wangxk * @author: wangxk
* @date: 2020/8/3 * @date: 2020/8/3
*/ */
...@@ -26,17 +25,17 @@ public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter { ...@@ -26,17 +25,17 @@ public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter {
DataSourceHive dataSourceHiveOrder = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_WDORDER_D); DataSourceHive dataSourceHiveOrder = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_WDORDER_D);
DataSourceHive dataSourceHiveOrderItem = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_ORDER_ITEM_D); DataSourceHive dataSourceHiveOrderItem = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_ORDER_ITEM_D);
protected static JavaRDD<TrdVirtualOrderBean> statisticsTypeHandle(JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd, AbstractFilterRequest request){ protected static JavaRDD<TrdVirtualOrderBean> statisticsTypeHandle(JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd, AbstractFilterRequest request) {
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD=orderRdd.mapPartitions(data->{ JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = orderRdd.mapPartitions(data -> {
List<TrdVirtualOrderBean> result=new ArrayList(); List<TrdVirtualOrderBean> result = new ArrayList();
while (data.hasNext()){ while (data.hasNext()) {
Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>> tp2=data.next(); Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>> tp2 = data.next();
TrdVirtualOrderBean consumeRecordBean=tp2._1(); TrdVirtualOrderBean consumeRecordBean = tp2._1();
switch (request.getStatisticsType()){ switch (request.getStatisticsType()) {
case COMMODITYBRAND: case COMMODITYBRAND:
if(tp2._2().isPresent()){ if (tp2._2().isPresent()) {
for(String ent_brand_id:tp2._2().get()){ for (String ent_brand_id : tp2._2().get()) {
if(request.getStatisticsValList().contains(ent_brand_id)){ if (request.getStatisticsValList().contains(ent_brand_id)) {
result.add(consumeRecordBean); result.add(consumeRecordBean);
break; break;
} }
...@@ -45,16 +44,17 @@ public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter { ...@@ -45,16 +44,17 @@ public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter {
break; break;
case CHANNEL: case CHANNEL:
if(request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getStore_info_id()))){ if (request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getStore_info_id()))) {
result.add(consumeRecordBean); result.add(consumeRecordBean);
} }
break; break;
case MCUINFO: case MCUINFO:
if(request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getArea_id()))){ if (request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getArea_id()))) {
result.add(consumeRecordBean); result.add(consumeRecordBean);
} }
break; break;
default:break; default:
break;
} }
} }
return result.iterator(); return result.iterator();
...@@ -62,35 +62,37 @@ public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter { ...@@ -62,35 +62,37 @@ public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter {
return consumeRecordRDD; return consumeRecordRDD;
} }
protected static boolean checkTime(AbstractFilterRequestTime request,long consumeTime){ protected static boolean checkTime(AbstractFilterRequestTime request, long consumeTime) {
Boolean result=false; Boolean result = false;
switch (request.getTimeRangeType()){ switch (request.getTimeRangeType()) {
case FIXATION: case FIXATION:
if(consumeTime>=request.getBeginTime().getTime() if (consumeTime >= request.getBeginTime().getTime()
&&consumeTime<=request.getEndTime().getTime()) && consumeTime <= request.getEndTime().getTime())
result=true; result = true;
break; break;
case LATELY: case LATELY:
switch (request.getYearMonthDayType()){ switch (request.getYearMonthDayType()) {
case DAY: case DAY:
if(consumeTime> DateUtil.addNumForDay(new Date(),-request.getTimeNum()).getTime()){ if (consumeTime > DateUtil.addNumForDay(new Date(), -request.getTimeNum()).getTime()) {
result=true; result = true;
} }
break; break;
case MONTH: case MONTH:
if(consumeTime>DateUtil.addNumForMonth(new Date(),-request.getTimeNum()).getTime()){ if (consumeTime > DateUtil.addNumForMonth(new Date(), -request.getTimeNum()).getTime()) {
result=true; result = true;
} }
break; break;
case YEAR: case YEAR:
if(consumeTime>DateUtil.addNumForYear(new Date(),-request.getTimeNum()).getTime()){ if (consumeTime > DateUtil.addNumForYear(new Date(), -request.getTimeNum()).getTime()) {
result=true; result = true;
} }
break; break;
default:break; default:
break;
} }
break; break;
default:break; default:
break;
} }
return result; return result;
} }
......
src/main/java/com/gic/spark/filter/TagAverageDiscountFactorFilter.java
...@@ -6,7 +6,6 @@ import com.gic.spark.entity.bean.TrdEcuBrandLabelBean; ...@@ -6,7 +6,6 @@ import com.gic.spark.entity.bean.TrdEcuBrandLabelBean;
import com.gic.spark.entity.bean.TrdEcuSalesLabelBean; import com.gic.spark.entity.bean.TrdEcuSalesLabelBean;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagConsumeDoubleRequest; import com.gic.spark.entity.request.TagConsumeDoubleRequest;
import com.gic.spark.entity.request.TagConsumeRequest;
import com.gic.spark.util.CommonUtil; import com.gic.spark.util.CommonUtil;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.Optional;
...@@ -16,21 +15,23 @@ import java.util.ArrayList; ...@@ -16,21 +15,23 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
/** /**
* @description: * @description: 平均折扣率
* 平均折扣率
* @author: wangxk * @author: wangxk
* @date: 2020/5/7 * @date: 2020/5/7
*/ */
public class TagAverageDiscountFactorFilter extends AbstractTagConsumFilter { public class TagAverageDiscountFactorFilter extends AbstractTagConsumFilter {
private static TagAverageDiscountFactorFilter instance; private static TagAverageDiscountFactorFilter instance;
public static TagAverageDiscountFactorFilter getInstance() { public static TagAverageDiscountFactorFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagAverageDiscountFactorFilter(); instance = new TagAverageDiscountFactorFilter();
} }
return instance; return instance;
} }
private TagAverageDiscountFactorFilter(){}
private TagAverageDiscountFactorFilter() {
}
@Override @Override
...@@ -43,62 +44,63 @@ public class TagAverageDiscountFactorFilter extends AbstractTagConsumFilter { ...@@ -43,62 +44,63 @@ public class TagAverageDiscountFactorFilter extends AbstractTagConsumFilter {
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeDoubleRequest consumeRequest=(TagConsumeDoubleRequest)request; TagConsumeDoubleRequest consumeRequest = (TagConsumeDoubleRequest) request;
JavaRDD<TrdEcuSalesLabelBean> salesLabelRDD=MysqlRddManager.getPojoFromDataset(dataSourceHiveSalesLabel.getDatasetByEntId(enterpriseId), TrdEcuSalesLabelBean.class).javaRDD(); JavaRDD<TrdEcuSalesLabelBean> salesLabelRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveSalesLabel.getDatasetByEntId(enterpriseId), TrdEcuSalesLabelBean.class).javaRDD();
JavaRDD<TrdEcuBrandLabelBean> brandLabelRDD=MysqlRddManager.getPojoFromDataset(dataSourceHiveBrandLabel.getDatasetByEntId(enterpriseId), TrdEcuBrandLabelBean.class).javaRDD(); JavaRDD<TrdEcuBrandLabelBean> brandLabelRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveBrandLabel.getDatasetByEntId(enterpriseId), TrdEcuBrandLabelBean.class).javaRDD();
JavaRDD<Tuple2<TrdEcuSalesLabelBean,Optional<Iterable<TrdEcuBrandLabelBean>>>>labelRDD=salesLabelRDD.mapToPair(data->Tuple2.apply(data.getEcu_id(),data)) JavaRDD<Tuple2<TrdEcuSalesLabelBean, Optional<Iterable<TrdEcuBrandLabelBean>>>> labelRDD = salesLabelRDD.mapToPair(data -> Tuple2.apply(data.getEcu_id(), data))
.leftOuterJoin(brandLabelRDD.mapToPair(data->Tuple2.apply(data.getEcu_id(),data)).groupByKey()) .leftOuterJoin(brandLabelRDD.mapToPair(data -> Tuple2.apply(data.getEcu_id(), data)).groupByKey())
.map(data->data._2()); .map(data -> data._2());
JavaRDD<TrdEcuSalesLabelBean> consumeRDD=statisticsTypeHandle(labelRDD,consumeRequest); JavaRDD<TrdEcuSalesLabelBean> consumeRDD = statisticsTypeHandle(labelRDD, consumeRequest);
int configStatus= CommonUtil.getConfigStatus(enterpriseId); int configStatus = CommonUtil.getConfigStatus(enterpriseId);
JavaRDD<Long>ecuRdd=consumeRDD.mapToPair(data-> Tuple2.apply(data.getEcu_id(),data)) JavaRDD<Long> ecuRdd = consumeRDD.mapToPair(data -> Tuple2.apply(data.getEcu_id(), data))
.reduceByKey((x,y)->{ .reduceByKey((x, y) -> {
x.setReceive_amt(x.getReceive_amt()+y.getReceive_amt()); x.setReceive_amt(x.getReceive_amt() + y.getReceive_amt());
x.setPay_amt(x.getPay_amt()+y.getPay_amt()); x.setPay_amt(x.getPay_amt() + y.getPay_amt());
x.setTotal_amt(x.getTotal_amt()+y.getTotal_amt()); x.setTotal_amt(x.getTotal_amt() + y.getTotal_amt());
return x; return x;
}) })
.mapPartitions(data->{ .mapPartitions(data -> {
List<Long>result=new ArrayList(); List<Long> result = new ArrayList();
while (data.hasNext()){ while (data.hasNext()) {
Tuple2<Long,TrdEcuSalesLabelBean> tp2=data.next(); Tuple2<Long, TrdEcuSalesLabelBean> tp2 = data.next();
double avgDiscountRate=1==configStatus?CommonUtil.isEmptyDouble2double(tp2._2().getPay_amt())/CommonUtil.isEmptyDouble2double(tp2._2().getTotal_amt()) double avgDiscountRate = 1 == configStatus ? CommonUtil.isEmptyDouble2double(tp2._2().getPay_amt()) / CommonUtil.isEmptyDouble2double(tp2._2().getTotal_amt())
:CommonUtil.isEmptyDouble2double(tp2._2().getReceive_amt())/CommonUtil.isEmptyDouble2double(tp2._2().getTotal_amt()); : CommonUtil.isEmptyDouble2double(tp2._2().getReceive_amt()) / CommonUtil.isEmptyDouble2double(tp2._2().getTotal_amt());
switch (consumeRequest.getNumberType()){ switch (consumeRequest.getNumberType()) {
case gt: case gt:
if(avgDiscountRate>consumeRequest.getBeginNum()){ if (avgDiscountRate > consumeRequest.getBeginNum()) {
result.add(tp2._1()); result.add(tp2._1());
} }
break; break;
case gte: case gte:
if(avgDiscountRate>=consumeRequest.getBeginNum()){ if (avgDiscountRate >= consumeRequest.getBeginNum()) {
result.add(tp2._1()); result.add(tp2._1());
} }
break; break;
case lt: case lt:
if(avgDiscountRate<consumeRequest.getEndNum()){ if (avgDiscountRate < consumeRequest.getEndNum()) {
result.add(tp2._1()); result.add(tp2._1());
} }
break; break;
case lte: case lte:
if(avgDiscountRate<=consumeRequest.getEndNum()){ if (avgDiscountRate <= consumeRequest.getEndNum()) {
result.add(tp2._1()); result.add(tp2._1());
} }
break; break;
case eq: case eq:
if(avgDiscountRate==consumeRequest.getEqualNum()){ if (avgDiscountRate == consumeRequest.getEqualNum()) {
result.add(tp2._1()); result.add(tp2._1());
} }
break; break;
case between: case between:
if(avgDiscountRate>=consumeRequest.getBeginNum() if (avgDiscountRate >= consumeRequest.getBeginNum()
&&avgDiscountRate<=consumeRequest.getEndNum()){ && avgDiscountRate <= consumeRequest.getEndNum()) {
result.add(tp2._1()); result.add(tp2._1());
} }
default:break; default:
break;
} }
} }
return result.iterator(); return result.iterator();
......
src/main/java/com/gic/spark/filter/TagConsumeCommodityFilter.java
package com.gic.spark.filter; package com.gic.spark.filter;
import com.gic.spark.datasource.entity.DataSourceEntity; import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceHive;
import com.gic.spark.datasource.mysql.MysqlRddManager; import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TrdVirtualOrderBean; import com.gic.spark.entity.bean.TrdVirtualOrderBean;
import com.gic.spark.entity.bean.TrdVirtualOrderItemBean; import com.gic.spark.entity.bean.TrdVirtualOrderItemBean;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagConsumeCommodityRequest; import com.gic.spark.entity.request.TagConsumeCommodityRequest;
import com.gic.spark.util.ConstantUtil;
import com.gic.spark.util.DateUtil; import com.gic.spark.util.DateUtil;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaPairRDD;
...@@ -25,14 +23,14 @@ import java.util.List; ...@@ -25,14 +23,14 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/12 * @date: 2020/8/12
*/ */
public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{ public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter {
private static TagConsumeCommodityFilter instance; private static TagConsumeCommodityFilter instance;
public static TagConsumeCommodityFilter getInstance() { public static TagConsumeCommodityFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagConsumeCommodityFilter(); instance = new TagConsumeCommodityFilter();
} }
return instance; return instance;
} }
...@@ -47,36 +45,36 @@ public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{ ...@@ -47,36 +45,36 @@ public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeCommodityRequest commodityRequest=(TagConsumeCommodityRequest)request; TagConsumeCommodityRequest commodityRequest = (TagConsumeCommodityRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
Dataset<Row>OrderItemDS= dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId); Dataset<Row> OrderItemDS = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderAndItemRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderAndItemRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id","ent_brand_id").javaRDD() .leftOuterJoin(OrderItemDS.select("virtual_order_id", "ent_brand_id").javaRDD()
.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))) .mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1)))
.groupByKey()) .groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderAndItemRdd,commodityRequest); consumeRecordRDD = statisticsTypeHandle(orderAndItemRdd, commodityRequest);
JavaPairRDD<Long,Long>orderRdd= consumeRecordRDD.filter(data-> StringUtils.isNotEmpty(data.getReceipts_time())) JavaPairRDD<Long, Long> orderRdd = consumeRecordRDD.filter(data -> StringUtils.isNotEmpty(data.getReceipts_time()))
.filter(data->checkTime(commodityRequest,DateUtil.strToDate(data.getReceipts_time(),DateUtil.FORMAT_DATETIME_19).getTime())) .filter(data -> checkTime(commodityRequest, DateUtil.strToDate(data.getReceipts_time(), DateUtil.FORMAT_DATETIME_19).getTime()))
.mapToPair(data-> Tuple2.apply(data.getVirtual_id(),data.getEcu_id())); .mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data.getEcu_id()));
JavaPairRDD<Long,Long> orderItemRDD=MysqlRddManager.getPojoFromDataset(OrderItemDS,TrdVirtualOrderItemBean.class).javaRDD() JavaPairRDD<Long, Long> orderItemRDD = MysqlRddManager.getPojoFromDataset(OrderItemDS, TrdVirtualOrderItemBean.class).javaRDD()
.filter(data->{ .filter(data -> {
if(StringUtils.isNotEmpty(data.getSku_code()) if (StringUtils.isNotEmpty(data.getSku_code())
&&commodityRequest.getSkuCodeList().contains(data.getSku_code())){ && commodityRequest.getSkuCodeList().contains(data.getSku_code())) {
return true; return true;
} }
return false; return false;
}).mapToPair(data->Tuple2.apply(data.getVirtual_order_id(),data.getVirtual_order_id())) }).mapToPair(data -> Tuple2.apply(data.getVirtual_order_id(), data.getVirtual_order_id()))
.reduceByKey((x,y)->x); .reduceByKey((x, y) -> x);
JavaRDD<Long>ecuRdd=orderRdd.leftOuterJoin(orderItemRDD) JavaRDD<Long> ecuRdd = orderRdd.leftOuterJoin(orderItemRDD)
.filter(data->data._2()._2().isPresent()) .filter(data -> data._2()._2().isPresent())
.map(data->data._2()._1()).distinct(); .map(data -> data._2()._1()).distinct();
return ecuRdd; return ecuRdd;
} }
......
src/main/java/com/gic/spark/filter/TagCouponFilter.java
...@@ -4,12 +4,13 @@ import com.gic.spark.datasource.entity.DataSourceEntity; ...@@ -4,12 +4,13 @@ import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceSharding; import com.gic.spark.datasource.entity.DataSourceSharding;
import com.gic.spark.datasource.mysql.MysqlRddManager; import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TagCouponBean; import com.gic.spark.entity.bean.TagCouponBean;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.util.AppEnvUtil; import com.gic.spark.util.AppEnvUtil;
import com.gic.spark.util.ConstantUtil; import com.gic.spark.util.ConstantUtil;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Column; import org.apache.spark.sql.Column;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
...@@ -22,15 +23,18 @@ import java.util.Set; ...@@ -22,15 +23,18 @@ import java.util.Set;
*/ */
public class TagCouponFilter implements BaseTagFilter { public class TagCouponFilter implements BaseTagFilter {
private DataSourceSharding dataSourceSharding=new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG); private DataSourceSharding dataSourceSharding = new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG);
private static TagCouponFilter instance; private static TagCouponFilter instance;
public static TagCouponFilter getInstance(){
if(null==instance){ public static TagCouponFilter getInstance() {
instance=new TagCouponFilter(); if (null == instance) {
instance = new TagCouponFilter();
} }
return instance; return instance;
} }
private TagCouponFilter(){}
private TagCouponFilter() {
}
@Override @Override
public List<DataSourceEntity> necessarySourceList() { public List<DataSourceEntity> necessarySourceList() {
...@@ -41,53 +45,54 @@ public class TagCouponFilter implements BaseTagFilter { ...@@ -41,53 +45,54 @@ public class TagCouponFilter implements BaseTagFilter {
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagCouponRequest couponRequest=(TagCouponRequest)request; TagCouponRequest couponRequest = (TagCouponRequest) request;
JavaRDD<TagCouponBean> couponBeanRDD= MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId) JavaRDD<TagCouponBean> couponBeanRDD = MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId)
.filter(new Column("status").isNotNull()),TagCouponBean.class).javaRDD() .filter(new Column("status").isNotNull()), TagCouponBean.class).javaRDD()
.filter(data->couponRequest.getCouponList().contains(data.getCoupon_id())); .filter(data -> couponRequest.getCouponList().contains(data.getCoupon_id()));
JavaRDD<Long>ecuRDD=couponBeanRDD.mapPartitions(data->{ JavaRDD<Long> ecuRDD = couponBeanRDD.mapPartitions(data -> {
Set<Long>result=new HashSet(); Set<Long> result = new HashSet();
while (data.hasNext()){ while (data.hasNext()) {
TagCouponBean couponBean=data.next(); TagCouponBean couponBean = data.next();
switch (couponRequest.getCouponType()){ switch (couponRequest.getCouponType()) {
case UNCLAIMED: case UNCLAIMED:
if(couponBean.getStatus()==3){ if (couponBean.getStatus() == 3) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case NO_CANCEL: case NO_CANCEL:
if(couponBean.getStatus()==4 if (couponBean.getStatus() == 4
&&couponBean.getEffect_End_Time().getTime()>System.currentTimeMillis()){ && couponBean.getEffect_End_Time().getTime() > System.currentTimeMillis()) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case NO_CANCEL_EXPIRES: case NO_CANCEL_EXPIRES:
if(couponBean.getStatus()==4 if (couponBean.getStatus() == 4
&&couponBean.getEffect_End_Time().getTime()<System.currentTimeMillis()){ && couponBean.getEffect_End_Time().getTime() < System.currentTimeMillis()) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case CANCEL: case CANCEL:
if(couponBean.getStatus()==7){ if (couponBean.getStatus() == 7) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case OCCUPY: case OCCUPY:
if(couponBean.getStatus()==8){ if (couponBean.getStatus() == 8) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case DONATION_IN: case DONATION_IN:
if(couponBean.getStatus()==9){ if (couponBean.getStatus() == 9) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
case YET_DONATION: case YET_DONATION:
if(couponBean.getStatus()==10){ if (couponBean.getStatus() == 10) {
result.add(couponBean.getEcu_Id()); result.add(couponBean.getEcu_Id());
} }
break; break;
default:break; default:
break;
} }
} }
return result.iterator(); return result.iterator();
......
src/main/java/com/gic/spark/filter/TagCurrentCouponNumFilter.java
...@@ -4,8 +4,8 @@ import com.gic.spark.datasource.entity.DataSourceEntity; ...@@ -4,8 +4,8 @@ import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceSharding; import com.gic.spark.datasource.entity.DataSourceSharding;
import com.gic.spark.datasource.mysql.MysqlRddManager; import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TagCouponBean; import com.gic.spark.entity.bean.TagCouponBean;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.util.AppEnvUtil; import com.gic.spark.util.AppEnvUtil;
import com.gic.spark.util.ConstantUtil; import com.gic.spark.util.ConstantUtil;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
...@@ -17,86 +17,91 @@ import java.util.List; ...@@ -17,86 +17,91 @@ import java.util.List;
import java.util.Set; import java.util.Set;
/** /**
* @description: * @description: 当前卡券数
* 当前卡券数
* @author: wangxk * @author: wangxk
* @date: 2020/4/20 * @date: 2020/4/20
*/ */
public class TagCurrentCouponNumFilter implements BaseTagFilter { public class TagCurrentCouponNumFilter implements BaseTagFilter {
private DataSourceSharding dataSourceSharding=new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG); private DataSourceSharding dataSourceSharding = new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG);
private static TagCurrentCouponNumFilter instance; private static TagCurrentCouponNumFilter instance;
public static TagCurrentCouponNumFilter getInstance() { public static TagCurrentCouponNumFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagCurrentCouponNumFilter(); instance = new TagCurrentCouponNumFilter();
} }
return instance; return instance;
} }
private TagCurrentCouponNumFilter(){};
private TagCurrentCouponNumFilter() {
}
;
@Override @Override
public List<DataSourceEntity> necessarySourceList() { public List<DataSourceEntity> necessarySourceList() {
List<DataSourceEntity> result = new ArrayList(); List<DataSourceEntity> result = new ArrayList();
System.out.println("dataSourceSharding==>"+dataSourceSharding); System.out.println("dataSourceSharding==>" + dataSourceSharding);
result.add(dataSourceSharding); result.add(dataSourceSharding);
return result; return result;
} }
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagCouponRequest couponRequest=(TagCouponRequest)request; TagCouponRequest couponRequest = (TagCouponRequest) request;
JavaRDD<TagCouponBean> couponBeanRDD=MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId),TagCouponBean.class).javaRDD() JavaRDD<TagCouponBean> couponBeanRDD = MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId), TagCouponBean.class).javaRDD()
.filter(data->{ .filter(data -> {
if(null!=data.getStatus() if (null != data.getStatus()
&&null!=data.getEffect_End_Time()){ && null != data.getEffect_End_Time()) {
return (data.getStatus()==3||data.getStatus()==4) return (data.getStatus() == 3 || data.getStatus() == 4)
&&data.getEffect_End_Time().getTime()>System.currentTimeMillis(); && data.getEffect_End_Time().getTime() > System.currentTimeMillis();
} }
return false; return false;
}); });
JavaRDD<Long>ecuRDD=couponBeanRDD.mapPartitionsToPair(data->{ JavaRDD<Long> ecuRDD = couponBeanRDD.mapPartitionsToPair(data -> {
List<Tuple2<String,Integer>>result=new ArrayList(); List<Tuple2<String, Integer>> result = new ArrayList();
while (data.hasNext()){ while (data.hasNext()) {
TagCouponBean couponBean=data.next(); TagCouponBean couponBean = data.next();
switch(couponRequest.getDomainType()){ switch (couponRequest.getDomainType()) {
case ECU_INFO: case ECU_INFO:
result.add(Tuple2.apply(couponBean.getEcu_Id().toString(),1)); result.add(Tuple2.apply(couponBean.getEcu_Id().toString(), 1));
break; break;
case ACU_INFO: case ACU_INFO:
if(null!=couponBean.getAcu_Id()&&couponRequest.getCuVals().contains(couponBean.getAcu_Id())){ if (null != couponBean.getAcu_Id() && couponRequest.getCuVals().contains(couponBean.getAcu_Id())) {
result.add(Tuple2.apply(couponBean.getAcu_Id()+"&"+couponBean.getEcu_Id(),1)); result.add(Tuple2.apply(couponBean.getAcu_Id() + "&" + couponBean.getEcu_Id(), 1));
} }
break; break;
case SCU_INFO: case SCU_INFO:
if(null!=couponBean.getAcu_Id()&&couponRequest.getCuVals().contains(couponBean.getScu_Id())){ if (null != couponBean.getAcu_Id() && couponRequest.getCuVals().contains(couponBean.getScu_Id())) {
result.add(Tuple2.apply(couponBean.getScu_Id()+"&"+couponBean.getEcu_Id(),1)); result.add(Tuple2.apply(couponBean.getScu_Id() + "&" + couponBean.getEcu_Id(), 1));
} }
break; break;
case MCU_INFO: case MCU_INFO:
if(null!=couponBean.getMcu_Id()&&couponRequest.getCuVals().contains(couponBean.getMcu_Id())){ if (null != couponBean.getMcu_Id() && couponRequest.getCuVals().contains(couponBean.getMcu_Id())) {
result.add(Tuple2.apply(couponBean.getMcu_Id()+"&"+couponBean.getEcu_Id(),1)); result.add(Tuple2.apply(couponBean.getMcu_Id() + "&" + couponBean.getEcu_Id(), 1));
} }
break; break;
default:break; default:
break;
} }
} }
return result.iterator(); return result.iterator();
}).reduceByKey((x,y)->x+y) }).reduceByKey((x, y) -> x + y)
.mapPartitions(data->{ .mapPartitions(data -> {
Set<Long> result=new HashSet(); Set<Long> result = new HashSet();
while (data.hasNext()){ while (data.hasNext()) {
Tuple2<String,Integer>tp2=data.next(); Tuple2<String, Integer> tp2 = data.next();
switch (couponRequest.getDomainType()){ switch (couponRequest.getDomainType()) {
case ECU_INFO: case ECU_INFO:
if(numberComputeHandle(couponRequest,tp2._2())){ if (numberComputeHandle(couponRequest, tp2._2())) {
result.add(Long.parseLong(tp2._1())); result.add(Long.parseLong(tp2._1()));
} }
break; break;
default: default:
if(numberComputeHandle(couponRequest,tp2._2())){ if (numberComputeHandle(couponRequest, tp2._2())) {
result.add(Long.parseLong(tp2._1().split("&")[1])); result.add(Long.parseLong(tp2._1().split("&")[1]));
} }
break; break;
...@@ -110,28 +115,29 @@ public class TagCurrentCouponNumFilter implements BaseTagFilter { ...@@ -110,28 +115,29 @@ public class TagCurrentCouponNumFilter implements BaseTagFilter {
return ecuRDD; return ecuRDD;
} }
public static boolean numberComputeHandle(TagCouponRequest couponRequest, int num){ public static boolean numberComputeHandle(TagCouponRequest couponRequest, int num) {
boolean result=false; boolean result = false;
switch (couponRequest.getNumberType()){ switch (couponRequest.getNumberType()) {
case eq: case eq:
result=num==couponRequest.getEqualNum(); result = num == couponRequest.getEqualNum();
break; break;
case lt: case lt:
result=num<couponRequest.getEndNum(); result = num < couponRequest.getEndNum();
break; break;
case lte: case lte:
result=num<=couponRequest.getEndNum(); result = num <= couponRequest.getEndNum();
break; break;
case gt: case gt:
result=num>couponRequest.getBeginNum(); result = num > couponRequest.getBeginNum();
break; break;
case gte: case gte:
result=num>=couponRequest.getBeginNum(); result = num >= couponRequest.getBeginNum();
break; break;
case between: case between:
result=num>=couponRequest.getBeginNum()&&num<=couponRequest.getEndNum(); result = num >= couponRequest.getBeginNum() && num <= couponRequest.getEndNum();
break;
default:
break; break;
default:break;
} }
return result; return result;
} }
......
src/main/java/com/gic/spark/filter/TagFirstOnlineConsumptionStoreFilter.java
...@@ -20,18 +20,19 @@ import java.util.List; ...@@ -20,18 +20,19 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/11 * @date: 2020/8/11
*/ */
public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecordFilter{ public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecordFilter {
private static TagFirstOnlineConsumptionStoreFilter instance; private static TagFirstOnlineConsumptionStoreFilter instance;
public static TagFirstOnlineConsumptionStoreFilter getInstance() { public static TagFirstOnlineConsumptionStoreFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagFirstOnlineConsumptionStoreFilter(); instance = new TagFirstOnlineConsumptionStoreFilter();
} }
return instance; return instance;
} }
private TagFirstOnlineConsumptionStoreFilter(){} private TagFirstOnlineConsumptionStoreFilter() {
}
@Override @Override
public List<DataSourceEntity> necessarySourceList() { public List<DataSourceEntity> necessarySourceList() {
...@@ -43,30 +44,30 @@ public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecor ...@@ -43,30 +44,30 @@ public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecor
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeStoreRequest storeRequest=(TagConsumeStoreRequest)request; TagConsumeStoreRequest storeRequest = (TagConsumeStoreRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
JavaRDD<Row>virtualOrderItemRdd=dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id","ent_brand_id").javaRDD(); JavaRDD<Row> virtualOrderItemRdd = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id", "ent_brand_id").javaRDD();
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))).groupByKey()) .leftOuterJoin(virtualOrderItemRdd.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1))).groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderRdd,storeRequest); consumeRecordRDD = statisticsTypeHandle(orderRdd, storeRequest);
JavaRDD<Long>ecuRdd=consumeRecordRDD.filter(data->data.getOrder_channel_code()!=1 JavaRDD<Long> ecuRdd = consumeRecordRDD.filter(data -> data.getOrder_channel_code() != 1
&& StringUtils.isNotEmpty(data.getReceipts_time()) && StringUtils.isNotEmpty(data.getReceipts_time())
&& null!=data.getShop_id()) && null != data.getShop_id())
.mapToPair(data-> Tuple2.apply(data.getEcu_id(),Tuple2.apply(data.getReceipts_time(),data.getShop_id()))) .mapToPair(data -> Tuple2.apply(data.getEcu_id(), Tuple2.apply(data.getReceipts_time(), data.getShop_id())))
.reduceByKey((x,y)->{ .reduceByKey((x, y) -> {
if(DateUtil.strToDate(x._1(),DateUtil.FORMAT_DATETIME_19).getTime() if (DateUtil.strToDate(x._1(), DateUtil.FORMAT_DATETIME_19).getTime()
<DateUtil.strToDate(y._1(),DateUtil.FORMAT_DATETIME_19).getTime()){ < DateUtil.strToDate(y._1(), DateUtil.FORMAT_DATETIME_19).getTime()) {
return x; return x;
}else{ } else {
return y; return y;
} }
}).filter(data->storeRequest.getStoreList().contains(String.valueOf(data._2()._2()))) }).filter(data -> storeRequest.getStoreList().contains(String.valueOf(data._2()._2())))
.map(data->data._1()); .map(data -> data._1());
return ecuRdd; return ecuRdd;
} }
} }
src/main/java/com/gic/spark/filter/TagHistoryConsumeCommodityFilter.java
package com.gic.spark.filter; package com.gic.spark.filter;
import com.gic.spark.datasource.entity.DataSourceEntity; import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceHive;
import com.gic.spark.datasource.mysql.MysqlRddManager; import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TrdVirtualOrderBean; import com.gic.spark.entity.bean.TrdVirtualOrderBean;
import com.gic.spark.entity.bean.TrdVirtualOrderItemBean; import com.gic.spark.entity.bean.TrdVirtualOrderItemBean;
import com.gic.spark.entity.request.AbstractFilterRequest; import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagConsumeCommodityRequest; import com.gic.spark.entity.request.TagConsumeCommodityRequest;
import com.gic.spark.util.ConstantUtil;
import com.gic.spark.util.DateUtil;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.Optional;
import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Dataset;
...@@ -25,13 +21,13 @@ import java.util.List; ...@@ -25,13 +21,13 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/12 * @date: 2020/8/12
*/ */
public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFilter{ public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFilter {
private static TagHistoryConsumeCommodityFilter instance; private static TagHistoryConsumeCommodityFilter instance;
public static TagHistoryConsumeCommodityFilter getInstance() { public static TagHistoryConsumeCommodityFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagHistoryConsumeCommodityFilter(); instance = new TagHistoryConsumeCommodityFilter();
} }
return instance; return instance;
} }
...@@ -46,25 +42,25 @@ public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFil ...@@ -46,25 +42,25 @@ public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFil
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeCommodityRequest commodityRequest=(TagConsumeCommodityRequest)request; TagConsumeCommodityRequest commodityRequest = (TagConsumeCommodityRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
Dataset<Row> OrderItemDS= dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId); Dataset<Row> OrderItemDS = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderAndItemRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderAndItemRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id","ent_brand_id").javaRDD() .leftOuterJoin(OrderItemDS.select("virtual_order_id", "ent_brand_id").javaRDD()
.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))) .mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1)))
.groupByKey()) .groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderAndItemRdd,commodityRequest); consumeRecordRDD = statisticsTypeHandle(orderAndItemRdd, commodityRequest);
JavaRDD<TrdVirtualOrderItemBean> orderItemRDD=MysqlRddManager.getPojoFromDataset(OrderItemDS,TrdVirtualOrderItemBean.class).javaRDD(); JavaRDD<TrdVirtualOrderItemBean> orderItemRDD = MysqlRddManager.getPojoFromDataset(OrderItemDS, TrdVirtualOrderItemBean.class).javaRDD();
JavaRDD<Long>ecuRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data.getEcu_id())) JavaRDD<Long> ecuRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data.getEcu_id()))
.leftOuterJoin(orderItemRDD.mapToPair(data->Tuple2.apply(data.getVirtual_order_id(),data.getSku_code())) .leftOuterJoin(orderItemRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_order_id(), data.getSku_code()))
.filter(data->StringUtils.isNotEmpty(data._2()) .filter(data -> StringUtils.isNotEmpty(data._2())
&&commodityRequest.getSkuCodeList().contains(data._2()))) && commodityRequest.getSkuCodeList().contains(data._2())))
.filter(data->data._2()._2().isPresent()) .filter(data -> data._2()._2().isPresent())
.map(data->data._2()._1()) .map(data -> data._2()._1())
.distinct(); .distinct();
return ecuRdd; return ecuRdd;
......
src/main/java/com/gic/spark/filter/TagHistoryConsumeTotalFilter.java
...@@ -19,18 +19,19 @@ import java.util.List; ...@@ -19,18 +19,19 @@ import java.util.List;
* @author: wangxk * @author: wangxk
* @date: 2020/8/6 * @date: 2020/8/6
*/ */
public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{ public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter {
private static TagHistoryConsumeTotalFilter instance; private static TagHistoryConsumeTotalFilter instance;
public static TagHistoryConsumeTotalFilter getInstance() { public static TagHistoryConsumeTotalFilter getInstance() {
if(null==instance){ if (null == instance) {
instance=new TagHistoryConsumeTotalFilter(); instance = new TagHistoryConsumeTotalFilter();
} }
return instance; return instance;
} }
private TagHistoryConsumeTotalFilter(){} private TagHistoryConsumeTotalFilter() {
}
@Override @Override
public List<DataSourceEntity> necessarySourceList() { public List<DataSourceEntity> necessarySourceList() {
...@@ -42,58 +43,59 @@ public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{ ...@@ -42,58 +43,59 @@ public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{
@Override @Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) { public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeAmountRequest consumeAmountRequest=(TagConsumeAmountRequest)request; TagConsumeAmountRequest consumeAmountRequest = (TagConsumeAmountRequest) request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD(); JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
JavaRDD<Row>virtualOrderItemRdd=dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id","ent_brand_id").javaRDD(); JavaRDD<Row> virtualOrderItemRdd = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id", "ent_brand_id").javaRDD();
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data)) JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))).groupByKey()) .leftOuterJoin(virtualOrderItemRdd.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1))).groupByKey())
.map(data->data._2()); .map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderRdd,consumeAmountRequest); consumeRecordRDD = statisticsTypeHandle(orderRdd, consumeAmountRequest);
int configStatus= CommonUtil.getConfigStatus(enterpriseId); int configStatus = CommonUtil.getConfigStatus(enterpriseId);
JavaRDD<Long>ecuRdd=consumeRecordRDD.mapToPair(data-> Tuple2.apply(data.getEcu_id(),configStatus==1?data.getPaid_amt():data.getPay_amt())) JavaRDD<Long> ecuRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getEcu_id(), configStatus == 1 ? data.getPaid_amt() : data.getPay_amt()))
.reduceByKey((x,y)->x+y) .reduceByKey((x, y) -> x + y)
.filter(data->{ .filter(data -> {
boolean result=false; boolean result = false;
switch (consumeAmountRequest.getNumberType()){ switch (consumeAmountRequest.getNumberType()) {
case between: case between:
if(data._2()>=consumeAmountRequest.getBeginNum() if (data._2() >= consumeAmountRequest.getBeginNum()
&&data._2()<=consumeAmountRequest.getEndNum()){ && data._2() <= consumeAmountRequest.getEndNum()) {
result=true; result = true;
} }
break; break;
case lt: case lt:
if(data._2()<consumeAmountRequest.getEndNum()){ if (data._2() < consumeAmountRequest.getEndNum()) {
result=true; result = true;
} }
break; break;
case gt: case gt:
if(data._2()>consumeAmountRequest.getBeginNum()){ if (data._2() > consumeAmountRequest.getBeginNum()) {
result=true; result = true;
} }
break; break;
case eq: case eq:
if(data._2()==consumeAmountRequest.getEqualNum()){ if (data._2() == consumeAmountRequest.getEqualNum()) {
result=true; result = true;
} }
break; break;
case lte: case lte:
if(data._2()<=consumeAmountRequest.getEndNum()){ if (data._2() <= consumeAmountRequest.getEndNum()) {
result=true; result = true;
} }
break; break;
case gte: case gte:
if(data._2()>=consumeAmountRequest.getBeginNum()){ if (data._2() >= consumeAmountRequest.getBeginNum()) {
result=true; result = true;
} }
break; break;
default:break; default:
break;
} }
return result; return result;
}).map(data->data._1()); }).map(data -> data._1());
return ecuRdd; return ecuRdd;
} }
} }
src/main/java/com/gic/spark/util/EsRequestUtil.java
package com.gic.spark.util; package com.gic.spark.util;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ClientProtocolException;
...@@ -7,7 +8,10 @@ import org.apache.http.client.methods.HttpGet; ...@@ -7,7 +8,10 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set;
/** /**
* Created by paste on 2018/7/21 16:29 * Created by paste on 2018/7/21 16:29
...@@ -16,10 +20,10 @@ import java.util.Map; ...@@ -16,10 +20,10 @@ import java.util.Map;
*/ */
public class EsRequestUtil { public class EsRequestUtil {
public static String getESIindexName(Integer enterpriseId,boolean isProduction) { public static String getESIindexName(Integer enterpriseId, boolean isProduction) {
String url = isProduction?"https://ideal.demogic.com/member-config/member/index-version/" + enterpriseId String url = isProduction ? "https://ideal.demogic.com/member-config/member/index-version/" + enterpriseId
:"https://four.gicdev.com/member-config/member/index-version/" + enterpriseId; : "https://four.gicdev.com/member-config/member/index-version/" + enterpriseId;
HttpResponse response = getHttpResponseByGet(url,enterpriseId); HttpResponse response = getHttpResponseByGet(url, enterpriseId);
int responseCode = response.getStatusLine().getStatusCode(); int responseCode = response.getStatusLine().getStatusCode();
if (responseCode == 200) { if (responseCode == 200) {
return HttpClient.getResponseString(response); return HttpClient.getResponseString(response);
...@@ -28,14 +32,14 @@ public class EsRequestUtil { ...@@ -28,14 +32,14 @@ public class EsRequestUtil {
} }
public static HttpResponse getHttpResponseByGet(String url,Integer enterpriseId) { public static HttpResponse getHttpResponseByGet(String url, Integer enterpriseId) {
org.apache.http.client.HttpClient httpClient = HttpClient.getHttpClient(); org.apache.http.client.HttpClient httpClient = HttpClient.getHttpClient();
HttpGet get = new HttpGet(url); HttpGet get = new HttpGet(url);
get.setHeader("sign",""+enterpriseId); get.setHeader("sign", "" + enterpriseId);
get.setHeader("route","/spark"); get.setHeader("route", "/spark");
get.setHeader("project","memberfour"); get.setHeader("project", "memberfour");
get.setHeader("isControl","true"); get.setHeader("isControl", "true");
get.setHeader("Refer",url); get.setHeader("Refer", url);
try { try {
HttpResponse response = httpClient.execute(get); HttpResponse response = httpClient.execute(get);
return response; return response;
...@@ -48,16 +52,16 @@ public class EsRequestUtil { ...@@ -48,16 +52,16 @@ public class EsRequestUtil {
return null; return null;
} }
public static String getIndexParam( Integer enterpriseId,Long id , boolean isProd) { public static String getIndexParam(Integer enterpriseId, Long id, boolean isProd) {
String url = isProd?"https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId="+id+"&enterpriseId="+enterpriseId String url = isProd ? "https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId=" + id + "&enterpriseId=" + enterpriseId
:"https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId="+id+"&enterpriseId="+enterpriseId; : "https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId=" + id + "&enterpriseId=" + enterpriseId;
HttpPost httpPost = new HttpPost(url); HttpPost httpPost = new HttpPost(url);
httpPost.addHeader("Content-Type", "application/json;charset=UTF-8"); httpPost.addHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.addHeader("project","member-tag"); httpPost.addHeader("project", "member-tag");
httpPost.addHeader("sign","1129"); httpPost.addHeader("sign", "1129");
httpPost.addHeader("route","get-member-size-by-condition"); httpPost.addHeader("route", "get-member-size-by-condition");
HttpResponse response = null; HttpResponse response = null;
try { try {
response = HttpClient.getHttpClient().execute(httpPost); response = HttpClient.getHttpClient().execute(httpPost);
...@@ -71,6 +75,7 @@ public class EsRequestUtil { ...@@ -71,6 +75,7 @@ public class EsRequestUtil {
} }
public static void main(String[] args) throws IOException { public static void main(String[] args) throws IOException {
System.out.println(getIndexParam(1129,183676455182233606L,true)); // System.out.println(getIndexParam(1129,183676455182233606L,true));
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment