Skip to content

G
gic-spark-tag-4.0
Commit 742d13ef by zhangyannao
Options

update

parent 292c03a3
Showing with 347 additions and 334 deletions
.gitignore 0 → 100644
/target
/gic-spark-tag-4.0.iml
src/main/java/com/gic/spark/SparkTagProcessByEnterpriseId.java
......@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager;
import com.gic.spark.config.SparkConfigManager;
import com.gic.spark.tag.TagProcessManager;
import java.util.ArrayList;
import java.util.List;
import java.util.HashSet;
import java.util.Set;
/**
* @description:
......@@ -15,7 +15,7 @@ import java.util.List;
*/
public class SparkTagProcessByEnterpriseId {
public static void main(String[] args){
public static void main(String[] args) {
if (args == null || args.length < 3) {
System.err.println("invalidate input params:");
System.err.println("1.is production: true/false");
......@@ -26,9 +26,9 @@ public class SparkTagProcessByEnterpriseId {
boolean isProd = Boolean.parseBoolean(args[0]);
boolean extractData = Boolean.parseBoolean(args[1]);
String[] arrs=args[2].split(",");
List<Integer>params=new ArrayList();
for(String s:arrs){
String[] arrs = args[2].split(",");
Set<Integer> params = new HashSet<>();
for (String s : arrs) {
params.add(Integer.parseInt(s));
}
ShardingConfigManager.initDefualt();
......
src/main/java/com/gic/spark/SparkTagProcessByTagGroupId.java
......@@ -5,8 +5,8 @@ import com.gic.spark.application.SparkEnvManager;
import com.gic.spark.config.SparkConfigManager;
import com.gic.spark.tag.TagProcessManager;
import java.util.ArrayList;
import java.util.List;
import java.util.HashSet;
import java.util.Set;
/**
* @description:
......@@ -15,7 +15,7 @@ import java.util.List;
*/
public class SparkTagProcessByTagGroupId {
public static void main(String[] args){
public static void main(String[] args) {
if (args == null || args.length < 3) {
System.err.println("invalidate input params:");
System.err.println("1.is production: true/false");
......@@ -26,9 +26,9 @@ public class SparkTagProcessByTagGroupId {
boolean isProd = Boolean.parseBoolean(args[0]);
boolean extractData = Boolean.parseBoolean(args[1]);
String[] arrs=args[2].split(",");
List<Long>params=new ArrayList();
for(String s:arrs){
String[] arrs = args[2].split(",");
Set<Long> params = new HashSet<>();
for (String s : arrs) {
params.add(Long.parseLong(s));
}
ShardingConfigManager.initDefualt();
......
src/main/java/com/gic/spark/datasource/entity/DataSourceMysql.java
......@@ -43,7 +43,7 @@ public class DataSourceMysql extends DataSourceEntity {
@Override
public String getHiveTableName() {
return getSchema()+"." + datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable;
return getSchema() + "." + datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable;
}
@Override
......@@ -51,7 +51,7 @@ public class DataSourceMysql extends DataSourceEntity {
SparkSession sparkSession = SparkEnvManager.getInstance().getSparkSession();
Dataset<Row> sourceDataset = datasource.buildRddManager().getDatasetByEnterpriseIds(targetTable, enterpriseList, null, null).repartition(new Column("enterprise_id"));
SparkHiveUtil.createHivePartitionTable(sourceDataset, "enterprise_id", getSchema(), datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable,sparkSession);
SparkHiveUtil.createHivePartitionTable(sourceDataset, "enterprise_id", getSchema(), datasource.getSchema().replaceAll("\\.", "_") + "_" + targetTable, sparkSession);
HivePartitionUtil.saveDatasetToPartitionTable(sparkSession, sourceDataset, getHiveTableName());
......
src/main/java/com/gic/spark/filter/AbstractTagConsumRecordFilter.java
......@@ -16,82 +16,84 @@ import java.util.List;
/**
* @description:
* 消费记录
* @description: 消费记录
* @author: wangxk
* @date: 2020/8/3
*/
public abstract class AbstractTagConsumRecordFilter implements BaseTagFilter {
DataSourceHive dataSourceHiveOrder = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_WDORDER_D);
DataSourceHive dataSourceHiveOrder = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_WDORDER_D);
DataSourceHive dataSourceHiveOrderItem = new DataSourceHive(ConstantUtil.DWD_GIC_TRD_VIRTUAL_ORDER_ITEM_D);
protected static JavaRDD<TrdVirtualOrderBean> statisticsTypeHandle(JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd, AbstractFilterRequest request){
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD=orderRdd.mapPartitions(data->{
List<TrdVirtualOrderBean> result=new ArrayList();
while (data.hasNext()){
Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>> tp2=data.next();
TrdVirtualOrderBean consumeRecordBean=tp2._1();
switch (request.getStatisticsType()){
case COMMODITYBRAND:
if(tp2._2().isPresent()){
for(String ent_brand_id:tp2._2().get()){
if(request.getStatisticsValList().contains(ent_brand_id)){
result.add(consumeRecordBean);
break;
}
}
}
protected static JavaRDD<TrdVirtualOrderBean> statisticsTypeHandle(JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd, AbstractFilterRequest request) {
JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = orderRdd.mapPartitions(data -> {
List<TrdVirtualOrderBean> result = new ArrayList();
while (data.hasNext()) {
Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>> tp2 = data.next();
TrdVirtualOrderBean consumeRecordBean = tp2._1();
switch (request.getStatisticsType()) {
case COMMODITYBRAND:
if (tp2._2().isPresent()) {
for (String ent_brand_id : tp2._2().get()) {
if (request.getStatisticsValList().contains(ent_brand_id)) {
result.add(consumeRecordBean);
break;
}
}
}
break;
case CHANNEL:
if(request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getStore_info_id()))){
result.add(consumeRecordBean);
}
break;
case MCUINFO:
if(request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getArea_id()))){
result.add(consumeRecordBean);
}
break;
default:break;
}
}
return result.iterator();
});
break;
case CHANNEL:
if (request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getStore_info_id()))) {
result.add(consumeRecordBean);
}
break;
case MCUINFO:
if (request.getStatisticsValList().contains(String.valueOf(consumeRecordBean.getArea_id()))) {
result.add(consumeRecordBean);
}
break;
default:
break;
}
}
return result.iterator();
});
return consumeRecordRDD;
}
protected static boolean checkTime(AbstractFilterRequestTime request,long consumeTime){
Boolean result=false;
switch (request.getTimeRangeType()){
protected static boolean checkTime(AbstractFilterRequestTime request, long consumeTime) {
Boolean result = false;
switch (request.getTimeRangeType()) {
case FIXATION:
if(consumeTime>=request.getBeginTime().getTime()
&&consumeTime<=request.getEndTime().getTime())
result=true;
if (consumeTime >= request.getBeginTime().getTime()
&& consumeTime <= request.getEndTime().getTime())
result = true;
break;
case LATELY:
switch (request.getYearMonthDayType()){
switch (request.getYearMonthDayType()) {
case DAY:
if(consumeTime> DateUtil.addNumForDay(new Date(),-request.getTimeNum()).getTime()){
result=true;
if (consumeTime > DateUtil.addNumForDay(new Date(), -request.getTimeNum()).getTime()) {
result = true;
}
break;
case MONTH:
if(consumeTime>DateUtil.addNumForMonth(new Date(),-request.getTimeNum()).getTime()){
result=true;
if (consumeTime > DateUtil.addNumForMonth(new Date(), -request.getTimeNum()).getTime()) {
result = true;
}
break;
case YEAR:
if(consumeTime>DateUtil.addNumForYear(new Date(),-request.getTimeNum()).getTime()){
result=true;
if (consumeTime > DateUtil.addNumForYear(new Date(), -request.getTimeNum()).getTime()) {
result = true;
}
break;
default:break;
default:
break;
}
break;
default:break;
default:
break;
}
return result;
return result;
}
}
src/main/java/com/gic/spark/filter/TagConsumeCommodityFilter.java
package com.gic.spark.filter;
import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceHive;
import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TrdVirtualOrderBean;
import com.gic.spark.entity.bean.TrdVirtualOrderItemBean;
import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagConsumeCommodityRequest;
import com.gic.spark.util.ConstantUtil;
import com.gic.spark.util.DateUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.spark.api.java.JavaPairRDD;
......@@ -25,14 +23,14 @@ import java.util.List;
* @author: wangxk
* @date: 2020/8/12
*/
public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{
public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter {
private static TagConsumeCommodityFilter instance;
public static TagConsumeCommodityFilter getInstance() {
if(null==instance){
instance=new TagConsumeCommodityFilter();
if (null == instance) {
instance = new TagConsumeCommodityFilter();
}
return instance;
}
......@@ -47,37 +45,37 @@ public class TagConsumeCommodityFilter extends AbstractTagConsumRecordFilter{
@Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeCommodityRequest commodityRequest=(TagConsumeCommodityRequest)request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD();
TagConsumeCommodityRequest commodityRequest = (TagConsumeCommodityRequest) request;
JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
Dataset<Row>OrderItemDS= dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
Dataset<Row> OrderItemDS = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderAndItemRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id","ent_brand_id").javaRDD()
.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1)))
.groupByKey())
.map(data->data._2());
JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderAndItemRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id", "ent_brand_id").javaRDD()
.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1)))
.groupByKey())
.map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderAndItemRdd,commodityRequest);
JavaPairRDD<Long,Long>orderRdd= consumeRecordRDD.filter(data-> StringUtils.isNotEmpty(data.getReceipts_time()))
.filter(data->checkTime(commodityRequest,DateUtil.strToDate(data.getReceipts_time(),DateUtil.FORMAT_DATETIME_19).getTime()))
.mapToPair(data-> Tuple2.apply(data.getVirtual_id(),data.getEcu_id()));
consumeRecordRDD = statisticsTypeHandle(orderAndItemRdd, commodityRequest);
JavaPairRDD<Long, Long> orderRdd = consumeRecordRDD.filter(data -> StringUtils.isNotEmpty(data.getReceipts_time()))
.filter(data -> checkTime(commodityRequest, DateUtil.strToDate(data.getReceipts_time(), DateUtil.FORMAT_DATETIME_19).getTime()))
.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data.getEcu_id()));
JavaPairRDD<Long,Long> orderItemRDD=MysqlRddManager.getPojoFromDataset(OrderItemDS,TrdVirtualOrderItemBean.class).javaRDD()
.filter(data->{
if(StringUtils.isNotEmpty(data.getSku_code())
&&commodityRequest.getSkuCodeList().contains(data.getSku_code())){
return true;
}
return false;
}).mapToPair(data->Tuple2.apply(data.getVirtual_order_id(),data.getVirtual_order_id()))
.reduceByKey((x,y)->x);
JavaPairRDD<Long, Long> orderItemRDD = MysqlRddManager.getPojoFromDataset(OrderItemDS, TrdVirtualOrderItemBean.class).javaRDD()
.filter(data -> {
if (StringUtils.isNotEmpty(data.getSku_code())
&& commodityRequest.getSkuCodeList().contains(data.getSku_code())) {
return true;
}
return false;
}).mapToPair(data -> Tuple2.apply(data.getVirtual_order_id(), data.getVirtual_order_id()))
.reduceByKey((x, y) -> x);
JavaRDD<Long>ecuRdd=orderRdd.leftOuterJoin(orderItemRDD)
.filter(data->data._2()._2().isPresent())
.map(data->data._2()._1()).distinct();
JavaRDD<Long> ecuRdd = orderRdd.leftOuterJoin(orderItemRDD)
.filter(data -> data._2()._2().isPresent())
.map(data -> data._2()._1()).distinct();
return ecuRdd;
return ecuRdd;
}
}
src/main/java/com/gic/spark/filter/TagCouponFilter.java
......@@ -4,12 +4,13 @@ import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceSharding;
import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TagCouponBean;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagCouponRequest;
import com.gic.spark.util.AppEnvUtil;
import com.gic.spark.util.ConstantUtil;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Column;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
......@@ -22,15 +23,18 @@ import java.util.Set;
*/
public class TagCouponFilter implements BaseTagFilter {
private DataSourceSharding dataSourceSharding=new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG);
private DataSourceSharding dataSourceSharding = new DataSourceSharding(AppEnvUtil.MARKETING_SHARDING, ConstantUtil.TAB_COUPON_LOG);
private static TagCouponFilter instance;
public static TagCouponFilter getInstance(){
if(null==instance){
instance=new TagCouponFilter();
public static TagCouponFilter getInstance() {
if (null == instance) {
instance = new TagCouponFilter();
}
return instance;
}
private TagCouponFilter(){}
private TagCouponFilter() {
}
@Override
public List<DataSourceEntity> necessarySourceList() {
......@@ -41,57 +45,58 @@ public class TagCouponFilter implements BaseTagFilter {
@Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagCouponRequest couponRequest=(TagCouponRequest)request;
JavaRDD<TagCouponBean> couponBeanRDD= MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId)
.filter(new Column("status").isNotNull()),TagCouponBean.class).javaRDD()
.filter(data->couponRequest.getCouponList().contains(data.getCoupon_id()));
JavaRDD<Long>ecuRDD=couponBeanRDD.mapPartitions(data->{
Set<Long>result=new HashSet();
while (data.hasNext()){
TagCouponBean couponBean=data.next();
switch (couponRequest.getCouponType()){
case UNCLAIMED:
if(couponBean.getStatus()==3){
result.add(couponBean.getEcu_Id());
}
break;
case NO_CANCEL:
if(couponBean.getStatus()==4
&&couponBean.getEffect_End_Time().getTime()>System.currentTimeMillis()){
result.add(couponBean.getEcu_Id());
}
break;
case NO_CANCEL_EXPIRES:
if(couponBean.getStatus()==4
&&couponBean.getEffect_End_Time().getTime()<System.currentTimeMillis()){
result.add(couponBean.getEcu_Id());
}
break;
case CANCEL:
if(couponBean.getStatus()==7){
result.add(couponBean.getEcu_Id());
}
break;
case OCCUPY:
if(couponBean.getStatus()==8){
result.add(couponBean.getEcu_Id());
}
break;
case DONATION_IN:
if(couponBean.getStatus()==9){
result.add(couponBean.getEcu_Id());
}
break;
case YET_DONATION:
if(couponBean.getStatus()==10){
result.add(couponBean.getEcu_Id());
}
break;
default:break;
}
}
return result.iterator();
});
TagCouponRequest couponRequest = (TagCouponRequest) request;
JavaRDD<TagCouponBean> couponBeanRDD = MysqlRddManager.getPojoFromDataset(dataSourceSharding.getDatasetByEnterpriseId(enterpriseId)
.filter(new Column("status").isNotNull()), TagCouponBean.class).javaRDD()
.filter(data -> couponRequest.getCouponList().contains(data.getCoupon_id()));
JavaRDD<Long> ecuRDD = couponBeanRDD.mapPartitions(data -> {
Set<Long> result = new HashSet();
while (data.hasNext()) {
TagCouponBean couponBean = data.next();
switch (couponRequest.getCouponType()) {
case UNCLAIMED:
if (couponBean.getStatus() == 3) {
result.add(couponBean.getEcu_Id());
}
break;
case NO_CANCEL:
if (couponBean.getStatus() == 4
&& couponBean.getEffect_End_Time().getTime() > System.currentTimeMillis()) {
result.add(couponBean.getEcu_Id());
}
break;
case NO_CANCEL_EXPIRES:
if (couponBean.getStatus() == 4
&& couponBean.getEffect_End_Time().getTime() < System.currentTimeMillis()) {
result.add(couponBean.getEcu_Id());
}
break;
case CANCEL:
if (couponBean.getStatus() == 7) {
result.add(couponBean.getEcu_Id());
}
break;
case OCCUPY:
if (couponBean.getStatus() == 8) {
result.add(couponBean.getEcu_Id());
}
break;
case DONATION_IN:
if (couponBean.getStatus() == 9) {
result.add(couponBean.getEcu_Id());
}
break;
case YET_DONATION:
if (couponBean.getStatus() == 10) {
result.add(couponBean.getEcu_Id());
}
break;
default:
break;
}
}
return result.iterator();
});
return ecuRDD;
}
......
src/main/java/com/gic/spark/filter/TagFirstOnlineConsumptionStoreFilter.java
......@@ -20,18 +20,19 @@ import java.util.List;
* @author: wangxk
* @date: 2020/8/11
*/
public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecordFilter{
public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecordFilter {
private static TagFirstOnlineConsumptionStoreFilter instance;
public static TagFirstOnlineConsumptionStoreFilter getInstance() {
if(null==instance){
instance=new TagFirstOnlineConsumptionStoreFilter();
if (null == instance) {
instance = new TagFirstOnlineConsumptionStoreFilter();
}
return instance;
}
private TagFirstOnlineConsumptionStoreFilter(){}
private TagFirstOnlineConsumptionStoreFilter() {
}
@Override
public List<DataSourceEntity> necessarySourceList() {
......@@ -43,30 +44,30 @@ public class TagFirstOnlineConsumptionStoreFilter extends AbstractTagConsumRecor
@Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeStoreRequest storeRequest=(TagConsumeStoreRequest)request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD();
TagConsumeStoreRequest storeRequest = (TagConsumeStoreRequest) request;
JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
JavaRDD<Row>virtualOrderItemRdd=dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id","ent_brand_id").javaRDD();
JavaRDD<Row> virtualOrderItemRdd = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id", "ent_brand_id").javaRDD();
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))).groupByKey())
.map(data->data._2());
JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1))).groupByKey())
.map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderRdd,storeRequest);
JavaRDD<Long>ecuRdd=consumeRecordRDD.filter(data->data.getOrder_channel_code()!=1
&& StringUtils.isNotEmpty(data.getReceipts_time())
&& null!=data.getShop_id())
.mapToPair(data-> Tuple2.apply(data.getEcu_id(),Tuple2.apply(data.getReceipts_time(),data.getShop_id())))
.reduceByKey((x,y)->{
if(DateUtil.strToDate(x._1(),DateUtil.FORMAT_DATETIME_19).getTime()
<DateUtil.strToDate(y._1(),DateUtil.FORMAT_DATETIME_19).getTime()){
return x;
}else{
return y;
}
}).filter(data->storeRequest.getStoreList().contains(String.valueOf(data._2()._2())))
.map(data->data._1());
consumeRecordRDD = statisticsTypeHandle(orderRdd, storeRequest);
JavaRDD<Long> ecuRdd = consumeRecordRDD.filter(data -> data.getOrder_channel_code() != 1
&& StringUtils.isNotEmpty(data.getReceipts_time())
&& null != data.getShop_id())
.mapToPair(data -> Tuple2.apply(data.getEcu_id(), Tuple2.apply(data.getReceipts_time(), data.getShop_id())))
.reduceByKey((x, y) -> {
if (DateUtil.strToDate(x._1(), DateUtil.FORMAT_DATETIME_19).getTime()
< DateUtil.strToDate(y._1(), DateUtil.FORMAT_DATETIME_19).getTime()) {
return x;
} else {
return y;
}
}).filter(data -> storeRequest.getStoreList().contains(String.valueOf(data._2()._2())))
.map(data -> data._1());
return ecuRdd;
}
}
src/main/java/com/gic/spark/filter/TagHistoryConsumeCommodityFilter.java
package com.gic.spark.filter;
import com.gic.spark.datasource.entity.DataSourceEntity;
import com.gic.spark.datasource.entity.DataSourceHive;
import com.gic.spark.datasource.mysql.MysqlRddManager;
import com.gic.spark.entity.bean.TrdVirtualOrderBean;
import com.gic.spark.entity.bean.TrdVirtualOrderItemBean;
import com.gic.spark.entity.request.AbstractFilterRequest;
import com.gic.spark.entity.request.TagConsumeCommodityRequest;
import com.gic.spark.util.ConstantUtil;
import com.gic.spark.util.DateUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.Optional;
import org.apache.spark.sql.Dataset;
......@@ -25,13 +21,13 @@ import java.util.List;
* @author: wangxk
* @date: 2020/8/12
*/
public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFilter{
public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFilter {
private static TagHistoryConsumeCommodityFilter instance;
public static TagHistoryConsumeCommodityFilter getInstance() {
if(null==instance){
instance=new TagHistoryConsumeCommodityFilter();
if (null == instance) {
instance = new TagHistoryConsumeCommodityFilter();
}
return instance;
}
......@@ -46,27 +42,27 @@ public class TagHistoryConsumeCommodityFilter extends AbstractTagConsumRecordFil
@Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeCommodityRequest commodityRequest=(TagConsumeCommodityRequest)request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId),TrdVirtualOrderBean.class).javaRDD();
TagConsumeCommodityRequest commodityRequest = (TagConsumeCommodityRequest) request;
JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
Dataset<Row> OrderItemDS= dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
Dataset<Row> OrderItemDS = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId);
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderAndItemRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id","ent_brand_id").javaRDD()
.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1)))
.groupByKey())
.map(data->data._2());
JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderAndItemRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(OrderItemDS.select("virtual_order_id", "ent_brand_id").javaRDD()
.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1)))
.groupByKey())
.map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderAndItemRdd,commodityRequest);
JavaRDD<TrdVirtualOrderItemBean> orderItemRDD=MysqlRddManager.getPojoFromDataset(OrderItemDS,TrdVirtualOrderItemBean.class).javaRDD();
JavaRDD<Long>ecuRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data.getEcu_id()))
.leftOuterJoin(orderItemRDD.mapToPair(data->Tuple2.apply(data.getVirtual_order_id(),data.getSku_code()))
.filter(data->StringUtils.isNotEmpty(data._2())
&&commodityRequest.getSkuCodeList().contains(data._2())))
.filter(data->data._2()._2().isPresent())
.map(data->data._2()._1())
.distinct();
consumeRecordRDD = statisticsTypeHandle(orderAndItemRdd, commodityRequest);
JavaRDD<TrdVirtualOrderItemBean> orderItemRDD = MysqlRddManager.getPojoFromDataset(OrderItemDS, TrdVirtualOrderItemBean.class).javaRDD();
JavaRDD<Long> ecuRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data.getEcu_id()))
.leftOuterJoin(orderItemRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_order_id(), data.getSku_code()))
.filter(data -> StringUtils.isNotEmpty(data._2())
&& commodityRequest.getSkuCodeList().contains(data._2())))
.filter(data -> data._2()._2().isPresent())
.map(data -> data._2()._1())
.distinct();
return ecuRdd;
return ecuRdd;
}
}
src/main/java/com/gic/spark/filter/TagHistoryConsumeTotalFilter.java
......@@ -19,18 +19,19 @@ import java.util.List;
* @author: wangxk
* @date: 2020/8/6
*/
public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{
public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter {
private static TagHistoryConsumeTotalFilter instance;
public static TagHistoryConsumeTotalFilter getInstance() {
if(null==instance){
instance=new TagHistoryConsumeTotalFilter();
if (null == instance) {
instance = new TagHistoryConsumeTotalFilter();
}
return instance;
}
private TagHistoryConsumeTotalFilter(){}
private TagHistoryConsumeTotalFilter() {
}
@Override
public List<DataSourceEntity> necessarySourceList() {
......@@ -42,58 +43,59 @@ public class TagHistoryConsumeTotalFilter extends AbstractTagConsumRecordFilter{
@Override
public JavaRDD<Long> filterValidMember(Integer enterpriseId, AbstractFilterRequest request) {
TagConsumeAmountRequest consumeAmountRequest=(TagConsumeAmountRequest)request;
JavaRDD<TrdVirtualOrderBean>consumeRecordRDD= MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
TagConsumeAmountRequest consumeAmountRequest = (TagConsumeAmountRequest) request;
JavaRDD<TrdVirtualOrderBean> consumeRecordRDD = MysqlRddManager.getPojoFromDataset(dataSourceHiveOrder.getDatasetByEntId(enterpriseId), TrdVirtualOrderBean.class).javaRDD();
JavaRDD<Row>virtualOrderItemRdd=dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id","ent_brand_id").javaRDD();
JavaRDD<Row> virtualOrderItemRdd = dataSourceHiveOrderItem.getDatasetByEntId(enterpriseId).select("virtual_order_id", "ent_brand_id").javaRDD();
JavaRDD<Tuple2<TrdVirtualOrderBean,Optional<Iterable<String>>>>orderRdd=consumeRecordRDD.mapToPair(data->Tuple2.apply(data.getVirtual_id(),data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row->Tuple2.apply(row.getLong(0),row.getString(1))).groupByKey())
.map(data->data._2());
JavaRDD<Tuple2<TrdVirtualOrderBean, Optional<Iterable<String>>>> orderRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getVirtual_id(), data))
.leftOuterJoin(virtualOrderItemRdd.mapToPair(row -> Tuple2.apply(row.getLong(0), row.getString(1))).groupByKey())
.map(data -> data._2());
consumeRecordRDD=statisticsTypeHandle(orderRdd,consumeAmountRequest);
int configStatus= CommonUtil.getConfigStatus(enterpriseId);
consumeRecordRDD = statisticsTypeHandle(orderRdd, consumeAmountRequest);
int configStatus = CommonUtil.getConfigStatus(enterpriseId);
JavaRDD<Long>ecuRdd=consumeRecordRDD.mapToPair(data-> Tuple2.apply(data.getEcu_id(),configStatus==1?data.getPaid_amt():data.getPay_amt()))
.reduceByKey((x,y)->x+y)
.filter(data->{
boolean result=false;
switch (consumeAmountRequest.getNumberType()){
case between:
if(data._2()>=consumeAmountRequest.getBeginNum()
&&data._2()<=consumeAmountRequest.getEndNum()){
result=true;
}
break;
case lt:
if(data._2()<consumeAmountRequest.getEndNum()){
result=true;
}
break;
case gt:
if(data._2()>consumeAmountRequest.getBeginNum()){
result=true;
}
break;
case eq:
if(data._2()==consumeAmountRequest.getEqualNum()){
result=true;
}
break;
case lte:
if(data._2()<=consumeAmountRequest.getEndNum()){
result=true;
}
break;
case gte:
if(data._2()>=consumeAmountRequest.getBeginNum()){
result=true;
}
break;
default:break;
}
return result;
}).map(data->data._1());
JavaRDD<Long> ecuRdd = consumeRecordRDD.mapToPair(data -> Tuple2.apply(data.getEcu_id(), configStatus == 1 ? data.getPaid_amt() : data.getPay_amt()))
.reduceByKey((x, y) -> x + y)
.filter(data -> {
boolean result = false;
switch (consumeAmountRequest.getNumberType()) {
case between:
if (data._2() >= consumeAmountRequest.getBeginNum()
&& data._2() <= consumeAmountRequest.getEndNum()) {
result = true;
}
break;
case lt:
if (data._2() < consumeAmountRequest.getEndNum()) {
result = true;
}
break;
case gt:
if (data._2() > consumeAmountRequest.getBeginNum()) {
result = true;
}
break;
case eq:
if (data._2() == consumeAmountRequest.getEqualNum()) {
result = true;
}
break;
case lte:
if (data._2() <= consumeAmountRequest.getEndNum()) {
result = true;
}
break;
case gte:
if (data._2() >= consumeAmountRequest.getBeginNum()) {
result = true;
}
break;
default:
break;
}
return result;
}).map(data -> data._1());
return ecuRdd;
}
}
src/main/java/com/gic/spark/util/EsRequestUtil.java
package com.gic.spark.util;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
......@@ -7,7 +8,10 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/**
* Created by paste on 2018/7/21 16:29
......@@ -16,26 +20,26 @@ import java.util.Map;
*/
public class EsRequestUtil {
public static String getESIindexName(Integer enterpriseId,boolean isProduction) {
String url = isProduction?"https://ideal.demogic.com/member-config/member/index-version/" + enterpriseId
:"https://four.gicdev.com/member-config/member/index-version/" + enterpriseId;
HttpResponse response = getHttpResponseByGet(url,enterpriseId);
public static String getESIindexName(Integer enterpriseId, boolean isProduction) {
String url = isProduction ? "https://ideal.demogic.com/member-config/member/index-version/" + enterpriseId
: "https://four.gicdev.com/member-config/member/index-version/" + enterpriseId;
HttpResponse response = getHttpResponseByGet(url, enterpriseId);
int responseCode = response.getStatusLine().getStatusCode();
if (responseCode == 200) {
return HttpClient.getResponseString(response);
return HttpClient.getResponseString(response);
}
return null;
}
public static HttpResponse getHttpResponseByGet(String url,Integer enterpriseId) {
public static HttpResponse getHttpResponseByGet(String url, Integer enterpriseId) {
org.apache.http.client.HttpClient httpClient = HttpClient.getHttpClient();
HttpGet get = new HttpGet(url);
get.setHeader("sign",""+enterpriseId);
get.setHeader("route","/spark");
get.setHeader("project","memberfour");
get.setHeader("isControl","true");
get.setHeader("Refer",url);
get.setHeader("sign", "" + enterpriseId);
get.setHeader("route", "/spark");
get.setHeader("project", "memberfour");
get.setHeader("isControl", "true");
get.setHeader("Refer", url);
try {
HttpResponse response = httpClient.execute(get);
return response;
......@@ -48,16 +52,16 @@ public class EsRequestUtil {
return null;
}
public static String getIndexParam( Integer enterpriseId,Long id , boolean isProd) {
String url = isProd?"https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId="+id+"&enterpriseId="+enterpriseId
:"https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId="+id+"&enterpriseId="+enterpriseId;
public static String getIndexParam(Integer enterpriseId, Long id, boolean isProd) {
String url = isProd ? "https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId=" + id + "&enterpriseId=" + enterpriseId
: "https://four.gicdev.com/member-label/get-member-size-by-condition?sceneCrowdId=" + id + "&enterpriseId=" + enterpriseId;
HttpPost httpPost = new HttpPost(url);
httpPost.addHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.addHeader("project","member-tag");
httpPost.addHeader("sign","1129");
httpPost.addHeader("route","get-member-size-by-condition");
httpPost.addHeader("project", "member-tag");
httpPost.addHeader("sign", "1129");
httpPost.addHeader("route", "get-member-size-by-condition");
HttpResponse response = null;
try {
response = HttpClient.getHttpClient().execute(httpPost);
......@@ -71,6 +75,7 @@ public class EsRequestUtil {
}
public static void main(String[] args) throws IOException {
System.out.println(getIndexParam(1129,183676455182233606L,true));
// System.out.println(getIndexParam(1129,183676455182233606L,true));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment