手撕項目-Flink電商實時分析六

目錄

 

回顧

頻道新鮮度編寫

總結

---

回顧

 

之前編寫了計算pvuv的mapredice，並且可以在本地進行測試，將計算結果存入hbase

 

 

頻道新鮮度編寫

 

新鮮度：每天的活躍用戶中，新用戶的數量和老用戶的數量的一個比例

map代碼：

public class PindaoXinXianDuMap implements FlatMapFunction<KafkaMessage,PidaoXinXianDu> {

    @Override
    public void flatMap(KafkaMessage value, Collector<PidaoXinXianDu> out) throws Exception {
        String jsonstring = value.getJsonmessage();
        long timestamp = value.getTimestamp();


        String hourtimestamp = DateUtil.getDateby(timestamp,"yyyyMMddhh");//小時
        String daytimestamp = DateUtil.getDateby(timestamp,"yyyyMMdd");//天
        String monthtimestamp = DateUtil.getDateby(timestamp,"yyyyMM");//月

        UserscanLog userscanLog = JSON.parseObject(jsonstring, UserscanLog.class);
        long pingdaoid = userscanLog.getPingdaoid();
        long userid = userscanLog.getUserid();

        UserState userState = PdvisterDao.getUserSatebyvistertime(userid+"",timestamp);
        boolean isFirsthour = userState.isFisrthour();
        boolean isFisrtday = userState.isFisrtday();
        boolean isFisrtmonth = userState.isFisrtmonth();

        PidaoXinXianDu pidaoXinXianDu = new PidaoXinXianDu();
        pidaoXinXianDu.setPingdaoid(pingdaoid);
        pidaoXinXianDu.setTimestamp(timestamp);
        /**
         * 新增用戶
         */
        long newuser = 0l;
        if(userState.isIsnew()){
            newuser = 1l;
        }
        pidaoXinXianDu.setNewcount(newuser);

        /**
         * 小時
         */
        long olduser = 0l;
        if(!userState.isIsnew()&&isFirsthour){
            olduser = 1l;
        }
        pidaoXinXianDu.setOldcount(olduser);
        pidaoXinXianDu.setTimestring(hourtimestamp);
        pidaoXinXianDu.setGroupbyfield(hourtimestamp+pingdaoid);
        out.collect(pidaoXinXianDu);
        System.out.println("小時=="+pidaoXinXianDu);
        /**
         * 天
         */
        olduser = 0l;
        if(!userState.isIsnew()&&isFisrtday){
            olduser = 1l;
        }
        pidaoXinXianDu.setOldcount(olduser);
        pidaoXinXianDu.setTimestring(daytimestamp);
        pidaoXinXianDu.setGroupbyfield(daytimestamp+pingdaoid);
        out.collect(pidaoXinXianDu);
        System.out.println("小時=="+pidaoXinXianDu);
        /**
         * 月
         */
        olduser = 0l;
        if(!userState.isIsnew()&&isFisrtmonth){
            olduser = 1l;
        }
        pidaoXinXianDu.setOldcount(olduser);
        pidaoXinXianDu.setTimestring(monthtimestamp);
        pidaoXinXianDu.setGroupbyfield(monthtimestamp+pingdaoid);
        out.collect(pidaoXinXianDu);
        System.out.println("小時=="+pidaoXinXianDu);
    }
}

 

reduce代碼：

public class PindaoXinxianduReduce implements ReduceFunction<PidaoXinXianDu> {

    @Override
    public PidaoXinXianDu reduce(PidaoXinXianDu value1, PidaoXinXianDu value2) throws Exception {
        System.out.println( "value1=="+value1);
        System.out.println( "value2=="+value2);
        long pingdaoid = value1.getPingdaoid();
        long timestampvalue = value1.getTimestamp();
        String timestring = value1.getTimestring();
        long newcountvalue1 = value1.getNewcount();
        long oldcountvalue1 = value1.getOldcount();

        long newcountvalue2 = value2.getNewcount();
        long oldcountvalue2 = value2.getOldcount();

        PidaoXinXianDu pidaoXinXianDu = new PidaoXinXianDu();
        pidaoXinXianDu.setPingdaoid(pingdaoid);
        pidaoXinXianDu.setTimestamp(timestampvalue);
        pidaoXinXianDu.setTimestring(timestring);
        pidaoXinXianDu.setNewcount(newcountvalue1+newcountvalue2);
        pidaoXinXianDu.setOldcount(oldcountvalue1+oldcountvalue2);
        System.out.println( "recuduce --pidaoXinXianDu=="+pidaoXinXianDu);
        return  pidaoXinXianDu;
    }
}

 

sink代碼：

public class PindaoXinXiandusinkreduce implements SinkFunction<PidaoXinXianDu> {
    @Override
    public void invoke(PidaoXinXianDu value, Context context) throws Exception {
        System.out.println( "recuducesinkd --pidaoPvUv=="+value);
        long pingdaoid = value.getPingdaoid();
        long newcount = value.getNewcount();
        long oldcount = value.getOldcount();
        String timestring = value.getTimestring();
        String newcountstring = HbaseUtil.getdata("pindaoinfo",pingdaoid+timestring,"info","xinxiandunewcount");
        String oldcountstring = HbaseUtil.getdata("pindaoinfo",pingdaoid+timestring,"info","xinxianduoldcount");
        if(StringUtils.isNotBlank(newcountstring)){
            newcount += newcount + Long.valueOf(newcountstring);
        }
        if(StringUtils.isNotBlank(oldcountstring)) {
            oldcount += oldcount + Long.valueOf(oldcountstring);
        }

        Map<String,String> datamap = new HashMap<String,String>();
        datamap.put("xinxiandunewcount",newcount+"");
        datamap.put("xinxianduoldcount",oldcount+"");
        System.out.println( "xinxiandu---- HbaseUtil.put(pindaoinfo+"+","+pingdaoid+timestring+",info"+datamap+")");
        HbaseUtil.put("pindaoinfo",pingdaoid+timestring,"info",datamap);
    }
}

 

總結

 

由於flnik的addSink（）方法沒有執行，還在調試，所以就不上結果圖了，後續編寫瀏覽地區分佈、用戶網絡、瀏覽器使用分析等

具體代碼可參照我的git項目地址,現有代碼均已通過測試可以使用，後續會持續更新，直到項目結束，不懂的細節，可以關注公衆號，後臺留言，會細緻解答。

git地址:https://github.com/jyqjyq/filnkDS.git