數據幀緩衝隊列（改進，數據可視化）（三）

《一個簡易的數據幀緩衝隊列 (一)》
《數據幀緩衝隊列（繼上一篇的簡單隊列，改進）（二）》
《數據幀緩衝隊列（改進，數據可視化）（三）》
一個c++實現的數據緩衝隊列，可以詳細分析數據的出入時間點，可視化數據。
基於上一篇，做以下改進：
添加log模塊，生成log文件，log文件可以通過python程序顯示成散點圖，分析數據的入隊列，出隊列詳細時間點。
python腳本程序在文末（在比較新的ubuntu上已經默認支持了python3 直接python3 XX 運行程序即可。wind上需要安裝python，並且安裝對應圖形庫）
ubuntu 18.04上實測。 數據可視化使用 python matplotlib工具。

/*
* queue.h  canok
*/
#ifndef  __QUEUE_HEAD_H__
#define __QUEUE_HEAD_H__
#include <pthread.h>
#include <semaphore.h>

typedef unsigned char   uint8_t;     //

#define MIN(a,b) (a)<(b)?(a):(b)
//#define SIGLEFRAME_LEN (1024*125) //125K
#define SIGLEFRAME_LEN (1024*512) //512k

#define USE_SEAMPHORE 0
#define EN_LOG_FILE 1

typedef struct _QUEUE_NODE_
{
	pthread_mutex_t mlock;
	int datalen;
	uint8_t data[SIGLEFRAME_LEN];
} QUEUE_NODE;



class CQueue
{
	public :
		CQueue(int frames,char*name=NULL);//the name max len 128
		~CQueue();
		
		/*
		* \\no block
		*\\ pram: data, the dest data buffer, len , dest data buffer lenth,
		*\\ return : real  output data lenth
		*/
		int pop(uint8_t *data,int len);

		
		/*
		*\\ pram: data, the src data, len , the src data lenth
		*\\ return : real  input data lenth
		*/
		int push(uint8_t *data,int len);

		/*	
		* \\ breif :get the data buffer pointer whith block, Manual release is required after use
		*\\  param : **pdata , *plen :the queue data len
		*\\  return -1; no data to get. other the que index will be return ,
		*/
		int getbuffer(uint8_t **pdata,int *plen);
		
		/*
		*\\ breif : get back the que data buffer
		*\\ pram:the que index 
		*\\ return : 0
		*/
		int releasebuffer(int index);

		/*
		*\\ breif : flush data
		*\\ return : 0
		*/
		int flush();

		int getmEffectives();

	private:
		void addInindex();
		void addOutindex();
		int IncreaseEffectives();
		int reduceEffectives();
	private:
        char sName[128];
		int maxFrames;
		QUEUE_NODE *que;
		int mInindex;
		int mOutindex;
		int mEffectives;
#if USE_SEAMPHORE
		sem_t mSem;
#else
	pthread_cond_t mCond;
	pthread_mutex_t mCondLock;
#endif

#if EN_LOG_FILE
	FILE* fp_log;
#endif
	pthread_mutex_t mIndexlock;
		
	
};
#endif

/*
* queue.cpp canok  g++ queue.cpp -lpthread
*/
#include<stdio.h>
#include<stdlib.h>
#include <string.h>
//#include "jnilog.h"
#include <sys/time.h>
#include "queue.h"
#define LOOP_ADD(i,r) ((i)+1)>=(r)?0:(i+1)

#define ERRO_INFO(fmt, args...) do {printf("[%s,%s,%d]",__FILE__,__FUNCTION__,__LINE__);printf(fmt,##args);} while(0)
#if 1
#define DEBUG(fmt, args...)		do {printf("[%s,%s,%d]",__FILE__,__FUNCTION__,__LINE__);printf(fmt,##args);} while(0)
//#define DEBUG(...)  __android_log_print(ANDROID_LOG_INFO,"Queue","[%d %s]",__LINE__,__FUNCTION__);__android_log_print(ANDROID_LOG_INFO,"Queue",__VA_ARGS__)
#else
#define DEBUG(fmt, args...)  
#endif



#define DEBUG_DIR  "/storage/emulated/0/"
#define LOG_FILE(name) DEBUG_DIR name
CQueue::CQueue(int frames,char*name):
maxFrames(frames),
mInindex(0),
mOutindex(0),
mEffectives(0)
{
	if(name)
	{
		snprintf(sName, sizeof(sName), "%s", name);
	}
	else
	{
		snprintf(sName,sizeof(sName),"log_maxFrames%02d",maxFrames);
	}
#if EN_LOG_FILE
	//fp_log = fopen(LOG_FILE(sName),"w+");
    char filepath[256];
    //snprintf(filepath,sizeof(filepath),"%s/%s","/storage/emulated/0/",sName);
    snprintf(filepath,sizeof(filepath),"%s/%s","./",sName);
    fp_log = fopen(filepath,"w+");
	if(fp_log == NULL)
	{
		DEBUG("fopen erro");
	}
	else
	{
		setbuf(fp_log,0); //立即寫入，不緩存
	}
#endif
	que = (QUEUE_NODE *)malloc(frames*sizeof(QUEUE_NODE));
	if(que == NULL)
	{
		printf("erro to malloc\n");
	}
	
	pthread_mutexattr_t mutextattr;
	pthread_mutexattr_init(&mutextattr);
	// 設置互斥鎖在進程之間共享
	pthread_mutexattr_setpshared(&mutextattr, PTHREAD_PROCESS_SHARED);
	int i =0;
	for(i=0;i++;i<maxFrames)
	{
		pthread_mutex_init(&que[i].mlock, &mutextattr);
	}
	pthread_mutex_init(&mIndexlock, &mutextattr);
#if USE_SEAMPHORE
	sem_init(&mSem, 0, 0);
#else
	//第二個參數爲NULL，使用默認屬性，不在進程間共享
	pthread_cond_init(&mCond,NULL);
	pthread_mutex_init(&mCondLock,NULL);//默認屬性

#endif
}

CQueue::~CQueue()
{
	int i =0;
	for(i=0;i++;i<maxFrames)
	{
		pthread_mutex_destroy(&que[i].mlock);
	}
	pthread_mutex_destroy(&mIndexlock);
#if USE_SEAMPHORE
	sem_destroy(&mSem);
#else
	pthread_cond_destroy(&mCond);
	pthread_mutex_destroy(&mCondLock);
#endif
#if EN_LOG_FILE
	if(fp_log)
	{
		fclose(fp_log);
	}
#endif
	if(que != NULL)
	{
		free(que);
		que = NULL;
	}
}
int CQueue::pop(uint8_t *data,int len)
{
	//DEBUG("in %d out %d len %d\n ",mInindex,mOutindex,len);
#if USE_SEAMPHORE
	//sem_wait(&mSem);
	sem_trywait(&mSem);
#endif
	int iout = mOutindex;
	
	pthread_mutex_t * ilock = &que[iout].mlock;
	pthread_mutex_lock(ilock);
	
	if(reduceEffectives() ==0)
	{	
		mEffectives = 0;
		pthread_mutex_unlock(ilock);
		DEBUG("no data\n");
		return 0;
	}
	
	int copylen = MIN(len,que[iout].datalen);
	if(data != NULL)
	{
		memcpy(data,que[iout].data,copylen);
	}
	
	addOutindex();
	pthread_mutex_unlock(ilock);
	return copylen;
}

int CQueue::push(uint8_t *data,int len)
{
	//DEBUG("in %d out %d len %d\n ",mInindex,mOutindex,len);
	int iIn = mInindex;
	pthread_mutex_t * ilock = &que[iIn].mlock;
	
	pthread_mutex_lock(ilock);

	DEBUG("maxFrames%d , push---- mEffectives %d\n",maxFrames,mEffectives);
#if EN_LOG_FILE
	if(fp_log)
	{
		struct timeval test_time;
		struct tm *st_tm = NULL;
		gettimeofday(&test_time,NULL);
		st_tm = gmtime(&test_time.tv_sec);
		long long m0 = test_time.tv_usec;

		fprintf(fp_log,"IN>>>mEffectives:%02d,lenth:%08d [%02d:%02d:%02d:%06ld] \n",mEffectives,len,st_tm->tm_hour,st_tm->tm_min,st_tm->tm_sec,test_time.tv_usec);
		//fwrite(data,1,len,fp_log);
	}
#endif
	int copylen = MIN(len,sizeof(que[iIn].data));
	
	if(data != NULL)
	{
		memcpy(que[iIn].data,data,copylen);
		que[iIn].datalen = copylen;
		if(copylen < len)
		{
			DEBUG("Buffer single frame too short ! Buffer_frame_len:%ld,data_len:%d \n ",sizeof(que[iIn].data),len);
		}
	}
	
	addInindex();
#if USE_SEAMPHORE
	sem_post(&mSem);
#else
	pthread_mutex_lock(&mCondLock);
	pthread_cond_signal(&mCond);
	pthread_mutex_unlock(&mCondLock);
#endif
	if(IncreaseEffectives() == maxFrames)
	{//覆蓋老數據
		DEBUG("cover  mEffectives %d ,maxFrames %d\n",mEffectives,maxFrames);
		//DEBUG("cover  mInindex %d ,mOutindex %d\n",mInindex,mOutindex);
#if USE_SEAMPHORE
		sem_trywait(&mSem);
#endif
		addOutindex();
	}


	pthread_mutex_unlock(ilock);
	

	return copylen;

}

int CQueue::flush()
{
    pthread_mutex_lock(&mIndexlock);
    while(mEffectives > 0)
    {
#if USE_SEAMPHORE
        sem_trywait(&mSem);
#endif
        mOutindex = LOOP_ADD(mOutindex, maxFrames);
        mEffectives --;
    }
    mEffectives =0;
    pthread_mutex_unlock(&mIndexlock);
    return 0;
}

int CQueue::getmEffectives()
{
	int ret = 0;
	pthread_mutex_lock(&mIndexlock);
	ret = mEffectives;
	pthread_mutex_unlock(&mIndexlock);
	return ret;
}

int CQueue::getbuffer(uint8_t **pdata,int *plen)
{
    DEBUG("maxFrames%d get , mEffectives %d\n",maxFrames,mEffectives);
	int val=0;
#if USE_SEAMPHORE
	sem_wait(&mSem); //阻塞 得有點不合常理，有時 geteffective 都到10了，這個才通過。----最後發現確實push端
	        //太快，1ms push了10多幀，當然 get不過來
#else
		if(getmEffectives()<=0) {
			pthread_cond_wait(&mCond,&mCondLock);
		}
#endif

	int iout = mOutindex;

	pthread_mutex_t * ilock = &que[iout].mlock;
	pthread_mutex_lock(ilock);
	//DEBUG("pop  mEffectives %d ,maxFrames %d\n",mEffectives,maxFrames);
	
	if(reduceEffectives() ==0 )
	{	
		*pdata = NULL;
		*plen =  0;
		pthread_mutex_unlock(ilock);
		DEBUG("no data\n");
		return -1;
	}

	*pdata = que[iout].data;
	*plen =  que[iout].datalen;

#if EN_LOG_FILE
	if(fp_log)
	{
		struct timeval test_time;
		struct tm *st_tm = NULL;
		gettimeofday(&test_time,NULL);
		st_tm = gmtime(&test_time.tv_sec);
		long long m0 = test_time.tv_usec;

		fprintf(fp_log,"out<<<mEffectives:%02d,lenth:%08d [%02d:%02d:%02d:%06ld] \n",mEffectives,*plen,st_tm->tm_hour,st_tm->tm_min,st_tm->tm_sec,test_time.tv_usec);
		//fwrite(data,1,len,fp_log);
	}
#endif
	
	addOutindex();
	//DEBUG("in %d out %d len %d unlock \n ",mInindex,mOutindex,len);
	//pthread_mutex_unlock(ilock);
	
	//DEBUG("get index  %d \n",iout);
	return iout;
}
int CQueue::releasebuffer(int index)
{
	//DEBUG("releas %d \n",index);
	if(0 <= index && index < maxFrames)
	{
		pthread_mutex_unlock(&que[index].mlock);
	}
	return 0;
}


void CQueue::addInindex()
{
	pthread_mutex_lock(&mIndexlock);
	mInindex = LOOP_ADD(mInindex,maxFrames);
	pthread_mutex_unlock(&mIndexlock);
}
void CQueue::addOutindex()
{
	pthread_mutex_lock(&mIndexlock);
	mOutindex = LOOP_ADD(mOutindex,maxFrames);
	pthread_mutex_unlock(&mIndexlock);
}

int CQueue::IncreaseEffectives()
{
	pthread_mutex_lock(&mIndexlock);
	int ret = mEffectives;
	mEffectives += 1;
	if(mEffectives > maxFrames)
	{	
		mEffectives = maxFrames;
	}
	pthread_mutex_unlock(&mIndexlock);
	return ret;
}

int CQueue::reduceEffectives()
{
	pthread_mutex_lock(&mIndexlock);
	
	int ret = mEffectives;
	mEffectives -= 1;
	if(mEffectives<0)
	{	
		mEffectives = 0;
	}
	pthread_mutex_unlock(&mIndexlock);
	return ret;
}



#if 1
#include <unistd.h>
FILE *fp_in  = NULL;
FILE *fp_out = NULL;
CQueue *pQueue =NULL;
void *product(void *pram)
{
	if(NULL == pQueue)
	{
		printf("erro , pQueue NULL!\n");
		return 0;
	}
	printf("start produce !\n");
	unsigned char* buf = (unsigned char*)malloc(1920);
	while(1)
	{
		if(fread(buf,1,1920,fp_in)<1920)
		{
			break;
		}
		printf("read %#x %#x \n",buf[0],buf[1]);
	        pQueue->push(buf,1920);
		usleep(10000*50);
	}
	free(buf);
};
int main()
{

	pQueue = new CQueue(20);
	fp_in = fopen("queue_test","r");
	fp_out = fopen("queue_out","w+");
	if(NULL == fp_in || NULL == fp_out )
	{
		printf("erro to open file\n");
		return -1;
	}

	pthread_t pid;
	if(pthread_create(&pid,NULL,product,NULL) < 0 )
	{
		printf("pthread create erro \n");
		return -1;
	}

	unsigned char *data =NULL;
	int packlen = 0;
	int index =-1;
	while(1)
	{
		do{
		index = pQueue->getbuffer(&data,&packlen);
		}while(index<0);
		
		printf("get data: %#x %#x %d \n\n\n",data[0],data[1],packlen);
		fwrite(data,1,packlen,fp_out);

		pQueue->releasebuffer(index);
	}
}
#endif

用命令 #g++ queue.cpp -lpthread 編譯。

log可視化：

#log.py canok 
import matplotlib.pyplot as plt


#print(int('123')-int('3')) #just for test
#file = open("./JM_TEK")
file = open("./live555_RECV")
#file = open("./UDP_CLIENT")
#file = open("./SER_AUDO_QUEUE")
start_lin =10
show_num = 600
num =show_num
index=0
in_values=[]
out_values=[]
diff_values=[]
lin_index =-1


in_num =0
out_num =0
for line in file:
    lin_index = lin_index +1
    if lin_index < start_lin:
        continue
    flag = line.split('mEffectives')
    #print(flag[0])
    if flag[0] == 'IN>>>':
        data_list = line.split(' ')
        time_list = data_list[1].split(':')
        usec = time_list[3].split(']')
        in_values.append(usec[0])
        in_num = in_num+1
        #print("in_num>>:",in_num,usec[0])
        #num-- #python不支持 i++ ++i等自增自減
        num = num-1
        index = index +1
        if num < 0:
            break
    elif flag[0] == 'out<<<':
        data_list = line.split(' ')
        time_list = data_list[1].split(':')
        usec = time_list[3].split(']')
        out_num = out_num+1
        out_values.append(usec[0])
        #print("out_num<<::",out_num,usec[0])

i=0
dif_num=(in_num if(in_num<out_num) else out_num)
print("dif_num::",dif_num)
while i < dif_num:
    diff = int(out_values[i])-int(in_values[i])
    #print(i,"diff:",diff)
    if(diff>0):
        diff_values.append(diff)
    i = i+1

file.close()

def uniform(ix_values,iy_values):
    #用來均勻分佈y軸，0.1ms, 一個撇腳的實現方式
    j=0
    while j < 1000:
        ix_values.append(1)
        iy_values.append(j*1000)
        j = j+1

def show_list(iy_values,color):
    x_in_values=[]
    y_in_values=[]
    #必須爲每一組點都加上這個實現，才能達到均勻顯示的目的，一個撇腳的實現方式
    uniform(x_in_values,y_in_values)
    in_num = len(iy_values)
    j =0
    while j < in_num:
        x_in_values.append(j)
        y_in_values.append(iy_values[j])
        j = j+1
    #retrun x_in_values,y_in_values
    #不返回列表，直接在裏面上圖
    plt.scatter(x_in_values,y_in_values,edgecolor='none',s=2,c=color)


show_list(in_values,'red')
show_list(out_values,'yellow')
show_list(diff_values,'green')
plt.show()

修改其中的文件名 file = open("./live555_RECV")， 換成log文件名稱，執行 python3 log.py 即可。
在ubuntu中結果如圖：（這是一個live555接受網絡數據包的實測log,當然，網絡傳輸可以通過wireshark抓包，更方便分析。放大按鈕可以局部放大圖形。可改進，在圖中添加文字說明.）
紅色點縱座標表示單個數據輸入時間點，（微秒——百萬分之一秒，爲單位）
黃色點縱座標表示單個數據輸出時間點，
綠色點縱座標表示單個數據在隊列中停留的時間值（最左側豎線除外）