opengl渲染yuv圖像
作者:史正
郵箱:[email protected]
如有錯誤還請及時指正
如果有錯誤的描述給您帶來不便還請見諒
如需交流請發送郵件,歡迎聯繫
- 我的csdn : https://blog.csdn.net/shizheng163
- 我的github : https://github.com/shizheng163
簡述
直接渲染Jpeg等編碼後的圖像(這裏使用的是Jpeg圖片)是特別慢的(當張圖片>40ms), 常用的做法是直接渲染yuv或者rgb的圖像。
這裏使用的是使用opengl渲染yuv的圖像, 當然也可以通過Qt的繪圖庫來渲染。不過opengl默認的渲染步驟是在gpu上執行的, 而Qt的繪圖庫是利於cpu進行計算,gpu進行渲染, 佔用cpu的資源比較多。
使用opengl渲染yuv圖像的時間單幀圖像爲(0-2]ms
渲染完成後使主線程等待40-渲染時間, 就會呈現出視頻的效果。
這一節的完整代碼:
這裏給出幾個ffmpeg將視頻分解爲jpeg然後再轉換爲yuv的命令(由於沒有找到視頻直接分解爲yuv圖片幀的辦法)
ffmpeg -i Suger.mp4 -b 3000k -ss 00:00:10 -t 10 SugerFrames/Frame_%04d.jpeg
ffmpeg -y -s 1920x1080 -i SugerFrames/yuv_0001.jpeg SugerYuvs/yuv_0001.yuv
由於沒有找到直接將多張jpeg圖片直接轉換爲多張yuv圖像的方法, 這裏寫了一個shell腳本來處理
for((i=1;i<=480;i++))
do
index=`printf "%04d" $i`
ffmpeg -y -s 1920x1080 -i SugerFrames/yuv_$index.jpeg SugerYuvs/yuv_$index.yuv
done
ffmpeg 查看yuv圖像或視頻的方法
ffplay.exe -video_size 1920x1080 yuv_0001.yuv
使用QOpenGLWidget進行yuv圖片渲染
除了可以選擇使用QPainter和標準的OpenGL渲染圖形,QOpenGLWidget類提供了在Qt應用程序中顯示OpenGL圖形的功能。
關於QOpenGLWidget
的介紹可以參考以下文章:
筆者對opengl基本上沒什麼瞭解, 渲染部分選自以下博客,感謝博主
另外glviewport
是opengl渲染的圖片顯示的位置, 座標原點爲左下角。
使用了opengl渲染之後,再使用QPainter渲染(當時只是想使用這個渲染默認的Jpeg圖標)就會導致opengl和QPainter渲染的效果無法使用。
解決方法還沒有找到, 所以乾脆就把默認圖片也轉成了yuv圖像。
.h文件
#ifndef VIDEOGLWIDGET_H
#define VIDEOGLWIDGET_H
#include <QWidget>
#include <QOpenGLWidget>
#include <QOpenGLFunctions>
#include <memory>
#include <mutex>
#include <QOpenGLShaderProgram>
#include <QOpenGLFunctions>
#include <QOpenGLTexture>
#include "fileutil.h"
class VideoGLWidget : public QOpenGLWidget, protected QOpenGLFunctions
{
QOBJECT_H
public:
VideoGLWidget(QWidget * pParent = 0);
virtual ~VideoGLWidget();
void PictureShow(fileutil::PictureFilePtr pPicture);
void DefaultPictureShow();
protected:
void initializeGL();
void resizeGL(int w, int h);
void paintGL();
private:
//繪圖是異步動作, 防止繪圖過程中,圖片被置爲NULL,所以需要加鎖
std::mutex m_mutexForShowYuvData;
bool m_bIsShowVideoIcon; //是否要顯示視頻默認圖標
fileutil::PictureFilePtr m_pDefaultPict;//默認圖片
fileutil::PictureFilePtr m_pYuvPictPtr;//當前需要顯示的Yuv數據
//顯示Opengl渲染後圖像位置
QRect m_drawRect;
//渲染yuv圖像變量
GLuint textureUniformY; //y紋理數據位置
GLuint textureUniformU; //u紋理數據位置
GLuint textureUniformV; //v紋理數據位置
GLuint id_y; //y紋理對象ID
GLuint id_u; //u紋理對象ID
GLuint id_v; //v紋理對象ID
QOpenGLTexture* m_pTextureY; //y紋理對象
QOpenGLTexture* m_pTextureU; //u紋理對象
QOpenGLTexture* m_pTextureV; //v紋理對象
QOpenGLShader * m_pVSHader; //頂點着色器程序對象
QOpenGLShader * m_pFSHader; //片段着色器對象
QOpenGLShaderProgram *m_pShaderProgram; //着色器程序容器
};
#endif // VIDEOGLWIDGET_H
PicturePtr結構如下
#include <string>
#include <stdint.h>
#include <memory>
struct FileRawData
{
FileRawData(){
m_pData = NULL;
}
FileRawData(FileRawData && filedata)
{
m_pData = filedata.m_pData;
m_uLen = filedata.m_uLen;
m_filename = filedata.m_filename;
filedata.m_pData = NULL;
filedata.m_uLen = 0;
filedata.m_filename.clear();
}
~FileRawData()
{
if(m_pData)
delete m_pData;
m_pData = NULL;
m_uLen = 0;
}
uint8_t * m_pData;
uint32_t m_uLen;
std::string m_filename;
};
typedef std::shared_ptr<FileRawData> FileRawDataPtr;
struct PictureFile: public FileRawData
{
enum PictureFormat
{
kFormatYuv,
kFormatJpeg
};
PictureFile(FileRawData & parent, uint32_t nWeight, uint32_t nHeight, PictureFormat pictFormat)
:FileRawData(std::move(parent))
,m_nWeight(nWeight)
,m_nHeight(nHeight)
,m_pictFormat(pictFormat)
{
}
uint32_t m_nWeight;
uint32_t m_nHeight;
PictureFormat m_pictFormat;
};
typedef std::shared_ptr<PictureFile> PictureFilePtr;
cpp文件
#include "videoglwidget.h"
#include <QPainter>
#include <QDebug>
#include <ctime>
#include <QOpenGLTexture>
#include <QOpenGLBuffer>
#include "logutil.h"
//opengl 渲染部分參考自Csdn博客:https://blog.csdn.net/su_vast/article/details/52214642
using namespace fileutil;
#define DEFAULT_ICO_VIEW_WIDTH 48
#define DEFAULT_ICO_VIEW_HEIGHT 48
#define DEFAULT_ICO_PIX_HEIGHT 512
#define DEFAULT_ICO_PIX_WIDTH 512
VideoGLWidget::VideoGLWidget(QWidget *pParent)
:QOpenGLWidget(pParent)
{
textureUniformY = 0;
textureUniformU = 0;
textureUniformV = 0;
id_y = 0;
id_u = 0;
id_v = 0;
m_pYuvPictPtr = NULL;
m_pVSHader = NULL;
m_pFSHader = NULL;
m_pShaderProgram = NULL;
m_pTextureY = NULL;
m_pTextureU = NULL;
m_pTextureV = NULL;
//默認顯示渲染位置爲默認視頻圖標位置
m_drawRect = QRect((width() - DEFAULT_ICO_VIEW_WIDTH)/2, height()/2, DEFAULT_ICO_VIEW_WIDTH, DEFAULT_ICO_VIEW_HEIGHT);
this->DefaultPictureShow();
}
VideoGLWidget::~VideoGLWidget()
{
if(m_pVSHader)
{
delete m_pVSHader;
m_pVSHader = NULL;
}
if(m_pFSHader)
{
delete m_pFSHader;
m_pFSHader = NULL;
}
if(m_pShaderProgram)
{
delete m_pShaderProgram;
m_pShaderProgram = NULL;
}
if(m_pTextureY)
{
delete m_pTextureY;
m_pTextureY = NULL;
}
if(m_pTextureU)
{
delete m_pTextureU;
m_pTextureU = NULL;
}
if(m_pTextureV)
{
delete m_pTextureV;
m_pTextureV = NULL;
}
}
void VideoGLWidget::PictureShow(PictureFilePtr pPicture)
{
std::unique_lock<std::mutex> locker(m_mutexForShowYuvData);
m_pYuvPictPtr = pPicture;
m_drawRect = QRect(0, 0, width(), height());
this->update();
}
void VideoGLWidget::DefaultPictureShow()
{
std::unique_lock<std::mutex> locker(m_mutexForShowYuvData);
if(!m_pDefaultPict)
{
FileRawDataPtr pFileData = fileutil::ReadFileRawData("../../images/video.yuv");
m_pDefaultPict = PictureFilePtr(new PictureFile(*pFileData.get(), DEFAULT_ICO_PIX_WIDTH, DEFAULT_ICO_PIX_HEIGHT, PictureFile::kFormatYuv));
}
m_pYuvPictPtr = m_pDefaultPict;
m_drawRect = QRect((width() - DEFAULT_ICO_VIEW_WIDTH)/2, (height() - DEFAULT_ICO_VIEW_HEIGHT) /2, DEFAULT_ICO_VIEW_WIDTH, DEFAULT_ICO_VIEW_HEIGHT);
this->update();
}
void VideoGLWidget::initializeGL()
{
initializeOpenGLFunctions();
glEnable(GL_DEPTH_TEST);
//現代opengl渲染管線依賴着色器來處理傳入的數據
//着色器:就是使用openGL着色語言(OpenGL Shading Language, GLSL)編寫的一個小函數,
//GLSL是構成所有OpenGL着色器的語言,具體的GLSL語言的語法需要讀者查找相關資料
//初始化頂點着色器 對象
m_pVSHader = new QOpenGLShader(QOpenGLShader::Vertex, this);
const char *vsrc ="\
attribute vec4 vertexIn; \
attribute vec2 textureIn; \
varying vec2 textureOut; \
void main(void) \
{ \
gl_Position = vertexIn; \
textureOut = textureIn; \
}";
m_pVSHader->compileSourceCode(vsrc);
//初始化片段着色器 功能gpu中yuv轉換成rgb
m_pFSHader = new QOpenGLShader(QOpenGLShader::Fragment, this);
//片段着色器源碼
const char *fsrc = "varying vec2 textureOut; \
uniform sampler2D tex_y; \
uniform sampler2D tex_u; \
uniform sampler2D tex_v; \
void main(void) \
{ \
vec3 yuv; \
vec3 rgb; \
yuv.x = texture2D(tex_y, textureOut).r; \
yuv.y = texture2D(tex_u, textureOut).r - 0.5; \
yuv.z = texture2D(tex_v, textureOut).r - 0.5; \
rgb = mat3( 1, 1, 1, \
0, -0.39465, 2.03211, \
1.13983, -0.58060, 0) * yuv; \
gl_FragColor = vec4(rgb, 1); \
}";
m_pFSHader->compileSourceCode(fsrc);
#define PROGRAM_VERTEX_ATTRIBUTE 0
#define PROGRAM_TEXCOORD_ATTRIBUTE 1
#define ATTRIB_VERTEX 3
#define ATTRIB_TEXTURE 4
//創建着色器程序容器
m_pShaderProgram = new QOpenGLShaderProgram;
//將片段着色器添加到程序容器
m_pShaderProgram->addShader(m_pFSHader);
//將頂點着色器添加到程序容器
m_pShaderProgram->addShader(m_pVSHader);
//綁定屬性vertexIn到指定位置ATTRIB_VERTEX,該屬性在頂點着色源碼其中有聲明
m_pShaderProgram->bindAttributeLocation("vertexIn", ATTRIB_VERTEX);
//綁定屬性textureIn到指定位置ATTRIB_TEXTURE,該屬性在頂點着色源碼其中有聲明
m_pShaderProgram->bindAttributeLocation("textureIn", ATTRIB_TEXTURE);
//鏈接所有所有添入到的着色器程序
m_pShaderProgram->link();
//激活所有鏈接
m_pShaderProgram->bind();
//讀取着色器中的數據變量tex_y, tex_u, tex_v的位置,這些變量的聲明可以在
//片段着色器源碼中可以看到
textureUniformY = m_pShaderProgram->uniformLocation("tex_y");
textureUniformU = m_pShaderProgram->uniformLocation("tex_u");
textureUniformV = m_pShaderProgram->uniformLocation("tex_v");
// 頂點矩陣
static const GLfloat vertexVertices[] = {
-1.0f, -1.0f,
1.0f, -1.0f,
-1.0f, 1.0f,
1.0f, 1.0f,
};
//紋理矩陣
static const GLfloat textureVertices[] = {
0.0f, 1.0f,
1.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f,
};
//設置屬性ATTRIB_VERTEX的頂點矩陣值以及格式
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, vertexVertices);
//設置屬性ATTRIB_TEXTURE的紋理矩陣值以及格式
glVertexAttribPointer(ATTRIB_TEXTURE, 2, GL_FLOAT, 0, 0, textureVertices);
//啓用ATTRIB_VERTEX屬性的數據,默認是關閉的
glEnableVertexAttribArray(ATTRIB_VERTEX);
//啓用ATTRIB_TEXTURE屬性的數據,默認是關閉的
glEnableVertexAttribArray(ATTRIB_TEXTURE);
//分別創建y,u,v紋理對象
m_pTextureY = new QOpenGLTexture(QOpenGLTexture::Target2D);
m_pTextureU = new QOpenGLTexture(QOpenGLTexture::Target2D);
m_pTextureV = new QOpenGLTexture(QOpenGLTexture::Target2D);
m_pTextureY->create();
m_pTextureU->create();
m_pTextureV->create();
//獲取返回y分量的紋理索引值
id_y = m_pTextureY->textureId();
//獲取返回u分量的紋理索引值
id_u = m_pTextureU->textureId();
//獲取返回v分量的紋理索引值
id_v = m_pTextureV->textureId();
// glClearColor(0.3,0.3,0.3,0.0);//設置背景色
}
void VideoGLWidget::resizeGL(int w, int h)
{
if(h == 0)// 防止被零除
{
h = 1;// 將高設爲1
}
glViewport(0, 0, w, h);
}
void VideoGLWidget::paintGL()
{
std::unique_lock<std::mutex> locker(m_mutexForShowYuvData);
if(m_pYuvPictPtr)
{
// clock_t start = clock();
uint32_t weight = m_pYuvPictPtr->m_nWeight;
uint32_t height = m_pYuvPictPtr->m_nHeight;
glViewport(m_drawRect.x(), m_drawRect.y(), m_drawRect.width(), m_drawRect.height());
//加載y數據紋理
//激活紋理單元GL_TEXTURE0
glActiveTexture(GL_TEXTURE0);
//使用來自y數據生成紋理
glBindTexture(GL_TEXTURE_2D, id_y);
//使用內存中m_pBufYuv420p數據創建真正的y數據紋理
glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, weight, height, 0, GL_RED, GL_UNSIGNED_BYTE, m_pYuvPictPtr->m_pData);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//加載u數據紋理
glActiveTexture(GL_TEXTURE1);//激活紋理單元GL_TEXTURE1
glBindTexture(GL_TEXTURE_2D, id_u);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, weight/2, height/2, 0, GL_RED, GL_UNSIGNED_BYTE, (char*)m_pYuvPictPtr->m_pData+weight*height);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//加載v數據紋理
glActiveTexture(GL_TEXTURE2);//激活紋理單元GL_TEXTURE2
glBindTexture(GL_TEXTURE_2D, id_v);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, weight/2, height/2, 0, GL_RED, GL_UNSIGNED_BYTE, (char*)m_pYuvPictPtr->m_pData+weight*height*5/4);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//指定y紋理要使用新值 只能用0,1,2等表示紋理單元的索引,這是opengl不人性化的地方
//0對應紋理單元GL_TEXTURE0 1對應紋理單元GL_TEXTURE1 2對應紋理的單元
glUniform1i(textureUniformY, 0);
//指定u紋理要使用新值
glUniform1i(textureUniformU, 1);
//指定v紋理要使用新值
glUniform1i(textureUniformV, 2);
//使用頂點數組方式繪製圖形
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// clock_t end = clock();
}
}