ICode9

精准搜索请尝试: 精确搜索
首页 > 其他分享> 文章详细

【音视频】视频混流-avfilter(2-2)

2021-09-21 21:58:21  阅读:156  来源: 互联网

标签:混流 err int avfilter 音视频 FILTER ERROR dstFrame filterInCtxs


要想将摄像头采集的视频流和桌面采集的视频流混成一股流,需要再次借助ffmpeg的avfilter功能库。

技术简介

借助fmpeg的filter功能,将两股视频合成一股视频流,从而实现两股视频的画中画效果

使用模块(库)

使用ffmpeg的avfilter库

主要流程和代码

1、初始化视频混流器。最需要关注的是filterDesc,即"[in0]setpts=PTS-STARTPTS,scale=%dx%d[main];[in1]setpts=PTS-STARTPTS,scale=%dx%d[over];[main][over]overlay=x=W-w:y=H-h:format=0[out]",因为这个最关键,这句话的意思是将[in0]设置从0开始的时间戳、scale视频大小并输出为标签[main],同理,[in1]输出为标签[over],然后将[over]在[main]上面使用overlay设置左上角位置、输出format为yuv420p并输出标签[out]

int VideoMixer::init(const VIDEO_FILTER_CTX& outCtx, const VIDEO_FILTER_CTX* inCtx0, const VIDEO_FILTER_CTX* inCtx1)
{
	int err = ERROR_CODE_OK;

	if (m_inited) {
		return err;
	}

	if (inCtx0 == nullptr || inCtx1 == nullptr) {
		err = ERROR_CODE_PARAMS_ERROR;
		return err;
	}

	do {
		m_filterGraph = avfilter_graph_alloc();
		if (m_filterGraph == nullptr) {
			err = ERROR_CODE_FILTER_ALLOC_GRAPH_FAILED;
			break;
		}

		m_filterInCtxs = new VIDEO_FILTER_CTX[VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE];
		memcpy_s(&m_filterInCtxs[0], sizeof(VIDEO_FILTER_CTX), inCtx0, sizeof(VIDEO_FILTER_CTX));
		memcpy_s(&m_filterInCtxs[1], sizeof(VIDEO_FILTER_CTX), inCtx1, sizeof(VIDEO_FILTER_CTX));
		m_filterOutCtx = outCtx;

		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			m_filterInCtxs[i].filterInout = avfilter_inout_alloc();
		}
		m_filterOutCtx.filterInout = avfilter_inout_alloc();

		char filterInArgs[2][512] = { 0 };
		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			sprintf_s(filterInArgs[i], sizeof(filterInArgs[i]), "video_size=%dx%d:pix_fmt=%d:frame_rate=%d:time_base=%d/%d:pixel_aspect=%d/%d",
				m_filterInCtxs[i].width, m_filterInCtxs[i].height, m_filterInCtxs[i].pixelFmt, m_filterInCtxs[i].framerate,
				m_filterInCtxs[i].timebase.num, m_filterInCtxs[i].timebase.den, m_filterInCtxs[i].pixelAspect.num, m_filterInCtxs[i].pixelAspect.den);
		}

		int ret = 0;
		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			char filterName[4] = { 0 };
			sprintf_s(filterName, sizeof(filterName), "in%d", i);
			ret = avfilter_graph_create_filter(&m_filterInCtxs[i].filterCtx, avfilter_get_by_name("buffer"), filterName, filterInArgs[i], nullptr, m_filterGraph);
			if (ret < 0) {
				err = ERROR_CODE_FILTER_CREATE_FILTER_FAILED;
				break;
			}
		}
		if (err != ERROR_CODE_OK) {
			break;
		}

		ret = avfilter_graph_create_filter(&m_filterOutCtx.filterCtx, avfilter_get_by_name("buffersink"), "out", nullptr, nullptr, m_filterGraph);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_CREATE_FILTER_FAILED;
			break;
		}

		av_opt_set_bin(m_filterOutCtx.filterCtx, "pix_fmts", (uint8_t*)&m_filterOutCtx.pixelFmt, sizeof(m_filterOutCtx.pixelFmt), AV_OPT_SEARCH_CHILDREN);

		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			char filterName[4] = { 0 };
			sprintf_s(filterName, sizeof(filterName), "in%d", i);
			m_filterInCtxs[i].filterInout->name = av_strdup(filterName);
			m_filterInCtxs[i].filterInout->filter_ctx = m_filterInCtxs[i].filterCtx;
			m_filterInCtxs[i].filterInout->pad_idx = 0;
			if (i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE - 1) {
				m_filterInCtxs[i].filterInout->next = m_filterInCtxs[i + 1].filterInout;
			}
			else {
				m_filterInCtxs[i].filterInout->next = nullptr;
			}
		}

		m_filterOutCtx.filterInout->name = av_strdup("out");
		m_filterOutCtx.filterInout->filter_ctx = m_filterOutCtx.filterCtx;
		m_filterOutCtx.filterInout->pad_idx = 0;
		m_filterOutCtx.filterInout->next = nullptr;

		AVFilterInOut* filterInOutputs[VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE];
		for (int i = 0; i < VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE; i++) {
			filterInOutputs[i] = m_filterInCtxs[i].filterInout;
		}

		char filterDesc[256] = { 0 };
		int smallWidth, smallHeight;
		getSmallResolutionByLarge(m_filterOutCtx.width, m_filterOutCtx.height, smallWidth, smallHeight);
		sprintf_s(filterDesc, sizeof(filterDesc), "[in0]setpts=PTS-STARTPTS,scale=%dx%d[main];[in1]setpts=PTS-STARTPTS,scale=%dx%d[over];[main][over]overlay=x=W-w:y=H-h:format=0[out]",
			m_filterOutCtx.width, m_filterOutCtx.height, smallWidth, smallHeight);
		ret = avfilter_graph_parse_ptr(m_filterGraph, filterDesc, &m_filterOutCtx.filterInout, filterInOutputs, nullptr);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_PARSE_PTR_FAILED;
			break;
		}

		ret = avfilter_graph_config(m_filterGraph, nullptr);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_CONFIG_FAILED;
			break;
		}

		m_inited = true;
	} while (0);

	if (err != ERROR_CODE_OK) {
		LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] init mixer, error: %s", __FUNCTION__, HCMDR_GET_ERROR_DESC(err));
		cleanup();
	}

	return err;
}

2、启动视频filter线程

int VideoMixer::start()
{
	int err = ERROR_CODE_OK;

	if (!m_inited) {
		err = ERROR_CODE_UNINITIALIZED;
		return err;
	}

	if (m_running) {
		return err;
	}

	m_running = true;
	m_thread = std::thread(std::bind(&VideoMixer::mixProcess, this));

	return err;
}

视频混流线程函数,从buffersink队列中获取一帧传给下一层

void VideoMixer::mixProcess()
{
	AVFrame* frame = av_frame_alloc();

	while (m_running) {
		std::unique_lock<std::mutex> lock(m_mutex);

		while (m_running && !m_conditionFlag) {
			m_condition.wait_for(lock, std::chrono::milliseconds(VIDEO_MIXER_WAIT_FOR_FRAME_TIMEOUT));
		}

		while (m_running && m_conditionFlag) {
			int ret = av_buffersink_get_frame(m_filterOutCtx.filterCtx, frame);
			if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
				break;
			}

			if (ret < 0) {
				LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] get sink frame, error: %s", __FUNCTION__, ret);
				if (m_onVideoFilterError != nullptr) {
					m_onVideoFilterError(ret);
				}
				break;
			}

			if (m_onVideoFilterData != nullptr) {
				m_onVideoFilterData(frame);
			}

			av_frame_unref(frame);
		}

		m_conditionFlag = false;
	}

	av_frame_free(&frame);
}

3、视频流输入,根据index可以将对应视频输入源的视频帧加入buffersrc队列

int VideoMixer::addFrame(AVFrame* frame, int index)
{
	int err = ERROR_CODE_OK;

	if (frame == nullptr) {
		err = ERROR_CODE_PARAMS_ERROR;
		return err;
	}
	if (0 > index || index >= VIDEO_MIXER_FILTER_IN_CTX_MAX_SIZE) {
		err = ERROR_CODE_FILTER_INVALID_FILTER;
		return err;
	}

	std::unique_lock<std::mutex> lock(m_mutex);

	do {
		AVFilterContext* filterCtx = m_filterInCtxs[index].filterCtx;
		int ret = av_buffersrc_add_frame_flags(filterCtx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
		if (ret < 0) {
			err = ERROR_CODE_FILTER_ADD_FRAME_FAILED;
			break;
		}
	} while (0);

	if (err != ERROR_CODE_OK) {
		LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] add frame, error: %s", __FUNCTION__, HCMDR_GET_ERROR_DESC(err));
	}

	m_conditionFlag = true;
	m_condition.notify_all();

	return err;
}

4、停止视频混流

int VideoMixer::stop()
{
	int err = ERROR_CODE_OK;

	if (!m_inited) {
		err = ERROR_CODE_UNINITIALIZED;
		return err;
	}

	if (!m_running) {
		return err;
	}

	m_running = false;

	m_conditionFlag = true;
	m_condition.notify_all();

	if (m_thread.joinable()) {
		m_thread.join();
	}

	return err;
}

这样,两个视频合成一股视频了。

补充:需要在FfmpegMuxer类(可以关注文章《【音视频】保存同步的音视频文件(九)》)中需要添加一个处理VideoFilter的数据回调,如:

void FfmpegMuxer::onVideoFilterData(AVFrame* frame)
{
	if (frame == nullptr) {
		return;
	}
	if (!m_running || m_paused || m_videoStream == nullptr || m_videoStream->vEncoder == nullptr) {
		return;
	}

	int err = ERROR_CODE_OK;
	AVFrame* dstFrame = frame;
	uint8_t* dstData = nullptr;
	uint32_t dstDataLen = 0;
	if (dstFrame->format == AV_PIX_FMT_YUV420P) {
		int dstOffset = 0;
		dstDataLen = dstFrame->width * dstFrame->height * 3 / 2;
		dstData = new uint8_t[dstDataLen];
		for (int i = 0; i < dstFrame->height; i++) {
			memcpy_s(dstData + i * dstFrame->width, dstFrame->width, dstFrame->data[0] + i * dstFrame->linesize[0], dstFrame->width);
		}
		dstOffset += dstFrame->width * dstFrame->height;
		for (int i = 0; i < dstFrame->height / 2; i++) {
			memcpy_s(dstData + dstOffset + i * dstFrame->width / 2, dstFrame->width / 2, dstFrame->data[1] + i * dstFrame->linesize[1], dstFrame->width / 2);
		}
		dstOffset += dstFrame->width * dstFrame->height / 4;
		for (int i = 0; i < dstFrame->height / 2; i++) {
			memcpy_s(dstData + dstOffset + i * dstFrame->width / 2, dstFrame->width / 2, dstFrame->data[2] + i * dstFrame->linesize[2], dstFrame->width / 2);
		}
	}
	if (dstData != nullptr && dstDataLen > 0) {
		err = m_videoStream->vEncoder->addFrame(dstData, dstDataLen, dstFrame);
		if (m_previewed && m_onMuxData != nullptr) {
			m_onMuxData((AVPixelFormat)dstFrame->format, dstData, dstDataLen, 0, 0);
		}
		delete[] dstData;
	}
}

这里需要注意的是YUV数据的处理,我这里混流之后的视频格式是YUV420P,看起来很麻烦的yuv数据拷贝其实是必须的,因为YUV420P原始数据Y占据height*linesize[0],U占据(height/2)*linesize[1],V占据(height/2)*linesize[2],而目标数据Y占据height*width,U和V各占据(height/2)*(width/2)

总结:需要研究一下ffmpeg中filter的使用,另外需要研究YUV数据格式。

标签:混流,err,int,avfilter,音视频,FILTER,ERROR,dstFrame,filterInCtxs
来源: https://blog.csdn.net/hcmonk/article/details/120405529

本站声明: 1. iCode9 技术分享网(下文简称本站)提供的所有内容,仅供技术学习、探讨和分享;
2. 关于本站的所有留言、评论、转载及引用,纯属内容发起人的个人观点,与本站观点和立场无关;
3. 关于本站的所有言论和文字,纯属内容发起人的个人观点,与本站观点和立场无关;
4. 本站文章均是网友提供,不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属;如您发现该文章侵犯了您的权益,可联系我们第一时间进行删除;
5. 本站为非盈利性的个人网站,所有内容不会用来进行牟利,也不会利用任何形式的广告来间接获益,纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。

专注分享技术,共同学习,共同进步。侵权联系[81616952@qq.com]

Copyright (C)ICode9.com, All Rights Reserved.

ICode9版权所有