关于c++:FFmpeg连载8视频合并以及替换视频背景音乐实战

前言

通过后面的实战，咱们实现音视频解封装提取、音视频解码、音视频编码、音频重采样等的性能，明天咱们就联合之前所做的性能，
来做一个短视频 APP 中常见的性能：

1、提取多个 mp3 文件中的音频，从新编码为合并为 aac
2、提取 mp4 中的视频，从新编码合并为 h264
3、h264 与 aac 合并成新的 mp4 文件

因为咱们的目标是以实战为主，为了囊括之前所学的一些知识点，在这个实战中咱们不仅仅须要实现音视频解封装提取、音视频解码、音视频编码、音频重采样这些性能，
咱们还须要联合多线程同步等知识点做好生产者消费者队列缓冲管制。还蕴含例如类成员函数作为线程执行函数的应用等知识点。

大抵框架

这里要阐明一个常识就是如果音频如果须要合并的话要保障两个音频的采样率、采样格局以及通道数统一，所以须要重采样，为了测试，笔者把音频都重采样为 22050hz。

同时视频也一样，如果视频须要合并也须要保障两个视频的分辨率是一样的，这里笔者对立把尺寸转换为 720×1280。

笔者文笔不好，常常一句卧槽走天下，间接看图吧。。。

代码实现

原本笔者想谋求简略，心愿用一个 cpp 文件实现的，前面写着写着发现代码量有点多，所以就拆分成了三个 cpp 文件，上面是代码详情：

AudioHandle.cpp

/**
 * 音频解决
 * 解码音频，并且重采样为 22050，而后编码成 aac
 */
#ifndef TARGET_AUDIO_SAMPLE_RATE
// 采样率
#define TARGET_AUDIO_SAMPLE_RATE  22050
#endif

#include <iostream>
#include <vector>

extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavutil/channel_layout.h>
#include <libavutil/audio_fifo.h>
#include <libswresample/swresample.h>
};

class AudioHandle {

public:
    void handle_audio(std::vector<char *> mp3_paths,
                      std::function<void(const AVCodecContext *, AVPacket *, bool)> callback) {
        // 音频编码器相干
        const AVCodec *avCodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
        audio_encoder_context = avcodec_alloc_context3(avCodec);
        audio_encoder_context->sample_rate = TARGET_AUDIO_SAMPLE_RATE;
        // 默认的 aac 编码器输出的 PCM 格局为:AV_SAMPLE_FMT_FLTP
        audio_encoder_context->sample_fmt = AV_SAMPLE_FMT_FLTP;
        audio_encoder_context->channel_layout = AV_CH_LAYOUT_STEREO;
//        audio_encoder_context->bit_rate = 128 * 1024;
        audio_encoder_context->codec_type = AVMEDIA_TYPE_AUDIO;
        audio_encoder_context->channels = av_get_channel_layout_nb_channels(audio_encoder_context->channel_layout);
        audio_encoder_context->profile = FF_PROFILE_AAC_LOW;
        //ffmpeg 默认的 aac 是不带 adts，而 fdk_aac 默认带 adts，这里咱们强制不带
        audio_encoder_context->flags = AV_CODEC_FLAG_GLOBAL_HEADER;
        int ret = avcodec_open2(audio_encoder_context, avCodec, nullptr);
        if (ret < 0) {
            std::cout << "音频编码器关上失败" << std::endl;
            return;
        }

        // 初始化 audiofifo
        audiofifo = av_audio_fifo_alloc(audio_encoder_context->sample_fmt, audio_encoder_context->channels,
                                        audio_encoder_context->frame_size);

        AVFormatContext *avFormatContext = nullptr;
        AVCodecContext *decoder_context = nullptr;
        AVPacket *avPacket = av_packet_alloc();
        AVFrame *avFrame = av_frame_alloc();
        std::vector<AVPacket *> pack_vector = std::vector<AVPacket *>();
        while (!mp3_paths.empty()) {
            // 先开释旧的
            avcodec_free_context(&decoder_context);
            avformat_free_context(avFormatContext);
            const char *mp3 = mp3_paths.at(0);
            mp3_paths.erase(mp3_paths.cbegin());
            avFormatContext = avformat_alloc_context();
            ret = avformat_open_input(&avFormatContext, mp3, nullptr, nullptr);
            if (ret < 0) {
                std::cout << "音频文件关上失败" << std::endl;
                break;
            }
            int audio_index = av_find_best_stream(avFormatContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
            if (audio_index < 0) {for (int i = 0; i < avFormatContext->nb_streams; ++i) {if (AVMEDIA_TYPE_AUDIO == avFormatContext->streams[i]->codecpar->codec_type) {
                        audio_index = i;
                        std::cout << "找到音频流，audio_index:" << audio_index << std::endl;
                        break;
                    }
                }
                if (audio_index < 0) {
                    std::cout << "没有找到音频流" << std::endl;
                    break;
                }
            }
            const AVCodec *avCodec = avcodec_find_decoder(avFormatContext->streams[audio_index]->codecpar->codec_id);
            decoder_context = avcodec_alloc_context3(avCodec);
            avcodec_parameters_to_context(decoder_context, avFormatContext->streams[audio_index]->codecpar);
            ret = avcodec_open2(decoder_context, avCodec, nullptr);
            if (ret < 0) {
                std::cout << "音频解码器关上失败" << std::endl;
                break;
            }

            while (true) {ret = av_read_frame(avFormatContext, avPacket);
                if (ret < 0) {
                    std::cout << "音频包读取结束" << std::endl;
                    break;
                }
                if (avPacket->stream_index != audio_index) {av_packet_unref(avPacket);
                    continue;
                }
                ret = avcodec_send_packet(decoder_context, avPacket);
                if (ret < 0) {
                    std::cout << "音频包发送解码失败" << std::endl;
                    break;
                }
                while (true) {ret = avcodec_receive_frame(decoder_context, avFrame);
                    if (ret == AVERROR(EAGAIN)) {
                        std::cout << "音频包获取解码帧：EAGAIN" << std::endl;
                        break;
                    } else if (ret < 0) {
                        std::cout << "音频包获取解码帧：fail" << std::endl;
                        break;
                    } else {
                        std::cout << "从新编码音频" << std::endl;
                        // 先进行重采样
                        resample_audio(avFrame);
                        pack_vector.clear();
                        encode_audio(pack_vector, out_frame);
                        while (!pack_vector.empty()) {AVPacket *packet = pack_vector.at(0);
                            pack_vector.erase(pack_vector.cbegin());
                            // 回调
                            callback(audio_encoder_context, packet, false);
                        }
                    }
                }
                av_packet_unref(avPacket);
            }
        }
        avcodec_free_context(&decoder_context);
        avformat_free_context(avFormatContext);
        // 回调完结
        callback(audio_encoder_context, nullptr, true);
    }

private:
    // 视频编码器
    AVCodecContext *audio_encoder_context = nullptr;

    AVFrame *encode_frame = nullptr;
    AVAudioFifo *audiofifo = nullptr;
    int64_t cur_pts = 0;

    // 重采样相干
    SwrContext *swrContext = nullptr;
    AVFrame *out_frame = nullptr;
    int64_t max_dst_nb_samples;

    void init_out_frame(int64_t dst_nb_samples){av_frame_free(&out_frame);
        out_frame = av_frame_alloc();
        out_frame->sample_rate = TARGET_AUDIO_SAMPLE_RATE;
        out_frame->format = AV_SAMPLE_FMT_FLTP;
        out_frame->channel_layout = AV_CH_LAYOUT_STEREO;
        out_frame->nb_samples = dst_nb_samples;
        // 调配 buffer
        av_frame_get_buffer(out_frame,0);
        av_frame_make_writable(out_frame);
    }

    /**
     * 重采样
     * @param avFrame
     */
    void resample_audio(AVFrame *avFrame){if (nullptr == swrContext) {
            /**
             * 以下能够应用 swr_alloc、av_opt_set_channel_layout、av_opt_set_int、av_opt_set_sample_fmt
             * 等 API 设置，更加灵便
             */
            swrContext = swr_alloc_set_opts(nullptr, AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_FLTP, TARGET_AUDIO_SAMPLE_RATE,
                                            avFrame->channel_layout, static_cast<AVSampleFormat>(avFrame->format),
                                            avFrame->sample_rate, 0, nullptr);
            swr_init(swrContext);
        }
        // 进行音频重采样
        int src_nb_sample = avFrame->nb_samples;
        // 为了放弃从采样后 dst_nb_samples / dest_sample = src_nb_sample / src_sample_rate
        max_dst_nb_samples = av_rescale_rnd(src_nb_sample, TARGET_AUDIO_SAMPLE_RATE, avFrame->sample_rate, AV_ROUND_UP);
        // 从采样器中会缓存一部分，获取缓存的长度
        int64_t delay = swr_get_delay(swrContext, avFrame->sample_rate);
        // 相当于 a *b/c
        int64_t dst_nb_samples = av_rescale_rnd(delay + avFrame->nb_samples, TARGET_AUDIO_SAMPLE_RATE, avFrame->sample_rate,
                                                AV_ROUND_UP);
        if(nullptr == out_frame){init_out_frame(dst_nb_samples);
        }

        if (dst_nb_samples > max_dst_nb_samples) {
            // 须要重新分配 buffer
            std::cout << "须要重新分配 buffer" << std::endl;
            init_out_frame(dst_nb_samples);
            max_dst_nb_samples = dst_nb_samples;
        }
        // 重采样
        int ret = swr_convert(swrContext, out_frame->data, dst_nb_samples,
                              const_cast<const uint8_t **>(avFrame->data), avFrame->nb_samples);
        if(ret < 0){std::cout << "重采样失败" << std::endl;} else{
            // 每帧音频数据量的大小
            int data_size = av_get_bytes_per_sample(static_cast<AVSampleFormat>(out_frame->format));
            // 返回值才是真正的重采样点数
            out_frame->nb_samples = ret;
            std::cout << "重采样胜利：" << ret << "----dst_nb_samples:" << dst_nb_samples  << "---data_size:" << data_size << std::endl;
        }
    }

    void encode_audio(std::vector<AVPacket *> &pack_vector, AVFrame *avFrame) {int cache_size = av_audio_fifo_size(audiofifo);
        std::cout << "cache_size:" << cache_size << std::endl;
        av_audio_fifo_realloc(audiofifo, cache_size + avFrame->nb_samples);
        av_audio_fifo_write(audiofifo, reinterpret_cast<void **>(avFrame->data), avFrame->nb_samples);

        if (nullptr == encode_frame) {encode_frame = av_frame_alloc();
            encode_frame->nb_samples = audio_encoder_context->frame_size;
            encode_frame->sample_rate = audio_encoder_context->sample_rate;
            encode_frame->channel_layout = audio_encoder_context->channel_layout;
            encode_frame->channels = audio_encoder_context->channels;
            encode_frame->format = audio_encoder_context->sample_fmt;
            av_frame_get_buffer(encode_frame, 0);
        }

        av_frame_make_writable(encode_frame);
        // todo 如果是冲刷最初几帧数据，不够的能够填充静音  av_samples_set_silence
        while (av_audio_fifo_size(audiofifo) > audio_encoder_context->frame_size) {int ret = av_audio_fifo_read(audiofifo, reinterpret_cast<void **>(encode_frame->data),
                                         audio_encoder_context->frame_size);
            if (ret < 0) {
                std::cout << "audiofifo 读取数据失败" << std::endl;
                return;
            }
            // 批改 pts
            cur_pts += encode_frame->nb_samples;
            encode_frame->pts = cur_pts;
            ret = avcodec_send_frame(audio_encoder_context, encode_frame);
            if (ret < 0) {
                std::cout << "发送编码失败" << std::endl;
                return;
            }
            while (true) {AVPacket *out_pack = av_packet_alloc();
                ret = avcodec_receive_packet(audio_encoder_context, out_pack);
                if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
                    std::cout << "avcodec_receive_packet end:" << ret << std::endl;
                    break;
                } else if (ret < 0) {
                    std::cout << "avcodec_receive_packet fail:" << ret << std::endl;
                    return;
                } else {pack_vector.push_back(out_pack);
                }
            }
        }
    }
};

VideoHandle.cpp

/**
 * 视频解决
 * 解码视频，而后转换成 720x1080，而后编码成 h264
 */

#ifndef TARGET_VIDEO_WIDTH
#define TARGET_VIDEO_WIDTH  720
#define TARGET_VIDEO_HEIGHT  1280
#endif

#include <iostream>
#include <vector>

extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libswscale/swscale.h>
};

class VideoHandle {

public:

    void handle_video(std::vector<char *> mp4_paths,
                      std::function<void(const AVCodecContext *, AVPacket *, bool)> callback) {
        // 视频编码器相干
        const AVCodec *video_codec = avcodec_find_encoder(AV_CODEC_ID_H264);
        video_encoder_context = avcodec_alloc_context3(video_codec);
        video_encoder_context->pix_fmt = AV_PIX_FMT_YUV420P;
        video_encoder_context->width = TARGET_VIDEO_WIDTH;
        video_encoder_context->height = TARGET_VIDEO_HEIGHT;
        video_encoder_context->bit_rate = 2000 * 1024;
        video_encoder_context->gop_size = 10;
        video_encoder_context->time_base = {1, 25};
        video_encoder_context->framerate = {25, 1};
        // b 帧的数量
        video_encoder_context->max_b_frames = 1;
        // 设置 H264 的编码器参数为提早模式，进步编码品质，然而会造成编码速度降落
//        av_opt_set(video_encoder_context->priv_data, "preset", "slow", 0);
        int ret = avcodec_open2(video_encoder_context, video_codec, nullptr);
        if (ret < 0) {
            std::cout << "视频编码器关上失败" << std::endl;
            return;
        }
        AVFormatContext *avFormatContext = nullptr;
        AVCodecContext *decoder_context = nullptr;
        AVPacket *avPacket = av_packet_alloc();
        AVFrame *avFrame = av_frame_alloc();
        std::vector<AVPacket *> pack_vector = std::vector<AVPacket *>();
        // 后面视频的 pts 累计
        int64_t previous_pts = 0;
        // 但前视频最初的 pts
        int64_t last_pts = 0;
        while (!mp4_paths.empty()) {
            // 先开释旧的
            previous_pts += last_pts;
            avcodec_free_context(&decoder_context);
            avformat_free_context(avFormatContext);
            const char *mp4 = mp4_paths.at(0);
            mp4_paths.erase(mp4_paths.cbegin());
            avFormatContext = avformat_alloc_context();
            ret = avformat_open_input(&avFormatContext, mp4, nullptr, nullptr);
            if (ret < 0) {
                std::cout << "视频文件关上失败" << std::endl;
                break;
            }
            int video_index = av_find_best_stream(avFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
            if (video_index < 0) {
                std::cout << "没有找到视频流" << std::endl;
                break;
            }
            const AVCodec *avCodec = avcodec_find_decoder(avFormatContext->streams[video_index]->codecpar->codec_id);
            decoder_context = avcodec_alloc_context3(avCodec);
            avcodec_parameters_to_context(decoder_context, avFormatContext->streams[video_index]->codecpar);
            ret = avcodec_open2(decoder_context, avCodec, nullptr);
            if (ret < 0) {
                std::cout << "视频解码器关上失败" << std::endl;
                break;
            }

            while (true) {ret = av_read_frame(avFormatContext, avPacket);
                if (ret < 0) {
                    std::cout << "视频包读取结束" << std::endl;
                    break;
                }
                if(avPacket->stream_index != video_index){av_packet_unref(avPacket);
                    continue;
                }
                ret = avcodec_send_packet(decoder_context, avPacket);
                if (ret < 0) {char error[1024];
                    av_strerror(ret,error,1024);
                    std::cout << "视频包发送解码失败" << error << std::endl;
                    break;
                }
                while (true) {ret = avcodec_receive_frame(decoder_context, avFrame);
                    if (ret == AVERROR(EAGAIN)) {
                        std::cout << "视频包获取解码帧：EAGAIN" << std::endl;
                        break;
                    } else if (ret < 0) {std::cout << "视频包获取解码帧：fail" << std::endl;} else {
                        std::cout << "从新编码视频" << std::endl;
                        pack_vector.clear();
                        // 转换成对立的 pts
                        last_pts = av_rescale_q(avFrame->pts,avFormatContext->streams[video_index]->time_base,AV_TIME_BASE_Q);
                        avFrame->pts = previous_pts + last_pts;
                        // 尺寸转换
                        scale_yuv(avFrame);
                        // 从新编码
                        encode_video(pack_vector,out_frame);
                        while (!pack_vector.empty()){AVPacket *packet = pack_vector.at(0);
                            // 回调
                            callback(video_encoder_context,packet, false);
                            pack_vector.erase(pack_vector.cbegin());
                        }
                    }
                }
                av_packet_unref(avPacket);
            }
        }
        avcodec_free_context(&decoder_context);
        avformat_free_context(avFormatContext);
        // 回调完结
        callback(video_encoder_context, nullptr, true);
    }

    void scale_yuv(AVFrame *frame){
        swsContext = sws_getCachedContext(swsContext,frame->width,frame->height,AV_PIX_FMT_YUV420P,TARGET_VIDEO_WIDTH,TARGET_VIDEO_HEIGHT,AV_PIX_FMT_YUV420P,
                             SWS_BILINEAR,
                             nullptr, nullptr, nullptr);

        if(nullptr == out_frame){out_frame = av_frame_alloc();
            out_frame->format = AV_PIX_FMT_YUV420P;
            out_frame->width = TARGET_VIDEO_WIDTH;
            out_frame->height = TARGET_VIDEO_HEIGHT;
            av_frame_get_buffer(out_frame,0);
        }
        // 转换
        int ret = sws_scale(swsContext,frame->data,frame->linesize,0,frame->height,out_frame->data,out_frame->linesize);
        // pts
        std::cout << "frame->pts:" << frame->pts << std::endl;
        out_frame->pts = frame->pts;
        if(ret < 0){
            std::cout << "图像缩放失败" << std::endl;
            return;
        }
    }

    void encode_video(std::vector<AVPacket *>& pack_vector, AVFrame *frame) {int ret = avcodec_send_frame(video_encoder_context, frame);
        if (ret < 0) {
            std::cout << "视频发送编码失败" << std::endl;
            return;
        }
        while (true) {AVPacket *packet = av_packet_alloc();
            ret = avcodec_receive_packet(video_encoder_context,packet);
            if(ret == AVERROR(EAGAIN)){
                std::cout << "视频编码：EAGAIN" << std::endl;
                break;
            } else if(ret < 0){
                std::cout << "视频编码：fail" << std::endl;
                break;
            } else{pack_vector.push_back(packet);
            }
        }
    }

    // 视频编码器
    AVCodecContext *video_encoder_context = nullptr;

    // 视频转换专用
    SwsContext *swsContext = nullptr;
    AVFrame *out_frame = nullptr;

};

ComplexMuxerCore.cpp

/**
 * 将视频和音频合并成 mp4 输入文件
 */

#ifndef MAX_QUEUE_SIZE
// 队列缓存最大值
#define MAX_QUEUE_SIZE 6
#endif

#include <iostream>
#include <vector>
#include <thread>
#include <AudioHandle.cpp>
#include <VideoHandle.cpp>

extern "C" {#include <libavformat/avformat.h>}

class ComplexMuxerCore {

public:

    ComplexMuxerCore() : audio_queue(new std::vector<AVPacket *>()), video_queue(new std::vector<AVPacket *>()) { }

    /**
     * 这个是主函数，也就是说在 main 中调用这个函数即可
     * @param mp3_paths
     * @param mp4_paths
     * @param mp4_out
     */
    void muxer_media(const std::vector<char *> mp3_paths, const std::vector<char *> mp4_paths,
                     const char *mp4_out) {audioHandle = new AudioHandle();
        auto a_call_back = std::bind(&ComplexMuxerCore::audio_callback, this, std::placeholders::_1,
                                     std::placeholders::_2, std::placeholders::_3);

        audio_thread = new std::thread(&AudioHandle::handle_audio, audioHandle, mp3_paths, a_call_back);

        videoHandle = new VideoHandle();
        auto v_call_back = std::bind(&ComplexMuxerCore::video_callback, this, std::placeholders::_1,
                                     std::placeholders::_2, std::placeholders::_3);
        video_thread = new std::thread(&VideoHandle::handle_video, videoHandle, mp4_paths, v_call_back);

        muxer_thread = new std::thread(&ComplexMuxerCore::muxer_out, this, mp4_out);

        muxer_thread->join();}

    ~ComplexMuxerCore() {// todo 开释资源}

private:
    VideoHandle *videoHandle = nullptr;
    AudioHandle *audioHandle = nullptr;

    // 音频解决线程
    std::thread *audio_thread = nullptr;
    // 视频解决线程
    std::thread *video_thread = nullptr;
    // 合并线程
    std::thread *muxer_thread = nullptr;
    // 音频队列
    std::vector<AVPacket *> *audio_queue = nullptr;
    // 视频队列
    std::vector<AVPacket *> *video_queue = nullptr;
    // 音频线程同步互斥量
    std::mutex audio_mutex;
    // 视频线程同步互斥量
    std::mutex video_mutex;
    // 合并线程同步互斥量
    std::mutex muxer_mutex;
    // 音频条件变量
    std::condition_variable audio_conditionVariable;
    // 视频条件变量
    std::condition_variable video_conditionVariable;
    // 合并条件变量
    std::condition_variable conditionVariable;
    // 输出音频是否处理完毕
    volatile bool is_audio_end;
    // 输出视频是否处理完毕
    volatile bool is_video_end;
    // 输入视频流的索引
    int out_video_stream_index = -1;
    // 输出视频流索引
    int out_audio_stream_index = -1;
    // 视频 pts
    double video_pts = 0;
    // 音频 pts
    double audio_pts = 0;

    AVFormatContext *out_format_context = nullptr;

    void muxer_out(const char *mp4_out) {out_format_context = avformat_alloc_context();
        const AVOutputFormat *avOutputFormat = av_guess_format(nullptr, mp4_out, nullptr);
        out_format_context->oformat = avOutputFormat;
        while (out_video_stream_index < 0 || out_audio_stream_index < 0) {
            std::cout << "视频流或音频流还没创立好，陷入期待" << std::endl;
            std::unique_lock<std::mutex> muxer_lock(muxer_mutex);
            conditionVariable.wait(muxer_lock);
        }
        int ret = avio_open(&out_format_context->pb, mp4_out, AVIO_FLAG_WRITE);
        if (ret < 0) {
            std::cout << "输入流关上失败" << std::endl;
            return;
        }
        ret = avformat_write_header(out_format_context, nullptr);
        if (ret < 0) {
            std::cout << "文件头写入失败" << std::endl;
            return;
        }
        while (!is_handle_end()) {
            std::cout << "muxer while" << std::endl;
            // 视频包的 pts 大于音频包或者视频包写完了则写音频包
            if((video_pts > audio_pts && !audio_queue->empty()) ||
                    (is_video_end && video_queue->empty())){
                // 写入音频包
                write_audio();} else {
                // 写入视频包
                write_video();}
        }

        std::cout << "开始写入文件头" << std::endl;
        ret = av_write_trailer(out_format_context);
        if (ret < 0) {std::cout << "文件尾写入失败" << std::endl;} else {std::cout << "合并实现" << std::endl;}
    }

    void write_audio(){while (audio_queue->empty() && !is_audio_end) {
            std::cout << "期待音频包生产" << std::endl;
            std::unique_lock<std::mutex> uniqueLock(audio_mutex);
            audio_conditionVariable.wait(uniqueLock);
        }

        if (!audio_queue->empty()) {
            // 锁住
            std::lock_guard<std::mutex> lockGuard(audio_mutex);
            AVPacket *pack = audio_queue->at(0);
            // pts 转换
//                    av_packet_rescale_ts(pack,pack->time_base,out_format_context->streams[out_audio_stream_index]->time_base);
            audio_pts = pack->pts * av_q2d(out_format_context->streams[out_audio_stream_index]->time_base);
            std::cout << "写入音频包  audio_pts:" << audio_pts << std::endl;
            av_write_frame(out_format_context, pack);
            av_packet_free(&pack);
            audio_queue->erase(audio_queue->cbegin());
        }
        // 唤醒
        audio_conditionVariable.notify_all();
        // 休眠一下，模仿生产比生产慢
        std::this_thread::sleep_for(std::chrono::milliseconds(10));
    }

    void write_video(){while (video_queue->empty() && !is_video_end) {
            std::cout << "期待视频包生产" << std::endl;
            std::unique_lock<std::mutex> uniqueLock(video_mutex);
            video_conditionVariable.wait(uniqueLock);
        }
        // 大括号括起来能够及时开释锁
        if (!video_queue->empty()) {
            // 加锁
            std::lock_guard<std::mutex> lockGuard(video_mutex);
            AVPacket *pack = video_queue->at(0);
            // 之前在 VideoHandle 中转换了对立的 pts，当初要转换回去
            av_packet_rescale_ts(pack,AV_TIME_BASE_Q,out_format_context->streams[out_video_stream_index]->time_base);
            video_pts = pack->pts * av_q2d(out_format_context->streams[out_video_stream_index]->time_base);
            std::cout << "写入视频包  video_pts:" << video_pts << std::endl;
            // pts 转换
//                    av_packet_rescale_ts(pack,pack->time_base,out_format_context->streams[out_video_stream_index]->time_base);
            av_write_frame(out_format_context, pack);
            av_packet_free(&pack);
            video_queue->erase(video_queue->cbegin());
        }
        // 唤醒
        video_conditionVariable.notify_all();
        // 休眠一下，模仿生产比生产慢
        std::this_thread::sleep_for(std::chrono::milliseconds(10));
    }

    void audio_callback(const AVCodecContext *codecContext, AVPacket *avPacket, bool is_end) {if (nullptr == out_format_context) {
            // 复用器还没初始化好
            std::cout << "复用器还没初始化" << std::endl;
            return;
        }
        if (out_audio_stream_index < 0) {
            // 加锁
            std::cout << "audio_callback" << std::endl;
            std::lock_guard<std::mutex> lockGuard(muxer_mutex);
            AVStream *audio_stream = avformat_new_stream(out_format_context, codecContext->codec);
            avcodec_parameters_from_context(audio_stream->codecpar, codecContext);
            out_audio_stream_index = audio_stream->index;
            // 唤醒
            conditionVariable.notify_all();}
        // 队列超了就阻塞在这里
        while (audio_queue->size() >= MAX_QUEUE_SIZE) {
            std::cout << "音频队列超出缓存，期待生产" << std::endl;
            std::unique_lock<std::mutex> uniqueLock(audio_mutex);
            audio_conditionVariable.wait(uniqueLock);
        }
        {if (nullptr != avPacket) {std::lock_guard<std::mutex> video_lock(audio_mutex);
                avPacket->stream_index = out_audio_stream_index;
                audio_queue->push_back(avPacket);
            }
        }
        is_audio_end = is_end;
        // 唤醒生产队列
        audio_conditionVariable.notify_all();}

    void video_callback(const AVCodecContext *codecContext, AVPacket *avPacket, bool is_end) {
        std::cout << "video_callback" << std::endl;
        // 队列超了就阻塞在这里
        if (nullptr == out_format_context) {
            // 复用器还没初始化好
            return;
        }
        if (out_video_stream_index < 0) {
            // 加锁
            std::cout << "video_callback" << std::endl;
            std::lock_guard<std::mutex> lockGuard(muxer_mutex);
            AVStream *video_stream = avformat_new_stream(out_format_context, codecContext->codec);
            avcodec_parameters_from_context(video_stream->codecpar, codecContext);
            out_video_stream_index = video_stream->index;
            std::cout << "创立视频输入流:" << out_video_stream_index << std::endl;
            // 唤醒
            conditionVariable.notify_all();}

        std::cout << "video_callback：" << video_queue->size() << std::endl;

        while (video_queue->size() >= MAX_QUEUE_SIZE) {
            std::cout << "视频队列超出缓存，期待生产" << std::endl;
            std::unique_lock<std::mutex> uniqueLock(video_mutex);
            video_conditionVariable.wait(uniqueLock);
        }

        {if (nullptr != avPacket) {std::lock_guard<std::mutex> video_lock(video_mutex);
                avPacket->stream_index = out_video_stream_index;
                video_queue->push_back(avPacket);
            }
        }
        is_video_end = is_end;
        // 唤醒生产队列
        video_conditionVariable.notify_all();}

    /**
     * 是否处理完毕
     * @return
     */
    bool is_handle_end() {
        return is_video_end &&
               is_audio_end &&
               (nullptr == audio_queue || audio_queue->empty()) &&
               (nullptr == video_queue || video_queue->empty());
    }

};

不得不说写技术博客是一个很耗时耗力的事件，原本为了更不便了解应该具体讲每个实现文件的细节问题的，因为工夫问题和文笔表白不大好还是算了，有趣味的私聊吧 …

思考

下面的代码在视频合并的过程中咱们通过 pts 的比拟穿插写入音频包或视频包。为什么须要这样做呢？假如在不思考多线程性能的前提下，能先写完音频再写入视频，或者能先写完视频再写入音频吗？

欢送大家沟通交流 …

系列举荐

FFmpeg 连载 1 - 开发环境搭建
FFmpeg 连载 2 - 拆散视频和音频
FFmpeg 连载 3 - 视频解码
FFmpeg 连载 4 - 音频解码
FFmpeg 连载 5 - 音视频编码
FFmpeg 连载 6 - 音频重采样
FFmpeg 连载 7 -mp3 转码 aac

关注我，一起提高，人生不止 coding！！！