Я начал изучать ffmpeg несколько слабых назад. На данный момент я могу транскодировать любое видео в mp4 с помощью кодека h264 / AVC. Основная схема примерно такая:

-открытый ввод -demux -decode -encode -mux

Фактический код ниже:

#include 
#include 
extern "C" 
{

#ifndef __STDC_CONSTANT_MACROS
#undef main /* Prevents SDL from overriding main() */
#  define __STDC_CONSTANT_MACROS
#endif



#pragma comment (lib,"avcodec.lib")
#pragma comment (lib,"avformat.lib")
#pragma comment (lib,"swscale.lib")
#pragma comment(lib,"avutil.lib")

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

}


using namespace std;


void open_video(AVFormatContext*oc , AVCodec *codec, AVStream * st)
{
int ret;
AVCodecContext *c ;
c = st->codec;

/*open codec */

cout < "probably starts here" < endl;
ret = avcodec_open2(c,codec,NULL);
cout < "and ends here" < endl;

if ( ret < 0)
{
    cout < ("Could not open video codec") < endl;
}


}






/*This function will add a new stream to our file.
@param
oc -> Format context that the new stream will be added.
codec -> codec of the stream, this will be passed.
codec_id -> 
chWidth->
chHeight->
*/

AVStream * addStream(AVFormatContext * oc, AVCodec **codec, enum AVCodecID codec_id, int chWidth,        int chHeight, int fps)
{
AVCodecContext *c;
AVStream *st;

//find encoder of the stream, it passes this information to @codec, later on
//it will be used in encoding the video @ avcodec_encode_video2 in loop.
*codec = avcodec_find_encoder(AV_CODEC_ID_H264);

if ( (*codec) == NULL)
    cout < "ERROR CAN NOT FIND ENCODER! ERROR! ERROR! AVCODEC_FIND_ENCODER FAILED !!!1 """ < endl;

if(!(*codec))
    printf ("Could not find encoder for ' %s ' ", avcodec_get_name(codec_id));


//create a new stream with the found codec inside oc(AVFormatContext).
st = avformat_new_stream ( oc, *codec);

if (!st)
    cout < " Cannot allocate stream " < endl;

//Setting the stream id.
//Since, there can be other streams in this AVFormatContext,
//we should find the first non used index. And this is oc->nb_streams(number of streams) - 1
st ->id = oc ->nb_streams - 1;

c = st->codec;

//setting the stream's codec's properties.
c-> codec_id = codec_id;
c->bit_rate = 4000000;
c->width = chWidth;
c->height = chHeight;
c->time_base.den = fps;
    //fps;
c->time_base.num = 1;
c->gop_size = 12;
c->pix_fmt = AV_PIX_FMT_YUV420P;

if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
        /* just for testing, we also add B frames */
        c->max_b_frames = 2;
    }


if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
        /* Needed to avoid using macroblocks in which some coeffs overflow.
         * This does not happen with normal video, it just happens here as
         * the motion of the chroma plane does not match the luma plane. */
        c->mb_decision = 2;
    }

/* Some formats want stream headers to be separate. */
 if (oc->oformat->flags & AVFMT_GLOBALHEADER)
    c->flags |= CODEC_FLAG_GLOBAL_HEADER;

//returning our lovely new brand stream.
return st;


}

int changeResolution ( string source, int format )
{
//Data members
struct SwsContext   *sws_ctx = NULL;
AVFrame             *pFrame = NULL;
AVFrame             *outFrame = NULL;   
AVPacket            packet;
uint8_t             *buffer = NULL;
uint8_t endcode[] = { 0, 0, 1, 0xb7 };
AVDictionary        *optionsDict = NULL;
AVFormatContext     *pFormatCtx = NULL; 
AVFormatContext     *outputContext = NULL;
AVCodecContext      *pCodecCtx;
AVCodec             *pCodec ;
AVCodec             *codec;
AVCodec             *videoCodec;
AVOutputFormat      *fmt;
AVStream            *video_stream;
int                 changeWidth;
int                  changeHeight;
int                 frameFinished;
int                 numBytes;
int                 fps;


int lock = 0;

//Register all codecs & other important stuff. Vital!..
av_register_all();


//Selects the desired resolution.
if (format == 0)
{
    changeWidth = 320;
    changeHeight = 180;
}

else if (format == 1)
{
    changeWidth = 640;
    changeHeight = 480;

}
else if (format == 2)
{
    changeWidth = 960;
    changeHeight = 540;

}
else if (format == 3)
{
    changeWidth = 1024;
    changeHeight = 768;

}
else
{
    changeWidth = 1280;
    changeHeight = 720;
}


// Open video file
int aaa;
aaa = avformat_open_input(&pFormatCtx, source.c_str(), NULL, NULL) ;
if(aaa !=0)
{
    cout < " cannot open input file \n" < endl;
    cout < "aaa = " < aaa  < endl;
     return -1; // Couldn't open file   
}

// Retrieve stream information
if(av_find_stream_info(pFormatCtx)