Showing preview only (380K chars total). Download the full file or copy to clipboard to get everything.
Repository: leandromoreira/ffmpeg-libav-tutorial
Branch: master
Commit: 68b5a5fb4202
Files: 21
Total size: 368.5 KB
Directory structure:
gitextract_utsib4k_/
├── .github/
│ └── workflows/
│ └── docker-image.yml
├── .gitignore
├── 0_hello_world.c
├── 2_remuxing.c
├── 3_transcoding.c
├── CMakeLists.txt
├── Dockerfile
├── LICENSE
├── Makefile
├── README-cn.md
├── README-es.md
├── README-ko.md
├── README-pt.md
├── README-ru.md
├── README-vn.md
├── README.md
├── build/
│ └── .gitignore
├── fetch_bbb_video.sh
├── remuxed_small_bunny_1080p_60fps.ts
├── video_debugging.c
└── video_debugging.h
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/workflows/docker-image.yml
================================================
name: Docker Image CI
on:
push:
branches: [ "master" ]
pull_request:
branches: [ "master" ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Run hello example
run: make make_hello
- name: Run remuxing
run: make make_remuxing
- name: Run transcoding
run: make make_transcoding
================================================
FILE: .gitignore
================================================
*pgm
build/*
bunny_1080p_60fps.mp4
bunny_1s_gop.mp4
bunny_1s_gop.mp4.ts
bunny_1s_gop.mp4.webm
.vscode
.clangd
compile_commands.json
================================================
FILE: 0_hello_world.c
================================================
/*
* http://ffmpeg.org/doxygen/trunk/index.html
*
* Main components
*
* Format (Container) - a wrapper, providing sync, metadata and muxing for the streams.
* Stream - a continuous stream (audio or video) of data over time.
* Codec - defines how data are enCOded (from Frame to Packet)
* and DECoded (from Packet to Frame).
* Packet - are the data (kind of slices of the stream data) to be decoded as raw frames.
* Frame - a decoded raw frame (to be encoded or filtered).
*/
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
// print out the steps and errors
static void logging(const char *fmt, ...);
// decode packets into frames
static int decode_packet(AVPacket *pPacket, AVCodecContext *pCodecContext, AVFrame *pFrame);
// save a frame into a .pgm file
static void save_gray_frame(unsigned char *buf, int wrap, int xsize, int ysize, char *filename);
int main(int argc, const char *argv[])
{
if (argc < 2) {
printf("You need to specify a media file.\n");
return -1;
}
logging("initializing all the containers, codecs and protocols.");
// AVFormatContext holds the header information from the format (Container)
// Allocating memory for this component
// http://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
AVFormatContext *pFormatContext = avformat_alloc_context();
if (!pFormatContext) {
logging("ERROR could not allocate memory for Format Context");
return -1;
}
logging("opening the input file (%s) and loading format (container) header", argv[1]);
// Open the file and read its header. The codecs are not opened.
// The function arguments are:
// AVFormatContext (the component we allocated memory for),
// url (filename),
// AVInputFormat (if you pass NULL it'll do the auto detect)
// and AVDictionary (which are options to the demuxer)
// http://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga31d601155e9035d5b0e7efedc894ee49
if (avformat_open_input(&pFormatContext, argv[1], NULL, NULL) != 0) {
logging("ERROR could not open the file");
return -1;
}
// now we have access to some information about our file
// since we read its header we can say what format (container) it's
// and some other information related to the format itself.
logging("format %s, duration %lld us, bit_rate %lld", pFormatContext->iformat->name, pFormatContext->duration, pFormatContext->bit_rate);
logging("finding stream info from format");
// read Packets from the Format to get stream information
// this function populates pFormatContext->streams
// (of size equals to pFormatContext->nb_streams)
// the arguments are:
// the AVFormatContext
// and options contains options for codec corresponding to i-th stream.
// On return each dictionary will be filled with options that were not found.
// https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#gad42172e27cddafb81096939783b157bb
if (avformat_find_stream_info(pFormatContext, NULL) < 0) {
logging("ERROR could not get the stream info");
return -1;
}
// the component that knows how to enCOde and DECode the stream
// it's the codec (audio or video)
// http://ffmpeg.org/doxygen/trunk/structAVCodec.html
AVCodec *pCodec = NULL;
// this component describes the properties of a codec used by the stream i
// https://ffmpeg.org/doxygen/trunk/structAVCodecParameters.html
AVCodecParameters *pCodecParameters = NULL;
int video_stream_index = -1;
// loop though all the streams and print its main information
for (int i = 0; i < pFormatContext->nb_streams; i++)
{
AVCodecParameters *pLocalCodecParameters = NULL;
pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
logging("AVStream->time_base before open coded %d/%d", pFormatContext->streams[i]->time_base.num, pFormatContext->streams[i]->time_base.den);
logging("AVStream->r_frame_rate before open coded %d/%d", pFormatContext->streams[i]->r_frame_rate.num, pFormatContext->streams[i]->r_frame_rate.den);
logging("AVStream->start_time %" PRId64, pFormatContext->streams[i]->start_time);
logging("AVStream->duration %" PRId64, pFormatContext->streams[i]->duration);
logging("finding the proper decoder (CODEC)");
AVCodec *pLocalCodec = NULL;
// finds the registered decoder for a codec ID
// https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga19a0ca553277f019dd5b0fec6e1f9dca
pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);
if (pLocalCodec==NULL) {
logging("ERROR unsupported codec!");
// In this example if the codec is not found we just skip it
continue;
}
// when the stream is a video we store its index, codec parameters and codec
if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO) {
if (video_stream_index == -1) {
video_stream_index = i;
pCodec = pLocalCodec;
pCodecParameters = pLocalCodecParameters;
}
logging("Video Codec: resolution %d x %d", pLocalCodecParameters->width, pLocalCodecParameters->height);
} else if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_AUDIO) {
logging("Audio Codec: %d channels, sample rate %d", pLocalCodecParameters->channels, pLocalCodecParameters->sample_rate);
}
// print its name, id and bitrate
logging("\tCodec %s ID %d bit_rate %lld", pLocalCodec->name, pLocalCodec->id, pLocalCodecParameters->bit_rate);
}
if (video_stream_index == -1) {
logging("File %s does not contain a video stream!", argv[1]);
return -1;
}
// https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
if (!pCodecContext)
{
logging("failed to allocated memory for AVCodecContext");
return -1;
}
// Fill the codec context based on the values from the supplied codec parameters
// https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#gac7b282f51540ca7a99416a3ba6ee0d16
if (avcodec_parameters_to_context(pCodecContext, pCodecParameters) < 0)
{
logging("failed to copy codec params to codec context");
return -1;
}
// Initialize the AVCodecContext to use the given AVCodec.
// https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga11f785a188d7d9df71621001465b0f1d
if (avcodec_open2(pCodecContext, pCodec, NULL) < 0)
{
logging("failed to open codec through avcodec_open2");
return -1;
}
// https://ffmpeg.org/doxygen/trunk/structAVFrame.html
AVFrame *pFrame = av_frame_alloc();
if (!pFrame)
{
logging("failed to allocate memory for AVFrame");
return -1;
}
// https://ffmpeg.org/doxygen/trunk/structAVPacket.html
AVPacket *pPacket = av_packet_alloc();
if (!pPacket)
{
logging("failed to allocate memory for AVPacket");
return -1;
}
int response = 0;
int how_many_packets_to_process = 8;
// fill the Packet with data from the Stream
// https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga4fdb3084415a82e3810de6ee60e46a61
while (av_read_frame(pFormatContext, pPacket) >= 0)
{
// if it's the video stream
if (pPacket->stream_index == video_stream_index) {
logging("AVPacket->pts %" PRId64, pPacket->pts);
response = decode_packet(pPacket, pCodecContext, pFrame);
if (response < 0)
break;
// stop it, otherwise we'll be saving hundreds of frames
if (--how_many_packets_to_process <= 0) break;
}
// https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html#ga63d5a489b419bd5d45cfd09091cbcbc2
av_packet_unref(pPacket);
}
logging("releasing all the resources");
avformat_close_input(&pFormatContext);
av_packet_free(&pPacket);
av_frame_free(&pFrame);
avcodec_free_context(&pCodecContext);
return 0;
}
static void logging(const char *fmt, ...)
{
va_list args;
fprintf( stderr, "LOG: " );
va_start( args, fmt );
vfprintf( stderr, fmt, args );
va_end( args );
fprintf( stderr, "\n" );
}
static int decode_packet(AVPacket *pPacket, AVCodecContext *pCodecContext, AVFrame *pFrame)
{
// Supply raw packet data as input to a decoder
// https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga58bc4bf1e0ac59e27362597e467efff3
int response = avcodec_send_packet(pCodecContext, pPacket);
if (response < 0) {
logging("Error while sending a packet to the decoder: %s", av_err2str(response));
return response;
}
while (response >= 0)
{
// Return decoded output data (into a frame) from a decoder
// https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga11e6542c4e66d3028668788a1a74217c
response = avcodec_receive_frame(pCodecContext, pFrame);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
logging("Error while receiving a frame from the decoder: %s", av_err2str(response));
return response;
}
if (response >= 0) {
logging(
"Frame %d (type=%c, size=%d bytes, format=%d) pts %d key_frame %d [DTS %d]",
pCodecContext->frame_number,
av_get_picture_type_char(pFrame->pict_type),
pFrame->pkt_size,
pFrame->format,
pFrame->pts,
pFrame->key_frame,
pFrame->coded_picture_number
);
char frame_filename[1024];
snprintf(frame_filename, sizeof(frame_filename), "%s-%d.pgm", "frame", pCodecContext->frame_number);
// Check if the frame is a planar YUV 4:2:0, 12bpp
// That is the format of the provided .mp4 file
// RGB formats will definitely not give a gray image
// Other YUV image may do so, but untested, so give a warning
if (pFrame->format != AV_PIX_FMT_YUV420P)
{
logging("Warning: the generated file may not be a grayscale image, but could e.g. be just the R component if the video format is RGB");
}
// save a grayscale frame into a .pgm file
save_gray_frame(pFrame->data[0], pFrame->linesize[0], pFrame->width, pFrame->height, frame_filename);
}
}
return 0;
}
static void save_gray_frame(unsigned char *buf, int wrap, int xsize, int ysize, char *filename)
{
FILE *f;
int i;
f = fopen(filename,"w");
// writing the minimal required header for a pgm file format
// portable graymap format -> https://en.wikipedia.org/wiki/Netpbm_format#PGM_example
fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
// writing line by line
for (i = 0; i < ysize; i++)
fwrite(buf + i * wrap, 1, xsize, f);
fclose(f);
}
================================================
FILE: 2_remuxing.c
================================================
// based on https://ffmpeg.org/doxygen/trunk/remuxing_8c-example.html
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
int main(int argc, char **argv)
{
AVFormatContext *input_format_context = NULL, *output_format_context = NULL;
AVPacket packet;
const char *in_filename, *out_filename;
int ret, i;
int stream_index = 0;
int *streams_list = NULL;
int number_of_streams = 0;
int fragmented_mp4_options = 0;
if (argc < 3) {
printf("You need to pass at least two parameters.\n");
return -1;
} else if (argc == 4) {
fragmented_mp4_options = 1;
}
in_filename = argv[1];
out_filename = argv[2];
if ((ret = avformat_open_input(&input_format_context, in_filename, NULL, NULL)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
if ((ret = avformat_find_stream_info(input_format_context, NULL)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
avformat_alloc_output_context2(&output_format_context, NULL, NULL, out_filename);
if (!output_format_context) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
number_of_streams = input_format_context->nb_streams;
streams_list = av_mallocz_array(number_of_streams, sizeof(*streams_list));
if (!streams_list) {
ret = AVERROR(ENOMEM);
goto end;
}
for (i = 0; i < input_format_context->nb_streams; i++) {
AVStream *out_stream;
AVStream *in_stream = input_format_context->streams[i];
AVCodecParameters *in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
streams_list[i] = -1;
continue;
}
streams_list[i] = stream_index++;
out_stream = avformat_new_stream(output_format_context, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
goto end;
}
}
// https://ffmpeg.org/doxygen/trunk/group__lavf__misc.html#gae2645941f2dc779c307eb6314fd39f10
av_dump_format(output_format_context, 0, out_filename, 1);
// unless it's a no file (we'll talk later about that) write to the disk (FLAG_WRITE)
// but basically it's a way to save the file to a buffer so you can store it
// wherever you want.
if (!(output_format_context->oformat->flags & AVFMT_NOFILE)) {
ret = avio_open(&output_format_context->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
AVDictionary* opts = NULL;
if (fragmented_mp4_options) {
// https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API/Transcoding_assets_for_MSE
av_dict_set(&opts, "movflags", "frag_keyframe+empty_moov+default_base_moof", 0);
}
// https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga18b7b10bb5b94c4842de18166bc677cb
ret = avformat_write_header(output_format_context, &opts);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
while (1) {
AVStream *in_stream, *out_stream;
ret = av_read_frame(input_format_context, &packet);
if (ret < 0)
break;
in_stream = input_format_context->streams[packet.stream_index];
if (packet.stream_index >= number_of_streams || streams_list[packet.stream_index] < 0) {
av_packet_unref(&packet);
continue;
}
packet.stream_index = streams_list[packet.stream_index];
out_stream = output_format_context->streams[packet.stream_index];
/* copy packet */
packet.pts = av_rescale_q_rnd(packet.pts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.dts = av_rescale_q_rnd(packet.dts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.duration = av_rescale_q(packet.duration, in_stream->time_base, out_stream->time_base);
// https://ffmpeg.org/doxygen/trunk/structAVPacket.html#ab5793d8195cf4789dfb3913b7a693903
packet.pos = -1;
//https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga37352ed2c63493c38219d935e71db6c1
ret = av_interleaved_write_frame(output_format_context, &packet);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&packet);
}
//https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga7f14007e7dc8f481f054b21614dfec13
av_write_trailer(output_format_context);
end:
avformat_close_input(&input_format_context);
/* close output */
if (output_format_context && !(output_format_context->oformat->flags & AVFMT_NOFILE))
avio_closep(&output_format_context->pb);
avformat_free_context(output_format_context);
av_freep(&streams_list);
if (ret < 0 && ret != AVERROR_EOF) {
fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
return 1;
}
return 0;
}
================================================
FILE: 3_transcoding.c
================================================
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/timestamp.h>
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <libavutil/opt.h>
#include <string.h>
#include <inttypes.h>
#include "video_debugging.h"
typedef struct StreamingParams {
char copy_video;
char copy_audio;
char *output_extension;
char *muxer_opt_key;
char *muxer_opt_value;
char *video_codec;
char *audio_codec;
char *codec_priv_key;
char *codec_priv_value;
} StreamingParams;
typedef struct StreamingContext {
AVFormatContext *avfc;
AVCodec *video_avc;
AVCodec *audio_avc;
AVStream *video_avs;
AVStream *audio_avs;
AVCodecContext *video_avcc;
AVCodecContext *audio_avcc;
int video_index;
int audio_index;
char *filename;
} StreamingContext;
int fill_stream_info(AVStream *avs, AVCodec **avc, AVCodecContext **avcc) {
*avc = avcodec_find_decoder(avs->codecpar->codec_id);
if (!*avc) {logging("failed to find the codec"); return -1;}
*avcc = avcodec_alloc_context3(*avc);
if (!*avcc) {logging("failed to alloc memory for codec context"); return -1;}
if (avcodec_parameters_to_context(*avcc, avs->codecpar) < 0) {logging("failed to fill codec context"); return -1;}
if (avcodec_open2(*avcc, *avc, NULL) < 0) {logging("failed to open codec"); return -1;}
return 0;
}
int open_media(const char *in_filename, AVFormatContext **avfc) {
*avfc = avformat_alloc_context();
if (!*avfc) {logging("failed to alloc memory for format"); return -1;}
if (avformat_open_input(avfc, in_filename, NULL, NULL) != 0) {logging("failed to open input file %s", in_filename); return -1;}
if (avformat_find_stream_info(*avfc, NULL) < 0) {logging("failed to get stream info"); return -1;}
return 0;
}
int prepare_decoder(StreamingContext *sc) {
for (int i = 0; i < sc->avfc->nb_streams; i++) {
if (sc->avfc->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
sc->video_avs = sc->avfc->streams[i];
sc->video_index = i;
if (fill_stream_info(sc->video_avs, &sc->video_avc, &sc->video_avcc)) {return -1;}
} else if (sc->avfc->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
sc->audio_avs = sc->avfc->streams[i];
sc->audio_index = i;
if (fill_stream_info(sc->audio_avs, &sc->audio_avc, &sc->audio_avcc)) {return -1;}
} else {
logging("skipping streams other than audio and video");
}
}
return 0;
}
int prepare_video_encoder(StreamingContext *sc, AVCodecContext *decoder_ctx, AVRational input_framerate, StreamingParams sp) {
sc->video_avs = avformat_new_stream(sc->avfc, NULL);
sc->video_avc = avcodec_find_encoder_by_name(sp.video_codec);
if (!sc->video_avc) {logging("could not find the proper codec"); return -1;}
sc->video_avcc = avcodec_alloc_context3(sc->video_avc);
if (!sc->video_avcc) {logging("could not allocated memory for codec context"); return -1;}
av_opt_set(sc->video_avcc->priv_data, "preset", "fast", 0);
if (sp.codec_priv_key && sp.codec_priv_value)
av_opt_set(sc->video_avcc->priv_data, sp.codec_priv_key, sp.codec_priv_value, 0);
sc->video_avcc->height = decoder_ctx->height;
sc->video_avcc->width = decoder_ctx->width;
sc->video_avcc->sample_aspect_ratio = decoder_ctx->sample_aspect_ratio;
if (sc->video_avc->pix_fmts)
sc->video_avcc->pix_fmt = sc->video_avc->pix_fmts[0];
else
sc->video_avcc->pix_fmt = decoder_ctx->pix_fmt;
sc->video_avcc->bit_rate = 2 * 1000 * 1000;
sc->video_avcc->rc_buffer_size = 4 * 1000 * 1000;
sc->video_avcc->rc_max_rate = 2 * 1000 * 1000;
sc->video_avcc->rc_min_rate = 2.5 * 1000 * 1000;
sc->video_avcc->time_base = av_inv_q(input_framerate);
sc->video_avs->time_base = sc->video_avcc->time_base;
if (avcodec_open2(sc->video_avcc, sc->video_avc, NULL) < 0) {logging("could not open the codec"); return -1;}
avcodec_parameters_from_context(sc->video_avs->codecpar, sc->video_avcc);
return 0;
}
int prepare_audio_encoder(StreamingContext *sc, int sample_rate, StreamingParams sp){
sc->audio_avs = avformat_new_stream(sc->avfc, NULL);
sc->audio_avc = avcodec_find_encoder_by_name(sp.audio_codec);
if (!sc->audio_avc) {logging("could not find the proper codec"); return -1;}
sc->audio_avcc = avcodec_alloc_context3(sc->audio_avc);
if (!sc->audio_avcc) {logging("could not allocated memory for codec context"); return -1;}
int OUTPUT_CHANNELS = 2;
int OUTPUT_BIT_RATE = 196000;
sc->audio_avcc->channels = OUTPUT_CHANNELS;
sc->audio_avcc->channel_layout = av_get_default_channel_layout(OUTPUT_CHANNELS);
sc->audio_avcc->sample_rate = sample_rate;
sc->audio_avcc->sample_fmt = sc->audio_avc->sample_fmts[0];
sc->audio_avcc->bit_rate = OUTPUT_BIT_RATE;
sc->audio_avcc->time_base = (AVRational){1, sample_rate};
sc->audio_avcc->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;
sc->audio_avs->time_base = sc->audio_avcc->time_base;
if (avcodec_open2(sc->audio_avcc, sc->audio_avc, NULL) < 0) {logging("could not open the codec"); return -1;}
avcodec_parameters_from_context(sc->audio_avs->codecpar, sc->audio_avcc);
return 0;
}
int prepare_copy(AVFormatContext *avfc, AVStream **avs, AVCodecParameters *decoder_par) {
*avs = avformat_new_stream(avfc, NULL);
avcodec_parameters_copy((*avs)->codecpar, decoder_par);
return 0;
}
int remux(AVPacket **pkt, AVFormatContext **avfc, AVRational decoder_tb, AVRational encoder_tb) {
av_packet_rescale_ts(*pkt, decoder_tb, encoder_tb);
if (av_interleaved_write_frame(*avfc, *pkt) < 0) { logging("error while copying stream packet"); return -1; }
return 0;
}
int encode_video(StreamingContext *decoder, StreamingContext *encoder, AVFrame *input_frame) {
if (input_frame) input_frame->pict_type = AV_PICTURE_TYPE_NONE;
AVPacket *output_packet = av_packet_alloc();
if (!output_packet) {logging("could not allocate memory for output packet"); return -1;}
int response = avcodec_send_frame(encoder->video_avcc, input_frame);
while (response >= 0) {
response = avcodec_receive_packet(encoder->video_avcc, output_packet);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
logging("Error while receiving packet from encoder: %s", av_err2str(response));
return -1;
}
output_packet->stream_index = decoder->video_index;
output_packet->duration = encoder->video_avs->time_base.den / encoder->video_avs->time_base.num / decoder->video_avs->avg_frame_rate.num * decoder->video_avs->avg_frame_rate.den;
av_packet_rescale_ts(output_packet, decoder->video_avs->time_base, encoder->video_avs->time_base);
response = av_interleaved_write_frame(encoder->avfc, output_packet);
if (response != 0) { logging("Error %d while receiving packet from decoder: %s", response, av_err2str(response)); return -1;}
}
av_packet_unref(output_packet);
av_packet_free(&output_packet);
return 0;
}
int encode_audio(StreamingContext *decoder, StreamingContext *encoder, AVFrame *input_frame) {
AVPacket *output_packet = av_packet_alloc();
if (!output_packet) {logging("could not allocate memory for output packet"); return -1;}
int response = avcodec_send_frame(encoder->audio_avcc, input_frame);
while (response >= 0) {
response = avcodec_receive_packet(encoder->audio_avcc, output_packet);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
logging("Error while receiving packet from encoder: %s", av_err2str(response));
return -1;
}
output_packet->stream_index = decoder->audio_index;
av_packet_rescale_ts(output_packet, decoder->audio_avs->time_base, encoder->audio_avs->time_base);
response = av_interleaved_write_frame(encoder->avfc, output_packet);
if (response != 0) { logging("Error %d while receiving packet from decoder: %s", response, av_err2str(response)); return -1;}
}
av_packet_unref(output_packet);
av_packet_free(&output_packet);
return 0;
}
int transcode_audio(StreamingContext *decoder, StreamingContext *encoder, AVPacket *input_packet, AVFrame *input_frame) {
int response = avcodec_send_packet(decoder->audio_avcc, input_packet);
if (response < 0) {logging("Error while sending packet to decoder: %s", av_err2str(response)); return response;}
while (response >= 0) {
response = avcodec_receive_frame(decoder->audio_avcc, input_frame);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
logging("Error while receiving frame from decoder: %s", av_err2str(response));
return response;
}
if (response >= 0) {
if (encode_audio(decoder, encoder, input_frame)) return -1;
}
av_frame_unref(input_frame);
}
return 0;
}
int transcode_video(StreamingContext *decoder, StreamingContext *encoder, AVPacket *input_packet, AVFrame *input_frame) {
int response = avcodec_send_packet(decoder->video_avcc, input_packet);
if (response < 0) {logging("Error while sending packet to decoder: %s", av_err2str(response)); return response;}
while (response >= 0) {
response = avcodec_receive_frame(decoder->video_avcc, input_frame);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
logging("Error while receiving frame from decoder: %s", av_err2str(response));
return response;
}
if (response >= 0) {
if (encode_video(decoder, encoder, input_frame)) return -1;
}
av_frame_unref(input_frame);
}
return 0;
}
int main(int argc, char *argv[])
{
/*
* H264 -> H265
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx265";
sp.codec_priv_key = "x265-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
//StreamingParams sp = {0};
//sp.copy_audio = 1;
//sp.copy_video = 0;
//sp.video_codec = "libx264";
//sp.codec_priv_key = "x264-params";
//sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - fragmented MP4
*/
//StreamingParams sp = {0};
//sp.copy_audio = 1;
//sp.copy_video = 0;
//sp.video_codec = "libx264";
//sp.codec_priv_key = "x264-params";
//sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
//sp.muxer_opt_key = "movflags";
//sp.muxer_opt_value = "frag_keyframe+empty_moov+delay_moov+default_base_moof";
/*
* H264 -> H264 (fixed gop)
* Audio -> AAC
* MP4 - MPEG-TS
*/
//StreamingParams sp = {0};
//sp.copy_audio = 0;
//sp.copy_video = 0;
//sp.video_codec = "libx264";
//sp.codec_priv_key = "x264-params";
//sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
//sp.audio_codec = "aac";
//sp.output_extension = ".ts";
/*
* H264 -> VP9
* Audio -> Vorbis
* MP4 - WebM
*/
//StreamingParams sp = {0};
//sp.copy_audio = 0;
//sp.copy_video = 0;
//sp.video_codec = "libvpx-vp9";
//sp.audio_codec = "libvorbis";
//sp.output_extension = ".webm";
StreamingContext *decoder = (StreamingContext*) calloc(1, sizeof(StreamingContext));
decoder->filename = argv[1];
StreamingContext *encoder = (StreamingContext*) calloc(1, sizeof(StreamingContext));
encoder->filename = argv[2];
if (sp.output_extension)
strcat(encoder->filename, sp.output_extension);
if (open_media(decoder->filename, &decoder->avfc)) return -1;
if (prepare_decoder(decoder)) return -1;
avformat_alloc_output_context2(&encoder->avfc, NULL, NULL, encoder->filename);
if (!encoder->avfc) {logging("could not allocate memory for output format");return -1;}
for (int i = 0; i < decoder->avfc->nb_streams; i++) {
if (decoder->avfc->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
{
if (!sp.copy_video) {
AVRational input_framerate = av_guess_frame_rate(decoder->avfc, decoder->video_avs, NULL);
prepare_video_encoder(encoder, decoder->video_avcc, input_framerate, sp);
} else {
if (prepare_copy(encoder->avfc, &encoder->video_avs, decoder->video_avs->codecpar)) {return -1;}
}
}
if (decoder->avfc->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)
{
if (!sp.copy_audio) {
if (prepare_audio_encoder(encoder, decoder->audio_avcc->sample_rate, sp)) {return -1;}
} else {
if (prepare_copy(encoder->avfc, &encoder->audio_avs, decoder->audio_avs->codecpar)) {return -1;}
}
}
}
if (encoder->avfc->oformat->flags & AVFMT_GLOBALHEADER)
encoder->avfc->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
if (!(encoder->avfc->oformat->flags & AVFMT_NOFILE)) {
if (avio_open(&encoder->avfc->pb, encoder->filename, AVIO_FLAG_WRITE) < 0) {
logging("could not open the output file");
return -1;
}
}
AVDictionary* muxer_opts = NULL;
if (sp.muxer_opt_key && sp.muxer_opt_value) {
av_dict_set(&muxer_opts, sp.muxer_opt_key, sp.muxer_opt_value, 0);
}
if (avformat_write_header(encoder->avfc, &muxer_opts) < 0) {logging("an error occurred when opening output file"); return -1;}
AVFrame *input_frame = av_frame_alloc();
if (!input_frame) {logging("failed to allocated memory for AVFrame"); return -1;}
AVPacket *input_packet = av_packet_alloc();
if (!input_packet) {logging("failed to allocated memory for AVPacket"); return -1;}
while (av_read_frame(decoder->avfc, input_packet) >= 0)
{
if (decoder->avfc->streams[input_packet->stream_index]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
if (!sp.copy_video) {
// TODO: refactor to be generic for audio and video (receiving a function pointer to the differences)
if (transcode_video(decoder, encoder, input_packet, input_frame)) return -1;
av_packet_unref(input_packet);
} else {
if (remux(&input_packet, &encoder->avfc, decoder->video_avs->time_base, encoder->video_avs->time_base)) return -1;
}
} else if (decoder->avfc->streams[input_packet->stream_index]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
if (!sp.copy_audio) {
if (transcode_audio(decoder, encoder, input_packet, input_frame)) return -1;
av_packet_unref(input_packet);
} else {
if (remux(&input_packet, &encoder->avfc, decoder->audio_avs->time_base, encoder->audio_avs->time_base)) return -1;
}
} else {
logging("ignoring all non video or audio packets");
}
}
if (!sp.copy_video)
if (encode_video(decoder, encoder, NULL)) return -1;
if (!sp.copy_audio)
if (encode_audio(decoder, encoder, NULL)) return -1;
av_write_trailer(encoder->avfc);
if (muxer_opts != NULL) {
av_dict_free(&muxer_opts);
muxer_opts = NULL;
}
if (input_frame != NULL) {
av_frame_free(&input_frame);
input_frame = NULL;
}
if (input_packet != NULL) {
av_packet_free(&input_packet);
input_packet = NULL;
}
avformat_close_input(&decoder->avfc);
avformat_free_context(decoder->avfc); decoder->avfc = NULL;
avformat_free_context(encoder->avfc); encoder->avfc = NULL;
avcodec_free_context(&decoder->video_avcc); decoder->video_avcc = NULL;
avcodec_free_context(&decoder->audio_avcc); decoder->audio_avcc = NULL;
free(decoder); decoder = NULL;
free(encoder); encoder = NULL;
return 0;
}
================================================
FILE: CMakeLists.txt
================================================
cmake_minimum_required(VERSION 3.17)
project(libav_tutorial)
# set out directory
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
# set ffmpeg root directory
if(NOT FFMPEG_DEV_ROOT)
message(FATAL_ERROR "set FFMPEG_DEV_ROOT to use ffmpeg libraries")
endif()
# set ffmpeg develop environment
include_directories(${FFMPEG_DEV_ROOT}/include)
link_directories(${FFMPEG_DEV_ROOT}/lib)
link_libraries(
avcodec
avformat
avfilter
avdevice
swresample
swscale
avutil
)
# copy dlls
file(GLOB ffmpeg_shared_libries ${FFMPEG_DEV_ROOT}/bin/*dll)
file(COPY ${ffmpeg_shared_libries} DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
# copy test file
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/small_bunny_1080p_60fps.mp4 DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
# add library
set(debug_src ${CMAKE_CURRENT_SOURCE_DIR}/video_debugging.c)
add_library(video_debug ${debug_src})
link_libraries(video_debug)
# add project/executables
file(GLOB srcs *.c)
list(REMOVE_ITEM srcs ${debug_src})
foreach(src ${srcs})
get_filename_component(TARGET ${src} NAME)
add_executable(${TARGET} ${src})
message(STATUS "${TARGET} added")
endforeach()
================================================
FILE: Dockerfile
================================================
# ffmpeg - http://ffmpeg.org/download.html
#
# From https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu
#
# https://hub.docker.com/r/jrottenberg/ffmpeg/
#
#
FROM ubuntu:20.04 AS base
WORKDIR /tmp/workdir
RUN apt-get -yqq update && \
apt-get install -yq --no-install-recommends ca-certificates expat libgomp1 && \
apt-get autoremove -y && \
apt-get clean -y
FROM base as build
ENV FFMPEG_VERSION=4.4 \
AOM_VERSION=v1.0.0 \
FDKAAC_VERSION=0.1.5 \
FONTCONFIG_VERSION=2.12.4 \
FREETYPE_VERSION=2.10.4 \
FRIBIDI_VERSION=0.19.7 \
KVAZAAR_VERSION=2.0.0 \
LAME_VERSION=3.100 \
LIBASS_VERSION=0.13.7 \
LIBPTHREAD_STUBS_VERSION=0.4 \
LIBVIDSTAB_VERSION=1.1.0 \
LIBXCB_VERSION=1.13.1 \
XCBPROTO_VERSION=1.13 \
OGG_VERSION=1.3.2 \
OPENCOREAMR_VERSION=0.1.5 \
OPUS_VERSION=1.2 \
OPENJPEG_VERSION=2.1.2 \
THEORA_VERSION=1.1.1 \
VORBIS_VERSION=1.3.5 \
VPX_VERSION=1.8.0 \
WEBP_VERSION=1.0.2 \
X264_VERSION=20170226-2245-stable \
X265_VERSION=3.4 \
XAU_VERSION=1.0.9 \
XORG_MACROS_VERSION=1.19.2 \
XPROTO_VERSION=7.0.31 \
XVID_VERSION=1.3.4 \
LIBXML2_VERSION=2.9.10 \
LIBBLURAY_VERSION=1.1.2 \
LIBZMQ_VERSION=4.3.2 \
LIBSRT_VERSION=1.4.1 \
LIBARIBB24_VERSION=1.0.3 \
LIBPNG_VERSION=1.6.9 \
LIBVMAF_VERSION=2.1.1 \
SRC=/usr/local
ARG FREETYPE_SHA256SUM="5eab795ebb23ac77001cfb68b7d4d50b5d6c7469247b0b01b2c953269f658dac freetype-2.10.4.tar.gz"
ARG FRIBIDI_SHA256SUM="3fc96fa9473bd31dcb5500bdf1aa78b337ba13eb8c301e7c28923fea982453a8 0.19.7.tar.gz"
ARG LIBASS_SHA256SUM="8fadf294bf701300d4605e6f1d92929304187fca4b8d8a47889315526adbafd7 0.13.7.tar.gz"
ARG LIBVIDSTAB_SHA256SUM="14d2a053e56edad4f397be0cb3ef8eb1ec3150404ce99a426c4eb641861dc0bb v1.1.0.tar.gz"
ARG OGG_SHA256SUM="e19ee34711d7af328cb26287f4137e70630e7261b17cbe3cd41011d73a654692 libogg-1.3.2.tar.gz"
ARG OPUS_SHA256SUM="77db45a87b51578fbc49555ef1b10926179861d854eb2613207dc79d9ec0a9a9 opus-1.2.tar.gz"
ARG THEORA_SHA256SUM="40952956c47811928d1e7922cda3bc1f427eb75680c3c37249c91e949054916b libtheora-1.1.1.tar.gz"
ARG VORBIS_SHA256SUM="6efbcecdd3e5dfbf090341b485da9d176eb250d893e3eb378c428a2db38301ce libvorbis-1.3.5.tar.gz"
ARG XVID_SHA256SUM="4e9fd62728885855bc5007fe1be58df42e5e274497591fec37249e1052ae316f xvidcore-1.3.4.tar.gz"
ARG LIBXML2_SHA256SUM="f07dab13bf42d2b8db80620cce7419b3b87827cc937c8bb20fe13b8571ee9501 libxml2-v2.9.10.tar.gz"
ARG LIBBLURAY_SHA256SUM="a3dd452239b100dc9da0d01b30e1692693e2a332a7d29917bf84bb10ea7c0b42 libbluray-1.1.2.tar.bz2"
ARG LIBZMQ_SHA256SUM="02ecc88466ae38cf2c8d79f09cfd2675ba299a439680b64ade733e26a349edeb v4.3.2.tar.gz"
ARG LIBARIBB24_SHA256SUM="f61560738926e57f9173510389634d8c06cabedfa857db4b28fb7704707ff128 v1.0.3.tar.gz"
ARG LIBVMAF_SHA256SUM="e7fc00ae1322a7eccfcf6d4f1cdf9c67eec8058709887c8c6c3795c617326f77 v2.1.1.tar.gz"
ARG LD_LIBRARY_PATH=/opt/ffmpeg/lib
ARG MAKEFLAGS="-j2"
ARG PKG_CONFIG_PATH="/opt/ffmpeg/share/pkgconfig:/opt/ffmpeg/lib/pkgconfig:/opt/ffmpeg/lib64/pkgconfig"
ARG PREFIX=/opt/ffmpeg
ARG LD_LIBRARY_PATH="/opt/ffmpeg/lib:/opt/ffmpeg/lib64"
ARG DEBIAN_FRONTEND=noninteractive
RUN buildDeps="autoconf \
automake \
cmake \
curl \
bzip2 \
libexpat1-dev \
g++ \
gcc \
git \
gperf \
libtool \
make \
meson \
nasm \
perl \
pkg-config \
python \
libssl-dev \
yasm \
zlib1g-dev" && \
apt-get -yqq update && \
apt-get install -yq --no-install-recommends ${buildDeps}
## libvmaf https://github.com/Netflix/vmaf
RUN \
if which meson || false; then \
echo "Building VMAF." && \
DIR=/tmp/vmaf && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/Netflix/vmaf/archive/v${LIBVMAF_VERSION}.tar.gz && \
tar -xz --strip-components=1 -f v${LIBVMAF_VERSION}.tar.gz && \
cd /tmp/vmaf/libvmaf && \
meson build --buildtype release --prefix=${PREFIX} && \
ninja -vC build && \
ninja -vC build install && \
mkdir -p ${PREFIX}/share/model/ && \
cp -r /tmp/vmaf/model/* ${PREFIX}/share/model/ && \
rm -rf ${DIR}; \
else \
echo "VMAF skipped."; \
fi
## opencore-amr https://sourceforge.net/projects/opencore-amr/
RUN \
DIR=/tmp/opencore-amr && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://versaweb.dl.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-${OPENCOREAMR_VERSION}.tar.gz | \
tar -zx --strip-components=1 && \
./configure --prefix="${PREFIX}" --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
## x264 http://www.videolan.org/developers/x264.html
RUN \
DIR=/tmp/x264 && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://download.videolan.org/pub/videolan/x264/snapshots/x264-snapshot-${X264_VERSION}.tar.bz2 | \
tar -jx --strip-components=1 && \
./configure --prefix="${PREFIX}" --enable-shared --enable-pic --disable-cli && \
make && \
make install && \
rm -rf ${DIR}
### x265 http://x265.org/
RUN \
DIR=/tmp/x265 && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://github.com/videolan/x265/archive/refs/tags/${X265_VERSION}.tar.gz | \
tar -zx && \
cd x265-${X265_VERSION}/build/linux && \
sed -i "/-DEXTRA_LIB/ s/$/ -DCMAKE_INSTALL_PREFIX=\${PREFIX}/" multilib.sh && \
sed -i "/^cmake/ s/$/ -DENABLE_CLI=OFF/" multilib.sh && \
./multilib.sh && \
make -C 8bit install && \
rm -rf ${DIR}
### libogg https://www.xiph.org/ogg/
RUN \
DIR=/tmp/ogg && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO http://downloads.xiph.org/releases/ogg/libogg-${OGG_VERSION}.tar.gz && \
echo ${OGG_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f libogg-${OGG_VERSION}.tar.gz && \
./configure --prefix="${PREFIX}" --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
### libopus https://www.opus-codec.org/
RUN \
DIR=/tmp/opus && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz && \
echo ${OPUS_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f opus-${OPUS_VERSION}.tar.gz && \
autoreconf -fiv && \
./configure --prefix="${PREFIX}" --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
### libvorbis https://xiph.org/vorbis/
RUN \
DIR=/tmp/vorbis && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO http://downloads.xiph.org/releases/vorbis/libvorbis-${VORBIS_VERSION}.tar.gz && \
echo ${VORBIS_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f libvorbis-${VORBIS_VERSION}.tar.gz && \
./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
### libtheora http://www.theora.org/
RUN \
DIR=/tmp/theora && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO http://downloads.xiph.org/releases/theora/libtheora-${THEORA_VERSION}.tar.gz && \
echo ${THEORA_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f libtheora-${THEORA_VERSION}.tar.gz && \
./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
### libvpx https://www.webmproject.org/code/
RUN \
DIR=/tmp/vpx && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://codeload.github.com/webmproject/libvpx/tar.gz/v${VPX_VERSION} | \
tar -zx --strip-components=1 && \
./configure --prefix="${PREFIX}" --enable-vp8 --enable-vp9 --enable-vp9-highbitdepth --enable-pic --enable-shared \
--disable-debug --disable-examples --disable-docs --disable-install-bins && \
make && \
make install && \
rm -rf ${DIR}
### libwebp https://developers.google.com/speed/webp/
RUN \
DIR=/tmp/vebp && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | \
tar -zx --strip-components=1 && \
./configure --prefix="${PREFIX}" --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
### libmp3lame http://lame.sourceforge.net/
RUN \
DIR=/tmp/lame && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://versaweb.dl.sourceforge.net/project/lame/lame/$(echo ${LAME_VERSION} | sed -e 's/[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)/\1.\2/')/lame-${LAME_VERSION}.tar.gz | \
tar -zx --strip-components=1 && \
./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" --enable-shared --enable-nasm --disable-frontend && \
make && \
make install && \
rm -rf ${DIR}
### xvid https://www.xvid.com/
RUN \
DIR=/tmp/xvid && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://xvid.com/downloads/xvidcore-${XVID_VERSION}.tar.gz && \
echo ${XVID_SHA256SUM} | sha256sum --check && \
tar -zx -f xvidcore-${XVID_VERSION}.tar.gz && \
cd xvidcore/build/generic && \
./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" && \
make && \
make install && \
rm -rf ${DIR}
### fdk-aac https://github.com/mstorsjo/fdk-aac
RUN \
DIR=/tmp/fdk-aac && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://github.com/mstorsjo/fdk-aac/archive/v${FDKAAC_VERSION}.tar.gz | \
tar -zx --strip-components=1 && \
autoreconf -fiv && \
./configure --prefix="${PREFIX}" --enable-shared --datadir="${DIR}" && \
make && \
make install && \
rm -rf ${DIR}
## openjpeg https://github.com/uclouvain/openjpeg
RUN \
DIR=/tmp/openjpeg && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | \
tar -zx --strip-components=1 && \
cmake -DBUILD_THIRDPARTY:BOOL=ON -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
make && \
make install && \
rm -rf ${DIR}
## freetype https://www.freetype.org/
RUN \
DIR=/tmp/freetype && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://download.savannah.gnu.org/releases/freetype/freetype-${FREETYPE_VERSION}.tar.gz && \
echo ${FREETYPE_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f freetype-${FREETYPE_VERSION}.tar.gz && \
./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
## libvstab https://github.com/georgmartius/vid.stab
RUN \
DIR=/tmp/vid.stab && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/georgmartius/vid.stab/archive/v${LIBVIDSTAB_VERSION}.tar.gz && \
echo ${LIBVIDSTAB_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f v${LIBVIDSTAB_VERSION}.tar.gz && \
cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
make && \
make install && \
rm -rf ${DIR}
## fridibi https://www.fribidi.org/
RUN \
DIR=/tmp/fribidi && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/fribidi/fribidi/archive/${FRIBIDI_VERSION}.tar.gz && \
echo ${FRIBIDI_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f ${FRIBIDI_VERSION}.tar.gz && \
sed -i 's/^SUBDIRS =.*/SUBDIRS=gen.tab charset lib bin/' Makefile.am && \
./bootstrap --no-config --auto && \
./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
make -j1 && \
make install && \
rm -rf ${DIR}
## fontconfig https://www.freedesktop.org/wiki/Software/fontconfig/
RUN \
DIR=/tmp/fontconfig && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://www.freedesktop.org/software/fontconfig/release/fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
tar -jx --strip-components=1 -f fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
## libass https://github.com/libass/libass
RUN \
DIR=/tmp/libass && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/libass/libass/archive/${LIBASS_VERSION}.tar.gz && \
echo ${LIBASS_SHA256SUM} | sha256sum --check && \
tar -zx --strip-components=1 -f ${LIBASS_VERSION}.tar.gz && \
./autogen.sh && \
./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
## kvazaar https://github.com/ultravideo/kvazaar
RUN \
DIR=/tmp/kvazaar && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/ultravideo/kvazaar/archive/v${KVAZAAR_VERSION}.tar.gz && \
tar -zx --strip-components=1 -f v${KVAZAAR_VERSION}.tar.gz && \
./autogen.sh && \
./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
RUN \
DIR=/tmp/aom && \
git clone --branch ${AOM_VERSION} --depth 1 https://aomedia.googlesource.com/aom ${DIR} ; \
cd ${DIR} ; \
rm -rf CMakeCache.txt CMakeFiles ; \
mkdir -p ./aom_build ; \
cd ./aom_build ; \
cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DBUILD_SHARED_LIBS=1 ..; \
make ; \
make install ; \
rm -rf ${DIR}
## libxcb (and supporting libraries) for screen capture https://xcb.freedesktop.org/
RUN \
DIR=/tmp/xorg-macros && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://www.x.org/archive//individual/util/util-macros-${XORG_MACROS_VERSION}.tar.gz && \
tar -zx --strip-components=1 -f util-macros-${XORG_MACROS_VERSION}.tar.gz && \
./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
make && \
make install && \
rm -rf ${DIR}
RUN \
DIR=/tmp/xproto && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://www.x.org/archive/individual/proto/xproto-${XPROTO_VERSION}.tar.gz && \
tar -zx --strip-components=1 -f xproto-${XPROTO_VERSION}.tar.gz && \
./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
make && \
make install && \
rm -rf ${DIR}
RUN \
DIR=/tmp/libXau && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://www.x.org/archive/individual/lib/libXau-${XAU_VERSION}.tar.gz && \
tar -zx --strip-components=1 -f libXau-${XAU_VERSION}.tar.gz && \
./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
make && \
make install && \
rm -rf ${DIR}
RUN \
DIR=/tmp/libpthread-stubs && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://xcb.freedesktop.org/dist/libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
tar -zx --strip-components=1 -f libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
./configure --prefix="${PREFIX}" && \
make && \
make install && \
rm -rf ${DIR}
RUN \
DIR=/tmp/libxcb-proto && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://xcb.freedesktop.org/dist/xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
tar -zx --strip-components=1 -f xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
./configure --prefix="${PREFIX}" && \
make && \
make install && \
rm -rf ${DIR}
RUN \
DIR=/tmp/libxcb && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://xcb.freedesktop.org/dist/libxcb-${LIBXCB_VERSION}.tar.gz && \
tar -zx --strip-components=1 -f libxcb-${LIBXCB_VERSION}.tar.gz && \
ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
## libxml2 - for libbluray
RUN \
DIR=/tmp/libxml2 && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://gitlab.gnome.org/GNOME/libxml2/-/archive/v${LIBXML2_VERSION}/libxml2-v${LIBXML2_VERSION}.tar.gz && \
echo ${LIBXML2_SHA256SUM} | sha256sum --check && \
tar -xz --strip-components=1 -f libxml2-v${LIBXML2_VERSION}.tar.gz && \
./autogen.sh --prefix="${PREFIX}" --with-ftp=no --with-http=no --with-python=no && \
make && \
make install && \
rm -rf ${DIR}
## libbluray - Requires libxml, freetype, and fontconfig
RUN \
DIR=/tmp/libbluray && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://download.videolan.org/pub/videolan/libbluray/${LIBBLURAY_VERSION}/libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
echo ${LIBBLURAY_SHA256SUM} | sha256sum --check && \
tar -jx --strip-components=1 -f libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
./configure --prefix="${PREFIX}" --disable-examples --disable-bdjava-jar --disable-static --enable-shared && \
make && \
make install && \
rm -rf ${DIR}
## libzmq https://github.com/zeromq/libzmq/
RUN \
DIR=/tmp/libzmq && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/zeromq/libzmq/archive/v${LIBZMQ_VERSION}.tar.gz && \
echo ${LIBZMQ_SHA256SUM} | sha256sum --check && \
tar -xz --strip-components=1 -f v${LIBZMQ_VERSION}.tar.gz && \
./autogen.sh && \
./configure --prefix="${PREFIX}" && \
make && \
make check && \
make install && \
rm -rf ${DIR}
## libsrt https://github.com/Haivision/srt
RUN \
DIR=/tmp/srt && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/Haivision/srt/archive/v${LIBSRT_VERSION}.tar.gz && \
tar -xz --strip-components=1 -f v${LIBSRT_VERSION}.tar.gz && \
cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
make && \
make install && \
rm -rf ${DIR}
## libpng
RUN \
DIR=/tmp/png && \
mkdir -p ${DIR} && \
cd ${DIR} && \
git clone https://git.code.sf.net/p/libpng/code ${DIR} -b v${LIBPNG_VERSION} --depth 1 && \
./autogen.sh && \
./configure --prefix="${PREFIX}" && \
make check && \
make install && \
rm -rf ${DIR}
## libaribb24
RUN \
DIR=/tmp/b24 && \
mkdir -p ${DIR} && \
cd ${DIR} && \
curl -sLO https://github.com/nkoriyama/aribb24/archive/v${LIBARIBB24_VERSION}.tar.gz && \
echo ${LIBARIBB24_SHA256SUM} | sha256sum --check && \
tar -xz --strip-components=1 -f v${LIBARIBB24_VERSION}.tar.gz && \
autoreconf -fiv && \
./configure CFLAGS="-I${PREFIX}/include -fPIC" --prefix="${PREFIX}" && \
make && \
make install && \
rm -rf ${DIR}
## ffmpeg https://ffmpeg.org/
RUN \
DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2
RUN \
DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
./configure \
--disable-debug \
--disable-doc \
--disable-ffplay \
--enable-shared \
--enable-avresample \
--enable-libopencore-amrnb \
--enable-libopencore-amrwb \
--enable-gpl \
--enable-libass \
--enable-fontconfig \
--enable-libfreetype \
--enable-libvidstab \
--enable-libmp3lame \
--enable-libopus \
--enable-libtheora \
--enable-libvorbis \
--enable-libvpx \
--enable-libwebp \
--enable-libxcb \
--enable-libx265 \
--enable-libxvid \
--enable-libx264 \
--enable-nonfree \
--enable-openssl \
--enable-libfdk_aac \
--enable-postproc \
--enable-small \
--enable-version3 \
--enable-libbluray \
--enable-libzmq \
--extra-libs=-ldl \
--prefix="${PREFIX}" \
--enable-libopenjpeg \
--enable-libkvazaar \
--enable-libaom \
--extra-libs=-lpthread \
--enable-libsrt \
--enable-libaribb24 \
--enable-libvmaf \
--extra-cflags="-I${PREFIX}/include" \
--extra-ldflags="-L${PREFIX}/lib" && \
make && \
make install && \
make tools/zmqsend && cp tools/zmqsend ${PREFIX}/bin/ && \
make distclean && \
hash -r && \
cd tools && \
make qt-faststart && cp qt-faststart ${PREFIX}/bin/
# Let's make sure the app built correctly
# Convenient to verify on https://hub.docker.com/r/jrottenberg/ffmpeg/builds/ console output
FROM base AS release
ENV LD_LIBRARY_PATH /opt/ffmpeg/lib:/usr/local/lib
RUN apt-get -yqq update && \
apt-get install -yq --no-install-recommends build-essential && \
apt-get autoremove -y && \
apt-get clean -y
COPY --from=build /opt/ffmpeg /opt/ffmpeg
================================================
FILE: LICENSE
================================================
BSD 3-Clause License
Copyright (c) 2017, Leandro Moreira
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================
FILE: Makefile
================================================
usage:
echo "make fetch_small_bunny_video && make run_hello"
all: clean fetch_bbb_video make_hello run_hello make_remuxing run_remuxing_ts run_remuxing_fragmented_mp4 make_transcoding
.PHONY: all
clean:
@rm -rf ./build/*
fetch_small_bunny_video:
./fetch_bbb_video.sh
make_hello: clean
docker run -i -w /files --rm -v `pwd`:/files leandromoreira/ffmpeg-devel:4.4 \
gcc -L/opt/ffmpeg/lib -I/opt/ffmpeg/include/ /files/0_hello_world.c \
-lavcodec -lavformat -lavfilter -lavdevice -lswresample -lswscale -lavutil \
-o /files/build/hello
run_hello: make_hello
docker run -i -w /files --rm -v `pwd`:/files leandromoreira/ffmpeg-devel:4.4 /files/build/hello /files/small_bunny_1080p_60fps.mp4
make_remuxing: clean
docker run -i -w /files --rm -v `pwd`:/files leandromoreira/ffmpeg-devel:4.4 \
gcc -L/opt/ffmpeg/lib -I/opt/ffmpeg/include/ /files/2_remuxing.c \
-lavcodec -lavformat -lavfilter -lavdevice -lswresample -lswscale -lavutil \
-o /files/build/remuxing
run_remuxing_ts: make_remuxing
docker run -i -w /files --rm -v `pwd`:/files leandromoreira/ffmpeg-devel:4.4 /files/build/remuxing /files/small_bunny_1080p_60fps.mp4 /files/remuxed_small_bunny_1080p_60fps.ts
run_remuxing_fragmented_mp4: make_remuxing
docker run -i -w /files --rm -v `pwd`:/files leandromoreira/ffmpeg-devel:4.4 /files/build/remuxing /files/small_bunny_1080p_60fps.mp4 /files/fragmented_small_bunny_1080p_60fps.mp4 fragmented
make_transcoding: clean
docker run -i -w /files --rm -v `pwd`:/files leandromoreira/ffmpeg-devel:4.4 \
gcc -g -Wall -L/opt/ffmpeg/lib -I/opt/ffmpeg/include/ /files/3_transcoding.c /files/video_debugging.c \
-lavcodec -lavformat -lavfilter -lavdevice -lswresample -lswscale -lavutil \
-o /files/build/3_transcoding
run_transcoding: make_transcoding
docker run -i -w /files --rm -v `pwd`:/files leandromoreira/ffmpeg-devel:4.4 ./build/3_transcoding /files/small_bunny_1080p_60fps.mp4 /files/bunny_1s_gop.mp4
================================================
FILE: README-cn.md
================================================
[](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)
起初我在寻找可以学习使用FFmpeg库(又名 libav)的教程或书籍，然后找到了名为["如何在1k行代码内实现视频播放器"](http://dranger.com/ffmpeg/)的指南。但该项目已经停止维护，因此我决定撰写此教程。
此项目主要使用C语言开发，**但请不用担心**：项目内容非常通俗易懂。FFmpeg libav具有许多其他语言的实现，例如[python](https://pyav.org/)，[go](https://github.com/imkira/go-libav)。即使其中没有你熟悉的编程语言，仍然可以通过 `ffi` 为它提供支持（这是一个 [Lua](https://github.com/daurnimator/ffmpeg-lua-ffi/blob/master/init.lua) 的示例）。
下文将会简单介绍什么是视频、音频、编解码和容器，然后我们将尝试使用 FFmpeg 命令行工具，最终使用代码实现一些功能。如果你拥有一些经验，可以随时跳过这些内容，直接阅读 [笨办法学 FFmpeg libav](#笨办法学-FFmpeg-libav) 章节。
许多人认为网络视频流媒体是传统 TV 的未来。无论如何，FFmpeg 值得我们深入学习。
__目录__
* [介绍](#介绍)
* [视频 - 目光所见](#视频---目光所见)
* [音频 - 耳朵所听](#音频---耳朵所听)
* [编解码 - 压缩数据](#编解码---压缩数据)
* [容器 - 整合音频和视频](#容器---整合音视频)
* [FFmpeg - 命令行](#FFmpeg---命令行)
* [FFmpeg 命令行工具 101](#FFmpeg-命令行工具-101)
* [通用视频操作](#通用视频操作)
* [转码](#转码)
* [转封装](#转封装)
* [转码率](#转码率)
* [转分辨率](#转分辨率)
* [自适应流](#自适应流)
* [更多](#更多)
* [笨办法学 FFmpeg libav](#笨办法学-FFmpeg-libav)
* [章节0 - 臭名昭著的 hello world](#章节0---臭名昭著的-hello-world)
* [FFmpeg libav 架构](#FFmpeg-libav-架构)
* [章节1 - 音视频同步](#章节-1---音视频同步)
* [章节2 - 重新封装](#章节-2---重新封装)
* [章节3 - 转码](#章节-3---转码)
# 介绍
## 视频 - 目光所见
如果以一定的频率播放一组图片([比如每秒24张图片](https://www.filmindependent.org/blog/hacking-film-24-frames-per-second/))，人将会产生[视觉暂留现象](https://en.wikipedia.org/wiki/Persistence_of_vision)。
概括来讲，视频的本质就是: **以给定频率播放的一系列图片/帧**.
<img src="https://upload.wikimedia.org/wikipedia/commons/1/1f/Linnet_kineograph_1886.jpg" title="flip book" height="280"></img>
当代插画 (1886)
## 音频 - 耳朵所听
尽管一个没有声音的视频也可以表达很多感受和情绪，但加入音频会带来更多的体验乐趣。
声音是指以压力波形式通过空气或其他介质（例如气体、液体或者固体）传播的振动。
> 在数字音频系统中，麦克风将声音转换为模拟电信号，然后通常使用脉冲编码调制（[PCM](https://en.wikipedia.org/wiki/Pulse-code_modulation)）的模数转换器（ADC）将模拟信号转换为数字信号。
>[图片来源](https://commons.wikimedia.org/wiki/File:CPT-Sound-ADC-DAC.svg)
## 编解码 - 压缩数据
> CODEC是用于压缩或解压缩数字音频/视频的硬件或软件。 它提供将原始（未压缩的）数字音频/视频与压缩格式相互转换的能力。
>
> https://en.wikipedia.org/wiki/Video_codec
如果我们选择打包数百万张图片来生成一个视频文件，那么该文件的大小将会非常惊人。让我们来计算一下：
假如我们创建一个 `1080x1920` (高x宽)的视频，每个像素占用 `3 bytes` 对颜色进行编码(或使用 [24 bit](https://en.wikipedia.org/wiki/Color_depth#True_color_.2824-bit.29) 真色彩, 这可以提供 16,777,216 种不同的颜色)，每秒 24 帧，视频时长为 30 分钟。
```c
toppf = 1080 * 1920 // 每帧所有的像素点
cpp = 3 // 每个像素的大小(bytes)
tis = 30 * 60 // 时长(秒)
fps = 24 // 每秒帧数
required_storage = tis * fps * toppf * cpp
```
计算结果显示，此视频需要大约 `250.28G` 的存储空间或 `1.19Gbps` 的带宽。这就是我们为什么需要使用 [CODEC](https://github.com/leandromoreira/digital_video_introduction#how-does-a-video-codec-work) 的原因。
## 容器 - 整合音视频
> 容器或者封装格式描述了不同的数据元素和元数据是如何在计算机文件中共存的。
> https://en.wikipedia.org/wiki/Digital_container_format
**单个这样的文件包含所有的流**（主要是音频和视频），并提供**同步和通用元数据**，比如标题、分辨率等等。
一般我们可以通过文件的后缀来判断文件格式：比如 video.webm 通常是一个使用 [`webm`](https://www.webmproject.org/) 容器格式的视频。
# FFmpeg - 命令行
> 这是一个完整的跨平台解决方案，可用于音视频的录制、转换和流式传输等。
我们使用非常优秀的工具/库 [FFmpeg](https://www.ffmpeg.org/) 来处理多媒体文件。你可能对它有些了解，也可能已经直接或者间接的在使用它了（你用过 [Chrome](https://www.chromium.org/developers/design-documents/video) 吗？）
`ffmpeg` 是该方案中简单而强大的命令行工具。例如，可以通过以下命令将一个 `mp4` 文件转换成 `avi` 格式：
```bash
$ ffmpeg -i input.mp4 output.avi
```
通过上述步骤，我们做了一次重新封装，从一个容器转换为另外一个容器。FFmpeg 也可以用于转码，我们稍后再针对它进行讨论。
## **FFmpeg 命令行工具 101**
FFmpeg 有一个非常完善的[文档](https://www.ffmpeg.org/ffmpeg.html)来介绍它是如何工作的。
简单来说，FFmpeg 命令行程序需要以下参数格式来执行操作： `ffmpeg {1} {2} -i {3} {4} {5}`，分别是:
1. 全局参数
2. 输入文件参数
3. 输入文件
4. 输出文件参数
5. 输出文件
选项 2、3、4、5 可以可以根据自己的需求进行添加。以下是一个易于理解的示例：
``` bash
# 警告：这个文件大约 300MB
$ wget -O bunny_1080p_60fps.mp4 http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_1080p_60fps_normal.mp4
$ ffmpeg \
-y \ # 全局参数
-c:a libfdk_aac \ # 输入文件参数
-i bunny_1080p_60fps.mp4 \ # 输入文件
-c:v libvpx-vp9 -c:a libvorbis \ # 输出文件参数
bunny_1080p_60fps_vp9.webm # 输出文件
```
这个命令行作用是将一个 `mp4` 文件（包含了 `aac` 格式的音频流，`h264` 编码格式的视频流）转换为 `webm`，同时改变了音视频的编码格式。
我们可以简化上述命令行，但请注意 FFmpeg 会猜测或采用默认值。例如我们仅输入 `ffmpeg -i input.avi output.mp4` 时，FFmpeg 会使用哪种音频/视频编码来生成 `output.mp4` 呢？
Werner Robitza 写了一篇 [关于 ffmpeg 编码和编辑的教程](https://slhck.info/ffmpeg-encoding-course/#/)。
# 通用视频操作
在处理音频/视频时，我们通常会执行一系列操作。
## 转码
**是什么?** 将其中一个流（视频流或音频流）从一种编码格式转换成另一种
**为什么?** 有时候有些设备（TV，智能手机等等）不支持 X ，但是支持 Y 和一些更新的编码方式，这些方式能提供更好的压缩比
**如何做?** 转换 `H264`（AVC）视频为 `H265`（HEVC）
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c:v libx265 \
bunny_1080p_60fps_h265.mp4
```
## 转封装
**是什么?** 将视频/音频从某一种格式（容器）转换成另一种
**为什么?** 有时候有些设备（TV，智能手机等等）不支持 X ，但是支持 Y 和一些新的容器，这些格式提供了更现代的功能/特征
**如何做?** 转换一个 `mp4` 为 `ts`
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c copy \ # 令 ffmpeg 跳过编解码过程
bunny_1080p_60fps.ts
```
## 转码率
**是什么?** 改变码率或生成其他版本。
**为什么?** 有的人使用用较为落后的智能手机通过 `2G` (edge) 的网络连接来观看视频，有些人使用 4K 电视及光纤网络来观看视频，因此我们需要提供不同的码率的视频来满足不同的需求。
**如何做?** 生成视频码率在 3856k 和 2000K 之间的版本。
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-minrate 964K -maxrate 3856K -bufsize 2000K \
bunny_1080p_60fps_transrating_964_3856.mp4
```
我们通常会同时使用改变码率和分辨率的操作。Werner Robitza 写了另一篇关于 [FFmpeg 码率控制](https://slhck.info/posts/) 的必知必会系列文章。
## 转分辨率
**是什么?** 将视频从一种分辨率转为其他分辨率的操作。正如上文所述，改变分辨率的操作通常与改变码率的操作同时使用。
**为什么?** 原因与转码率相同，需要满足不同情况下的不同需求。
**如何做?** 将视频从 `1080p` 转换为 `480p`
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-vf scale=480:-1 \
bunny_1080p_60fps_transsizing_480.mp4
```
## 自适应流
**是什么?** 生成很多不同分辨率/码率的视频并分块，通过http进行传输。
**为什么?** 为了在不同的终端和网络环境下提供更加灵活的观看体验，比如低端智能手机或者4K电视。这也使得扩展和部署更为简单方便，但是会增加延迟。
**如何做?** 用 DASH 创建一个自适应的 WebM。
```bash
# 视频流
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 160x90 -b:v 250k -keyint_min 150 -g 150 -an -f webm -dash 1 video_160x90_250k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 320x180 -b:v 500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_320x180_500k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 750k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_750k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 1000k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_1000k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 1280x720 -b:v 1500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_1280x720_1500k.webm
# 音频流
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:a libvorbis -b:a 128k -vn -f webm -dash 1 audio_128k.webm
# DASH 格式
$ ffmpeg \
-f webm_dash_manifest -i video_160x90_250k.webm \
-f webm_dash_manifest -i video_320x180_500k.webm \
-f webm_dash_manifest -i video_640x360_750k.webm \
-f webm_dash_manifest -i video_640x360_1000k.webm \
-f webm_dash_manifest -i video_1280x720_500k.webm \
-f webm_dash_manifest -i audio_128k.webm \
-c copy -map 0 -map 1 -map 2 -map 3 -map 4 -map 5 \
-f webm_dash_manifest \
-adaptation_sets "id=0,streams=0,1,2,3,4 id=1,streams=5" \
manifest.mpd
```
PS: 该样例借鉴自 [使用 DASH 播放自适应 WebM](http://wiki.webmproject.org/adaptive-streaming/instructions-to-playback-adaptive-webm-using-dash)
## 更多
FFmpeg 还有很多[其他用法](https://github.com/leandromoreira/digital_video_introduction/blob/master/encoding_pratical_examples.md#split-and-merge-smoothly)。我会利用 FFmpeg 结合 iMovie 为 YouTube 编辑视频，你当然也可以更专业地使用它。
# 笨办法学 FFmpeg libav
> Don't you wonder sometimes 'bout sound and vision?
> **David Robert Jones**
既然 [FFmpeg](#ffmpeg---command-line) 作为命令行工具对多媒体文件进行基本处理这么有效，那么我们如何在自己的程序里使用它呢？
FFmpeg 是由几个可以集成到程序里的[lib库](https://www.ffmpeg.org/doxygen/trunk/index.html)组成的。通常在安装FFmpeg时，会自动安装这些库。我们将这些库统一叫做 **FFmpeg libav**。
> 这个标题是对 Zed Shaw 的[笨办法学XX](https://learncodethehardway.org/)系列丛书的致敬，特别是笨办法学C语言。
## 章节0 - 臭名昭著的 hello world
这里说的 hello world 实际上不是在终端里输出 “hello world” :tongue:，而是**输出视频信息**，例如：格式、时长、分辨率、音频轨道，最后我们将**解码一些帧，并保存为图片**。
### FFmpeg libav 架构
在我们开始之前，我们需要先了解一下**FFmpeg libav 架构**的工作流程和各个组件之间的工作方式。
下面是一张视频解码的处理流程图：
首先，我们需要加载媒体文件到 [AVFormatContext](https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html) 组件（为便于理解，容器看作是文件格式即可）。这个过程并不是加载整个文件，它通常只是加载了文件头。
我们加载**容器的头部信息**后，就可以访问媒体文件流（流可以认为是基本的音频和视频数据）。每个流在 [`AVStream`](https://ffmpeg.org/doxygen/trunk/structAVStream.html) 组件中可用。
> 流是数据流的一个昵称
假设我们的视频文件包含两个流：一个是 [AAC](https://en.wikipedia.org/wiki/Advanced_Audio_Coding) 音频流，一个是 [H264（AVC）](https://en.wikipedia.org/wiki/H.264/MPEG-4_AVC)视频流。我们可以从每一个流中提取出被称为数据包的数据片段（切片），这些数据包将被加载到 [AVPacket](https://ffmpeg.org/doxygen/trunk/structAVPacket.html) 组件中。
**数据包中的数据仍然是被编码的**（被压缩），为了解码这些数据，我们需要将这些数据给到 [`AVCodec`](https://ffmpeg.org/doxygen/trunk/structAVCodec.html)。
`AVCodec` 将解码这些数据到 [`AVFrame`](https://ffmpeg.org/doxygen/trunk/structAVFrame.html)，最后我们将得到**解码后的帧**。注意，视频流和音频流共用此处理流程。
### 构建要求
由于有些人编译或者运行示例时会遇到许多[问题](https://github.com/leandromoreira/ffmpeg-libav-tutorial/issues?utf8=%E2%9C%93&q=is%3Aissue+is%3Aopen+compiling)，因此我们使用 `Docker` 来构建开发/运行环境。我们将使用一个 Big Buck Bunny 的视频来作为示例，如果你没有这个视频，运行 `make fetch_small_bunny_video` 来获取。
### 章节 0 - 代码一览
> 展示[代码](/0_hello_world.c)并执行。
>
> ```bash
> $ make run_hello
> ```
我们将跳过一些细节，不过不用担心，[代码](https://github.com/leandromoreira/ffmpeg-libav-tutorial/blob/master/0_hello_world.c)都在Github上维护。
我们首先为 [`AVFormatContext`](https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html) 分配内存，利用它可以获得相关格式（容器）的信息。
```c
AVFormatContext *pFormatContext = avformat_alloc_context();
```
我们将打开一个文件并读取文件的头信息，利用相关格式的简要信息填充 `AVFormatContext`（注意，编解码器通常不会被打开）。需要使用 [`avformat_open_input`](https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga31d601155e9035d5b0e7efedc894ee49) 函数，该函数需要 `AVFormatContext`、文件名和两个可选参数：[`AVInputFormat`](https://ffmpeg.org/doxygen/trunk/structAVInputFormat.html)（如果为NULL，FFmpeg将猜测格式）、[`AVDictionary`](https://ffmpeg.org/doxygen/trunk/structAVDictionary.html)（解封装参数）。
```c
avformat_open_input(&pFormatContext, filename, NULL, NULL);
```
可以输出视频的格式和时长：
```c
printf("Format %s, duration %lld us", pFormatContext->iformat->long_name, pFormatContext->duration);
```
为了访问数据流，我们需要从媒体文件中读取数据。需要利用函数 [`avformat_find_stream_info`](https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#gad42172e27cddafb81096939783b157bb)完成此步骤。`pFormatContext->nb_streams` 将获取所有的流信息，并且通过 `pFormatContext->streams[i]` 获取到指定的 `i` 数据流（[`AVStream`](https://ffmpeg.org/doxygen/trunk/structAVStream.html))。
```c
avformat_find_stream_info(pFormatContext, NULL);
```
可以使用循环来获取所有流数据：
```c
for (int i = 0; i < pFormatContext->nb_streams; i++)
{
//
}
```
针对每个流维护一个对应的 [`AVCodecParameters`](https://ffmpeg.org/doxygen/trunk/structAVCodecParameters.html)，该结构体描述了被编码流的各种属性。
```c
AVCodecParameters *pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
```
通过codec id和 [`avcodec_find_decoder`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga19a0ca553277f019dd5b0fec6e1f9dca) 函数可以找到对应已经注册的解码器，返回 [`AVCodec`](https://ffmpeg.org/doxygen/trunk/structAVCodec.html) 指针，该组件能让我们知道如何编解码这个流。
```c
AVCodec *pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);
```
现在可以输出一些编解码信息。
```c
// 用于视频和音频
if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO) {
printf("Video Codec: resolution %d x %d", pLocalCodecParameters->width, pLocalCodecParameters->height);
} else if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_AUDIO) {
printf("Audio Codec: %d channels, sample rate %d", pLocalCodecParameters->channels, pLocalCodecParameters->sample_rate);
}
// 通用
printf("\tCodec %s ID %d bit_rate %lld", pLocalCodec->long_name, pLocalCodec->id, pCodecParameters->bit_rate);
```
利用刚刚获取的 `AVCodec` 为 [`AVCodecContext`](https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html) 分配内存，它将维护解码/编码过程的上下文。 然后需要使用 [`avcodec_parameters_to_context`](https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#gac7b282f51540ca7a99416a3ba6ee0d16)和被编码流的参数(`AVCodecParameters`) 来填充 `AVCodecContext`。
完成上下文填充后，使用 [`avcodec_open2`](https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga11f785a188d7d9df71621001465b0f1d) 来打开解码器。
```c
AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
avcodec_parameters_to_context(pCodecContext, pCodecParameters);
avcodec_open2(pCodecContext, pCodec, NULL);
```
现在我们将从流中读取数据包并将它们解码为帧。但首先，需要为 [`AVPacket`](https://ffmpeg.org/doxygen/trunk/structAVPacket.html) 和 [`AVFrame`](https://ffmpeg.org/doxygen/trunk/structAVFrame.html) 分配内存。
```c
AVPacket *pPacket = av_packet_alloc();
AVFrame *pFrame = av_frame_alloc();
```
使用函数 [`av_read_frame`](https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga4fdb3084415a82e3810de6ee60e46a61) 读取帧数据来填充数据包。
```c
while (av_read_frame(pFormatContext, pPacket) >= 0) {
//...
}
```
使用函数 [`avcodec_send_packet`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga58bc4bf1e0ac59e27362597e467efff3) 来把**原始数据包**（未解压的帧）发送给解码器。
```c
avcodec_send_packet(pCodecContext, pPacket);
```
使用函数 [`avcodec_receive_frame`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga11e6542c4e66d3028668788a1a74217c) 从解码器接受原始数据帧（解压后的帧）。
```c
avcodec_receive_frame(pCodecContext, pFrame);
```
可以输出 frame 编号、[PTS](https://en.wikipedia.org/wiki/Presentation_timestamp)、DTS、[frame 类型](https://en.wikipedia.org/wiki/Video_compression_picture_types)等其他信息。
```c
printf(
"Frame %c (%d) pts %d dts %d key_frame %d [coded_picture_number %d, display_picture_number %d]",
av_get_picture_type_char(pFrame->pict_type),
pCodecContext->frame_number,
pFrame->pts,
pFrame->pkt_dts,
pFrame->key_frame,
pFrame->coded_picture_number,
pFrame->display_picture_number
);
```
最后，我们可以将解码后的帧保存为[灰度图](https://en.wikipedia.org/wiki/Netpbm#PGM_example)。处理过程非常简单，使用 `pFrame->data`，它的索引与 [Y, Cb 和 Cr 分量](https://en.wikipedia.org/wiki/YCbCr) 相关联。我们只选择 `0`（Y 分量）数据保存灰度图。
```c
save_gray_frame(pFrame->data[0], pFrame->linesize[0], pFrame->width, pFrame->height, frame_filename);
static void save_gray_frame(unsigned char *buf, int wrap, int xsize, int ysize, char *filename)
{
FILE *f;
int i;
f = fopen(filename,"w");
// 编写 pgm 格式所需的最小文件头
// portable graymap format -> https://en.wikipedia.org/wiki/Netpbm_format#PGM_example
fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
// 逐行写入
for (i = 0; i < ysize; i++)
fwrite(buf + i * wrap, 1, xsize, f);
fclose(f);
}
```
现在将得到一张2MB大小的灰度图：
## 章节 1 - 音视频同步
> **Be the player** - 一个年轻 JS 开发者开发的新 MSE 视频播放器。
在我们学习 [重新封装](#章节-2---重新封装) 之前，我们来谈谈timing（时机/时间点），或者说播放器如何知道在正确的时间来播放每一帧。
在上一个例子中，我们保存了一些帧：
当我们在设计一个播放器的时候，需要**以给定的速度播放每一帧**。否则，我们很难获得好的体验，因为在观看的过程中很可能播放得太快或者太慢。
因此我们需要引入一些机制来流畅地播放每一帧。每一帧都将拥有一个**播放时间戳**（PTS）。它是一个将**timebase**（时基，FFmpeg中一种特殊的时间度量单位，**timescale**可以认为是它的倒数）作为单位的递增数字。
我们来模仿几个场景，通过以下示例可以更迅速地理解。
例如 `fps=60/1` ， `timebase=1/60000`，PTS 将以 `timescale / fps = 1000` 进行递增，因此每一帧对应的 PTS 如下（假设开始为0）:
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 1000, PTS_TIME = PTS * timebase = 0.016`
* `frame=2, PTS = 2000, PTS_TIME = PTS * timebase = 0.033`
相同情况下，将 timebase 修改为 `1/60`：
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 1, PTS_TIME = PTS * timebase = 0.016`
* `frame=2, PTS = 2, PTS_TIME = PTS * timebase = 0.033`
* `frame=3, PTS = 3, PTS_TIME = PTS * timebase = 0.050`
如 `fps=25`，`timebase=1/75`，PTS 将以 `timescale / fps = 3` 进行递增，因此每一帧对应的 PTS 如下（假设开始为0）：
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 3, PTS_TIME = PTS * timebase = 0.04`
* `frame=2, PTS = 6, PTS_TIME = PTS * timebase = 0.08`
* `frame=3, PTS = 9, PTS_TIME = PTS * timebase = 0.12`
* ...
* `frame=24, PTS = 72, PTS_TIME = PTS * timebase = 0.96`
* ...
* `frame=4064, PTS = 12192, PTS_TIME = PTS * timebase = 162.56`
通过 `pts_time`， 我们可以找到一种渲染它和音频的 `pts_time` 或系统时钟进行同步的方式。FFmpeg libav 提供了获取这些信息的接口：
- fps = [`AVStream->avg_frame_rate`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#a946e1e9b89eeeae4cab8a833b482c1ad)
- tbr = [`AVStream->r_frame_rate`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#ad63fb11cc1415e278e09ddc676e8a1ad)
- tbn = [`AVStream->time_base`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#a9db755451f14e2bf590d4b85d82b32e6)
被保存的帧按照 DTS 顺序发送(frames：1,6,4,2,3,5)，按照 PTS 顺序播放(frames：1,2,3,4,5)。同时，我们可以发现B帧相对于P帧和I帧压缩率更高，更加节省空间。
```
LOG: AVStream->r_frame_rate 60/1
LOG: AVStream->time_base 1/60000
...
LOG: Frame 1 (type=I, size=153797 bytes) pts 6000 key_frame 1 [DTS 0]
LOG: Frame 2 (type=B, size=8117 bytes) pts 7000 key_frame 0 [DTS 3]
LOG: Frame 3 (type=B, size=8226 bytes) pts 8000 key_frame 0 [DTS 4]
LOG: Frame 4 (type=B, size=17699 bytes) pts 9000 key_frame 0 [DTS 2]
LOG: Frame 5 (type=B, size=6253 bytes) pts 10000 key_frame 0 [DTS 5]
LOG: Frame 6 (type=P, size=34992 bytes) pts 11000 key_frame 0 [DTS 1]
```
## 章节 2 - 重新封装
重新封装是将文件从一种格式转换为另一种格式。例如：我们可以非常容易地利用 FFmpeg 将 [MPEG-4](https://en.wikipedia.org/wiki/MPEG-4_Part_14) 格式的视频 转换成 [MPEG-TS](https://en.wikipedia.org/wiki/MPEG_transport_stream) 格式。
```bash
ffmpeg input.mp4 -c copy output.ts
```
以上命令将在不编解码的情况下（`-c copy`）来对 mp4 做解封装，然重新后封装为 `mpegts` 文件。如果不用 `-f` 参数来指定格式的话，ffmpeg 会根据文件扩展名来进行猜测。
FFmpeg 或 libav 的一般用法遵循以下模式/架构或工作流：
* **[协议层](https://ffmpeg.org/doxygen/trunk/protocols_8c.html)** - 接收一个输入（例如一个文件，也可以是 `rtmp` 或 `http`）
* **[格式层](https://ffmpeg.org/doxygen/trunk/group__libavf.html)** - 解封装数据内容，暴露出元数据和流信息
* **[编码层](https://ffmpeg.org/doxygen/trunk/group__libavc.html)** - 解码原始数据流 <sup>*可选*</sup>
* **[像素层](https://ffmpeg.org/doxygen/trunk/group__lavfi.html)** - 可以对原始帧应用一些 `filters`（例如调整大小）<sup>*可选*</sup>
* 然后反过来做相同的操作
* **[编码层](https://ffmpeg.org/doxygen/trunk/group__libavc.html)** - 编码（重新编码或者转码）原始帧<sup>*可选*</sup>
* **[格式层](https://ffmpeg.org/doxygen/trunk/group__libavf.html)** - 封装（或重新封装）原始数据流（压缩数据）
* **[协议层](https://ffmpeg.org/doxygen/trunk/protocols_8c.html)** - 将封装后数据输出 (另外的文件或远程服务器)
> 这张图的灵感来自 [Leixiaohua's](https://leixiaohua1020.github.io/#ffmpeg-development-examples) 和 [Slhck's](https://slhck.info/ffmpeg-encoding-course/#/9) 的作品。
现在我们将使用 libav 编写一个示例，完成与此命令行相同的效果: `ffmpeg input.mp4 -c copy output.ts`
我们读取一个输入文件（`input_format_context`)，并且将修改保存至输出（`output_format_context`)。
```c
AVFormatContext *input_format_context = NULL;
AVFormatContext *output_format_context = NULL;
```
通常我们的做法是分配内存并打开输入文件。对于这个示例，我们将打开一个输入文件并为一个输出文件分配内存。
```c
if ((ret = avformat_open_input(&input_format_context, in_filename, NULL, NULL)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
if ((ret = avformat_find_stream_info(input_format_context, NULL)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
avformat_alloc_output_context2(&output_format_context, NULL, NULL, out_filename);
if (!output_format_context) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
```
我们将重新封装视频、音频、字幕流，因此需要将用到的这些流存入一个数组中。
```c
number_of_streams = input_format_context->nb_streams;
streams_list = av_mallocz_array(number_of_streams, sizeof(*streams_list));
```
分配完所需要的内存之后，我们将遍历所有的流，然后利用 [avformat_new_stream](https://ffmpeg.org/doxygen/trunk/group__lavf__core.html#gadcb0fd3e507d9b58fe78f61f8ad39827) 为每一个流创建一个对应的输出流。注意，当前只需要针对视频、音频、字幕流进行处理。
```c
for (i = 0; i < input_format_context->nb_streams; i++) {
AVStream *out_stream;
AVStream *in_stream = input_format_context->streams[i];
AVCodecParameters *in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
streams_list[i] = -1;
continue;
}
streams_list[i] = stream_index++;
out_stream = avformat_new_stream(output_format_context, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
goto end;
}
}
```
现在，我们需要创建一个输出文件。
```c
if (!(output_format_context->oformat->flags & AVFMT_NOFILE)) {
ret = avio_open(&output_format_context->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
ret = avformat_write_header(output_format_context, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
```
完成上述操作之后，我们就可以将输入流逐个数据包复制到输出流。我们通过（`av_read_frame`）循环读取每一个数据包。对于每一数据包，我们都要重新计算 PTS 和 DTS，最终通过 `av_interleaved_write_frame` 写入输出格式的上下文。
```c
while (1) {
AVStream *in_stream, *out_stream;
ret = av_read_frame(input_format_context, &packet);
if (ret < 0)
break;
in_stream = input_format_context->streams[packet.stream_index];
if (packet.stream_index >= number_of_streams || streams_list[packet.stream_index] < 0) {
av_packet_unref(&packet);
continue;
}
packet.stream_index = streams_list[packet.stream_index];
out_stream = output_format_context->streams[packet.stream_index];
/* 赋值数据包 */
packet.pts = av_rescale_q_rnd(packet.pts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.dts = av_rescale_q_rnd(packet.dts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.duration = av_rescale_q(packet.duration, in_stream->time_base, out_stream->time_base);
// https://ffmpeg.org/doxygen/trunk/structAVPacket.html#ab5793d8195cf4789dfb3913b7a693903
packet.pos = -1;
//https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga37352ed2c63493c38219d935e71db6c1
ret = av_interleaved_write_frame(output_format_context, &packet);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&packet);
}
```
最后我们要使用函数 [av_write_trailer](https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga7f14007e7dc8f481f054b21614dfec13) 输出文件尾。
```c
av_write_trailer(output_format_context);
```
现在可以进行测试了，首先我们将文件从 MP4 转换成 MPEG-TS 格式。使用 libav 来代替命令行 `ffmpeg input.mp4 -c copy output.ts `的作用。
```bash
make run_remuxing_ts
```
它起作用了！！！不相信吗？我们可以使用 ffprobe 来检测一下：
```bash
ffprobe -i remuxed_small_bunny_1080p_60fps.ts
Input #0, mpegts, from 'remuxed_small_bunny_1080p_60fps.ts':
Duration: 00:00:10.03, start: 0.000000, bitrate: 2751 kb/s
Program 1
Metadata:
service_name : Service01
service_provider: FFmpeg
Stream #0:0[0x100]: Video: h264 (High) ([27][0][0][0] / 0x001B), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 60 fps, 60 tbr, 90k tbn, 120 tbc
Stream #0:1[0x101]: Audio: ac3 ([129][0][0][0] / 0x0081), 48000 Hz, 5.1(side), fltp, 320 kb/s
```
下图中总结了我们所做的工作，我们可以回顾一下之前关于[libav如何工作](https://github.com/leandromoreira/ffmpeg-libav-tutorial#ffmpeg-libav-architecture)的介绍。但图中也表明我们跳过了编解码的部分。
在结束本章之前，我想展示一下重新封装中的一个重要功能 — — **使用选项**。比如我们想要 [MPEG-DASH](https://developer.mozilla.org/en-US/docs/Web/Apps/Fundamentals/Audio_and_video_delivery/Setting_up_adaptive_streaming_media_sources#MPEG-DASH_Encoding) 格式的文件，需要使用 [fragmented mp4](https://stackoverflow.com/a/35180327)（有时称为fmp4）而不是 MPEG-TS 或者普通的 MPEG-4。
使用[命令行](https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API/Transcoding_assets_for_MSE#Fragmenting)可以简单地实现该功能：
```
ffmpeg -i non_fragmented.mp4 -movflags frag_keyframe+empty_moov+default_base_moof fragmented.mp4
```
使用 libav 进行实现也非常简单，只需要在写入输出头时（复制数据包之前），传递相应选项即可。
```c
AVDictionary* opts = NULL;
av_dict_set(&opts, "movflags", "frag_keyframe+empty_moov+default_base_moof", 0);
ret = avformat_write_header(output_format_context, &opts);
```
现在可以生成 fragmented mp4 文件：
```bash
make run_remuxing_fragmented_mp4
```
可以使用非常优秀的 [gpac/mp4box.js](https://gpac.github.io/mp4box.js/)，或者在线工具 [http://mp4parser.com/](http://mp4parser.com/) 来对比差异。首先加载普通mp4：
如你所见，`mdat` atom/box 是**存放实际音视频帧数据**的地方。现在我们加载 fragmented mp4，看看它是如何组织 `mdat` 的。
## 章节 3 - 转码
> #### 展示代码并执行
>
> ```bash
> $ make run_transcoding
> ```
>
> 我们将跳过一些细节，但是请不用担心：[代码](https://github.com/leandromoreira/ffmpeg-libav-tutorial/blob/master/3_transcoding.c)维护在 github。
在这一章，我们将用 C 写一个精简的转码器，使用 **FFmpeg/libav库**，特别是[libavcodec](https://ffmpeg.org/libavcodec.html)、libavformat 和 libavutil，将 H264 编码的视频转换为 H265。
> 简单回顾：[**AVFormatContext**](https://www.ffmpeg.org/doxygen/trunk/structAVFormatContext.html) 是多媒体文件格式的抽象（例如：MKV，MP4，Webm，TS）。 [**AVStream**](https://www.ffmpeg.org/doxygen/trunk/structAVStream.html) 代表给定格式的数据类型（例如：音频，视频，字幕，元数据）。 [**AVPacket**](https://www.ffmpeg.org/doxygen/trunk/structAVPacket.html) 是从 `AVStream` 获得的压缩数据的切片，可由 [**AVCodec**](https://www.ffmpeg.org/doxygen/trunk/structAVCodec.html)（例如av1，h264，vp9，hevc）解码，从而生成称为 [**AVFrame**](https://www.ffmpeg.org/doxygen/trunk/structAVFrame.html) 的原始数据。
### 转封装
我们将从简单的转封装操作开始，然后在此代码基础上进行构建，第一步需要**加载输入文件**。
```c
// 为 AVFormatContext 分配内存
avfc = avformat_alloc_context();
// 打开一个输入流并读取头信息
avformat_open_input(avfc, in_filename, NULL, NULL);
// 读取文件数据包以获取流信息
avformat_find_stream_info(avfc, NULL);
```
现在需要设置解码器，`AVFormatContext` 将使我们能够访问所有 `AVStream` 组件，获取它们的 `AVCodec` 并创建特定的 `AVCodecContext`，最后我们可以打开给定的编解码器进行解码。
> [**AVCodecContext**](https://www.ffmpeg.org/doxygen/trunk/structAVCodecContext.html) 保存相关媒体文件的数据包括：码率，帧率，采样率，通道、高度等等。
```c
for (int i = 0; i < avfc->nb_streams; i++)
{
AVStream *avs = avfc->streams[i];
AVCodec *avc = avcodec_find_decoder(avs->codecpar->codec_id);
AVCodecContext *avcc = avcodec_alloc_context3(*avc);
avcodec_parameters_to_context(*avcc, avs->codecpar);
avcodec_open2(*avcc, *avc, NULL);
}
```
现在我们需要准备输出文件，首先为 `AVFormatContext` **分配内存**。我们为为输出的格式创建**每一个流**。为了正确打包这些流，我们从解码器中**复制编解码参数**。
通过设置 `AV_CODEC_FLAG_GLOBAL_HEADER` 来告诉编码器可以使用这个全局头信息，最终打开输出文件写入文件头。
```c
avformat_alloc_output_context2(&encoder_avfc, NULL, NULL, out_filename);
AVStream *avs = avformat_new_stream(encoder_avfc, NULL);
avcodec_parameters_copy(avs->codecpar, decoder_avs->codecpar);
if (encoder_avfc->oformat->flags & AVFMT_GLOBALHEADER)
encoder_avfc->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
avio_open(&encoder_avfc->pb, encoder->filename, AVIO_FLAG_WRITE);
avformat_write_header(encoder->avfc, &muxer_opts);
```
我们从解码器获得 `AVPacket`，调整时间戳后写到输出文件。尽管 `av_interleaved_write_frame` 从函数名上来看是 “写入帧信息”，但我们实际是在存储数据包。最后通过写入文件尾来结束转封装操作。
```c
AVFrame *input_frame = av_frame_alloc();
AVPacket *input_packet = av_packet_alloc();
while (av_read_frame(decoder_avfc, input_packet) >= 0)
{
av_packet_rescale_ts(input_packet, decoder_video_avs->time_base, encoder_video_avs->time_base);
av_interleaved_write_frame(*avfc, input_packet) < 0));
}
av_write_trailer(encoder_avfc);
```
### 转码
前面的章节展示了一个转封装的程序，现在我们将添加对文件做编码的功能，具体是将视频从 `h264` 编码转换为 `h265`。
在我们设置解码器之后及准备输出文件之前，需要设置编码器。
* 使用 [`avformat_new_stream`](https://www.ffmpeg.org/doxygen/trunk/group__lavf__core.html#gadcb0fd3e507d9b58fe78f61f8ad39827) 和编码器创建 `AVStream`
* 使用名为 `libx265` 的 `AVCodec`，利用 [`avcodec_find_encoder_by_name`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__encoding.html#gaa614ffc38511c104bdff4a3afa086d37) 获取
* 利用 [`avcodec_alloc_context3`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#gae80afec6f26df6607eaacf39b561c315) 及编解码器创建 `AVCodecContext`
* 为编解码设置基础属性
* 打开编解码器，使用 [`avcodec_open2`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga11f785a188d7d9df71621001465b0f1d) 和 [`avcodec_parameters_from_context`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga0c7058f764778615e7978a1821ab3cfe) 将参数从上下文复制到流中
```c
AVRational input_framerate = av_guess_frame_rate(decoder_avfc, decoder_video_avs, NULL);
AVStream *video_avs = avformat_new_stream(encoder_avfc, NULL);
char *codec_name = "libx265";
char *codec_priv_key = "x265-params";
// 我们将对 x265 使用内置的参数
// 禁用场景切换并且把 GOP 调整为 60 帧
char *codec_priv_value = "keyint=60:min-keyint=60:scenecut=0";
AVCodec *video_avc = avcodec_find_encoder_by_name(codec_name);
AVCodecContext *video_avcc = avcodec_alloc_context3(video_avc);
// 编码参数
av_opt_set(sc->video_avcc->priv_data, codec_priv_key, codec_priv_value, 0);
video_avcc->height = decoder_ctx->height;
video_avcc->width = decoder_ctx->width;
video_avcc->pix_fmt = video_avc->pix_fmts[0];
// 控制码率
video_avcc->bit_rate = 2 * 1000 * 1000;
video_avcc->rc_buffer_size = 4 * 1000 * 1000;
video_avcc->rc_max_rate = 2 * 1000 * 1000;
video_avcc->rc_min_rate = 2.5 * 1000 * 1000;
// 时间基数
video_avcc->time_base = av_inv_q(input_framerate);
video_avs->time_base = sc->video_avcc->time_base;
avcodec_open2(sc->video_avcc, sc->video_avc, NULL);
avcodec_parameters_from_context(sc->video_avs->codecpar, sc->video_avcc);
```
为了视频流转码，我们需要拓展解码的步骤：
- 利用 [`avcodec_send_packet`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga58bc4bf1e0ac59e27362597e467efff3) 发送空的 `AVPacket` 给解码器
- 利用 [`avcodec_receive_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga11e6542c4e66d3028668788a1a74217c) 接收未压缩的 `AVFrame`
- 开始转码原始数据
- 使用 [`avcodec_send_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga9395cb802a5febf1f00df31497779169) 发送原始数据
- 基于编解码器和 `AVPacket`，利用 [`avcodec_receive_packet`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decodinghtml#ga5b8eff59cf259747cf0b31563e38ded6) 接受编码数据
- 设置时间戳，调用 [`av_packet_rescale_ts`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__packet.html#gae5c86e4d93f6e7aa62ef2c60763ea67e)
- 写入输出文件 [`av_interleaved_write_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga37352ed2c63493c38219d935e71db6c1)
```c
AVFrame *input_frame = av_frame_alloc();
AVPacket *input_packet = av_packet_alloc();
while (av_read_frame(decoder_avfc, input_packet) >= 0)
{
int response = avcodec_send_packet(decoder_video_avcc, input_packet);
while (response >= 0) {
response = avcodec_receive_frame(decoder_video_avcc, input_frame);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
return response;
}
if (response >= 0) {
encode(encoder_avfc, decoder_video_avs, encoder_video_avs, decoder_video_avcc, input_packet->stream_index);
}
av_frame_unref(input_frame);
}
av_packet_unref(input_packet);
}
av_write_trailer(encoder_avfc);
// used function
int encode(AVFormatContext *avfc, AVStream *dec_video_avs, AVStream *enc_video_avs, AVCodecContext video_avcc int index) {
AVPacket *output_packet = av_packet_alloc();
int response = avcodec_send_frame(video_avcc, input_frame);
while (response >= 0) {
response = avcodec_receive_packet(video_avcc, output_packet);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
return -1;
}
output_packet->stream_index = index;
output_packet->duration = enc_video_avs->time_base.den / enc_video_avs->time_base.num / dec_video_avs->avg_frame_rate.num * dec_video_avs->avg_frame_rate.den;
av_packet_rescale_ts(output_packet, dec_video_avs->time_base, enc_video_avs->time_base);
response = av_interleaved_write_frame(avfc, output_packet);
}
av_packet_unref(output_packet);
av_packet_free(&output_packet);
return 0;
}
```
我们将媒体流从 `h264` 编码转换为 `h265`，和预期的一样，`h265` 编码的文件相较于 h264 更小。本次[创建的程序](/3_transcoding.c)能够完成以下转换：
```c
/*
* H264 -> H265
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx265";
sp.codec_priv_key = "x265-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - fragmented MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
sp.muxer_opt_key = "movflags";
sp.muxer_opt_value = "frag_keyframe+empty_moov+delay_moov+default_base_moof";
/*
* H264 -> H264 (fixed gop)
* Audio -> AAC
* MP4 - MPEG-TS
*/
StreamingParams sp = {0};
sp.copy_audio = 0;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
sp.audio_codec = "aac";
sp.output_extension = ".ts";
/* WIP :P -> it's not playing on VLC, the final bit rate is huge
* H264 -> VP9
* Audio -> Vorbis
* MP4 - WebM
*/
//StreamingParams sp = {0};
//sp.copy_audio = 0;
//sp.copy_video = 0;
//sp.video_codec = "libvpx-vp9";
//sp.audio_codec = "libvorbis";
//sp.output_extension = ".webm";
```
> 老实说，完成这个教程[比我想象中的难](https://github.com/leandromoreira/ffmpeg-libav-tutorial/pull/54)，必须深入理解 [FFmpeg 命令行源码](https://github.com/leandromoreira/ffmpeg-libav-tutorial/pull/54#issuecomment-570746749)并进行大量测试。而且我想我肯定遗漏了一些细节，因为我必须强制执行 `force-cfr` 才能使 h264 正常工作，并且现在仍然会出现一些 warning 信息，例如 `warning messages (forced frame type (5) at 80 was changed to frame type (3))`。
================================================
FILE: README-es.md
================================================
[](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)
Estaba buscando un tutorial/libro que pudiera enseñarme como usar [FFmpeg](https://www.ffmpeg.org/) como una librería (alias libav) y encontré el tutorial de ["How to write a video player in less than 1k lines"](http://dranger.com/ffmpeg/). Desafortunadamente estaba obsoleto, así que decidí escribir el siguiente tutorial.
La mayoría del código aquí estará en C, **pero no te preocupes**: tu podrás entenderlo fácilmente y aplicarlo a tu lenguaje preferido. FFmpeg libav tiene montones de bindings para muchos lenguajes como [python](https://pyav.org/), [go](https://github.com/imkira/go-libav) e incluso si tu lenguaje no lo tiene, aún es posible darle soporte mediante `ffi` (aquí hay un ejemplo en [Lua](https://github.com/daurnimator/ffmpeg-lua-ffi/blob/master/init.lua)).
Empezaremos con una lección rápida de lo que es video, audio, códec y contenedor, entonces iremos a un curso rápido en como usar el comando `FFmpeg` y finalmente, escribiremos algo de código, siéntete libre de saltar directamente[ ](http://newmediarockstars.com/wp-content/uploads/2015/11/nintendo-direct-iwata.jpg "Secret Leandro´s Easter Egg")a la sección [Aprender FFmpeg libav de la manera difícil.](#learn-ffmpeg-libav-the-hard-way)
Algunas personas solían decir que la transmisión de video por internet era el futuro de la televisión tradicional, en cualquier caso, FFmpeg es algo que vale la pena estudiar.
__Tabla de Contenido__
* [Intro](#intro)
* [video - ¡lo que ves!](#video---¡lo-que-ves!)
* [audio - ¡lo que escuchas!](#audio---¡lo-que-escuchas!)
* [códec - comprimiendo datos](#códec---comprimiendo-datos)
* [Contenedor - un lugar cómodo para audio y video](#Contenedor---un-lugar-cómodo-para-audio-y-video)
* [FFmpeg - línea de comandos](#FFmpeg---línea-de-comandos)
* [FFmpeg herramienta de línea de comandos 101](#FFmpeg-herramienta-de-línea-de-comandos-101)
* [Operaciones de video comunes](#Operaciones-de-video-comunes)
* [Transcoding](#transcoding)
* [Transmuxing](#transmuxing)
* [Transrating](#transrating)
* [Transsizing](#transsizing)
* [Round Bonus: Transmisión adaptativa](#Round-Bonus-Transmisión-adaptativa)
* [Ve más allá](#Ve-más-allá)
* [Aprende FFmpeg libav de la manera difícil](#Aprende-FFmpeg-libav-de-la-manera-difícil)
* [Capítulo 0 - El infame hola mundo](#Capítulo-0---El-infame-hola-mundo)
* [Arquitectura de FFmpeg libav](#Arquitectura-de-FFmpeg-libav)
* [Capítulo 1 - timing](#Capítulo-1---sincronizando-audio-y-video)
* [Capítulo 2 - remuxing](#Capítulo-2---remuxing)
* [Capítulo 3 - transcoding](#Capítulo-3---transcoding)
# Intro
## Video - ¡lo que ves!
Si tu tienes una secuencia de imágenes en serie y las cambias a cierta frecuencia (digamos [24 imagenes por segundo](https://www.filmindependent.org/blog/hacking-film-24-frames-per-second/)), crearías una [ilusion de movimiento](https://en.wikipedia.org/wiki/Persistence_of_vision).
En resumen, esta es una muy básica idea detrás de un video: **una serie de imágenes / cuadros, corriendo a una velocidad dada**.
<img src="https://upload.wikimedia.org/wikipedia/commons/1/1f/Linnet_kineograph_1886.jpg" title="flip book" height="280"></img>
Ilustración Zeitgenössische (1886)
## Audio - ¡lo que escuchas!
Aunque un video mudo puede expresar una variedad de sentimientos, el agregarle sonido lo vuelve una experiencia mas placentera.
El sonido es la vibración que se propaga como una onda de presión, a través del aire o de cualquier otro medio de transmisión, como un gas, líquido o sólido.
> En un sistema de audio digital, el micrófono convierte sonido a una señal eléctrica analógica, después un convertidor analógico-a-digital (ADC) — típicamente se usa [pulse-code modulation (PCM)](https://en.wikipedia.org/wiki/Pulse-code_modulation) - que convierte la señal analógica en una señal digital.
>[Fuente](https://commons.wikimedia.org/wiki/File:CPT-Sound-ADC-DAC.svg)
## Códec - comprimiendo datos
> CODEC es un circuito electrónico o software que **comprime o descomprime audio/video digital.**
Convierte audio/video digital en bruto (raw) a un formato comprimido o vice versa.
> https://en.wikipedia.org/wiki/Video_codec
Pero si deseamos empaquetar millones de imágenes dentro de un solo archivo y generamos una película, entonces terminaríamos con un archivo enorme. Veamos las matemáticas:
Supongamos que creamos el video con una resolución de `1080 x 1920` (altura x anchura) y que utilizaremos `3 bytes` por píxel (la unidad mínima en una pantalla) para codificar el color (o [un color de 24 bit](https://en.wikipedia.org/wiki/Color_depth#True_color_.2824-bit.29), que nos da 16,777,216 diferentes colores) y este video se reproduce a `24 cuadros por segundo` entonces serán `30 minutos` de duración.
```c
toppf = 1080 * 1920 //total_de_pixeles_por_cuadro
cpp = 3 //costo_por_pixel
tis = 30 * 60 //tiempo_en_segundos
fps = 24 //cuadros_por_segundo
almacenamiento_requerido = tis * fps * toppf * cpp
```
¡Este video requeriría aproximadamente `250.28GB` de almacenamiento o `1.19 Gbps` de banda ancha! Es por esto que necesitamos hacer uso de un [CODEC](https://github.com/leandromoreira/digital_video_introduction#how-does-a-video-codec-work).
## Contenedor - un lugar cómodo para audio y video
> Un contenedor o formato de envoltura es un formato de meta-archivos cuyas especificaciones describen que diferentes elementos de datos y metadatos coexisten en un mismo archivo de computadora.
>
> https://en.wikipedia.org/wiki/Digital_container_format
Es un **sólo archivo que contiene todos los streams (en su mayoría de audio y video) y también provee una sincronización y metadatos generales**, como un titulo, resolución, etc.
Usualmente, podemos inferir el formato de un archivo al ver su extensión: por ejemplo un `video.webm` es probablemente un video usando el contenedor [`webm`](https://www.webmproject.org/).
# FFmpeg - línea de comandos
> Una completa solución multi-plataforma para grabar, convertir y transmitir audio y video.
Para trabajar con multimedia podemos hacer uso de esta MARAVILLOSA herramienta/librería llamada [FFmpeg](https://www.ffmpeg.org/). Existen posibilidades de que ya la conoces/usas, directa o indirectamente (¿usas [Chrome](https://www.chromium.org/developers/design-documents/video)?).
Éste tiene una programa para línea de comandos llamado `ffmpeg`,un binario muy simple y poderoso. Por ejemplo, puedes convertir desde un contenedor `mp4`a uno `avi` solo escribiendo el siguiente comando:
```bash
$ ffmpeg -i input.mp4 output.avi
```
Acabamos de hacer **remuxing** (remultiplexación) aquí, el cual consiste convertir de un contenedor a otro. Técnicamente, FFmpeg puede también hacer un transcoding, pero hablaremos de eso después.
## FFmpeg herramienta de línea de comandos 101
FFmpeg posee [documentación](https://www.ffmpeg.org/ffmpeg.html) que hace un gran trabajo explicando como funciona.
```bash
# tambien puedes ver la documentacion usando la linea de comandos
ffmpeg -h full | grep -A 10 -B 10 avoid_negative_ts
```
Para ser breves, el comando de línea para FFmpeg espera el siguiente formato de argumentos para realizar sus acciones `ffmpeg {1} {2} -i {3} {4} {5}`, donde:
1. Opciones globales
2. Opciones de archivo de entrada
3. URL de entrada
4. Opciones de archivo de salida
5. URL de salida
Las partes 2,3,4 y 5 pueden ser tantas como sean necesarias.
Es mas fácil entender este formato de argumentos en acción:
``` bash
# ADVERTENCIA: este archivo pesa alrededor de 300MB
$ wget -O bunny_1080p_60fps.mp4 http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_1080p_60fps_normal.mp4
$ ffmpeg \
-y \ # opciones globales
-c:a libfdk_aac \ # opciones de entrada
-i bunny_1080p_60fps.mp4 \ # url de entrada
-c:v libvpx-vp9 -c:a libvorbis \ # opciones de salida
bunny_1080p_60fps_vp9.webm # url de salida
```
Este comando toma el archivo de entrada `mp4` que contiene 2 streams (un audio codificado con el CODEC `aac` y el video codificado usando el CODEC `h264`) y va a convertirlo a `webm`, cambiando también los CODECs de audio y video.
Podríamos simplificar el comando de arriba pero tenemos que saber que FFmpeg adoptará o supondrá los valores predeterminados por ti.
Por ejemplo, cuando tu introduces `ffmpeg -i input.avi output.mp4` ¿qué CODEC para audio/video va a usar para producir `output.mp4`?
Werner Robitza escribió un [tutorial acerca de codificacion y edicion con FFmpeg](http://slhck.info/ffmpeg-encoding-course/#/) que se tiene que leer/realizar para una mejor comprensión.
# Operaciones de video comunes
Cuando trabajamos con audio/video nosotros usualmente hacemos una serie de tareas con archivos multimedia.
## Transcoding
**¿Qué?** el acto de convertir uno de los flujos de transmisión (audio o video) de un CODEC a otro.
**¿Por qué?** en ocasiones algunos dispositivos (TVs, smartphones, consolas, etc.) no soportan X pero si Y y nuevos CODECs proveen mejor tasa de compresión.
**¿Cómo?** convirtiendo un video `H264` (AVC) a un `H265` (HEVC).
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c:v libx265 \
bunny_1080p_60fps_h265.mp4
```
## Transmuxing
**¿Qué?** el acto de convertir un formato (contenedor) a otro.
**¿Por qué?** en ocasiones algunos dispositivos (TVs, smartphones, consolas, etc.) no soportan X pero si Y y a veces nuevos contenedores proveen características modernas que son requeridas.
**¿Cómo?** convirtiendo de `mp4` a `webm`.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c copy \ # con esto se dice a ffmpeg que se salte la codificación
bunny_1080p_60fps.webm
```
## Transrating
**¿Qué?** el acto de cambiar la tasa de bits, o produciendo otras presentaciones.
**¿Por qué?** las personas intentaran ver tu video usando una conexión `2G` (edge) en un smartphone de baja gama o una conexión por `fibra` a Internet en los televisores a 4K, por lo tanto tu deberías ofrecer mas de una presentación para el mismo video a diferente tasa de bits.
**¿Cómo?** produciendo una presentación con una tasa de bits entre 964K y 3856K.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-minrate 964K -maxrate 3856K -bufsize 2000K \
bunny_1080p_60fps_transrating_964_3856.mp4
```
Usualmente vamos a estar usando transrating con transsizing. Werner Robitza escribió otra [serie de posts acerca del control de tasa para FFmpeg](http://slhck.info/posts/) que debes leer/realizar.
# Transsizing
**¿Qué?** el acto de convertir desde una resolución a otro. Como antes se dijo, transsizing es usualmente usado con transrating.
**¿Por qué?** las razones serian las mismas que las de transrating.
**¿Cómo?** convirtiendo de una resolución de `1080p` a `480p`.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-vf scale=480:-1 \
bunny_1080p_60fps_transsizing_480.mp4
```
## Round Bonus: Transmisión adaptativa
**¿Qué?** el acto de producir varias resoluciones (tasas de bits) y dividir el contenido en porciones y después servirlos mediante http.
**¿Por qué?** para proveer un contenido flexible que puede ser observado en un smartphone de baja gama o en una televisión en 4K, también es fácil de escalar y desplegar pero puede agregar latencia.
**¿Cómo?** creando un WebM adaptativo usando DASH.
```bash
# emisiones de video
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 160x90 -b:v 250k -keyint_min 150 -g 150 -an -f webm -dash 1 video_160x90_250k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 320x180 -b:v 500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_320x180_500k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 750k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_750k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 1000k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_1000k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 1280x720 -b:v 1500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_1280x720_1500k.webm
# emisiones de audio
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:a libvorbis -b:a 128k -vn -f webm -dash 1 audio_128k.webm
# el manifiesto DASH
$ ffmpeg \
-f webm_dash_manifest -i video_160x90_250k.webm \
-f webm_dash_manifest -i video_320x180_500k.webm \
-f webm_dash_manifest -i video_640x360_750k.webm \
-f webm_dash_manifest -i video_640x360_1000k.webm \
-f webm_dash_manifest -i video_1280x720_500k.webm \
-f webm_dash_manifest -i audio_128k.webm \
-c copy -map 0 -map 1 -map 2 -map 3 -map 4 -map 5 \
-f webm_dash_manifest \
-adaptation_sets "id=0,streams=0,1,2,3,4 id=1,streams=5" \
manifest.mpd
```
PD: Tomé este ejemplo desde las [Instrucciones de la reproducción de WebM adaptativo usando DASH](http://wiki.webmproject.org/adaptive-streaming/instructions-to-playback-adaptive-webm-using-dash)
## Ve más allá
Hay [muchos y bastantes mas usos para FFmpeg](https://github.com/leandromoreira/digital_video_introduction/blob/master/encoding_pratical_examples.md#split-and-merge-smoothly).
Yo lo uso en conjunto con *iMovie* para producir/editar algunos videos de Youtube y tu ciertamente puedes usarle de manera profesional.
# Aprende FFmpeg libav de la manera difícil
> ¿A veces no te preguntas acerca de el sonido y la visión?
> **David Robert Jones**
Sabiendo que [FFmpeg]() es tan útil como una herramienta de línea de comandos para realizar tareas esenciales en archivos multimedia, pero ¿cómo se pueden usar en nuestros programas?
FFmpeg está [compuesto de multiples librerías](https://www.ffmpeg.org/doxygen/trunk/index.html) que pueden ser integradas en nuestros propios programas.
Usualmente, cuando instalas FFmpeg, se instalan automáticamente todas esas librerías. De aquí en adelante, me voy a referir a estas set de librerías como **FFmpeg libav**.
> Este título es un homenaje a las series de Zed Shaw [Aprende X de la manera difícil](https://learncodethehardway.org/), particularmente a su libro Aprende C de la manera difícil.
## Capítulo 0 - El infame hola mundo
Éste hola mundo, de hecho, no enseñara el mensaje de `"hola mundo"` en la terminal :tongue: En su lugar, vamos a **imprimir la información acerca del video**. cosas como su formato (contenedor), duración, resolución, canales de audio y, al final, vamos a **decodificar algunos cuadros y a guardarlos como archivos de imagen**.
### Arquitectura de FFmpeg libav
Pero antes de que podamos empezar a codificar, vamos a aprender como la **Arquitectura de FFmpeg libav** funciona y como sus componentes se comunican con otros.
Aquí hay un diagrama del proceso de decodificación de video:
Primero, vas a necesitar cargar tu archivo multimedia dentro de un componente llamado [`AVFormatContext`](https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html)(el contenedor de video es también conocido como formato).
De hecho, no se carga todo el archivo: usualmente solo lee el encabezado (header) del mismo.
Una vez cargamos el **encabezado de nuestro contenedor** en su forma mínima, nosotros podemos acceder a sus streams (piensa de ellos como datos rudimentarios de audio y video).
Cada stream estará disponible en un componente llamado [`AVStream`](https://ffmpeg.org/doxygen/trunk/structAVStream.html).
> Stream es un nombre elegante para un flujo continuo de datos.
Supongamos que nuestro video tiene dos streams: un audio codificado con [AAC CODEC](https://en.wikipedia.org/wiki/Advanced_Audio_Coding) y un video codificado con [H264 (AVC) CODEC](https://en.wikipedia.org/wiki/H.264/MPEG-4_AVC). Por cada stream, nosotros podemos extraer **piezas de datos** llamados paquetes, los que serán cargados en componentes llamados [`AVPacket`](https://ffmpeg.org/doxygen/trunk/structAVPacket.html).
Los **datos dentro de los paquetes siguen codificados** (comprimidos) y para decodificar los paquetes, necesitamos pasarlos a un [`AVCodec`](https://ffmpeg.org/doxygen/trunk/structAVCodec.html) específico.
El `AVCodec` va a decodificarlos dentro de un [`AVFrame`](https://ffmpeg.org/doxygen/trunk/structAVFrame.html) y finalmente, este componente nos da **el cuadro (frame) descomprimido**. Hay que poner atención en que se usa la misma terminología o mismo proceso es usado de igual manera por un stream de audio y video.
### Requerimientos
Debido a que algunas personas estuvieron [enfrentandose a varios problemas durante la compilacion o ejecucion de los ejemplos](https://github.com/leandromoreira/ffmpeg-libav-tutorial/issues?utf8=%E2%9C%93&q=is%3Aissue+is%3Aopen+compiling) **vamos a usar [`Docker`](https://docs.docker.com/install/) como nuestro entorno de desarrollo/ejecución, también haremos uso del video: "The Big Buck Bunny", que en caso de no contar con él de manera local, solo ejecuta el comando `make fetch_small_bunny_video`.
### Capítulo 0 - el código, paso a paso
> #### TLDR; enséname el [codigo](/0_hello_world.c) y ejecuta.
> ```bash
> $ make run_hello
> ```
Vamos a saltarnos unos detalles, pero no te preocupes: el [código fuente esta disponible en GitHub](/0_hello_world.c).
Vamos a acomodar (allocate) la memoria para el componente [`AVFormatContext`](http://ffmpeg.org/doxygen/trunk/structAVFormatContext.html), el cual va a contener la información acerca del formato (contenedor).
```c
AVFormatContext *pFormatContext = avformat_alloc_context();
```
Ahora vamos a abrir el archivo y leer su encabezado para llenar el `AVFormatContext` con la información mínima acerca del formato (note que usualmente los códecs no son abiertos).
La función usada para hacer esto es [`avformat_open_input`](http://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga31d601155e9035d5b0e7efedc894ee49). Éste espera un `AVFormatContext`, un archivo (`filename`) y dos argumentos opcionales: el [`AVInputFormat`](https://ffmpeg.org/doxygen/trunk/structAVInputFormat.html) (si tu colocas un `NULL`, FFmpeg va a suponer el formato por ti) y el [`AVDictionary`](https://ffmpeg.org/doxygen/trunk/structAVDictionary.html) (el cual son las opciones para el desmultiplexador).
```c
avformat_open_input(&pFormatContext, filename, NULL, NULL);
```
Podemos imprimir el nombre del formato y la duración media:
```c
printf("Format %s, duration %lld us", pFormatContext->iformat->long_name, pFormatContext->duration);
```
Para acceder a los `streams`, necesitamos leer los datos del archivo. La función [`avformat_find_stream_info`](https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#gad42172e27cddafb81096939783b157bb) hace eso.
Ahora, el `pFormatContext->nb_streams` contendrá el numero de streams y el `pFormatContext->streams[i]` nos dará el stream `i`([`AVStream`](https://ffmpeg.org/doxygen/trunk/structAVStream.html)).
```c
avformat_find_stream_info(pFormatContext, NULL);
```
Ahora, navegaremos por todos los streams.
```c
for (int i = 0; i < pFormatContext->nb_streams; i++)
{
//
}
```
Por cada stream, vamos a mantener los [`AVCodecParameters`](https://ffmpeg.org/doxygen/trunk/structAVCodecParameters.html), los cuales describen las propiedades de un códec usado por el stream `i`.
```c
AVCodecParameters *pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
```
Ya con las propiedades del códec, podremos ver el CODEC apropiado solicitándolo a la función [`avcodec_find_decoder`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga19a0ca553277f019dd5b0fec6e1f9dca) y encontrar el decodificador para un códec id y regresar un [`AVCodec`](http://ffmpeg.org/doxygen/trunk/structAVCodec.html), el componente que conoce como **CO**dificar y **DEC**odificar el stream.
```c
AVCodec *pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);
```
Ahora, vamos a imprimir la información acerca de los códecs.
```c
// especifico para video y audio
if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO) {
printf("Video Codec: resolution %d x %d", pLocalCodecParameters->width, pLocalCodecParameters->height);
} else if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_AUDIO) {
printf("Audio Codec: %d channels, sample rate %d", pLocalCodecParameters->channels, pLocalCodecParameters->sample_rate);
}
// general
printf("\tCodec %s ID %d bit_rate %lld", pLocalCodec->long_name, pLocalCodec->id, pLocalCodecParameters->bit_rate);
```
Con el códec, podemos acomodar memoria para el [`AVCodecContext`](https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html), el cual va a contener el contexto para nuestro proceso de decodificación/codificación, pero antes debemos llenar el contexto del códec con los parámetros CODEC; esto lo hacemos con [`avcodec_parameters_to_context`](https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#gac7b282f51540ca7a99416a3ba6ee0d16).
Una vez llenado el contexto del códec, necesitamos abrirlo. Entonces tenemos que llamar a la función [`avcodec_open2`](https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga11f785a188d7d9df71621001465b0f1d) y después de ello, lo podremos usar.
```c
AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
avcodec_parameters_to_context(pCodecContext, pCodecParameters);
avcodec_open2(pCodecContext, pCodec, NULL);
```
Ahora, vamos a leer los paquetes desde el stream y decodificarlos dentro de cuadros, vamos a necesitar acomodar la memoria para ambos componentes, el [`AVPacket`](https://ffmpeg.org/doxygen/trunk/structAVPacket.html) y [`AVFrame`](https://ffmpeg.org/doxygen/trunk/structAVFrame.html).
```c
AVPacket *pPacket = av_packet_alloc();
AVFrame *pFrame = av_frame_alloc();
```
Hay que sustraer nuestros paquetes desde los streams con la función [`av_read_frame`](https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga4fdb3084415a82e3810de6ee60e46a61) mientras contenga paquetes.
```c
while (av_read_frame(pFormatContext, pPacket) >= 0) {
//...
}
```
Ahora, hay que **mandar los paquetes de datos en bruto** (cuadro comprimido) al decodificador, mediante el contexto del códec, usando la función [`avcodec_send_packet`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga58bc4bf1e0ac59e27362597e467efff3).
```c
avcodec_send_packet(pCodecContext, pPacket);
```
Y vamos a **recibir el cuadro de datos en bruto** (cuadro descomprimido) desde el decodificador, mediante el mismo contexto del códec, usando la función [`avcodec_receive_frame`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga11e6542c4e66d3028668788a1a74217c).
```c
avcodec_receive_frame(pCodecContext, pFrame);
```
Podemos imprimir el numero de cuadro, el [PTS](https://en.wikipedia.org/wiki/Presentation_timestamp), DTS, [frame type](https://en.wikipedia.org/wiki/Video_compression_picture_types), etc.
```c
printf(
"Frame %c (%d) pts %d dts %d key_frame %d [coded_picture_number %d, display_picture_number %d]",
av_get_picture_type_char(pFrame->pict_type),
pCodecContext->frame_number,
pFrame->pts,
pFrame->pkt_dts,
pFrame->key_frame,
pFrame->coded_picture_number,
pFrame->display_picture_number
);
```
Finalmente, podemos guardar nuestro cuadro decodificado dentro de una [imagen gris simple](https://en.wikipedia.org/wiki/Netpbm_format#PGM_example). El proceso es muy sencillo, nosotros usaremos el `pFrame->data,`, donde el índice esta relacionado con los [planos Y, Cb y Cr](https://en.wikipedia.org/wiki/YCbCr), nosotros solo seleccionamos `0` (Y) para guardar nuestra imagen gris.
```c
save_gray_frame(pFrame->data[0], pFrame->linesize[0], pFrame->width, pFrame->height, frame_filename);
static void save_gray_frame(unsigned char *buf, int wrap, int xsize, int ysize, char *filename)
{
FILE *f;
int i;
f = fopen(filename,"w");
// escribiendo el encabezado mínimo para un formato de un archivo pgm
// portable graymap format -> https://en.wikipedia.org/wiki/Netpbm_format#PGM_example
fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
// escribiendo linea por linea
for (i = 0; i < ysize; i++)
fwrite(buf + i * wrap, 1, xsize, f);
fclose(f);
}
```
¡Y voilà! Ahora nosotros tenemos una imagen gris a escala de 2MB:
## Capítulo 1 - sincronizando audio y video
> **Sé el jugador** - un joven desarrollador de JS escribiendo un nuevo reproductor de video MSE.
Antes de que nos movamos a [codificar un ejemplo de transcoding](#capitulo2-transcoding) ahora vamos a hablar acerca de la **sincronización (timing)**, o como el reproductor de video lo conoce, el tiempo correcto para reproducir un cuadro.
En el ultimo ejemplo, hemos guardado algunos cuadros que pueden verse aquí:
Cuando nosotros estamos diseñando un reproductor de video, nosotros necesitamos **reproducir cada cuadro a su debido tiempo**, de otra forma sería difícil ver un video de manera agradable, porque se estaría reproduciendo demasiado rápido o lento.
Por lo tanto, necesitamos introducir algo de lógica para reproducir sin complicaciones cada cuadro. Para ello, cada cuadro tiene un **Timestamp de presentación** (PTS) el cual tiene un numero creciente factorizado en un **timebase (tiempo base)**, que es un numero racional (donde el denominador es conocido como **timescale**) divisible por el **frame rate (fps)**.
Es fácil entender cuando vemos algunos ejemplos, vamos a simular varios escenarios.
Para un `fps=60/1` y `timebase=1/60000` cada PTS se incrementará `timescale / fps = 1000`, por lo tanto el **PTS en tiempo real** por cada cuadro podría ser (suponiendo que empieza en 0):
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 1000, PTS_TIME = PTS * timebase = 0.016`
* `frame=2, PTS = 2000, PTS_TIME = PTS * timebase = 0.033`
Para casi el mismo escenario pero con un timebase igual a `1/60`.
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 1, PTS_TIME = PTS * timebase = 0.016`
* `frame=2, PTS = 2, PTS_TIME = PTS * timebase = 0.033`
* `frame=3, PTS = 3, PTS_TIME = PTS * timebase = 0.050`
Para un `fps=25/1` y `timebase=1/75` cada PTS se incrementará `timescale / fps = 3` y el tiempo PTS podría ser:
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 3, PTS_TIME = PTS * timebase = 0.04`
* `frame=2, PTS = 6, PTS_TIME = PTS * timebase = 0.08`
* `frame=3, PTS = 9, PTS_TIME = PTS * timebase = 0.12`
* ...
* `frame=24, PTS = 72, PTS_TIME = PTS * timebase = 0.96`
* ...
* `frame=4064, PTS = 12192, PTS_TIME = PTS * timebase = 162.56`
Ahora con el `pts_time` podemos encontrar una forma de renderizarlo, esto es sincronizándolo con el audio `pts_time` o con el reloj del sistema. El FFmpeg libav provee esa información a través de su API:
- fps = [`AVStream->avg_frame_rate`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#a946e1e9b89eeeae4cab8a833b482c1ad)
- tbr = [`AVStream->r_frame_rate`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#ad63fb11cc1415e278e09ddc676e8a1ad)
- tbn = [`AVStream->time_base`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#a9db755451f14e2bf590d4b85d82b32e6)
Por pura curiosidad, observa que los cuadros fueron guardados en el orden DTS (cuadros: 1, 6, 4, 2, 3, 5) pero reproducidos en un orden PTS (cuadros: 1, 2, 3, 4, 5). Además, nota que poco costo tienen los cuadros-B en comparación con los cuadros-P o cuadros-I.
```
LOG: AVStream->r_frame_rate 60/1
LOG: AVStream->time_base 1/60000
...
LOG: Frame 1 (type=I, size=153797 bytes) pts 6000 key_frame 1 [DTS 0]
LOG: Frame 2 (type=B, size=8117 bytes) pts 7000 key_frame 0 [DTS 3]
LOG: Frame 3 (type=B, size=8226 bytes) pts 8000 key_frame 0 [DTS 4]
LOG: Frame 4 (type=B, size=17699 bytes) pts 9000 key_frame 0 [DTS 2]
LOG: Frame 5 (type=B, size=6253 bytes) pts 10000 key_frame 0 [DTS 5]
LOG: Frame 6 (type=P, size=34992 bytes) pts 11000 key_frame 0 [DTS 1]
```
## Capítulo 2 - remuxing
Remuxing (remultiplexar) es el acto de cambiar de un formato (contenedor) a otro, por ejemplo, nosotros podemos cambiar un video [MPEG-4](https://en.wikipedia.org/wiki/MPEG-4_Part_14) a uno [MPEG-TS](https://en.wikipedia.org/wiki/MPEG_transport_stream) sin muchos problemas usando FFmpeg:
```bash
ffmpeg input.mp4 -c copy output.ts
```
Esto va a desmultiplexar (demux) el mp4 pero no lo va a decodificar o codificar (`-c copy`) y al final, esto lo multiplexa (mux) dentro de un archivo `mpegts`. Si tu no provees el formato `-f`, entonces FFmpeg va a tener que determinarlo en base de la extensión del archivo.
El uso general de FFmpeg o libav sigue un patrón/arquitectura o flujo de trabajo:
* **[protocol layer](https://ffmpeg.org/doxygen/trunk/protocols_8c.html)** - este acepta una entrada (`input`) (un archivo o `file`, o por ejemplo la entrada también podría ser `rtmp` o `HTTP`)
* **[format layer](https://ffmpeg.org/doxygen/trunk/group__libavf.html)** - este desmultiplexa (`demuxes`) su contenido, revelando, en mayor parte, los metadatos y sus streams
* **[codec layer](https://ffmpeg.org/doxygen/trunk/group__libavc.html)** - esto decodifica (`decodes`) sus datos de stream comprimidos<sup>*opcional*</sup>
* **[pixel layer](https://ffmpeg.org/doxygen/trunk/group__lavfi.html)** - aquí también se pueden aplicar filtros (`filters`) a los cuadros en bruto (como resizing)<sup>*optional*</sup>
* y después lo hace en el sentido contrario.
* **[codec layer](https://ffmpeg.org/doxygen/trunk/group__libavc.html)** - esto codifica (`encodes`) (o re-encodifica (`re-encodes`) o incluso transcodifican o `transcodes`) los cuadros en bruto<sup>*opcional*</sup>
* **[format layer](https://ffmpeg.org/doxygen/trunk/group__libavf.html)** - esto multiplexa (`muxes`) (o remultiplexa (`remuxes`) los streams en bruto (los datos comprimidos)
* **[protocol layer](https://ffmpeg.org/doxygen/trunk/protocols_8c.html)** - y finalmente los datos multiplexados son enviados a una salida o `output` (otro archivo o quizás, un servidor remoto en la red)
> Esta imagen está fuertemente inspirada por los trabajos de [Leixiaohua](http://leixiaohua1020.github.io/#ffmpeg-development-examples) y [Slhck](https://slhck.info/ffmpeg-encoding-course/#/9).
Ahora vamos a codificar un ejemplo usando libav para proveer el mismo efecto que en `ffmpeg input.mp4 -c copy output.ts`.
```c
AVFormatContext *input_format_context = NULL;
AVFormatContext *output_format_context = NULL;
```
Como en los ejemplos anteriores, empezaremos por acomodar la memoria y abrir el formato de la entrada. Para este caso en específico, vamos a abrir un archivo de entrada y acomodar memora para un archivo de salida.
```c
if ((ret = avformat_open_input(&input_format_context, in_filename, NULL, NULL)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
if ((ret = avformat_find_stream_info(input_format_context, NULL)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
avformat_alloc_output_context2(&output_format_context, NULL, NULL, out_filename);
if (!output_format_context) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
```
Vamos a remultiplexar solamente los tipos de streams de video, audio y subtítulos, así que vamos a obtener que streams vamos a estar usando dentro de un arreglo de índices.
```c
number_of_streams = input_format_context->nb_streams;
streams_list = av_mallocz_array(number_of_streams, sizeof(*streams_list));
```
Después de haber acomodado la memoria requerida, vamos a navegar por todos los streams, por cada uno necesitaremos crear un nuevo stream dentro de nuestro contexto de formato de salida, usando la función [avformat_new_stream](https://ffmpeg.org/doxygen/trunk/group__lavf__core.html#gadcb0fd3e507d9b58fe78f61f8ad39827). Nota como estamos marcando todos los streams que no son video, audio o subtitulo, así que podemos saltarlos para luego.
```c
for (i = 0; i < input_format_context->nb_streams; i++) {
AVStream *out_stream;
AVStream *in_stream = input_format_context->streams[i];
AVCodecParameters *in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
streams_list[i] = -1;
continue;
}
streams_list[i] = stream_index++;
out_stream = avformat_new_stream(output_format_context, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
goto end;
}
}
```
Ahora, podemos crear un archivo de salida.
```c
if (!(output_format_context->oformat->flags & AVFMT_NOFILE)) {
ret = avio_open(&output_format_context->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
ret = avformat_write_header(output_format_context, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
```
Después, podemos copiar los streams, paquete por paquete, desde nuestros streams de entrada a los de salida. Continuaremos navegando por los paquetes, mientras estos sigan llegando (`av_read_frame`), por cada paquete vamos a necesitar recalcular el PTS y el DTS, para finalmente escribirlo (`av_interleaved_write_frame`) a nuestro contexto de formato de salida.
```c
while (1) {
AVStream *in_stream, *out_stream;
ret = av_read_frame(input_format_context, &packet);
if (ret < 0)
break;
in_stream = input_format_context->streams[packet.stream_index];
if (packet.stream_index >= number_of_streams || streams_list[packet.stream_index] < 0) {
av_packet_unref(&packet);
continue;
}
packet.stream_index = streams_list[packet.stream_index];
out_stream = output_format_context->streams[packet.stream_index];
/* copiar paquete */
packet.pts = av_rescale_q_rnd(packet.pts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.dts = av_rescale_q_rnd(packet.dts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.duration = av_rescale_q(packet.duration, in_stream->time_base, out_stream->time_base);
// https://ffmpeg.org/doxygen/trunk/structAVPacket.html#ab5793d8195cf4789dfb3913b7a693903
packet.pos = -1;
//https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga37352ed2c63493c38219d935e71db6c1
ret = av_interleaved_write_frame(output_format_context, &packet);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&packet);
}
```
Para finalizar, necesitamos escribir el stream trailer a un archivo multimedia de salida con la función [av_write_trailer](https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga7f14007e7dc8f481f054b21614dfec13).
```c
av_write_trailer(output_format_context);
```
Ahora, estamos listos para probarlo y la primera prueba va a ser una conversión de formato (contenedor de video) de un video MP4 a un video MPEG-TS. Estamos básicamente realizando la línea de comando `ffmpeg input.mp4 -c copy output.ts` con libav.
```bash
make run_remuxing_ts
```
¡Funciona! !¿No me crees?! no deberías, podemos checarlo con `ffprobe`:
```bash
ffprobe -i remuxed_small_bunny_1080p_60fps.ts
Input #0, mpegts, from 'remuxed_small_bunny_1080p_60fps.ts':
Duration: 00:00:10.03, start: 0.000000, bitrate: 2751 kb/s
Program 1
Metadata:
service_name : Service01
service_provider: FFmpeg
Stream #0:0[0x100]: Video: h264 (High) ([27][0][0][0] / 0x001B), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 60 fps, 60 tbr, 90k tbn, 120 tbc
Stream #0:1[0x101]: Audio: ac3 ([129][0][0][0] / 0x0081), 48000 Hz, 5.1(side), fltp, 320 kb/s
```
Para resumir todo lo que hicimos esto en una imagen, podemos revisitar nuestra [idea inicial acerca de cómo libav funciona](https://github.com/leandromoreira/ffmpeg-libav-tutorial#ffmpeg-libav-architecture) pero observa que nos saltamos la parte del códec.
Antes de terminar este capítulo, me gustaría enseñarte una parte importante del proceso de remultiplexación, **tu puedes pasar esas opciones al multiplexor**. Digamos que se desea entregar un formato [MPEG-DASH](https://developer.mozilla.org/en-US/docs/Web/Apps/Fundamentals/Audio_and_video_delivery/Setting_up_adaptive_streaming_media_sources#MPEG-DASH_Encoding), para eso, necesitamos usar [mp4 fragmentado](https://stackoverflow.com/a/35180327) (a veces es referido como `fmp4`) en lugar de MPEG-TS o MPEG-4 plano.
Con la [línea de comando, podemos hacer eso fácilmente](https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API/Transcoding_assets_for_MSE#Fragmenting).
```
ffmpeg -i non_fragmented.mp4 -movflags frag_keyframe+empty_moov+default_base_moof fragmented.mp4
```
Casi igual de fácil como en la línea de comando, para su versión en libav, solamente debemos pasar las opciones y después escribir el encabezado de salida, justo antes de copiar los paquetes.
```c
AVDictionary* opts = NULL;
av_dict_set(&opts, "movflags", "frag_keyframe+empty_moov+default_base_moof", 0);
ret = avformat_write_header(output_format_context, &opts);
```
Ahora podemos generar este archivo mp4 fragmentado:
```bash
make run_remuxing_fragmented_mp4
```
Para asegurarte que no te estoy mintiendo. Puedes usar esta maravillosa página/herramienta [gpac/mp4box.js](http://download.tsi.telecom-paristech.fr/gpac/mp4box.js/filereader.html) o el sitio [http://mp4parser.com/](http://mp4parser.com/) para ver las diferencias, primero carga el mp4 "común".
Como podrás ver, este tiene un solo atom (o caja) `mdat`, **este es el espacio donde se encuentran los cuadros de video y audio**. Ahora carga el mp4 fragmentado y ve lo que despliega de las cajas `mdat`.
## Capítulo 3 - transcoding
> #### TLDR; enséñame el [código](/3_transcoding.c) y ejecuta.
> ```bash
> $ make run_transcoding
> ```
> Vamos a saltarnos unos detalles, pero no te preocupes: el [el codigo fuente está disponible en GitHub](/3_transcoding.c).
En este capitulo, vamos a crear un transcoder minimalista, escrito en C, que pueda convertir videos codificados en H264 a H265 usando la librería **FFmpeg/libav**, específicamente [libavcodec](https://ffmpeg.org/libavcodec.html), libavformat, y libavutil.
> _Solo una recapitulación rápida:_ El [**AVFormatContext**](https://www.ffmpeg.org/doxygen/trunk/structAVFormatContext.html) es la abstracción del formato para un archivo multimedia, alias contenedor (ej. MKV, MP4, Webm, TS). El [**AVStream**](https://www.ffmpeg.org/doxygen/trunk/structAVStream.html) representa cada tipo de datos para un formato dado (ej: audio, video, subtitulo, metadatos). El [**AVPacket**](https://www.ffmpeg.org/doxygen/trunk/structAVPacket.html) es una porción de datos comprimidos, los cuales son adquiridos desde `AVStream` y que pueden ser decodificados por un [**AVCodec**](https://www.ffmpeg.org/doxygen/trunk/structAVCodec.html) (ej: av1, h264, vp9, hevc) generando datos en bruto, llamados [**AVFrame**](https://www.ffmpeg.org/doxygen/trunk/structAVFrame.html).
### Transmuxing
Vamos a empezar con una simple operación de transmultiplexación (transmuxing) y después podemos construir sobre este código, el primer paso es **cargar el archivo de entrada**.
```c
// Acomoda un AVFormatContext
avfc = avformat_alloc_context();
// Abre un stream de entrada y lee el encabezado.
avformat_open_input(avfc, in_filename, NULL, NULL);
// Lee los paquetes del archivo para obtener la informacion de streams.
avformat_find_stream_info(avfc, NULL);
```
Ahora vamos a poner en pie el decodificador, el `AVFormatContext` nos va a dar acceso a todos los componentes `AVStream` y por cada uno de ellos, podremos obtener su `AVCodec` y crear su `AVCodecContext` en particular y finalmente podremos abrir el códec dado, así entonces podremos proceder con el proceso de decodificación.
> El [**AVCodecContext**](https://www.ffmpeg.org/doxygen/trunk/structAVCodecContext.html) contiene datos acerca de la configuración del archivo como la tasa de bits (bit rate), tasa de cuadros (frame rate), tasa de muestreo (sample rate), canales (channels), altura (height), así como muchos otros.
```c
for (int i = 0; i < avfc->nb_streams; i++)
{
AVStream *avs = avfc->streams[i];
AVCodec *avc = avcodec_find_decoder(avs->codecpar->codec_id);
AVCodecContext *avcc = avcodec_alloc_context3(*avc);
avcodec_parameters_to_context(*avcc, avs->codecpar);
avcodec_open2(*avcc, *avc, NULL);
}
```
Necesitamos preparar el archivo multimedia para transmultiplexación también, primero debemos **acomodar memoria** para la salida `AVFormatContext`. Creamos **cada uno de los streams** en el formato de salida. Para poder empaquetar propiamente el stream, **copiamos los parámetros del códec** desde el decodificador.
**Establecemos la bandera** `AV_CODEC_FLAG_GLOBAL_HEADER` el cual le dice al encodificador que puede usar los encabezados globales y finalmente abrimos el **archivo de salida para vaciar los datos** y mantener los encabezados.
```c
avformat_alloc_output_context2(&encoder_avfc, NULL, NULL, out_filename);
AVStream *avs = avformat_new_stream(encoder_avfc, NULL);
avcodec_parameters_copy(avs->codecpar, decoder_avs->codecpar);
if (encoder_avfc->oformat->flags & AVFMT_GLOBALHEADER)
encoder_avfc->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
avio_open(&encoder_avfc->pb, encoder->filename, AVIO_FLAG_WRITE);
avformat_write_header(encoder->avfc, &muxer_opts);
```
Nosotros conseguiremos los `AVPacket` desde el decodificador, ajustando los timestamps, y así poder escribir apropiadamente el paquete en el archivo de salida. Aunque la función `av_interleaved_write_frame` dice "write frame" (escribir cuadro), estamos guardando el paquete. Terminaremos el proceso de transmultiplexación escribiendo el stream del trailer, que se encuentra dentro del archivo.
```c
AVFrame *input_frame = av_frame_alloc();
AVPacket *input_packet = av_packet_alloc();
while (av_read_frame(decoder_avfc, input_packet) >= 0)
{
av_packet_rescale_ts(input_packet, decoder_video_avs->time_base, encoder_video_avs->time_base);
av_interleaved_write_frame(*avfc, input_packet) < 0));
}
av_write_trailer(encoder_avfc);
```
### Transcoding
La sección previa mostró un programa transmultiplexador, ahora vamos a agregar la capacidad para codificar los archivos, específicamente, vamos a habilitarlo para transcodificar videos desde `h264` a `h265`.
Después de que preparamos el decodificador, pero antes de acomodar el archivo de salida multimedia, vamos a configurar el encodificador.
* Crea el video `AVStream` en el encodificador, [`avformat_new_stream`](https://www.ffmpeg.org/doxygen/trunk/group__lavf__core.html#gadcb0fd3e507d9b58fe78f61f8ad39827)
* Usa el `AVCodec` llamado `libx265`, [`avcodec_find_encoder_by_name`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__encoding.html#gaa614ffc38511c104bdff4a3afa086d37)
* Crear el `AVCodecContext` basado en el códec creado, [`avcodec_alloc_context3`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#gae80afec6f26df6607eaacf39b561c315)
* Configurar los atributos básicos para la sesión de transcodificación, y
* Abre el códec y copia los parámetros del contexto al stream. [`avcodec_open2`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga11f785a188d7d9df71621001465b0f1d) y [`avcodec_parameters_from_context`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga0c7058f764778615e7978a1821ab3cfe)
```c
AVRational input_framerate = av_guess_frame_rate(decoder_avfc, decoder_video_avs, NULL);
AVStream *video_avs = avformat_new_stream(encoder_avfc, NULL);
char *codec_name = "libx265";
char *codec_priv_key = "x265-params";
// vamos a usar las opciones internas para x265
// esto deshabilita la deteccion de cambio de escena y despues fija
// GOP en 60 cuadros.
char *codec_priv_value = "keyint=60:min-keyint=60:scenecut=0";
AVCodec *video_avc = avcodec_find_encoder_by_name(codec_name);
AVCodecContext *video_avcc = avcodec_alloc_context3(video_avc);
// parametros de codec para el encoder
av_opt_set(sc->video_avcc->priv_data, codec_priv_key, codec_priv_value, 0);
video_avcc->height = decoder_ctx->height;
video_avcc->width = decoder_ctx->width;
video_avcc->pix_fmt = video_avc->pix_fmts[0];
// control de tasa
video_avcc->bit_rate = 2 * 1000 * 1000;
video_avcc->rc_buffer_size = 4 * 1000 * 1000;
video_avcc->rc_max_rate = 2 * 1000 * 1000;
video_avcc->rc_min_rate = 2.5 * 1000 * 1000;
// tiempo base
video_avcc->time_base = av_inv_q(input_framerate);
video_avs->time_base = sc->video_avcc->time_base;
avcodec_open2(sc->video_avcc, sc->video_avc, NULL);
avcodec_parameters_from_context(sc->video_avs->codecpar, sc->video_avcc);
```
Necesitamos expandir nuestro ciclo decodificador para la transcodificación del stream de video:
* Envía el `AVPacket` vacío al decodificador, [`avcodec_send_packet`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga58bc4bf1e0ac59e27362597e467efff3)
* Recibir el `AVFrame` descomprimido, [`avcodec_receive_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga11e6542c4e66d3028668788a1a74217c)
* Empezar a transcodificar este cuadro en bruto,
* Enviar este cuadro en bruto, [`avcodec_send_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga9395cb802a5febf1f00df31497779169)
* Recibe el contenido comprimido, basado en nuestro códec, `AVPacket`, [`avcodec_receive_packet`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga5b8eff59cf259747cf0b31563e38ded6)
* Establece el timestamp, y[`av_packet_rescale_ts`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__packet.html#gae5c86e4d93f6e7aa62ef2c60763ea67e)
* Escríbelo a un archivo de salida. [`av_interleaved_write_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga37352ed2c63493c38219d935e71db6c1)
```c
AVFrame *input_frame = av_frame_alloc();
AVPacket *input_packet = av_packet_alloc();
while (av_read_frame(decoder_avfc, input_packet) >= 0)
{
int response = avcodec_send_packet(decoder_video_avcc, input_packet);
while (response >= 0) {
response = avcodec_receive_frame(decoder_video_avcc, input_frame);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
return response;
}
if (response >= 0) {
encode(encoder_avfc, decoder_video_avs, encoder_video_avs, decoder_video_avcc, input_packet->stream_index);
}
av_frame_unref(input_frame);
}
av_packet_unref(input_packet);
}
av_write_trailer(encoder_avfc);
// funcion usada
int encode(AVFormatContext *avfc, AVStream *dec_video_avs, AVStream *enc_video_avs, AVCodecContext video_avcc int index) {
AVPacket *output_packet = av_packet_alloc();
int response = avcodec_send_frame(video_avcc, input_frame);
while (response >= 0) {
response = avcodec_receive_packet(video_avcc, output_packet);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
return -1;
}
output_packet->stream_index = index;
output_packet->duration = enc_video_avs->time_base.den / enc_video_avs->time_base.num / dec_video_avs->avg_frame_rate.num * dec_video_avs->avg_frame_rate.den;
av_packet_rescale_ts(output_packet, dec_video_avs->time_base, enc_video_avs->time_base);
response = av_interleaved_write_frame(avfc, output_packet);
}
av_packet_unref(output_packet);
av_packet_free(&output_packet);
return 0;
}
```
Vamos a convertir el stream desde `h264` a `h265`, como se espera de la versión `h265`, el archivo es más pequeño que el `h264` sin embargo el [programa creado](/3_transcoding.c) es capaz de:
```c
/*
* H264 -> H265
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx265";
sp.codec_priv_key = "x265-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - fragmented MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
sp.muxer_opt_key = "movflags";
sp.muxer_opt_value = "frag_keyframe+empty_moov+delay_moov+default_base_moof";
/*
* H264 -> H264 (fixed gop)
* Audio -> AAC
* MP4 - MPEG-TS
*/
StreamingParams sp = {0};
sp.copy_audio = 0;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
sp.audio_codec = "aac";
sp.output_extension = ".ts";
/* WIP :P -> it's not playing on VLC, the final bit rate is huge
* H264 -> VP9
* Audio -> Vorbis
* MP4 - WebM
*/
//StreamingParams sp = {0};
//sp.copy_audio = 0;
//sp.copy_video = 0;
//sp.video_codec = "libvpx-vp9";
//sp.audio_codec = "libvorbis";
//sp.output_extension = ".webm";
```
> Ahora, para ser honesto, esto fue [más difícil de lo que pensé](https://github.com/leandromoreira/ffmpeg-libav-tutorial/pull/54), voy a tener que y ya me he metido dentro de [el código fuente de la linea de comandos FFmpeg](https://github.com/leandromoreira/ffmpeg-libav-tutorial/pull/54#issuecomment-570746749) y probarlo bastante, y también pienso que estoy olvidando algo, ya que cuando tengo que forzar `force-cfr` para que el `h264` funcione, me sigue arrojando algunos mensajes como `warning messages (forced frame type (5) at 80 was changed to frame type (3))`.
================================================
FILE: README-ko.md
================================================
[](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)
[FFmpeg](https://www.ffmpeg.org/)을 라이브러리처럼(a.k.a. libav) 사용하려면 어떻게 시작해야할지 알려줄만할 튜토리얼/책을 찾아봤었습니다. 그리고는 ["How to write a video player in less than 1k lines"](http://dranger.com/ffmpeg/) 라는 튜토리얼을 찾았죠.
하지만 안타깝게도 그건 더이상 관리가 안되고 있어서 이 글을 쓰기로 결정했습니다.
여기서 사용된 대부분의 코드는 C로 되어있습니다. **하지만 걱정하지 마세요**: 당신도 쉽게 이해할 것이고 선호하는 언어에도 적용하실 수 있을겁니다.
FFmpeg libav는 [python](https://pyav.org/), [go](https://github.com/imkira/go-libav)와 같은 다양한 언어로 된 많은 bindings을 제공합니다. 만약 사용하려는 언어에 그것이 없다면 `ffi`를 통해서도 지원할 수 있습니다. ([Lua](https://github.com/daurnimator/ffmpeg-lua-ffi/blob/master/init.lua) 예시)
우리는 비디오와 오디오, 코덱, 컨테이너가 무엇인지에 대해 빠르게 학습한 후에 `FFmpeg` 명령을 어떻게 사용하는지 대해서 파헤쳐보고 마지막으로 코드도 작성해볼 것입니다, [삽질하면서 FFmpeg libav 배우기](#삽질하면서-FFmpeg-libav-배우기) 섹션으로 바로 넘어가셔도 좋습니다.
혹자는 인터넷 비디오 스트리밍이 전통적인 TV의 미래라고 이야기하기도 합니다. 어떻게 되든 FFmpeg은 공부해둘만한 가치가 있는 것입니다.
__목차__
* [소개](#intro)
* [비디오 - 당신이 무엇을 보는지!](#비디오---당신이-무엇을-보는지!)
* [오디오 - 당신이 무엇을 듣는지!](#오디오---당신이-무엇을-듣는지!)
* [코덱 - 데이터를 줄이기](#코덱---데이터를-줄이기)
* [컨테이너 - 오디오와 비디오의 안식처](#[컨테이너---오디오와-비디오의-안식처)
* [FFmpeg - 명령줄 도구](#FFmpeg---명령줄-도구)
* [FFmpeg 명령줄 도구 101](#FFmpeg-명령줄-도구-101)
* [공통 비디오 연산](#공통-비디오-연산)
* [트랜스코딩 (Transcoding)](#트랜스코딩-(Transcoding))
* [트랜스먹싱 (Transmuxing)](#트랜스먹싱-(Transmuxing))
* [트랜스레이팅 (Transrating)](#트랜스레이팅-(Transrating))
* [트랜스사이징 (Transsizing)](#트랜스사이징-(Transsizing))
* [보너스: 적응형 스트리밍 (Adaptive Streaming)](#보너스:-적응형-스트리밍-(Adaptive-Streaming))
* [더 들어가기](#더-들어가기)
* [삽질하면서 FFmpeg libav 배우기](#삽질하면서-FFmpeg-libav-배우기)
* [챕터 0 - 악명 높은 hello world](#챕터-0---악명-높은-hello-world)
* [FFmpeg libav 아키텍처](#FFmpeg-libav-아키텍처)
* [챕터 1 - 타이밍 (timing)](#챕터-1---오디오와-비디오-동기화)
* [챕터 2 - 리먹싱 (remuxing)](#챕터-2---리먹싱-(remuxing))
* [챕터 3 - 트랜스코딩 (transcoding)](#챕터-3---트랜스코딩-(transcoding))
# 소개
## 비디오 - 당신이 무엇을 보는지!
만약 당신이 여러 연속된 이미지들을 가지고 있고 이것들을 주어진 주파수에 맞게 변화시킨다면 (이를테면 [초당 24장의 이미지](https://www.filmindependent.org/blog/hacking-film-24-frames-per-second/)), [움직임의 잔상](https://en.wikipedia.org/wiki/Persistence_of_vision)을 만들게 될 것입니다.
요약하면 이게 비디오라는 것의 가장 기본적인 아이디어입니다: **정해진 속도에 맞게 돌아가는 연속된 사진들 / 프레임들**.
<img src="https://upload.wikimedia.org/wikipedia/commons/1/1f/Linnet_kineograph_1886.jpg" title="flip book" height="280"></img>
Zeitgenössische Illustration (1886)
## 오디오 - 당신이 무엇을 듣는지!
음소거된 비디오만으로도 다양한 감정들을 표현할 수는 있지만 여기에 소리를 더해준다면 훨씬 더 즐거운 경험을 가져다 줄 것입니다.
소리는 공기 혹은 가스, 액체, 고체와 같은 다른 매체들을 통해 압력의 파동 형태로 전파되는 진동입니다.
> 디지털 오디오 시스템에서는 마이크가 소리를 아날로그 전기 신호로 전환하고, 아날로그-디지털 변환기 (ADC) - 보통 [펄스-부호 변조 (PCM)](https://en.wikipedia.org/wiki/Pulse-code_modulation)를 이용하여 - 아날로그 신호를 디지탈 신호로 변환합니다.
>[원문](https://commons.wikimedia.org/wiki/File:CPT-Sound-ADC-DAC.svg)
## 코덱 - 데이터를 줄이기
> CODEC은 **디지털 오디오/비디오를 압축하거나 압축해제하는** 전자회로나 소프트웨어입니다. 이것은 raw (압축이안된) 디지털 오디오/비디오를 압축된 형태로 혹은 그 반대로 변환합니다.
> https://en.wikipedia.org/wiki/Video_codec
만약 우리가 수많은 이미지들을 차곡차곡 채워서 영화라고 부르는 하나의 파일로 만든다면, 결과적으로 엄청나게 큰 하나의 파일을 접하게 될 것 입니다. 한번 계산해봅시다:
한번 가정해봅시다. 해상도가 `1080 x 1920` (높이 x 너비)인 비디오를 하나 만들건데 색을 인코딩하는데 픽셀당 `3 bytes` (화면의 최소 화소)를 쓸 것입니다. (혹은 [24비트 컬러](https://en.wikipedia.org/wiki/Color_depth#True_color_.2824-bit.29), 16,777,216개의 다른 색상을 제공) 그리고 이 비디오는 `초당 24프레임`으로 재생되고 `30분` 정도 길이입니다.
```c
toppf = 1080 * 1920 //total_of_pixels_per_frame
cpp = 3 //cost_per_pixel
tis = 30 * 60 //time_in_seconds
fps = 24 //frames_per_second
required_storage = tis * fps * toppf * cpp
```
이 비디오는 거의 `250.28GB`의 저장 용량이 필요하며 `1.19Gbps`의 대역폭이 요구됩니다! 이것이 바로 우리가 [CODEC](https://github.com/leandromoreira/digital_video_introduction#how-does-a-video-codec-work)을 사용해야하는 이유입니다.
## 컨테이너 - 오디오와 비디오의 안식처
> 컨테이너 혹은 래퍼(wrapper) 포맷은 데이터와 메타데이터의 다양한 요소들이 어떻게 하나의 컴퓨터 파일에 구성되어있는지를 기술하는 스펙을 담은 메타파일 포맷입니다.
> https://en.wikipedia.org/wiki/Digital_container_format
**하나의 파일이 모든 스트림을 담고 있고** (주로 오디오와 비디오) 이것은 또 동기화와 제목, 해상도 등과 같은 일반적인 메타데이터도 제공합니다.
보통 우리는 파일의 확장자를 보고 포맷을 유추할 수 있습니다: 예를들면 `video.webm`은 아마도 [`webm`](https://www.webmproject.org/)를 컨테이너로 사용하는 비디오겠죠.
# FFmpeg - 명령줄 도구
> 오디오와 비디오를 녹화하고 변환하고 스트리밍할 수 있는 완전한 크로스-플랫폼 솔루션.
멀티미디어 작업을 한다면 우리는 [FFmpeg](https://www.ffmpeg.org/)이라고 하는 정말 쩌는 툴/라이브러리를 사용할 수 있습니다. 아마도 여러분도 이것을 직간접적으로 알고있거나/사용했던 기회가 있었을 것입니다. ([Chrome](https://www.chromium.org/developers/design-documents/video) 사용시죠?).
이것은 `ffmpeg`이라고하는 아주 단순하지만 파워풀한 바이너리 형태의 명려줄 프로그램도 제공합니다.
예를들어, 아래 명령을 치는 것만으로도 컨테이너를 `mp4`에서 `avi`로 변환할 수 있습니다:
```bash
$ ffmpeg -i input.mp4 output.avi
```
우리는 방금 어떤 컨테이너에서 다른 컨테이너로 변환하는 과정인 **remuxing**을 해보았습니다.
기술적으로 FFmpeg은 트랜스코딩(transcoding)도 할 수 있습니다만 이것들에 대해서는 뒤에서 다시 이야기하겠습니다.
## FFmpeg 명령줄 도구 101
FFmpeg이 어떻게 동작하는지를 아주 잘 설명하고 있는 [문서](https://www.ffmpeg.org/ffmpeg.html)가 있습니다.
간단히 정리하면, FFmpeg 명령줄 프로그램은 실행하기 위해 다음과 같은 형식의 인자를 갖춰야합니다 `ffmpeg {1} {2} -i {3} {4} {5}`, 여기서:
1. 전역 옵션
2. 입력 파일 옵션
3. 입력 url
4. 출력 파일 옵션
5. 출력 url
2, 3, 4, 5 부분은 필요한만큼 많아질 수 있습니다.
실제로 수행해보면 이 인자 형식을 더 쉽게 이해할 수 있습니다:
``` bash
# WARNING: this file is around 300MB
$ wget -O bunny_1080p_60fps.mp4 http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_1080p_60fps_normal.mp4
$ ffmpeg \
-y \ # 전역 옵션
-c:a libfdk_aac \ # 입력 파일 옵션
-i bunny_1080p_60fps.mp4 \ # 입력 url
-c:v libvpx-vp9 -c:a libvorbis \ # 출력 파일 옵션
bunny_1080p_60fps_vp9.webm # 출력 url
```
이 명령은 두개의 스트림(`aac` 코덱으로 인코딩된 오디오와 `h264` 코덱으로 인코딩된 비디오)을 포함하는 `mp4`를 입력 파일로 받고 이를 `webm`으로 변환합니다, 물론 그 안의 오디오와 비디오 코덱들도 변환하고 있죠.
위의 명령을 더 단순화할 수도 있는데 그러면 FFmpeg이 기본값들을 사용하거나 추측하게될 것입니다.
예를들어 `ffmpeg -i input.avi output.mp4` 이렇게만 친다면 어떤 오디오/비디오 코덱이 `output.mp4`를 만들기 위해 사용될까요?
Werner Robitza가 작성한 꼭 읽고/실행해볼만한 [FFmpeg으로 인코딩하고 편집하는 것에 대한 튜토리얼](http://slhck.info/ffmpeg-encoding-course/#/)이 있습니다.
# 공통 비디오 연산
오디오/비디오 작업 중 보통 미디어에 대해 일련의 작업을 수행하게 됩니다.
## 트랜스코딩 (Transcoding)
**무엇인가?** 스트림 (오디오 또는 비디오) 중에 하나를 기존 코덱에서 다른 코덱으로 변환하는 작업.
**왜?** 가끔 어떤 장치들은 (텔레비전, 스마트폰, 콘솔 등) X는 지원하지 않지만 Y를 지원합니다. 그리고 더 새로운 코덱들은 더 나은 압축률을 제공하기도 합니다.
**어떻게?** `H264` (AVC) 비디오를 `H265` (HEVC)로 변환하기.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c:v libx265 \
bunny_1080p_60fps_h265.mp4
```
## 트랜스먹싱 (Transmuxing)
**무엇인가?** 하나의 포맷을 (컨테이너) 다른 포맷으로 변환하는 작업.
**왜?** 가끔 어떤 장치들은 (텔레비전, 스마트폰, 콘솔 등) X는 지원하지 않지만 Y를 지원합니다. 그리고 때때로 더 새로운 컨테이터들은 최신으로 요구되는 피처들을 제공합니다.
**어떻게?** `mp4`에서 `webm`으로 변환하기.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c copy \ # just saying to ffmpeg to skip encoding
bunny_1080p_60fps.webm
```
## 트랜스레이팅 (Transrating)
**무엇인가?** 비트레이트를 변환하거나 다른 변환본(renditions)을 만드는 작업.
**왜?** 사람들은 `2G` (edge)가 연결된 저사양의 스마트폰에서든 `광통신` 인터넷이 연결된 4K 텔레비전에든 당신의 비디오 볼 것이다. 그래서 같은 비디오라도 여러 비트레이트를 가진 하나 이상의 변환본을 제공해야합니다.
**어떻게?** 3856K와 2000K 사이의 비트레이트를 가진 변환본을 생성하기.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-minrate 964K -maxrate 3856K -bufsize 2000K \
bunny_1080p_60fps_transrating_964_3856.mp4
```
보통 트랜스레이팅(transrating)은 트랜스사이징(transsizing)과 함께 사용합니다. Werner Robitza가 작성한 또 다른 필독/실행물 [FFmpeg rate 제어에 대한 연재 포스팅](http://slhck.info/posts/)가 있습니다.
## 트랜스사이징 (Transsizing)
**무엇인가?** 하나의 해상도에서 다른 것으로 변환하는 작업. 이전에 언급한 것처럼 트랜스사이징(transsizing)은 주로 트랜스레이팅(transrating)과 함께 사용됩니다.
**왜?** 트랜스레이팅(transrating)에서의 이유와 동일함.
**어떻게?** `1080p`의 해상도를 `480p`로 변환하기.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-vf scale=480:-1 \
bunny_1080p_60fps_transsizing_480.mp4
```
## 보너스: 적응형 스트리밍 (Adaptive Streaming)
**무엇인가?** 다양한 (비트레이트의) 해상도를 생성하고 미디어들을 여러 청크로 나눠서 http를 통해 서비스하는 작업.
**왜?** 저사양 스마트폰 혹은 4K TV에서 시청할 수 있는 유연한 미디어를 제공하기 위해. 또한 이렇게 하면 확장이나 배포하기가 쉽습니다. 다만 지연시간이 생길 수 있습니다.
**어떻게?** DASH를 이용하여 적응형 WebM을 생성하기.
```bash
# video streams
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 160x90 -b:v 250k -keyint_min 150 -g 150 -an -f webm -dash 1 video_160x90_250k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 320x180 -b:v 500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_320x180_500k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 750k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_750k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 1000k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_1000k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 1280x720 -b:v 1500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_1280x720_1500k.webm
# audio streams
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:a libvorbis -b:a 128k -vn -f webm -dash 1 audio_128k.webm
# the DASH manifest
$ ffmpeg \
-f webm_dash_manifest -i video_160x90_250k.webm \
-f webm_dash_manifest -i video_320x180_500k.webm \
-f webm_dash_manifest -i video_640x360_750k.webm \
-f webm_dash_manifest -i video_640x360_1000k.webm \
-f webm_dash_manifest -i video_1280x720_500k.webm \
-f webm_dash_manifest -i audio_128k.webm \
-c copy -map 0 -map 1 -map 2 -map 3 -map 4 -map 5 \
-f webm_dash_manifest \
-adaptation_sets "id=0,streams=0,1,2,3,4 id=1,streams=5" \
manifest.mpd
```
PS: 저는 이 예제를 [DASH를 이용한 Adaptive WebM 재생에 대한 지침](http://wiki.webmproject.org/adaptive-streaming/instructions-to-playback-adaptive-webm-using-dash)에서 가져왔습니다.
## 더 들어가기
[FFmpeg에 대한 아주 수많은 다른 사용방법들이](https://github.com/leandromoreira/digital_video_introduction/blob/master/encoding_pratical_examples.md#split-and-merge-smoothly) 있습니다.
저는 이걸 YouTube 용 동영상들을 만들고/편집하는데 *iMovie*와 함께 사용합니다. 물론 여러분도 프로페셔널처럼 사용하실 수 있습니다.
# 삽질하면서 FFmpeg libav 배우기
> 가끔 '소리나는 것과 보이는 것이' 궁금하지 않으세요?
> **David Robert Jones**
[FFmpeg](#ffmpeg---command-line)는 미디어 파일들에 대한 필수 작업들을 수행하는 명령줄 도구로써 매우 유용합니다. 어떻게 우리의 프로그램에 이용할 수 있을까요?
FFmpeg는 우리의 프로그램에 통합될 수 있는 [여러 라이브러리들로 구성](https://www.ffmpeg.org/doxygen/trunk/index.html)되어있습니다.
보통, FFmpeg을 설치할때 이 모든 라이브러리들도 자동으로 설치됩니다. 이 라이브러리 모음들을 **FFmpeg libav**라고 해보죠.
> 이 제목은 Zed Shaw의 [Learn X the Hard Way](https://learncodethehardway.org/) 시리즈, 특히 그의 책 Learn C the Hard Way에 대한 오마주입니다.
## 챕터 0 - 악명 높은 hello world
이 hello world는 실제로 `"hello world"` 메시지를 터미널에 보여주진 않습니다. :tongue:
대신 우리는 **비디오의 정보를 출력**할 것입니다. 비디오의 포맷 (컨테이너), 길이, 해상도, 오디오 채널들 같은 것들을 말이죠. 그리고 마지막으로 **몇몇 프레임들을 디코딩하고 이미지 파일로 저장**해보겠습니다.
### FFmpeg libav 아키텍처
하지만 코딩을 시작하기 전에, **FFmpeg libav 아키텍처**가 어떻게 동작하는지 이것들의 컴포넌트들이 서로 어떻게 통신하는지를 배워봅시다.
여기 비디오가 디코딩되는 프로세스를 담은 다이어그램이 하나 있습니다.
우선 여러분의 미디어 파일을 [`AVFormatContext`](https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html) (비디오 컨테이너는 포맷이라고도 합니다)라고 불리는 컴포넌트로 불러올 필요가 있습니다.
이건 사실 파일 전체를 불러오는건 아닙니다: 종종 헤더만을 읽죠.
일단 최소한의 **컨테이너 헤더**를 불러왔다면, 우리는 이것의 스트림 (기본적이고 필수적인 오디오와 비디오 데이터라고 간주하시면 됩니다)에 접근할 수 있습니다.
각 스트림은 [`AVStream`](https://ffmpeg.org/doxygen/trunk/structAVStream.html)라고 하는 컴포넌트로 접근 가능합니다.
> 스트림은 데이터의 연속적인 흐름을 의미하는 fancy한 이름입니다.
비디오가 두개의 스트림을 가지고 있다고 해봅시다: 오디오는 [AAC CODEC](https://en.wikipedia.org/wiki/Advanced_Audio_Coding)로 인코딩되어있고 비디오는 [H264 (AVC) CODEC](https://en.wikipedia.org/wiki/H.264/MPEG-4_AVC)로 인코딩되어있습니다. 각 스트림으로부터 [`AVPacket`](https://ffmpeg.org/doxygen/trunk/structAVPacket.html) 컴포넌트로 로드될 패킷이라 칭하는 **데이터의 조각들**을 추출할 수 있습니다.
**패킷안의 데이터는 여전히 인코딩되어 있습니다** (압축된상태). 이 패킷을 디코딩하기 위해서 우리는 이것들을 특정한 [`AVCodec`](https://ffmpeg.org/doxygen/trunk/structAVCodec.html)에 넘겨야합니다.
`AVCodec`은 그것들을 [`AVFrame`](https://ffmpeg.org/doxygen/trunk/structAVFrame.html)으로 디코딩하며 최종적으로 우리에게 **압축 해제된 프레임**을 넘겨줍니다. 오디오 및 비디오 스트림에서 동일한 용어/프로세스가 사용된다는 점을 유의하십시오.
### 요구 사항
간혹 예제를 컴파일하고 실행하는데 이슈들을 겪는 분들이 계셔서 **우리의 개발/실행 환경으로 [`Docker`](https://docs.docker.com/install/)를 사용할 것입니다,** 우리는 또한 big buck bunny 비디오를 사용할 것인데 따로 로컬에 가지고 있지 않다면 `make fetch_small_bunny_video` 명령만 실행해주시면 됩니다.
### 챕터 0 - 몸풀기 코드
> #### TLDR; [코드](/0_hello_world.c)랑 실행하는거나 보여주세요.
> ```bash
> $ make run_hello
> ```
> 좀 상세한 부분은 넘어가겠습니다. 그러나 걱정하진 마세요: [소스 코드는 github에 있습니다](/0_hello_world.c).
포맷 (컨테이너)에 관한 정보를 담고 있는 [`AVFormatContext`](http://ffmpeg.org/doxygen/trunk/structAVFormatContext.html) 컴포넌트에게 메모리를 할당합니다.
```c
AVFormatContext *pFormatContext = avformat_alloc_context();
```
이제 우리는 파일을 열고 헤더를 읽어서 `AVFormatContext`에 포맷에 관한 기본적인 정보를 채워줄 것입니다 (보통 코덱은 열리지 않음).
이를 위해 사용할 함수는 [`avformat_open_input`](http://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga31d601155e9035d5b0e7efedc894ee49)입니다. 이 함수는 `AVFormatContext`와 `filename` 두개의 옵셔널 인자를 받습니다: [`AVInputFormat`](https://ffmpeg.org/doxygen/trunk/structAVInputFormat.html) (`NULL`을 넘기면 FFmpeg이 포맷을 추측)과 [`AVDictionary`](https://ffmpeg.org/doxygen/trunk/structAVDictionary.html) (demuxer에 대한 옵션)
```c
avformat_open_input(&pFormatContext, filename, NULL, NULL);
```
포맷 이름과 미디어 길이를 출력할 수 있습니다:
```c
printf("Format %s, duration %lld us", pFormatContext->iformat->long_name, pFormatContext->duration);
```
`streams`에 접근하기 위해서는, 미디어로부터 데이터를 읽어야합니다. [`avformat_find_stream_info`](https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#gad42172e27cddafb81096939783b157bb) 함수가 그 일을 하죠.
`pFormatContext->nb_streams`가 스트림의 개수를 가지고 있고 `pFormatContext->streams[i]`는 `i`번째 스트림 ([`AVStream`](https://ffmpeg.org/doxygen/trunk/structAVStream.html))을 반환합니다.
```c
avformat_find_stream_info(pFormatContext, NULL);
```
이제 모든 스트림에 대해 루프를 돌아보겠습니다.
```c
for (int i = 0; i < pFormatContext->nb_streams; i++)
{
//
}
```
각 스트림에 대해서, `i`번째 스트림에 사용된 코덱 속성들을 담고있는 [`AVCodecParameters`](https://ffmpeg.org/doxygen/trunk/structAVCodecParameters.html)를 가져오겠습니다.
```c
AVCodecParameters *pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
```
이 코덱 속성을 이용하여 [`avcodec_find_decoder`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga19a0ca553277f019dd5b0fec6e1f9dca) 함수를 통해 적절한 코덱을 찾을 수 있습니다. 코덱 id에 맞는 등록된 디코더를 찾고 스트림을 어떻게 en**CO**de와 **DEC**ode할지를 알고 있는 [`AVCodec`](http://ffmpeg.org/doxygen/trunk/structAVCodec.html) 컴포넌트를 반환받을 수 있습니다.
```c
AVCodec *pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);
```
이제 코덱에 관한 정보를 출력할 수 있습니다.
```c
// specific for video and audio
if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO) {
printf("Video Codec: resolution %d x %d", pLocalCodecParameters->width, pLocalCodecParameters->height);
} else if (pLocalCodecParameters->codec_type == AVMEDIA_TYPE_AUDIO) {
printf("Audio Codec: %d channels, sample rate %d", pLocalCodecParameters->channels, pLocalCodecParameters->sample_rate);
}
// general
printf("\tCodec %s ID %d bit_rate %lld", pLocalCodec->long_name, pLocalCodec->id, pLocalCodecParameters->bit_rate);
```
이 코덱을 기반으로 디코딩/인코딩 프로세스에 대한 컨텍스트를 담고있는 [`AVCodecContext`](https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html)의 메모리를 할당할 수 있습니다. 그 다음 코덱 파라미터로 코덱 컨텍스트를 채워줍니다; [`avcodec_parameters_to_context`](https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#gac7b282f51540ca7a99416a3ba6ee0d16)로 가능합니다.
일단 코덱 컨텍스트를 채웠다면 이제 코덱을 열 수 있습니다. [`avcodec_open2`](https://ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga11f785a188d7d9df71621001465b0f1d)로 가능합니다.
```c
AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
avcodec_parameters_to_context(pCodecContext, pCodecParameters);
avcodec_open2(pCodecContext, pCodec, NULL);
```
이제 스트림으로부터 패킷을 읽고 디코딩하여 프레임으로 만들어볼 예정입니다. 그러나 우선, [`AVPacket`](https://ffmpeg.org/doxygen/trunk/structAVPacket.html)와 [`AVFrame`](https://ffmpeg.org/doxygen/trunk/structAVFrame.html) 두 컴포넌트에 대해 메모리 할당이 필요합니다.
```c
AVPacket *pPacket = av_packet_alloc();
AVFrame *pFrame = av_frame_alloc();
```
패킷이 존재하는 동안 루프를 돌면서 [`av_read_frame`](https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html#ga4fdb3084415a82e3810de6ee60e46a61) 함수를 이용해 스트림으로부터 패킷을 받아오겠습니다.
```c
while (av_read_frame(pFormatContext, pPacket) >= 0) {
//...
}
```
코덱 컨텍스트를 [`avcodec_send_packet`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga58bc4bf1e0ac59e27362597e467efff3) 함수를 통해 디코더에 **raw 데이터 패킷 (압축된 프레임)을 보내**봅시다.
```c
avcodec_send_packet(pCodecContext, pPacket);
```
그리고 마찬가지로 코덱 컨텍스트를 [`avcodec_receive_frame`](https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga11e6542c4e66d3028668788a1a74217c) 함수를 통해 디코더로부터 **raw 데이터 프레임 (압축 해제된 프레임)를 받아**봅시다.
```c
avcodec_receive_frame(pCodecContext, pFrame);
```
프레임 번호, [PTS](https://en.wikipedia.org/wiki/Presentation_timestamp), DTS, [프레임 타입](https://en.wikipedia.org/wiki/Video_compression_picture_types) 등을 출력해볼 수 있습니다.
```c
printf(
"Frame %c (%d) pts %d dts %d key_frame %d [coded_picture_number %d, display_picture_number %d]",
av_get_picture_type_char(pFrame->pict_type),
pCodecContext->frame_number,
pFrame->pts,
pFrame->pkt_dts,
pFrame->key_frame,
pFrame->coded_picture_number,
pFrame->display_picture_number
);
```
마지막으로 디코딩된 프레임을 [심플 흑백 이미지](https://en.wikipedia.org/wiki/Netpbm_format#PGM_example)로 저장해볼 수 있습니다. 이 과정은 매우 단순합니다, 인덱스가 [planes Y, Cb, Cr](https://en.wikipedia.org/wiki/YCbCr)를 참조하고 있는 `pFrame->data`를 사용할 것입니다. 우리는 흑백 이미지를 저장하기 위해 `0` (Y) 인덱스를 선택했습니다.
```c
save_gray_frame(pFrame->data[0], pFrame->linesize[0], pFrame->width, pFrame->height, frame_filename);
static void save_gray_frame(unsigned char *buf, int wrap, int xsize, int ysize, char *filename)
{
FILE *f;
int i;
f = fopen(filename,"w");
// writing the minimal required header for a pgm file format
// portable graymap format -> https://en.wikipedia.org/wiki/Netpbm_format#PGM_example
fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
// writing line by line
for (i = 0; i < ysize; i++)
fwrite(buf + i * wrap, 1, xsize, f);
fclose(f);
}
```
voilà! 이제 우리는 2MB짜리 흑백 이미지를 얻어냈습니다:
## 챕터 1 - 오디오와 비디오 동기화
> **플레이어가 되세요** - 신규 MSE 비디오 플레이어를 작성 중인 젊은 JS 개발자
[트랜스코딩 예제 코드](#챕터-3---트랜스코딩-(transcoding))로 넘어가기 전에 **타이밍** 혹은 어떻게 비디오 플레이어가 하나의 프레임을 제시간에 재생해야하는지에 대해서 이야기해봅시다.
지난 예제에서, 우리는 이렇게 보이는 프레임들을 저장했습니다.
비디오 플레이어를 디자인 할때 **각 프레임을 주어진 속도에 재생**해야합니다, 그렇지 않으면 너무 빠르거나 너무 느리게 재생되기 때문에 비디오를 제대로 즐기기 어려울 것입니다.
그래서 뭔가 프레임을 원활하게 재생할 수 있는 로직을 소개할 필요가 있습니다. 이 이슈를 위해, 각 프레임은 **프리젠테이션 타임스탬프** (PTS)를 갖게 되는데 이것은 **프레임속도(fps)** 로 나누어지는 **타임베이스(timebase)** 라고 하는 유리수(분모가 **타임스케일(timescale)** 로 알려진)로 구성된(factored) 증가하는 숫자입니다.
예제를 좀 본다면 이해가 더 쉬울 것입니다, 몇개의 시나리오를 시뮬레이션해죠.
`fps=60/1` 이고 `timebase=1/60000` 라면 각 PTS는 `timescale / fps = 1000`를 증가할 것 입니다. 그래서 각 프레임의 **PTS 실제 시간**은 이렇게 됩니다 (0부터 시작한다고 하면):
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 1000, PTS_TIME = PTS * timebase = 0.016`
* `frame=2, PTS = 2000, PTS_TIME = PTS * timebase = 0.033`
동일한 시나리오지만 타임베이스가 `1/60`이라면.
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 1, PTS_TIME = PTS * timebase = 0.016`
* `frame=2, PTS = 2, PTS_TIME = PTS * timebase = 0.033`
* `frame=3, PTS = 3, PTS_TIME = PTS * timebase = 0.050`
`fps=25/1`와 `timebase=1/75`에 대해서는 각 PTS는 `timescale / fps = 3`만큼 증가할 것이고 PTS 시간은 이렇게 될 것 입니다:
* `frame=0, PTS = 0, PTS_TIME = 0`
* `frame=1, PTS = 3, PTS_TIME = PTS * timebase = 0.04`
* `frame=2, PTS = 6, PTS_TIME = PTS * timebase = 0.08`
* `frame=3, PTS = 9, PTS_TIME = PTS * timebase = 0.12`
* ...
* `frame=24, PTS = 72, PTS_TIME = PTS * timebase = 0.96`
* ...
* `frame=4064, PTS = 12192, PTS_TIME = PTS * timebase = 162.56`
이제 이 `pts_time`으로 오디오의 `pts_time` 혹은 시스템 시간과 동기화해서 재생할 방법을 찾을 수 있습니다. FFmpeg libav는 그 정보들을 아래 API를 통해 제공합니다.
- fps = [`AVStream->avg_frame_rate`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#a946e1e9b89eeeae4cab8a833b482c1ad)
- tbr = [`AVStream->r_frame_rate`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#ad63fb11cc1415e278e09ddc676e8a1ad)
- tbn = [`AVStream->time_base`](https://ffmpeg.org/doxygen/trunk/structAVStream.html#a9db755451f14e2bf590d4b85d82b32e6)
호기심에 보자면, 우리가 저장했던 프레임들을 DTS 순으로 (frames: 1,6,4,2,3,5) 보내졌지만 재생은 PTS 순 (frames: 1,2,3,4,5)로 되었습니다. 또한, B-프레임이 P 혹은 I-프레임 대비 얼마나 저렴한지도 알 수 있죠.
```
LOG: AVStream->r_frame_rate 60/1
LOG: AVStream->time_base 1/60000
...
LOG: Frame 1 (type=I, size=153797 bytes) pts 6000 key_frame 1 [DTS 0]
LOG: Frame 2 (type=B, size=8117 bytes) pts 7000 key_frame 0 [DTS 3]
LOG: Frame 3 (type=B, size=8226 bytes) pts 8000 key_frame 0 [DTS 4]
LOG: Frame 4 (type=B, size=17699 bytes) pts 9000 key_frame 0 [DTS 2]
LOG: Frame 5 (type=B, size=6253 bytes) pts 10000 key_frame 0 [DTS 5]
LOG: Frame 6 (type=P, size=34992 bytes) pts 11000 key_frame 0 [DTS 1]
```
## 챕터 2 - 리먹싱 (remuxing)
Remuxing은 하나의 포맷 (컨테이너)에서 다른 것으로 변경하는 작업입니다. 다음 예제처럼 FFmpeg을 쓰면 별로 어렵지 않게 [MPEG-4](https://en.wikipedia.org/wiki/MPEG-4_Part_14) 비디오를 [MPEG-TS](https://en.wikipedia.org/wiki/MPEG_transport_stream)로 변경할 수 있습니다:
```bash
ffmpeg input.mp4 -c copy output.ts
```
이것은 mp4를 demux하지만 디코딩이나 인코딩은 하지 않습니다. (`-c copy`) 최종적으로 `mpegts` 파일로 mux할 것입니다. 만약 포맷을 의미하는 `-f`를 제공하지 않으면 ffmpeg은 파일 확장자로 포맷을 추측할 것입니다.
FFmpeg 혹은 libav의 일반적인 사용법은 아래 패턴/아키텍처 또는 워크플로우를 따릅니다:
* **[프로토콜 레이어](https://ffmpeg.org/doxygen/trunk/protocols_8c.html)** - `input`을 받음 (예를들면 `file`이지만 `rtmp` 또는 `HTTP` 입력도 가능).
* **[포맷 레이어](https://ffmpeg.org/doxygen/trunk/group__libavf.html)** - 컨텐츠를 `demuxes`, 대부분 메타데이터와 스트림을 열어봄
* **[코덱 레이어](https://ffmpeg.org/doxygen/trunk/group__libavc.html)** - 압축된 스트림 데이터를 `decodes` <sup>*optional*</sup>
* **[픽셀 레이어](https://ffmpeg.org/doxygen/trunk/group__lavfi.html)** - raw 프레임에 대해 (리사이징 같은) `filters`를 적용할 수도 있음 <sup>*optional*</sup>
* and then it does the reverse path
* **[코덱 레이어](https://ffmpeg.org/doxygen/trunk/group__libavc.html)** - raw 프레임을 `encodes` (또는 `re-encodes` 혹은 `transcodes` 까지도) <sup>*optional*</sup>
* **[포맷 레이어](https://ffmpeg.org/doxygen/trunk/group__libavf.html)** - raw 스트림 (압축된 데이터)를 `muxes` (또는 `remuxes`)
* **[프로토콜 레이어](https://ffmpeg.org/doxygen/trunk/protocols_8c.html)** - 그리고 마지막으로 muxed된 데이터를 `output`으로 전송 (또다른 파일 혹은 네트워크 원격 서버일 수도 있음)
> 이 그래프는 [Leixiaohua's](http://leixiaohua1020.github.io/#ffmpeg-development-examples)와 [Slhck's](https://slhck.info/ffmpeg-encoding-course/#/9)의 작업으로부터 큰 영감을 받은 것입니다.
자 이제 `ffmpeg input.mp4 -c copy output.ts`와 동일한 효과를 제공할 수 있도록 libav 를 이용한 예제를 하나 구현해봅시다.
입력 (`input_format_context`)으로부터 읽은 것을 다른 출력 (`output_format_context`)으로 변환해보겠습니다.
```c
AVFormatContext *input_format_context = NULL;
AVFormatContext *output_format_context = NULL;
```
일반적으로 메모리 할당을 시작하고 입력 포맷을 엽니다. 이번같은 특정한 경우에는, 입력 파일을 열고나서 출력 파일을 위한 메모리를 할당하겠습니다.
```c
if ((ret = avformat_open_input(&input_format_context, in_filename, NULL, NULL)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
if ((ret = avformat_find_stream_info(input_format_context, NULL)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
avformat_alloc_output_context2(&output_format_context, NULL, NULL, out_filename);
if (!output_format_context) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
```
비디오, 오디오, 자막 타입의 스트림만 remux할 것이며 사용하게될 스트림을 인덱스 배열에 들고 있겠습니다.
```c
number_of_streams = input_format_context->nb_streams;
streams_list = av_mallocz_array(number_of_streams, sizeof(*streams_list));
```
필요한만큼의 메모리를 할당한 후, 모든 스트림에 대해 각각 루프를 돌면서 [avformat_new_stream](https://ffmpeg.org/doxygen/trunk/group__lavf__core.html#gadcb0fd3e507d9b58fe78f61f8ad39827) 함수를 통해 출력 포맷 컨텍스트에다가 새로운 출력 스트림을 생성해야합니다. 비디오, 오디오, 자막이 아닌 모든 스트림들에 대해서는 마킹을 해서 나중에 스킵할 수 있게 하겠습니다.
```c
for (i = 0; i < input_format_context->nb_streams; i++) {
AVStream *out_stream;
AVStream *in_stream = input_format_context->streams[i];
AVCodecParameters *in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
streams_list[i] = -1;
continue;
}
streams_list[i] = stream_index++;
out_stream = avformat_new_stream(output_format_context, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
goto end;
}
}
```
이제 출력 파일을 생성할 수 있습니다.
```c
if (!(output_format_context->oformat->flags & AVFMT_NOFILE)) {
ret = avio_open(&output_format_context->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
ret = avformat_write_header(output_format_context, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
```
그런 후에, 입력 스트림에서 패킷을 하나씩 출력 스트림으로 복사하겠습니다. 패킷이 존재하는 동안 (`av_read_frame`), 각 패킷에 대해 PTS와 DTS를 다시 계산하고 마지막으로 포맷 컨텍스트에 (`av_interleaved_write_frame`) 씁니다.
```c
while (1) {
AVStream *in_stream, *out_stream;
ret = av_read_frame(input_format_context, &packet);
if (ret < 0)
break;
in_stream = input_format_context->streams[packet.stream_index];
if (packet.stream_index >= number_of_streams || streams_list[packet.stream_index] < 0) {
av_packet_unref(&packet);
continue;
}
packet.stream_index = streams_list[packet.stream_index];
out_stream = output_format_context->streams[packet.stream_index];
/* copy packet */
packet.pts = av_rescale_q_rnd(packet.pts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.dts = av_rescale_q_rnd(packet.dts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
packet.duration = av_rescale_q(packet.duration, in_stream->time_base, out_stream->time_base);
// https://ffmpeg.org/doxygen/trunk/structAVPacket.html#ab5793d8195cf4789dfb3913b7a693903
packet.pos = -1;
//https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga37352ed2c63493c38219d935e71db6c1
ret = av_interleaved_write_frame(output_format_context, &packet);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&packet);
}
```
마무리를 위해 [av_write_trailer](https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga7f14007e7dc8f481f054b21614dfec13) 함수를 통해 스트림 트레일러(trailer)를 출력 미디어 파일에 씁니다.
```c
av_write_trailer(output_format_context);
```
이제 테스트할 준비가 되었습니다. 첫번째 테스트는 MP4에서 MPEG-TS 비디오 파일로의 포맷 (비디오 컨테이너) 변환입니다. 우리는 기본적으로 `ffmpeg input.mp4 -c copy output.ts` 명령줄을 libav를 이용해 만든 것입니다.
```bash
make run_remuxing_ts
```
동작합니다!!! 절 믿지 않았나요?! 그러시면 안되죠, `ffprobe`로 한번 확인해보겠습니다:
```bash
ffprobe -i remuxed_small_bunny_1080p_60fps.ts
Input #0, mpegts, from 'remuxed_small_bunny_1080p_60fps.ts':
Duration: 00:00:10.03, start: 0.000000, bitrate: 2751 kb/s
Program 1
Metadata:
service_name : Service01
service_provider: FFmpeg
Stream #0:0[0x100]: Video: h264 (High) ([27][0][0][0] / 0x001B), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 60 fps, 60 tbr, 90k tbn, 120 tbc
Stream #0:1[0x101]: Audio: ac3 ([129][0][0][0] / 0x0081), 48000 Hz, 5.1(side), fltp, 320 kb/s
```
우리가 했던 것을 그래프로 정리하기 위해, 초반 [libav의 동작 방식에 대한 아이디어](https://github.com/leandromoreira/ffmpeg-libav-tutorial#ffmpeg-li기bav-architecture)를 다시 한번 살펴보면 코덱 부분만 건너뛴걸 볼 수 있습니다.
이 챕터를 끝내기 전에 리먹싱(remuxing) 프로세스의 중요한 부분을 보여드리고자 합니다, **muxer에 옵션을 줄 수 있다**는 것인데요. 만약에 전송을 [MPEG-DASH](https://developer.mozilla.org/en-US/docs/Web/Apps/Fundamentals/Audio_and_video_delivery/Setting_up_adaptive_streaming_media_sources#MPEG-DASH_Encoding) 포맷으로 하고 싶다면 MPEG-TS나 기본 MPEG-4 대신 (`fmp4`라고 부르는) [fragmented mp4](https://stackoverflow.com/a/35180327)를 사용해야합니다.
[명령줄로는 이렇게 쉽게 할 수 있습니다](https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API/Transcoding_assets_for_MSE#Fragmenting).
```
ffmpeg -i non_fragmented.mp4 -movflags frag_keyframe+empty_moov+default_base_moof fragmented.mp4
```
libav 버전도 거의 명령줄 만큼이나 쉽습니다. 패킷을 복사하기 바로 전, 출력 헤더를 쓸때 해당 옵션을 넘겨주기만 하면 됩니다.
```c
AVDictionary* opts = NULL;
av_dict_set(&opts, "movflags", "frag_keyframe+empty_moov+default_base_moof", 0);
ret = avformat_write_header(output_format_context, &opts);
```
이제 fragmented mp4 파일을 생성할 수 있습니다:
```bash
make run_remuxing_fragmented_mp4
```
제가 여러분께 거짓말하고 있지 않다는걸 보여드리죠. 결과물의 차이를 확인하기 위해 [gpac/mp4box.js](http://download.tsi.telecom-paristech.fr/gpac/mp4box.js/filereader.html) 혹은 [http://mp4parser.com/](http://mp4parser.com/) 같은 아주 훌륭한 사이트/툴을 이용할 수 있습니다. 일단 "common" mp4 파일을 로드해보세요.
보시다시피 단 하나의 `mdat` 박스(atom)가 있습니다, **여기에 비디오와 오디오 프레임이 담겨있습니다**. 이번엔 fragmented mp4를 로드해서 `mdat` 박스가 어떻게 흩어져있는지 보시겠습니다.
## 챕터 3 - 트랜스코딩 (transcoding)
> #### TLDR; [코드](/3_transcoding.c)랑 실행하는거나 보여주세요.
> ```bash
> $ make run_transcoding
> ```
> 좀 상세한 부분은 넘어가겠습니다, 그러나 걱정하진 마세요: [소스 코드는 github에 있습니다](/3_transcoding.c).
이번 챕터에서는 아주 간단한 트랜스코더를 만들어보겠습니다, C로 작성할 것이고, 이것으로 H264로 인코딩된 비디오를 H265로 변환할 수 있을겁니다. **FFmpeg/libav** 라이브러리, 특히 [libavcodec](https://ffmpeg.org/libavcodec.html), libavformat, libavutil를 이용하겠습니다.
> _빠르게 복습해보면:_ [**AVFormatContext**](https://www.ffmpeg.org/doxygen/trunk/structAVFormatContext.html)는 컨테이너 (ex: MKV, MP4, Webm, TS) 같은 미디어 파일 포맷에 대한 추상화 구조체입니다. [**AVStream**](https://www.ffmpeg.org/doxygen/trunk/structAVStream.html)는 주어진 포맷 (ex: 오디오, 비디오, 자막, 메타데이터)에 대한 각 데이터 유형을 나타냅니다. [**AVPacket**](https://www.ffmpeg.org/doxygen/trunk/structAVPacket.html)은 `AVStream`으로부터 얻어진 압축된 데이터의 조각입니다. 그리고 이것은 [**AVCodec**](https://www.ffmpeg.org/doxygen/trunk/structAVCodec.html) (ex: av1, h264, vp9, hevc)에 의해 디코딩되어 [**AVFrame**](https://www.ffmpeg.org/doxygen/trunk/structAVFrame.html)라고 불리는 raw 데이터로 만들어집니다.
### 트랜스먹싱 (Transmuxing)
간단한 트랜스먹싱 작업을 시작해봅시다. 그리고나서 이 코드 기반으로 빌드할 수 있을겁니다. 첫번째 단계는 **입력 파일 로드하기**입니다.
```c
// Allocate an AVFormatContext
avfc = avformat_alloc_context();
// Open an input stream and read the header.
avformat_open_input(avfc, in_filename, NULL, NULL);
// Read packets of a media file to get stream information.
avformat_find_stream_info(avfc, NULL);
```
이제 디코더를 설정할 것인데, `AVFormatContext`가 모든 `AVStream` 컴포넌트에 접근할 수 있게 해줄 것입니다. 그리고 각각의 스트림에 대해서, `AVCodec`을 가져와서 특정 `AVCodecContext`를 생성합니다. 그리고 마지막으로 주어진 코덱을 열게되고 디코딩 프로세스를 수행할 수 있습니다.
> [**AVCodecContext**](https://www.ffmpeg.org/doxygen/trunk/structAVCodecContext.html)는 비트레이트, 프레임 속도, 샘플레이트, 채널, 높이 등과 같은 미디어 설정에 대한 데이터를 가지고 있습니다.
```c
for (int i = 0; i < avfc->nb_streams; i++)
{
AVStream *avs = avfc->streams[i];
AVCodec *avc = avcodec_find_decoder(avs->codecpar->codec_id);
AVCodecContext *avcc = avcodec_alloc_context3(*avc);
avcodec_parameters_to_context(*avcc, avs->codecpar);
avcodec_open2(*avcc, *avc, NULL);
}
```
마찬가지로 트랜스먹싱에서도 출력 미디어 파일을 준비해둬야합니다, 우선 출력 `AVFormatContext`에 대해 **메모리를 할당**합니다. 이 출력 포맷에 **각 스트림**을 생성합니다. 스트림을 제대로 적재시키기 위해 디코더로부터 **코덱 파라미터를 복사**합니다.
인코더가 글로벌 헤더를 사용할 수 있도록 지정하는 `AV_CODEC_FLAG_GLOBAL_HEADER` **플래그를 설정**합니다. 그리고 출력으로 **쓰기 위한 파일**을 열고 헤더를 저장합니다.
```c
avformat_alloc_output_context2(&encoder_avfc, NULL, NULL, out_filename);
AVStream *avs = avformat_new_stream(encoder_avfc, NULL);
avcodec_parameters_copy(avs->codecpar, decoder_avs->codecpar);
if (encoder_avfc->oformat->flags & AVFMT_GLOBALHEADER)
encoder_avfc->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
avio_open(&encoder_avfc->pb, encoder->filename, AVIO_FLAG_WRITE);
avformat_write_header(encoder->avfc, &muxer_opts);
```
디코더로부터 `AVPacket`을 얻어서, 타임스탬프를 조정하고, 패킷을 출력 파일에 제대로 씁니다. `av_interleaved_write_frame` 이 함수 이름이 "write frame"라고 되어있긴 하지만 이것은 패킷을 저장합니다 . 이제 파일에 스트림 트레일러를 쓰면서 트랜스먹싱 프로세스를 마무리합니다.
```c
AVFrame *input_frame = av_frame_alloc();
AVPacket *input_packet = av_packet_alloc();
while (av_read_frame(decoder_avfc, input_packet) >= 0)
{
av_packet_rescale_ts(input_packet, decoder_video_avs->time_base, encoder_video_avs->time_base);
av_interleaved_write_frame(*avfc, input_packet) < 0));
}
av_write_trailer(encoder_avfc);
```
### 트랜스코딩 (Transcoding)
이전 섹션에서 간단한 트랜스먹서 프로그램을 봤는데요, 이번엔 여기에 인코딩을 기능을 추가해보겠습니다. 특히, `h264`에서 `h265`로 비디오를 트랜스코딩할 수 있게 하겠습니다.
디코더를 준비한 후, 그리고 출력 미디어 파일을 다루기 전에 인코더를 설정할 것입니다.
* 인코더에 비디오 `AVStream`를 생성합니다, [`avformat_new_stream`](https://www.ffmpeg.org/doxygen/trunk/group__lavf__core.html#gadcb0fd3e507d9b58fe78f61f8ad39827)
* `libx265`라고 하는 `AVCodec`을 사용합니다, [`avcodec_find_encoder_by_name`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__encoding.html#gaa614ffc38511c104bdff4a3afa086d37)
* 생성한 코덱을 기반으로 `AVCodecContext`를 생성합니다,[`avcodec_alloc_context3`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#gae80afec6f26df6607eaacf39b561c315)
* 트랜스코딩 세션에 대해 기본적인 속성들을 설정합니다, 그리고
* 코덱을 열고 컨텍스트에서 스트림으로 파라미터를 복사합니다. [`avcodec_open2`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga11f785a188d7d9df71621001465b0f1d), [`avcodec_parameters_from_context`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__core.html#ga0c7058f764778615e7978a1821ab3cfe)
```c
AVRational input_framerate = av_guess_frame_rate(decoder_avfc, decoder_video_avs, NULL);
AVStream *video_avs = avformat_new_stream(encoder_avfc, NULL);
char *codec_name = "libx265";
char *codec_priv_key = "x265-params";
// we're going to use internal options for the x265
// it disables the scene change detection and fix then
// GOP on 60 frames.
char *codec_priv_value = "keyint=60:min-keyint=60:scenecut=0";
AVCodec *video_avc = avcodec_find_encoder_by_name(codec_name);
AVCodecContext *video_avcc = avcodec_alloc_context3(video_avc);
// encoder codec params
av_opt_set(sc->video_avcc->priv_data, codec_priv_key, codec_priv_value, 0);
video_avcc->height = decoder_ctx->height;
video_avcc->width = decoder_ctx->width;
video_avcc->pix_fmt = video_avc->pix_fmts[0];
// control rate
video_avcc->bit_rate = 2 * 1000 * 1000;
video_avcc->rc_buffer_size = 4 * 1000 * 1000;
video_avcc->rc_max_rate = 2 * 1000 * 1000;
video_avcc->rc_min_rate = 2.5 * 1000 * 1000;
// time base
video_avcc->time_base = av_inv_q(input_framerate);
video_avs->time_base = sc->video_avcc->time_base;
avcodec_open2(sc->video_avcc, sc->video_avc, NULL);
avcodec_parameters_from_context(sc->video_avs->codecpar, sc->video_avcc);
```
비디오 스트림의 트랜스코딩을 위해 디코딩 루프를 확장해야합니다.
* 디코더에 빈 `AVPacket`를 전송합니다, [`avcodec_send_packet`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga58bc4bf1e0ac59e27362597e467efff3)
* 압축이 해제된 `AVFrame`를 받아옵니다, [`avcodec_receive_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga11e6542c4e66d3028668788a1a74217c)
* 이 raw 프레임의 트랜스코딩을 시작합니다,
* raw 프레임을 (인코더에) 보내고, [`avcodec_send_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga9395cb802a5febf1f00df31497779169)
* 코덱에 맞게 압축된 `AVPacket`을 받아옵니다, [`avcodec_receive_packet`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__decoding.html#ga5b8eff59cf259747cf0b31563e38ded6)
* 타임스탬프를 설정하고, [`av_packet_rescale_ts`](https://www.ffmpeg.org/doxygen/trunk/group__lavc__packet.html#gae5c86e4d93f6e7aa62ef2c60763ea67e)
* 패킷을 출력 파일에 씁니다. [`av_interleaved_write_frame`](https://www.ffmpeg.org/doxygen/trunk/group__lavf__encoding.html#ga37352ed2c63493c38219d935e71db6c1)
```c
AVFrame *input_frame = av_frame_alloc();
AVPacket *input_packet = av_packet_alloc();
while (av_read_frame(decoder_avfc, input_packet) >= 0)
{
int response = avcodec_send_packet(decoder_video_avcc, input_packet);
while (response >= 0) {
response = avcodec_receive_frame(decoder_video_avcc, input_frame);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
return response;
}
if (response >= 0) {
encode(encoder_avfc, decoder_video_avs, encoder_video_avs, decoder_video_avcc, input_packet->stream_index);
}
av_frame_unref(input_frame);
}
av_packet_unref(input_packet);
}
av_write_trailer(encoder_avfc);
// used function
int encode(AVFormatContext *avfc, AVStream *dec_video_avs, AVStream *enc_video_avs, AVCodecContext video_avcc int index) {
AVPacket *output_packet = av_packet_alloc();
int response = avcodec_send_frame(video_avcc, input_frame);
while (response >= 0) {
response = avcodec_receive_packet(video_avcc, output_packet);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
break;
} else if (response < 0) {
return -1;
}
output_packet->stream_index = index;
output_packet->duration = enc_video_avs->time_base.den / enc_video_avs->time_base.num / dec_video_avs->avg_frame_rate.num * dec_video_avs->avg_frame_rate.den;
av_packet_rescale_ts(output_packet, dec_video_avs->time_base, enc_video_avs->time_base);
response = av_interleaved_write_frame(avfc, output_packet);
}
av_packet_unref(output_packet);
av_packet_free(&output_packet);
return 0;
}
```
아시다시피 `h265` 버전의 미디어 파일이 `h264`보다 사이즈가 작기 때문에 미디어 스트림을 `h264`에서 `h265`로 변환했습니다. 하지만 [작성한 프로그램](/3_transcoding.c)은 다음의 작업들도 수행할 수 있습니다:
```c
/*
* H264 -> H265
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx265";
sp.codec_priv_key = "x265-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
/*
* H264 -> H264 (fixed gop)
* Audio -> remuxed (untouched)
* MP4 - fragmented MP4
*/
StreamingParams sp = {0};
sp.copy_audio = 1;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
sp.muxer_opt_key = "movflags";
sp.muxer_opt_value = "frag_keyframe+empty_moov+delay_moov+default_base_moof";
/*
* H264 -> H264 (fixed gop)
* Audio -> AAC
* MP4 - MPEG-TS
*/
StreamingParams sp = {0};
sp.copy_audio = 0;
sp.copy_video = 0;
sp.video_codec = "libx264";
sp.codec_priv_key = "x264-params";
sp.codec_priv_value = "keyint=60:min-keyint=60:scenecut=0:force-cfr=1";
sp.audio_codec = "aac";
sp.output_extension = ".ts";
/* WIP :P -> it's not playing on VLC, the final bit rate is huge
* H264 -> VP9
* Audio -> Vorbis
* MP4 - WebM
*/
//StreamingParams sp = {0};
//sp.copy_audio = 0;
//sp.copy_video = 0;
//sp.video_codec = "libvpx-vp9";
//sp.audio_codec = "libvorbis";
//sp.output_extension = ".webm";
```
> 이제서야 솔직히 말하자면, [제가 생각했던 것보다 더 삽질](https://github.com/leandromoreira/ffmpeg-libav-tutorial/pull/53)했는데요. [FFmpeg 명령줄 소스코드](https://github.com/leandromoreira/ffmpeg-libav-tutorial/pull/54#issuecomment-570746749)를 파봐야했고 테스트도 엄청 돌려봤습니다. 그리고 제가 뭔가 놓치는게 있는 것 같은데요, 왜냐하면 `force-cfr`을 강제로 넣어줘야지만 `h264`가 작용하고 `warning messages (forced frame type (5) at 80 was changed to frame type (3))` 같은 경고 메시지도 여전히 나고 있기 때문이죠.
================================================
FILE: README-pt.md
================================================
[](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)
Eu estava procurando por um tutorial/livro que me ensinasse a começar a usar o [FFmpeg](https://www.ffmpeg.org/) como biblioteca (também conhecida como libav) e então encontrei o tutorial ["Como escrever um player de vídeo em menos de 1k linhas"](http://dranger.com/ffmpeg/).
Infelizmente, ele foi descontinuado, então decidi escrever este.
A maior parte do código aqui será em C **mas não se preocupe**: você pode facilmente entender e aplicá-lo à sua linguagem preferida.
O FFmpeg libav tem muitas ligações para várias linguagens, como [python](https://pyav.org/), [go](https://github.com/imkira/go-libav) e mesmo que sua linguagem não tenha, ainda é possível suportá-la através do `ffi` (aqui está um exemplo com [Lua](https://github.com/daurnimator/ffmpeg-lua-ffi/blob/master/init.lua)).
Começaremos com uma breve lição sobre o que é vídeo, áudio, codec e contêiner e depois faremos um curso intensivo sobre como usar a linha de comando do `FFmpeg` e, finalmente, escreveremos código. Sinta-se à vontade para pular diretamente para a seção [Aprenda o FFmpeg libav do jeito difícil.](#aprenda-o-ffmpeg-libav-do-modo-difícil)
Algumas pessoas costumavam dizer que o streaming de vídeo na internet é o futuro da TV tradicional, de qualquer forma, o FFmpeg é algo que vale a pena estudar.
__Índice__
* [Introdução](#introdução)
* [Vídeo - O que você vê!](#vídeo---o-que-você-vê)
* [Áudio - O que você ouve!](#áudio---o-que-você-ouve)
* [Codec - reduzindo dados](#codec---reduzindo-dados)
* [Container - um lugar confortável para áudio e vídeo](#container---um-lugar-confortável-para-áudio-e-vídeo)
* [FFmpeg - linha de comando](#ffmpeg---linha-de-comando)
* [Ferramenta de linha de comando do FFmpeg 101](#ferramenta-de-linha-de-comando-do-ffmpeg-101)
* [Operações comuns de vídeo](#operações-comuns-de-vídeo)
* [Transcodificação](#transcodificação)
* [Transmuxing](#transmuxing)
* [Transcodificação de Taxa de Bits](#transcodificação-de-taxa-de-bits)
* [Transdimensionamento](#transdimensionamento)
* [Bônus: Streaming Adaptativo](#bônus-streaming-adaptativo)
* [Indo além](#indo-além)
* [Aprenda o FFmpeg libav do modo difícil](#aprenda-o-ffmpeg-libav-do-modo-difícil)
* [Capítulo 0 - O famoso "hello world"](#capítulo-0---o-famoso-hello-world)
* [Arquitetura da biblioteca FFmpeg libav](#arquitetura-da-biblioteca-ffmpeg-libav)
* [Capítulo 1 - Sincronização de áudio e vídeo](#capítulo-1---sincronização-de-áudio-e-vídeo)
* [Capítulo 2 - Remuxing](#capítulo-2---remuxing)
* [Capítulo 3 - Transcoding](#capítulo-3---transcoding)
# Introdução
## Vídeo - O que você vê!
Se você tem uma sequência de imagens e as altera com uma determinada frequência (digamos [24 imagens por segundo](https://www.filmindependent.org/blog/hacking-film-24-frames-per-second/)), você criará uma [ilusão de movimento](https://en.wikipedia.org/wiki/Persistence_of_vision).
Em resumo, essa é a ideia básica por trás de um vídeo: **uma série de imagens/quadros sendo executados a uma determinada taxa**.
<img src="https://upload.wikimedia.org/wikipedia/commons/1/1f/Linnet_kineograph_1886.jpg" title="flip book" height="280"></img>
Zeitgenössische Illustration (1886)
## Áudio - O que você ouve!
Embora um vídeo sem som possa expressar uma variedade de sentimentos, adicionar som a ele traz mais prazer à experiência.
O som é a vibração que se propaga como uma onda de pressão, através do ar ou qualquer outro meio de transmissão, como um gás, líquido ou sólido.
> Em um sistema de áudio digital, um microfone converte o som em um sinal elétrico analógico, em seguida, um conversor analógico-digital (ADC) - tipicamente usando [modulação por código de pulso (PCM)](https://en.wikipedia.org/wiki/Pulse-code_modulation) - converte o sinal analógico em um sinal digital.
>[Fonte](https://commons.wikimedia.org/wiki/File:CPT-Sound-ADC-DAC.svg)
## Codec - reduzindo dados
> CODEC é um circuito eletrônico ou software que **comprime ou descomprime áudio/vídeo digital.** Ele converte áudio/vídeo digital bruto (não comprimido) para um formato comprimido ou vice-versa.
> https://en.wikipedia.org/wiki/Video_codec
Mas se escolhermos empacotar milhões de imagens em um único arquivo e chamá-lo de filme, podemos acabar com um arquivo enorme. Vamos fazer as contas:
Suponha que estamos criando um vídeo com resolução `1080 x 1920` (altura x largura) e que gastaremos `3 bytes` por pixel (o ponto mínimo em uma tela) para codificar a cor (ou [cor de 24 bits](https://en.wikipedia.org/wiki/Color_depth#True_color_.2824-bit.29), o que nos dá 16.777.216 cores diferentes) e este vídeo é executado a `24 quadros por segundo` e tem `30 minutos` de duração.
```c
toppf = 1080 * 1920 // total_de_pixels_por_quadro
cpp = 3 //custo_por_pixel
tis = 30 * 60 //tempo_em_segundos
fps = 24 //quadros_por_segundo
armazenamento_necessário = tis * fps * toppf * cpp
```
Este vídeo exigiria aproximadamente `250,28 GB` de armazenamento ou `1,19 Gbps` de largura de banda! É por isso que precisamos usar um [CODEC](https://github.com/leandromoreira/digital_video_introduction#how-does-a-video-codec-work).
## container - um lugar confortável para áudio e vídeo
> Um formato de container ou envoltório é um formato de metafile cuja especificação descreve como diferentes elementos de dados e metadados coexistem em um arquivo de computador.
> https://en.wikipedia.org/wiki/Digital_container_format
Um **único arquivo que contém todos os fluxos** (principalmente áudio e vídeo) e também fornece **sincronização e metadados gerais**, como título, resolução, entre outros.
Normalmente, podemos inferir o formato de um arquivo ao olhar para sua extensão: por exemplo, um `video.webm` provavelmente é um vídeo usando o container [`webm`](https://www.webmproject.org/).
# FFmpeg - linha de comando
> Uma solução completa e multiplataforma para gravar, converter e transmitir áudio e vídeo.
Para trabalhar com multimídia, podemos usar a FERRAMENTA/BIBLIOTECA incrível chamada [FFmpeg](https://www.ffmpeg.org/). Provavelmente, você já a conhece/usa diretamente ou indiretamente (você usa o [Chrome?](https://www.chromium.org/developers/design-documents/video)).
Ele tem um programa de linha de comando chamado `ffmpeg`, um binário muito simples, porém poderoso.
Por exemplo, você pode converter de `mp4` para o contêiner `avi` apenas digitando o seguinte comando:
```bash
$ ffmpeg -i input.mp4 output.avi
```
Acabamos de fazer um **remuxing** aqui, que é converter de um contêiner para outro.
Tecnicamente, o FFmpeg também poderia estar fazendo uma transcodificação, mas falaremos sobre isso mais tarde.
## Ferramenta de linha de comando do FFmpeg 101
O FFmpeg possui uma [documentação](https://www.ffmpeg.org/ffmpeg.html) que faz um ótimo trabalho explicando como ele funciona.
```bash
# você também pode procurar a documentação usando a linha de comando
ffmpeg -h full | grep -A 10 -B 10 avoid_negative_ts
```
Resumidamente, o programa de linha de comando do FFmpeg espera o seguinte formato de argumento para executar suas ações: `ffmpeg {1} {2} -i {3} {4} {5}`, onde:
1. opções globais
2. opções do arquivo de entrada
3. URL de entrada
4. opções do arquivo de saída
5. URL de saída
As partes 2, 3, 4 e 5 podem ser quantas você precisar.
É mais fácil entender esse formato de argumento na prática:
```bash
# ATENÇÃO: este arquivo tem cerca de 300MB
$ wget -O bunny_1080p_60fps.mp4 http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_1080p_60fps_normal.mp4
$ ffmpeg \
-y \ # opções globais
-c:a libfdk_aac \ # opções de entrada
-i bunny_1080p_60fps.mp4 \ # URL de entrada
-c:v libvpx-vp9 -c:a libvorbis \ # opções de saída
bunny_1080p_60fps_vp9.webm # URL de saída
```
Este comando leva um arquivo de entrada `mp4` contendo dois fluxos (um áudio codificado com `aac` CODEC e um vídeo codificado usando `h264` CODEC) e o converte para `webm`, mudando seus CODECs de áudio e vídeo também.
Podemos simplificar o comando acima, mas esteja ciente de que o FFmpeg adotará ou adivinhará os valores padrão para você.
Por exemplo, quando você apenas digita `ffmpeg -i input.avi output.mp4`, que CODEC de áudio/vídeo ele usa para produzir o `output.mp4`?
Werner Robitza escreveu um [tutorial obrigatório para ler/executar sobre codificação e edição com FFmpeg](http://slhck.info/ffmpeg-encoding-course/#/).
# Operações comuns de vídeo
Ao trabalhar com áudio/vídeo, geralmente realizamos um conjunto de tarefas com a mídia.
## Transcodificação
**O que é?** É o ato de converter um dos fluxos (áudio ou vídeo) de um CODEC para outro.
**Por que?** Às vezes, alguns dispositivos (TVs, smartphones, consoles etc.) não suportam X, mas sim Y, e os novos CODECs fornecem melhor taxa de compressão.
**Como?** Convertendo um vídeo `H264` (AVC) para `H265` (HEVC).
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c:v libx265 \
bunny_1080p_60fps_h265.mp4
```
## Transmuxing
**O que é?** É o ato de converter de um formato (container) para outro.
**Por que?** Às vezes, alguns dispositivos (TVs, smartphones, consoles, etc.) não suportam o formato X, mas suportam o Y e, às vezes, os novos formatos (containers) fornecem recursos modernos necessários.
**Como?** Converter um arquivo `mp4` para `ts`.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-c copy \ # just saying to ffmpeg to skip encoding
bunny_1080p_60fps.ts
```
## Transcodificação de Taxa de Bits
**O que é?** É a alteração da taxa de bits de um vídeo, ou a produção de outras versões do mesmo vídeo.
**Por que fazer?** As pessoas podem tentar assistir ao seu vídeo em uma conexão de rede `2G` (edge) usando um smartphone menos potente ou em uma conexão de fibra óptica em suas TVs 4K. Portanto, você deve oferecer mais de uma versão do mesmo vídeo com diferentes taxas de bits.
**Como fazer?** Produzindo uma versão com taxa de bits entre 3856K e 2000K.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-minrate 964K -maxrate 3856K -bufsize 2000K \
bunny_1080p_60fps_transrating_964_3856.mp4
```
Geralmente, a transcodificação de taxa de bits é usada em conjunto com a transcodificação de tamanho de vídeo. Werner Robitza escreveu outra série de posts que deve ser lida/executada sobre o controle de taxa do FFmpeg (http://slhck.info/posts/).
## Transdimensionamento
**O que é?** a ação de converter de uma resolução para outra. Como mencionado antes, o transdimensionamento é frequentemente usado junto com o transrating.
**Por quê?** as razões são as mesmas que para o transrating.
**Como?** convertendo uma resolução `1080p` para `480p`.
```bash
$ ffmpeg \
-i bunny_1080p_60fps.mp4 \
-vf scale=480:-1 \
bunny_1080p_60fps_transsizing_480.mp4
```
## Bônus: Streaming Adaptativo
**O que é?** A produção de várias resoluções (taxas de bits) e a divisão da mídia em pedaços para serem servidos por HTTP.
**Por que?** Para fornecer uma mídia flexível que possa ser assistida em um smartphone de baixo desempenho ou em uma TV 4K, além de ser fácil de dimensionar e implantar, mas pode adicionar latência.
**Como?** Criando um WebM adaptativo usando o DASH.
```bash
# video streams
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 160x90 -b:v 250k -keyint_min 150 -g 150 -an -f webm -dash 1 video_160x90_250k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 320x180 -b:v 500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_320x180_500k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 750k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_750k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 640x360 -b:v 1000k -keyint_min 150 -g 150 -an -f webm -dash 1 video_640x360_1000k.webm
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:v libvpx-vp9 -s 1280x720 -b:v 1500k -keyint_min 150 -g 150 -an -f webm -dash 1 video_1280x720_1500k.webm
# audio streams
$ ffmpeg -i bunny_1080p_60fps.mp4 -c:a libvorbis -b:a 128k -vn -f webm -dash 1 audio_128k.webm
# the DASH manifest
$ ffmpeg \
-f webm_dash_manifest -i video_160x90_250k.webm \
-f webm_dash_manifest -i video_320x180_500k.webm \
-f webm_dash_manifest -i video_640x360_750k.webm \
-f webm_dash_manifest -i video_640x360_1000k.webm \
-f webm_dash_manifest -i video_1280x720_500k.webm \
-f webm_dash_manifest -i audio_128k.webm \
-c copy -map 0 -map 1 -map 2 -map 3 -map 4 -map 5 \
-f webm_dash_manifest \
-adaptation_sets "id=0,streams=0,1,2,3,4 id=1,streams=5" \
manifest.mpd
```
PS: Eu roubei esse exemplo das [Instruções para reproduzir Adaptive WebM usando DASH](http://wiki.webmproject.org/adaptive-streaming/instructions-to-playback-adaptive-webm-using-dash)
## Indo além
Existem [muitos outros usos para o FFmpeg](https://github.com/leandromoreira/digital_video_introduction/blob/master/encoding_pratical_examples.md#split-and-merge-smoothly).
Eu uso em conjunto com o *iMovie* para produzir/editar alguns vídeos para o YouTube e certamente você pode usá-lo profissionalmente.
# Aprenda o FFmpeg libav do modo difícil
> Você nunca se perguntou sobre som e visão?
> **David Robert Jones**
Já que o [FFmpeg](#ffmpeg---linha-de-comando) é tão útil como uma ferramenta de linha de comando para realizar tarefas essenciais em arquivos de mídia, como podemos usá-lo em nossos programas?
O FFmpeg é [composto por diversas bibliotecas](https://www.ffmpeg.org/doxygen/trunk/index.html) que podem ser integradas em nossos próprios programas. Geralmente, quando você instala o FFmpeg, ele instala automaticamente todas essas bibliotecas. Estarei me referindo a esse conjunto de bibliotecas como **FFmpeg libav**.
> Este título é uma homenagem à série de Zed Shaw [Aprenda X do Modo Difícil](https://learncodethehardway.org/), em particular seu livro Aprenda C do Modo Difícil.
## Capítulo 0 - O famoso "hello world"
Este "hello world" na verdade não mostrará a mensagem "hello world" no terminal :tongue:
Em vez disso, vamos **imprimir informações sobre o vídeo**, como seu formato (container), duração, resolução, canais de áudio e, no final, vamos **decodificar alguns quadros e salvá-los como arquivos de imagem**.
### Arquitetura da biblioteca FFmpeg libav
Mas antes de começarmos a programar, vamos aprender como funciona a **arquitetura da biblioteca FFmpeg libav** e como seus componentes se comunicam entre si.
Aqui está um diagrama do processo de decodificação de um vídeo:
Você primeiro precisará carregar seu arquivo de mídia em um componente chamado [`AVFormatContext`](https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html) (o contêiner de vídeo também é conhecido como formato).
Na verdade, ele não carrega todo o arquivo: muitas vezes ele lê apenas o cabeçalho.
Depois de carregar o **cabeçalho mínimo do nosso contêiner**, podemos acessar suas streams (pense nelas como dado
gitextract_utsib4k_/ ├── .github/ │ └── workflows/ │ └── docker-image.yml ├── .gitignore ├── 0_hello_world.c ├── 2_remuxing.c ├── 3_transcoding.c ├── CMakeLists.txt ├── Dockerfile ├── LICENSE ├── Makefile ├── README-cn.md ├── README-es.md ├── README-ko.md ├── README-pt.md ├── README-ru.md ├── README-vn.md ├── README.md ├── build/ │ └── .gitignore ├── fetch_bbb_video.sh ├── remuxed_small_bunny_1080p_60fps.ts ├── video_debugging.c └── video_debugging.h
SYMBOL INDEX (22 symbols across 4 files)
FILE: 0_hello_world.c
function main (line 29) | int main(int argc, const char *argv[])
function logging (line 203) | static void logging(const char *fmt, ...)
function decode_packet (line 213) | static int decode_packet(AVPacket *pPacket, AVCodecContext *pCodecContex...
function save_gray_frame (line 265) | static void save_gray_frame(unsigned char *buf, int wrap, int xsize, int...
FILE: 2_remuxing.c
function main (line 5) | int main(int argc, char **argv)
FILE: 3_transcoding.c
type StreamingParams (line 12) | typedef struct StreamingParams {
type StreamingContext (line 24) | typedef struct StreamingContext {
function fill_stream_info (line 37) | int fill_stream_info(AVStream *avs, AVCodec **avc, AVCodecContext **avcc) {
function open_media (line 50) | int open_media(const char *in_filename, AVFormatContext **avfc) {
function prepare_decoder (line 60) | int prepare_decoder(StreamingContext *sc) {
function prepare_video_encoder (line 80) | int prepare_video_encoder(StreamingContext *sc, AVCodecContext *decoder_...
function prepare_audio_encoder (line 114) | int prepare_audio_encoder(StreamingContext *sc, int sample_rate, Streami...
function prepare_copy (line 141) | int prepare_copy(AVFormatContext *avfc, AVStream **avs, AVCodecParameter...
function remux (line 147) | int remux(AVPacket **pkt, AVFormatContext **avfc, AVRational decoder_tb,...
function encode_video (line 153) | int encode_video(StreamingContext *decoder, StreamingContext *encoder, A...
function encode_audio (line 182) | int encode_audio(StreamingContext *decoder, StreamingContext *encoder, A...
function transcode_audio (line 208) | int transcode_audio(StreamingContext *decoder, StreamingContext *encoder...
function transcode_video (line 229) | int transcode_video(StreamingContext *decoder, StreamingContext *encoder...
function main (line 250) | int main(int argc, char *argv[])
FILE: video_debugging.c
function logging (line 12) | void logging(const char *fmt, ...)
function log_packet (line 22) | void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt)
function print_timing (line 33) | void print_timing(char *name, AVFormatContext *avf, AVCodecContext *avc,...
Condensed preview — 21 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (389K chars).
[
{
"path": ".github/workflows/docker-image.yml",
"chars": 374,
"preview": "name: Docker Image CI\n\non:\n push:\n branches: [ \"master\" ]\n pull_request:\n branches: [ \"master\" ]\n\njobs:\n\n build"
},
{
"path": ".gitignore",
"chars": 131,
"preview": "*pgm\nbuild/*\nbunny_1080p_60fps.mp4\nbunny_1s_gop.mp4\nbunny_1s_gop.mp4.ts\nbunny_1s_gop.mp4.webm\n.vscode\n.clangd\ncompile_co"
},
{
"path": "0_hello_world.c",
"chars": 10636,
"preview": "/*\n * http://ffmpeg.org/doxygen/trunk/index.html\n *\n * Main components\n *\n * Format (Container) - a wrapper, providing s"
},
{
"path": "2_remuxing.c",
"chars": 5219,
"preview": "// based on https://ffmpeg.org/doxygen/trunk/remuxing_8c-example.html\n#include <libavutil/timestamp.h>\n#include <libavfo"
},
{
"path": "3_transcoding.c",
"chars": 15524,
"preview": "#include <libavcodec/avcodec.h>\n#include <libavformat/avformat.h>\n#include <libavutil/timestamp.h>\n#include <stdio.h>\n#i"
},
{
"path": "CMakeLists.txt",
"chars": 1297,
"preview": "cmake_minimum_required(VERSION 3.17)\nproject(libav_tutorial)\n\n# set out directory\nset(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${C"
},
{
"path": "Dockerfile",
"chars": 22741,
"preview": "# ffmpeg - http://ffmpeg.org/download.html\n#\n# From https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu\n#\n# https://hub"
},
{
"path": "LICENSE",
"chars": 1515,
"preview": "BSD 3-Clause License\n\nCopyright (c) 2017, Leandro Moreira\nAll rights reserved.\n\nRedistribution and use in source and bin"
},
{
"path": "Makefile",
"chars": 1952,
"preview": "usage:\n\techo \"make fetch_small_bunny_video && make run_hello\"\n\nall: clean fetch_bbb_video make_hello run_hello make_remu"
},
{
"path": "README-cn.md",
"chars": 33784,
"preview": "[](https://img.shields.io/badge/license-BSD--3--"
},
{
"path": "README-es.md",
"chars": 50875,
"preview": "[](https://img.shields.io/badge/license-BSD--3--"
},
{
"path": "README-ko.md",
"chars": 39084,
"preview": "[](https://img.shields.io/badge/license-BSD--3--"
},
{
"path": "README-pt.md",
"chars": 49185,
"preview": "[](https://img.shields.io/badge/license-BSD--3--"
},
{
"path": "README-ru.md",
"chars": 43663,
"preview": "[🇨🇳](/README-cn.md \"Simplified Chinese\")\n[🇰🇷](/README-ko.md \"Korean\")\n[🇪🇸](/README-es.md \"Spanish\")\n[🇻🇳](/README-vn.md \""
},
{
"path": "README-vn.md",
"chars": 50270,
"preview": "[](https://img.shields.io/badge/license-BSD--3--"
},
{
"path": "README.md",
"chars": 47604,
"preview": "[🇨🇳](/README-cn.md \"Simplified Chinese\")\n[🇰🇷](/README-ko.md \"Korean\")\n[🇪🇸](/README-es.md \"Spanish\")\n[🇻🇳](/README-vn.md \""
},
{
"path": "build/.gitignore",
"chars": 71,
"preview": "# Ignore everything in this directory\n*\n# Except this file\n!.gitignore\n"
},
{
"path": "fetch_bbb_video.sh",
"chars": 318,
"preview": "#!/bin/bash\n# the link doesn't work anymore\n# wget -O bunny_1080p_60fps.mp4 http://distribution.bbb3d.renderfarming.net"
},
{
"path": "video_debugging.c",
"chars": 2683,
"preview": "#include <libavcodec/avcodec.h>\n#include <libavformat/avformat.h>\n#include <libavutil/timestamp.h>\n#include <stdio.h>\n#i"
},
{
"path": "video_debugging.h",
"chars": 423,
"preview": "#include <libavcodec/avcodec.h>\n#include <libavformat/avformat.h>\n#include <libavutil/timestamp.h>\n#include <stdio.h>\n#i"
}
]
// ... and 1 more files (download for full content)
About this extraction
This page contains the full source code of the leandromoreira/ffmpeg-libav-tutorial GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 21 files (368.5 KB), approximately 118.9k tokens, and a symbol index with 22 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.