OpenShot Library | libopenshot  0.1.8
FFmpegWriter.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegWriter class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegWriter.h"
32 
33 using namespace openshot;
34 
36  path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), audio_pts(0), video_pts(0), samples(NULL),
37  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
38  initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(8), num_of_rescalers(32),
39  rescaler_position(0), video_codec(NULL), audio_codec(NULL), is_writing(false), write_video_count(0), write_audio_count(0),
40  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
41  write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL)
42 {
43 
44  // Disable audio & video (so they can be independently enabled)
45  info.has_audio = false;
46  info.has_video = false;
47 
48  // Initialize FFMpeg, and register all formats and codecs
49  av_register_all();
50 
51  // auto detect format
52  auto_detect_format();
53 }
54 
55 // Open the writer
57 {
58  // Open the writer
59  is_open = true;
60 
61  // Prepare streams (if needed)
62  if (!prepare_streams)
64 
65  // Write header (if needed)
66  if (!write_header)
67  WriteHeader();
68 }
69 
70 // auto detect format (from path)
71 void FFmpegWriter::auto_detect_format()
72 {
73  // Auto detect the output format from the name. default is mpeg.
74  fmt = av_guess_format(NULL, path.c_str(), NULL);
75  if (!fmt)
76  throw InvalidFormat("Could not deduce output format from file extension.", path);
77 
78  // Allocate the output media context
79  oc = avformat_alloc_context();
80  if (!oc)
81  throw OutOfMemory("Could not allocate memory for AVFormatContext.", path);
82 
83  // Set the AVOutputFormat for the current AVFormatContext
84  oc->oformat = fmt;
85 
86  // Update codec names
87  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
88  // Update video codec name
89  info.vcodec = avcodec_find_encoder(fmt->video_codec)->name;
90 
91  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
92  // Update audio codec name
93  info.acodec = avcodec_find_encoder(fmt->audio_codec)->name;
94 }
95 
96 // initialize streams
97 void FFmpegWriter::initialize_streams()
98 {
99  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::initialize_streams", "fmt->video_codec", fmt->video_codec, "fmt->audio_codec", fmt->audio_codec, "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE, "", -1, "", -1, "", -1);
100 
101  // Add the audio and video streams using the default format codecs and initialize the codecs
102  video_st = NULL;
103  audio_st = NULL;
104  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
105  // Add video stream
106  video_st = add_video_stream();
107 
108  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
109  // Add audio stream
110  audio_st = add_audio_stream();
111 }
112 
113 // Set video export options
114 void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
116 {
117  // Set the video options
118  if (codec.length() > 0)
119  {
120  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
121  if (new_codec == NULL)
122  throw InvalidCodec("A valid video codec could not be found for this file.", path);
123  else {
124  // Set video codec
125  info.vcodec = new_codec->name;
126 
127  // Update video codec in fmt
128  fmt->video_codec = new_codec->id;
129  }
130  }
131  if (fps.num > 0)
132  {
133  // Set frames per second (if provided)
134  info.fps.num = fps.num;
135  info.fps.den = fps.den;
136 
137  // Set the timebase (inverse of fps)
140  }
141  if (width >= 1)
142  info.width = width;
143  if (height >= 1)
144  info.height = height;
145  if (pixel_ratio.num > 0)
146  {
147  info.pixel_ratio.num = pixel_ratio.num;
148  info.pixel_ratio.den = pixel_ratio.den;
149  }
150  if (bit_rate >= 1000)
151  info.video_bit_rate = bit_rate;
152 
153  info.interlaced_frame = interlaced;
154  info.top_field_first = top_field_first;
155 
156  // Calculate the DAR (display aspect ratio)
158 
159  // Reduce size fraction
160  size.Reduce();
161 
162  // Set the ratio based on the reduced fraction
163  info.display_ratio.num = size.num;
164  info.display_ratio.den = size.den;
165 
166  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetVideoOptions (" + codec + ")", "width", width, "height", height, "size.num", size.num, "size.den", size.den, "fps.num", fps.num, "fps.den", fps.den);
167 
168  // Enable / Disable video
169  info.has_video = has_video;
170 }
171 
172 // Set audio export options
173 void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
175 {
176  // Set audio options
177  if (codec.length() > 0)
178  {
179  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
180  if (new_codec == NULL)
181  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
182  else
183  {
184  // Set audio codec
185  info.acodec = new_codec->name;
186 
187  // Update audio codec in fmt
188  fmt->audio_codec = new_codec->id;
189  }
190  }
191  if (sample_rate > 7999)
192  info.sample_rate = sample_rate;
193  if (channels > 0)
194  info.channels = channels;
195  if (bit_rate > 999)
196  info.audio_bit_rate = bit_rate;
197  info.channel_layout = channel_layout;
198 
199  // init resample options (if zero)
200  if (original_sample_rate == 0)
201  original_sample_rate = info.sample_rate;
202  if (original_channels == 0)
203  original_channels = info.channels;
204 
205  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetAudioOptions (" + codec + ")", "sample_rate", sample_rate, "channels", channels, "bit_rate", bit_rate, "", -1, "", -1, "", -1);
206 
207  // Enable / Disable audio
208  info.has_audio = has_audio;
209 }
210 
211 // Set custom options (some codecs accept additional params)
212 void FFmpegWriter::SetOption(StreamType stream, string name, string value) throw(NoStreamsFound, InvalidOptions)
213 {
214  // Declare codec context
215  AVCodecContext *c = NULL;
216  stringstream convert(value);
217 
218  if (info.has_video && stream == VIDEO_STREAM && video_st)
219  c = video_st->codec;
220  else if (info.has_audio && stream == AUDIO_STREAM && audio_st)
221  c = audio_st->codec;
222  else
223  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
224 
225  // Init AVOption
226  const AVOption *option = NULL;
227 
228  // Was a codec / stream found?
229  if (c)
230  // Find AVOption (if it exists)
231  #if LIBAVFORMAT_VERSION_MAJOR <= 53
232  option = av_find_opt(c->priv_data, name.c_str(), NULL, NULL, NULL);
233  #else
234  option = av_opt_find(c->priv_data, name.c_str(), NULL, 0, 0);
235  #endif
236 
237  // Was option found?
238  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
239  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate"))
240  {
241  // Check for specific named options
242  if (name == "g")
243  // Set gop_size
244  convert >> c->gop_size;
245 
246  else if (name == "qmin")
247  // Minimum quantizer
248  convert >> c->qmin;
249 
250  else if (name == "qmax")
251  // Maximum quantizer
252  convert >> c->qmax;
253 
254  else if (name == "max_b_frames")
255  // Maximum number of B-frames between non-B-frames
256  convert >> c->max_b_frames;
257 
258  else if (name == "mb_decision")
259  // Macroblock decision mode
260  convert >> c->mb_decision;
261 
262  else if (name == "level")
263  // Set codec level
264  convert >> c->level;
265 
266  else if (name == "profile")
267  // Set codec profile
268  convert >> c->profile;
269 
270  else if (name == "slices")
271  // Indicates number of picture subdivisions
272  convert >> c->slices;
273 
274  else if (name == "rc_min_rate")
275  // Minimum bitrate
276  convert >> c->rc_min_rate;
277 
278  else if (name == "rc_max_rate")
279  // Maximum bitrate
280  convert >> c->rc_max_rate;
281 
282  else if (name == "rc_buffer_size")
283  // Buffer size
284  convert >> c->rc_buffer_size;
285 
286  else
287  // Set AVOption
288  #if LIBAVFORMAT_VERSION_MAJOR <= 53
289  av_set_string3 (c->priv_data, name.c_str(), value.c_str(), 0, NULL);
290  #else
291  av_opt_set (c->priv_data, name.c_str(), value.c_str(), 0);
292  #endif
293 
294  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetOption (" + (string)name + ")", "stream == VIDEO_STREAM", stream == VIDEO_STREAM, "", -1, "", -1, "", -1, "", -1, "", -1);
295 
296  }
297  else
298  throw InvalidOptions("The option is not valid for this codec.", path);
299 
300 }
301 
302 // Prepare & initialize streams and open codecs
304 {
305  if (!info.has_audio && !info.has_video)
306  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
307 
308  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::PrepareStreams [" + path + "]", "info.has_audio", info.has_audio, "info.has_video", info.has_video, "", -1, "", -1, "", -1, "", -1);
309 
310  // Initialize the streams (i.e. add the streams)
311  initialize_streams();
312 
313  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
314  if (info.has_video && video_st)
315  open_video(oc, video_st);
316  if (info.has_audio && audio_st)
317  open_audio(oc, audio_st);
318 
319  // Mark as 'prepared'
320  prepare_streams = true;
321 }
322 
323 // Write the file header (after the options are set)
325 {
326  if (!info.has_audio && !info.has_video)
327  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
328 
329  // Open the output file, if needed
330  if (!(fmt->flags & AVFMT_NOFILE)) {
331  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
332  throw InvalidFile("Could not open or write file.", path);
333  }
334 
335  // Force the output filename (which doesn't always happen for some reason)
336  snprintf(oc->filename, sizeof(oc->filename), "%s", path.c_str());
337 
338  // Write the stream header, if any
339  // TODO: add avoptions / parameters instead of NULL
340  avformat_write_header(oc, NULL);
341 
342  // Mark as 'written'
343  write_header = true;
344 
345  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
346 }
347 
348 // Add a frame to the queue waiting to be encoded.
349 void FFmpegWriter::WriteFrame(std::shared_ptr<Frame> frame) throw(ErrorEncodingVideo, WriterClosed)
350 {
351  // Check for open reader (or throw exception)
352  if (!is_open)
353  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
354 
355  // Add frame pointer to "queue", waiting to be processed the next
356  // time the WriteFrames() method is called.
357  if (info.has_video && video_st)
358  spooled_video_frames.push_back(frame);
359 
360  if (info.has_audio && audio_st)
361  spooled_audio_frames.push_back(frame);
362 
363  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame", "frame->number", frame->number, "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "cache_size", cache_size, "is_writing", is_writing, "", -1);
364 
365  // Write the frames once it reaches the correct cache size
366  if (spooled_video_frames.size() == cache_size || spooled_audio_frames.size() == cache_size)
367  {
368  // Is writer currently writing?
369  if (!is_writing)
370  // Write frames to video file
371  write_queued_frames();
372 
373  else
374  {
375  // YES, WRITING... so wait until it finishes, before writing again
376  while (is_writing)
377  usleep(250000); // sleep for 250 milliseconds
378 
379  // Write frames to video file
380  write_queued_frames();
381  }
382  }
383 
384  // Keep track of the last frame added
385  last_frame = frame;
386 }
387 
388 // Write all frames in the queue to the video file.
389 void FFmpegWriter::write_queued_frames() throw (ErrorEncodingVideo)
390 {
391  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_queued_frames", "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "", -1, "", -1, "", -1, "", -1);
392 
393  // Flip writing flag
394  is_writing = true;
395 
396  // Transfer spool to queue
397  queued_video_frames = spooled_video_frames;
398  queued_audio_frames = spooled_audio_frames;
399 
400  // Empty spool
401  spooled_video_frames.clear();
402  spooled_audio_frames.clear();
403 
404  // Set the number of threads in OpenMP
405  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
406  // Allow nested OpenMP sections
407  omp_set_nested(true);
408 
409  // Create blank exception
410  bool has_error_encoding_video = false;
411 
412  #pragma omp parallel
413  {
414  #pragma omp single
415  {
416  // Process all audio frames (in a separate thread)
417  if (info.has_audio && audio_st && !queued_audio_frames.empty())
418  write_audio_packets(false);
419 
420  // Loop through each queued image frame
421  while (!queued_video_frames.empty())
422  {
423  // Get front frame (from the queue)
424  std::shared_ptr<Frame> frame = queued_video_frames.front();
425 
426  // Add to processed queue
427  processed_frames.push_back(frame);
428 
429  // Encode and add the frame to the output file
430  if (info.has_video && video_st)
431  process_video_packet(frame);
432 
433  // Remove front item
434  queued_video_frames.pop_front();
435 
436  } // end while
437  } // end omp single
438 
439  #pragma omp single
440  {
441  // Loop back through the frames (in order), and write them to the video file
442  while (!processed_frames.empty())
443  {
444  // Get front frame (from the queue)
445  std::shared_ptr<Frame> frame = processed_frames.front();
446 
447  if (info.has_video && video_st)
448  {
449  // Add to deallocate queue (so we can remove the AVFrames when we are done)
450  deallocate_frames.push_back(frame);
451 
452  // Does this frame's AVFrame still exist
453  if (av_frames.count(frame))
454  {
455  // Get AVFrame
456  AVFrame *frame_final = av_frames[frame];
457 
458  // Write frame to video file
459  bool success = write_video_packet(frame, frame_final);
460  if (!success)
461  has_error_encoding_video = true;
462  }
463  }
464 
465  // Remove front item
466  processed_frames.pop_front();
467  }
468 
469  // Loop through, and deallocate AVFrames
470  while (!deallocate_frames.empty())
471  {
472  // Get front frame (from the queue)
473  std::shared_ptr<Frame> frame = deallocate_frames.front();
474 
475  // Does this frame's AVFrame still exist
476  if (av_frames.count(frame))
477  {
478  // Get AVFrame
479  AVFrame *av_frame = av_frames[frame];
480 
481  // Deallocate AVPicture and AVFrame
482  av_freep(&(av_frame->data[0]));
483  AV_FREE_FRAME(&av_frame);
484  av_frames.erase(frame);
485  }
486 
487  // Remove front item
488  deallocate_frames.pop_front();
489  }
490 
491  // Done writing
492  is_writing = false;
493 
494  } // end omp single
495  } // end omp parallel
496 
497  // Raise exception from main thread
498  if (has_error_encoding_video)
499  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
500 }
501 
502 // Write a block of frames from a reader
503 void FFmpegWriter::WriteFrame(ReaderBase* reader, long int start, long int length) throw(ErrorEncodingVideo, WriterClosed)
504 {
505  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame (from Reader)", "start", start, "length", length, "", -1, "", -1, "", -1, "", -1);
506 
507  // Loop through each frame (and encoded it)
508  for (long int number = start; number <= length; number++)
509  {
510  // Get the frame
511  std::shared_ptr<Frame> f = reader->GetFrame(number);
512 
513  // Encode frame
514  WriteFrame(f);
515  }
516 }
517 
518 // Write the file trailer (after all frames are written)
520 {
521  // Write any remaining queued frames to video file
522  write_queued_frames();
523 
524  // Process final audio frame (if any)
525  if (info.has_audio && audio_st)
526  write_audio_packets(true);
527 
528  // Flush encoders (who sometimes hold on to frames)
529  flush_encoders();
530 
531  /* write the trailer, if any. The trailer must be written
532  * before you close the CodecContexts open when you wrote the
533  * header; otherwise write_trailer may try to use memory that
534  * was freed on av_codec_close() */
535  av_write_trailer(oc);
536 
537  // Mark as 'written'
538  write_trailer = true;
539 
540  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
541 }
542 
543 // Flush encoders
544 void FFmpegWriter::flush_encoders()
545 {
546  if (info.has_audio && audio_codec && audio_st->codec->codec_type == AVMEDIA_TYPE_AUDIO && audio_codec->frame_size <= 1)
547  return;
548  if (info.has_video && video_st->codec->codec_type == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && video_codec->codec->id == AV_CODEC_ID_RAWVIDEO)
549  return;
550 
551  int error_code = 0;
552  int stop_encoding = 1;
553 
554  // FLUSH VIDEO ENCODER
555  if (info.has_video)
556  for (;;) {
557 
558  // Increment PTS (in frames and scaled to the codec's timebase)
559  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
560 
561  AVPacket pkt;
562  av_init_packet(&pkt);
563  pkt.data = NULL;
564  pkt.size = 0;
565 
566  // Pointer for video buffer (if using old FFmpeg version)
567  uint8_t *video_outbuf = NULL;
568 
569  /* encode the image */
570  int got_packet = 0;
571  int error_code = 0;
572 
573  #if LIBAVFORMAT_VERSION_MAJOR >= 54
574  // Newer versions of FFMpeg
575  error_code = avcodec_encode_video2(video_codec, &pkt, NULL, &got_packet);
576 
577  #else
578  // Older versions of FFmpeg (much sloppier)
579 
580  // Encode Picture and Write Frame
581  int video_outbuf_size = 0;
582 
583  /* encode the image */
584  int out_size = avcodec_encode_video(video_codec, NULL, video_outbuf_size, NULL);
585 
586  /* if zero size, it means the image was buffered */
587  if (out_size > 0) {
588  if(video_codec->coded_frame->key_frame)
589  pkt.flags |= AV_PKT_FLAG_KEY;
590  pkt.data= video_outbuf;
591  pkt.size= out_size;
592 
593  // got data back (so encode this frame)
594  got_packet = 1;
595  }
596  #endif
597 
598  if (error_code < 0) {
599  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
600  }
601  if (!got_packet) {
602  stop_encoding = 1;
603  break;
604  }
605 
606  // Override PTS (in frames and scaled to the codec's timebase)
607  //pkt.pts = write_video_count;
608 
609  // set the timestamp
610  if (pkt.pts != AV_NOPTS_VALUE)
611  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
612  if (pkt.dts != AV_NOPTS_VALUE)
613  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
614  if (pkt.duration > 0)
615  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
616  pkt.stream_index = video_st->index;
617 
618  // Write packet
619  error_code = av_interleaved_write_frame(oc, &pkt);
620  if (error_code < 0) {
621  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
622  }
623 
624  // Deallocate memory (if needed)
625  if (video_outbuf)
626  av_freep(&video_outbuf);
627  }
628 
629  // FLUSH AUDIO ENCODER
630  if (info.has_audio)
631  for (;;) {
632 
633  // Increment PTS (in samples and scaled to the codec's timebase)
634 #if LIBAVFORMAT_VERSION_MAJOR >= 54
635  // for some reason, it requires me to multiply channels X 2
636  write_audio_count += av_rescale_q(audio_input_position / (audio_codec->channels * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)), (AVRational){1, info.sample_rate}, audio_codec->time_base);
637 #else
638  write_audio_count += av_rescale_q(audio_input_position / audio_codec->channels, (AVRational){1, info.sample_rate}, audio_codec->time_base);
639 #endif
640 
641  AVPacket pkt;
642  av_init_packet(&pkt);
643  pkt.data = NULL;
644  pkt.size = 0;
645  pkt.pts = pkt.dts = write_audio_count;
646 
647  /* encode the image */
648  int got_packet = 0;
649  error_code = avcodec_encode_audio2(audio_codec, &pkt, NULL, &got_packet);
650  if (error_code < 0) {
651  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
652  }
653  if (!got_packet) {
654  stop_encoding = 1;
655  break;
656  }
657 
658  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
659  // but it fixes lots of PTS related issues when I do this.
660  pkt.pts = pkt.dts = write_audio_count;
661 
662  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
663  if (pkt.pts != AV_NOPTS_VALUE)
664  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
665  if (pkt.dts != AV_NOPTS_VALUE)
666  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
667  if (pkt.duration > 0)
668  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
669 
670  // set stream
671  pkt.stream_index = audio_st->index;
672  pkt.flags |= AV_PKT_FLAG_KEY;
673 
674  // Write packet
675  error_code = av_interleaved_write_frame(oc, &pkt);
676  if (error_code < 0) {
677  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
678  }
679 
680  // deallocate memory for packet
681  AV_FREE_PACKET(&pkt);
682  }
683 
684 
685 }
686 
687 // Close the video codec
688 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
689 {
690  avcodec_close(st->codec);
691  video_codec = NULL;
692 }
693 
694 // Close the audio codec
695 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
696 {
697  avcodec_close(st->codec);
698  audio_codec = NULL;
699 
700  // Clear buffers
701  delete[] samples;
702  delete[] audio_outbuf;
703  delete[] audio_encoder_buffer;
704  samples = NULL;
705  audio_outbuf = NULL;
706  audio_encoder_buffer = NULL;
707 
708  // Deallocate resample buffer
709  if (avr) {
710  avresample_close(avr);
711  avresample_free(&avr);
712  avr = NULL;
713  }
714 
715  if (avr_planar) {
716  avresample_close(avr_planar);
717  avresample_free(&avr_planar);
718  avr_planar = NULL;
719  }
720 }
721 
722 // Close the writer
724 {
725  // Write trailer (if needed)
726  if (!write_trailer)
727  WriteTrailer();
728 
729  // Close each codec
730  if (video_st)
731  close_video(oc, video_st);
732  if (audio_st)
733  close_audio(oc, audio_st);
734 
735  // Deallocate image scalers
736  if (image_rescalers.size() > 0)
737  RemoveScalers();
738 
739  // Free the streams
740  for (int i = 0; i < oc->nb_streams; i++) {
741  av_freep(&oc->streams[i]->codec);
742  av_freep(&oc->streams[i]);
743  }
744 
745  if (!(fmt->flags & AVFMT_NOFILE)) {
746  /* close the output file */
747  avio_close(oc->pb);
748  }
749 
750  // Reset frame counters
751  write_video_count = 0;
752  write_audio_count = 0;
753 
754  // Free the context
755  av_freep(&oc);
756 
757  // Close writer
758  is_open = false;
759  prepare_streams = false;
760  write_header = false;
761  write_trailer = false;
762 
763  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
764 }
765 
766 // Add an AVFrame to the cache
767 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame* av_frame)
768 {
769  // Add AVFrame to map (if it does not already exist)
770  if (!av_frames.count(frame))
771  {
772  // Add av_frame
773  av_frames[frame] = av_frame;
774  }
775  else
776  {
777  // Do not add, and deallocate this AVFrame
778  AV_FREE_FRAME(&av_frame);
779  }
780 }
781 
782 // Add an audio output stream
783 AVStream* FFmpegWriter::add_audio_stream()
784 {
785  AVCodecContext *c;
786  AVStream *st;
787 
788  // Find the audio codec
789  AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
790  if (codec == NULL)
791  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
792 
793  // Create a new audio stream
794  st = avformat_new_stream(oc, codec);
795  if (!st)
796  throw OutOfMemory("Could not allocate memory for the audio stream.", path);
797 
798  // Set default values
799  avcodec_get_context_defaults3(st->codec, codec);
800 
801  c = st->codec;
802  c->codec_id = codec->id;
803 #if LIBAVFORMAT_VERSION_MAJOR >= 53
804  c->codec_type = AVMEDIA_TYPE_AUDIO;
805 #else
806  c->codec_type = CODEC_TYPE_AUDIO;
807 #endif
808 
809  // Set the sample parameters
810  c->bit_rate = info.audio_bit_rate;
811  c->channels = info.channels;
812 
813  // Set valid sample rate (or throw error)
814  if (codec->supported_samplerates) {
815  int i;
816  for (i = 0; codec->supported_samplerates[i] != 0; i++)
817  if (info.sample_rate == codec->supported_samplerates[i])
818  {
819  // Set the valid sample rate
820  c->sample_rate = info.sample_rate;
821  break;
822  }
823  if (codec->supported_samplerates[i] == 0)
824  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
825  } else
826  // Set sample rate
827  c->sample_rate = info.sample_rate;
828 
829 
830  // Set a valid number of channels (or throw error)
831  int channel_layout = info.channel_layout;
832  if (codec->channel_layouts) {
833  int i;
834  for (i = 0; codec->channel_layouts[i] != 0; i++)
835  if (channel_layout == codec->channel_layouts[i])
836  {
837  // Set valid channel layout
838  c->channel_layout = channel_layout;
839  break;
840  }
841  if (codec->channel_layouts[i] == 0)
842  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
843  } else
844  // Set valid channel layout
845  c->channel_layout = channel_layout;
846 
847  // Choose a valid sample_fmt
848  if (codec->sample_fmts) {
849  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++)
850  {
851  // Set sample format to 1st valid format (and then exit loop)
852  c->sample_fmt = codec->sample_fmts[i];
853  break;
854  }
855  }
856  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
857  // Default if no sample formats found
858  c->sample_fmt = AV_SAMPLE_FMT_S16;
859  }
860 
861  // some formats want stream headers to be separate
862  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
863  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
864 
865  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_audio_stream", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->channels", c->channels, "c->sample_fmt", c->sample_fmt, "c->channel_layout", c->channel_layout, "c->sample_rate", c->sample_rate);
866 
867  return st;
868 }
869 
870 // Add a video output stream
871 AVStream* FFmpegWriter::add_video_stream()
872 {
873  AVCodecContext *c;
874  AVStream *st;
875 
876  // Find the audio codec
877  AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
878  if (codec == NULL)
879  throw InvalidCodec("A valid video codec could not be found for this file.", path);
880 
881  // Create a new stream
882  st = avformat_new_stream(oc, codec);
883  if (!st)
884  throw OutOfMemory("Could not allocate memory for the video stream.", path);
885 
886  // Set default values
887  avcodec_get_context_defaults3(st->codec, codec);
888 
889  c = st->codec;
890  c->codec_id = codec->id;
891 #if LIBAVFORMAT_VERSION_MAJOR >= 53
892  c->codec_type = AVMEDIA_TYPE_VIDEO;
893 #else
894  c->codec_type = CODEC_TYPE_VIDEO;
895 #endif
896 
897  /* Init video encoder options */
898  c->bit_rate = info.video_bit_rate;
899  c->rc_min_rate = info.video_bit_rate - (info.video_bit_rate / 6);
900  c->rc_max_rate = info.video_bit_rate;
901  c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
902  c->qmin = 2;
903  c->qmax = 30;
904 
905  /* resolution must be a multiple of two */
906  // TODO: require /2 height and width
907  c->width = info.width;
908  c->height = info.height;
909 
910  /* time base: this is the fundamental unit of time (in seconds) in terms
911  of which frame timestamps are represented. for fixed-fps content,
912  timebase should be 1/framerate and timestamp increments should be
913  identically 1. */
914  c->time_base.num = info.video_timebase.num;
915  c->time_base.den = info.video_timebase.den;
916  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
917  c->max_b_frames = 10;
918  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
919  /* just for testing, we also add B frames */
920  c->max_b_frames = 2;
921  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
922  /* Needed to avoid using macroblocks in which some coeffs overflow.
923  This does not happen with normal video, it just happens here as
924  the motion of the chroma plane does not match the luma plane. */
925  c->mb_decision = 2;
926  // some formats want stream headers to be separate
927  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
928  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
929 
930  // Find all supported pixel formats for this codec
931  const PixelFormat* supported_pixel_formats = codec->pix_fmts;
932  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
933  // Assign the 1st valid pixel format (if one is missing)
934  if (c->pix_fmt == PIX_FMT_NONE)
935  c->pix_fmt = *supported_pixel_formats;
936  ++supported_pixel_formats;
937  }
938 
939  // Codec doesn't have any pix formats?
940  if (c->pix_fmt == PIX_FMT_NONE) {
941  if(fmt->video_codec == AV_CODEC_ID_RAWVIDEO) {
942  // Raw video should use RGB24
943  c->pix_fmt = PIX_FMT_RGB24;
944 
945  if (strcmp(fmt->name, "gif") != 0)
946  // If not GIF format, skip the encoding process
947  // Set raw picture flag (so we don't encode this video)
948  oc->oformat->flags |= AVFMT_RAWPICTURE;
949  } else {
950  // Set the default codec
951  c->pix_fmt = PIX_FMT_YUV420P;
952  }
953  }
954 
955  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (string)fmt->name + " : " + (string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags, "AVFMT_RAWPICTURE", AVFMT_RAWPICTURE, "", -1);
956 
957  return st;
958 }
959 
960 // open audio codec
961 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st)
962 {
963  AVCodec *codec;
964  audio_codec = st->codec;
965 
966  // Set number of threads equal to number of processors (not to exceed 16)
967  audio_codec->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
968 
969  // Find the audio encoder
970  codec = avcodec_find_encoder(audio_codec->codec_id);
971  if (!codec)
972  throw InvalidCodec("Could not find codec", path);
973 
974  // Open the codec
975  if (avcodec_open2(audio_codec, codec, NULL) < 0)
976  throw InvalidCodec("Could not open codec", path);
977 
978  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
979  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
980  if (audio_codec->frame_size <= 1) {
981  // No frame size found... so calculate
982  audio_input_frame_size = 50000 / info.channels;
983 
984  switch (st->codec->codec_id) {
985  case AV_CODEC_ID_PCM_S16LE:
986  case AV_CODEC_ID_PCM_S16BE:
987  case AV_CODEC_ID_PCM_U16LE:
988  case AV_CODEC_ID_PCM_U16BE:
989  audio_input_frame_size >>= 1;
990  break;
991  default:
992  break;
993  }
994  } else {
995  // Set frame size based on the codec
996  audio_input_frame_size = audio_codec->frame_size;
997  }
998 
999  // Set the initial frame size (since it might change during resampling)
1000  initial_audio_input_frame_size = audio_input_frame_size;
1001 
1002  // Allocate array for samples
1003  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1004 
1005  // Set audio output buffer (used to store the encoded audio)
1006  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1007  audio_outbuf = new uint8_t[audio_outbuf_size];
1008 
1009  // Set audio packet encoding buffer
1010  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1011  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1012 
1013  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_audio", "audio_codec->thread_count", audio_codec->thread_count, "audio_input_frame_size", audio_input_frame_size, "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE, "", -1, "", -1, "", -1);
1014 
1015 }
1016 
1017 // open video codec
1018 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st)
1019 {
1020  AVCodec *codec;
1021  video_codec = st->codec;
1022 
1023  // Set number of threads equal to number of processors (not to exceed 16)
1024  video_codec->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
1025 
1026  /* find the video encoder */
1027  codec = avcodec_find_encoder(video_codec->codec_id);
1028  if (!codec)
1029  throw InvalidCodec("Could not find codec", path);
1030 
1031  /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */
1032  if(video_codec->max_b_frames && video_codec->codec_id != AV_CODEC_ID_MPEG4 && video_codec->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1033  video_codec->max_b_frames = 0;
1034 
1035  /* open the codec */
1036  if (avcodec_open2(video_codec, codec, NULL) < 0)
1037  throw InvalidCodec("Could not open codec", path);
1038 
1039  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video", "video_codec->thread_count", video_codec->thread_count, "", -1, "", -1, "", -1, "", -1, "", -1);
1040 
1041 }
1042 
1043 // write all queued frames' audio to the video file
1044 void FFmpegWriter::write_audio_packets(bool final)
1045 {
1046  #pragma omp task firstprivate(final)
1047  {
1048  // Init audio buffers / variables
1049  int total_frame_samples = 0;
1050  int frame_position = 0;
1051  int channels_in_frame = 0;
1052  int sample_rate_in_frame = 0;
1053  int samples_in_frame = 0;
1054  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1055 
1056  // Create a new array (to hold all S16 audio samples, for the current queued frames
1057  int16_t* all_queued_samples = (int16_t*)av_malloc((sizeof(int16_t)*(queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE)));
1058  int16_t* all_resampled_samples = NULL;
1059  int16_t* final_samples_planar = NULL;
1060  int16_t* final_samples = NULL;
1061 
1062  // Loop through each queued audio frame
1063  while (!queued_audio_frames.empty())
1064  {
1065  // Get front frame (from the queue)
1066  std::shared_ptr<Frame> frame = queued_audio_frames.front();
1067 
1068  // Get the audio details from this frame
1069  sample_rate_in_frame = frame->SampleRate();
1070  samples_in_frame = frame->GetAudioSamplesCount();
1071  channels_in_frame = frame->GetAudioChannelsCount();
1072  channel_layout_in_frame = frame->ChannelsLayout();
1073 
1074 
1075  // Get audio sample array
1076  float* frame_samples_float = NULL;
1077  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1078  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
1079 
1080 
1081  // Calculate total samples
1082  total_frame_samples = samples_in_frame * channels_in_frame;
1083 
1084  // Translate audio sample values back to 16 bit integers
1085  for (int s = 0; s < total_frame_samples; s++, frame_position++)
1086  // Translate sample value and copy into buffer
1087  all_queued_samples[frame_position] = int(frame_samples_float[s] * (1 << 15));
1088 
1089 
1090  // Deallocate float array
1091  delete[] frame_samples_float;
1092 
1093  // Remove front item
1094  queued_audio_frames.pop_front();
1095 
1096  } // end while
1097 
1098 
1099  // Update total samples (since we've combined all queued frames)
1100  total_frame_samples = frame_position;
1101  int remaining_frame_samples = total_frame_samples;
1102  int samples_position = 0;
1103 
1104 
1105  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "final", final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO);
1106 
1107  // Keep track of the original sample format
1108  AVSampleFormat output_sample_fmt = audio_codec->sample_fmt;
1109 
1110  AVFrame *audio_frame = NULL;
1111  if (!final) {
1112  // Create input frame (and allocate arrays)
1113  audio_frame = AV_ALLOCATE_FRAME();
1114  AV_RESET_FRAME(audio_frame);
1115  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1116 
1117  // Fill input frame with sample data
1118  avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples,
1119  audio_encoder_buffer_size, 0);
1120 
1121  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1122  switch (audio_codec->sample_fmt)
1123  {
1124  case AV_SAMPLE_FMT_FLTP:
1125  {
1126  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1127  break;
1128  }
1129  case AV_SAMPLE_FMT_S32P:
1130  {
1131  output_sample_fmt = AV_SAMPLE_FMT_S32;
1132  break;
1133  }
1134  case AV_SAMPLE_FMT_S16P:
1135  {
1136  output_sample_fmt = AV_SAMPLE_FMT_S16;
1137  break;
1138  }
1139  case AV_SAMPLE_FMT_U8P:
1140  {
1141  output_sample_fmt = AV_SAMPLE_FMT_U8;
1142  break;
1143  }
1144  }
1145 
1146  // Update total samples & input frame size (due to bigger or smaller data types)
1147  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1148  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1149 
1150  // Set remaining samples
1151  remaining_frame_samples = total_frame_samples;
1152 
1153  // Create output frame (and allocate arrays)
1154  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1155  AV_RESET_FRAME(audio_converted);
1156  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1157  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1158 
1159  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (1st resampling)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", output_sample_fmt, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
1160 
1161  // setup resample context
1162  if (!avr) {
1163  avr = avresample_alloc_context();
1164  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1165  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1166  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1167  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1168  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1169  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1170  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1171  av_opt_set_int(avr, "out_channels", info.channels, 0);
1172  avresample_open(avr);
1173  }
1174  int nb_samples = 0;
1175 
1176  // Convert audio samples
1177  nb_samples = avresample_convert(avr, // audio resample context
1178  audio_converted->data, // output data pointers
1179  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1180  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1181  audio_frame->data, // input data pointers
1182  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1183  audio_frame->nb_samples); // number of input samples to convert
1184 
1185  // Create a new array (to hold all resampled S16 audio samples)
1186  all_resampled_samples = (int16_t*)av_malloc(sizeof(int16_t) * nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1187 
1188  // Copy audio samples over original samples
1189  memcpy(all_resampled_samples, audio_converted->data[0], nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1190 
1191  // Remove converted audio
1192  av_freep(&(audio_frame->data[0]));
1193  AV_FREE_FRAME(&audio_frame);
1194  av_freep(&audio_converted->data[0]);
1195  AV_FREE_FRAME(&audio_converted);
1196  all_queued_samples = NULL; // this array cleared with above call
1197 
1198  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)", "nb_samples", nb_samples, "remaining_frame_samples", remaining_frame_samples, "", -1, "", -1, "", -1, "", -1);
1199  }
1200 
1201  // Loop until no more samples
1202  while (remaining_frame_samples > 0 || final) {
1203  // Get remaining samples needed for this packet
1204  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1205 
1206  // Determine how many samples we need
1207  int diff = 0;
1208  if (remaining_frame_samples >= remaining_packet_samples)
1209  diff = remaining_packet_samples;
1210  else if (remaining_frame_samples < remaining_packet_samples)
1211  diff = remaining_frame_samples;
1212 
1213  // Copy frame samples into the packet samples array
1214  if (!final)
1215  //TODO: Make this more sane
1216  memcpy(samples + (audio_input_position * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))), all_resampled_samples + samples_position, diff * av_get_bytes_per_sample(output_sample_fmt));
1217 
1218  // Increment counters
1219  audio_input_position += diff;
1220  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1221  remaining_frame_samples -= diff;
1222  remaining_packet_samples -= diff;
1223 
1224  // Do we have enough samples to proceed?
1225  if (audio_input_position < (audio_input_frame_size * info.channels) && !final)
1226  // Not enough samples to encode... so wait until the next frame
1227  break;
1228 
1229  // Convert to planar (if needed by audio codec)
1230  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1231  AV_RESET_FRAME(frame_final);
1232  if (av_sample_fmt_is_planar(audio_codec->sample_fmt))
1233  {
1234  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)", "in_sample_fmt", output_sample_fmt, "out_sample_fmt", audio_codec->sample_fmt, "in_sample_rate", info.sample_rate, "out_sample_rate", info.sample_rate, "in_channels", info.channels, "out_channels", info.channels);
1235 
1236  // setup resample context
1237  if (!avr_planar) {
1238  avr_planar = avresample_alloc_context();
1239  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1240  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1241  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1242  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec->sample_fmt, 0); // planar not allowed here
1243  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1244  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1245  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1246  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1247  avresample_open(avr_planar);
1248  }
1249 
1250  // Create input frame (and allocate arrays)
1251  audio_frame = AV_ALLOCATE_FRAME();
1252  AV_RESET_FRAME(audio_frame);
1253  audio_frame->nb_samples = audio_input_position / info.channels;
1254 
1255  // Create a new array
1256  final_samples_planar = (int16_t*)av_malloc(sizeof(int16_t) * audio_frame->nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1257 
1258  // Copy audio into buffer for frame
1259  memcpy(final_samples_planar, samples, audio_frame->nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1260 
1261  // Fill input frame with sample data
1262  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt, (uint8_t *) final_samples_planar,
1263  audio_encoder_buffer_size, 0);
1264 
1265  // Create output frame (and allocate arrays)
1266  frame_final->nb_samples = audio_input_frame_size;
1267  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels, frame_final->nb_samples, audio_codec->sample_fmt, 0);
1268 
1269  // Convert audio samples
1270  int nb_samples = avresample_convert(avr_planar, // audio resample context
1271  frame_final->data, // output data pointers
1272  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1273  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1274  audio_frame->data, // input data pointers
1275  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1276  audio_frame->nb_samples); // number of input samples to convert
1277 
1278  // Copy audio samples over original samples
1279  if (nb_samples > 0)
1280  memcpy(samples, frame_final->data[0], nb_samples * av_get_bytes_per_sample(audio_codec->sample_fmt) * info.channels);
1281 
1282  // deallocate AVFrame
1283  av_freep(&(audio_frame->data[0]));
1284  AV_FREE_FRAME(&audio_frame);
1285  all_queued_samples = NULL; // this array cleared with above call
1286 
1287  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)", "nb_samples", nb_samples, "", -1, "", -1, "", -1, "", -1, "", -1);
1288 
1289  } else {
1290  // Create a new array
1291  final_samples = new int16_t[audio_input_position * (av_get_bytes_per_sample(audio_codec->sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))];
1292 
1293  // Copy audio into buffer for frame
1294  memcpy(final_samples, samples, audio_input_position * av_get_bytes_per_sample(audio_codec->sample_fmt));
1295 
1296  // Init the nb_samples property
1297  frame_final->nb_samples = audio_input_frame_size;
1298 
1299  // Fill the final_frame AVFrame with audio (non planar)
1300  avcodec_fill_audio_frame(frame_final, audio_codec->channels, audio_codec->sample_fmt, (uint8_t *) final_samples,
1301  audio_encoder_buffer_size, 0);
1302  }
1303 
1304  // Increment PTS (in samples)
1305  write_audio_count += FFMIN(audio_input_frame_size, audio_input_position);
1306  frame_final->pts = write_audio_count; // Set the AVFrame's PTS
1307 
1308  // Init the packet
1309  AVPacket pkt;
1310  av_init_packet(&pkt);
1311  pkt.data = audio_encoder_buffer;
1312  pkt.size = audio_encoder_buffer_size;
1313 
1314  // Set the packet's PTS prior to encoding
1315  pkt.pts = pkt.dts = write_audio_count;
1316 
1317  /* encode the audio samples */
1318  int got_packet_ptr = 0;
1319  int error_code = avcodec_encode_audio2(audio_codec, &pkt, frame_final, &got_packet_ptr);
1320 
1321  /* if zero size, it means the image was buffered */
1322  if (error_code == 0 && got_packet_ptr) {
1323 
1324  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1325  // but it fixes lots of PTS related issues when I do this.
1326  pkt.pts = pkt.dts = write_audio_count;
1327 
1328  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1329  if (pkt.pts != AV_NOPTS_VALUE)
1330  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
1331  if (pkt.dts != AV_NOPTS_VALUE)
1332  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
1333  if (pkt.duration > 0)
1334  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
1335 
1336  // set stream
1337  pkt.stream_index = audio_st->index;
1338  pkt.flags |= AV_PKT_FLAG_KEY;
1339 
1340  /* write the compressed frame in the media file */
1341  int error_code = av_interleaved_write_frame(oc, &pkt);
1342  if (error_code < 0)
1343  {
1344  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1345  }
1346  }
1347 
1348  if (error_code < 0)
1349  {
1350  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1351  }
1352 
1353  // deallocate AVFrame
1354  av_freep(&(frame_final->data[0]));
1355  AV_FREE_FRAME(&frame_final);
1356 
1357  // deallocate memory for packet
1358  AV_FREE_PACKET(&pkt);
1359 
1360  // Reset position
1361  audio_input_position = 0;
1362  final = false;
1363  }
1364 
1365  // Delete arrays (if needed)
1366  if (all_resampled_samples) {
1367  av_freep(&all_resampled_samples);
1368  all_resampled_samples = NULL;
1369  }
1370  if (all_queued_samples) {
1371  av_freep(&all_queued_samples);
1372  all_queued_samples = NULL;
1373  }
1374 
1375  } // end task
1376 }
1377 
1378 // Allocate an AVFrame object
1379 AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer)
1380 {
1381  // Create an RGB AVFrame
1382  AVFrame *new_av_frame = NULL;
1383 
1384  // Allocate an AVFrame structure
1385  new_av_frame = AV_ALLOCATE_FRAME();
1386  if (new_av_frame == NULL)
1387  throw OutOfMemory("Could not allocate AVFrame", path);
1388 
1389  // Determine required buffer size and allocate buffer
1390  *buffer_size = avpicture_get_size(pix_fmt, width, height);
1391 
1392  // Create buffer (if not provided)
1393  if (!new_buffer)
1394  {
1395  // New Buffer
1396  new_buffer = (uint8_t*)av_malloc(*buffer_size * sizeof(uint8_t));
1397  // Attach buffer to AVFrame
1398  avpicture_fill((AVPicture *)new_av_frame, new_buffer, pix_fmt, width, height);
1399  new_av_frame->width = width;
1400  new_av_frame->height = height;
1401  new_av_frame->format = pix_fmt;
1402  }
1403 
1404  // return AVFrame
1405  return new_av_frame;
1406 }
1407 
1408 // process video frame
1409 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame)
1410 {
1411  // Determine the height & width of the source image
1412  int source_image_width = frame->GetWidth();
1413  int source_image_height = frame->GetHeight();
1414 
1415  // Do nothing if size is 1x1 (i.e. no image in this frame)
1416  if (source_image_height == 1 && source_image_width == 1)
1417  return;
1418 
1419  // Init rescalers (if not initialized yet)
1420  if (image_rescalers.size() == 0)
1421  InitScalers(source_image_width, source_image_height);
1422 
1423  // Get a unique rescaler (for this thread)
1424  SwsContext *scaler = image_rescalers[rescaler_position];
1425  rescaler_position++;
1426  if (rescaler_position == num_of_rescalers)
1427  rescaler_position = 0;
1428 
1429  #pragma omp task firstprivate(frame, scaler, source_image_width, source_image_height)
1430  {
1431  // Allocate an RGB frame & final output frame
1432  int bytes_source = 0;
1433  int bytes_final = 0;
1434  AVFrame *frame_source = NULL;
1435  const uchar *pixels = NULL;
1436 
1437  // Get a list of pixels from source image
1438  pixels = frame->GetPixels();
1439 
1440  // Init AVFrame for source image & final (converted image)
1441  frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t*) pixels);
1442  AVFrame *frame_final = allocate_avframe(video_codec->pix_fmt, info.width, info.height, &bytes_final, NULL);
1443 
1444  // Fill with data
1445  avpicture_fill((AVPicture *) frame_source, (uint8_t*)pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
1446  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::process_video_packet", "frame->number", frame->number, "bytes_source", bytes_source, "bytes_final", bytes_final, "", -1, "", -1, "", -1);
1447 
1448  // Resize & convert pixel format
1449  sws_scale(scaler, frame_source->data, frame_source->linesize, 0,
1450  source_image_height, frame_final->data, frame_final->linesize);
1451 
1452  // Add resized AVFrame to av_frames map
1453  #pragma omp critical (av_frames_section)
1454  add_avframe(frame, frame_final);
1455 
1456  // Deallocate memory
1457  AV_FREE_FRAME(&frame_source);
1458 
1459  } // end task
1460 
1461 }
1462 
1463 // write video frame
1464 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame* frame_final)
1465 {
1466  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet", "frame->number", frame->number, "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE, "", -1, "", -1, "", -1, "", -1);
1467 
1468  if (oc->oformat->flags & AVFMT_RAWPICTURE) {
1469  // Raw video case.
1470  AVPacket pkt;
1471  av_init_packet(&pkt);
1472 
1473  pkt.flags |= AV_PKT_FLAG_KEY;
1474  pkt.stream_index= video_st->index;
1475  pkt.data= (uint8_t*)frame_final->data;
1476  pkt.size= sizeof(AVPicture);
1477 
1478  // Increment PTS (in frames and scaled to the codec's timebase)
1479  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1480  pkt.pts = write_video_count;
1481 
1482  /* write the compressed frame in the media file */
1483  int error_code = av_interleaved_write_frame(oc, &pkt);
1484  if (error_code < 0)
1485  {
1486  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1487  return false;
1488  }
1489 
1490  // Deallocate packet
1491  AV_FREE_PACKET(&pkt);
1492 
1493  } else {
1494 
1495  AVPacket pkt;
1496  av_init_packet(&pkt);
1497  pkt.data = NULL;
1498  pkt.size = 0;
1499  pkt.pts = pkt.dts = AV_NOPTS_VALUE;
1500 
1501  // Pointer for video buffer (if using old FFmpeg version)
1502  uint8_t *video_outbuf = NULL;
1503 
1504  // Increment PTS (in frames and scaled to the codec's timebase)
1505  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1506 
1507  // Assign the initial AVFrame PTS from the frame counter
1508  frame_final->pts = write_video_count;
1509 
1510  /* encode the image */
1511  int got_packet_ptr = 0;
1512  int error_code = 0;
1513  #if LIBAVFORMAT_VERSION_MAJOR >= 54
1514  // Newer versions of FFMpeg
1515  error_code = avcodec_encode_video2(video_codec, &pkt, frame_final, &got_packet_ptr);
1516 
1517  #else
1518  // Older versions of FFmpeg (much sloppier)
1519 
1520  // Encode Picture and Write Frame
1521  int video_outbuf_size = 200000;
1522  video_outbuf = (uint8_t*) av_malloc(200000);
1523 
1524  /* encode the image */
1525  int out_size = avcodec_encode_video(video_codec, video_outbuf, video_outbuf_size, frame_final);
1526 
1527  /* if zero size, it means the image was buffered */
1528  if (out_size > 0) {
1529  if(video_codec->coded_frame->key_frame)
1530  pkt.flags |= AV_PKT_FLAG_KEY;
1531  pkt.data= video_outbuf;
1532  pkt.size= out_size;
1533 
1534  // got data back (so encode this frame)
1535  got_packet_ptr = 1;
1536  }
1537  #endif
1538 
1539  /* if zero size, it means the image was buffered */
1540  if (error_code == 0 && got_packet_ptr) {
1541 
1542  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1543  // but it fixes lots of PTS related issues when I do this.
1544  //pkt.pts = pkt.dts = write_video_count;
1545 
1546  // set the timestamp
1547  if (pkt.pts != AV_NOPTS_VALUE)
1548  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
1549  if (pkt.dts != AV_NOPTS_VALUE)
1550  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
1551  if (pkt.duration > 0)
1552  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
1553  pkt.stream_index = video_st->index;
1554 
1555  /* write the compressed frame in the media file */
1556  int error_code = av_interleaved_write_frame(oc, &pkt);
1557  if (error_code < 0)
1558  {
1559  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1560  return false;
1561  }
1562  }
1563 
1564  // Deallocate memory (if needed)
1565  if (video_outbuf)
1566  delete[] video_outbuf;
1567 
1568  // Deallocate packet
1569  AV_FREE_PACKET(&pkt);
1570  }
1571 
1572  // Success
1573  return true;
1574 }
1575 
1576 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
1578 {
1579  // output debug info
1580  av_dump_format(oc, 0, path.c_str(), 1);
1581 }
1582 
1583 // Init a collection of software rescalers (thread safe)
1584 void FFmpegWriter::InitScalers(int source_width, int source_height)
1585 {
1586  // Get the codec
1587  AVCodecContext *c;
1588  c = video_st->codec;
1589 
1590  // Init software rescalers vector (many of them, one for each thread)
1591  for (int x = 0; x < num_of_rescalers; x++)
1592  {
1593  // Init the software scaler from FFMpeg
1594  img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, c->pix_fmt, SWS_FAST_BILINEAR, NULL, NULL, NULL);
1595 
1596  // Add rescaler to vector
1597  image_rescalers.push_back(img_convert_ctx);
1598  }
1599 }
1600 
1601 // Set audio resample options
1602 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
1603  original_sample_rate = sample_rate;
1604  original_channels = channels;
1605 }
1606 
1607 // Remove & deallocate all software scalers
1609 {
1610  // Close all rescalers
1611  for (int x = 0; x < num_of_rescalers; x++)
1612  sws_freeContext(image_rescalers[x]);
1613 
1614  // Clear vector
1615  image_rescalers.clear();
1616 }
#define AV_RESET_FRAME(av_frame)
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:72
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:64
#define AV_FREE_FRAME(av_frame)
void SetOption(StreamType stream, string name, string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
int num
Numerator for the fraction.
Definition: Fraction.h:44
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:92
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:65
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:60
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: WriterBase.h:61
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:112
#define PIX_FMT_RGB24
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:69
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:63
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:95
int width
The width of the video (in pixels)
Definition: WriterBase.h:57
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:70
#define OPEN_MP_NUM_PROCESSORS
void WriteFrame(std::shared_ptr< Frame > frame)
Add a frame to the stack waiting to be encoded.
Exception when encoding audio packet.
Definition: Exceptions.h:101
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:172
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:66
void Open()
Open writer.
void SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
Exception when memory could not be allocated.
Definition: Exceptions.h:224
Exception when invalid encoding options are used.
Definition: Exceptions.h:162
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
Definition: Exceptions.h:192
void RemoveScalers()
Remove & deallocate all software scalers.
#define AV_ALLOCATE_FRAME()
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:68
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
FFmpegWriter(string path)
Constructor for FFmpegWriter. Throws one of the following exceptions.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
#define PIX_FMT_YUV420P
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: WriterBase.h:62
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
#define PixelFormat
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
#define av_err2str(errnum)
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
void Close()
Close the writer.
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:67
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:71
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:59
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
This namespace is the default namespace for all code in the openshot library.
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:52
#define PIX_FMT_RGBA
void SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:264
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:73
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
int height
The height of the video (in pixels)
Definition: WriterBase.h:56
#define PIX_FMT_NONE
int den
Denominator for the fraction.
Definition: Fraction.h:45
Exception when no valid format is found for a file.
Definition: Exceptions.h:142
#define AUDIO_PACKET_ENCODING_SIZE
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:62