OpenShot Library | libopenshot  0.1.9
FFmpegWriter.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegWriter class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegWriter.h"
32 
33 using namespace openshot;
34 
36  path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), audio_pts(0), video_pts(0), samples(NULL),
37  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
38  initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(8), num_of_rescalers(32),
39  rescaler_position(0), video_codec(NULL), audio_codec(NULL), is_writing(false), write_video_count(0), write_audio_count(0),
40  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
41  write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL)
42 {
43 
44  // Disable audio & video (so they can be independently enabled)
45  info.has_audio = false;
46  info.has_video = false;
47 
48  // Initialize FFMpeg, and register all formats and codecs
49  av_register_all();
50 
51  // auto detect format
52  auto_detect_format();
53 }
54 
55 // Open the writer
57 {
58  // Open the writer
59  is_open = true;
60 
61  // Prepare streams (if needed)
62  if (!prepare_streams)
64 
65  // Write header (if needed)
66  if (!write_header)
67  WriteHeader();
68 }
69 
70 // auto detect format (from path)
71 void FFmpegWriter::auto_detect_format()
72 {
73  // Auto detect the output format from the name. default is mpeg.
74  fmt = av_guess_format(NULL, path.c_str(), NULL);
75  if (!fmt)
76  throw InvalidFormat("Could not deduce output format from file extension.", path);
77 
78  // Allocate the output media context
79  oc = avformat_alloc_context();
80  if (!oc)
81  throw OutOfMemory("Could not allocate memory for AVFormatContext.", path);
82 
83  // Set the AVOutputFormat for the current AVFormatContext
84  oc->oformat = fmt;
85 
86  // Update codec names
87  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
88  // Update video codec name
89  info.vcodec = avcodec_find_encoder(fmt->video_codec)->name;
90 
91  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
92  // Update audio codec name
93  info.acodec = avcodec_find_encoder(fmt->audio_codec)->name;
94 }
95 
96 // initialize streams
97 void FFmpegWriter::initialize_streams()
98 {
99  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::initialize_streams", "fmt->video_codec", fmt->video_codec, "fmt->audio_codec", fmt->audio_codec, "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE, "", -1, "", -1, "", -1);
100 
101  // Add the audio and video streams using the default format codecs and initialize the codecs
102  video_st = NULL;
103  audio_st = NULL;
104  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
105  // Add video stream
106  video_st = add_video_stream();
107 
108  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
109  // Add audio stream
110  audio_st = add_audio_stream();
111 }
112 
113 // Set video export options
114 void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
115 {
116  // Set the video options
117  if (codec.length() > 0)
118  {
119  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
120  if (new_codec == NULL)
121  throw InvalidCodec("A valid video codec could not be found for this file.", path);
122  else {
123  // Set video codec
124  info.vcodec = new_codec->name;
125 
126  // Update video codec in fmt
127  fmt->video_codec = new_codec->id;
128  }
129  }
130  if (fps.num > 0)
131  {
132  // Set frames per second (if provided)
133  info.fps.num = fps.num;
134  info.fps.den = fps.den;
135 
136  // Set the timebase (inverse of fps)
139  }
140  if (width >= 1)
141  info.width = width;
142  if (height >= 1)
143  info.height = height;
144  if (pixel_ratio.num > 0)
145  {
146  info.pixel_ratio.num = pixel_ratio.num;
147  info.pixel_ratio.den = pixel_ratio.den;
148  }
149  if (bit_rate >= 1000)
150  info.video_bit_rate = bit_rate;
151 
152  info.interlaced_frame = interlaced;
153  info.top_field_first = top_field_first;
154 
155  // Calculate the DAR (display aspect ratio)
157 
158  // Reduce size fraction
159  size.Reduce();
160 
161  // Set the ratio based on the reduced fraction
162  info.display_ratio.num = size.num;
163  info.display_ratio.den = size.den;
164 
165  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetVideoOptions (" + codec + ")", "width", width, "height", height, "size.num", size.num, "size.den", size.den, "fps.num", fps.num, "fps.den", fps.den);
166 
167  // Enable / Disable video
168  info.has_video = has_video;
169 }
170 
171 // Set audio export options
172 void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
173 {
174  // Set audio options
175  if (codec.length() > 0)
176  {
177  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
178  if (new_codec == NULL)
179  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
180  else
181  {
182  // Set audio codec
183  info.acodec = new_codec->name;
184 
185  // Update audio codec in fmt
186  fmt->audio_codec = new_codec->id;
187  }
188  }
189  if (sample_rate > 7999)
190  info.sample_rate = sample_rate;
191  if (channels > 0)
192  info.channels = channels;
193  if (bit_rate > 999)
194  info.audio_bit_rate = bit_rate;
195  info.channel_layout = channel_layout;
196 
197  // init resample options (if zero)
198  if (original_sample_rate == 0)
199  original_sample_rate = info.sample_rate;
200  if (original_channels == 0)
201  original_channels = info.channels;
202 
203  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetAudioOptions (" + codec + ")", "sample_rate", sample_rate, "channels", channels, "bit_rate", bit_rate, "", -1, "", -1, "", -1);
204 
205  // Enable / Disable audio
206  info.has_audio = has_audio;
207 }
208 
209 // Set custom options (some codecs accept additional params)
210 void FFmpegWriter::SetOption(StreamType stream, string name, string value)
211 {
212  // Declare codec context
213  AVCodecContext *c = NULL;
214  stringstream convert(value);
215 
216  if (info.has_video && stream == VIDEO_STREAM && video_st)
217  c = video_st->codec;
218  else if (info.has_audio && stream == AUDIO_STREAM && audio_st)
219  c = audio_st->codec;
220  else
221  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
222 
223  // Init AVOption
224  const AVOption *option = NULL;
225 
226  // Was a codec / stream found?
227  if (c)
228  // Find AVOption (if it exists)
229  #if LIBAVFORMAT_VERSION_MAJOR <= 53
230  option = av_find_opt(c->priv_data, name.c_str(), NULL, NULL, NULL);
231  #else
232  option = av_opt_find(c->priv_data, name.c_str(), NULL, 0, 0);
233  #endif
234 
235  // Was option found?
236  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
237  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate"))
238  {
239  // Check for specific named options
240  if (name == "g")
241  // Set gop_size
242  convert >> c->gop_size;
243 
244  else if (name == "qmin")
245  // Minimum quantizer
246  convert >> c->qmin;
247 
248  else if (name == "qmax")
249  // Maximum quantizer
250  convert >> c->qmax;
251 
252  else if (name == "max_b_frames")
253  // Maximum number of B-frames between non-B-frames
254  convert >> c->max_b_frames;
255 
256  else if (name == "mb_decision")
257  // Macroblock decision mode
258  convert >> c->mb_decision;
259 
260  else if (name == "level")
261  // Set codec level
262  convert >> c->level;
263 
264  else if (name == "profile")
265  // Set codec profile
266  convert >> c->profile;
267 
268  else if (name == "slices")
269  // Indicates number of picture subdivisions
270  convert >> c->slices;
271 
272  else if (name == "rc_min_rate")
273  // Minimum bitrate
274  convert >> c->rc_min_rate;
275 
276  else if (name == "rc_max_rate")
277  // Maximum bitrate
278  convert >> c->rc_max_rate;
279 
280  else if (name == "rc_buffer_size")
281  // Buffer size
282  convert >> c->rc_buffer_size;
283 
284  else
285  // Set AVOption
286  #if LIBAVFORMAT_VERSION_MAJOR <= 53
287  av_set_string3 (c->priv_data, name.c_str(), value.c_str(), 0, NULL);
288  #else
289  av_opt_set (c->priv_data, name.c_str(), value.c_str(), 0);
290  #endif
291 
292  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetOption (" + (string)name + ")", "stream == VIDEO_STREAM", stream == VIDEO_STREAM, "", -1, "", -1, "", -1, "", -1, "", -1);
293 
294  }
295  else
296  throw InvalidOptions("The option is not valid for this codec.", path);
297 
298 }
299 
300 // Prepare & initialize streams and open codecs
302 {
303  if (!info.has_audio && !info.has_video)
304  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
305 
306  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::PrepareStreams [" + path + "]", "info.has_audio", info.has_audio, "info.has_video", info.has_video, "", -1, "", -1, "", -1, "", -1);
307 
308  // Initialize the streams (i.e. add the streams)
309  initialize_streams();
310 
311  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
312  if (info.has_video && video_st)
313  open_video(oc, video_st);
314  if (info.has_audio && audio_st)
315  open_audio(oc, audio_st);
316 
317  // Mark as 'prepared'
318  prepare_streams = true;
319 }
320 
321 // Write the file header (after the options are set)
323 {
324  if (!info.has_audio && !info.has_video)
325  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
326 
327  // Open the output file, if needed
328  if (!(fmt->flags & AVFMT_NOFILE)) {
329  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
330  throw InvalidFile("Could not open or write file.", path);
331  }
332 
333  // Force the output filename (which doesn't always happen for some reason)
334  snprintf(oc->filename, sizeof(oc->filename), "%s", path.c_str());
335 
336  // Write the stream header, if any
337  // TODO: add avoptions / parameters instead of NULL
338  avformat_write_header(oc, NULL);
339 
340  // Mark as 'written'
341  write_header = true;
342 
343  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
344 }
345 
346 // Add a frame to the queue waiting to be encoded.
347 void FFmpegWriter::WriteFrame(std::shared_ptr<Frame> frame)
348 {
349  // Check for open reader (or throw exception)
350  if (!is_open)
351  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
352 
353  // Add frame pointer to "queue", waiting to be processed the next
354  // time the WriteFrames() method is called.
355  if (info.has_video && video_st)
356  spooled_video_frames.push_back(frame);
357 
358  if (info.has_audio && audio_st)
359  spooled_audio_frames.push_back(frame);
360 
361  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame", "frame->number", frame->number, "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "cache_size", cache_size, "is_writing", is_writing, "", -1);
362 
363  // Write the frames once it reaches the correct cache size
364  if (spooled_video_frames.size() == cache_size || spooled_audio_frames.size() == cache_size)
365  {
366  // Is writer currently writing?
367  if (!is_writing)
368  // Write frames to video file
369  write_queued_frames();
370 
371  else
372  {
373  // YES, WRITING... so wait until it finishes, before writing again
374  while (is_writing)
375  usleep(250000); // sleep for 250 milliseconds
376 
377  // Write frames to video file
378  write_queued_frames();
379  }
380  }
381 
382  // Keep track of the last frame added
383  last_frame = frame;
384 }
385 
386 // Write all frames in the queue to the video file.
387 void FFmpegWriter::write_queued_frames()
388 {
389  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_queued_frames", "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "", -1, "", -1, "", -1, "", -1);
390 
391  // Flip writing flag
392  is_writing = true;
393 
394  // Transfer spool to queue
395  queued_video_frames = spooled_video_frames;
396  queued_audio_frames = spooled_audio_frames;
397 
398  // Empty spool
399  spooled_video_frames.clear();
400  spooled_audio_frames.clear();
401 
402  // Set the number of threads in OpenMP
403  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
404  // Allow nested OpenMP sections
405  omp_set_nested(true);
406 
407  // Create blank exception
408  bool has_error_encoding_video = false;
409 
410  #pragma omp parallel
411  {
412  #pragma omp single
413  {
414  // Process all audio frames (in a separate thread)
415  if (info.has_audio && audio_st && !queued_audio_frames.empty())
416  write_audio_packets(false);
417 
418  // Loop through each queued image frame
419  while (!queued_video_frames.empty())
420  {
421  // Get front frame (from the queue)
422  std::shared_ptr<Frame> frame = queued_video_frames.front();
423 
424  // Add to processed queue
425  processed_frames.push_back(frame);
426 
427  // Encode and add the frame to the output file
428  if (info.has_video && video_st)
429  process_video_packet(frame);
430 
431  // Remove front item
432  queued_video_frames.pop_front();
433 
434  } // end while
435  } // end omp single
436 
437  #pragma omp single
438  {
439  // Loop back through the frames (in order), and write them to the video file
440  while (!processed_frames.empty())
441  {
442  // Get front frame (from the queue)
443  std::shared_ptr<Frame> frame = processed_frames.front();
444 
445  if (info.has_video && video_st)
446  {
447  // Add to deallocate queue (so we can remove the AVFrames when we are done)
448  deallocate_frames.push_back(frame);
449 
450  // Does this frame's AVFrame still exist
451  if (av_frames.count(frame))
452  {
453  // Get AVFrame
454  AVFrame *frame_final = av_frames[frame];
455 
456  // Write frame to video file
457  bool success = write_video_packet(frame, frame_final);
458  if (!success)
459  has_error_encoding_video = true;
460  }
461  }
462 
463  // Remove front item
464  processed_frames.pop_front();
465  }
466 
467  // Loop through, and deallocate AVFrames
468  while (!deallocate_frames.empty())
469  {
470  // Get front frame (from the queue)
471  std::shared_ptr<Frame> frame = deallocate_frames.front();
472 
473  // Does this frame's AVFrame still exist
474  if (av_frames.count(frame))
475  {
476  // Get AVFrame
477  AVFrame *av_frame = av_frames[frame];
478 
479  // Deallocate AVPicture and AVFrame
480  av_freep(&(av_frame->data[0]));
481  AV_FREE_FRAME(&av_frame);
482  av_frames.erase(frame);
483  }
484 
485  // Remove front item
486  deallocate_frames.pop_front();
487  }
488 
489  // Done writing
490  is_writing = false;
491 
492  } // end omp single
493  } // end omp parallel
494 
495  // Raise exception from main thread
496  if (has_error_encoding_video)
497  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
498 }
499 
500 // Write a block of frames from a reader
501 void FFmpegWriter::WriteFrame(ReaderBase* reader, int64_t start, int64_t length)
502 {
503  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame (from Reader)", "start", start, "length", length, "", -1, "", -1, "", -1, "", -1);
504 
505  // Loop through each frame (and encoded it)
506  for (int64_t number = start; number <= length; number++)
507  {
508  // Get the frame
509  std::shared_ptr<Frame> f = reader->GetFrame(number);
510 
511  // Encode frame
512  WriteFrame(f);
513  }
514 }
515 
516 // Write the file trailer (after all frames are written)
518 {
519  // Write any remaining queued frames to video file
520  write_queued_frames();
521 
522  // Process final audio frame (if any)
523  if (info.has_audio && audio_st)
524  write_audio_packets(true);
525 
526  // Flush encoders (who sometimes hold on to frames)
527  flush_encoders();
528 
529  /* write the trailer, if any. The trailer must be written
530  * before you close the CodecContexts open when you wrote the
531  * header; otherwise write_trailer may try to use memory that
532  * was freed on av_codec_close() */
533  av_write_trailer(oc);
534 
535  // Mark as 'written'
536  write_trailer = true;
537 
538  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
539 }
540 
541 // Flush encoders
542 void FFmpegWriter::flush_encoders()
543 {
544  if (info.has_audio && audio_codec && audio_st->codec->codec_type == AVMEDIA_TYPE_AUDIO && audio_codec->frame_size <= 1)
545  return;
546 
547  int error_code = 0;
548  int stop_encoding = 1;
549 
550  // FLUSH VIDEO ENCODER
551  if (info.has_video)
552  for (;;) {
553 
554  // Increment PTS (in frames and scaled to the codec's timebase)
555  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
556 
557  AVPacket pkt;
558  av_init_packet(&pkt);
559  pkt.data = NULL;
560  pkt.size = 0;
561 
562  // Pointer for video buffer (if using old FFmpeg version)
563  uint8_t *video_outbuf = NULL;
564 
565  /* encode the image */
566  int got_packet = 0;
567  int error_code = 0;
568 
569  #if LIBAVFORMAT_VERSION_MAJOR >= 54
570  // Newer versions of FFMpeg
571  error_code = avcodec_encode_video2(video_codec, &pkt, NULL, &got_packet);
572 
573  #else
574  // Older versions of FFmpeg (much sloppier)
575 
576  // Encode Picture and Write Frame
577  int video_outbuf_size = 0;
578 
579  /* encode the image */
580  int out_size = avcodec_encode_video(video_codec, NULL, video_outbuf_size, NULL);
581 
582  /* if zero size, it means the image was buffered */
583  if (out_size > 0) {
584  if(video_codec->coded_frame->key_frame)
585  pkt.flags |= AV_PKT_FLAG_KEY;
586  pkt.data= video_outbuf;
587  pkt.size= out_size;
588 
589  // got data back (so encode this frame)
590  got_packet = 1;
591  }
592  #endif
593 
594  if (error_code < 0) {
595  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
596  }
597  if (!got_packet) {
598  stop_encoding = 1;
599  break;
600  }
601 
602  // Override PTS (in frames and scaled to the codec's timebase)
603  //pkt.pts = write_video_count;
604 
605  // set the timestamp
606  if (pkt.pts != AV_NOPTS_VALUE)
607  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
608  if (pkt.dts != AV_NOPTS_VALUE)
609  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
610  if (pkt.duration > 0)
611  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
612  pkt.stream_index = video_st->index;
613 
614  // Write packet
615  error_code = av_interleaved_write_frame(oc, &pkt);
616  if (error_code < 0) {
617  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
618  }
619 
620  // Deallocate memory (if needed)
621  if (video_outbuf)
622  av_freep(&video_outbuf);
623  }
624 
625  // FLUSH AUDIO ENCODER
626  if (info.has_audio)
627  for (;;) {
628 
629  // Increment PTS (in samples and scaled to the codec's timebase)
630 #if LIBAVFORMAT_VERSION_MAJOR >= 54
631  // for some reason, it requires me to multiply channels X 2
632  write_audio_count += av_rescale_q(audio_input_position / (audio_codec->channels * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)), (AVRational){1, info.sample_rate}, audio_codec->time_base);
633 #else
634  write_audio_count += av_rescale_q(audio_input_position / audio_codec->channels, (AVRational){1, info.sample_rate}, audio_codec->time_base);
635 #endif
636 
637  AVPacket pkt;
638  av_init_packet(&pkt);
639  pkt.data = NULL;
640  pkt.size = 0;
641  pkt.pts = pkt.dts = write_audio_count;
642 
643  /* encode the image */
644  int got_packet = 0;
645  error_code = avcodec_encode_audio2(audio_codec, &pkt, NULL, &got_packet);
646  if (error_code < 0) {
647  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
648  }
649  if (!got_packet) {
650  stop_encoding = 1;
651  break;
652  }
653 
654  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
655  // but it fixes lots of PTS related issues when I do this.
656  pkt.pts = pkt.dts = write_audio_count;
657 
658  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
659  if (pkt.pts != AV_NOPTS_VALUE)
660  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
661  if (pkt.dts != AV_NOPTS_VALUE)
662  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
663  if (pkt.duration > 0)
664  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
665 
666  // set stream
667  pkt.stream_index = audio_st->index;
668  pkt.flags |= AV_PKT_FLAG_KEY;
669 
670  // Write packet
671  error_code = av_interleaved_write_frame(oc, &pkt);
672  if (error_code < 0) {
673  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
674  }
675 
676  // deallocate memory for packet
677  AV_FREE_PACKET(&pkt);
678  }
679 
680 
681 }
682 
683 // Close the video codec
684 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
685 {
686  avcodec_close(st->codec);
687  video_codec = NULL;
688 }
689 
690 // Close the audio codec
691 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
692 {
693  avcodec_close(st->codec);
694  audio_codec = NULL;
695 
696  // Clear buffers
697  delete[] samples;
698  delete[] audio_outbuf;
699  delete[] audio_encoder_buffer;
700  samples = NULL;
701  audio_outbuf = NULL;
702  audio_encoder_buffer = NULL;
703 
704  // Deallocate resample buffer
705  if (avr) {
706  avresample_close(avr);
707  avresample_free(&avr);
708  avr = NULL;
709  }
710 
711  if (avr_planar) {
712  avresample_close(avr_planar);
713  avresample_free(&avr_planar);
714  avr_planar = NULL;
715  }
716 }
717 
718 // Close the writer
720 {
721  // Write trailer (if needed)
722  if (!write_trailer)
723  WriteTrailer();
724 
725  // Close each codec
726  if (video_st)
727  close_video(oc, video_st);
728  if (audio_st)
729  close_audio(oc, audio_st);
730 
731  // Deallocate image scalers
732  if (image_rescalers.size() > 0)
733  RemoveScalers();
734 
735  // Free the streams
736  for (int i = 0; i < oc->nb_streams; i++) {
737  av_freep(&oc->streams[i]->codec);
738  av_freep(&oc->streams[i]);
739  }
740 
741  if (!(fmt->flags & AVFMT_NOFILE)) {
742  /* close the output file */
743  avio_close(oc->pb);
744  }
745 
746  // Reset frame counters
747  write_video_count = 0;
748  write_audio_count = 0;
749 
750  // Free the context
751  av_freep(&oc);
752 
753  // Close writer
754  is_open = false;
755  prepare_streams = false;
756  write_header = false;
757  write_trailer = false;
758 
759  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
760 }
761 
762 // Add an AVFrame to the cache
763 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame* av_frame)
764 {
765  // Add AVFrame to map (if it does not already exist)
766  if (!av_frames.count(frame))
767  {
768  // Add av_frame
769  av_frames[frame] = av_frame;
770  }
771  else
772  {
773  // Do not add, and deallocate this AVFrame
774  AV_FREE_FRAME(&av_frame);
775  }
776 }
777 
778 // Add an audio output stream
779 AVStream* FFmpegWriter::add_audio_stream()
780 {
781  AVCodecContext *c;
782  AVStream *st;
783 
784  // Find the audio codec
785  AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
786  if (codec == NULL)
787  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
788 
789  // Create a new audio stream
790  st = avformat_new_stream(oc, codec);
791  if (!st)
792  throw OutOfMemory("Could not allocate memory for the audio stream.", path);
793 
794  // Set default values
795  avcodec_get_context_defaults3(st->codec, codec);
796 
797  c = st->codec;
798  c->codec_id = codec->id;
799 #if LIBAVFORMAT_VERSION_MAJOR >= 53
800  c->codec_type = AVMEDIA_TYPE_AUDIO;
801 #else
802  c->codec_type = CODEC_TYPE_AUDIO;
803 #endif
804 
805  // Set the sample parameters
806  c->bit_rate = info.audio_bit_rate;
807  c->channels = info.channels;
808 
809  // Set valid sample rate (or throw error)
810  if (codec->supported_samplerates) {
811  int i;
812  for (i = 0; codec->supported_samplerates[i] != 0; i++)
813  if (info.sample_rate == codec->supported_samplerates[i])
814  {
815  // Set the valid sample rate
816  c->sample_rate = info.sample_rate;
817  break;
818  }
819  if (codec->supported_samplerates[i] == 0)
820  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
821  } else
822  // Set sample rate
823  c->sample_rate = info.sample_rate;
824 
825 
826  // Set a valid number of channels (or throw error)
827  int channel_layout = info.channel_layout;
828  if (codec->channel_layouts) {
829  int i;
830  for (i = 0; codec->channel_layouts[i] != 0; i++)
831  if (channel_layout == codec->channel_layouts[i])
832  {
833  // Set valid channel layout
834  c->channel_layout = channel_layout;
835  break;
836  }
837  if (codec->channel_layouts[i] == 0)
838  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
839  } else
840  // Set valid channel layout
841  c->channel_layout = channel_layout;
842 
843  // Choose a valid sample_fmt
844  if (codec->sample_fmts) {
845  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++)
846  {
847  // Set sample format to 1st valid format (and then exit loop)
848  c->sample_fmt = codec->sample_fmts[i];
849  break;
850  }
851  }
852  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
853  // Default if no sample formats found
854  c->sample_fmt = AV_SAMPLE_FMT_S16;
855  }
856 
857  // some formats want stream headers to be separate
858  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
859  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
860 
861  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_audio_stream", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->channels", c->channels, "c->sample_fmt", c->sample_fmt, "c->channel_layout", c->channel_layout, "c->sample_rate", c->sample_rate);
862 
863  return st;
864 }
865 
866 // Add a video output stream
867 AVStream* FFmpegWriter::add_video_stream()
868 {
869  AVCodecContext *c;
870  AVStream *st;
871 
872  // Find the audio codec
873  AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
874  if (codec == NULL)
875  throw InvalidCodec("A valid video codec could not be found for this file.", path);
876 
877  // Create a new stream
878  st = avformat_new_stream(oc, codec);
879  if (!st)
880  throw OutOfMemory("Could not allocate memory for the video stream.", path);
881 
882  // Set default values
883  avcodec_get_context_defaults3(st->codec, codec);
884 
885  c = st->codec;
886  c->codec_id = codec->id;
887 #if LIBAVFORMAT_VERSION_MAJOR >= 53
888  c->codec_type = AVMEDIA_TYPE_VIDEO;
889 #else
890  c->codec_type = CODEC_TYPE_VIDEO;
891 #endif
892 
893  /* Init video encoder options */
894  c->bit_rate = info.video_bit_rate;
895 
896  //TODO: Implement variable bitrate feature (which actually works). This implementation throws
897  //invalid bitrate errors and rc buffer underflow errors, etc...
898  //c->rc_min_rate = info.video_bit_rate;
899  //c->rc_max_rate = info.video_bit_rate;
900  //c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
901  //if ( !c->rc_initial_buffer_occupancy )
902  // c->rc_initial_buffer_occupancy = c->rc_buffer_size * 3/4;
903  c->qmin = 2;
904  c->qmax = 30;
905 
906  /* resolution must be a multiple of two */
907  // TODO: require /2 height and width
908  c->width = info.width;
909  c->height = info.height;
910 
911  /* time base: this is the fundamental unit of time (in seconds) in terms
912  of which frame timestamps are represented. for fixed-fps content,
913  timebase should be 1/framerate and timestamp increments should be
914  identically 1. */
915  c->time_base.num = info.video_timebase.num;
916  c->time_base.den = info.video_timebase.den;
917  st->time_base.num = info.video_timebase.num;
918  st->time_base.den = info.video_timebase.den;
919 
920  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
921  c->max_b_frames = 10;
922  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
923  /* just for testing, we also add B frames */
924  c->max_b_frames = 2;
925  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
926  /* Needed to avoid using macroblocks in which some coeffs overflow.
927  This does not happen with normal video, it just happens here as
928  the motion of the chroma plane does not match the luma plane. */
929  c->mb_decision = 2;
930  // some formats want stream headers to be separate
931  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
932  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
933 
934  // Find all supported pixel formats for this codec
935  const PixelFormat* supported_pixel_formats = codec->pix_fmts;
936  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
937  // Assign the 1st valid pixel format (if one is missing)
938  if (c->pix_fmt == PIX_FMT_NONE)
939  c->pix_fmt = *supported_pixel_formats;
940  ++supported_pixel_formats;
941  }
942 
943  // Codec doesn't have any pix formats?
944  if (c->pix_fmt == PIX_FMT_NONE) {
945  if(fmt->video_codec == AV_CODEC_ID_RAWVIDEO) {
946  // Raw video should use RGB24
947  c->pix_fmt = PIX_FMT_RGB24;
948  }
949  }
950 
951  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (string)fmt->name + " : " + (string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags, "", -1, "", -1);
952 
953  return st;
954 }
955 
956 // open audio codec
957 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st)
958 {
959  AVCodec *codec;
960  audio_codec = st->codec;
961 
962  // Set number of threads equal to number of processors (not to exceed 16)
963  audio_codec->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
964 
965  // Find the audio encoder
966  codec = avcodec_find_encoder_by_name(info.acodec.c_str());
967  if (!codec)
968  codec = avcodec_find_encoder(audio_codec->codec_id);
969  if (!codec)
970  throw InvalidCodec("Could not find codec", path);
971 
972  // Open the codec
973  if (avcodec_open2(audio_codec, codec, NULL) < 0)
974  throw InvalidCodec("Could not open codec", path);
975 
976  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
977  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
978  if (audio_codec->frame_size <= 1) {
979  // No frame size found... so calculate
980  audio_input_frame_size = 50000 / info.channels;
981 
982  switch (st->codec->codec_id) {
983  case AV_CODEC_ID_PCM_S16LE:
984  case AV_CODEC_ID_PCM_S16BE:
985  case AV_CODEC_ID_PCM_U16LE:
986  case AV_CODEC_ID_PCM_U16BE:
987  audio_input_frame_size >>= 1;
988  break;
989  default:
990  break;
991  }
992  } else {
993  // Set frame size based on the codec
994  audio_input_frame_size = audio_codec->frame_size;
995  }
996 
997  // Set the initial frame size (since it might change during resampling)
998  initial_audio_input_frame_size = audio_input_frame_size;
999 
1000  // Allocate array for samples
1001  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1002 
1003  // Set audio output buffer (used to store the encoded audio)
1004  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1005  audio_outbuf = new uint8_t[audio_outbuf_size];
1006 
1007  // Set audio packet encoding buffer
1008  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1009  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1010 
1011  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_audio", "audio_codec->thread_count", audio_codec->thread_count, "audio_input_frame_size", audio_input_frame_size, "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + AV_INPUT_BUFFER_PADDING_SIZE, "", -1, "", -1, "", -1);
1012 
1013 }
1014 
1015 // open video codec
1016 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st)
1017 {
1018  AVCodec *codec;
1019  video_codec = st->codec;
1020 
1021  // Set number of threads equal to number of processors (not to exceed 16)
1022  video_codec->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
1023 
1024  /* find the video encoder */
1025  codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1026  if (!codec)
1027  codec = avcodec_find_encoder(video_codec->codec_id);
1028  if (!codec)
1029  throw InvalidCodec("Could not find codec", path);
1030 
1031  /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */
1032  if(video_codec->max_b_frames && video_codec->codec_id != AV_CODEC_ID_MPEG4 && video_codec->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1033  video_codec->max_b_frames = 0;
1034 
1035  /* open the codec */
1036  if (avcodec_open2(video_codec, codec, NULL) < 0)
1037  throw InvalidCodec("Could not open codec", path);
1038 
1039  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video", "video_codec->thread_count", video_codec->thread_count, "", -1, "", -1, "", -1, "", -1, "", -1);
1040 
1041 }
1042 
1043 // write all queued frames' audio to the video file
1044 void FFmpegWriter::write_audio_packets(bool final)
1045 {
1046  #pragma omp task firstprivate(final)
1047  {
1048  // Init audio buffers / variables
1049  int total_frame_samples = 0;
1050  int frame_position = 0;
1051  int channels_in_frame = 0;
1052  int sample_rate_in_frame = 0;
1053  int samples_in_frame = 0;
1054  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1055 
1056  // Create a new array (to hold all S16 audio samples, for the current queued frames
1057  int16_t* all_queued_samples = (int16_t*)av_malloc((sizeof(int16_t)*(queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE)));
1058  int16_t* all_resampled_samples = NULL;
1059  int16_t* final_samples_planar = NULL;
1060  int16_t* final_samples = NULL;
1061 
1062  // Loop through each queued audio frame
1063  while (!queued_audio_frames.empty())
1064  {
1065  // Get front frame (from the queue)
1066  std::shared_ptr<Frame> frame = queued_audio_frames.front();
1067 
1068  // Get the audio details from this frame
1069  sample_rate_in_frame = frame->SampleRate();
1070  samples_in_frame = frame->GetAudioSamplesCount();
1071  channels_in_frame = frame->GetAudioChannelsCount();
1072  channel_layout_in_frame = frame->ChannelsLayout();
1073 
1074 
1075  // Get audio sample array
1076  float* frame_samples_float = NULL;
1077  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1078  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
1079 
1080 
1081  // Calculate total samples
1082  total_frame_samples = samples_in_frame * channels_in_frame;
1083 
1084  // Translate audio sample values back to 16 bit integers
1085  for (int s = 0; s < total_frame_samples; s++, frame_position++)
1086  // Translate sample value and copy into buffer
1087  all_queued_samples[frame_position] = int(frame_samples_float[s] * (1 << 15));
1088 
1089 
1090  // Deallocate float array
1091  delete[] frame_samples_float;
1092 
1093  // Remove front item
1094  queued_audio_frames.pop_front();
1095 
1096  } // end while
1097 
1098 
1099  // Update total samples (since we've combined all queued frames)
1100  total_frame_samples = frame_position;
1101  int remaining_frame_samples = total_frame_samples;
1102  int samples_position = 0;
1103 
1104 
1105  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "final", final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO);
1106 
1107  // Keep track of the original sample format
1108  AVSampleFormat output_sample_fmt = audio_codec->sample_fmt;
1109 
1110  AVFrame *audio_frame = NULL;
1111  if (!final) {
1112  // Create input frame (and allocate arrays)
1113  audio_frame = AV_ALLOCATE_FRAME();
1114  AV_RESET_FRAME(audio_frame);
1115  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1116 
1117  // Fill input frame with sample data
1118  avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples,
1119  audio_encoder_buffer_size, 0);
1120 
1121  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1122  switch (audio_codec->sample_fmt)
1123  {
1124  case AV_SAMPLE_FMT_FLTP:
1125  {
1126  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1127  break;
1128  }
1129  case AV_SAMPLE_FMT_S32P:
1130  {
1131  output_sample_fmt = AV_SAMPLE_FMT_S32;
1132  break;
1133  }
1134  case AV_SAMPLE_FMT_S16P:
1135  {
1136  output_sample_fmt = AV_SAMPLE_FMT_S16;
1137  break;
1138  }
1139  case AV_SAMPLE_FMT_U8P:
1140  {
1141  output_sample_fmt = AV_SAMPLE_FMT_U8;
1142  break;
1143  }
1144  }
1145 
1146  // Update total samples & input frame size (due to bigger or smaller data types)
1147  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1148  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1149 
1150  // Set remaining samples
1151  remaining_frame_samples = total_frame_samples;
1152 
1153  // Create output frame (and allocate arrays)
1154  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1155  AV_RESET_FRAME(audio_converted);
1156  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1157  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1158 
1159  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (1st resampling)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", output_sample_fmt, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
1160 
1161  // setup resample context
1162  if (!avr) {
1163  avr = avresample_alloc_context();
1164  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1165  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1166  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1167  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1168  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1169  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1170  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1171  av_opt_set_int(avr, "out_channels", info.channels, 0);
1172  avresample_open(avr);
1173  }
1174  int nb_samples = 0;
1175 
1176  // Convert audio samples
1177  nb_samples = avresample_convert(avr, // audio resample context
1178  audio_converted->data, // output data pointers
1179  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1180  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1181  audio_frame->data, // input data pointers
1182  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1183  audio_frame->nb_samples); // number of input samples to convert
1184 
1185  // Create a new array (to hold all resampled S16 audio samples)
1186  all_resampled_samples = (int16_t*)av_malloc(sizeof(int16_t) * nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1187 
1188  // Copy audio samples over original samples
1189  memcpy(all_resampled_samples, audio_converted->data[0], nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1190 
1191  // Remove converted audio
1192  av_freep(&(audio_frame->data[0]));
1193  AV_FREE_FRAME(&audio_frame);
1194  av_freep(&audio_converted->data[0]);
1195  AV_FREE_FRAME(&audio_converted);
1196  all_queued_samples = NULL; // this array cleared with above call
1197 
1198  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)", "nb_samples", nb_samples, "remaining_frame_samples", remaining_frame_samples, "", -1, "", -1, "", -1, "", -1);
1199  }
1200 
1201  // Loop until no more samples
1202  while (remaining_frame_samples > 0 || final) {
1203  // Get remaining samples needed for this packet
1204  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1205 
1206  // Determine how many samples we need
1207  int diff = 0;
1208  if (remaining_frame_samples >= remaining_packet_samples)
1209  diff = remaining_packet_samples;
1210  else if (remaining_frame_samples < remaining_packet_samples)
1211  diff = remaining_frame_samples;
1212 
1213  // Copy frame samples into the packet samples array
1214  if (!final)
1215  //TODO: Make this more sane
1216  memcpy(samples + (audio_input_position * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))), all_resampled_samples + samples_position, diff * av_get_bytes_per_sample(output_sample_fmt));
1217 
1218  // Increment counters
1219  audio_input_position += diff;
1220  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1221  remaining_frame_samples -= diff;
1222  remaining_packet_samples -= diff;
1223 
1224  // Do we have enough samples to proceed?
1225  if (audio_input_position < (audio_input_frame_size * info.channels) && !final)
1226  // Not enough samples to encode... so wait until the next frame
1227  break;
1228 
1229  // Convert to planar (if needed by audio codec)
1230  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1231  AV_RESET_FRAME(frame_final);
1232  if (av_sample_fmt_is_planar(audio_codec->sample_fmt))
1233  {
1234  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)", "in_sample_fmt", output_sample_fmt, "out_sample_fmt", audio_codec->sample_fmt, "in_sample_rate", info.sample_rate, "out_sample_rate", info.sample_rate, "in_channels", info.channels, "out_channels", info.channels);
1235 
1236  // setup resample context
1237  if (!avr_planar) {
1238  avr_planar = avresample_alloc_context();
1239  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1240  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1241  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1242  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec->sample_fmt, 0); // planar not allowed here
1243  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1244  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1245  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1246  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1247  avresample_open(avr_planar);
1248  }
1249 
1250  // Create input frame (and allocate arrays)
1251  audio_frame = AV_ALLOCATE_FRAME();
1252  AV_RESET_FRAME(audio_frame);
1253  audio_frame->nb_samples = audio_input_position / info.channels;
1254 
1255  // Create a new array
1256  final_samples_planar = (int16_t*)av_malloc(sizeof(int16_t) * audio_frame->nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1257 
1258  // Copy audio into buffer for frame
1259  memcpy(final_samples_planar, samples, audio_frame->nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1260 
1261  // Fill input frame with sample data
1262  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt, (uint8_t *) final_samples_planar,
1263  audio_encoder_buffer_size, 0);
1264 
1265  // Create output frame (and allocate arrays)
1266  frame_final->nb_samples = audio_input_frame_size;
1267  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels, frame_final->nb_samples, audio_codec->sample_fmt, 0);
1268 
1269  // Convert audio samples
1270  int nb_samples = avresample_convert(avr_planar, // audio resample context
1271  frame_final->data, // output data pointers
1272  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1273  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1274  audio_frame->data, // input data pointers
1275  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1276  audio_frame->nb_samples); // number of input samples to convert
1277 
1278  // Copy audio samples over original samples
1279  if (nb_samples > 0)
1280  memcpy(samples, frame_final->data[0], nb_samples * av_get_bytes_per_sample(audio_codec->sample_fmt) * info.channels);
1281 
1282  // deallocate AVFrame
1283  av_freep(&(audio_frame->data[0]));
1284  AV_FREE_FRAME(&audio_frame);
1285  all_queued_samples = NULL; // this array cleared with above call
1286 
1287  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)", "nb_samples", nb_samples, "", -1, "", -1, "", -1, "", -1, "", -1);
1288 
1289  } else {
1290  // Create a new array
1291  final_samples = new int16_t[audio_input_position * (av_get_bytes_per_sample(audio_codec->sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))];
1292 
1293  // Copy audio into buffer for frame
1294  memcpy(final_samples, samples, audio_input_position * av_get_bytes_per_sample(audio_codec->sample_fmt));
1295 
1296  // Init the nb_samples property
1297  frame_final->nb_samples = audio_input_frame_size;
1298 
1299  // Fill the final_frame AVFrame with audio (non planar)
1300  avcodec_fill_audio_frame(frame_final, audio_codec->channels, audio_codec->sample_fmt, (uint8_t *) final_samples,
1301  audio_encoder_buffer_size, 0);
1302  }
1303 
1304  // Increment PTS (in samples)
1305  write_audio_count += FFMIN(audio_input_frame_size, audio_input_position);
1306  frame_final->pts = write_audio_count; // Set the AVFrame's PTS
1307 
1308  // Init the packet
1309  AVPacket pkt;
1310  av_init_packet(&pkt);
1311  pkt.data = audio_encoder_buffer;
1312  pkt.size = audio_encoder_buffer_size;
1313 
1314  // Set the packet's PTS prior to encoding
1315  pkt.pts = pkt.dts = write_audio_count;
1316 
1317  /* encode the audio samples */
1318  int got_packet_ptr = 0;
1319  int error_code = avcodec_encode_audio2(audio_codec, &pkt, frame_final, &got_packet_ptr);
1320 
1321  /* if zero size, it means the image was buffered */
1322  if (error_code == 0 && got_packet_ptr) {
1323 
1324  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1325  // but it fixes lots of PTS related issues when I do this.
1326  pkt.pts = pkt.dts = write_audio_count;
1327 
1328  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1329  if (pkt.pts != AV_NOPTS_VALUE)
1330  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
1331  if (pkt.dts != AV_NOPTS_VALUE)
1332  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
1333  if (pkt.duration > 0)
1334  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
1335 
1336  // set stream
1337  pkt.stream_index = audio_st->index;
1338  pkt.flags |= AV_PKT_FLAG_KEY;
1339 
1340  /* write the compressed frame in the media file */
1341  int error_code = av_interleaved_write_frame(oc, &pkt);
1342  if (error_code < 0)
1343  {
1344  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1345  }
1346  }
1347 
1348  if (error_code < 0)
1349  {
1350  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1351  }
1352 
1353  // deallocate AVFrame
1354  av_freep(&(frame_final->data[0]));
1355  AV_FREE_FRAME(&frame_final);
1356 
1357  // deallocate memory for packet
1358  AV_FREE_PACKET(&pkt);
1359 
1360  // Reset position
1361  audio_input_position = 0;
1362  final = false;
1363  }
1364 
1365  // Delete arrays (if needed)
1366  if (all_resampled_samples) {
1367  av_freep(&all_resampled_samples);
1368  all_resampled_samples = NULL;
1369  }
1370  if (all_queued_samples) {
1371  av_freep(&all_queued_samples);
1372  all_queued_samples = NULL;
1373  }
1374 
1375  } // end task
1376 }
1377 
1378 // Allocate an AVFrame object
1379 AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer)
1380 {
1381  // Create an RGB AVFrame
1382  AVFrame *new_av_frame = NULL;
1383 
1384  // Allocate an AVFrame structure
1385  new_av_frame = AV_ALLOCATE_FRAME();
1386  if (new_av_frame == NULL)
1387  throw OutOfMemory("Could not allocate AVFrame", path);
1388 
1389  // Determine required buffer size and allocate buffer
1390  *buffer_size = avpicture_get_size(pix_fmt, width, height);
1391 
1392  // Create buffer (if not provided)
1393  if (!new_buffer)
1394  {
1395  // New Buffer
1396  new_buffer = (uint8_t*)av_malloc(*buffer_size * sizeof(uint8_t));
1397  // Attach buffer to AVFrame
1398  avpicture_fill((AVPicture *)new_av_frame, new_buffer, pix_fmt, width, height);
1399  new_av_frame->width = width;
1400  new_av_frame->height = height;
1401  new_av_frame->format = pix_fmt;
1402  }
1403 
1404  // return AVFrame
1405  return new_av_frame;
1406 }
1407 
1408 // process video frame
1409 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame)
1410 {
1411  // Determine the height & width of the source image
1412  int source_image_width = frame->GetWidth();
1413  int source_image_height = frame->GetHeight();
1414 
1415  // Do nothing if size is 1x1 (i.e. no image in this frame)
1416  if (source_image_height == 1 && source_image_width == 1)
1417  return;
1418 
1419  // Init rescalers (if not initialized yet)
1420  if (image_rescalers.size() == 0)
1421  InitScalers(source_image_width, source_image_height);
1422 
1423  // Get a unique rescaler (for this thread)
1424  SwsContext *scaler = image_rescalers[rescaler_position];
1425  rescaler_position++;
1426  if (rescaler_position == num_of_rescalers)
1427  rescaler_position = 0;
1428 
1429  #pragma omp task firstprivate(frame, scaler, source_image_width, source_image_height)
1430  {
1431  // Allocate an RGB frame & final output frame
1432  int bytes_source = 0;
1433  int bytes_final = 0;
1434  AVFrame *frame_source = NULL;
1435  const uchar *pixels = NULL;
1436 
1437  // Get a list of pixels from source image
1438  pixels = frame->GetPixels();
1439 
1440  // Init AVFrame for source image & final (converted image)
1441  frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t*) pixels);
1442  AVFrame *frame_final = allocate_avframe(video_codec->pix_fmt, info.width, info.height, &bytes_final, NULL);
1443 
1444  // Fill with data
1445  avpicture_fill((AVPicture *) frame_source, (uint8_t*)pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
1446  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::process_video_packet", "frame->number", frame->number, "bytes_source", bytes_source, "bytes_final", bytes_final, "", -1, "", -1, "", -1);
1447 
1448  // Resize & convert pixel format
1449  sws_scale(scaler, frame_source->data, frame_source->linesize, 0,
1450  source_image_height, frame_final->data, frame_final->linesize);
1451 
1452  // Add resized AVFrame to av_frames map
1453  #pragma omp critical (av_frames_section)
1454  add_avframe(frame, frame_final);
1455 
1456  // Deallocate memory
1457  AV_FREE_FRAME(&frame_source);
1458 
1459  } // end task
1460 
1461 }
1462 
1463 // write video frame
1464 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame* frame_final)
1465 {
1466  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet", "frame->number", frame->number, "", -1, "", -1, "", -1, "", -1, "", -1);
1467 
1468  AVPacket pkt;
1469  av_init_packet(&pkt);
1470  pkt.data = NULL;
1471  pkt.size = 0;
1472  pkt.pts = pkt.dts = AV_NOPTS_VALUE;
1473 
1474  // Pointer for video buffer (if using old FFmpeg version)
1475  uint8_t *video_outbuf = NULL;
1476 
1477  // Increment PTS (in frames and scaled to the codec's timebase)
1478  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1479 
1480  // Assign the initial AVFrame PTS from the frame counter
1481  frame_final->pts = write_video_count;
1482 
1483  /* encode the image */
1484  int got_packet_ptr = 0;
1485  int error_code = 0;
1486  #if LIBAVFORMAT_VERSION_MAJOR >= 54
1487  // Newer versions of FFMpeg
1488  error_code = avcodec_encode_video2(video_codec, &pkt, frame_final, &got_packet_ptr);
1489 
1490  #else
1491  // Older versions of FFmpeg (much sloppier)
1492 
1493  // Encode Picture and Write Frame
1494  int video_outbuf_size = 200000;
1495  video_outbuf = (uint8_t*) av_malloc(200000);
1496 
1497  /* encode the image */
1498  int out_size = avcodec_encode_video(video_codec, video_outbuf, video_outbuf_size, frame_final);
1499 
1500  /* if zero size, it means the image was buffered */
1501  if (out_size > 0) {
1502  if(video_codec->coded_frame->key_frame)
1503  pkt.flags |= AV_PKT_FLAG_KEY;
1504  pkt.data= video_outbuf;
1505  pkt.size= out_size;
1506 
1507  // got data back (so encode this frame)
1508  got_packet_ptr = 1;
1509  }
1510  #endif
1511 
1512  /* if zero size, it means the image was buffered */
1513  if (error_code == 0 && got_packet_ptr) {
1514 
1515  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1516  // but it fixes lots of PTS related issues when I do this.
1517  //pkt.pts = pkt.dts = write_video_count;
1518 
1519  // set the timestamp
1520  if (pkt.pts != AV_NOPTS_VALUE)
1521  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
1522  if (pkt.dts != AV_NOPTS_VALUE)
1523  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
1524  if (pkt.duration > 0)
1525  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
1526  pkt.stream_index = video_st->index;
1527 
1528  /* write the compressed frame in the media file */
1529  int error_code = av_interleaved_write_frame(oc, &pkt);
1530  if (error_code < 0)
1531  {
1532  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1533  return false;
1534  }
1535  }
1536 
1537  // Deallocate memory (if needed)
1538  if (video_outbuf)
1539  delete[] video_outbuf;
1540 
1541  // Deallocate packet
1542  AV_FREE_PACKET(&pkt);
1543 
1544  // Success
1545  return true;
1546 }
1547 
1548 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
1550 {
1551  // output debug info
1552  av_dump_format(oc, 0, path.c_str(), 1);
1553 }
1554 
1555 // Init a collection of software rescalers (thread safe)
1556 void FFmpegWriter::InitScalers(int source_width, int source_height)
1557 {
1558  // Get the codec
1559  AVCodecContext *c;
1560  c = video_st->codec;
1561 
1562  // Init software rescalers vector (many of them, one for each thread)
1563  for (int x = 0; x < num_of_rescalers; x++)
1564  {
1565  // Init the software scaler from FFMpeg
1566  img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, c->pix_fmt, SWS_BILINEAR, NULL, NULL, NULL);
1567 
1568  // Add rescaler to vector
1569  image_rescalers.push_back(img_convert_ctx);
1570  }
1571 }
1572 
1573 // Set audio resample options
1574 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
1575  original_sample_rate = sample_rate;
1576  original_channels = channels;
1577 }
1578 
1579 // Remove & deallocate all software scalers
1581 {
1582  // Close all rescalers
1583  for (int x = 0; x < num_of_rescalers; x++)
1584  sws_freeContext(image_rescalers[x]);
1585 
1586  // Clear vector
1587  image_rescalers.clear();
1588 }
#define AV_RESET_FRAME(av_frame)
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:72
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:64
#define AV_FREE_FRAME(av_frame)
void SetOption(StreamType stream, string name, string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
int num
Numerator for the fraction.
Definition: Fraction.h:44
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:92
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:65
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:60
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: WriterBase.h:61
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:112
#define PIX_FMT_RGB24
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:69
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:63
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:95
int width
The width of the video (in pixels)
Definition: WriterBase.h:57
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:70
#define OPEN_MP_NUM_PROCESSORS
void WriteFrame(std::shared_ptr< Frame > frame)
Add a frame to the stack waiting to be encoded.
Exception when encoding audio packet.
Definition: Exceptions.h:101
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:172
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:66
void Open()
Open writer.
void SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
virtual std::shared_ptr< Frame > GetFrame(int64_t number)=0
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
Exception when memory could not be allocated.
Definition: Exceptions.h:224
Exception when invalid encoding options are used.
Definition: Exceptions.h:162
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
Definition: Exceptions.h:192
void RemoveScalers()
Remove & deallocate all software scalers.
#define AV_ALLOCATE_FRAME()
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:68
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
FFmpegWriter(string path)
Constructor for FFmpegWriter. Throws one of the following exceptions.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: WriterBase.h:62
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
#define PixelFormat
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
#define av_err2str(errnum)
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
void Close()
Close the writer.
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:67
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:71
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:59
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
This namespace is the default namespace for all code in the openshot library.
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:52
#define PIX_FMT_RGBA
void SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:264
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:73
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
int height
The height of the video (in pixels)
Definition: WriterBase.h:56
#define PIX_FMT_NONE
int den
Denominator for the fraction.
Definition: Fraction.h:45
Exception when no valid format is found for a file.
Definition: Exceptions.h:142
#define AUDIO_PACKET_ENCODING_SIZE
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:62