OpenShot Library | libopenshot  0.2.2
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegReader.h"
32 
33 using namespace openshot;
34 
36  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
41  packet(NULL), use_omp_threads(true) {
42 
43  // Initialize FFMpeg, and register all formats and codecs
46 
47  // Init cache
51 
52  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
53  Open();
54  Close();
55 }
56 
57 FFmpegReader::FFmpegReader(string path, bool inspect_reader)
58  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
59  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
60  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
61  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
62  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
63  packet(NULL), use_omp_threads(true) {
64 
65  // Initialize FFMpeg, and register all formats and codecs
68 
69  // Init cache
73 
74  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
75  if (inspect_reader) {
76  Open();
77  Close();
78  }
79 }
80 
82  if (is_open)
83  // Auto close reader if not already done
84  Close();
85 }
86 
87 // This struct holds the associated video frame and starting sample # for an audio packet.
88 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount)
89 {
90  // Is frame even close to this one?
91  if (abs(location.frame - frame) >= 2)
92  // This is too far away to be considered
93  return false;
94 
95  // Note that samples_per_frame can vary slightly frame to frame when the
96  // audio sampling rate is not an integer multiple of the video fps.
97  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
98  if (abs(diff) <= amount)
99  // close
100  return true;
101 
102  // not close
103  return false;
104 }
105 
107 {
108  // Open reader if not already open
109  if (!is_open)
110  {
111  // Initialize format context
112  pFormatCtx = NULL;
113 
114  // Open video file
115  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
116  throw InvalidFile("File could not be opened.", path);
117 
118  // Retrieve stream information
119  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
120  throw NoStreamsFound("No streams found in file.", path);
121 
122  videoStream = -1;
123  audioStream = -1;
124  // Loop through each stream, and identify the video and audio stream index
125  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
126  {
127  // Is this a video stream?
128  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
129  videoStream = i;
130  }
131  // Is this an audio stream?
132  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
133  audioStream = i;
134  }
135  }
136  if (videoStream == -1 && audioStream == -1)
137  throw NoStreamsFound("No video or audio streams found in this file.", path);
138 
139  // Is there a video stream?
140  if (videoStream != -1)
141  {
142  // Set the stream index
143  info.video_stream_index = videoStream;
144 
145  // Set the codec and codec context pointers
146  pStream = pFormatCtx->streams[videoStream];
147 
148  // Find the codec ID from stream
149  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
150 
151  // Get codec and codec context from stream
152  AVCodec *pCodec = avcodec_find_decoder(codecId);
153  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
154 
155  // Set number of threads equal to number of processors (not to exceed 16)
156  pCodecCtx->thread_count = min(FF_NUM_PROCESSORS, 16);
157 
158  if (pCodec == NULL) {
159  throw InvalidCodec("A valid video codec could not be found for this file.", path);
160  }
161 
162  // Init options
163  AVDictionary *opts = NULL;
164  av_dict_set(&opts, "strict", "experimental", 0);
165 
166  // Open video codec
167  if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
168  throw InvalidCodec("A video codec was found, but could not be opened.", path);
169 
170  // Free options
171  av_dict_free(&opts);
172 
173  // Update the File Info struct with video details (if a video stream is found)
174  UpdateVideoInfo();
175  }
176 
177  // Is there an audio stream?
178  if (audioStream != -1)
179  {
180  // Set the stream index
181  info.audio_stream_index = audioStream;
182 
183  // Get a pointer to the codec context for the audio stream
184  aStream = pFormatCtx->streams[audioStream];
185 
186  // Find the codec ID from stream
187  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
188 
189  // Get codec and codec context from stream
190  AVCodec *aCodec = avcodec_find_decoder(codecId);
191  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
192 
193  // Set number of threads equal to number of processors (not to exceed 16)
194  aCodecCtx->thread_count = min(FF_NUM_PROCESSORS, 16);
195 
196  if (aCodec == NULL) {
197  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
198  }
199 
200  // Init options
201  AVDictionary *opts = NULL;
202  av_dict_set(&opts, "strict", "experimental", 0);
203 
204  // Open audio codec
205  if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
206  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
207 
208  // Free options
209  av_dict_free(&opts);
210 
211  // Update the File Info struct with audio details (if an audio stream is found)
212  UpdateAudioInfo();
213  }
214 
215  // Add format metadata (if any)
216  AVDictionaryEntry *tag = NULL;
217  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
218  QString str_key = tag->key;
219  QString str_value = tag->value;
220  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
221  }
222 
223  // Init previous audio location to zero
224  previous_packet_location.frame = -1;
225  previous_packet_location.sample_start = 0;
226 
227  // Adjust cache size based on size of frame and audio
231 
232  // Initialize OMP threading support
233  use_omp_threads = openshot::IsOMPEnabled();
234 
235  // Mark as "open"
236  is_open = true;
237  }
238 }
239 
241 {
242  // Close all objects, if reader is 'open'
243  if (is_open)
244  {
245  // Mark as "closed"
246  is_open = false;
247 
248  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
249 
250  // Close the codec
251  if (info.has_video)
252  {
253  avcodec_flush_buffers(pCodecCtx);
254  AV_FREE_CONTEXT(pCodecCtx);
255  }
256  if (info.has_audio)
257  {
258  avcodec_flush_buffers(aCodecCtx);
259  AV_FREE_CONTEXT(aCodecCtx);
260  }
261 
262  // Clear final cache
263  final_cache.Clear();
264  working_cache.Clear();
265  missing_frames.Clear();
266 
267  // Clear processed lists
268  {
269  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
270  processed_video_frames.clear();
271  processed_audio_frames.clear();
272  processing_video_frames.clear();
273  processing_audio_frames.clear();
274  missing_audio_frames.clear();
275  missing_video_frames.clear();
276  missing_audio_frames_source.clear();
277  missing_video_frames_source.clear();
278  checked_frames.clear();
279  }
280 
281  // Close the video file
282  avformat_close_input(&pFormatCtx);
283  av_freep(&pFormatCtx);
284 
285  // Reset some variables
286  last_frame = 0;
287  largest_frame_processed = 0;
288  seek_audio_frame_found = 0;
289  seek_video_frame_found = 0;
290  current_video_frame = 0;
291  has_missing_frames = false;
292  }
293 }
294 
295 void FFmpegReader::UpdateAudioInfo()
296 {
297  // Set values of FileInfo struct
298  info.has_audio = true;
299  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
300  info.acodec = aCodecCtx->codec->name;
301  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
302  if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
303  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout( AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels );
304  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
305  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
306  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
307 
308  // Set audio timebase
309  info.audio_timebase.num = aStream->time_base.num;
310  info.audio_timebase.den = aStream->time_base.den;
311 
312  // Get timebase of audio stream (if valid) and greater than the current duration
313  if (aStream->duration > 0.0f && aStream->duration > info.duration)
314  info.duration = aStream->duration * info.audio_timebase.ToDouble();
315 
316  // Check for an invalid video length
317  if (info.has_video && info.video_length <= 0)
318  {
319  // Calculate the video length from the audio duration
321  }
322 
323  // Set video timebase (if no video stream was found)
324  if (!info.has_video)
325  {
326  // Set a few important default video settings (so audio can be divided into frames)
327  info.fps.num = 24;
328  info.fps.den = 1;
329  info.video_timebase.num = 1;
330  info.video_timebase.den = 24;
332  info.width = 720;
333  info.height = 480;
334  }
335 
336  // Fix invalid video lengths for certain types of files (MP3 for example)
337  if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
339  }
340 
341  // Add audio metadata (if any found)
342  AVDictionaryEntry *tag = NULL;
343  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
344  QString str_key = tag->key;
345  QString str_value = tag->value;
346  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
347  }
348 }
349 
350 void FFmpegReader::UpdateVideoInfo()
351 {
352  if (check_fps)
353  // Already initialized all the video metadata, no reason to do it again
354  return;
355 
356  // Set values of FileInfo struct
357  info.has_video = true;
358  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
359  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
360  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
361  info.vcodec = pCodecCtx->codec->name;
362  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
363 
364  // set frames per second (fps)
365  info.fps.num = pStream->avg_frame_rate.num;
366  info.fps.den = pStream->avg_frame_rate.den;
367 
368  if (pStream->sample_aspect_ratio.num != 0)
369  {
370  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
371  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
372  }
373  else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0)
374  {
375  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
376  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
377  }
378  else
379  {
380  info.pixel_ratio.num = 1;
381  info.pixel_ratio.den = 1;
382  }
383  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
384 
385  // Calculate the DAR (display aspect ratio)
387 
388  // Reduce size fraction
389  size.Reduce();
390 
391  // Set the ratio based on the reduced fraction
392  info.display_ratio.num = size.num;
393  info.display_ratio.den = size.den;
394 
395  // Set the video timebase
396  info.video_timebase.num = pStream->time_base.num;
397  info.video_timebase.den = pStream->time_base.den;
398 
399  // Set the duration in seconds, and video length (# of frames)
400  info.duration = pStream->duration * info.video_timebase.ToDouble();
401 
402  // Check for valid duration (if found)
403  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
404  // Use the format's duration
405  info.duration = pFormatCtx->duration / AV_TIME_BASE;
406 
407  // Calculate duration from filesize and bitrate (if any)
408  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
409  // Estimate from bitrate, total bytes, and framerate
411 
412  // No duration found in stream of file
413  if (info.duration <= 0.0f)
414  {
415  // No duration is found in the video stream
416  info.duration = -1;
417  info.video_length = -1;
418  is_duration_known = false;
419  }
420  else
421  {
422  // Yes, a duration was found
423  is_duration_known = true;
424 
425  // Calculate number of frames
427  }
428 
429  // Override an invalid framerate
430  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
431  // Calculate FPS, duration, video bit rate, and video length manually
432  // by scanning through all the video stream packets
433  CheckFPS();
434  }
435 
436  // Add video metadata (if any)
437  AVDictionaryEntry *tag = NULL;
438  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
439  QString str_key = tag->key;
440  QString str_value = tag->value;
441  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
442  }
443 }
444 
445 
446 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame)
447 {
448  // Check for open reader (or throw exception)
449  if (!is_open)
450  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
451 
452  // Adjust for a requested frame that is too small or too large
453  if (requested_frame < 1)
454  requested_frame = 1;
455  if (requested_frame > info.video_length && is_duration_known)
456  requested_frame = info.video_length;
457  if (info.has_video && info.video_length == 0)
458  // Invalid duration of video file
459  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
460 
461  // Debug output
462  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame, "", -1, "", -1, "", -1, "", -1);
463 
464  // Check the cache for this frame
465  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
466  if (frame) {
467  // Debug output
468  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
469 
470  // Return the cached frame
471  return frame;
472  }
473  else
474  {
475  #pragma omp critical (ReadStream)
476  {
477  // Check the cache a 2nd time (due to a potential previous lock)
478  if (has_missing_frames)
479  CheckMissingFrame(requested_frame);
480  frame = final_cache.GetFrame(requested_frame);
481  if (frame) {
482  // Debug output
483  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
484 
485  // Return the cached frame
486  }
487  else {
488  // Frame is not in cache
489  // Reset seek count
490  seek_count = 0;
491 
492  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
493  if (last_frame == 0 && requested_frame != 1)
494  // Get first frame
495  ReadStream(1);
496 
497  // Are we within X frames of the requested frame?
498  int64_t diff = requested_frame - last_frame;
499  if (diff >= 1 && diff <= 20)
500  {
501  // Continue walking the stream
502  frame = ReadStream(requested_frame);
503  }
504  else
505  {
506  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
507  if (enable_seek)
508  // Only seek if enabled
509  Seek(requested_frame);
510 
511  else if (!enable_seek && diff < 0)
512  {
513  // Start over, since we can't seek, and the requested frame is smaller than our position
514  Close();
515  Open();
516  }
517 
518  // Then continue walking the stream
519  frame = ReadStream(requested_frame);
520  }
521  }
522  } //omp critical
523  return frame;
524  }
525 }
526 
527 // Read the stream until we find the requested Frame
528 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
529 {
530  // Allocate video frame
531  bool end_of_stream = false;
532  bool check_seek = false;
533  bool frame_finished = false;
534  int packet_error = -1;
535 
536  // Minimum number of packets to process (for performance reasons)
537  int packets_processed = 0;
538  int minimum_packets = OPEN_MP_NUM_PROCESSORS;
539  int max_packets = 4096;
540 
541  // Set the number of threads in OpenMP
542  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
543  // Allow nested OpenMP sections
544  omp_set_nested(true);
545 
546  // Debug output
547  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS, "", -1, "", -1, "", -1, "", -1);
548 
549  #pragma omp parallel
550  {
551  #pragma omp single
552  {
553  // Loop through the stream until the correct frame is found
554  while (true)
555  {
556  // Get the next packet into a local variable called packet
557  packet_error = GetNextPacket();
558 
559  int processing_video_frames_size = 0;
560  int processing_audio_frames_size = 0;
561  {
562  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
563  processing_video_frames_size = processing_video_frames.size();
564  processing_audio_frames_size = processing_audio_frames.size();
565  }
566 
567  // Wait if too many frames are being processed
568  while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
569  usleep(2500);
570  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
571  processing_video_frames_size = processing_video_frames.size();
572  processing_audio_frames_size = processing_audio_frames.size();
573  }
574 
575  // Get the next packet (if any)
576  if (packet_error < 0)
577  {
578  // Break loop when no more packets found
579  end_of_stream = true;
580  break;
581  }
582 
583  // Debug output
584  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
585 
586  // Video packet
587  if (info.has_video && packet->stream_index == videoStream)
588  {
589  // Reset this counter, since we have a video packet
590  num_packets_since_video_frame = 0;
591 
592  // Check the status of a seek (if any)
593  if (is_seeking)
594  #pragma omp critical (openshot_seek)
595  check_seek = CheckSeek(true);
596  else
597  check_seek = false;
598 
599  if (check_seek) {
600  // Jump to the next iteration of this loop
601  continue;
602  }
603 
604  // Get the AVFrame from the current packet
605  frame_finished = GetAVFrame();
606 
607  // Check if the AVFrame is finished and set it
608  if (frame_finished)
609  {
610  // Update PTS / Frame Offset (if any)
611  UpdatePTSOffset(true);
612 
613  // Process Video Packet
614  ProcessVideoPacket(requested_frame);
615 
616  if (!use_omp_threads) {
617  // Wait on each OMP task to complete before moving on to the next one. This slows
618  // down processing considerably, but might be more stable on some systems.
619  #pragma omp taskwait
620  }
621  }
622 
623  }
624  // Audio packet
625  else if (info.has_audio && packet->stream_index == audioStream)
626  {
627  // Increment this (to track # of packets since the last video packet)
628  num_packets_since_video_frame++;
629 
630  // Check the status of a seek (if any)
631  if (is_seeking)
632  #pragma omp critical (openshot_seek)
633  check_seek = CheckSeek(false);
634  else
635  check_seek = false;
636 
637  if (check_seek) {
638  // Jump to the next iteration of this loop
639  continue;
640  }
641 
642  // Update PTS / Frame Offset (if any)
643  UpdatePTSOffset(false);
644 
645  // Determine related video frame and starting sample # from audio PTS
646  AudioLocation location = GetAudioPTSLocation(packet->pts);
647 
648  // Process Audio Packet
649  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
650  }
651 
652  // Check if working frames are 'finished'
653  if (!is_seeking) {
654  // Check for any missing frames
655  CheckMissingFrame(requested_frame);
656 
657  // Check for final frames
658  CheckWorkingFrames(false, requested_frame);
659  }
660 
661  // Check if requested 'final' frame is available
662  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
663 
664  // Increment frames processed
665  packets_processed++;
666 
667  // Break once the frame is found
668  if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
669  break;
670 
671  } // end while
672 
673  } // end omp single
674 
675  } // end omp parallel
676 
677  // Debug output
678  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count(), "", -1, "", -1);
679 
680  // End of stream?
681  if (end_of_stream)
682  // Mark the any other working frames as 'finished'
683  CheckWorkingFrames(end_of_stream, requested_frame);
684 
685  // Return requested frame (if found)
686  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
687  if (frame)
688  // Return prepared frame
689  return frame;
690  else {
691 
692  // Check if largest frame is still cached
693  frame = final_cache.GetFrame(largest_frame_processed);
694  if (frame) {
695  // return the largest processed frame (assuming it was the last in the video file)
696  return frame;
697  }
698  else {
699  // The largest processed frame is no longer in cache, return a blank frame
700  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
701  f->AddColor(info.width, info.height, "#000");
702  return f;
703  }
704  }
705 
706 }
707 
708 // Get the next packet (if any)
709 int FFmpegReader::GetNextPacket()
710 {
711  int found_packet = 0;
712  AVPacket *next_packet = new AVPacket();
713  found_packet = av_read_frame(pFormatCtx, next_packet);
714 
715  if (packet) {
716  // Remove previous packet before getting next one
717  RemoveAVPacket(packet);
718  packet = NULL;
719  }
720 
721  if (found_packet >= 0)
722  {
723  // Update current packet pointer
724  packet = next_packet;
725  }
726 
727  // Return if packet was found (or error number)
728  return found_packet;
729 }
730 
731 // Get an AVFrame (if any)
732 bool FFmpegReader::GetAVFrame()
733 {
734  int frameFinished = -1;
735  int ret = 0;
736 
737  // Decode video frame
738  AVFrame *next_frame = AV_ALLOCATE_FRAME();
739  #pragma omp critical (packet_cache)
740  {
741  #if IS_FFMPEG_3_2
742  frameFinished = 0;
743  ret = avcodec_send_packet(pCodecCtx, packet);
744  if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
745  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
746  }
747  else {
748  pFrame = new AVFrame();
749  while (ret >= 0) {
750  ret = avcodec_receive_frame(pCodecCtx, next_frame);
751  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
752  break;
753  }
754  // TODO also handle possible further frames
755  // Use only the first frame like avcodec_decode_video2
756  if (frameFinished == 0 ) {
757  frameFinished = 1;
758  av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
759  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
760  (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
761  if (!check_interlace) {
762  check_interlace = true;
763  info.interlaced_frame = next_frame->interlaced_frame;
764  info.top_field_first = next_frame->top_field_first;
765  }
766  }
767  }
768  }
769  #else
770  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
771 
772  // is frame finished
773  if (frameFinished) {
774  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
775  // must make a copy of the image data before this method is called again.
776  pFrame = AV_ALLOCATE_FRAME();
777  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
778  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
779  info.height);
780 
781  // Detect interlaced frame (only once)
782  if (!check_interlace) {
783  check_interlace = true;
784  info.interlaced_frame = next_frame->interlaced_frame;
785  info.top_field_first = next_frame->top_field_first;
786  }
787  }
788  #endif
789  }
790 
791  // deallocate the frame
792  AV_FREE_FRAME(&next_frame);
793 
794  // Did we get a video frame?
795  return frameFinished;
796 }
797 
798 // Check the current seek position and determine if we need to seek again
799 bool FFmpegReader::CheckSeek(bool is_video)
800 {
801  // Are we seeking for a specific frame?
802  if (is_seeking)
803  {
804  // Determine if both an audio and video packet have been decoded since the seek happened.
805  // If not, allow the ReadStream method to keep looping
806  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
807  return false;
808 
809  // Check for both streams
810  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
811  return false;
812 
813  // Determine max seeked frame
814  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
815  if (seek_video_frame_found > max_seeked_frame)
816  max_seeked_frame = seek_video_frame_found;
817 
818  // determine if we are "before" the requested frame
819  if (max_seeked_frame >= seeking_frame)
820  {
821  // SEEKED TOO FAR
822  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
823 
824  // Seek again... to the nearest Keyframe
825  Seek(seeking_frame - (10 * seek_count * seek_count));
826  }
827  else
828  {
829  // SEEK WORKED
830  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
831 
832  // Seek worked, and we are "before" the requested frame
833  is_seeking = false;
834  seeking_frame = 0;
835  seeking_pts = -1;
836  }
837  }
838 
839  // return the pts to seek to (if any)
840  return is_seeking;
841 }
842 
843 // Process a video packet
844 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
845 {
846  // Calculate current frame #
847  int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
848 
849  // Track 1st video packet after a successful seek
850  if (!seek_video_frame_found && is_seeking)
851  seek_video_frame_found = current_frame;
852 
853  // Are we close enough to decode the frame? and is this frame # valid?
854  if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
855  {
856  // Remove frame and packet
857  RemoveAVFrame(pFrame);
858 
859  // Debug output
860  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
861 
862  // Skip to next frame without decoding or caching
863  return;
864  }
865 
866  // Debug output
867  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
868 
869  // Init some things local (for OpenMP)
870  PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
871  int height = info.height;
872  int width = info.width;
873  int64_t video_length = info.video_length;
874  AVFrame *my_frame = pFrame;
875 
876  // Add video frame to list of processing video frames
877  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
878  processing_video_frames[current_frame] = current_frame;
879 
880  #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt)
881  {
882  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
883  AVFrame *pFrameRGB = NULL;
884  int numBytes;
885  uint8_t *buffer = NULL;
886 
887  // Allocate an AVFrame structure
888  pFrameRGB = AV_ALLOCATE_FRAME();
889  if (pFrameRGB == NULL)
890  throw OutOfBoundsFrame("Convert Image Broke!", current_frame, video_length);
891 
892  // Determine if video needs to be scaled down (for performance reasons)
893  // Timelines pass their size to the clips, which pass their size to the readers (as max size)
894  // If a clip is being scaled larger, it will set max_width and max_height = 0 (which means don't down scale)
895  int original_height = height;
896  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
897  // Override width and height (but maintain aspect ratio)
898  float ratio = float(width) / float(height);
899  int possible_width = round(max_height * ratio);
900  int possible_height = round(max_width / ratio);
901 
902  if (possible_width <= max_width) {
903  // use calculated width, and max_height
904  width = possible_width;
905  height = max_height;
906  } else {
907  // use max_width, and calculated height
908  width = max_width;
909  height = possible_height;
910  }
911  }
912 
913  // Determine required buffer size and allocate buffer
914  numBytes = AV_GET_IMAGE_SIZE(PIX_FMT_RGBA, width, height);
915 
916  #pragma omp critical (video_buffer)
917  buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
918 
919  // Copy picture data from one AVFrame (or AVPicture) to another one.
920  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
921 
922  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
923  height, PIX_FMT_RGBA, SWS_LANCZOS, NULL, NULL, NULL);
924 
925  // Resize / Convert to RGB
926  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
927  original_height, pFrameRGB->data, pFrameRGB->linesize);
928 
929  // Create or get the existing frame object
930  std::shared_ptr<Frame> f = CreateFrame(current_frame);
931 
932  // Add Image data to frame
933  f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
934 
935  // Update working cache
936  working_cache.Add(f);
937 
938  // Keep track of last last_video_frame
939  #pragma omp critical (video_buffer)
940  last_video_frame = f;
941 
942  // Free the RGB image
943  av_free(buffer);
944  AV_FREE_FRAME(&pFrameRGB);
945 
946  // Remove frame and packet
947  RemoveAVFrame(my_frame);
948  sws_freeContext(img_convert_ctx);
949 
950  // Remove video frame from list of processing video frames
951  {
952  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
953  processing_video_frames.erase(current_frame);
954  processed_video_frames[current_frame] = current_frame;
955  }
956 
957  // Debug output
958  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "", -1, "", -1, "", -1);
959 
960  } // end omp task
961 
962 }
963 
964 // Process an audio packet
965 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample)
966 {
967  // Track 1st audio packet after a successful seek
968  if (!seek_audio_frame_found && is_seeking)
969  seek_audio_frame_found = target_frame;
970 
971  // Are we close enough to decode the frame's audio?
972  if (target_frame < (requested_frame - 20))
973  {
974  // Debug output
975  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
976 
977  // Skip to next frame without decoding or caching
978  return;
979  }
980 
981  // Debug output
982  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
983 
984  // Init an AVFrame to hold the decoded audio samples
985  int frame_finished = 0;
986  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
987  AV_RESET_FRAME(audio_frame);
988 
989  int packet_samples = 0;
990  int data_size = 0;
991 
992  // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call)
994  #pragma omp critical (ProcessAudioPacket)
995  {
996  #if IS_FFMPEG_3_2
997  int ret = 0;
998  frame_finished = 1;
999  while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1000  frame_finished = 0;
1001  ret = avcodec_send_packet(aCodecCtx, packet);
1002  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1003  avcodec_send_packet(aCodecCtx, NULL);
1004  break;
1005  }
1006  if (ret >= 0)
1007  packet->size = 0;
1008  ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1009  if (ret >= 0)
1010  frame_finished = 1;
1011  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1012  avcodec_flush_buffers(aCodecCtx);
1013  ret = 0;
1014  }
1015  if (ret >= 0) {
1016  ret = frame_finished;
1017  }
1018  }
1019  if (!packet->data && !frame_finished)
1020  {
1021  ret = -1;
1022  }
1023  #else
1024  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1025 #endif
1026  }
1027 
1028  if (frame_finished) {
1029 
1030  // determine how many samples were decoded
1031  int planar = av_sample_fmt_is_planar((AVSampleFormat)AV_GET_CODEC_PIXEL_FORMAT(aStream, aCodecCtx));
1032  int plane_size = -1;
1033  data_size = av_samples_get_buffer_size(&plane_size,
1034  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1035  audio_frame->nb_samples,
1036  (AVSampleFormat)(AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1037 
1038  // Calculate total number of samples
1039  packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1040  }
1041 
1042  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1043  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1044 
1045  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
1046  int64_t adjusted_pts = packet->pts + audio_pts_offset;
1047  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
1048  double sample_seconds = double(pts_total) / info.sample_rate;
1049 
1050  // Debug output
1051  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
1052  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
1053 
1054  // DEBUG (FOR AUDIO ISSUES)
1055  prev_pts = adjusted_pts;
1056  pts_total += pts_remaining_samples;
1057  pts_counter++;
1058  prev_samples = pts_remaining_samples;
1059 
1060  // Add audio frame to list of processing audio frames
1061  {
1062  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1063  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1064  }
1065 
1066  while (pts_remaining_samples)
1067  {
1068  // Get Samples per frame (for this frame number)
1069  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1070 
1071  // Calculate # of samples to add to this frame
1072  int samples = samples_per_frame - previous_packet_location.sample_start;
1073  if (samples > pts_remaining_samples)
1074  samples = pts_remaining_samples;
1075 
1076  // Decrement remaining samples
1077  pts_remaining_samples -= samples;
1078 
1079  if (pts_remaining_samples > 0) {
1080  // next frame
1081  previous_packet_location.frame++;
1082  previous_packet_location.sample_start = 0;
1083 
1084  // Add audio frame to list of processing audio frames
1085  {
1086  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1087  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1088  }
1089 
1090  } else {
1091  // Increment sample start
1092  previous_packet_location.sample_start += samples;
1093  }
1094  }
1095 
1096 
1097  // Allocate audio buffer
1098  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE];
1099 
1100  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16, "", -1);
1101 
1102  // Create output frame
1103  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1104  AV_RESET_FRAME(audio_converted);
1105  audio_converted->nb_samples = audio_frame->nb_samples;
1106  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1107 
1108  SWRCONTEXT *avr = NULL;
1109  int nb_samples = 0;
1110 
1111  // setup resample context
1112  avr = SWR_ALLOC();
1113  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1114  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1115  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1116  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1117  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1118  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1119  av_opt_set_int(avr, "in_channels", info.channels, 0);
1120  av_opt_set_int(avr, "out_channels", info.channels, 0);
1121  int r = SWR_INIT(avr);
1122 
1123  // Convert audio samples
1124  nb_samples = SWR_CONVERT(avr, // audio resample context
1125  audio_converted->data, // output data pointers
1126  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1127  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1128  audio_frame->data, // input data pointers
1129  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1130  audio_frame->nb_samples); // number of input samples to convert
1131 
1132  // Copy audio samples over original samples
1133  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1134 
1135  // Deallocate resample buffer
1136  SWR_CLOSE(avr);
1137  SWR_FREE(&avr);
1138  avr = NULL;
1139 
1140  // Free AVFrames
1141  av_free(audio_converted->data[0]);
1142  AV_FREE_FRAME(&audio_converted);
1143 
1144  int64_t starting_frame_number = -1;
1145  bool partial_frame = true;
1146  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
1147  {
1148  // Array of floats (to hold samples for each channel)
1149  starting_frame_number = target_frame;
1150  int channel_buffer_size = packet_samples / info.channels;
1151  float *channel_buffer = new float[channel_buffer_size];
1152 
1153  // Init buffer array
1154  for (int z = 0; z < channel_buffer_size; z++)
1155  channel_buffer[z] = 0.0f;
1156 
1157  // Loop through all samples and add them to our Frame based on channel.
1158  // Toggle through each channel number, since channel data is stored like (left right left right)
1159  int channel = 0;
1160  int position = 0;
1161  for (int sample = 0; sample < packet_samples; sample++)
1162  {
1163  // Only add samples for current channel
1164  if (channel_filter == channel)
1165  {
1166  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1167  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1168 
1169  // Increment audio position
1170  position++;
1171  }
1172 
1173  // increment channel (if needed)
1174  if ((channel + 1) < info.channels)
1175  // move to next channel
1176  channel ++;
1177  else
1178  // reset channel
1179  channel = 0;
1180  }
1181 
1182  // Loop through samples, and add them to the correct frames
1183  int start = starting_sample;
1184  int remaining_samples = channel_buffer_size;
1185  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1186  while (remaining_samples > 0)
1187  {
1188  // Get Samples per frame (for this frame number)
1189  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1190 
1191  // Calculate # of samples to add to this frame
1192  int samples = samples_per_frame - start;
1193  if (samples > remaining_samples)
1194  samples = remaining_samples;
1195 
1196  // Create or get the existing frame object
1197  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1198 
1199  // Determine if this frame was "partially" filled in
1200  if (samples_per_frame == start + samples)
1201  partial_frame = false;
1202  else
1203  partial_frame = true;
1204 
1205  // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent
1206  // some louder samples from maxing out at 1.0 (not sure why this happens)
1207  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1208 
1209  // Debug output
1210  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1211 
1212  // Add or update cache
1213  working_cache.Add(f);
1214 
1215  // Decrement remaining samples
1216  remaining_samples -= samples;
1217 
1218  // Increment buffer (to next set of samples)
1219  if (remaining_samples > 0)
1220  iterate_channel_buffer += samples;
1221 
1222  // Increment frame number
1223  starting_frame_number++;
1224 
1225  // Reset starting sample #
1226  start = 0;
1227  }
1228 
1229  // clear channel buffer
1230  delete[] channel_buffer;
1231  channel_buffer = NULL;
1232  iterate_channel_buffer = NULL;
1233  }
1234 
1235  // Clean up some arrays
1236  delete[] audio_buf;
1237  audio_buf = NULL;
1238 
1239  // Remove audio frame from list of processing audio frames
1240  {
1241  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1242  // Update all frames as completed
1243  for (int64_t f = target_frame; f < starting_frame_number; f++) {
1244  // Remove the frame # from the processing list. NOTE: If more than one thread is
1245  // processing this frame, the frame # will be in this list multiple times. We are only
1246  // removing a single instance of it here.
1247  processing_audio_frames.erase(processing_audio_frames.find(f));
1248 
1249  // Check and see if this frame is also being processed by another thread
1250  if (processing_audio_frames.count(f) == 0)
1251  // No other thread is processing it. Mark the audio as processed (final)
1252  processed_audio_frames[f] = f;
1253  }
1254 
1255  if (target_frame == starting_frame_number) {
1256  // This typically never happens, but just in case, remove the currently processing number
1257  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1258  }
1259  }
1260 
1261  // Free audio frame
1262  AV_FREE_FRAME(&audio_frame);
1263 
1264  // Debug output
1265  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1, "", -1, "", -1, "", -1);
1266 
1267 }
1268 
1269 
1270 
1271 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1272 void FFmpegReader::Seek(int64_t requested_frame)
1273 {
1274  // Adjust for a requested frame that is too small or too large
1275  if (requested_frame < 1)
1276  requested_frame = 1;
1277  if (requested_frame > info.video_length)
1278  requested_frame = info.video_length;
1279 
1280  int processing_video_frames_size = 0;
1281  int processing_audio_frames_size = 0;
1282  {
1283  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1284  processing_video_frames_size = processing_video_frames.size();
1285  processing_audio_frames_size = processing_audio_frames.size();
1286  }
1287 
1288  // Debug output
1289  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "video_pts_offset", video_pts_offset);
1290 
1291  // Wait for any processing frames to complete
1292  while (processing_video_frames_size + processing_audio_frames_size > 0) {
1293  usleep(2500);
1294  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1295  processing_video_frames_size = processing_video_frames.size();
1296  processing_audio_frames_size = processing_audio_frames.size();
1297  }
1298 
1299  // Clear working cache (since we are seeking to another location in the file)
1300  working_cache.Clear();
1301  missing_frames.Clear();
1302 
1303  // Clear processed lists
1304  {
1305  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1306  processing_audio_frames.clear();
1307  processing_video_frames.clear();
1308  processed_video_frames.clear();
1309  processed_audio_frames.clear();
1310  missing_audio_frames.clear();
1311  missing_video_frames.clear();
1312  missing_audio_frames_source.clear();
1313  missing_video_frames_source.clear();
1314  checked_frames.clear();
1315  }
1316 
1317  // Reset the last frame variable
1318  last_frame = 0;
1319  current_video_frame = 0;
1320  largest_frame_processed = 0;
1321  num_checks_since_final = 0;
1322  num_packets_since_video_frame = 0;
1323  has_missing_frames = false;
1324  bool has_audio_override = info.has_audio;
1325  bool has_video_override = info.has_video;
1326 
1327  // Increment seek count
1328  seek_count++;
1329 
1330  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1331  int buffer_amount = max(OPEN_MP_NUM_PROCESSORS, 8);
1332  if (requested_frame - buffer_amount < 20)
1333  {
1334  // Close and re-open file (basically seeking to frame 1)
1335  Close();
1336  Open();
1337 
1338  // Update overrides (since closing and re-opening might update these)
1339  info.has_audio = has_audio_override;
1340  info.has_video = has_video_override;
1341 
1342  // Not actually seeking, so clear these flags
1343  is_seeking = false;
1344  if (seek_count == 1) {
1345  // Don't redefine this on multiple seek attempts for a specific frame
1346  seeking_frame = 1;
1347  seeking_pts = ConvertFrameToVideoPTS(1);
1348  }
1349  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1350  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1351  }
1352  else
1353  {
1354  // Seek to nearest key-frame (aka, i-frame)
1355  bool seek_worked = false;
1356  int64_t seek_target = 0;
1357 
1358  // Seek video stream (if any)
1359  if (!seek_worked && info.has_video)
1360  {
1361  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1362  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1363  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1364  } else
1365  {
1366  // VIDEO SEEK
1367  is_video_seek = true;
1368  seek_worked = true;
1369  }
1370  }
1371 
1372  // Seek audio stream (if not already seeked... and if an audio stream is found)
1373  if (!seek_worked && info.has_audio)
1374  {
1375  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1376  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1377  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1378  } else
1379  {
1380  // AUDIO SEEK
1381  is_video_seek = false;
1382  seek_worked = true;
1383  }
1384  }
1385 
1386  // Was the seek successful?
1387  if (seek_worked)
1388  {
1389  // Flush audio buffer
1390  if (info.has_audio)
1391  avcodec_flush_buffers(aCodecCtx);
1392 
1393  // Flush video buffer
1394  if (info.has_video)
1395  avcodec_flush_buffers(pCodecCtx);
1396 
1397  // Reset previous audio location to zero
1398  previous_packet_location.frame = -1;
1399  previous_packet_location.sample_start = 0;
1400 
1401  // init seek flags
1402  is_seeking = true;
1403  if (seek_count == 1) {
1404  // Don't redefine this on multiple seek attempts for a specific frame
1405  seeking_pts = seek_target;
1406  seeking_frame = requested_frame;
1407  }
1408  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1409  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1410 
1411  }
1412  else
1413  {
1414  // seek failed
1415  is_seeking = false;
1416  seeking_pts = 0;
1417  seeking_frame = 0;
1418 
1419  // dislable seeking for this reader (since it failed)
1420  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1421  enable_seek = false;
1422 
1423  // Close and re-open file (basically seeking to frame 1)
1424  Close();
1425  Open();
1426 
1427  // Update overrides (since closing and re-opening might update these)
1428  info.has_audio = has_audio_override;
1429  info.has_video = has_video_override;
1430  }
1431  }
1432 }
1433 
1434 // Get the PTS for the current video packet
1435 int64_t FFmpegReader::GetVideoPTS()
1436 {
1437  int64_t current_pts = 0;
1438  if(packet->dts != AV_NOPTS_VALUE)
1439  current_pts = packet->dts;
1440 
1441  // Return adjusted PTS
1442  return current_pts;
1443 }
1444 
1445 // Update PTS Offset (if any)
1446 void FFmpegReader::UpdatePTSOffset(bool is_video)
1447 {
1448  // Determine the offset between the PTS and Frame number (only for 1st frame)
1449  if (is_video)
1450  {
1451  // VIDEO PACKET
1452  if (video_pts_offset == 99999) // Has the offset been set yet?
1453  {
1454  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1455  video_pts_offset = 0 - max(GetVideoPTS(), (int64_t) info.video_timebase.ToInt() * 10);
1456 
1457  // debug output
1458  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1459  }
1460  }
1461  else
1462  {
1463  // AUDIO PACKET
1464  if (audio_pts_offset == 99999) // Has the offset been set yet?
1465  {
1466  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1467  audio_pts_offset = 0 - max(packet->pts, (int64_t) info.audio_timebase.ToInt() * 10);
1468 
1469  // debug output
1470  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1471  }
1472  }
1473 }
1474 
1475 // Convert PTS into Frame Number
1476 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1477 {
1478  // Apply PTS offset
1479  pts = pts + video_pts_offset;
1480  int64_t previous_video_frame = current_video_frame;
1481 
1482  // Get the video packet start time (in seconds)
1483  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1484 
1485  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1486  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1487 
1488  // Keep track of the expected video frame #
1489  if (current_video_frame == 0)
1490  current_video_frame = frame;
1491  else {
1492 
1493  // Sometimes frames are duplicated due to identical (or similar) timestamps
1494  if (frame == previous_video_frame) {
1495  // return -1 frame number
1496  frame = -1;
1497  }
1498  else
1499  // Increment expected frame
1500  current_video_frame++;
1501 
1502  if (current_video_frame < frame)
1503  // has missing frames
1504  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame, "", -1, "", -1, "", -1);
1505 
1506  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1507  // if we are missing a video frame.
1508  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1509  while (current_video_frame < frame) {
1510  if (!missing_video_frames.count(current_video_frame)) {
1511  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame, "", -1, "", -1, "", -1, "", -1);
1512  missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1513  missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1514  }
1515 
1516  // Mark this reader as containing missing frames
1517  has_missing_frames = true;
1518 
1519  // Increment current frame
1520  current_video_frame++;
1521  }
1522  }
1523 
1524  // Return frame #
1525  return frame;
1526 }
1527 
1528 // Convert Frame Number into Video PTS
1529 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1530 {
1531  // Get timestamp of this frame (in seconds)
1532  double seconds = double(frame_number) / info.fps.ToDouble();
1533 
1534  // Calculate the # of video packets in this timestamp
1535  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1536 
1537  // Apply PTS offset (opposite)
1538  return video_pts - video_pts_offset;
1539 }
1540 
1541 // Convert Frame Number into Video PTS
1542 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1543 {
1544  // Get timestamp of this frame (in seconds)
1545  double seconds = double(frame_number) / info.fps.ToDouble();
1546 
1547  // Calculate the # of audio packets in this timestamp
1548  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1549 
1550  // Apply PTS offset (opposite)
1551  return audio_pts - audio_pts_offset;
1552 }
1553 
1554 // Calculate Starting video frame and sample # for an audio PTS
1555 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1556 {
1557  // Apply PTS offset
1558  pts = pts + audio_pts_offset;
1559 
1560  // Get the audio packet start time (in seconds)
1561  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1562 
1563  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1564  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1565 
1566  // Frame # as a whole number (no more decimals)
1567  int64_t whole_frame = int64_t(frame);
1568 
1569  // Remove the whole number, and only get the decimal of the frame
1570  double sample_start_percentage = frame - double(whole_frame);
1571 
1572  // Get Samples per frame
1573  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1574 
1575  // Calculate the sample # to start on
1576  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1577 
1578  // Protect against broken (i.e. negative) timestamps
1579  if (whole_frame < 1)
1580  whole_frame = 1;
1581  if (sample_start < 0)
1582  sample_start = 0;
1583 
1584  // Prepare final audio packet location
1585  AudioLocation location = {whole_frame, sample_start};
1586 
1587  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1588  if (previous_packet_location.frame != -1) {
1589  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1590  {
1591  int64_t orig_frame = location.frame;
1592  int orig_start = location.sample_start;
1593 
1594  // Update sample start, to prevent gaps in audio
1595  location.sample_start = previous_packet_location.sample_start;
1596  location.frame = previous_packet_location.frame;
1597 
1598  // Debug output
1599  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1);
1600 
1601  } else {
1602  // Debug output
1603  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1, "", -1);
1604 
1605  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1606  for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
1607  if (!missing_audio_frames.count(audio_frame)) {
1608  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1);
1609  missing_audio_frames.insert(pair<int64_t, int64_t>(previous_packet_location.frame - 1, audio_frame));
1610  }
1611  }
1612  }
1613  }
1614 
1615  // Set previous location
1616  previous_packet_location = location;
1617 
1618  // Return the associated video frame and starting sample #
1619  return location;
1620 }
1621 
1622 // Create a new Frame (or return an existing one) and add it to the working queue.
1623 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1624 {
1625  // Check working cache
1626  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
1627  if (!output)
1628  {
1629  // Create a new frame on the working cache
1630  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
1631  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
1632  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
1633  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
1634 
1635  working_cache.Add(output);
1636 
1637  // Set the largest processed frame (if this is larger)
1638  if (requested_frame > largest_frame_processed)
1639  largest_frame_processed = requested_frame;
1640  }
1641 
1642  // Return new frame
1643  return output;
1644 }
1645 
1646 // Determine if frame is partial due to seek
1647 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1648 
1649  // Sometimes a seek gets partial frames, and we need to remove them
1650  bool seek_trash = false;
1651  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1652  if (seek_video_frame_found > max_seeked_frame)
1653  max_seeked_frame = seek_video_frame_found;
1654  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1655  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1656  seek_trash = true;
1657 
1658  return seek_trash;
1659 }
1660 
1661 // Check if a frame is missing and attempt to replace it's frame image (and
1662 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1663 {
1664  // Lock
1665  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1666 
1667  // Init # of times this frame has been checked so far
1668  int checked_count = 0;
1669 
1670  // Increment check count for this frame (or init to 1)
1671  if (checked_frames.count(requested_frame) == 0)
1672  checked_frames[requested_frame] = 1;
1673  else
1674  checked_frames[requested_frame]++;
1675  checked_count = checked_frames[requested_frame];
1676 
1677  // Debug output
1678  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_count, "", -1, "", -1);
1679 
1680  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
1681  map<int64_t, int64_t>::iterator itr;
1682  bool found_missing_frame = false;
1683 
1684  // Check if requested frame is a missing frame
1685  if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1686  int64_t missing_source_frame = -1;
1687  if (missing_video_frames.count(requested_frame))
1688  missing_source_frame = missing_video_frames.find(requested_frame)->second;
1689  else if (missing_audio_frames.count(requested_frame))
1690  missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1691 
1692  // Increment missing source frame check count (or init to 1)
1693  if (checked_frames.count(missing_source_frame) == 0)
1694  checked_frames[missing_source_frame] = 1;
1695  else
1696  checked_frames[missing_source_frame]++;
1697 
1698  // Get the previous frame of this missing frame (if it's available in missing cache)
1699  std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
1700  if (parent_frame == NULL) {
1701  parent_frame = final_cache.GetFrame(missing_source_frame);
1702  if (parent_frame != NULL) {
1703  // Add missing final frame to missing cache
1704  missing_frames.Add(parent_frame);
1705  }
1706  }
1707 
1708  // Create blank missing frame
1709  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1710 
1711  // Debug output
1712  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1713 
1714  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
1715  if (parent_frame != NULL) {
1716  // Debug output
1717  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1718 
1719  // Add this frame to the processed map (since it's already done)
1720  std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1721  if (parent_image) {
1722  missing_frame->AddImage(std::shared_ptr<QImage>(new QImage(*parent_image)));
1723 
1724  processed_video_frames[missing_frame->number] = missing_frame->number;
1725  processed_audio_frames[missing_frame->number] = missing_frame->number;
1726 
1727  // Move frame to final cache
1728  final_cache.Add(missing_frame);
1729 
1730  // Remove frame from working cache
1731  working_cache.Remove(missing_frame->number);
1732 
1733  // Update last_frame processed
1734  last_frame = missing_frame->number;
1735  }
1736  }
1737 
1738  }
1739 
1740  return found_missing_frame;
1741 }
1742 
1743 // Check the working queue, and move finished frames to the finished queue
1744 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame)
1745 {
1746  // Loop through all working queue frames
1747  bool checked_count_tripped = false;
1748  int max_checked_count = 80;
1749 
1750  while (true)
1751  {
1752  // Get the front frame of working cache
1753  std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
1754 
1755  // Was a frame found?
1756  if (!f)
1757  // No frames found
1758  break;
1759 
1760  // Remove frames which are too old
1761  if (f && f->number < (requested_frame - (OPEN_MP_NUM_PROCESSORS * 2))) {
1762  working_cache.Remove(f->number);
1763  }
1764 
1765  // Check if this frame is 'missing'
1766  CheckMissingFrame(f->number);
1767 
1768  // Init # of times this frame has been checked so far
1769  int checked_count = 0;
1770  int checked_frames_size = 0;
1771 
1772  bool is_video_ready = false;
1773  bool is_audio_ready = false;
1774  { // limit scope of next few lines
1775  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1776  is_video_ready = processed_video_frames.count(f->number);
1777  is_audio_ready = processed_audio_frames.count(f->number);
1778 
1779  // Get check count for this frame
1780  checked_frames_size = checked_frames.size();
1781  if (!checked_count_tripped || f->number >= requested_frame)
1782  checked_count = checked_frames[f->number];
1783  else
1784  // Force checked count over the limit
1785  checked_count = max_checked_count;
1786  }
1787 
1788  if (previous_packet_location.frame == f->number && !end_of_stream)
1789  is_audio_ready = false; // don't finalize the last processed audio frame
1790  bool is_seek_trash = IsPartialFrame(f->number);
1791 
1792  // Adjust for available streams
1793  if (!info.has_video) is_video_ready = true;
1794  if (!info.has_audio) is_audio_ready = true;
1795 
1796  // Make final any frames that get stuck (for whatever reason)
1797  if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1798  // Debug output
1799  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1800 
1801  // Trigger checked count tripped mode (clear out all frames before requested frame)
1802  checked_count_tripped = true;
1803 
1804  if (info.has_video && !is_video_ready && last_video_frame) {
1805  // Copy image from last frame
1806  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1807  is_video_ready = true;
1808  }
1809 
1810  if (info.has_audio && !is_audio_ready) {
1811  // Mark audio as processed, and indicate the frame has audio data
1812  is_audio_ready = true;
1813  }
1814  }
1815 
1816  // Debug output
1817  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1818 
1819  // Check if working frame is final
1820  if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1821  {
1822  // Debug output
1823  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream);
1824 
1825  if (!is_seek_trash)
1826  {
1827  // Add missing image (if needed - sometimes end_of_stream causes frames with only audio)
1828  if (info.has_video && !is_video_ready && last_video_frame)
1829  // Copy image from last frame
1830  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1831 
1832  // Reset counter since last 'final' frame
1833  num_checks_since_final = 0;
1834 
1835  // Move frame to final cache
1836  final_cache.Add(f);
1837 
1838  // Add to missing cache (if another frame depends on it)
1839  {
1840  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1841  if (missing_video_frames_source.count(f->number)) {
1842  // Debug output
1843  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1);
1844  missing_frames.Add(f);
1845  }
1846 
1847  // Remove from 'checked' count
1848  checked_frames.erase(f->number);
1849  }
1850 
1851  // Remove frame from working cache
1852  working_cache.Remove(f->number);
1853 
1854  // Update last frame processed
1855  last_frame = f->number;
1856 
1857  } else {
1858  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
1859  working_cache.Remove(f->number);
1860  }
1861  }
1862  else
1863  // Stop looping
1864  break;
1865  }
1866 }
1867 
1868 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
1869 void FFmpegReader::CheckFPS()
1870 {
1871  check_fps = true;
1872 
1873  int first_second_counter = 0;
1874  int second_second_counter = 0;
1875  int third_second_counter = 0;
1876  int forth_second_counter = 0;
1877  int fifth_second_counter = 0;
1878  int frames_detected = 0;
1879  int64_t pts = 0;
1880 
1881  // Loop through the stream
1882  while (true)
1883  {
1884  // Get the next packet (if any)
1885  if (GetNextPacket() < 0)
1886  // Break loop when no more packets found
1887  break;
1888 
1889  // Video packet
1890  if (packet->stream_index == videoStream)
1891  {
1892  // Check if the AVFrame is finished and set it
1893  if (GetAVFrame())
1894  {
1895  // Update PTS / Frame Offset (if any)
1896  UpdatePTSOffset(true);
1897 
1898  // Get PTS of this packet
1899  pts = GetVideoPTS();
1900 
1901  // Remove pFrame
1902  RemoveAVFrame(pFrame);
1903 
1904  // Apply PTS offset
1905  pts += video_pts_offset;
1906 
1907  // Get the video packet start time (in seconds)
1908  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1909 
1910  // Increment the correct counter
1911  if (video_seconds <= 1.0)
1912  first_second_counter++;
1913  else if (video_seconds > 1.0 && video_seconds <= 2.0)
1914  second_second_counter++;
1915  else if (video_seconds > 2.0 && video_seconds <= 3.0)
1916  third_second_counter++;
1917  else if (video_seconds > 3.0 && video_seconds <= 4.0)
1918  forth_second_counter++;
1919  else if (video_seconds > 4.0 && video_seconds <= 5.0)
1920  fifth_second_counter++;
1921 
1922  // Increment counters
1923  frames_detected++;
1924  }
1925  }
1926  }
1927 
1928  // Double check that all counters have greater than zero (or give up)
1929  if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
1930  // Calculate average FPS (average of first few seconds)
1931  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1932  int avg_fps = round(sum_fps / 4.0f);
1933 
1934  // Update FPS
1935  info.fps = Fraction(avg_fps, 1);
1936 
1937  // Update Duration and Length
1938  info.video_length = frames_detected;
1939  info.duration = frames_detected / (sum_fps / 4.0f);
1940 
1941  // Update video bit rate
1943  } else if (second_second_counter != 0 && third_second_counter != 0) {
1944  // Calculate average FPS (only on second 2)
1945  int sum_fps = second_second_counter;
1946 
1947  // Update FPS
1948  info.fps = Fraction(sum_fps, 1);
1949 
1950  // Update Duration and Length
1951  info.video_length = frames_detected;
1952  info.duration = frames_detected / float(sum_fps);
1953 
1954  // Update video bit rate
1956  } else {
1957  // Too short to determine framerate, just default FPS
1958  // Set a few important default video settings (so audio can be divided into frames)
1959  info.fps.num = 30;
1960  info.fps.den = 1;
1961 
1962  // Calculate number of frames
1963  info.video_length = frames_detected;
1964  info.duration = frames_detected / info.fps.ToFloat();
1965  }
1966 }
1967 
1968 // Remove AVFrame from cache (and deallocate it's memory)
1969 void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame)
1970 {
1971  // Remove pFrame (if exists)
1972  if (remove_frame)
1973  {
1974  // Free memory
1975  av_freep(&remove_frame->data[0]);
1976  }
1977 }
1978 
1979 // Remove AVPacket from cache (and deallocate it's memory)
1980 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1981 {
1982  // deallocate memory for packet
1983  AV_FREE_PACKET(remove_packet);
1984 
1985  // Delete the object
1986  delete remove_packet;
1987 }
1988 
1989 /// Get the smallest video frame that is still being processed
1990 int64_t FFmpegReader::GetSmallestVideoFrame()
1991 {
1992  // Loop through frame numbers
1993  map<int64_t, int64_t>::iterator itr;
1994  int64_t smallest_frame = -1;
1995  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1996  for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1997  {
1998  if (itr->first < smallest_frame || smallest_frame == -1)
1999  smallest_frame = itr->first;
2000  }
2001 
2002  // Return frame number
2003  return smallest_frame;
2004 }
2005 
2006 /// Get the smallest audio frame that is still being processed
2007 int64_t FFmpegReader::GetSmallestAudioFrame()
2008 {
2009  // Loop through frame numbers
2010  map<int64_t, int64_t>::iterator itr;
2011  int64_t smallest_frame = -1;
2012  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
2013  for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
2014  {
2015  if (itr->first < smallest_frame || smallest_frame == -1)
2016  smallest_frame = itr->first;
2017  }
2018 
2019  // Return frame number
2020  return smallest_frame;
2021 }
2022 
2023 // Generate JSON string of this object
2025 
2026  // Return formatted string
2027  return JsonValue().toStyledString();
2028 }
2029 
2030 // Generate Json::JsonValue for this object
2032 
2033  // Create root json object
2034  Json::Value root = ReaderBase::JsonValue(); // get parent properties
2035  root["type"] = "FFmpegReader";
2036  root["path"] = path;
2037 
2038  // return JsonValue
2039  return root;
2040 }
2041 
2042 // Load JSON string into this object
2043 void FFmpegReader::SetJson(string value) {
2044 
2045  // Parse JSON string into JSON objects
2046  Json::Value root;
2047  Json::Reader reader;
2048  bool success = reader.parse( value, root );
2049  if (!success)
2050  // Raise exception
2051  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
2052 
2053  try
2054  {
2055  // Set all values that match
2056  SetJsonValue(root);
2057  }
2058  catch (exception e)
2059  {
2060  // Error parsing JSON (or missing keys)
2061  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
2062  }
2063 }
2064 
2065 // Load Json::JsonValue into this object
2066 void FFmpegReader::SetJsonValue(Json::Value root) {
2067 
2068  // Set parent data
2070 
2071  // Set data from Json (if key is found)
2072  if (!root["path"].isNull())
2073  path = root["path"].asString();
2074 
2075  // Re-Open path, and re-init everything (if needed)
2076  if (is_open)
2077  {
2078  Close();
2079  Open();
2080  }
2081 }
#define AV_RESET_FRAME(av_frame)
int max_height
The maximium image height needed by this clip (used for optimizations)
Definition: ReaderBase.h:104
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
#define SWRCONTEXT
#define AV_FIND_DECODER_CODEC_ID(av_stream)
CriticalSection processingCriticalSection
Definition: ReaderBase.h:101
#define SWR_INIT(ctx)
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_REGISTER_ALL
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:79
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:46
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define SWR_CLOSE(ctx)
#define SWR_FREE(ctx)
#define OPEN_MP_NUM_PROCESSORS
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AVCODEC_REGISTER_ALL
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
void Close()
Close File.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:72
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:65
FFmpegReader(string path)
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:80
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:84
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
#define FF_NUM_PROCESSORS
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
Definition: Exceptions.h:192
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:113
auto AV_GET_CODEC_CONTEXT
#define PixelFormat
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:168
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:112
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
std::map< string, string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:86
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
int64_t Count()
Count the frames in the queue.
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:71
This namespace is the default namespace for all code in the openshot library.
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Exception for invalid JSON.
Definition: Exceptions.h:152
#define AV_GET_CODEC_TYPE(av_stream)
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:68
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:59
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:70
#define SWR_ALLOC()
#define PIX_FMT_RGBA
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:85
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:73
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:220
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:75
int max_width
The maximum image width needed by this clip (used for optimizations)
Definition: ReaderBase.h:103
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:521
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81