FFmpeg  4.4
avfoundation.m
Go to the documentation of this file.
1 /*
2  * AVFoundation input device
3  * Copyright (c) 2014 Thilo Borgmann <thilo.borgmann@mail.de>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * AVFoundation input device
25  * @author Thilo Borgmann <thilo.borgmann@mail.de>
26  */
27 
28 #import <AVFoundation/AVFoundation.h>
29 #include <pthread.h>
30 
31 #include "libavutil/pixdesc.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/avstring.h"
34 #include "libavformat/internal.h"
35 #include "libavutil/internal.h"
36 #include "libavutil/parseutils.h"
37 #include "libavutil/time.h"
38 #include "libavutil/imgutils.h"
39 #include "avdevice.h"
40 
41 static const int avf_time_base = 1000000;
42 
43 static const AVRational avf_time_base_q = {
44  .num = 1,
45  .den = avf_time_base
46 };
47 
49  enum AVPixelFormat ff_id;
50  OSType avf_id;
51 };
52 
53 static const struct AVFPixelFormatSpec avf_pixel_formats[] = {
54  { AV_PIX_FMT_MONOBLACK, kCVPixelFormatType_1Monochrome },
55  { AV_PIX_FMT_RGB555BE, kCVPixelFormatType_16BE555 },
56  { AV_PIX_FMT_RGB555LE, kCVPixelFormatType_16LE555 },
57  { AV_PIX_FMT_RGB565BE, kCVPixelFormatType_16BE565 },
58  { AV_PIX_FMT_RGB565LE, kCVPixelFormatType_16LE565 },
59  { AV_PIX_FMT_RGB24, kCVPixelFormatType_24RGB },
60  { AV_PIX_FMT_BGR24, kCVPixelFormatType_24BGR },
61  { AV_PIX_FMT_0RGB, kCVPixelFormatType_32ARGB },
62  { AV_PIX_FMT_BGR0, kCVPixelFormatType_32BGRA },
63  { AV_PIX_FMT_0BGR, kCVPixelFormatType_32ABGR },
64  { AV_PIX_FMT_RGB0, kCVPixelFormatType_32RGBA },
65  { AV_PIX_FMT_BGR48BE, kCVPixelFormatType_48RGB },
66  { AV_PIX_FMT_UYVY422, kCVPixelFormatType_422YpCbCr8 },
67  { AV_PIX_FMT_YUVA444P, kCVPixelFormatType_4444YpCbCrA8R },
68  { AV_PIX_FMT_YUVA444P16LE, kCVPixelFormatType_4444AYpCbCr16 },
69  { AV_PIX_FMT_YUV444P, kCVPixelFormatType_444YpCbCr8 },
70  { AV_PIX_FMT_YUV422P16, kCVPixelFormatType_422YpCbCr16 },
71  { AV_PIX_FMT_YUV422P10, kCVPixelFormatType_422YpCbCr10 },
72  { AV_PIX_FMT_YUV444P10, kCVPixelFormatType_444YpCbCr10 },
73  { AV_PIX_FMT_YUV420P, kCVPixelFormatType_420YpCbCr8Planar },
74  { AV_PIX_FMT_NV12, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },
75  { AV_PIX_FMT_YUYV422, kCVPixelFormatType_422YpCbCr8_yuvs },
76 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
77  { AV_PIX_FMT_GRAY8, kCVPixelFormatType_OneComponent8 },
78 #endif
79  { AV_PIX_FMT_NONE, 0 }
80 };
81 
82 typedef struct
83 {
84  AVClass* class;
85 
88  int64_t first_pts;
89  int64_t first_audio_pts;
93 
95  int width, height;
96 
103 
109 
112 
114 
118  int audio_be;
122 
125 
126  enum AVPixelFormat pixel_format;
127 
128  AVCaptureSession *capture_session;
129  AVCaptureVideoDataOutput *video_output;
130  AVCaptureAudioDataOutput *audio_output;
131  CMSampleBufferRef current_frame;
132  CMSampleBufferRef current_audio_frame;
133 
134  AVCaptureDevice *observed_device;
135 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
136  AVCaptureDeviceTransportControlsPlaybackMode observed_mode;
137 #endif
139 } AVFContext;
140 
142 {
143  pthread_mutex_lock(&ctx->frame_lock);
144 }
145 
147 {
148  pthread_mutex_unlock(&ctx->frame_lock);
149 }
150 
151 /** FrameReciever class - delegate for AVCaptureSession
152  */
153 @interface AVFFrameReceiver : NSObject
154 {
156 }
157 
158 - (id)initWithContext:(AVFContext*)context;
159 
160 - (void) captureOutput:(AVCaptureOutput *)captureOutput
161  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
162  fromConnection:(AVCaptureConnection *)connection;
163 
164 @end
165 
166 @implementation AVFFrameReceiver
167 
168 - (id)initWithContext:(AVFContext*)context
169 {
170  if (self = [super init]) {
171  _context = context;
172 
173  // start observing if a device is set for it
174 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
175  if (_context->observed_device) {
176  NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
177  NSKeyValueObservingOptions options = NSKeyValueObservingOptionNew;
178 
179  [_context->observed_device addObserver: self
180  forKeyPath: keyPath
181  options: options
182  context: _context];
183  }
184 #endif
185  }
186  return self;
187 }
188 
189 - (void)dealloc {
190  // stop observing if a device is set for it
191 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
192  if (_context->observed_device) {
193  NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
194  [_context->observed_device removeObserver: self forKeyPath: keyPath];
195  }
196 #endif
197  [super dealloc];
198 }
199 
200 - (void)observeValueForKeyPath:(NSString *)keyPath
201  ofObject:(id)object
202  change:(NSDictionary *)change
203  context:(void *)context {
204  if (context == _context) {
205 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
206  AVCaptureDeviceTransportControlsPlaybackMode mode =
207  [change[NSKeyValueChangeNewKey] integerValue];
208 
209  if (mode != _context->observed_mode) {
210  if (mode == AVCaptureDeviceTransportControlsNotPlayingMode) {
211  _context->observed_quit = 1;
212  }
213  _context->observed_mode = mode;
214  }
215 #endif
216  } else {
217  [super observeValueForKeyPath: keyPath
218  ofObject: object
219  change: change
220  context: context];
221  }
222 }
223 
224 - (void) captureOutput:(AVCaptureOutput *)captureOutput
225  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
226  fromConnection:(AVCaptureConnection *)connection
227 {
229 
230  if (_context->current_frame != nil) {
231  CFRelease(_context->current_frame);
232  }
233 
234  _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
235 
237 
238  ++_context->frames_captured;
239 }
240 
241 @end
242 
243 /** AudioReciever class - delegate for AVCaptureSession
244  */
245 @interface AVFAudioReceiver : NSObject
246 {
248 }
249 
250 - (id)initWithContext:(AVFContext*)context;
251 
252 - (void) captureOutput:(AVCaptureOutput *)captureOutput
253  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
254  fromConnection:(AVCaptureConnection *)connection;
255 
256 @end
257 
258 @implementation AVFAudioReceiver
259 
260 - (id)initWithContext:(AVFContext*)context
261 {
262  if (self = [super init]) {
263  _context = context;
264  }
265  return self;
266 }
267 
268 - (void) captureOutput:(AVCaptureOutput *)captureOutput
269  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
270  fromConnection:(AVCaptureConnection *)connection
271 {
273 
274  if (_context->current_audio_frame != nil) {
275  CFRelease(_context->current_audio_frame);
276  }
277 
278  _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
279 
281 
282  ++_context->audio_frames_captured;
283 }
284 
285 @end
286 
288 {
289  [ctx->capture_session stopRunning];
290 
291  [ctx->capture_session release];
292  [ctx->video_output release];
293  [ctx->audio_output release];
294  [ctx->avf_delegate release];
295  [ctx->avf_audio_delegate release];
296 
297  ctx->capture_session = NULL;
298  ctx->video_output = NULL;
299  ctx->audio_output = NULL;
300  ctx->avf_delegate = NULL;
301  ctx->avf_audio_delegate = NULL;
302 
303  av_freep(&ctx->audio_buffer);
304 
305  pthread_mutex_destroy(&ctx->frame_lock);
306 
307  if (ctx->current_frame) {
308  CFRelease(ctx->current_frame);
309  }
310 }
311 
313 {
314  AVFContext *ctx = (AVFContext*)s->priv_data;
315  char *tmp = av_strdup(s->url);
316  char *save;
317 
318  if (tmp[0] != ':') {
319  ctx->video_filename = av_strtok(tmp, ":", &save);
320  ctx->audio_filename = av_strtok(NULL, ":", &save);
321  } else {
322  ctx->audio_filename = av_strtok(tmp, ":", &save);
323  }
324 }
325 
326 /**
327  * Configure the video device.
328  *
329  * Configure the video device using a run-time approach to access properties
330  * since formats, activeFormat are available since iOS >= 7.0 or OSX >= 10.7
331  * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.
332  *
333  * The NSUndefinedKeyException must be handled by the caller of this function.
334  *
335  */
336 static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
337 {
338  AVFContext *ctx = (AVFContext*)s->priv_data;
339 
340  double framerate = av_q2d(ctx->framerate);
341  NSObject *range = nil;
342  NSObject *format = nil;
343  NSObject *selected_range = nil;
344  NSObject *selected_format = nil;
345 
346  // try to configure format by formats list
347  // might raise an exception if no format list is given
348  // (then fallback to default, no configuration)
349  @try {
350  for (format in [video_device valueForKey:@"formats"]) {
351  CMFormatDescriptionRef formatDescription;
352  CMVideoDimensions dimensions;
353 
354  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
355  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
356 
357  if ((ctx->width == 0 && ctx->height == 0) ||
358  (dimensions.width == ctx->width && dimensions.height == ctx->height)) {
359 
360  selected_format = format;
361 
362  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
363  double max_framerate;
364 
365  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
366  if (fabs (framerate - max_framerate) < 0.01) {
367  selected_range = range;
368  break;
369  }
370  }
371  }
372  }
373 
374  if (!selected_format) {
375  av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device.\n",
376  ctx->width, ctx->height);
377  goto unsupported_format;
378  }
379 
380  if (!selected_range) {
381  av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n",
382  framerate);
383  if (ctx->video_is_muxed) {
384  av_log(s, AV_LOG_ERROR, "Falling back to default.\n");
385  } else {
386  goto unsupported_format;
387  }
388  }
389 
390  if ([video_device lockForConfiguration:NULL] == YES) {
391  if (selected_format) {
392  [video_device setValue:selected_format forKey:@"activeFormat"];
393  }
394  if (selected_range) {
395  NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];
396  [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];
397  [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];
398  }
399  } else {
400  av_log(s, AV_LOG_ERROR, "Could not lock device for configuration.\n");
401  return AVERROR(EINVAL);
402  }
403  } @catch(NSException *e) {
404  av_log(ctx, AV_LOG_WARNING, "Configuration of video device failed, falling back to default.\n");
405  }
406 
407  return 0;
408 
409 unsupported_format:
410 
411  av_log(s, AV_LOG_ERROR, "Supported modes:\n");
412  for (format in [video_device valueForKey:@"formats"]) {
413  CMFormatDescriptionRef formatDescription;
414  CMVideoDimensions dimensions;
415 
416  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
417  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
418 
419  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
420  double min_framerate;
421  double max_framerate;
422 
423  [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];
424  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
425  av_log(s, AV_LOG_ERROR, " %dx%d@[%f %f]fps\n",
426  dimensions.width, dimensions.height,
427  min_framerate, max_framerate);
428  }
429  }
430  return AVERROR(EINVAL);
431 }
432 
433 static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
434 {
435  AVFContext *ctx = (AVFContext*)s->priv_data;
436  int ret;
437  NSError *error = nil;
438  AVCaptureInput* capture_input = nil;
439  struct AVFPixelFormatSpec pxl_fmt_spec;
440  NSNumber *pixel_format;
441  NSDictionary *capture_dict;
442  dispatch_queue_t queue;
443 
444  if (ctx->video_device_index < ctx->num_video_devices) {
445  capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
446  } else {
447  capture_input = (AVCaptureInput*) video_device;
448  }
449 
450  if (!capture_input) {
451  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
452  [[error localizedDescription] UTF8String]);
453  return 1;
454  }
455 
456  if ([ctx->capture_session canAddInput:capture_input]) {
457  [ctx->capture_session addInput:capture_input];
458  } else {
459  av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");
460  return 1;
461  }
462 
463  // Attaching output
464  ctx->video_output = [[AVCaptureVideoDataOutput alloc] init];
465 
466  if (!ctx->video_output) {
467  av_log(s, AV_LOG_ERROR, "Failed to init AV video output\n");
468  return 1;
469  }
470 
471  // Configure device framerate and video size
472  @try {
473  if ((ret = configure_video_device(s, video_device)) < 0) {
474  return ret;
475  }
476  } @catch (NSException *exception) {
477  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
478  av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);
479  return AVERROR_EXTERNAL;
480  }
481  }
482 
483  // select pixel format
484  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
485 
486  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
487  if (ctx->pixel_format == avf_pixel_formats[i].ff_id) {
488  pxl_fmt_spec = avf_pixel_formats[i];
489  break;
490  }
491  }
492 
493  // check if selected pixel format is supported by AVFoundation
494  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
495  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by AVFoundation.\n",
496  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
497  return 1;
498  }
499 
500  // check if the pixel format is available for this device
501  if ([[ctx->video_output availableVideoCVPixelFormatTypes] indexOfObject:[NSNumber numberWithInt:pxl_fmt_spec.avf_id]] == NSNotFound) {
502  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by the input device.\n",
503  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
504 
505  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
506 
507  av_log(s, AV_LOG_ERROR, "Supported pixel formats:\n");
508  for (NSNumber *pxl_fmt in [ctx->video_output availableVideoCVPixelFormatTypes]) {
509  struct AVFPixelFormatSpec pxl_fmt_dummy;
510  pxl_fmt_dummy.ff_id = AV_PIX_FMT_NONE;
511  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
512  if ([pxl_fmt intValue] == avf_pixel_formats[i].avf_id) {
513  pxl_fmt_dummy = avf_pixel_formats[i];
514  break;
515  }
516  }
517 
518  if (pxl_fmt_dummy.ff_id != AV_PIX_FMT_NONE) {
519  av_log(s, AV_LOG_ERROR, " %s\n", av_get_pix_fmt_name(pxl_fmt_dummy.ff_id));
520 
521  // select first supported pixel format instead of user selected (or default) pixel format
522  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
523  pxl_fmt_spec = pxl_fmt_dummy;
524  }
525  }
526  }
527 
528  // fail if there is no appropriate pixel format or print a warning about overriding the pixel format
529  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
530  return 1;
531  } else {
532  av_log(s, AV_LOG_WARNING, "Overriding selected pixel format to use %s instead.\n",
533  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
534  }
535  }
536 
537  // set videoSettings to an empty dict for receiving raw data of muxed devices
538  if (ctx->capture_raw_data) {
539  ctx->pixel_format = pxl_fmt_spec.ff_id;
540  ctx->video_output.videoSettings = @{ };
541  } else {
542  ctx->pixel_format = pxl_fmt_spec.ff_id;
543  pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
544  capture_dict = [NSDictionary dictionaryWithObject:pixel_format
545  forKey:(id)kCVPixelBufferPixelFormatTypeKey];
546 
547  [ctx->video_output setVideoSettings:capture_dict];
548  }
549  [ctx->video_output setAlwaysDiscardsLateVideoFrames:ctx->drop_late_frames];
550 
551 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
552  // check for transport control support and set observer device if supported
553  if (!ctx->video_is_screen) {
554  int trans_ctrl = [video_device transportControlsSupported];
555  AVCaptureDeviceTransportControlsPlaybackMode trans_mode = [video_device transportControlsPlaybackMode];
556 
557  if (trans_ctrl) {
558  ctx->observed_mode = trans_mode;
559  ctx->observed_device = video_device;
560  }
561  }
562 #endif
563 
564  ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];
565 
566  queue = dispatch_queue_create("avf_queue", NULL);
567  [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];
568  dispatch_release(queue);
569 
570  if ([ctx->capture_session canAddOutput:ctx->video_output]) {
571  [ctx->capture_session addOutput:ctx->video_output];
572  } else {
573  av_log(s, AV_LOG_ERROR, "can't add video output to capture session\n");
574  return 1;
575  }
576 
577  return 0;
578 }
579 
580 static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
581 {
582  AVFContext *ctx = (AVFContext*)s->priv_data;
583  NSError *error = nil;
584  AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];
585  dispatch_queue_t queue;
586 
587  if (!audio_dev_input) {
588  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
589  [[error localizedDescription] UTF8String]);
590  return 1;
591  }
592 
593  if ([ctx->capture_session canAddInput:audio_dev_input]) {
594  [ctx->capture_session addInput:audio_dev_input];
595  } else {
596  av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");
597  return 1;
598  }
599 
600  // Attaching output
601  ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];
602 
603  if (!ctx->audio_output) {
604  av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");
605  return 1;
606  }
607 
608  ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];
609 
610  queue = dispatch_queue_create("avf_audio_queue", NULL);
611  [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];
612  dispatch_release(queue);
613 
614  if ([ctx->capture_session canAddOutput:ctx->audio_output]) {
615  [ctx->capture_session addOutput:ctx->audio_output];
616  } else {
617  av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");
618  return 1;
619  }
620 
621  return 0;
622 }
623 
625 {
626  AVFContext *ctx = (AVFContext*)s->priv_data;
627  CVImageBufferRef image_buffer;
628  CMBlockBufferRef block_buffer;
629  CGSize image_buffer_size;
630  AVStream* stream = avformat_new_stream(s, NULL);
631 
632  if (!stream) {
633  return 1;
634  }
635 
636  // Take stream info from the first frame.
637  while (ctx->frames_captured < 1) {
638  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
639  }
640 
641  lock_frames(ctx);
642 
643  ctx->video_stream_index = stream->index;
644 
645  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
646 
647  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
648  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
649 
650  if (image_buffer) {
651  image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
652 
653  stream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
654  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
655  stream->codecpar->width = (int)image_buffer_size.width;
656  stream->codecpar->height = (int)image_buffer_size.height;
657  stream->codecpar->format = ctx->pixel_format;
658  } else {
659  stream->codecpar->codec_id = AV_CODEC_ID_DVVIDEO;
660  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
661  stream->codecpar->format = ctx->pixel_format;
662  }
663 
664  CFRelease(ctx->current_frame);
665  ctx->current_frame = nil;
666 
668 
669  return 0;
670 }
671 
673 {
674  AVFContext *ctx = (AVFContext*)s->priv_data;
675  CMFormatDescriptionRef format_desc;
676  AVStream* stream = avformat_new_stream(s, NULL);
677 
678  if (!stream) {
679  return 1;
680  }
681 
682  // Take stream info from the first frame.
683  while (ctx->audio_frames_captured < 1) {
684  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
685  }
686 
687  lock_frames(ctx);
688 
689  ctx->audio_stream_index = stream->index;
690 
691  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
692 
693  format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
694  const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
695 
696  if (!basic_desc) {
697  av_log(s, AV_LOG_ERROR, "audio format not available\n");
698  return 1;
699  }
700 
701  stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
702  stream->codecpar->sample_rate = basic_desc->mSampleRate;
703  stream->codecpar->channels = basic_desc->mChannelsPerFrame;
704  stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
705 
706  ctx->audio_channels = basic_desc->mChannelsPerFrame;
707  ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
708  ctx->audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
709  ctx->audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
710  ctx->audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
711  ctx->audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
712  ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;
713 
714  if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
715  ctx->audio_float &&
716  ctx->audio_bits_per_sample == 32 &&
717  ctx->audio_packed) {
718  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
719  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
720  ctx->audio_signed_integer &&
721  ctx->audio_bits_per_sample == 16 &&
722  ctx->audio_packed) {
723  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
724  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
725  ctx->audio_signed_integer &&
726  ctx->audio_bits_per_sample == 24 &&
727  ctx->audio_packed) {
728  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
729  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
730  ctx->audio_signed_integer &&
731  ctx->audio_bits_per_sample == 32 &&
732  ctx->audio_packed) {
733  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
734  } else {
735  av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
736  return 1;
737  }
738 
739  if (ctx->audio_non_interleaved) {
740  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
741  ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer);
742  ctx->audio_buffer = av_malloc(ctx->audio_buffer_size);
743  if (!ctx->audio_buffer) {
744  av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
745  return 1;
746  }
747  }
748 
749  CFRelease(ctx->current_audio_frame);
750  ctx->current_audio_frame = nil;
751 
753 
754  return 0;
755 }
756 
758 {
759  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
760  uint32_t num_screens = 0;
761  AVFContext *ctx = (AVFContext*)s->priv_data;
762  AVCaptureDevice *video_device = nil;
763  AVCaptureDevice *audio_device = nil;
764  // Find capture device
765  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
766  NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
767 
768  ctx->num_video_devices = [devices count] + [devices_muxed count];
769  ctx->first_pts = av_gettime();
770  ctx->first_audio_pts = av_gettime();
771 
772  pthread_mutex_init(&ctx->frame_lock, NULL);
773 
774 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
775  CGGetActiveDisplayList(0, NULL, &num_screens);
776 #endif
777 
778  // List devices if requested
779  if (ctx->list_devices) {
780  int index = 0;
781  av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
782  for (AVCaptureDevice *device in devices) {
783  const char *name = [[device localizedName] UTF8String];
784  index = [devices indexOfObject:device];
785  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
786  }
787  for (AVCaptureDevice *device in devices_muxed) {
788  const char *name = [[device localizedName] UTF8String];
789  index = [devices count] + [devices_muxed indexOfObject:device];
790  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
791  }
792 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
793  if (num_screens > 0) {
794  CGDirectDisplayID screens[num_screens];
795  CGGetActiveDisplayList(num_screens, screens, &num_screens);
796  for (int i = 0; i < num_screens; i++) {
797  av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i);
798  }
799  }
800 #endif
801 
802  av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
803  devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
804  for (AVCaptureDevice *device in devices) {
805  const char *name = [[device localizedName] UTF8String];
806  int index = [devices indexOfObject:device];
807  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
808  }
809  goto fail;
810  }
811 
812  // parse input filename for video and audio device
814 
815  // check for device index given in filename
816  if (ctx->video_device_index == -1 && ctx->video_filename) {
817  sscanf(ctx->video_filename, "%d", &ctx->video_device_index);
818  }
819  if (ctx->audio_device_index == -1 && ctx->audio_filename) {
820  sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);
821  }
822 
823  if (ctx->video_device_index >= 0) {
824  if (ctx->video_device_index < ctx->num_video_devices) {
825  if (ctx->video_device_index < [devices count]) {
826  video_device = [devices objectAtIndex:ctx->video_device_index];
827  } else {
828  video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])];
829  ctx->video_is_muxed = 1;
830  }
831  } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
832 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
833  CGDirectDisplayID screens[num_screens];
834  CGGetActiveDisplayList(num_screens, screens, &num_screens);
835  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];
836 
837  if (ctx->framerate.num > 0) {
838  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
839  }
840 
841 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
842  if (ctx->capture_cursor) {
843  capture_screen_input.capturesCursor = YES;
844  } else {
845  capture_screen_input.capturesCursor = NO;
846  }
847 #endif
848 
849  if (ctx->capture_mouse_clicks) {
850  capture_screen_input.capturesMouseClicks = YES;
851  } else {
852  capture_screen_input.capturesMouseClicks = NO;
853  }
854 
855  video_device = (AVCaptureDevice*) capture_screen_input;
856  ctx->video_is_screen = 1;
857 #endif
858  } else {
859  av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");
860  goto fail;
861  }
862  } else if (ctx->video_filename &&
863  strncmp(ctx->video_filename, "none", 4)) {
864  if (!strncmp(ctx->video_filename, "default", 7)) {
865  video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
866  } else {
867  // looking for video inputs
868  for (AVCaptureDevice *device in devices) {
869  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
870  video_device = device;
871  break;
872  }
873  }
874  // looking for muxed inputs
875  for (AVCaptureDevice *device in devices_muxed) {
876  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
877  video_device = device;
878  ctx->video_is_muxed = 1;
879  break;
880  }
881  }
882 
883 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
884  // looking for screen inputs
885  if (!video_device) {
886  int idx;
887  if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
888  CGDirectDisplayID screens[num_screens];
889  CGGetActiveDisplayList(num_screens, screens, &num_screens);
890  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
891  video_device = (AVCaptureDevice*) capture_screen_input;
892  ctx->video_device_index = ctx->num_video_devices + idx;
893  ctx->video_is_screen = 1;
894 
895  if (ctx->framerate.num > 0) {
896  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
897  }
898 
899 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
900  if (ctx->capture_cursor) {
901  capture_screen_input.capturesCursor = YES;
902  } else {
903  capture_screen_input.capturesCursor = NO;
904  }
905 #endif
906 
907  if (ctx->capture_mouse_clicks) {
908  capture_screen_input.capturesMouseClicks = YES;
909  } else {
910  capture_screen_input.capturesMouseClicks = NO;
911  }
912  }
913  }
914 #endif
915  }
916 
917  if (!video_device) {
918  av_log(ctx, AV_LOG_ERROR, "Video device not found\n");
919  goto fail;
920  }
921  }
922 
923  // get audio device
924  if (ctx->audio_device_index >= 0) {
925  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
926 
927  if (ctx->audio_device_index >= [devices count]) {
928  av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
929  goto fail;
930  }
931 
932  audio_device = [devices objectAtIndex:ctx->audio_device_index];
933  } else if (ctx->audio_filename &&
934  strncmp(ctx->audio_filename, "none", 4)) {
935  if (!strncmp(ctx->audio_filename, "default", 7)) {
936  audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
937  } else {
938  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
939 
940  for (AVCaptureDevice *device in devices) {
941  if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
942  audio_device = device;
943  break;
944  }
945  }
946  }
947 
948  if (!audio_device) {
949  av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
950  goto fail;
951  }
952  }
953 
954  // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio
955  if (!video_device && !audio_device) {
956  av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
957  goto fail;
958  }
959 
960  if (video_device) {
961  if (ctx->video_device_index < ctx->num_video_devices) {
962  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);
963  } else {
964  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);
965  }
966  }
967  if (audio_device) {
968  av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);
969  }
970 
971  // Initialize capture session
972  ctx->capture_session = [[AVCaptureSession alloc] init];
973 
974  if (video_device && add_video_device(s, video_device)) {
975  goto fail;
976  }
977  if (audio_device && add_audio_device(s, audio_device)) {
978  }
979 
980  [ctx->capture_session startRunning];
981 
982  /* Unlock device configuration only after the session is started so it
983  * does not reset the capture formats */
984  if (!ctx->video_is_screen) {
985  [video_device unlockForConfiguration];
986  }
987 
988  if (video_device && get_video_config(s)) {
989  goto fail;
990  }
991 
992  // set audio stream
993  if (audio_device && get_audio_config(s)) {
994  goto fail;
995  }
996 
997  [pool release];
998  return 0;
999 
1000 fail:
1001  [pool release];
1003  return AVERROR(EIO);
1004 }
1005 
1007  CVPixelBufferRef image_buffer,
1008  AVPacket *pkt)
1009 {
1010  AVFContext *ctx = s->priv_data;
1011  int src_linesize[4];
1012  const uint8_t *src_data[4];
1013  int width = CVPixelBufferGetWidth(image_buffer);
1014  int height = CVPixelBufferGetHeight(image_buffer);
1015  int status;
1016 
1017  memset(src_linesize, 0, sizeof(src_linesize));
1018  memset(src_data, 0, sizeof(src_data));
1019 
1020  status = CVPixelBufferLockBaseAddress(image_buffer, 0);
1021  if (status != kCVReturnSuccess) {
1022  av_log(s, AV_LOG_ERROR, "Could not lock base address: %d (%dx%d)\n", status, width, height);
1023  return AVERROR_EXTERNAL;
1024  }
1025 
1026  if (CVPixelBufferIsPlanar(image_buffer)) {
1027  size_t plane_count = CVPixelBufferGetPlaneCount(image_buffer);
1028  int i;
1029  for(i = 0; i < plane_count; i++){
1030  src_linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(image_buffer, i);
1031  src_data[i] = CVPixelBufferGetBaseAddressOfPlane(image_buffer, i);
1032  }
1033  } else {
1034  src_linesize[0] = CVPixelBufferGetBytesPerRow(image_buffer);
1035  src_data[0] = CVPixelBufferGetBaseAddress(image_buffer);
1036  }
1037 
1038  status = av_image_copy_to_buffer(pkt->data, pkt->size,
1039  src_data, src_linesize,
1040  ctx->pixel_format, width, height, 1);
1041 
1042 
1043 
1044  CVPixelBufferUnlockBaseAddress(image_buffer, 0);
1045 
1046  return status;
1047 }
1048 
1050 {
1051  AVFContext* ctx = (AVFContext*)s->priv_data;
1052 
1053  do {
1054  CVImageBufferRef image_buffer;
1055  CMBlockBufferRef block_buffer;
1056  lock_frames(ctx);
1057 
1058  if (ctx->current_frame != nil) {
1059  int status;
1060  int length = 0;
1061 
1062  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
1063  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
1064 
1065  if (image_buffer != nil) {
1066  length = (int)CVPixelBufferGetDataSize(image_buffer);
1067  } else if (block_buffer != nil) {
1068  length = (int)CMBlockBufferGetDataLength(block_buffer);
1069  } else {
1070  return AVERROR(EINVAL);
1071  }
1072 
1073  if (av_new_packet(pkt, length) < 0) {
1074  return AVERROR(EIO);
1075  }
1076 
1077  CMItemCount count;
1078  CMSampleTimingInfo timing_info;
1079 
1080  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
1081  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1082  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1083  }
1084 
1085  pkt->stream_index = ctx->video_stream_index;
1087 
1088  if (image_buffer) {
1089  status = copy_cvpixelbuffer(s, image_buffer, pkt);
1090  } else {
1091  status = 0;
1092  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1093  if (ret != kCMBlockBufferNoErr) {
1094  status = AVERROR(EIO);
1095  }
1096  }
1097  CFRelease(ctx->current_frame);
1098  ctx->current_frame = nil;
1099 
1100  if (status < 0)
1101  return status;
1102  } else if (ctx->current_audio_frame != nil) {
1103  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
1104  int block_buffer_size = CMBlockBufferGetDataLength(block_buffer);
1105 
1106  if (!block_buffer || !block_buffer_size) {
1107  return AVERROR(EIO);
1108  }
1109 
1110  if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
1111  return AVERROR_BUFFER_TOO_SMALL;
1112  }
1113 
1114  if (av_new_packet(pkt, block_buffer_size) < 0) {
1115  return AVERROR(EIO);
1116  }
1117 
1118  CMItemCount count;
1119  CMSampleTimingInfo timing_info;
1120 
1121  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
1122  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1123  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1124  }
1125 
1126  pkt->stream_index = ctx->audio_stream_index;
1128 
1129  if (ctx->audio_non_interleaved) {
1130  int sample, c, shift, num_samples;
1131 
1132  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
1133  if (ret != kCMBlockBufferNoErr) {
1134  return AVERROR(EIO);
1135  }
1136 
1137  num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
1138 
1139  // transform decoded frame into output format
1140  #define INTERLEAVE_OUTPUT(bps) \
1141  { \
1142  int##bps##_t **src; \
1143  int##bps##_t *dest; \
1144  src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*)); \
1145  if (!src) return AVERROR(EIO); \
1146  for (c = 0; c < ctx->audio_channels; c++) { \
1147  src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
1148  } \
1149  dest = (int##bps##_t*)pkt->data; \
1150  shift = bps - ctx->audio_bits_per_sample; \
1151  for (sample = 0; sample < num_samples; sample++) \
1152  for (c = 0; c < ctx->audio_channels; c++) \
1153  *dest++ = src[c][sample] << shift; \
1154  av_freep(&src); \
1155  }
1156 
1157  if (ctx->audio_bits_per_sample <= 16) {
1158  INTERLEAVE_OUTPUT(16)
1159  } else {
1160  INTERLEAVE_OUTPUT(32)
1161  }
1162  } else {
1163  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1164  if (ret != kCMBlockBufferNoErr) {
1165  return AVERROR(EIO);
1166  }
1167  }
1168 
1169  CFRelease(ctx->current_audio_frame);
1170  ctx->current_audio_frame = nil;
1171  } else {
1172  pkt->data = NULL;
1173  unlock_frames(ctx);
1174  if (ctx->observed_quit) {
1175  return AVERROR_EOF;
1176  } else {
1177  return AVERROR(EAGAIN);
1178  }
1179  }
1180 
1181  unlock_frames(ctx);
1182  } while (!pkt->data);
1183 
1184  return 0;
1185 }
1186 
1188 {
1189  AVFContext* ctx = (AVFContext*)s->priv_data;
1191  return 0;
1192 }
1193 
1194 static const AVOption options[] = {
1195  { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1196  { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1197  { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1198  { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},
1199  { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1200  { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },
1201  { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1202  { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1203  { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1204  { "drop_late_frames", "drop frames that are available later than expected", offsetof(AVFContext, drop_late_frames), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1205 
1206  { NULL },
1207 };
1208 
1209 static const AVClass avf_class = {
1210  .class_name = "AVFoundation indev",
1211  .item_name = av_default_item_name,
1212  .option = options,
1213  .version = LIBAVUTIL_VERSION_INT,
1215 };
1216 
1218  .name = "avfoundation",
1219  .long_name = NULL_IF_CONFIG_SMALL("AVFoundation input device"),
1220  .priv_data_size = sizeof(AVFContext),
1223  .read_close = avf_close,
1224  .flags = AVFMT_NOFILE,
1225  .priv_class = &avf_class,
1226 };
static const char *const format[]
Definition: af_aiir.c:456
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31)))) #define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac) { } void ff_audio_convert_free(AudioConvert **ac) { if(! *ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);} AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map) { AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method !=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2) { ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc) { av_free(ac);return NULL;} return ac;} in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar) { ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar ? ac->channels :1;} else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;} int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in) { int use_generic=1;int len=in->nb_samples;int p;if(ac->dc) { av_log(ac->avr, AV_LOG_TRACE, "%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
uint8_t
int32_t
Main libavdevice API header.
#define AVFMT_NOFILE
Demuxer will use avio_open, no opened file should be provided by the caller.
Definition: avformat.h:458
static int get_audio_config(AVFormatContext *s)
Definition: avfoundation.m:672
static void lock_frames(AVFContext *ctx)
Definition: avfoundation.m:141
static const AVClass avf_class
static int avf_close(AVFormatContext *s)
static const AVOption options[]
static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Definition: avfoundation.m:433
static void unlock_frames(AVFContext *ctx)
Definition: avfoundation.m:146
static const AVRational avf_time_base_q
Definition: avfoundation.m:43
static void parse_device_name(AVFormatContext *s)
Definition: avfoundation.m:312
static const struct AVFPixelFormatSpec avf_pixel_formats[]
Definition: avfoundation.m:53
static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
Definition: avfoundation.m:580
static void destroy_context(AVFContext *ctx)
Definition: avfoundation.m:287
static int avf_read_header(AVFormatContext *s)
Definition: avfoundation.m:757
static int copy_cvpixelbuffer(AVFormatContext *s, CVPixelBufferRef image_buffer, AVPacket *pkt)
static const int avf_time_base
Definition: avfoundation.m:41
AVInputFormat ff_avfoundation_demuxer
static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Configure the video device.
Definition: avfoundation.m:336
#define INTERLEAVE_OUTPUT(bps)
static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
static int get_video_config(AVFormatContext *s)
Definition: avfoundation.m:624
static int read_packet(void *opaque, uint8_t *buf, int buf_size)
Definition: avio_reading.c:42
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:31
#define flags(name, subs,...)
Definition: cbs_av1.c:561
static int FUNC() timing_info(CodedBitstreamContext *ctx, RWContext *rw, AV1RawTimingInfo *current)
#define s(width, name)
Definition: cbs_vp9.c:257
#define fail()
Definition: checkasm.h:133
#define NULL
Definition: coverity.c:32
static __device__ float fabs(float a)
Definition: cuda_runtime.h:182
mode
Use these values in ebur128_init (or'ed).
Definition: ebur128.h:83
enum AVCodecID id
int
#define pthread_mutex_lock(a)
Definition: ffprobe.c:63
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:67
static int read_header(FFV1Context *f)
Definition: ffv1dec.c:527
#define sample
@ AV_OPT_TYPE_IMAGE_SIZE
offset must point to two consecutive integers
Definition: opt.h:235
@ AV_OPT_TYPE_PIXEL_FMT
Definition: opt.h:236
@ AV_OPT_TYPE_VIDEO_RATE
offset must point to AVRational
Definition: opt.h:238
@ AV_OPT_TYPE_INT
Definition: opt.h:225
@ AV_OPT_TYPE_BOOL
Definition: opt.h:242
int64_t av_get_default_channel_layout(int nb_channels)
Return default channel layout for a given number of channels.
@ AV_CODEC_ID_PCM_F32LE
Definition: codec_id.h:334
@ AV_CODEC_ID_PCM_S24BE
Definition: codec_id.h:326
@ AV_CODEC_ID_RAWVIDEO
Definition: codec_id.h:62
@ AV_CODEC_ID_PCM_S16LE
Definition: codec_id.h:313
@ AV_CODEC_ID_PCM_F32BE
Definition: codec_id.h:333
@ AV_CODEC_ID_PCM_S16BE
Definition: codec_id.h:314
@ AV_CODEC_ID_PCM_S24LE
Definition: codec_id.h:325
@ AV_CODEC_ID_PCM_S32LE
Definition: codec_id.h:321
@ AV_CODEC_ID_DVVIDEO
Definition: codec_id.h:73
@ AV_CODEC_ID_PCM_S32BE
Definition: codec_id.h:322
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: packet.h:410
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: avpacket.c:99
AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)
Add a new stream to a media file.
Definition: utils.c:4505
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
#define AVERROR_EOF
End of file.
Definition: error.h:55
#define AVERROR(e)
Definition: error.h:43
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:215
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
#define AV_LOG_INFO
Standard information.
Definition: log.h:205
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
static AVRational av_make_q(int num, int den)
Create an AVRational.
Definition: rational.h:71
static double av_q2d(AVRational a)
Convert an AVRational to a double.
Definition: rational.h:104
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:253
@ AVMEDIA_TYPE_AUDIO
Definition: avutil.h:202
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)
Copy image data from an image into a buffer.
Definition: imgutils.c:505
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:186
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
int index
Definition: gxfenc.c:89
misc image utilities
int i
Definition: input.c:407
AudioReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:246
AVFContext * _context
Definition: avfoundation.m:247
FrameReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:154
AVFContext * _context
Definition: avfoundation.m:155
void avpriv_set_pts_info(AVStream *s, int pts_wrap_bits, unsigned int pts_num, unsigned int pts_den)
Set the time base and wrapping info for a given stream.
Definition: utils.c:4941
common internal API header
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:117
static av_cold int read_close(AVFormatContext *ctx)
Definition: libcdio.c:145
@ AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT
Definition: log.h:42
AVOptions.
#define AV_OPT_FLAG_DECODING_PARAM
a generic parameter which can be set by the user for demuxing or decoding
Definition: opt.h:279
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
_fmutex pthread_mutex_t
Definition: os2threads.h:53
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
misc parsing utilities
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:400
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
@ AV_PIX_FMT_MONOBLACK
Y , 1bpp, 0 is black, 1 is white, in each byte pixels are ordered from the msb to the lsb.
Definition: pixfmt.h:76
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:240
@ AV_PIX_FMT_RGB555BE
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), big-endian , X=unused/undefined
Definition: pixfmt.h:107
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:74
@ AV_PIX_FMT_BGR48BE
packed RGB 16:16:16, 48bpp, 16B, 16G, 16R, the 2-byte value for each R/G/B component is stored as big...
Definition: pixfmt.h:148
@ AV_PIX_FMT_UYVY422
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:81
@ AV_PIX_FMT_0BGR
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
Definition: pixfmt.h:239
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:71
@ AV_PIX_FMT_YUVA444P
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:177
@ AV_PIX_FMT_YUVA444P16LE
planar YUV 4:4:4 64bpp, (1 Cr & Cb sample per 1x1 Y & A samples, little-endian)
Definition: pixfmt.h:195
@ AV_PIX_FMT_RGB565LE
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), little-endian
Definition: pixfmt.h:106
@ AV_PIX_FMT_RGB555LE
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), little-endian, X=unused/undefined
Definition: pixfmt.h:108
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:238
@ AV_PIX_FMT_RGB565BE
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), big-endian
Definition: pixfmt.h:105
@ AV_PIX_FMT_YUYV422
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:67
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
@ AV_PIX_FMT_0RGB
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
Definition: pixfmt.h:237
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:411
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:402
const char * name
Definition: qsvenc.c:46
typedef void(RENAME(mix_any_func_type))
static int shift(int a, int b)
Definition: sonic.c:82
Describe the class of an AVClass context structure.
Definition: log.h:67
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
int drop_late_frames
Definition: avfoundation.m:100
CMSampleBufferRef current_frame
Definition: avfoundation.m:131
int32_t * audio_buffer
Definition: avfoundation.m:123
int64_t first_audio_pts
Definition: avfoundation.m:89
int video_is_muxed
Definition: avfoundation.m:101
int capture_raw_data
Definition: avfoundation.m:99
int audio_packed
Definition: avfoundation.m:120
int64_t first_pts
Definition: avfoundation.m:88
int audio_stream_index
Definition: avfoundation.m:108
AVCaptureAudioDataOutput * audio_output
Definition: avfoundation.m:130
int video_stream_index
Definition: avfoundation.m:106
char * video_filename
Definition: avfoundation.m:110
int num_video_devices
Definition: avfoundation.m:113
int capture_mouse_clicks
Definition: avfoundation.m:98
int audio_bits_per_sample
Definition: avfoundation.m:116
pthread_mutex_t frame_lock
Definition: avfoundation.m:90
int audio_device_index
Definition: avfoundation.m:107
int video_is_screen
Definition: avfoundation.m:102
int audio_non_interleaved
Definition: avfoundation.m:121
char * audio_filename
Definition: avfoundation.m:111
AVCaptureVideoDataOutput * video_output
Definition: avfoundation.m:129
int audio_signed_integer
Definition: avfoundation.m:119
int audio_frames_captured
Definition: avfoundation.m:87
AVRational framerate
Definition: avfoundation.m:94
int observed_quit
Definition: avfoundation.m:138
int audio_buffer_size
Definition: avfoundation.m:124
AVCaptureSession * capture_session
Definition: avfoundation.m:128
int frames_captured
Definition: avfoundation.m:86
int video_device_index
Definition: avfoundation.m:105
CMSampleBufferRef current_audio_frame
Definition: avfoundation.m:132
int capture_cursor
Definition: avfoundation.m:97
int list_devices
Definition: avfoundation.m:104
AVCaptureDevice * observed_device
Definition: avfoundation.m:134
id avf_audio_delegate
Definition: avfoundation.m:92
int audio_channels
Definition: avfoundation.m:115
enum AVPixelFormat ff_id
Definition: avfoundation.m:49
Format I/O context.
Definition: avformat.h:1232
const char * name
A comma separated list of short names for the format.
Definition: avformat.h:645
AVOption.
Definition: opt.h:248
This structure stores compressed data.
Definition: packet.h:346
int stream_index
Definition: packet.h:371
int flags
A combination of AV_PKT_FLAG values.
Definition: packet.h:375
int size
Definition: packet.h:370
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: packet.h:362
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed.
Definition: packet.h:368
uint8_t * data
Definition: packet.h:369
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int num
Numerator.
Definition: rational.h:59
Stream structure.
Definition: avformat.h:873
#define av_freep(p)
#define av_malloc(s)
#define av_log(a,...)
static void error(const char *err)
static uint8_t tmp[11]
Definition: aes_ctr.c:27
int framerate
Definition: h264_levels.c:65
AVPacket * pkt
Definition: movenc.c:59
AVFormatContext * ctx
Definition: movenc.c:48
#define height
#define width
int64_t av_gettime(void)
Get the current time in microseconds.
Definition: time.c:39
if(ret< 0)
Definition: vf_mcdeint.c:282
static double c[64]