FFmpeg  4.4
videotoolboxenc.c
Go to the documentation of this file.
1 /*
2  * copyright (c) 2015 Rick Kern <kernrj@gmail.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <VideoToolbox/VideoToolbox.h>
22 #include <CoreVideo/CoreVideo.h>
23 #include <CoreMedia/CoreMedia.h>
24 #include <TargetConditionals.h>
25 #include <Availability.h>
26 #include "avcodec.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/avassert.h"
29 #include "libavutil/avstring.h"
30 #include "libavcodec/avcodec.h"
31 #include "libavutil/pixdesc.h"
32 #include "internal.h"
33 #include <pthread.h>
34 #include "atsc_a53.h"
35 #include "h264.h"
36 #include "h264_sei.h"
37 #include <dlfcn.h>
38 
39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
41 #endif
42 
43 #if !HAVE_KCMVIDEOCODECTYPE_HEVCWITHALPHA
45 #endif
46 
47 #if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
50 #endif
51 
52 typedef OSStatus (*getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc,
53  size_t parameterSetIndex,
54  const uint8_t **parameterSetPointerOut,
55  size_t *parameterSetSizeOut,
56  size_t *parameterSetCountOut,
57  int *NALUnitHeaderLengthOut);
58 
59 //These symbols may not be present
60 static struct{
64 
68 
90 
93 
96 
99 
102 
103 #define GET_SYM(symbol, defaultVal) \
104 do{ \
105  CFStringRef* handle = (CFStringRef*)dlsym(RTLD_DEFAULT, #symbol); \
106  if(!handle) \
107  compat_keys.symbol = CFSTR(defaultVal); \
108  else \
109  compat_keys.symbol = *handle; \
110 }while(0)
111 
113 
114 static void loadVTEncSymbols(){
115  compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex =
116  (getParameterSetAtIndex)dlsym(
117  RTLD_DEFAULT,
118  "CMVideoFormatDescriptionGetHEVCParameterSetAtIndex"
119  );
120 
124 
128 
129  GET_SYM(kVTProfileLevel_H264_Baseline_4_0, "H264_Baseline_4_0");
130  GET_SYM(kVTProfileLevel_H264_Baseline_4_2, "H264_Baseline_4_2");
131  GET_SYM(kVTProfileLevel_H264_Baseline_5_0, "H264_Baseline_5_0");
132  GET_SYM(kVTProfileLevel_H264_Baseline_5_1, "H264_Baseline_5_1");
133  GET_SYM(kVTProfileLevel_H264_Baseline_5_2, "H264_Baseline_5_2");
134  GET_SYM(kVTProfileLevel_H264_Baseline_AutoLevel, "H264_Baseline_AutoLevel");
135  GET_SYM(kVTProfileLevel_H264_Main_4_2, "H264_Main_4_2");
136  GET_SYM(kVTProfileLevel_H264_Main_5_1, "H264_Main_5_1");
137  GET_SYM(kVTProfileLevel_H264_Main_5_2, "H264_Main_5_2");
138  GET_SYM(kVTProfileLevel_H264_Main_AutoLevel, "H264_Main_AutoLevel");
139  GET_SYM(kVTProfileLevel_H264_High_3_0, "H264_High_3_0");
140  GET_SYM(kVTProfileLevel_H264_High_3_1, "H264_High_3_1");
141  GET_SYM(kVTProfileLevel_H264_High_3_2, "H264_High_3_2");
142  GET_SYM(kVTProfileLevel_H264_High_4_0, "H264_High_4_0");
143  GET_SYM(kVTProfileLevel_H264_High_4_1, "H264_High_4_1");
144  GET_SYM(kVTProfileLevel_H264_High_4_2, "H264_High_4_2");
145  GET_SYM(kVTProfileLevel_H264_High_5_1, "H264_High_5_1");
146  GET_SYM(kVTProfileLevel_H264_High_5_2, "H264_High_5_2");
147  GET_SYM(kVTProfileLevel_H264_High_AutoLevel, "H264_High_AutoLevel");
148  GET_SYM(kVTProfileLevel_H264_Extended_5_0, "H264_Extended_5_0");
149  GET_SYM(kVTProfileLevel_H264_Extended_AutoLevel, "H264_Extended_AutoLevel");
150 
151  GET_SYM(kVTProfileLevel_HEVC_Main_AutoLevel, "HEVC_Main_AutoLevel");
152  GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel, "HEVC_Main10_AutoLevel");
153 
156  "TargetQualityForAlpha");
157 
159  "EnableHardwareAcceleratedVideoEncoder");
161  "RequireHardwareAcceleratedVideoEncoder");
162 }
163 
164 typedef enum VT_H264Profile {
172 
173 typedef enum VTH264Entropy{
176  VT_CABAC
178 
179 typedef enum VT_HEVCProfile {
185 
186 static const uint8_t start_code[] = { 0, 0, 0, 1 };
187 
188 typedef struct ExtraSEI {
189  void *data;
190  size_t size;
191 } ExtraSEI;
192 
193 typedef struct BufNode {
194  CMSampleBufferRef cm_buffer;
196  struct BufNode* next;
197  int error;
198 } BufNode;
199 
200 typedef struct VTEncContext {
201  AVClass *class;
202  enum AVCodecID codec_id;
203  VTCompressionSessionRef session;
204  CFStringRef ycbcr_matrix;
205  CFStringRef color_primaries;
206  CFStringRef transfer_function;
208 
211 
213 
216 
217  int64_t frame_ct_out;
218  int64_t frame_ct_in;
219 
220  int64_t first_pts;
221  int64_t dts_delta;
222 
223  int64_t profile;
224  int64_t level;
225  int64_t entropy;
226  int64_t realtime;
227  int64_t frames_before;
228  int64_t frames_after;
229 
230  int64_t allow_sw;
231  int64_t require_sw;
233 
234  bool flushing;
237 
238  /* can't be bool type since AVOption will access it as int */
239  int a53_cc;
240 } VTEncContext;
241 
242 static int vtenc_populate_extradata(AVCodecContext *avctx,
243  CMVideoCodecType codec_type,
244  CFStringRef profile_level,
245  CFNumberRef gamma_level,
246  CFDictionaryRef enc_info,
247  CFDictionaryRef pixel_buffer_info);
248 
249 /**
250  * NULL-safe release of *refPtr, and sets value to NULL.
251  */
252 static void vt_release_num(CFNumberRef* refPtr){
253  if (!*refPtr) {
254  return;
255  }
256 
257  CFRelease(*refPtr);
258  *refPtr = NULL;
259 }
260 
261 static void set_async_error(VTEncContext *vtctx, int err)
262 {
263  BufNode *info;
264 
265  pthread_mutex_lock(&vtctx->lock);
266 
267  vtctx->async_error = err;
268 
269  info = vtctx->q_head;
270  vtctx->q_head = vtctx->q_tail = NULL;
271 
272  while (info) {
273  BufNode *next = info->next;
274  CFRelease(info->cm_buffer);
275  av_free(info);
276  info = next;
277  }
278 
279  pthread_mutex_unlock(&vtctx->lock);
280 }
281 
282 static void clear_frame_queue(VTEncContext *vtctx)
283 {
284  set_async_error(vtctx, 0);
285 }
286 
287 static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
288 {
289  BufNode *info;
290 
291  pthread_mutex_lock(&vtctx->lock);
292 
293  if (vtctx->async_error) {
294  pthread_mutex_unlock(&vtctx->lock);
295  return vtctx->async_error;
296  }
297 
298  if (vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out) {
299  *buf = NULL;
300 
301  pthread_mutex_unlock(&vtctx->lock);
302  return 0;
303  }
304 
305  while (!vtctx->q_head && !vtctx->async_error && wait && !vtctx->flushing) {
306  pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
307  }
308 
309  if (!vtctx->q_head) {
310  pthread_mutex_unlock(&vtctx->lock);
311  *buf = NULL;
312  return 0;
313  }
314 
315  info = vtctx->q_head;
316  vtctx->q_head = vtctx->q_head->next;
317  if (!vtctx->q_head) {
318  vtctx->q_tail = NULL;
319  }
320 
321  vtctx->frame_ct_out++;
322  pthread_mutex_unlock(&vtctx->lock);
323 
324  *buf = info->cm_buffer;
325  if (sei && *buf) {
326  *sei = info->sei;
327  } else if (info->sei) {
328  if (info->sei->data) av_free(info->sei->data);
329  av_free(info->sei);
330  }
331  av_free(info);
332 
333 
334  return 0;
335 }
336 
337 static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
338 {
339  BufNode *info = av_malloc(sizeof(BufNode));
340  if (!info) {
341  set_async_error(vtctx, AVERROR(ENOMEM));
342  return;
343  }
344 
345  CFRetain(buffer);
346  info->cm_buffer = buffer;
347  info->sei = sei;
348  info->next = NULL;
349 
350  pthread_mutex_lock(&vtctx->lock);
351 
352  if (!vtctx->q_head) {
353  vtctx->q_head = info;
354  } else {
355  vtctx->q_tail->next = info;
356  }
357 
358  vtctx->q_tail = info;
359 
361  pthread_mutex_unlock(&vtctx->lock);
362 }
363 
364 static int count_nalus(size_t length_code_size,
365  CMSampleBufferRef sample_buffer,
366  int *count)
367 {
368  size_t offset = 0;
369  int status;
370  int nalu_ct = 0;
371  uint8_t size_buf[4];
372  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
373  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
374 
375  if (length_code_size > 4)
376  return AVERROR_INVALIDDATA;
377 
378  while (offset < src_size) {
379  size_t curr_src_len;
380  size_t box_len = 0;
381  size_t i;
382 
383  status = CMBlockBufferCopyDataBytes(block,
384  offset,
385  length_code_size,
386  size_buf);
387 
388  for (i = 0; i < length_code_size; i++) {
389  box_len <<= 8;
390  box_len |= size_buf[i];
391  }
392 
393  curr_src_len = box_len + length_code_size;
394  offset += curr_src_len;
395 
396  nalu_ct++;
397  }
398 
399  *count = nalu_ct;
400  return 0;
401 }
402 
403 static CMVideoCodecType get_cm_codec_type(enum AVCodecID id,
404  enum AVPixelFormat fmt,
405  double alpha_quality)
406 {
407  switch (id) {
408  case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
409  case AV_CODEC_ID_HEVC:
410  if (fmt == AV_PIX_FMT_BGRA && alpha_quality > 0.0) {
412  }
413  return kCMVideoCodecType_HEVC;
414  default: return 0;
415  }
416 }
417 
418 /**
419  * Get the parameter sets from a CMSampleBufferRef.
420  * @param dst If *dst isn't NULL, the parameters are copied into existing
421  * memory. *dst_size must be set accordingly when *dst != NULL.
422  * If *dst is NULL, it will be allocated.
423  * In all cases, *dst_size is set to the number of bytes used starting
424  * at *dst.
425  */
426 static int get_params_size(
427  AVCodecContext *avctx,
428  CMVideoFormatDescriptionRef vid_fmt,
429  size_t *size)
430 {
431  VTEncContext *vtctx = avctx->priv_data;
432  size_t total_size = 0;
433  size_t ps_count;
434  int is_count_bad = 0;
435  size_t i;
436  int status;
437  status = vtctx->get_param_set_func(vid_fmt,
438  0,
439  NULL,
440  NULL,
441  &ps_count,
442  NULL);
443  if (status) {
444  is_count_bad = 1;
445  ps_count = 0;
446  status = 0;
447  }
448 
449  for (i = 0; i < ps_count || is_count_bad; i++) {
450  const uint8_t *ps;
451  size_t ps_size;
452  status = vtctx->get_param_set_func(vid_fmt,
453  i,
454  &ps,
455  &ps_size,
456  NULL,
457  NULL);
458  if (status) {
459  /*
460  * When ps_count is invalid, status != 0 ends the loop normally
461  * unless we didn't get any parameter sets.
462  */
463  if (i > 0 && is_count_bad) status = 0;
464 
465  break;
466  }
467 
468  total_size += ps_size + sizeof(start_code);
469  }
470 
471  if (status) {
472  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set sizes: %d\n", status);
473  return AVERROR_EXTERNAL;
474  }
475 
476  *size = total_size;
477  return 0;
478 }
479 
480 static int copy_param_sets(
481  AVCodecContext *avctx,
482  CMVideoFormatDescriptionRef vid_fmt,
483  uint8_t *dst,
484  size_t dst_size)
485 {
486  VTEncContext *vtctx = avctx->priv_data;
487  size_t ps_count;
488  int is_count_bad = 0;
489  int status;
490  size_t offset = 0;
491  size_t i;
492 
493  status = vtctx->get_param_set_func(vid_fmt,
494  0,
495  NULL,
496  NULL,
497  &ps_count,
498  NULL);
499  if (status) {
500  is_count_bad = 1;
501  ps_count = 0;
502  status = 0;
503  }
504 
505 
506  for (i = 0; i < ps_count || is_count_bad; i++) {
507  const uint8_t *ps;
508  size_t ps_size;
509  size_t next_offset;
510 
511  status = vtctx->get_param_set_func(vid_fmt,
512  i,
513  &ps,
514  &ps_size,
515  NULL,
516  NULL);
517  if (status) {
518  if (i > 0 && is_count_bad) status = 0;
519 
520  break;
521  }
522 
523  next_offset = offset + sizeof(start_code) + ps_size;
524  if (dst_size < next_offset) {
525  av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n");
527  }
528 
529  memcpy(dst + offset, start_code, sizeof(start_code));
530  offset += sizeof(start_code);
531 
532  memcpy(dst + offset, ps, ps_size);
533  offset = next_offset;
534  }
535 
536  if (status) {
537  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data: %d\n", status);
538  return AVERROR_EXTERNAL;
539  }
540 
541  return 0;
542 }
543 
544 static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
545 {
546  CMVideoFormatDescriptionRef vid_fmt;
547  size_t total_size;
548  int status;
549 
550  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
551  if (!vid_fmt) {
552  av_log(avctx, AV_LOG_ERROR, "No video format.\n");
553  return AVERROR_EXTERNAL;
554  }
555 
556  status = get_params_size(avctx, vid_fmt, &total_size);
557  if (status) {
558  av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
559  return status;
560  }
561 
562  avctx->extradata = av_mallocz(total_size + AV_INPUT_BUFFER_PADDING_SIZE);
563  if (!avctx->extradata) {
564  return AVERROR(ENOMEM);
565  }
566  avctx->extradata_size = total_size;
567 
568  status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
569 
570  if (status) {
571  av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
572  return status;
573  }
574 
575  return 0;
576 }
577 
579  void *ctx,
580  void *sourceFrameCtx,
581  OSStatus status,
582  VTEncodeInfoFlags flags,
583  CMSampleBufferRef sample_buffer)
584 {
585  AVCodecContext *avctx = ctx;
586  VTEncContext *vtctx = avctx->priv_data;
587  ExtraSEI *sei = sourceFrameCtx;
588 
589  if (vtctx->async_error) {
590  return;
591  }
592 
593  if (status) {
594  av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status);
596  return;
597  }
598 
599  if (!sample_buffer) {
600  return;
601  }
602 
603  if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) {
604  int set_status = set_extradata(avctx, sample_buffer);
605  if (set_status) {
606  set_async_error(vtctx, set_status);
607  return;
608  }
609  }
610 
611  vtenc_q_push(vtctx, sample_buffer, sei);
612 }
613 
615  AVCodecContext *avctx,
616  CMSampleBufferRef sample_buffer,
617  size_t *size)
618 {
619  VTEncContext *vtctx = avctx->priv_data;
620  CMVideoFormatDescriptionRef vid_fmt;
621  int isize;
622  int status;
623 
624  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
625  if (!vid_fmt) {
626  av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n");
627  return AVERROR_EXTERNAL;
628  }
629 
630  status = vtctx->get_param_set_func(vid_fmt,
631  0,
632  NULL,
633  NULL,
634  NULL,
635  &isize);
636  if (status) {
637  av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status);
638  return AVERROR_EXTERNAL;
639  }
640 
641  *size = isize;
642  return 0;
643 }
644 
645 /*
646  * Returns true on success.
647  *
648  * If profile_level_val is NULL and this method returns true, don't specify the
649  * profile/level to the encoder.
650  */
652  CFStringRef *profile_level_val)
653 {
654  VTEncContext *vtctx = avctx->priv_data;
655  int64_t profile = vtctx->profile;
656 
657  if (profile == H264_PROF_AUTO && vtctx->level) {
658  //Need to pick a profile if level is not auto-selected.
660  }
661 
662  *profile_level_val = NULL;
663 
664  switch (profile) {
665  case H264_PROF_AUTO:
666  return true;
667 
668  case H264_PROF_BASELINE:
669  switch (vtctx->level) {
670  case 0: *profile_level_val =
671  compat_keys.kVTProfileLevel_H264_Baseline_AutoLevel; break;
672  case 13: *profile_level_val = kVTProfileLevel_H264_Baseline_1_3; break;
673  case 30: *profile_level_val = kVTProfileLevel_H264_Baseline_3_0; break;
674  case 31: *profile_level_val = kVTProfileLevel_H264_Baseline_3_1; break;
675  case 32: *profile_level_val = kVTProfileLevel_H264_Baseline_3_2; break;
676  case 40: *profile_level_val =
677  compat_keys.kVTProfileLevel_H264_Baseline_4_0; break;
678  case 41: *profile_level_val = kVTProfileLevel_H264_Baseline_4_1; break;
679  case 42: *profile_level_val =
680  compat_keys.kVTProfileLevel_H264_Baseline_4_2; break;
681  case 50: *profile_level_val =
682  compat_keys.kVTProfileLevel_H264_Baseline_5_0; break;
683  case 51: *profile_level_val =
684  compat_keys.kVTProfileLevel_H264_Baseline_5_1; break;
685  case 52: *profile_level_val =
686  compat_keys.kVTProfileLevel_H264_Baseline_5_2; break;
687  }
688  break;
689 
690  case H264_PROF_MAIN:
691  switch (vtctx->level) {
692  case 0: *profile_level_val =
693  compat_keys.kVTProfileLevel_H264_Main_AutoLevel; break;
694  case 30: *profile_level_val = kVTProfileLevel_H264_Main_3_0; break;
695  case 31: *profile_level_val = kVTProfileLevel_H264_Main_3_1; break;
696  case 32: *profile_level_val = kVTProfileLevel_H264_Main_3_2; break;
697  case 40: *profile_level_val = kVTProfileLevel_H264_Main_4_0; break;
698  case 41: *profile_level_val = kVTProfileLevel_H264_Main_4_1; break;
699  case 42: *profile_level_val =
700  compat_keys.kVTProfileLevel_H264_Main_4_2; break;
701  case 50: *profile_level_val = kVTProfileLevel_H264_Main_5_0; break;
702  case 51: *profile_level_val =
703  compat_keys.kVTProfileLevel_H264_Main_5_1; break;
704  case 52: *profile_level_val =
705  compat_keys.kVTProfileLevel_H264_Main_5_2; break;
706  }
707  break;
708 
709  case H264_PROF_HIGH:
710  switch (vtctx->level) {
711  case 0: *profile_level_val =
712  compat_keys.kVTProfileLevel_H264_High_AutoLevel; break;
713  case 30: *profile_level_val =
714  compat_keys.kVTProfileLevel_H264_High_3_0; break;
715  case 31: *profile_level_val =
716  compat_keys.kVTProfileLevel_H264_High_3_1; break;
717  case 32: *profile_level_val =
718  compat_keys.kVTProfileLevel_H264_High_3_2; break;
719  case 40: *profile_level_val =
720  compat_keys.kVTProfileLevel_H264_High_4_0; break;
721  case 41: *profile_level_val =
722  compat_keys.kVTProfileLevel_H264_High_4_1; break;
723  case 42: *profile_level_val =
724  compat_keys.kVTProfileLevel_H264_High_4_2; break;
725  case 50: *profile_level_val = kVTProfileLevel_H264_High_5_0; break;
726  case 51: *profile_level_val =
727  compat_keys.kVTProfileLevel_H264_High_5_1; break;
728  case 52: *profile_level_val =
729  compat_keys.kVTProfileLevel_H264_High_5_2; break;
730  }
731  break;
732  case H264_PROF_EXTENDED:
733  switch (vtctx->level) {
734  case 0: *profile_level_val =
735  compat_keys.kVTProfileLevel_H264_Extended_AutoLevel; break;
736  case 50: *profile_level_val =
737  compat_keys.kVTProfileLevel_H264_Extended_5_0; break;
738  }
739  break;
740  }
741 
742  if (!*profile_level_val) {
743  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
744  return false;
745  }
746 
747  return true;
748 }
749 
750 /*
751  * Returns true on success.
752  *
753  * If profile_level_val is NULL and this method returns true, don't specify the
754  * profile/level to the encoder.
755  */
757  CFStringRef *profile_level_val)
758 {
759  VTEncContext *vtctx = avctx->priv_data;
760  int64_t profile = vtctx->profile;
761 
762  *profile_level_val = NULL;
763 
764  switch (profile) {
765  case HEVC_PROF_AUTO:
766  return true;
767  case HEVC_PROF_MAIN:
768  *profile_level_val =
769  compat_keys.kVTProfileLevel_HEVC_Main_AutoLevel;
770  break;
771  case HEVC_PROF_MAIN10:
772  *profile_level_val =
773  compat_keys.kVTProfileLevel_HEVC_Main10_AutoLevel;
774  break;
775  }
776 
777  if (!*profile_level_val) {
778  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
779  return false;
780  }
781 
782  return true;
783 }
784 
786  enum AVPixelFormat fmt,
787  enum AVColorRange range,
788  int* av_pixel_format,
789  int* range_guessed)
790 {
791  if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG &&
792  range != AVCOL_RANGE_JPEG;
793 
794  //MPEG range is used when no range is set
795  if (fmt == AV_PIX_FMT_NV12) {
796  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
797  kCVPixelFormatType_420YpCbCr8BiPlanarFullRange :
798  kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
799  } else if (fmt == AV_PIX_FMT_YUV420P) {
800  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
801  kCVPixelFormatType_420YpCbCr8PlanarFullRange :
802  kCVPixelFormatType_420YpCbCr8Planar;
803  } else if (fmt == AV_PIX_FMT_BGRA) {
804  *av_pixel_format = kCVPixelFormatType_32BGRA;
805  } else if (fmt == AV_PIX_FMT_P010LE) {
806  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
809  } else {
810  return AVERROR(EINVAL);
811  }
812 
813  return 0;
814 }
815 
816 static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict) {
817  VTEncContext *vtctx = avctx->priv_data;
818 
819  if (vtctx->color_primaries) {
820  CFDictionarySetValue(dict,
821  kCVImageBufferColorPrimariesKey,
822  vtctx->color_primaries);
823  }
824 
825  if (vtctx->transfer_function) {
826  CFDictionarySetValue(dict,
827  kCVImageBufferTransferFunctionKey,
828  vtctx->transfer_function);
829  }
830 
831  if (vtctx->ycbcr_matrix) {
832  CFDictionarySetValue(dict,
833  kCVImageBufferYCbCrMatrixKey,
834  vtctx->ycbcr_matrix);
835  }
836 }
837 
839  CFMutableDictionaryRef* dict)
840 {
841  CFNumberRef cv_color_format_num = NULL;
842  CFNumberRef width_num = NULL;
843  CFNumberRef height_num = NULL;
844  CFMutableDictionaryRef pixel_buffer_info = NULL;
845  int cv_color_format;
846  int status = get_cv_pixel_format(avctx,
847  avctx->pix_fmt,
848  avctx->color_range,
849  &cv_color_format,
850  NULL);
851  if (status) return status;
852 
853  pixel_buffer_info = CFDictionaryCreateMutable(
854  kCFAllocatorDefault,
855  20,
856  &kCFCopyStringDictionaryKeyCallBacks,
857  &kCFTypeDictionaryValueCallBacks);
858 
859  if (!pixel_buffer_info) goto pbinfo_nomem;
860 
861  cv_color_format_num = CFNumberCreate(kCFAllocatorDefault,
862  kCFNumberSInt32Type,
863  &cv_color_format);
864  if (!cv_color_format_num) goto pbinfo_nomem;
865 
866  CFDictionarySetValue(pixel_buffer_info,
867  kCVPixelBufferPixelFormatTypeKey,
868  cv_color_format_num);
869  vt_release_num(&cv_color_format_num);
870 
871  width_num = CFNumberCreate(kCFAllocatorDefault,
872  kCFNumberSInt32Type,
873  &avctx->width);
874  if (!width_num) return AVERROR(ENOMEM);
875 
876  CFDictionarySetValue(pixel_buffer_info,
877  kCVPixelBufferWidthKey,
878  width_num);
879  vt_release_num(&width_num);
880 
881  height_num = CFNumberCreate(kCFAllocatorDefault,
882  kCFNumberSInt32Type,
883  &avctx->height);
884  if (!height_num) goto pbinfo_nomem;
885 
886  CFDictionarySetValue(pixel_buffer_info,
887  kCVPixelBufferHeightKey,
888  height_num);
889  vt_release_num(&height_num);
890 
891  add_color_attr(avctx, pixel_buffer_info);
892 
893  *dict = pixel_buffer_info;
894  return 0;
895 
896 pbinfo_nomem:
897  vt_release_num(&cv_color_format_num);
898  vt_release_num(&width_num);
899  vt_release_num(&height_num);
900  if (pixel_buffer_info) CFRelease(pixel_buffer_info);
901 
902  return AVERROR(ENOMEM);
903 }
904 
906  CFStringRef *primaries)
907 {
908  enum AVColorPrimaries pri = avctx->color_primaries;
909  switch (pri) {
911  *primaries = NULL;
912  break;
913 
914  case AVCOL_PRI_BT470BG:
915  *primaries = kCVImageBufferColorPrimaries_EBU_3213;
916  break;
917 
918  case AVCOL_PRI_SMPTE170M:
919  *primaries = kCVImageBufferColorPrimaries_SMPTE_C;
920  break;
921 
922  case AVCOL_PRI_BT709:
923  *primaries = kCVImageBufferColorPrimaries_ITU_R_709_2;
924  break;
925 
926  case AVCOL_PRI_BT2020:
927  *primaries = compat_keys.kCVImageBufferColorPrimaries_ITU_R_2020;
928  break;
929 
930  default:
931  av_log(avctx, AV_LOG_ERROR, "Color primaries %s is not supported.\n", av_color_primaries_name(pri));
932  *primaries = NULL;
933  return -1;
934  }
935 
936  return 0;
937 }
938 
940  CFStringRef *transfer_fnc,
941  CFNumberRef *gamma_level)
942 {
943  enum AVColorTransferCharacteristic trc = avctx->color_trc;
944  Float32 gamma;
945  *gamma_level = NULL;
946 
947  switch (trc) {
949  *transfer_fnc = NULL;
950  break;
951 
952  case AVCOL_TRC_BT709:
953  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_709_2;
954  break;
955 
956  case AVCOL_TRC_SMPTE240M:
957  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
958  break;
959 
960 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ
961  case AVCOL_TRC_SMPTE2084:
962  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
963  break;
964 #endif
965 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_LINEAR
966  case AVCOL_TRC_LINEAR:
967  *transfer_fnc = kCVImageBufferTransferFunction_Linear;
968  break;
969 #endif
970 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
972  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
973  break;
974 #endif
975 
976  case AVCOL_TRC_GAMMA22:
977  gamma = 2.2;
978  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
979  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
980  break;
981 
982  case AVCOL_TRC_GAMMA28:
983  gamma = 2.8;
984  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
985  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
986  break;
987 
988  case AVCOL_TRC_BT2020_10:
989  case AVCOL_TRC_BT2020_12:
990  *transfer_fnc = compat_keys.kCVImageBufferTransferFunction_ITU_R_2020;
991  break;
992 
993  default:
994  *transfer_fnc = NULL;
995  av_log(avctx, AV_LOG_ERROR, "Transfer function %s is not supported.\n", av_color_transfer_name(trc));
996  return -1;
997  }
998 
999  return 0;
1000 }
1001 
1002 static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix) {
1003  switch(avctx->colorspace) {
1004  case AVCOL_SPC_BT709:
1005  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
1006  break;
1007 
1008  case AVCOL_SPC_UNSPECIFIED:
1009  *matrix = NULL;
1010  break;
1011 
1012  case AVCOL_SPC_BT470BG:
1013  case AVCOL_SPC_SMPTE170M:
1014  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
1015  break;
1016 
1017  case AVCOL_SPC_SMPTE240M:
1018  *matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
1019  break;
1020 
1021  case AVCOL_SPC_BT2020_NCL:
1022  *matrix = compat_keys.kCVImageBufferYCbCrMatrix_ITU_R_2020;
1023  break;
1024 
1025  default:
1026  av_log(avctx, AV_LOG_ERROR, "Color space %s is not supported.\n", av_color_space_name(avctx->colorspace));
1027  return -1;
1028  }
1029 
1030  return 0;
1031 }
1032 
1033 // constant quality only on Macs with Apple Silicon
1034 static bool vtenc_qscale_enabled(void)
1035 {
1036  return TARGET_OS_OSX && TARGET_CPU_ARM64;
1037 }
1038 
1040  CMVideoCodecType codec_type,
1041  CFStringRef profile_level,
1042  CFNumberRef gamma_level,
1043  CFDictionaryRef enc_info,
1044  CFDictionaryRef pixel_buffer_info,
1045  VTCompressionSessionRef *session)
1046 {
1047  VTEncContext *vtctx = avctx->priv_data;
1048  SInt32 bit_rate = avctx->bit_rate;
1049  SInt32 max_rate = avctx->rc_max_rate;
1050  Float32 quality = avctx->global_quality / FF_QP2LAMBDA;
1051  CFNumberRef bit_rate_num;
1052  CFNumberRef quality_num;
1053  CFNumberRef bytes_per_second;
1054  CFNumberRef one_second;
1055  CFArrayRef data_rate_limits;
1056  int64_t bytes_per_second_value = 0;
1057  int64_t one_second_value = 0;
1058  void *nums[2];
1059 
1060  int status = VTCompressionSessionCreate(kCFAllocatorDefault,
1061  avctx->width,
1062  avctx->height,
1063  codec_type,
1064  enc_info,
1065  pixel_buffer_info,
1066  kCFAllocatorDefault,
1068  avctx,
1069  session);
1070 
1071  if (status || !vtctx->session) {
1072  av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression session: %d\n", status);
1073 
1074 #if !TARGET_OS_IPHONE
1075  if (!vtctx->allow_sw) {
1076  av_log(avctx, AV_LOG_ERROR, "Try -allow_sw 1. The hardware encoder may be busy, or not supported.\n");
1077  }
1078 #endif
1079 
1080  return AVERROR_EXTERNAL;
1081  }
1082 
1083  if (avctx->flags & AV_CODEC_FLAG_QSCALE && !vtenc_qscale_enabled()) {
1084  av_log(avctx, AV_LOG_ERROR, "Error: -q:v qscale not available for encoder. Use -b:v bitrate instead.\n");
1085  return AVERROR_EXTERNAL;
1086  }
1087 
1088  if (avctx->flags & AV_CODEC_FLAG_QSCALE) {
1089  quality = quality >= 100 ? 1.0 : quality / 100;
1090  quality_num = CFNumberCreate(kCFAllocatorDefault,
1091  kCFNumberFloat32Type,
1092  &quality);
1093  if (!quality_num) return AVERROR(ENOMEM);
1094 
1095  status = VTSessionSetProperty(vtctx->session,
1096  kVTCompressionPropertyKey_Quality,
1097  quality_num);
1098  CFRelease(quality_num);
1099  } else {
1100  bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
1101  kCFNumberSInt32Type,
1102  &bit_rate);
1103  if (!bit_rate_num) return AVERROR(ENOMEM);
1104 
1105  status = VTSessionSetProperty(vtctx->session,
1106  kVTCompressionPropertyKey_AverageBitRate,
1107  bit_rate_num);
1108  CFRelease(bit_rate_num);
1109  }
1110 
1111  if (status) {
1112  av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n", status);
1113  return AVERROR_EXTERNAL;
1114  }
1115 
1116  if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
1117  // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
1118  bytes_per_second_value = max_rate >> 3;
1119  bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
1120  kCFNumberSInt64Type,
1121  &bytes_per_second_value);
1122  if (!bytes_per_second) {
1123  return AVERROR(ENOMEM);
1124  }
1125  one_second_value = 1;
1126  one_second = CFNumberCreate(kCFAllocatorDefault,
1127  kCFNumberSInt64Type,
1128  &one_second_value);
1129  if (!one_second) {
1130  CFRelease(bytes_per_second);
1131  return AVERROR(ENOMEM);
1132  }
1133  nums[0] = (void *)bytes_per_second;
1134  nums[1] = (void *)one_second;
1135  data_rate_limits = CFArrayCreate(kCFAllocatorDefault,
1136  (const void **)nums,
1137  2,
1138  &kCFTypeArrayCallBacks);
1139 
1140  if (!data_rate_limits) {
1141  CFRelease(bytes_per_second);
1142  CFRelease(one_second);
1143  return AVERROR(ENOMEM);
1144  }
1145  status = VTSessionSetProperty(vtctx->session,
1146  kVTCompressionPropertyKey_DataRateLimits,
1147  data_rate_limits);
1148 
1149  CFRelease(bytes_per_second);
1150  CFRelease(one_second);
1151  CFRelease(data_rate_limits);
1152 
1153  if (status) {
1154  av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
1155  return AVERROR_EXTERNAL;
1156  }
1157  }
1158 
1159  if (vtctx->codec_id == AV_CODEC_ID_HEVC) {
1160  if (avctx->pix_fmt == AV_PIX_FMT_BGRA && vtctx->alpha_quality > 0.0) {
1161  CFNumberRef alpha_quality_num = CFNumberCreate(kCFAllocatorDefault,
1162  kCFNumberDoubleType,
1163  &vtctx->alpha_quality);
1164  if (!alpha_quality_num) return AVERROR(ENOMEM);
1165 
1166  status = VTSessionSetProperty(vtctx->session,
1167  compat_keys.kVTCompressionPropertyKey_TargetQualityForAlpha,
1168  alpha_quality_num);
1169  CFRelease(alpha_quality_num);
1170  }
1171  }
1172 
1173  if (profile_level) {
1174  status = VTSessionSetProperty(vtctx->session,
1175  kVTCompressionPropertyKey_ProfileLevel,
1176  profile_level);
1177  if (status) {
1178  av_log(avctx, AV_LOG_ERROR, "Error setting profile/level property: %d. Output will be encoded using a supported profile/level combination.\n", status);
1179  }
1180  }
1181 
1182  if (avctx->gop_size > 0) {
1183  CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault,
1184  kCFNumberIntType,
1185  &avctx->gop_size);
1186  if (!interval) {
1187  return AVERROR(ENOMEM);
1188  }
1189 
1190  status = VTSessionSetProperty(vtctx->session,
1191  kVTCompressionPropertyKey_MaxKeyFrameInterval,
1192  interval);
1193  CFRelease(interval);
1194 
1195  if (status) {
1196  av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame interval' property: %d\n", status);
1197  return AVERROR_EXTERNAL;
1198  }
1199  }
1200 
1201  if (vtctx->frames_before) {
1202  status = VTSessionSetProperty(vtctx->session,
1203  kVTCompressionPropertyKey_MoreFramesBeforeStart,
1204  kCFBooleanTrue);
1205 
1206  if (status == kVTPropertyNotSupportedErr) {
1207  av_log(avctx, AV_LOG_WARNING, "frames_before property is not supported on this device. Ignoring.\n");
1208  } else if (status) {
1209  av_log(avctx, AV_LOG_ERROR, "Error setting frames_before property: %d\n", status);
1210  }
1211  }
1212 
1213  if (vtctx->frames_after) {
1214  status = VTSessionSetProperty(vtctx->session,
1215  kVTCompressionPropertyKey_MoreFramesAfterEnd,
1216  kCFBooleanTrue);
1217 
1218  if (status == kVTPropertyNotSupportedErr) {
1219  av_log(avctx, AV_LOG_WARNING, "frames_after property is not supported on this device. Ignoring.\n");
1220  } else if (status) {
1221  av_log(avctx, AV_LOG_ERROR, "Error setting frames_after property: %d\n", status);
1222  }
1223  }
1224 
1225  if (avctx->sample_aspect_ratio.num != 0) {
1226  CFNumberRef num;
1227  CFNumberRef den;
1228  CFMutableDictionaryRef par;
1229  AVRational *avpar = &avctx->sample_aspect_ratio;
1230 
1231  av_reduce(&avpar->num, &avpar->den,
1232  avpar->num, avpar->den,
1233  0xFFFFFFFF);
1234 
1235  num = CFNumberCreate(kCFAllocatorDefault,
1236  kCFNumberIntType,
1237  &avpar->num);
1238 
1239  den = CFNumberCreate(kCFAllocatorDefault,
1240  kCFNumberIntType,
1241  &avpar->den);
1242 
1243 
1244 
1245  par = CFDictionaryCreateMutable(kCFAllocatorDefault,
1246  2,
1247  &kCFCopyStringDictionaryKeyCallBacks,
1248  &kCFTypeDictionaryValueCallBacks);
1249 
1250  if (!par || !num || !den) {
1251  if (par) CFRelease(par);
1252  if (num) CFRelease(num);
1253  if (den) CFRelease(den);
1254 
1255  return AVERROR(ENOMEM);
1256  }
1257 
1258  CFDictionarySetValue(
1259  par,
1260  kCMFormatDescriptionKey_PixelAspectRatioHorizontalSpacing,
1261  num);
1262 
1263  CFDictionarySetValue(
1264  par,
1265  kCMFormatDescriptionKey_PixelAspectRatioVerticalSpacing,
1266  den);
1267 
1268  status = VTSessionSetProperty(vtctx->session,
1269  kVTCompressionPropertyKey_PixelAspectRatio,
1270  par);
1271 
1272  CFRelease(par);
1273  CFRelease(num);
1274  CFRelease(den);
1275 
1276  if (status) {
1277  av_log(avctx,
1278  AV_LOG_ERROR,
1279  "Error setting pixel aspect ratio to %d:%d: %d.\n",
1280  avctx->sample_aspect_ratio.num,
1281  avctx->sample_aspect_ratio.den,
1282  status);
1283 
1284  return AVERROR_EXTERNAL;
1285  }
1286  }
1287 
1288 
1289  if (vtctx->transfer_function) {
1290  status = VTSessionSetProperty(vtctx->session,
1291  kVTCompressionPropertyKey_TransferFunction,
1292  vtctx->transfer_function);
1293 
1294  if (status) {
1295  av_log(avctx, AV_LOG_WARNING, "Could not set transfer function: %d\n", status);
1296  }
1297  }
1298 
1299 
1300  if (vtctx->ycbcr_matrix) {
1301  status = VTSessionSetProperty(vtctx->session,
1302  kVTCompressionPropertyKey_YCbCrMatrix,
1303  vtctx->ycbcr_matrix);
1304 
1305  if (status) {
1306  av_log(avctx, AV_LOG_WARNING, "Could not set ycbcr matrix: %d\n", status);
1307  }
1308  }
1309 
1310 
1311  if (vtctx->color_primaries) {
1312  status = VTSessionSetProperty(vtctx->session,
1313  kVTCompressionPropertyKey_ColorPrimaries,
1314  vtctx->color_primaries);
1315 
1316  if (status) {
1317  av_log(avctx, AV_LOG_WARNING, "Could not set color primaries: %d\n", status);
1318  }
1319  }
1320 
1321  if (gamma_level) {
1322  status = VTSessionSetProperty(vtctx->session,
1323  kCVImageBufferGammaLevelKey,
1324  gamma_level);
1325 
1326  if (status) {
1327  av_log(avctx, AV_LOG_WARNING, "Could not set gamma level: %d\n", status);
1328  }
1329  }
1330 
1331  if (!vtctx->has_b_frames) {
1332  status = VTSessionSetProperty(vtctx->session,
1333  kVTCompressionPropertyKey_AllowFrameReordering,
1334  kCFBooleanFalse);
1335 
1336  if (status) {
1337  av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame reordering' property: %d\n", status);
1338  return AVERROR_EXTERNAL;
1339  }
1340  }
1341 
1342  if (vtctx->entropy != VT_ENTROPY_NOT_SET) {
1343  CFStringRef entropy = vtctx->entropy == VT_CABAC ?
1344  compat_keys.kVTH264EntropyMode_CABAC:
1345  compat_keys.kVTH264EntropyMode_CAVLC;
1346 
1347  status = VTSessionSetProperty(vtctx->session,
1348  compat_keys.kVTCompressionPropertyKey_H264EntropyMode,
1349  entropy);
1350 
1351  if (status) {
1352  av_log(avctx, AV_LOG_ERROR, "Error setting entropy property: %d\n", status);
1353  }
1354  }
1355 
1356  if (vtctx->realtime) {
1357  status = VTSessionSetProperty(vtctx->session,
1358  compat_keys.kVTCompressionPropertyKey_RealTime,
1359  kCFBooleanTrue);
1360 
1361  if (status) {
1362  av_log(avctx, AV_LOG_ERROR, "Error setting realtime property: %d\n", status);
1363  }
1364  }
1365 
1366  status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
1367  if (status) {
1368  av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n", status);
1369  return AVERROR_EXTERNAL;
1370  }
1371 
1372  return 0;
1373 }
1374 
1376 {
1377  CFMutableDictionaryRef enc_info;
1378  CFMutableDictionaryRef pixel_buffer_info;
1379  CMVideoCodecType codec_type;
1380  VTEncContext *vtctx = avctx->priv_data;
1381  CFStringRef profile_level;
1382  CFNumberRef gamma_level = NULL;
1383  int status;
1384 
1385  codec_type = get_cm_codec_type(avctx->codec_id, avctx->pix_fmt, vtctx->alpha_quality);
1386  if (!codec_type) {
1387  av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
1388  return AVERROR(EINVAL);
1389  }
1390 
1391  vtctx->codec_id = avctx->codec_id;
1392  avctx->max_b_frames = 16;
1393 
1394  if (vtctx->codec_id == AV_CODEC_ID_H264) {
1395  vtctx->get_param_set_func = CMVideoFormatDescriptionGetH264ParameterSetAtIndex;
1396 
1397  vtctx->has_b_frames = avctx->max_b_frames > 0;
1398  if(vtctx->has_b_frames && vtctx->profile == H264_PROF_BASELINE){
1399  av_log(avctx, AV_LOG_WARNING, "Cannot use B-frames with baseline profile. Output will not contain B-frames.\n");
1400  vtctx->has_b_frames = 0;
1401  }
1402 
1403  if (vtctx->entropy == VT_CABAC && vtctx->profile == H264_PROF_BASELINE) {
1404  av_log(avctx, AV_LOG_WARNING, "CABAC entropy requires 'main' or 'high' profile, but baseline was requested. Encode will not use CABAC entropy.\n");
1405  vtctx->entropy = VT_ENTROPY_NOT_SET;
1406  }
1407 
1408  if (!get_vt_h264_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1409  } else {
1410  vtctx->get_param_set_func = compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex;
1411  if (!vtctx->get_param_set_func) return AVERROR(EINVAL);
1412  if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1413  // HEVC has b-byramid
1414  vtctx->has_b_frames = avctx->max_b_frames > 0 ? 2 : 0;
1415  }
1416 
1417  enc_info = CFDictionaryCreateMutable(
1418  kCFAllocatorDefault,
1419  20,
1420  &kCFCopyStringDictionaryKeyCallBacks,
1421  &kCFTypeDictionaryValueCallBacks
1422  );
1423 
1424  if (!enc_info) return AVERROR(ENOMEM);
1425 
1426 #if !TARGET_OS_IPHONE
1427  if(vtctx->require_sw) {
1428  CFDictionarySetValue(enc_info,
1429  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1430  kCFBooleanFalse);
1431  } else if (!vtctx->allow_sw) {
1432  CFDictionarySetValue(enc_info,
1433  compat_keys.kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
1434  kCFBooleanTrue);
1435  } else {
1436  CFDictionarySetValue(enc_info,
1437  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1438  kCFBooleanTrue);
1439  }
1440 #endif
1441 
1442  if (avctx->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX) {
1443  status = create_cv_pixel_buffer_info(avctx, &pixel_buffer_info);
1444  if (status)
1445  goto init_cleanup;
1446  } else {
1447  pixel_buffer_info = NULL;
1448  }
1449 
1450  vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
1451 
1452  get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level);
1453  get_cv_ycbcr_matrix(avctx, &vtctx->ycbcr_matrix);
1454  get_cv_color_primaries(avctx, &vtctx->color_primaries);
1455 
1456 
1457  if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
1458  status = vtenc_populate_extradata(avctx,
1459  codec_type,
1460  profile_level,
1461  gamma_level,
1462  enc_info,
1463  pixel_buffer_info);
1464  if (status)
1465  goto init_cleanup;
1466  }
1467 
1468  status = vtenc_create_encoder(avctx,
1469  codec_type,
1470  profile_level,
1471  gamma_level,
1472  enc_info,
1473  pixel_buffer_info,
1474  &vtctx->session);
1475 
1476 init_cleanup:
1477  if (gamma_level)
1478  CFRelease(gamma_level);
1479 
1480  if (pixel_buffer_info)
1481  CFRelease(pixel_buffer_info);
1482 
1483  CFRelease(enc_info);
1484 
1485  return status;
1486 }
1487 
1489 {
1490  VTEncContext *vtctx = avctx->priv_data;
1491  CFBooleanRef has_b_frames_cfbool;
1492  int status;
1493 
1495 
1496  pthread_mutex_init(&vtctx->lock, NULL);
1498 
1499  vtctx->session = NULL;
1500  status = vtenc_configure_encoder(avctx);
1501  if (status) return status;
1502 
1503  status = VTSessionCopyProperty(vtctx->session,
1504  kVTCompressionPropertyKey_AllowFrameReordering,
1505  kCFAllocatorDefault,
1506  &has_b_frames_cfbool);
1507 
1508  if (!status && has_b_frames_cfbool) {
1509  //Some devices don't output B-frames for main profile, even if requested.
1510  // HEVC has b-pyramid
1511  vtctx->has_b_frames = (CFBooleanGetValue(has_b_frames_cfbool) && avctx->codec_id == AV_CODEC_ID_HEVC) ? 2 : 1;
1512  CFRelease(has_b_frames_cfbool);
1513  }
1514  avctx->has_b_frames = vtctx->has_b_frames;
1515 
1516  return 0;
1517 }
1518 
1519 static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
1520 {
1521  CFArrayRef attachments;
1522  CFDictionaryRef attachment;
1523  CFBooleanRef not_sync;
1524  CFIndex len;
1525 
1526  attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, false);
1527  len = !attachments ? 0 : CFArrayGetCount(attachments);
1528 
1529  if (!len) {
1530  *is_key_frame = true;
1531  return;
1532  }
1533 
1534  attachment = CFArrayGetValueAtIndex(attachments, 0);
1535 
1536  if (CFDictionaryGetValueIfPresent(attachment,
1537  kCMSampleAttachmentKey_NotSync,
1538  (const void **)&not_sync))
1539  {
1540  *is_key_frame = !CFBooleanGetValue(not_sync);
1541  } else {
1542  *is_key_frame = true;
1543  }
1544 }
1545 
1546 static int is_post_sei_nal_type(int nal_type){
1547  return nal_type != H264_NAL_SEI &&
1548  nal_type != H264_NAL_SPS &&
1549  nal_type != H264_NAL_PPS &&
1550  nal_type != H264_NAL_AUD;
1551 }
1552 
1553 /*
1554  * Finds the sei message start/size of type find_sei_type.
1555  * If more than one of that type exists, the last one is returned.
1556  */
1557 static int find_sei_end(AVCodecContext *avctx,
1558  uint8_t *nal_data,
1559  size_t nal_size,
1560  uint8_t **sei_end)
1561 {
1562  int nal_type;
1563  size_t sei_payload_size = 0;
1564  int sei_payload_type = 0;
1565  *sei_end = NULL;
1566  uint8_t *nal_start = nal_data;
1567 
1568  if (!nal_size)
1569  return 0;
1570 
1571  nal_type = *nal_data & 0x1F;
1572  if (nal_type != H264_NAL_SEI)
1573  return 0;
1574 
1575  nal_data++;
1576  nal_size--;
1577 
1578  if (nal_data[nal_size - 1] == 0x80)
1579  nal_size--;
1580 
1581  while (nal_size > 0 && *nal_data > 0) {
1582  do{
1583  sei_payload_type += *nal_data;
1584  nal_data++;
1585  nal_size--;
1586  } while (nal_size > 0 && *nal_data == 0xFF);
1587 
1588  if (!nal_size) {
1589  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing type.\n");
1590  return AVERROR_INVALIDDATA;
1591  }
1592 
1593  do{
1594  sei_payload_size += *nal_data;
1595  nal_data++;
1596  nal_size--;
1597  } while (nal_size > 0 && *nal_data == 0xFF);
1598 
1599  if (nal_size < sei_payload_size) {
1600  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing size.\n");
1601  return AVERROR_INVALIDDATA;
1602  }
1603 
1604  nal_data += sei_payload_size;
1605  nal_size -= sei_payload_size;
1606  }
1607 
1608  *sei_end = nal_data;
1609 
1610  return nal_data - nal_start + 1;
1611 }
1612 
1613 /**
1614  * Copies the data inserting emulation prevention bytes as needed.
1615  * Existing data in the destination can be taken into account by providing
1616  * dst with a dst_offset > 0.
1617  *
1618  * @return The number of bytes copied on success. On failure, the negative of
1619  * the number of bytes needed to copy src is returned.
1620  */
1621 static int copy_emulation_prev(const uint8_t *src,
1622  size_t src_size,
1623  uint8_t *dst,
1624  ssize_t dst_offset,
1625  size_t dst_size)
1626 {
1627  int zeros = 0;
1628  int wrote_bytes;
1629  uint8_t* dst_start;
1630  uint8_t* dst_end = dst + dst_size;
1631  const uint8_t* src_end = src + src_size;
1632  int start_at = dst_offset > 2 ? dst_offset - 2 : 0;
1633  int i;
1634  for (i = start_at; i < dst_offset && i < dst_size; i++) {
1635  if (!dst[i])
1636  zeros++;
1637  else
1638  zeros = 0;
1639  }
1640 
1641  dst += dst_offset;
1642  dst_start = dst;
1643  for (; src < src_end; src++, dst++) {
1644  if (zeros == 2) {
1645  int insert_ep3_byte = *src <= 3;
1646  if (insert_ep3_byte) {
1647  if (dst < dst_end)
1648  *dst = 3;
1649  dst++;
1650  }
1651 
1652  zeros = 0;
1653  }
1654 
1655  if (dst < dst_end)
1656  *dst = *src;
1657 
1658  if (!*src)
1659  zeros++;
1660  else
1661  zeros = 0;
1662  }
1663 
1664  wrote_bytes = dst - dst_start;
1665 
1666  if (dst > dst_end)
1667  return -wrote_bytes;
1668 
1669  return wrote_bytes;
1670 }
1671 
1672 static int write_sei(const ExtraSEI *sei,
1673  int sei_type,
1674  uint8_t *dst,
1675  size_t dst_size)
1676 {
1677  uint8_t *sei_start = dst;
1678  size_t remaining_sei_size = sei->size;
1679  size_t remaining_dst_size = dst_size;
1680  int header_bytes;
1681  int bytes_written;
1682  ssize_t offset;
1683 
1684  if (!remaining_dst_size)
1685  return AVERROR_BUFFER_TOO_SMALL;
1686 
1687  while (sei_type && remaining_dst_size != 0) {
1688  int sei_byte = sei_type > 255 ? 255 : sei_type;
1689  *dst = sei_byte;
1690 
1691  sei_type -= sei_byte;
1692  dst++;
1693  remaining_dst_size--;
1694  }
1695 
1696  if (!dst_size)
1697  return AVERROR_BUFFER_TOO_SMALL;
1698 
1699  while (remaining_sei_size && remaining_dst_size != 0) {
1700  int size_byte = remaining_sei_size > 255 ? 255 : remaining_sei_size;
1701  *dst = size_byte;
1702 
1703  remaining_sei_size -= size_byte;
1704  dst++;
1705  remaining_dst_size--;
1706  }
1707 
1708  if (remaining_dst_size < sei->size)
1709  return AVERROR_BUFFER_TOO_SMALL;
1710 
1711  header_bytes = dst - sei_start;
1712 
1713  offset = header_bytes;
1714  bytes_written = copy_emulation_prev(sei->data,
1715  sei->size,
1716  sei_start,
1717  offset,
1718  dst_size);
1719  if (bytes_written < 0)
1720  return AVERROR_BUFFER_TOO_SMALL;
1721 
1722  bytes_written += header_bytes;
1723  return bytes_written;
1724 }
1725 
1726 /**
1727  * Copies NAL units and replaces length codes with
1728  * H.264 Annex B start codes. On failure, the contents of
1729  * dst_data may have been modified.
1730  *
1731  * @param length_code_size Byte length of each length code
1732  * @param sample_buffer NAL units prefixed with length codes.
1733  * @param sei Optional A53 closed captions SEI data.
1734  * @param dst_data Must be zeroed before calling this function.
1735  * Contains the copied NAL units prefixed with
1736  * start codes when the function returns
1737  * successfully.
1738  * @param dst_size Length of dst_data
1739  * @return 0 on success
1740  * AVERROR_INVALIDDATA if length_code_size is invalid
1741  * AVERROR_BUFFER_TOO_SMALL if dst_data is too small
1742  * or if a length_code in src_data specifies data beyond
1743  * the end of its buffer.
1744  */
1746  AVCodecContext *avctx,
1747  size_t length_code_size,
1748  CMSampleBufferRef sample_buffer,
1749  ExtraSEI *sei,
1750  uint8_t *dst_data,
1751  size_t dst_size)
1752 {
1753  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1754  size_t remaining_src_size = src_size;
1755  size_t remaining_dst_size = dst_size;
1756  size_t src_offset = 0;
1757  int wrote_sei = 0;
1758  int status;
1759  uint8_t size_buf[4];
1760  uint8_t nal_type;
1761  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
1762 
1763  if (length_code_size > 4) {
1764  return AVERROR_INVALIDDATA;
1765  }
1766 
1767  while (remaining_src_size > 0) {
1768  size_t curr_src_len;
1769  size_t curr_dst_len;
1770  size_t box_len = 0;
1771  size_t i;
1772 
1773  uint8_t *dst_box;
1774 
1775  status = CMBlockBufferCopyDataBytes(block,
1776  src_offset,
1777  length_code_size,
1778  size_buf);
1779  if (status) {
1780  av_log(avctx, AV_LOG_ERROR, "Cannot copy length: %d\n", status);
1781  return AVERROR_EXTERNAL;
1782  }
1783 
1784  status = CMBlockBufferCopyDataBytes(block,
1785  src_offset + length_code_size,
1786  1,
1787  &nal_type);
1788 
1789  if (status) {
1790  av_log(avctx, AV_LOG_ERROR, "Cannot copy type: %d\n", status);
1791  return AVERROR_EXTERNAL;
1792  }
1793 
1794  nal_type &= 0x1F;
1795 
1796  for (i = 0; i < length_code_size; i++) {
1797  box_len <<= 8;
1798  box_len |= size_buf[i];
1799  }
1800 
1801  if (sei && !wrote_sei && is_post_sei_nal_type(nal_type)) {
1802  //No SEI NAL unit - insert.
1803  int wrote_bytes;
1804 
1805  memcpy(dst_data, start_code, sizeof(start_code));
1806  dst_data += sizeof(start_code);
1807  remaining_dst_size -= sizeof(start_code);
1808 
1809  *dst_data = H264_NAL_SEI;
1810  dst_data++;
1811  remaining_dst_size--;
1812 
1813  wrote_bytes = write_sei(sei,
1815  dst_data,
1816  remaining_dst_size);
1817 
1818  if (wrote_bytes < 0)
1819  return wrote_bytes;
1820 
1821  remaining_dst_size -= wrote_bytes;
1822  dst_data += wrote_bytes;
1823 
1824  if (remaining_dst_size <= 0)
1825  return AVERROR_BUFFER_TOO_SMALL;
1826 
1827  *dst_data = 0x80;
1828 
1829  dst_data++;
1830  remaining_dst_size--;
1831 
1832  wrote_sei = 1;
1833  }
1834 
1835  curr_src_len = box_len + length_code_size;
1836  curr_dst_len = box_len + sizeof(start_code);
1837 
1838  if (remaining_src_size < curr_src_len) {
1839  return AVERROR_BUFFER_TOO_SMALL;
1840  }
1841 
1842  if (remaining_dst_size < curr_dst_len) {
1843  return AVERROR_BUFFER_TOO_SMALL;
1844  }
1845 
1846  dst_box = dst_data + sizeof(start_code);
1847 
1848  memcpy(dst_data, start_code, sizeof(start_code));
1849  status = CMBlockBufferCopyDataBytes(block,
1850  src_offset + length_code_size,
1851  box_len,
1852  dst_box);
1853 
1854  if (status) {
1855  av_log(avctx, AV_LOG_ERROR, "Cannot copy data: %d\n", status);
1856  return AVERROR_EXTERNAL;
1857  }
1858 
1859  if (sei && !wrote_sei && nal_type == H264_NAL_SEI) {
1860  //Found SEI NAL unit - append.
1861  int wrote_bytes;
1862  int old_sei_length;
1863  int extra_bytes;
1864  uint8_t *new_sei;
1865  old_sei_length = find_sei_end(avctx, dst_box, box_len, &new_sei);
1866  if (old_sei_length < 0)
1867  return status;
1868 
1869  wrote_bytes = write_sei(sei,
1871  new_sei,
1872  remaining_dst_size - old_sei_length);
1873  if (wrote_bytes < 0)
1874  return wrote_bytes;
1875 
1876  if (new_sei + wrote_bytes >= dst_data + remaining_dst_size)
1877  return AVERROR_BUFFER_TOO_SMALL;
1878 
1879  new_sei[wrote_bytes++] = 0x80;
1880  extra_bytes = wrote_bytes - (dst_box + box_len - new_sei);
1881 
1882  dst_data += extra_bytes;
1883  remaining_dst_size -= extra_bytes;
1884 
1885  wrote_sei = 1;
1886  }
1887 
1888  src_offset += curr_src_len;
1889  dst_data += curr_dst_len;
1890 
1891  remaining_src_size -= curr_src_len;
1892  remaining_dst_size -= curr_dst_len;
1893  }
1894 
1895  return 0;
1896 }
1897 
1898 /**
1899  * Returns a sufficient number of bytes to contain the sei data.
1900  * It may be greater than the minimum required.
1901  */
1902 static int get_sei_msg_bytes(const ExtraSEI* sei, int type){
1903  int copied_size;
1904  if (sei->size == 0)
1905  return 0;
1906 
1907  copied_size = -copy_emulation_prev(sei->data,
1908  sei->size,
1909  NULL,
1910  0,
1911  0);
1912 
1913  if ((sei->size % 255) == 0) //may result in an extra byte
1914  copied_size++;
1915 
1916  return copied_size + sei->size / 255 + 1 + type / 255 + 1;
1917 }
1918 
1920  AVCodecContext *avctx,
1921  CMSampleBufferRef sample_buffer,
1922  AVPacket *pkt,
1923  ExtraSEI *sei)
1924 {
1925  VTEncContext *vtctx = avctx->priv_data;
1926 
1927  int status;
1928  bool is_key_frame;
1929  bool add_header;
1930  size_t length_code_size;
1931  size_t header_size = 0;
1932  size_t in_buf_size;
1933  size_t out_buf_size;
1934  size_t sei_nalu_size = 0;
1935  int64_t dts_delta;
1936  int64_t time_base_num;
1937  int nalu_count;
1938  CMTime pts;
1939  CMTime dts;
1940  CMVideoFormatDescriptionRef vid_fmt;
1941 
1942 
1943  vtenc_get_frame_info(sample_buffer, &is_key_frame);
1944  status = get_length_code_size(avctx, sample_buffer, &length_code_size);
1945  if (status) return status;
1946 
1947  add_header = is_key_frame && !(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER);
1948 
1949  if (add_header) {
1950  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
1951  if (!vid_fmt) {
1952  av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n");
1953  return AVERROR_EXTERNAL;
1954  }
1955 
1956  int status = get_params_size(avctx, vid_fmt, &header_size);
1957  if (status) return status;
1958  }
1959 
1960  status = count_nalus(length_code_size, sample_buffer, &nalu_count);
1961  if(status)
1962  return status;
1963 
1964  if (sei) {
1965  size_t msg_size = get_sei_msg_bytes(sei,
1967 
1968  sei_nalu_size = sizeof(start_code) + 1 + msg_size + 1;
1969  }
1970 
1971  in_buf_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1972  out_buf_size = header_size +
1973  in_buf_size +
1974  sei_nalu_size +
1975  nalu_count * ((int)sizeof(start_code) - (int)length_code_size);
1976 
1977  status = ff_alloc_packet2(avctx, pkt, out_buf_size, out_buf_size);
1978  if (status < 0)
1979  return status;
1980 
1981  if (add_header) {
1982  status = copy_param_sets(avctx, vid_fmt, pkt->data, out_buf_size);
1983  if(status) return status;
1984  }
1985 
1986  status = copy_replace_length_codes(
1987  avctx,
1988  length_code_size,
1989  sample_buffer,
1990  sei,
1991  pkt->data + header_size,
1992  pkt->size - header_size
1993  );
1994 
1995  if (status) {
1996  av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d\n", status);
1997  return status;
1998  }
1999 
2000  if (is_key_frame) {
2002  }
2003 
2004  pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer);
2005  dts = CMSampleBufferGetDecodeTimeStamp (sample_buffer);
2006 
2007  if (CMTIME_IS_INVALID(dts)) {
2008  if (!vtctx->has_b_frames) {
2009  dts = pts;
2010  } else {
2011  av_log(avctx, AV_LOG_ERROR, "DTS is invalid.\n");
2012  return AVERROR_EXTERNAL;
2013  }
2014  }
2015 
2016  dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0;
2017  time_base_num = avctx->time_base.num;
2018  pkt->pts = pts.value / time_base_num;
2019  pkt->dts = dts.value / time_base_num - dts_delta;
2020  pkt->size = out_buf_size;
2021 
2022  return 0;
2023 }
2024 
2025 /*
2026  * contiguous_buf_size is 0 if not contiguous, and the size of the buffer
2027  * containing all planes if so.
2028  */
2030  AVCodecContext *avctx,
2031  const AVFrame *frame,
2032  int *color,
2033  int *plane_count,
2034  size_t *widths,
2035  size_t *heights,
2036  size_t *strides,
2037  size_t *contiguous_buf_size)
2038 {
2039  VTEncContext *vtctx = avctx->priv_data;
2040  int av_format = frame->format;
2041  int av_color_range = frame->color_range;
2042  int i;
2043  int range_guessed;
2044  int status;
2045 
2046  status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed);
2047  if (status) {
2048  av_log(avctx,
2049  AV_LOG_ERROR,
2050  "Could not get pixel format for color format '%s' range '%s'.\n",
2051  av_get_pix_fmt_name(av_format),
2052  av_color_range > AVCOL_RANGE_UNSPECIFIED &&
2053  av_color_range < AVCOL_RANGE_NB ?
2054  av_color_range_name(av_color_range) :
2055  "Unknown");
2056 
2057  return AVERROR(EINVAL);
2058  }
2059 
2060  if (range_guessed) {
2061  if (!vtctx->warned_color_range) {
2062  vtctx->warned_color_range = true;
2063  av_log(avctx,
2065  "Color range not set for %s. Using MPEG range.\n",
2066  av_get_pix_fmt_name(av_format));
2067  }
2068  }
2069 
2070  switch (av_format) {
2071  case AV_PIX_FMT_NV12:
2072  *plane_count = 2;
2073 
2074  widths [0] = avctx->width;
2075  heights[0] = avctx->height;
2076  strides[0] = frame ? frame->linesize[0] : avctx->width;
2077 
2078  widths [1] = (avctx->width + 1) / 2;
2079  heights[1] = (avctx->height + 1) / 2;
2080  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) & -2;
2081  break;
2082 
2083  case AV_PIX_FMT_YUV420P:
2084  *plane_count = 3;
2085 
2086  widths [0] = avctx->width;
2087  heights[0] = avctx->height;
2088  strides[0] = frame ? frame->linesize[0] : avctx->width;
2089 
2090  widths [1] = (avctx->width + 1) / 2;
2091  heights[1] = (avctx->height + 1) / 2;
2092  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
2093 
2094  widths [2] = (avctx->width + 1) / 2;
2095  heights[2] = (avctx->height + 1) / 2;
2096  strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
2097  break;
2098 
2099  case AV_PIX_FMT_BGRA:
2100  *plane_count = 1;
2101 
2102  widths [0] = avctx->width;
2103  heights[0] = avctx->height;
2104  strides[0] = frame ? frame->linesize[0] : avctx->width * 4;
2105  break;
2106 
2107  case AV_PIX_FMT_P010LE:
2108  *plane_count = 2;
2109  widths[0] = avctx->width;
2110  heights[0] = avctx->height;
2111  strides[0] = frame ? frame->linesize[0] : (avctx->width * 2 + 63) & -64;
2112 
2113  widths[1] = (avctx->width + 1) / 2;
2114  heights[1] = (avctx->height + 1) / 2;
2115  strides[1] = frame ? frame->linesize[1] : ((avctx->width + 1) / 2 + 63) & -64;
2116  break;
2117 
2118  default:
2119  av_log(
2120  avctx,
2121  AV_LOG_ERROR,
2122  "Could not get frame format info for color %d range %d.\n",
2123  av_format,
2124  av_color_range);
2125 
2126  return AVERROR(EINVAL);
2127  }
2128 
2129  *contiguous_buf_size = 0;
2130  for (i = 0; i < *plane_count; i++) {
2131  if (i < *plane_count - 1 &&
2132  frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) {
2133  *contiguous_buf_size = 0;
2134  break;
2135  }
2136 
2137  *contiguous_buf_size += strides[i] * heights[i];
2138  }
2139 
2140  return 0;
2141 }
2142 
2143 //Not used on OSX - frame is never copied.
2145  const AVFrame *frame,
2146  CVPixelBufferRef cv_img,
2147  const size_t *plane_strides,
2148  const size_t *plane_rows)
2149 {
2150  int i, j;
2151  size_t plane_count;
2152  int status;
2153  int rows;
2154  int src_stride;
2155  int dst_stride;
2156  uint8_t *src_addr;
2157  uint8_t *dst_addr;
2158  size_t copy_bytes;
2159 
2160  status = CVPixelBufferLockBaseAddress(cv_img, 0);
2161  if (status) {
2162  av_log(
2163  avctx,
2164  AV_LOG_ERROR,
2165  "Error: Could not lock base address of CVPixelBuffer: %d.\n",
2166  status
2167  );
2168  }
2169 
2170  if (CVPixelBufferIsPlanar(cv_img)) {
2171  plane_count = CVPixelBufferGetPlaneCount(cv_img);
2172  for (i = 0; frame->data[i]; i++) {
2173  if (i == plane_count) {
2174  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2175  av_log(avctx,
2176  AV_LOG_ERROR,
2177  "Error: different number of planes in AVFrame and CVPixelBuffer.\n"
2178  );
2179 
2180  return AVERROR_EXTERNAL;
2181  }
2182 
2183  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
2184  src_addr = (uint8_t*)frame->data[i];
2185  dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
2186  src_stride = plane_strides[i];
2187  rows = plane_rows[i];
2188 
2189  if (dst_stride == src_stride) {
2190  memcpy(dst_addr, src_addr, src_stride * rows);
2191  } else {
2192  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2193 
2194  for (j = 0; j < rows; j++) {
2195  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2196  }
2197  }
2198  }
2199  } else {
2200  if (frame->data[1]) {
2201  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2202  av_log(avctx,
2203  AV_LOG_ERROR,
2204  "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"
2205  );
2206 
2207  return AVERROR_EXTERNAL;
2208  }
2209 
2210  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img);
2211  src_addr = (uint8_t*)frame->data[0];
2212  dst_stride = CVPixelBufferGetBytesPerRow(cv_img);
2213  src_stride = plane_strides[0];
2214  rows = plane_rows[0];
2215 
2216  if (dst_stride == src_stride) {
2217  memcpy(dst_addr, src_addr, src_stride * rows);
2218  } else {
2219  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2220 
2221  for (j = 0; j < rows; j++) {
2222  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2223  }
2224  }
2225  }
2226 
2227  status = CVPixelBufferUnlockBaseAddress(cv_img, 0);
2228  if (status) {
2229  av_log(avctx, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status);
2230  return AVERROR_EXTERNAL;
2231  }
2232 
2233  return 0;
2234 }
2235 
2237  const AVFrame *frame,
2238  CVPixelBufferRef *cv_img)
2239 {
2240  int plane_count;
2241  int color;
2242  size_t widths [AV_NUM_DATA_POINTERS];
2243  size_t heights[AV_NUM_DATA_POINTERS];
2244  size_t strides[AV_NUM_DATA_POINTERS];
2245  int status;
2246  size_t contiguous_buf_size;
2247  CVPixelBufferPoolRef pix_buf_pool;
2248  VTEncContext* vtctx = avctx->priv_data;
2249 
2250  if (avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX) {
2252 
2253  *cv_img = (CVPixelBufferRef)frame->data[3];
2254  av_assert0(*cv_img);
2255 
2256  CFRetain(*cv_img);
2257  return 0;
2258  }
2259 
2260  memset(widths, 0, sizeof(widths));
2261  memset(heights, 0, sizeof(heights));
2262  memset(strides, 0, sizeof(strides));
2263 
2264  status = get_cv_pixel_info(
2265  avctx,
2266  frame,
2267  &color,
2268  &plane_count,
2269  widths,
2270  heights,
2271  strides,
2272  &contiguous_buf_size
2273  );
2274 
2275  if (status) {
2276  av_log(
2277  avctx,
2278  AV_LOG_ERROR,
2279  "Error: Cannot convert format %d color_range %d: %d\n",
2280  frame->format,
2281  frame->color_range,
2282  status
2283  );
2284 
2285  return AVERROR_EXTERNAL;
2286  }
2287 
2288  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2289  if (!pix_buf_pool) {
2290  /* On iOS, the VT session is invalidated when the APP switches from
2291  * foreground to background and vice versa. Fetch the actual error code
2292  * of the VT session to detect that case and restart the VT session
2293  * accordingly. */
2294  OSStatus vtstatus;
2295 
2296  vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
2297  if (vtstatus == kVTInvalidSessionErr) {
2298  CFRelease(vtctx->session);
2299  vtctx->session = NULL;
2300  status = vtenc_configure_encoder(avctx);
2301  if (status == 0)
2302  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2303  }
2304  if (!pix_buf_pool) {
2305  av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
2306  return AVERROR_EXTERNAL;
2307  }
2308  else
2309  av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
2310  "kVTInvalidSessionErr error.\n");
2311  }
2312 
2313  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2314  pix_buf_pool,
2315  cv_img);
2316 
2317 
2318  if (status) {
2319  av_log(avctx, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status);
2320  return AVERROR_EXTERNAL;
2321  }
2322 
2323  status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights);
2324  if (status) {
2325  CFRelease(*cv_img);
2326  *cv_img = NULL;
2327  return status;
2328  }
2329 
2330  return 0;
2331 }
2332 
2334  CFDictionaryRef* dict_out)
2335 {
2336  CFDictionaryRef dict = NULL;
2337  if (frame->pict_type == AV_PICTURE_TYPE_I) {
2338  const void *keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
2339  const void *vals[] = { kCFBooleanTrue };
2340 
2341  dict = CFDictionaryCreate(NULL, keys, vals, 1, NULL, NULL);
2342  if(!dict) return AVERROR(ENOMEM);
2343  }
2344 
2345  *dict_out = dict;
2346  return 0;
2347 }
2348 
2350  VTEncContext *vtctx,
2351  const AVFrame *frame)
2352 {
2353  CMTime time;
2354  CFDictionaryRef frame_dict;
2355  CVPixelBufferRef cv_img = NULL;
2356  AVFrameSideData *side_data = NULL;
2357  ExtraSEI *sei = NULL;
2358  int status = create_cv_pixel_buffer(avctx, frame, &cv_img);
2359 
2360  if (status) return status;
2361 
2362  status = create_encoder_dict_h264(frame, &frame_dict);
2363  if (status) {
2364  CFRelease(cv_img);
2365  return status;
2366  }
2367 
2369  if (vtctx->a53_cc && side_data && side_data->size) {
2370  sei = av_mallocz(sizeof(*sei));
2371  if (!sei) {
2372  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2373  } else {
2374  int ret = ff_alloc_a53_sei(frame, 0, &sei->data, &sei->size);
2375  if (ret < 0) {
2376  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2377  av_free(sei);
2378  sei = NULL;
2379  }
2380  }
2381  }
2382 
2383  time = CMTimeMake(frame->pts * avctx->time_base.num, avctx->time_base.den);
2384  status = VTCompressionSessionEncodeFrame(
2385  vtctx->session,
2386  cv_img,
2387  time,
2388  kCMTimeInvalid,
2389  frame_dict,
2390  sei,
2391  NULL
2392  );
2393 
2394  if (frame_dict) CFRelease(frame_dict);
2395  CFRelease(cv_img);
2396 
2397  if (status) {
2398  av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n", status);
2399  return AVERROR_EXTERNAL;
2400  }
2401 
2402  return 0;
2403 }
2404 
2406  AVCodecContext *avctx,
2407  AVPacket *pkt,
2408  const AVFrame *frame,
2409  int *got_packet)
2410 {
2411  VTEncContext *vtctx = avctx->priv_data;
2412  bool get_frame;
2413  int status;
2414  CMSampleBufferRef buf = NULL;
2415  ExtraSEI *sei = NULL;
2416 
2417  if (frame) {
2418  status = vtenc_send_frame(avctx, vtctx, frame);
2419 
2420  if (status) {
2421  status = AVERROR_EXTERNAL;
2422  goto end_nopkt;
2423  }
2424 
2425  if (vtctx->frame_ct_in == 0) {
2426  vtctx->first_pts = frame->pts;
2427  } else if(vtctx->frame_ct_in == vtctx->has_b_frames) {
2428  vtctx->dts_delta = frame->pts - vtctx->first_pts;
2429  }
2430 
2431  vtctx->frame_ct_in++;
2432  } else if(!vtctx->flushing) {
2433  vtctx->flushing = true;
2434 
2435  status = VTCompressionSessionCompleteFrames(vtctx->session,
2436  kCMTimeIndefinite);
2437 
2438  if (status) {
2439  av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n", status);
2440  status = AVERROR_EXTERNAL;
2441  goto end_nopkt;
2442  }
2443  }
2444 
2445  *got_packet = 0;
2446  get_frame = vtctx->dts_delta >= 0 || !frame;
2447  if (!get_frame) {
2448  status = 0;
2449  goto end_nopkt;
2450  }
2451 
2452  status = vtenc_q_pop(vtctx, !frame, &buf, &sei);
2453  if (status) goto end_nopkt;
2454  if (!buf) goto end_nopkt;
2455 
2456  status = vtenc_cm_to_avpacket(avctx, buf, pkt, sei);
2457  if (sei) {
2458  if (sei->data) av_free(sei->data);
2459  av_free(sei);
2460  }
2461  CFRelease(buf);
2462  if (status) goto end_nopkt;
2463 
2464  *got_packet = 1;
2465  return 0;
2466 
2467 end_nopkt:
2469  return status;
2470 }
2471 
2473  CMVideoCodecType codec_type,
2474  CFStringRef profile_level,
2475  CFNumberRef gamma_level,
2476  CFDictionaryRef enc_info,
2477  CFDictionaryRef pixel_buffer_info)
2478 {
2479  VTEncContext *vtctx = avctx->priv_data;
2480  int status;
2481  CVPixelBufferPoolRef pool = NULL;
2482  CVPixelBufferRef pix_buf = NULL;
2483  CMTime time;
2484  CMSampleBufferRef buf = NULL;
2485 
2486  status = vtenc_create_encoder(avctx,
2487  codec_type,
2488  profile_level,
2489  gamma_level,
2490  enc_info,
2491  pixel_buffer_info,
2492  &vtctx->session);
2493  if (status)
2494  goto pe_cleanup;
2495 
2496  pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2497  if(!pool){
2498  av_log(avctx, AV_LOG_ERROR, "Error getting pixel buffer pool.\n");
2499  goto pe_cleanup;
2500  }
2501 
2502  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2503  pool,
2504  &pix_buf);
2505 
2506  if(status != kCVReturnSuccess){
2507  av_log(avctx, AV_LOG_ERROR, "Error creating frame from pool: %d\n", status);
2508  goto pe_cleanup;
2509  }
2510 
2511  time = CMTimeMake(0, avctx->time_base.den);
2512  status = VTCompressionSessionEncodeFrame(vtctx->session,
2513  pix_buf,
2514  time,
2515  kCMTimeInvalid,
2516  NULL,
2517  NULL,
2518  NULL);
2519 
2520  if (status) {
2521  av_log(avctx,
2522  AV_LOG_ERROR,
2523  "Error sending frame for extradata: %d\n",
2524  status);
2525 
2526  goto pe_cleanup;
2527  }
2528 
2529  //Populates extradata - output frames are flushed and param sets are available.
2530  status = VTCompressionSessionCompleteFrames(vtctx->session,
2531  kCMTimeIndefinite);
2532 
2533  if (status)
2534  goto pe_cleanup;
2535 
2536  status = vtenc_q_pop(vtctx, 0, &buf, NULL);
2537  if (status) {
2538  av_log(avctx, AV_LOG_ERROR, "popping: %d\n", status);
2539  goto pe_cleanup;
2540  }
2541 
2542  CFRelease(buf);
2543 
2544 
2545 
2546 pe_cleanup:
2547  if(vtctx->session)
2548  CFRelease(vtctx->session);
2549 
2550  vtctx->session = NULL;
2551  vtctx->frame_ct_out = 0;
2552 
2553  av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0));
2554 
2555  return status;
2556 }
2557 
2559 {
2560  VTEncContext *vtctx = avctx->priv_data;
2561 
2562  if(!vtctx->session) {
2564  pthread_mutex_destroy(&vtctx->lock);
2565  return 0;
2566  }
2567 
2568  VTCompressionSessionCompleteFrames(vtctx->session,
2569  kCMTimeIndefinite);
2570  clear_frame_queue(vtctx);
2572  pthread_mutex_destroy(&vtctx->lock);
2573  CFRelease(vtctx->session);
2574  vtctx->session = NULL;
2575 
2576  if (vtctx->color_primaries) {
2577  CFRelease(vtctx->color_primaries);
2578  vtctx->color_primaries = NULL;
2579  }
2580 
2581  if (vtctx->transfer_function) {
2582  CFRelease(vtctx->transfer_function);
2583  vtctx->transfer_function = NULL;
2584  }
2585 
2586  if (vtctx->ycbcr_matrix) {
2587  CFRelease(vtctx->ycbcr_matrix);
2588  vtctx->ycbcr_matrix = NULL;
2589  }
2590 
2591  return 0;
2592 }
2593 
2594 static const enum AVPixelFormat avc_pix_fmts[] = {
2599 };
2600 
2601 static const enum AVPixelFormat hevc_pix_fmts[] = {
2608 };
2609 
2610 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
2611 #define COMMON_OPTIONS \
2612  { "allow_sw", "Allow software encoding", OFFSET(allow_sw), AV_OPT_TYPE_BOOL, \
2613  { .i64 = 0 }, 0, 1, VE }, \
2614  { "require_sw", "Require software encoding", OFFSET(require_sw), AV_OPT_TYPE_BOOL, \
2615  { .i64 = 0 }, 0, 1, VE }, \
2616  { "realtime", "Hint that encoding should happen in real-time if not faster (e.g. capturing from camera).", \
2617  OFFSET(realtime), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2618  { "frames_before", "Other frames will come before the frames in this session. This helps smooth concatenation issues.", \
2619  OFFSET(frames_before), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2620  { "frames_after", "Other frames will come after the frames in this session. This helps smooth concatenation issues.", \
2621  OFFSET(frames_after), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
2622 
2623 #define OFFSET(x) offsetof(VTEncContext, x)
2624 static const AVOption h264_options[] = {
2625  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = H264_PROF_AUTO }, H264_PROF_AUTO, H264_PROF_COUNT, VE, "profile" },
2626  { "baseline", "Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_BASELINE }, INT_MIN, INT_MAX, VE, "profile" },
2627  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2628  { "high", "High Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_HIGH }, INT_MIN, INT_MAX, VE, "profile" },
2629  { "extended", "Extend Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_EXTENDED }, INT_MIN, INT_MAX, VE, "profile" },
2630 
2631  { "level", "Level", OFFSET(level), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 52, VE, "level" },
2632  { "1.3", "Level 1.3, only available with Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = 13 }, INT_MIN, INT_MAX, VE, "level" },
2633  { "3.0", "Level 3.0", 0, AV_OPT_TYPE_CONST, { .i64 = 30 }, INT_MIN, INT_MAX, VE, "level" },
2634  { "3.1", "Level 3.1", 0, AV_OPT_TYPE_CONST, { .i64 = 31 }, INT_MIN, INT_MAX, VE, "level" },
2635  { "3.2", "Level 3.2", 0, AV_OPT_TYPE_CONST, { .i64 = 32 }, INT_MIN, INT_MAX, VE, "level" },
2636  { "4.0", "Level 4.0", 0, AV_OPT_TYPE_CONST, { .i64 = 40 }, INT_MIN, INT_MAX, VE, "level" },
2637  { "4.1", "Level 4.1", 0, AV_OPT_TYPE_CONST, { .i64 = 41 }, INT_MIN, INT_MAX, VE, "level" },
2638  { "4.2", "Level 4.2", 0, AV_OPT_TYPE_CONST, { .i64 = 42 }, INT_MIN, INT_MAX, VE, "level" },
2639  { "5.0", "Level 5.0", 0, AV_OPT_TYPE_CONST, { .i64 = 50 }, INT_MIN, INT_MAX, VE, "level" },
2640  { "5.1", "Level 5.1", 0, AV_OPT_TYPE_CONST, { .i64 = 51 }, INT_MIN, INT_MAX, VE, "level" },
2641  { "5.2", "Level 5.2", 0, AV_OPT_TYPE_CONST, { .i64 = 52 }, INT_MIN, INT_MAX, VE, "level" },
2642 
2643  { "coder", "Entropy coding", OFFSET(entropy), AV_OPT_TYPE_INT, { .i64 = VT_ENTROPY_NOT_SET }, VT_ENTROPY_NOT_SET, VT_CABAC, VE, "coder" },
2644  { "cavlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2645  { "vlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2646  { "cabac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2647  { "ac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2648 
2649  { "a53cc", "Use A53 Closed Captions (if available)", OFFSET(a53_cc), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, VE },
2650 
2652  { NULL },
2653 };
2654 
2656  .class_name = "h264_videotoolbox",
2657  .item_name = av_default_item_name,
2658  .option = h264_options,
2659  .version = LIBAVUTIL_VERSION_INT,
2660 };
2661 
2663  .name = "h264_videotoolbox",
2664  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"),
2665  .type = AVMEDIA_TYPE_VIDEO,
2666  .id = AV_CODEC_ID_H264,
2667  .priv_data_size = sizeof(VTEncContext),
2669  .init = vtenc_init,
2670  .encode2 = vtenc_frame,
2671  .close = vtenc_close,
2672  .capabilities = AV_CODEC_CAP_DELAY,
2673  .priv_class = &h264_videotoolbox_class,
2674  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2676 };
2677 
2678 static const AVOption hevc_options[] = {
2679  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = HEVC_PROF_AUTO }, HEVC_PROF_AUTO, HEVC_PROF_COUNT, VE, "profile" },
2680  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2681  { "main10", "Main10 Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
2682 
2683  { "alpha_quality", "Compression quality for the alpha channel", OFFSET(alpha_quality), AV_OPT_TYPE_DOUBLE, { .dbl = 0.0 }, 0.0, 1.0, VE },
2684 
2686  { NULL },
2687 };
2688 
2690  .class_name = "hevc_videotoolbox",
2691  .item_name = av_default_item_name,
2692  .option = hevc_options,
2693  .version = LIBAVUTIL_VERSION_INT,
2694 };
2695 
2697  .name = "hevc_videotoolbox",
2698  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.265 Encoder"),
2699  .type = AVMEDIA_TYPE_VIDEO,
2700  .id = AV_CODEC_ID_HEVC,
2701  .priv_data_size = sizeof(VTEncContext),
2703  .init = vtenc_init,
2704  .encode2 = vtenc_frame,
2705  .close = vtenc_close,
2706  .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
2707  .priv_class = &hevc_videotoolbox_class,
2708  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2710  .wrapper_name = "videotoolbox",
2711 };
int ff_alloc_a53_sei(const AVFrame *frame, size_t prefix_len, void **data, size_t *sei_size)
Check AVFrame for A53 side data and allocate and fill SEI message with A53 info.
Definition: atsc_a53.c:25
#define av_cold
Definition: attributes.h:88
uint8_t
simple assert() macros that are a bit more flexible than ISO C assert().
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
Libavcodec external API header.
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:31
#define flags(name, subs,...)
Definition: cbs_av1.c:561
static int FUNC() sei(CodedBitstreamContext *ctx, RWContext *rw, H264RawSEI *current)
#define NULL
Definition: coverity.c:32
static AVFrame * frame
int ff_alloc_packet2(AVCodecContext *avctx, AVPacket *avpkt, int64_t size, int64_t min_size)
Check AVPacket size and/or allocate data.
Definition: encode.c:33
int
#define pthread_mutex_lock(a)
Definition: ffprobe.c:63
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:67
#define AV_NUM_DATA_POINTERS
Definition: frame.h:319
@ AV_OPT_TYPE_CONST
Definition: opt.h:234
@ AV_OPT_TYPE_INT
Definition: opt.h:225
@ AV_OPT_TYPE_DOUBLE
Definition: opt.h:227
@ AV_OPT_TYPE_BOOL
Definition: opt.h:242
#define AV_CODEC_CAP_DELAY
Encoder or decoder requires flushing with NULL input at the end in order to give the complete and cor...
Definition: codec.h:77
#define AV_CODEC_FLAG_QSCALE
Use fixed qscale.
Definition: avcodec.h:275
#define AV_CODEC_FLAG_GLOBAL_HEADER
Place global headers in extradata instead of every keyframe.
Definition: avcodec.h:329
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:46
#define AV_CODEC_CAP_HARDWARE
Codec is backed by a hardware implementation.
Definition: codec.h:157
@ AV_CODEC_ID_H264
Definition: codec_id.h:76
@ AV_CODEC_ID_HEVC
Definition: codec_id.h:223
#define AV_INPUT_BUFFER_PADDING_SIZE
Required number of additionally allocated bytes at the end of the input bitstream for decoding.
Definition: avcodec.h:215
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:634
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: packet.h:410
#define FF_QP2LAMBDA
factor to convert from H.263 QP to lambda
Definition: avutil.h:227
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
#define AVERROR(e)
Definition: error.h:43
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:738
@ AV_FRAME_DATA_A53_CC
ATSC A53 Part 4 Closed Captions.
Definition: frame.h:58
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
int av_reduce(int *dst_num, int *dst_den, int64_t num, int64_t den, int64_t max)
Reduce a fraction.
Definition: rational.c:35
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:237
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
H.264 common definitions.
@ H264_NAL_PPS
Definition: h264.h:42
@ H264_NAL_SPS
Definition: h264.h:41
@ H264_NAL_SEI
Definition: h264.h:40
@ H264_NAL_AUD
Definition: h264.h:43
cl_device_type type
int i
Definition: input.c:407
#define FF_CODEC_CAP_INIT_THREADSAFE
The codec does not modify any global variables in the init function, allowing to call the init functi...
Definition: internal.h:41
#define FF_CODEC_CAP_INIT_CLEANUP
The codec allows calling the close function for deallocation even if the init function returned a fai...
Definition: internal.h:49
common internal API header
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:117
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:303
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
AVOptions.
static av_always_inline int pthread_cond_signal(pthread_cond_t *cond)
Definition: os2threads.h:152
static av_always_inline int pthread_cond_destroy(pthread_cond_t *cond)
Definition: os2threads.h:144
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
#define PTHREAD_ONCE_INIT
Definition: os2threads.h:71
static av_always_inline int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
Definition: os2threads.h:133
static av_always_inline int pthread_once(pthread_once_t *once_control, void(*init_routine)(void))
Definition: os2threads.h:210
_fmutex pthread_mutex_t
Definition: os2threads.h:53
static av_always_inline int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
Definition: os2threads.h:192
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
const char * av_color_transfer_name(enum AVColorTransferCharacteristic transfer)
Definition: pixdesc.c:2940
const char * av_color_range_name(enum AVColorRange range)
Definition: pixdesc.c:2901
const char * av_color_space_name(enum AVColorSpace space)
Definition: pixdesc.c:2961
const char * av_color_primaries_name(enum AVColorPrimaries primaries)
Definition: pixdesc.c:2919
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
AVColorRange
Visual content value range.
Definition: pixfmt.h:551
@ AVCOL_RANGE_MPEG
Narrow or limited range content.
Definition: pixfmt.h:569
@ AVCOL_RANGE_UNSPECIFIED
Definition: pixfmt.h:552
@ AVCOL_RANGE_NB
Not part of ABI.
Definition: pixfmt.h:587
@ AVCOL_RANGE_JPEG
Full range content.
Definition: pixfmt.h:586
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
@ AV_PIX_FMT_P010LE
like NV12, with 10bpp per component, data in the high bits, zeros in the low bits,...
Definition: pixfmt.h:284
@ AV_PIX_FMT_VIDEOTOOLBOX
hardware decoding through Videotoolbox
Definition: pixfmt.h:282
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
@ AV_PIX_FMT_BGRA
packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
Definition: pixfmt.h:95
AVColorPrimaries
Chromaticity coordinates of the source primaries.
Definition: pixfmt.h:458
@ AVCOL_PRI_BT470BG
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:465
@ AVCOL_PRI_BT709
also ITU-R BT1361 / IEC 61966-2-4 / SMPTE RP177 Annex B
Definition: pixfmt.h:460
@ AVCOL_PRI_UNSPECIFIED
Definition: pixfmt.h:461
@ AVCOL_PRI_SMPTE170M
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:466
@ AVCOL_PRI_BT2020
ITU-R BT2020.
Definition: pixfmt.h:469
AVColorTransferCharacteristic
Color Transfer Characteristic.
Definition: pixfmt.h:483
@ AVCOL_TRC_SMPTE2084
SMPTE ST 2084 for 10-, 12-, 14- and 16-bit systems.
Definition: pixfmt.h:500
@ AVCOL_TRC_GAMMA22
also ITU-R BT470M / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:488
@ AVCOL_TRC_SMPTE240M
Definition: pixfmt.h:491
@ AVCOL_TRC_LINEAR
"Linear transfer characteristics"
Definition: pixfmt.h:492
@ AVCOL_TRC_GAMMA28
also ITU-R BT470BG
Definition: pixfmt.h:489
@ AVCOL_TRC_ARIB_STD_B67
ARIB STD-B67, known as "Hybrid log-gamma".
Definition: pixfmt.h:504
@ AVCOL_TRC_BT2020_12
ITU-R BT2020 for 12-bit system.
Definition: pixfmt.h:499
@ AVCOL_TRC_BT2020_10
ITU-R BT2020 for 10-bit system.
Definition: pixfmt.h:498
@ AVCOL_TRC_UNSPECIFIED
Definition: pixfmt.h:486
@ AVCOL_TRC_BT709
also ITU-R BT1361
Definition: pixfmt.h:485
@ AVCOL_SPC_BT709
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:514
@ AVCOL_SPC_BT470BG
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601
Definition: pixfmt.h:518
@ AVCOL_SPC_BT2020_NCL
ITU-R BT2020 non-constant luminance system.
Definition: pixfmt.h:523
@ AVCOL_SPC_UNSPECIFIED
Definition: pixfmt.h:515
@ AVCOL_SPC_SMPTE170M
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:519
@ AVCOL_SPC_SMPTE240M
functionally identical to above
Definition: pixfmt.h:520
mfxU16 profile
Definition: qsvenc.c:45
enum AVMediaType codec_type
Definition: rtp.c:37
static char buffer[20]
Definition: seek.c:32
@ SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35
Definition: sei.h:34
Describe the class of an AVClass context structure.
Definition: log.h:67
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
main external API structure.
Definition: avcodec.h:536
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:746
int width
picture width / height.
Definition: avcodec.h:709
int global_quality
Global quality for codecs which cannot change it per frame.
Definition: avcodec.h:602
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: avcodec.h:1171
enum AVColorPrimaries color_primaries
Chromaticity coordinates of the source primaries.
Definition: avcodec.h:1150
int max_b_frames
maximum number of B-frames between non-B-frames Note: The output will be delayed by max_b_frames+1 re...
Definition: avcodec.h:796
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel.
Definition: avcodec.h:915
int has_b_frames
Size of the frame reordering buffer in the decoder.
Definition: avcodec.h:826
int64_t bit_rate
the average bitrate
Definition: avcodec.h:586
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:1164
int gop_size
the number of pictures in a group of pictures, or 0 for intra_only
Definition: avcodec.h:731
int64_t rc_max_rate
maximum bitrate
Definition: avcodec.h:1416
enum AVColorTransferCharacteristic color_trc
Color Transfer Characteristic.
Definition: avcodec.h:1157
AVRational time_base
This is the fundamental unit of time (in seconds) in terms of which frame timestamps are represented.
Definition: avcodec.h:659
int flags
AV_CODEC_FLAG_*.
Definition: avcodec.h:616
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:637
enum AVCodecID codec_id
Definition: avcodec.h:546
int extradata_size
Definition: avcodec.h:638
void * priv_data
Definition: avcodec.h:563
AVCodec.
Definition: codec.h:197
const char * name
Name of the codec implementation.
Definition: codec.h:204
Structure to hold side data for an AVFrame.
Definition: frame.h:220
This structure describes decoded (raw) audio or video data.
Definition: frame.h:318
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:411
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:332
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: frame.h:562
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:349
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:391
enum AVPictureType pict_type
Picture type of the frame.
Definition: frame.h:401
AVOption.
Definition: opt.h:248
This structure stores compressed data.
Definition: packet.h:346
int flags
A combination of AV_PKT_FLAG values.
Definition: packet.h:375
int size
Definition: packet.h:370
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: packet.h:362
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed.
Definition: packet.h:368
uint8_t * data
Definition: packet.h:369
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int num
Numerator.
Definition: rational.h:59
int den
Denominator.
Definition: rational.h:60
CMSampleBufferRef cm_buffer
struct BufNode * next
ExtraSEI * sei
int64_t frames_before
pthread_cond_t cv_sample_sent
CFStringRef transfer_function
int64_t frame_ct_out
BufNode * q_tail
double alpha_quality
int64_t frames_after
getParameterSetAtIndex get_param_set_func
CFStringRef ycbcr_matrix
enum AVCodecID codec_id
int64_t require_sw
pthread_mutex_t lock
BufNode * q_head
VTCompressionSessionRef session
CFStringRef color_primaries
int64_t frame_ct_in
uint8_t level
Definition: svq3.c:206
#define av_free(p)
#define av_malloc(s)
#define av_log(a,...)
#define src
Definition: vp8dsp.c:255
static int16_t block[64]
Definition: dct.c:116
AVPacket * pkt
Definition: movenc.c:59
AVFormatContext * ctx
Definition: movenc.c:48
static int64_t pts
int size
static int get_frame(AVFilterContext *ctx, int is_second)
Definition: vf_nnedi.c:666
static const uint8_t offset[127][2]
Definition: vf_spp.c:107
static int write_sei(const ExtraSEI *sei, int sei_type, uint8_t *dst, size_t dst_size)
static const AVClass h264_videotoolbox_class
CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder
static av_cold int vtenc_init(AVCodecContext *avctx)
CFStringRef kVTProfileLevel_H264_High_4_0
static int create_encoder_dict_h264(const AVFrame *frame, CFDictionaryRef *dict_out)
CFStringRef kVTProfileLevel_H264_Main_5_2
@ kCMVideoCodecType_HEVC
static int vtenc_create_encoder(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info, VTCompressionSessionRef *session)
static int create_cv_pixel_buffer_info(AVCodecContext *avctx, CFMutableDictionaryRef *dict)
static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
static CMVideoCodecType get_cm_codec_type(enum AVCodecID id, enum AVPixelFormat fmt, double alpha_quality)
CFStringRef kCVImageBufferYCbCrMatrix_ITU_R_2020
static int is_post_sei_nal_type(int nal_type)
CFStringRef kVTProfileLevel_H264_High_3_0
CFStringRef kVTProfileLevel_H264_Main_5_1
CFStringRef kVTProfileLevel_H264_Baseline_4_2
static void vtenc_output_callback(void *ctx, void *sourceFrameCtx, OSStatus status, VTEncodeInfoFlags flags, CMSampleBufferRef sample_buffer)
CFStringRef kVTProfileLevel_H264_High_4_1
static void set_async_error(VTEncContext *vtctx, int err)
static av_cold int vtenc_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet)
getParameterSetAtIndex CMVideoFormatDescriptionGetHEVCParameterSetAtIndex
CFStringRef kVTCompressionPropertyKey_RealTime
#define VE
CFStringRef kVTProfileLevel_H264_Baseline_5_1
VT_HEVCProfile
@ HEVC_PROF_AUTO
@ HEVC_PROF_MAIN
@ HEVC_PROF_MAIN10
@ HEVC_PROF_COUNT
CFStringRef kVTProfileLevel_H264_High_3_1
#define COMMON_OPTIONS
static int copy_emulation_prev(const uint8_t *src, size_t src_size, uint8_t *dst, ssize_t dst_offset, size_t dst_size)
Copies the data inserting emulation prevention bytes as needed.
@ kCMVideoCodecType_HEVCWithAlpha
static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict)
static void clear_frame_queue(VTEncContext *vtctx)
CFStringRef kVTH264EntropyMode_CAVLC
static pthread_once_t once_ctrl
@ kCVPixelFormatType_420YpCbCr10BiPlanarFullRange
static bool get_vt_h264_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
static int vtenc_send_frame(AVCodecContext *avctx, VTEncContext *vtctx, const AVFrame *frame)
static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef cv_img, const size_t *plane_strides, const size_t *plane_rows)
static const AVOption h264_options[]
CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder
static int get_cv_transfer_function(AVCodecContext *avctx, CFStringRef *transfer_fnc, CFNumberRef *gamma_level)
CFStringRef kVTProfileLevel_H264_Baseline_5_2
static void vt_release_num(CFNumberRef *refPtr)
NULL-safe release of *refPtr, and sets value to NULL.
static const AVOption hevc_options[]
static enum AVPixelFormat hevc_pix_fmts[]
static bool get_vt_hevc_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
static enum AVPixelFormat avc_pix_fmts[]
static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
static int vtenc_cm_to_avpacket(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, AVPacket *pkt, ExtraSEI *sei)
CFStringRef kVTProfileLevel_H264_High_4_2
CFStringRef kVTProfileLevel_H264_High_3_2
VT_H264Profile
@ H264_PROF_EXTENDED
@ H264_PROF_COUNT
@ H264_PROF_AUTO
@ H264_PROF_HIGH
@ H264_PROF_BASELINE
@ H264_PROF_MAIN
OSStatus(* getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc, size_t parameterSetIndex, const uint8_t **parameterSetPointerOut, size_t *parameterSetSizeOut, size_t *parameterSetCountOut, int *NALUnitHeaderLengthOut)
static int get_cv_color_primaries(AVCodecContext *avctx, CFStringRef *primaries)
static struct @163 compat_keys
static int find_sei_end(AVCodecContext *avctx, uint8_t *nal_data, size_t nal_size, uint8_t **sei_end)
CFStringRef kVTProfileLevel_H264_Baseline_4_0
CFStringRef kVTProfileLevel_H264_Extended_AutoLevel
CFStringRef kCVImageBufferColorPrimaries_ITU_R_2020
VTH264Entropy
@ VT_CAVLC
@ VT_ENTROPY_NOT_SET
@ VT_CABAC
static const uint8_t start_code[]
AVCodec ff_h264_videotoolbox_encoder
static int copy_param_sets(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, uint8_t *dst, size_t dst_size)
static bool vtenc_qscale_enabled(void)
CFStringRef kVTProfileLevel_HEVC_Main_AutoLevel
CFStringRef kCVImageBufferTransferFunction_ITU_R_2020
static void loadVTEncSymbols()
static int count_nalus(size_t length_code_size, CMSampleBufferRef sample_buffer, int *count)
CFStringRef kVTProfileLevel_H264_High_AutoLevel
static int get_length_code_size(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, size_t *size)
static av_cold int vtenc_close(AVCodecContext *avctx)
CFStringRef kVTCompressionPropertyKey_H264EntropyMode
static int vtenc_populate_extradata(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info)
CFStringRef kVTH264EntropyMode_CABAC
CFStringRef kVTProfileLevel_H264_Main_AutoLevel
static const AVClass hevc_videotoolbox_class
static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
AVCodec ff_hevc_videotoolbox_encoder
#define OFFSET(x)
CFStringRef kVTCompressionPropertyKey_TargetQualityForAlpha
static int copy_replace_length_codes(AVCodecContext *avctx, size_t length_code_size, CMSampleBufferRef sample_buffer, ExtraSEI *sei, uint8_t *dst_data, size_t dst_size)
Copies NAL units and replaces length codes with H.264 Annex B start codes.
static int get_cv_pixel_info(AVCodecContext *avctx, const AVFrame *frame, int *color, int *plane_count, size_t *widths, size_t *heights, size_t *strides, size_t *contiguous_buf_size)
static int get_cv_pixel_format(AVCodecContext *avctx, enum AVPixelFormat fmt, enum AVColorRange range, int *av_pixel_format, int *range_guessed)
CFStringRef kVTProfileLevel_H264_High_5_2
CFStringRef kVTProfileLevel_H264_High_5_1
static int create_cv_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef *cv_img)
static int get_sei_msg_bytes(const ExtraSEI *sei, int type)
Returns a sufficient number of bytes to contain the sei data.
#define GET_SYM(symbol, defaultVal)
static int get_params_size(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, size_t *size)
Get the parameter sets from a CMSampleBufferRef.
CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel
static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix)
CFStringRef kVTProfileLevel_H264_Baseline_5_0
CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel
static int vtenc_configure_encoder(AVCodecContext *avctx)
CFStringRef kVTProfileLevel_H264_Main_4_2
@ kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
CFStringRef kVTProfileLevel_H264_Extended_5_0
int len