Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix VT tone mapping and add transpose_vt #352

Merged
merged 5 commits into from
Mar 9, 2024
Merged

fix VT tone mapping and add transpose_vt #352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
588959a
fix: force nv12 when doing tone mapping
gnattu Mar 9, 2024
990a4ea
feat: backport transpose_vt
gnattu Mar 9, 2024
8ee9707
fix: add transpose_vt to series
gnattu Mar 9, 2024
ac2e812
feat: add format option for scale_vt
gnattu Mar 9, 2024
2e3649a
fix: relax format check for output
gnattu Mar 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions debian/patches/0061-backport-scale-vt.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ diff --git a/libavfilter/vf_scale_vt.c b/libavfilter/vf_scale_vt.c
new file mode 100644
--- /dev/null (revision c2c96c4c2419859c4d1b11e6f907e58afb6dfa3c)
+++ b/libavfilter/vf_scale_vt.c (revision c2c96c4c2419859c4d1b11e6f907e58afb6dfa3c)
@@ -0,0 +1,268 @@
@@ -0,0 +1,293 @@
+/*
+ * Copyright (c) 2023 Zhao Zhili <zhilizhao@tencent.com>
+ *
Expand Down Expand Up @@ -132,11 +132,26 @@ new file mode 100644
+ enum AVColorPrimaries colour_primaries;
+ enum AVColorTransferCharacteristic colour_transfer;
+ enum AVColorSpace colour_matrix;
+ enum AVPixelFormat format;
+ char *colour_primaries_string;
+ char *colour_transfer_string;
+ char *colour_matrix_string;
+} ScaleVtContext;
+
+static const enum AVPixelFormat supported_formats[] = {
+ AV_PIX_FMT_NV12,
+ AV_PIX_FMT_P010,
+ AV_PIX_FMT_NONE,
+};
+
+static int format_is_supported(enum AVPixelFormat fmt)
+{
+ for (int i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++)
+ if (supported_formats[i] == fmt)
+ return 1;
+ return 0;
+}
+
+static av_cold int scale_vt_init(AVFilterContext *avctx)
+{
+ ScaleVtContext *s = avctx->priv;
Expand Down Expand Up @@ -271,6 +286,7 @@ new file mode 100644
+ AVFilterLink *inlink = outlink->src->inputs[0];
+ AVHWFramesContext *hw_frame_ctx_in;
+ AVHWFramesContext *hw_frame_ctx_out;
+ enum AVPixelFormat out_format;
+
+ err = ff_scale_eval_dimensions(s, s->w_expr, s->h_expr, inlink, outlink,
+ &s->output_width,
Expand All @@ -290,11 +306,18 @@ new file mode 100644
+
+ hw_frame_ctx_in = (AVHWFramesContext *)inlink->hw_frames_ctx->data;
+
+ out_format = (s->format == AV_PIX_FMT_NONE) ? hw_frame_ctx_in->sw_format : s->format;
+ if (!format_is_supported(s->format)) {
+ av_log(s, AV_LOG_ERROR, "Unsupported output format: %s\n",
+ av_get_pix_fmt_name(out_format));
+ return AVERROR(ENOSYS);
+ }
+
+ av_buffer_unref(&outlink->hw_frames_ctx);
+ outlink->hw_frames_ctx = av_hwframe_ctx_alloc(hw_frame_ctx_in->device_ref);
+ hw_frame_ctx_out = (AVHWFramesContext *)outlink->hw_frames_ctx->data;
+ hw_frame_ctx_out->format = AV_PIX_FMT_VIDEOTOOLBOX;
+ hw_frame_ctx_out->sw_format = hw_frame_ctx_in->sw_format;
+ hw_frame_ctx_out->sw_format = out_format;
+ hw_frame_ctx_out->width = outlink->w;
+ hw_frame_ctx_out->height = outlink->h;
+
Expand Down Expand Up @@ -326,6 +349,8 @@ new file mode 100644
+ OFFSET(colour_primaries_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
+ { "color_transfer", "Output colour transfer characteristics",
+ OFFSET(colour_transfer_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
+ { "format", "Output pixel format",
+ OFFSET(format), AV_OPT_TYPE_PIXEL_FMT, { .i64 = AV_PIX_FMT_NONE }, AV_PIX_FMT_NONE, INT_MAX, FLAGS, "fmt" },
+ { NULL },
+};
+
Expand Down
307 changes: 307 additions & 0 deletions debian/patches/0067-backport-transpose_vt.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,307 @@
diff --git a/configure b/configure
index 3cd3bdfb44..7bf621590e 100755
--- a/configure
+++ b/configure
@@ -3751,6 +3751,7 @@ tonemap_opencl_filter_deps="opencl const_nan"
transpose_opencl_filter_deps="opencl"
transpose_vaapi_filter_deps="vaapi VAProcPipelineCaps_rotation_flags"
transpose_vulkan_filter_deps="vulkan spirv_compiler"
+transpose_vt_filter_deps="coreimage videotoolbox"
unsharp_opencl_filter_deps="opencl"
uspp_filter_deps="gpl avcodec"
vaguedenoiser_filter_deps="gpl"
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index b3d3d981dd..f4ef8cd062 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -516,6 +516,7 @@ OBJS-$(CONFIG_TRANSPOSE_NPP_FILTER) += vf_transpose_npp.o
OBJS-$(CONFIG_TRANSPOSE_OPENCL_FILTER) += vf_transpose_opencl.o opencl.o opencl/transpose.o
OBJS-$(CONFIG_TRANSPOSE_VAAPI_FILTER) += vf_transpose_vaapi.o vaapi_vpp.o
OBJS-$(CONFIG_TRANSPOSE_VULKAN_FILTER) += vf_transpose_vulkan.o vulkan.o vulkan_filter.o
+OBJS-$(CONFIG_TRANSPOSE_VT_FILTER) += vf_transpose_vt.o
OBJS-$(CONFIG_TRIM_FILTER) += trim.o
OBJS-$(CONFIG_UNPREMULTIPLY_FILTER) += vf_premultiply.o framesync.o
OBJS-$(CONFIG_UNSHARP_FILTER) += vf_unsharp.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index d7db46c2af..4d95949027 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -486,6 +486,7 @@ extern const AVFilter ff_vf_transpose_npp;
extern const AVFilter ff_vf_transpose_opencl;
extern const AVFilter ff_vf_transpose_vaapi;
extern const AVFilter ff_vf_transpose_vulkan;
+extern const AVFilter ff_vf_transpose_vt;
extern const AVFilter ff_vf_trim;
extern const AVFilter ff_vf_unpremultiply;
extern const AVFilter ff_vf_unsharp;
diff --git a/libavfilter/vf_transpose_vt.m b/libavfilter/vf_transpose_vt.m
new file mode 100644
index 0000000000..bc3b727a10
--- /dev/null
+++ b/libavfilter/vf_transpose_vt.m
@@ -0,0 +1,265 @@
+/*
+ * Copyright (c) 2024 Gnattu OC <gnattuoc@me.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <CoreImage/CoreImage.h>
+
+#include "libavutil/hwcontext.h"
+#include "libavutil/opt.h"
+#include "internal.h"
+#include "transpose.h"
+#include "video.h"
+
+typedef struct TransposeVtContext {
+ AVClass *class;
+ CIContext *ci_ctx;
+ CGImagePropertyOrientation orientation;
+
+ int dir;
+ int passthrough;
+} TransposeVtContext;
+
+static av_cold int transpose_vt_init(AVFilterContext *avctx)
+{
+ TransposeVtContext *s = avctx->priv;
+ s->ci_ctx = CFBridgingRetain([CIContext context]);
+ if (!s->ci_ctx) {
+ av_log(avctx, AV_LOG_ERROR, "CoreImage Context create failed\n");
+ return AVERROR_EXTERNAL;
+ }
+
+ return 0;
+}
+
+static av_cold void transpose_vt_uninit(AVFilterContext *avctx)
+{
+ TransposeVtContext *s = avctx->priv;
+ if (s->ci_ctx) {
+ CFRelease(s->ci_ctx);
+ s->ci_ctx = NULL;
+ }
+}
+
+static int transpose_vt_filter_frame(AVFilterLink *link, AVFrame *in)
+{
+ int ret;
+ AVFilterContext *ctx = link->dst;
+ TransposeVtContext *s = ctx->priv;
+ AVFilterLink *outlink = ctx->outputs[0];
+ CVPixelBufferRef src;
+ CVPixelBufferRef dst;
+ AVFrame *out;
+ CIImage *source_image = NULL;
+ CIImage *transposed_image = NULL;
+
+ if (s->passthrough)
+ return ff_filter_frame(outlink, in);
+
+ out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+ if (!out) {
+ ret = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ ret = av_frame_copy_props(out, in);
+ if (ret < 0)
+ goto fail;
+
+ src = (CVPixelBufferRef)in->data[3];
+ dst = (CVPixelBufferRef)out->data[3];
+
+ source_image = CFBridgingRetain([CIImage imageWithCVPixelBuffer: src]);
+ transposed_image = CFBridgingRetain([source_image imageByApplyingCGOrientation: s->orientation]);
+ if (!transposed_image) {
+ CFRelease(source_image);
+ av_log(ctx, AV_LOG_ERROR, "transpose image failed, %d\n", ret);
+ ret = AVERROR_EXTERNAL;
+ goto fail;
+ }
+ [(__bridge CIContext*)s->ci_ctx render: (__bridge CIImage*)transposed_image toCVPixelBuffer: dst];
+ CFRelease(source_image);
+ CFRelease(transposed_image);
+ CVBufferPropagateAttachments(src, dst);
+
+ av_frame_free(&in);
+
+ return ff_filter_frame(outlink, out);
+
+ fail:
+ av_frame_free(&in);
+ av_frame_free(&out);
+ return ret;
+}
+
+static int transpose_vt_recreate_hw_ctx(AVFilterLink *outlink)
+{
+ AVFilterContext *avctx = outlink->src;
+ AVFilterLink *inlink = outlink->src->inputs[0];
+ AVHWFramesContext *hw_frame_ctx_in;
+ AVHWFramesContext *hw_frame_ctx_out;
+ int err;
+
+ av_buffer_unref(&outlink->hw_frames_ctx);
+
+ hw_frame_ctx_in = (AVHWFramesContext *)inlink->hw_frames_ctx->data;
+ outlink->hw_frames_ctx = av_hwframe_ctx_alloc(hw_frame_ctx_in->device_ref);
+ hw_frame_ctx_out = (AVHWFramesContext *)outlink->hw_frames_ctx->data;
+ hw_frame_ctx_out->format = AV_PIX_FMT_VIDEOTOOLBOX;
+ hw_frame_ctx_out->sw_format = hw_frame_ctx_in->sw_format;
+ hw_frame_ctx_out->width = outlink->w;
+ hw_frame_ctx_out->height = outlink->h;
+
+ err = ff_filter_init_hw_frames(avctx, outlink, 1);
+ if (err < 0)
+ return err;
+
+ err = av_hwframe_ctx_init(outlink->hw_frames_ctx);
+ if (err < 0) {
+ av_log(avctx, AV_LOG_ERROR,
+ "Failed to init videotoolbox frame context, %s\n",
+ av_err2str(err));
+ return err;
+ }
+
+ return 0;
+}
+
+static int transpose_vt_config_output(AVFilterLink *outlink)
+{
+ int err;
+ AVFilterContext *avctx = outlink->src;
+ TransposeVtContext *s = avctx->priv;
+ AVFilterLink *inlink = outlink->src->inputs[0];
+ int swap_w_h = 0;
+
+ av_buffer_unref(&outlink->hw_frames_ctx);
+ outlink->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
+
+ if ((inlink->w >= inlink->h && s->passthrough == TRANSPOSE_PT_TYPE_LANDSCAPE) ||
+ (inlink->w <= inlink->h && s->passthrough == TRANSPOSE_PT_TYPE_PORTRAIT)) {
+ av_log(avctx, AV_LOG_VERBOSE,
+ "w:%d h:%d -> w:%d h:%d (passthrough mode)\n",
+ inlink->w, inlink->h, inlink->w, inlink->h);
+ s->orientation = kCGImagePropertyOrientationUp;
+ return 0;
+ }
+
+ s->passthrough = TRANSPOSE_PT_TYPE_NONE;
+
+ switch (s->dir) {
+ case TRANSPOSE_CCLOCK_FLIP:
+ s->orientation = kCGImagePropertyOrientationLeftMirrored;
+ swap_w_h = 1;
+ break;
+ case TRANSPOSE_CCLOCK:
+ s->orientation = kCGImagePropertyOrientationLeft;
+ swap_w_h = 1;
+ break;
+ case TRANSPOSE_CLOCK:
+ s->orientation = kCGImagePropertyOrientationRight;
+ swap_w_h = 1;
+ break;
+ case TRANSPOSE_CLOCK_FLIP:
+ s->orientation = kCGImagePropertyOrientationRightMirrored;
+ swap_w_h = 1;
+ break;
+ case TRANSPOSE_REVERSAL:
+ s->orientation = kCGImagePropertyOrientationDown;
+ break;
+ case TRANSPOSE_HFLIP:
+ s->orientation = kCGImagePropertyOrientationUpMirrored;
+ break;
+ case TRANSPOSE_VFLIP:
+ s->orientation = kCGImagePropertyOrientationDownMirrored;
+ break;
+ default:
+ av_log(avctx, AV_LOG_ERROR, "Failed to set direction to %d\n", s->dir);
+ return AVERROR(EINVAL);
+ }
+
+ if (!swap_w_h)
+ return 0;
+
+ outlink->w = inlink->h;
+ outlink->h = inlink->w;
+ return transpose_vt_recreate_hw_ctx(outlink);
+}
+
+#define OFFSET(x) offsetof(TransposeVtContext, x)
+#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
+static const AVOption transpose_vt_options[] = {
+ { "dir", "set transpose direction",
+ OFFSET(dir), AV_OPT_TYPE_INT, { .i64 = TRANSPOSE_CCLOCK_FLIP }, 0, 6, FLAGS, .unit = "dir" },
+ { "cclock_flip", "rotate counter-clockwise with vertical flip",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CCLOCK_FLIP }, .flags=FLAGS, .unit = "dir" },
+ { "clock", "rotate clockwise",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CLOCK }, .flags=FLAGS, .unit = "dir" },
+ { "cclock", "rotate counter-clockwise",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CCLOCK }, .flags=FLAGS, .unit = "dir" },
+ { "clock_flip", "rotate clockwise with vertical flip",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CLOCK_FLIP }, .flags=FLAGS, .unit = "dir" },
+ { "reversal", "rotate by half-turn",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_REVERSAL }, .flags=FLAGS, .unit = "dir" },
+ { "hflip", "flip horizontally",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_HFLIP }, .flags=FLAGS, .unit = "dir" },
+ { "vflip", "flip vertically",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_VFLIP }, .flags=FLAGS, .unit = "dir" },
+
+ { "passthrough", "do not apply transposition if the input matches the specified geometry",
+ OFFSET(passthrough), AV_OPT_TYPE_INT, { .i64=TRANSPOSE_PT_TYPE_NONE }, 0, INT_MAX, FLAGS, .unit = "passthrough" },
+ { "none", "always apply transposition",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_PT_TYPE_NONE }, INT_MIN, INT_MAX, FLAGS, .unit = "passthrough" },
+ { "portrait", "preserve portrait geometry",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_PT_TYPE_PORTRAIT }, INT_MIN, INT_MAX, FLAGS, .unit = "passthrough" },
+ { "landscape", "preserve landscape geometry",
+ 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_PT_TYPE_LANDSCAPE }, INT_MIN, INT_MAX, FLAGS, .unit = "passthrough" },
+
+ { NULL }
+};
+
+AVFILTER_DEFINE_CLASS(transpose_vt);
+
+static const AVFilterPad transpose_vt_inputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .filter_frame = &transpose_vt_filter_frame,
+ },
+};
+
+static const AVFilterPad transpose_vt_outputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = &transpose_vt_config_output,
+ },
+};
+
+const AVFilter ff_vf_transpose_vt = {
+ .name = "transpose_vt",
+ .description = NULL_IF_CONFIG_SMALL("Transpose Videotoolbox frames"),
+ .priv_size = sizeof(TransposeVtContext),
+ .init = transpose_vt_init,
+ .uninit = transpose_vt_uninit,
+ FILTER_INPUTS(transpose_vt_inputs),
+ FILTER_OUTPUTS(transpose_vt_outputs),
+ FILTER_SINGLE_PIXFMT(AV_PIX_FMT_VIDEOTOOLBOX),
+ .priv_class = &transpose_vt_class,
+ .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
+};
1 change: 1 addition & 0 deletions debian/patches/series
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,4 @@
0064-backport-enhanced-vt.patch
0065-add-cuda-transpose-filter-impl.patch
0066-add-flip-feat-to-opencl-transpose-filter.patch
0067-backport-transpose_vt.patch