libavfilter/vf_framepack.c

   1 /*
   2  * Copyright (c) 2013 Vittorio Giovara
   3  *
   4  * This file is part of Libav.
   5  *
   6  * Libav is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * Libav is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with Libav; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 /**
  22  * @file
  23  * Generate a frame packed video, by combining two views in a single surface.
  24  */
  25
  26 #include <string.h>
  27
  28 #include "libavutil/imgutils.h"
  29 #include "libavutil/opt.h"
  30 #include "libavutil/pixdesc.h"
  31 #include "libavutil/rational.h"
  32 #include "libavutil/stereo3d.h"
  33
  34 #include "avfilter.h"
  35 #include "formats.h"
  36 #include "internal.h"
  37 #include "video.h"
  38
  39 #define LEFT  0
  40 #define RIGHT 1
  41
  42 typedef struct FramepackContext {
  43     const AVClass *class;
  44
  45     const AVPixFmtDescriptor *pix_desc; ///< agreed pixel format
  46
  47     enum AVStereo3DType format;         ///< frame pack type output
  48
  49     AVFrame *input_views[2];            ///< input frames
  50
  51     int64_t double_pts;                 ///< new pts for frameseq mode
  52 } FramepackContext;
  53
  54 static const enum AVPixelFormat formats_supported[] = {
  55     AV_PIX_FMT_YUV420P,  AV_PIX_FMT_YUV422P,  AV_PIX_FMT_YUV444P,
  56     AV_PIX_FMT_YUV410P,  AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVJ420P,
  57     AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVJ440P,
  58     AV_PIX_FMT_NONE
  59 };
  60
  61 static int query_formats(AVFilterContext *ctx)
  62 {
  63     // this will ensure that formats are the same on all pads
  64     ff_set_common_formats(ctx, ff_make_format_list(formats_supported));
  65     return 0;
  66 }
  67
  68 static av_cold void framepack_uninit(AVFilterContext *ctx)
  69 {
  70     FramepackContext *s = ctx->priv;
  71
  72     // clean any leftover frame
  73     av_frame_free(&s->input_views[LEFT]);
  74     av_frame_free(&s->input_views[RIGHT]);
  75 }
  76
  77 static int config_output(AVFilterLink *outlink)
  78 {
  79     AVFilterContext *ctx = outlink->src;
  80     FramepackContext *s  = outlink->src->priv;
  81
  82     int width            = ctx->inputs[LEFT]->w;
  83     int height           = ctx->inputs[LEFT]->h;
  84     AVRational time_base = ctx->inputs[LEFT]->time_base;
  85
  86     // check size and fps match on the other input
  87     if (width  != ctx->inputs[RIGHT]->w ||
  88         height != ctx->inputs[RIGHT]->h) {
  89         av_log(ctx, AV_LOG_ERROR,
  90                "Left and right sizes differ (%dx%d vs %dx%d).\n",
  91                width, height,
  92                ctx->inputs[RIGHT]->w, ctx->inputs[RIGHT]->h);
  93         return AVERROR_INVALIDDATA;
  94     } else if (av_cmp_q(time_base, ctx->inputs[RIGHT]->time_base) != 0) {
  95         av_log(ctx, AV_LOG_ERROR,
  96                "Left and right framerates differ (%d/%d vs %d/%d).\n",
  97                time_base.num, time_base.den,
  98                ctx->inputs[RIGHT]->time_base.num,
  99                ctx->inputs[RIGHT]->time_base.den);
 100         return AVERROR_INVALIDDATA;
 101     }
 102
 103     s->pix_desc = av_pix_fmt_desc_get(outlink->format);
 104     if (!s->pix_desc)
 105         return AVERROR_BUG;
 106
 107     // modify output properties as needed
 108     switch (s->format) {
 109     case AV_STEREO3D_FRAMESEQUENCE:
 110         time_base.den *= 2;
 111         s->double_pts = AV_NOPTS_VALUE;
 112         break;
 113     case AV_STEREO3D_COLUMNS:
 114     case AV_STEREO3D_SIDEBYSIDE:
 115         width *= 2;
 116         break;
 117     case AV_STEREO3D_LINES:
 118     case AV_STEREO3D_TOPBOTTOM:
 119         height *= 2;
 120         break;
 121     default:
 122         av_log(ctx, AV_LOG_ERROR, "Unknown packing mode.");
 123         return AVERROR_INVALIDDATA;
 124     }
 125
 126     outlink->w         = width;
 127     outlink->h         = height;
 128     outlink->time_base = time_base;
 129
 130     return 0;
 131 }
 132
 133 static void horizontal_frame_pack(AVFilterLink *outlink,
 134                                   AVFrame *out,
 135                                   int interleaved)
 136 {
 137     AVFilterContext *ctx = outlink->src;
 138     FramepackContext *s = ctx->priv;
 139     int i, plane;
 140
 141     if (interleaved) {
 142         const uint8_t *leftp  = s->input_views[LEFT]->data[0];
 143         const uint8_t *rightp = s->input_views[RIGHT]->data[0];
 144         uint8_t *dstp         = out->data[0];
 145         int length = out->width / 2;
 146         int lines  = out->height;
 147
 148         for (plane = 0; plane < s->pix_desc->nb_components; plane++) {
 149             if (plane == 1 || plane == 2) {
 150                 length = -(-(out->width / 2) >> s->pix_desc->log2_chroma_w);
 151                 lines  = -(-(out->height)    >> s->pix_desc->log2_chroma_h);
 152             }
 153             for (i = 0; i < lines; i++) {
 154                 int j;
 155                 leftp  = s->input_views[LEFT]->data[plane] +
 156                          s->input_views[LEFT]->linesize[plane] * i;
 157                 rightp = s->input_views[RIGHT]->data[plane] +
 158                          s->input_views[RIGHT]->linesize[plane] * i;
 159                 dstp   = out->data[plane] + out->linesize[plane] * i;
 160                 for (j = 0; j < length; j++) {
 161                     // interpolate chroma as necessary
 162                     if ((s->pix_desc->log2_chroma_w ||
 163                          s->pix_desc->log2_chroma_h) &&
 164                         (plane == 1 || plane == 2)) {
 165                         *dstp++ = (*leftp + *rightp) / 2;
 166                         *dstp++ = (*leftp + *rightp) / 2;
 167                     } else {
 168                         *dstp++ = *leftp;
 169                         *dstp++ = *rightp;
 170                     }
 171                     leftp += 1;
 172                     rightp += 1;
 173                 }
 174             }
 175         }
 176     } else {
 177         for (i = 0; i < 2; i++) {
 178             const uint8_t *src[4];
 179             uint8_t *dst[4];
 180             int sub_w = s->input_views[i]->width >> s->pix_desc->log2_chroma_w;
 181
 182             src[0] = s->input_views[i]->data[0];
 183             src[1] = s->input_views[i]->data[1];
 184             src[2] = s->input_views[i]->data[2];
 185
 186             dst[0] = out->data[0] + i * s->input_views[i]->width;
 187             dst[1] = out->data[1] + i * sub_w;
 188             dst[2] = out->data[2] + i * sub_w;
 189
 190             av_image_copy(dst, out->linesize, src, s->input_views[i]->linesize,
 191                           s->input_views[i]->format,
 192                           s->input_views[i]->width,
 193                           s->input_views[i]->height);
 194         }
 195     }
 196 }
 197
 198 static void vertical_frame_pack(AVFilterLink *outlink,
 199                                 AVFrame *out,
 200                                 int interleaved)
 201 {
 202     AVFilterContext *ctx = outlink->src;
 203     FramepackContext *s = ctx->priv;
 204     int i;
 205
 206     for (i = 0; i < 2; i++) {
 207         const uint8_t *src[4];
 208         uint8_t *dst[4];
 209         int linesizes[4];
 210         int sub_h = s->input_views[i]->height >> s->pix_desc->log2_chroma_h;
 211
 212         src[0] = s->input_views[i]->data[0];
 213         src[1] = s->input_views[i]->data[1];
 214         src[2] = s->input_views[i]->data[2];
 215
 216         dst[0] = out->data[0] + i * out->linesize[0] *
 217                  (interleaved + s->input_views[i]->height * (1 - interleaved));
 218         dst[1] = out->data[1] + i * out->linesize[1] *
 219                  (interleaved + sub_h * (1 - interleaved));
 220         dst[2] = out->data[2] + i * out->linesize[2] *
 221                  (interleaved + sub_h * (1 - interleaved));
 222
 223         linesizes[0] = out->linesize[0] +
 224                        interleaved * out->linesize[0];
 225         linesizes[1] = out->linesize[1] +
 226                        interleaved * out->linesize[1];
 227         linesizes[2] = out->linesize[2] +
 228                        interleaved * out->linesize[2];
 229
 230         av_image_copy(dst, linesizes, src, s->input_views[i]->linesize,
 231                       s->input_views[i]->format,
 232                       s->input_views[i]->width,
 233                       s->input_views[i]->height);
 234     }
 235 }
 236
 237 static av_always_inline void spatial_frame_pack(AVFilterLink *outlink,
 238                                                 AVFrame *dst)
 239 {
 240     AVFilterContext *ctx = outlink->src;
 241     FramepackContext *s = ctx->priv;
 242     switch (s->format) {
 243     case AV_STEREO3D_SIDEBYSIDE:
 244         horizontal_frame_pack(outlink, dst, 0);
 245         break;
 246     case AV_STEREO3D_COLUMNS:
 247         horizontal_frame_pack(outlink, dst, 1);
 248         break;
 249     case AV_STEREO3D_TOPBOTTOM:
 250         vertical_frame_pack(outlink, dst, 0);
 251         break;
 252     case AV_STEREO3D_LINES:
 253         vertical_frame_pack(outlink, dst, 1);
 254         break;
 255     }
 256 }
 257
 258 static int filter_frame_left(AVFilterLink *inlink, AVFrame *frame)
 259 {
 260     FramepackContext *s = inlink->dst->priv;
 261     s->input_views[LEFT] = frame;
 262     return 0;
 263 }
 264
 265 static int filter_frame_right(AVFilterLink *inlink, AVFrame *frame)
 266 {
 267     FramepackContext *s = inlink->dst->priv;
 268     s->input_views[RIGHT] = frame;
 269     return 0;
 270 }
 271
 272 static int request_frame(AVFilterLink *outlink)
 273 {
 274     AVFilterContext *ctx = outlink->src;
 275     FramepackContext *s = ctx->priv;
 276     AVStereo3D *stereo;
 277     int ret, i;
 278
 279     /* get a frame on the either input, stop as soon as a video ends */
 280     for (i = 0; i < 2; i++) {
 281         if (!s->input_views[i]) {
 282             ret = ff_request_frame(ctx->inputs[i]);
 283             if (ret < 0)
 284                 return ret;
 285         }
 286     }
 287
 288     if (s->format == AV_STEREO3D_FRAMESEQUENCE) {
 289         if (s->double_pts == AV_NOPTS_VALUE)
 290             s->double_pts = s->input_views[LEFT]->pts;
 291
 292         for (i = 0; i < 2; i++) {
 293             // set correct timestamps
 294             s->input_views[i]->pts = s->double_pts++;
 295
 296             // set stereo3d side data
 297             stereo = av_stereo3d_create_side_data(s->input_views[i]);
 298             if (!stereo)
 299                 return AVERROR(ENOMEM);
 300             stereo->type = s->format;
 301
 302             // filter the frame and immediately relinquish its pointer
 303             ret = ff_filter_frame(outlink, s->input_views[i]);
 304             s->input_views[i] = NULL;
 305             if (ret < 0)
 306                 return ret;
 307         }
 308         return ret;
 309     } else {
 310         AVFrame *dst = ff_get_video_buffer(outlink, outlink->w, outlink->h);
 311         if (!dst)
 312             return AVERROR(ENOMEM);
 313
 314         spatial_frame_pack(outlink, dst);
 315
 316         // get any property from the original frame
 317         ret = av_frame_copy_props(dst, s->input_views[LEFT]);
 318         if (ret < 0) {
 319             av_frame_free(&dst);
 320             return ret;
 321         }
 322
 323         for (i = 0; i < 2; i++)
 324             av_frame_free(&s->input_views[i]);
 325
 326         // set stereo3d side data
 327         stereo = av_stereo3d_create_side_data(dst);
 328         if (!stereo) {
 329             av_frame_free(&dst);
 330             return AVERROR(ENOMEM);
 331         }
 332         stereo->type = s->format;
 333
 334         return ff_filter_frame(outlink, dst);
 335     }
 336 }
 337
 338 #define OFFSET(x) offsetof(FramepackContext, x)
 339 #define V AV_OPT_FLAG_VIDEO_PARAM
 340 static const AVOption options[] = {
 341     { "format", "Frame pack output format", OFFSET(format), AV_OPT_TYPE_INT,
 342         { .i64 = AV_STEREO3D_SIDEBYSIDE }, 0, INT_MAX, .flags = V, .unit = "format" },
 343     { "sbs", "Views are packed next to each other", 0, AV_OPT_TYPE_CONST,
 344         { .i64 = AV_STEREO3D_SIDEBYSIDE }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
 345     { "tab", "Views are packed on top of each other", 0, AV_OPT_TYPE_CONST,
 346         { .i64 = AV_STEREO3D_TOPBOTTOM }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
 347     { "frameseq", "Views are one after the other", 0, AV_OPT_TYPE_CONST,
 348         { .i64 = AV_STEREO3D_FRAMESEQUENCE }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
 349     { "lines", "Views are interleaved by lines", 0, AV_OPT_TYPE_CONST,
 350         { .i64 = AV_STEREO3D_LINES }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
 351     { "columns", "Views are interleaved by columns", 0, AV_OPT_TYPE_CONST,
 352         { .i64 = AV_STEREO3D_COLUMNS }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
 353     { NULL },
 354 };
 355
 356 static const AVClass framepack_class = {
 357     .class_name = "framepack",
 358     .item_name  = av_default_item_name,
 359     .option     = options,
 360     .version    = LIBAVUTIL_VERSION_INT,
 361 };
 362
 363 static const AVFilterPad framepack_inputs[] = {
 364     {
 365         .name         = "left",
 366         .type         = AVMEDIA_TYPE_VIDEO,
 367         .filter_frame = filter_frame_left,
 368         .needs_fifo   = 1,
 369     },
 370     {
 371         .name         = "right",
 372         .type         = AVMEDIA_TYPE_VIDEO,
 373         .filter_frame = filter_frame_right,
 374         .needs_fifo   = 1,
 375     },
 376     { NULL }
 377 };
 378
 379 static const AVFilterPad framepack_outputs[] = {
 380     {
 381         .name          = "packed",
 382         .type          = AVMEDIA_TYPE_VIDEO,
 383         .config_props  = config_output,
 384         .request_frame = request_frame,
 385     },
 386     { NULL }
 387 };
 388
 389 AVFilter ff_vf_framepack = {
 390     .name          = "framepack",
 391     .description   = NULL_IF_CONFIG_SMALL("Generate a frame packed stereoscopic video."),
 392     .priv_size     = sizeof(FramepackContext),
 393     .priv_class    = &framepack_class,
 394     .query_formats = query_formats,
 395     .inputs        = framepack_inputs,
 396     .outputs       = framepack_outputs,
 397     .uninit        = framepack_uninit,
 398 };