Add video converter

Makefile

@@ -7,6 +7,7 @@ XAV_DIR = c_src/xav
 PRIV_DIR = $(MIX_APP_PATH)/priv
 XAV_DECODER_SO = $(PRIV_DIR)/libxavdecoder.so
 XAV_READER_SO = $(PRIV_DIR)/libxavreader.so
+XAV_VIDEO_CONVERTER_SO = $(PRIV_DIR)/libxavvideoconverter.so
 
 # uncomment to compile with debug logs
 # XAV_DEBUG_LOGS = -DXAV_DEBUG=1
@@ -17,6 +18,9 @@ DECODER_SOURCES = $(XAV_DIR)/xav_decoder.c $(XAV_DIR)/decoder.c $(XAV_DIR)/video
 READER_HEADERS = $(XAV_DIR)/xav_reader.h $(XAV_DIR)/reader.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/audio_converter.h $(XAV_DIR)/utils.h $(XAV_DIR)/channel_layout.h
 READER_SOURCES = $(XAV_DIR)/xav_reader.c $(XAV_DIR)/reader.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/audio_converter.c $(XAV_DIR)/utils.c
 
+VIDEO_CONVERTER_HEADERS = $(XAV_DIR)/xav_video_converter.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/utils.h
+VIDEO_CONVERTER_SOURCES = $(XAV_DIR)/xav_video_converter.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/utils.c
+
 CFLAGS = $(XAV_DEBUG_LOGS) -fPIC -shared
 IFLAGS = -I$(ERTS_INCLUDE_DIR) -I$(XAV_DIR)
 LDFLAGS = -lavcodec -lswscale -lavutil -lavformat -lavdevice -lswresample
@@ -38,7 +42,7 @@ ifneq (,$(wildcard /etc/fedora-release))
 	LFLAGS += $$(pkg-config --libs-only-L libavcodec libswscale libavutil libavformat libavdevice libswresample)
 endif
 
-all: $(XAV_DECODER_SO) $(XAV_READER_SO)
+all: $(XAV_DECODER_SO) $(XAV_READER_SO) $(XAV_VIDEO_CONVERTER_SO)
 
 $(XAV_DECODER_SO): Makefile $(DECODER_SOURCES) $(DECODER_HEADERS)
 	mkdir -p $(PRIV_DIR)
@@ -48,6 +52,10 @@ $(XAV_READER_SO): Makefile $(READER_SOURCES) $(READER_HEADERS)
 	mkdir -p $(PRIV_DIR)
 	$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(READER_SOURCES) -o $(XAV_READER_SO) $(LDFLAGS)
 
+$(XAV_VIDEO_CONVERTER_SO): Makefile $(VIDEO_CONVERTER_SOURCES) $(VIDEO_CONVERTER_HEADERS)
+	mkdir -p $(PRIV_DIR)
+	$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(VIDEO_CONVERTER_SOURCES) -o $(XAV_VIDEO_CONVERTER_SO) $(LDFLAGS)
+
 format:
 	clang-format -i $(XAV_DIR)/*
 

c_src/xav/video_converter.c

@@ -1,43 +1,75 @@
 #include "video_converter.h"
+#include "utils.h"
 
-int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame,
-                            enum AVPixelFormat out_format) {
-  int ret;
+static inline unsigned int video_converter_resolution_changed(struct VideoConverter *converter, AVFrame *frame) {
+  return converter->in_format != frame->format || 
+          converter->in_width != frame->width || 
+          converter->in_height != frame->height;
+}
 
-  *dst_frame = av_frame_alloc();
-  if (!*dst_frame) {
-    return -1;
+struct VideoConverter *video_converter_alloc() {
+  struct VideoConverter *converter =
+      (struct VideoConverter *)XAV_ALLOC(sizeof(struct VideoConverter));
+  if(converter) {
+    converter->sws_ctx = NULL;
+    converter->dst_frame = av_frame_alloc();
   }
+  return converter;
+}
 
-  (*dst_frame)->width = src_frame->width;
-  (*dst_frame)->height = src_frame->height;
-  (*dst_frame)->format = out_format;
-  (*dst_frame)->pts = src_frame->pts;
+int video_converter_init(struct VideoConverter *converter, int in_width, int in_height, 
+                          enum AVPixelFormat in_format, enum AVPixelFormat out_format) {                            
+  converter->in_width = in_width;
+  converter->in_height = in_height;
+  converter->in_format = in_format;
+  converter->out_format = out_format;
 
-  ret = av_frame_get_buffer(*dst_frame, 0);
-  if (ret < 0) {
-    return ret;
-  }
+  av_frame_unref(converter->dst_frame);
 
-  struct SwsContext *sws_ctx =
-      sws_getContext(src_frame->width, src_frame->height, src_frame->format, src_frame->width,
-                     src_frame->height, out_format, SWS_BILINEAR, NULL, NULL, NULL);
+  converter->dst_frame->width = in_width;
+  converter->dst_frame->height = in_height;
+  converter->dst_frame->format = out_format;
 
-  if (ret < 0) {
+  int ret = av_frame_get_buffer(converter->dst_frame, 0);
+  if (ret < 0)
     return ret;
+
+  converter->sws_ctx = sws_getContext(in_width, in_height, in_format, in_width, in_height, out_format, 
+                                  SWS_BILINEAR, NULL, NULL, NULL);
+
+  if (!converter->sws_ctx) {
+    XAV_LOG_DEBUG("Couldn't get sws context");
+    return -1;
   }
 
-  // is this (const uint8_t * const*) cast really correct?
-  ret = sws_scale(sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
-                  src_frame->height, (*dst_frame)->data, (*dst_frame)->linesize);
+  return 0;
+}
 
-  if (ret < 0) {
-    av_frame_free(dst_frame);
-    sws_freeContext(sws_ctx);
-    return ret;
+int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame) {
+  int ret;
+
+  if (video_converter_resolution_changed(converter, src_frame)) {
+    XAV_LOG_DEBUG("Frame resolution changed");
+    sws_freeContext(converter->sws_ctx);
+    ret = video_converter_init(converter, src_frame->width, src_frame->height, 
+                                src_frame->format, converter->out_format);
+    if (ret < 0) {
+      return ret;
+    }
   }
 
-  sws_freeContext(sws_ctx);
+  converter->dst_frame->pts = src_frame->pts;
 
-  return ret;
+  // is this (const uint8_t * const*) cast really correct?
+  return sws_scale(converter->sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
+                  src_frame->height, converter->dst_frame->data, converter->dst_frame->linesize);
+}
+
+void video_converter_free(struct VideoConverter **converter) {
+  if (*converter != NULL) {
+    sws_freeContext((*converter)->sws_ctx);
+    av_frame_free(&(*converter)->dst_frame);
+    XAV_FREE(*converter);
+    *converter = NULL;
+  }
 }

c_src/xav/video_converter.h

@@ -5,4 +5,20 @@
 #include <libswscale/swscale.h>
 #include <stdint.h>
 
-int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame, enum AVPixelFormat out_format);
+struct VideoConverter {
+    struct SwsContext *sws_ctx;
+    int in_width;
+    int in_height;
+    enum AVPixelFormat in_format;
+    enum AVPixelFormat out_format;
+    AVFrame *dst_frame;
+};
+
+struct VideoConverter *video_converter_alloc();
+
+int video_converter_init(struct VideoConverter* converter, int in_width, int in_height, 
+                         enum AVPixelFormat in_format, enum AVPixelFormat out_format);
+
+int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame);
+
+void video_converter_free(struct VideoConverter **converter);

c_src/xav/xav_decoder.c

@@ -1,10 +1,10 @@
 #include "xav_decoder.h"
 #include "audio_converter.h"
-#include "video_converter.h"
 
 ErlNifResourceType *xav_decoder_resource_type;
 
 static int init_audio_converter(struct XavDecoder *xav_decoder);
+static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame);
 
 void free_frames(AVFrame **frames, int size) {
   for (int i = 0; i < size; i++) {
@@ -86,6 +86,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
       enif_alloc_resource(xav_decoder_resource_type, sizeof(struct XavDecoder));
   xav_decoder->decoder = NULL;
   xav_decoder->ac = NULL;
+  xav_decoder->vc = NULL;
   xav_decoder->out_audio_fmt = out_audo_fmt;
   xav_decoder->out_video_fmt = out_video_fmt;
   xav_decoder->out_sample_rate = out_sample_rate;
@@ -119,15 +120,20 @@ ERL_NIF_TERM convert(ErlNifEnv *env, struct XavDecoder *xav_decoder, AVFrame *fr
       return xav_nif_video_frame_to_term(env, frame);
     }
 
-    AVFrame *dst_frame;
-    ret = video_converter_convert(frame, &dst_frame, xav_decoder->out_video_fmt);
-    if (ret <= 0) {
-      return xav_nif_raise(env, "failed_to_decode");
+    if (xav_decoder->vc == NULL) {
+      ret = init_video_converter(xav_decoder, frame);
+      if (ret < 0) {
+        return xav_nif_raise(env, "failed_to_init_converter");
+      }
+    }
+
+    ret = video_converter_convert(xav_decoder->vc, frame);
+    if (ret < 0) {
+      return xav_nif_raise(env, "failed_to_convert");
     }
 
-    frame_term = xav_nif_video_frame_to_term(env, dst_frame);
+    frame_term = xav_nif_video_frame_to_term(env, xav_decoder->vc->dst_frame);
 
-    av_frame_free(&dst_frame);
   } else if (xav_decoder->decoder->media_type == AVMEDIA_TYPE_AUDIO) {
     XAV_LOG_DEBUG("Converting audio to desired out format");
 
@@ -286,6 +292,17 @@ static int init_audio_converter(struct XavDecoder *xav_decoder) {
                               xav_decoder->out_audio_fmt);
 }
 
+static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame) {
+  xav_decoder->vc = video_converter_alloc();
+  if (xav_decoder->vc == NULL) {
+    XAV_LOG_DEBUG("Couldn't allocate video converter");
+    return -1;
+  }
+
+  return video_converter_init(xav_decoder->vc, frame->width, frame->height, 
+                                  frame->format, xav_decoder->out_video_fmt);
+}
+
 void free_xav_decoder(ErlNifEnv *env, void *obj) {
   XAV_LOG_DEBUG("Freeing XavDecoder object");
   struct XavDecoder *xav_decoder = (struct XavDecoder *)obj;
@@ -296,6 +313,10 @@ void free_xav_decoder(ErlNifEnv *env, void *obj) {
   if (xav_decoder->ac != NULL) {
     audio_converter_free(&xav_decoder->ac);
   }
+
+  if (xav_decoder->vc != NULL) {
+    video_converter_free(&xav_decoder->vc);
+  }
 }
 
 static ErlNifFunc xav_funcs[] = {{"new", 4, new},

c_src/xav/xav_decoder.h

@@ -1,11 +1,13 @@
 #include "audio_converter.h"
+#include "video_converter.h"
 #include "decoder.h"
 
 #include <libavutil/pixfmt.h>
 
 struct XavDecoder {
   struct Decoder *decoder;
   struct AudioConverter *ac;
+  struct VideoConverter *vc;
   enum AVPixelFormat out_video_fmt;
   enum AVSampleFormat out_audio_fmt;
   int out_sample_rate;

c_src/xav/xav_reader.c

@@ -1,7 +1,8 @@
 #include "xav_reader.h"
-#include "video_converter.h"
 
 static int init_audio_converter(struct XavReader *xav_reader);
+static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame);
+
 ErlNifResourceType *xav_reader_resource_type;
 
 ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
@@ -56,6 +57,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
       enif_alloc_resource(xav_reader_resource_type, sizeof(struct XavReader));
   xav_reader->reader = NULL;
   xav_reader->ac = NULL;
+  xav_reader->vc = NULL;
   xav_reader->out_format = out_format;
   xav_reader->out_sample_rate = out_sample_rate;
   xav_reader->out_channels = out_channels;
@@ -111,7 +113,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
   } else if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
     ERL_NIF_TERM in_format_term =
         enif_make_atom(env, av_get_pix_fmt_name(xav_reader->reader->c->pix_fmt));
-    ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb");
+    ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb24");
     ERL_NIF_TERM framerate_num_term = enif_make_int(env, xav_reader->reader->framerate.num);
     ERL_NIF_TERM framerate_den_term = enif_make_int(env, xav_reader->reader->framerate.den);
     ERL_NIF_TERM framerate_term = enif_make_tuple(env, 2, framerate_num_term, framerate_den_term);
@@ -147,15 +149,19 @@ ERL_NIF_TERM next_frame(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
   if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
     XAV_LOG_DEBUG("Converting video to RGB");
 
-    AVFrame *dst_frame;
-    ret = video_converter_convert(xav_reader->reader->frame, &dst_frame, AV_PIX_FMT_RGB24);
+    if (xav_reader->vc == NULL) {
+      ret = init_video_converter(xav_reader, xav_reader->reader->frame);
+      if (ret < 0) {
+        return xav_nif_raise(env, "failed_to_init_converter");
+      }
+    }
+
+    ret = video_converter_convert(xav_reader->vc, xav_reader->reader->frame);
     if (ret <= 0) {
       return xav_nif_raise(env, "failed_to_read");
     }
 
-    frame_term = xav_nif_video_frame_to_term(env, dst_frame);
-
-    av_frame_free(&dst_frame);
+    frame_term = xav_nif_video_frame_to_term(env, xav_reader->vc->dst_frame);
   } else if (xav_reader->reader->media_type == AVMEDIA_TYPE_AUDIO) {
     XAV_LOG_DEBUG("Converting audio to desired out format");
 
@@ -277,6 +283,17 @@ static int init_audio_converter(struct XavReader *xav_reader) {
                               out_sample_fmt);
 }
 
+static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame) {
+  xav_reader->vc = video_converter_alloc();
+  if (xav_reader->vc == NULL) {
+    XAV_LOG_DEBUG("Couldn't allocate video converter");
+    return -1;
+  }
+
+  return video_converter_init(xav_reader->vc, frame->width, frame->height, 
+                                  frame->format, AV_PIX_FMT_RGB24);
+}
+
 void free_xav_reader(ErlNifEnv *env, void *obj) {
   XAV_LOG_DEBUG("Freeing XavReader object");
   struct XavReader *xav_reader = (struct XavReader *)obj;
@@ -287,6 +304,10 @@ void free_xav_reader(ErlNifEnv *env, void *obj) {
   if (xav_reader->ac != NULL) {
     audio_converter_free(&xav_reader->ac);
   }
+
+  if (xav_reader->vc != NULL) {
+    video_converter_free(&xav_reader->vc);
+  }
 }
 
 static ErlNifFunc xav_funcs[] = {{"new", 6, new},

c_src/xav/xav_reader.h

@@ -1,9 +1,11 @@
 #include "audio_converter.h"
+#include "video_converter.h"
 #include "reader.h"
 
 struct XavReader {
   struct Reader *reader;
   struct AudioConverter *ac;
+  struct VideoConverter *vc;
   char *out_format;
   int out_sample_rate;
   int out_channels;