Skip to content

Commit

Permalink
Add video converter (#25)
Browse files Browse the repository at this point in the history
  • Loading branch information
gBillal authored Jan 13, 2025
1 parent a26f39a commit b52115d
Show file tree
Hide file tree
Showing 17 changed files with 771 additions and 43 deletions.
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ XAV_DIR = c_src/xav
PRIV_DIR = $(MIX_APP_PATH)/priv
XAV_DECODER_SO = $(PRIV_DIR)/libxavdecoder.so
XAV_READER_SO = $(PRIV_DIR)/libxavreader.so
XAV_VIDEO_CONVERTER_SO = $(PRIV_DIR)/libxavvideoconverter.so

# uncomment to compile with debug logs
# XAV_DEBUG_LOGS = -DXAV_DEBUG=1
Expand All @@ -17,11 +18,14 @@ DECODER_SOURCES = $(XAV_DIR)/xav_decoder.c $(XAV_DIR)/decoder.c $(XAV_DIR)/video
READER_HEADERS = $(XAV_DIR)/xav_reader.h $(XAV_DIR)/reader.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/audio_converter.h $(XAV_DIR)/utils.h $(XAV_DIR)/channel_layout.h
READER_SOURCES = $(XAV_DIR)/xav_reader.c $(XAV_DIR)/reader.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/audio_converter.c $(XAV_DIR)/utils.c

VIDEO_CONVERTER_HEADERS = $(XAV_DIR)/xav_video_converter.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/utils.h
VIDEO_CONVERTER_SOURCES = $(XAV_DIR)/xav_video_converter.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/utils.c

CFLAGS = $(XAV_DEBUG_LOGS) -fPIC -shared
IFLAGS = -I$(ERTS_INCLUDE_DIR) -I$(XAV_DIR)
LDFLAGS = -lavcodec -lswscale -lavutil -lavformat -lavdevice -lswresample

# Falgs for MacOS
# Flags for MacOS
ifeq ($(shell uname -s),Darwin)
ifeq ($(shell uname -m),arm64)
IFLAGS += $$(pkg-config --cflags-only-I libavcodec libswscale libavutil libavformat libavdevice libswresample)
Expand All @@ -38,7 +42,7 @@ ifneq (,$(wildcard /etc/fedora-release))
LFLAGS += $$(pkg-config --libs-only-L libavcodec libswscale libavutil libavformat libavdevice libswresample)
endif

all: $(XAV_DECODER_SO) $(XAV_READER_SO)
all: $(XAV_DECODER_SO) $(XAV_READER_SO) $(XAV_VIDEO_CONVERTER_SO)

$(XAV_DECODER_SO): Makefile $(DECODER_SOURCES) $(DECODER_HEADERS)
mkdir -p $(PRIV_DIR)
Expand All @@ -48,6 +52,10 @@ $(XAV_READER_SO): Makefile $(READER_SOURCES) $(READER_HEADERS)
mkdir -p $(PRIV_DIR)
$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(READER_SOURCES) -o $(XAV_READER_SO) $(LDFLAGS)

$(XAV_VIDEO_CONVERTER_SO): Makefile $(VIDEO_CONVERTER_SOURCES) $(VIDEO_CONVERTER_HEADERS)
mkdir -p $(PRIV_DIR)
$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(VIDEO_CONVERTER_SOURCES) -o $(XAV_VIDEO_CONVERTER_SO) $(LDFLAGS)

format:
clang-format -i $(XAV_DIR)/*

Expand Down
16 changes: 16 additions & 0 deletions c_src/xav/utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,22 @@ ERL_NIF_TERM xav_nif_raise(ErlNifEnv *env, char *msg) {
return enif_raise_exception(env, reason);
}

int xav_get_atom(ErlNifEnv *env, ERL_NIF_TERM atom, char **value) {
unsigned int atom_len;
if (!enif_get_atom_length(env, atom, &atom_len, ERL_NIF_LATIN1)) {
return 0;
}

char *format = (char *)XAV_ALLOC((atom_len * 1) * sizeof(char *));
if (!enif_get_atom(env, atom, format, atom_len + 1, ERL_NIF_LATIN1)) {
XAV_FREE(format);
return 0;
}

*value = format;
return 1;
}

ERL_NIF_TERM xav_nif_audio_frame_to_term(ErlNifEnv *env, uint8_t **out_data, int out_samples,
int out_size, enum AVSampleFormat out_format, int pts) {
ERL_NIF_TERM data_term;
Expand Down
1 change: 1 addition & 0 deletions c_src/xav/utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
ERL_NIF_TERM xav_nif_ok(ErlNifEnv *env, ERL_NIF_TERM data_term);
ERL_NIF_TERM xav_nif_error(ErlNifEnv *env, char *reason);
ERL_NIF_TERM xav_nif_raise(ErlNifEnv *env, char *msg);
int xav_get_atom(ErlNifEnv *env, ERL_NIF_TERM atom, char **value);
ERL_NIF_TERM xav_nif_video_frame_to_term(ErlNifEnv *env, AVFrame *frame);
ERL_NIF_TERM xav_nif_audio_frame_to_term(ErlNifEnv *env, uint8_t **out_data, int out_samples,
int out_size, enum AVSampleFormat out_format, int pts);
91 changes: 65 additions & 26 deletions c_src/xav/video_converter.c
Original file line number Diff line number Diff line change
@@ -1,43 +1,82 @@
#include "video_converter.h"
#include "utils.h"

int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame,
enum AVPixelFormat out_format) {
int ret;
static inline unsigned int video_converter_resolution_changed(struct VideoConverter *converter, AVFrame *frame) {
return converter->in_format != frame->format ||
converter->in_width != frame->width ||
converter->in_height != frame->height;
}

*dst_frame = av_frame_alloc();
if (!*dst_frame) {
return -1;
struct VideoConverter *video_converter_alloc() {
struct VideoConverter *converter =
(struct VideoConverter *)XAV_ALLOC(sizeof(struct VideoConverter));
if(converter) {
converter->sws_ctx = NULL;
converter->dst_frame = av_frame_alloc();
}
return converter;
}

int video_converter_init(struct VideoConverter *converter, int in_width, int in_height,
enum AVPixelFormat in_format, enum AVPixelFormat out_format) {
converter->in_width = in_width;
converter->in_height = in_height;
converter->in_format = in_format;
converter->out_format = out_format;

av_frame_unref(converter->dst_frame);

(*dst_frame)->width = src_frame->width;
(*dst_frame)->height = src_frame->height;
(*dst_frame)->format = out_format;
(*dst_frame)->pts = src_frame->pts;
converter->dst_frame->width = in_width;
converter->dst_frame->height = in_height;
converter->dst_frame->format = out_format;

ret = av_frame_get_buffer(*dst_frame, 0);
if (ret < 0) {
int ret = av_frame_get_buffer(converter->dst_frame, 0);
if (ret < 0)
return ret;

converter->sws_ctx = sws_getContext(in_width, in_height, in_format, in_width, in_height, out_format,
SWS_BILINEAR, NULL, NULL, NULL);

if (!converter->sws_ctx) {
XAV_LOG_DEBUG("Couldn't get sws context");
return -1;
}

struct SwsContext *sws_ctx =
sws_getContext(src_frame->width, src_frame->height, src_frame->format, src_frame->width,
src_frame->height, out_format, SWS_BILINEAR, NULL, NULL, NULL);
return 0;
}

if (ret < 0) {
return ret;
int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame) {
int ret;

if (video_converter_resolution_changed(converter, src_frame)) {
XAV_LOG_DEBUG("Frame resolution changed");
sws_freeContext(converter->sws_ctx);
ret = video_converter_init(converter, src_frame->width, src_frame->height,
src_frame->format, converter->out_format);
if (ret < 0) {
return ret;
}
}

converter->dst_frame->pts = src_frame->pts;

// is this (const uint8_t * const*) cast really correct?
ret = sws_scale(sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
src_frame->height, (*dst_frame)->data, (*dst_frame)->linesize);
return sws_scale(converter->sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
src_frame->height, converter->dst_frame->data, converter->dst_frame->linesize);
}

if (ret < 0) {
av_frame_free(dst_frame);
sws_freeContext(sws_ctx);
return ret;
}
void video_converter_free(struct VideoConverter **converter) {
struct VideoConverter* vc = *converter;
if (vc != NULL) {
if (vc->sws_ctx != NULL) {
sws_freeContext((*converter)->sws_ctx);
}

sws_freeContext(sws_ctx);
if (vc->dst_frame != NULL) {
av_frame_free(&(*converter)->dst_frame);
}

return ret;
XAV_FREE(vc);
*converter = NULL;
}
}
18 changes: 17 additions & 1 deletion c_src/xav/video_converter.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,20 @@
#include <libswscale/swscale.h>
#include <stdint.h>

int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame, enum AVPixelFormat out_format);
struct VideoConverter {
struct SwsContext *sws_ctx;
int in_width;
int in_height;
enum AVPixelFormat in_format;
enum AVPixelFormat out_format;
AVFrame *dst_frame;
};

struct VideoConverter *video_converter_alloc();

int video_converter_init(struct VideoConverter* converter, int in_width, int in_height,
enum AVPixelFormat in_format, enum AVPixelFormat out_format);

int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame);

void video_converter_free(struct VideoConverter **converter);
35 changes: 28 additions & 7 deletions c_src/xav/xav_decoder.c
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
#include "xav_decoder.h"
#include "audio_converter.h"
#include "video_converter.h"

ErlNifResourceType *xav_decoder_resource_type;

static int init_audio_converter(struct XavDecoder *xav_decoder);
static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame);

void free_frames(AVFrame **frames, int size) {
for (int i = 0; i < size; i++) {
Expand Down Expand Up @@ -86,6 +86,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
enif_alloc_resource(xav_decoder_resource_type, sizeof(struct XavDecoder));
xav_decoder->decoder = NULL;
xav_decoder->ac = NULL;
xav_decoder->vc = NULL;
xav_decoder->out_audio_fmt = out_audo_fmt;
xav_decoder->out_video_fmt = out_video_fmt;
xav_decoder->out_sample_rate = out_sample_rate;
Expand Down Expand Up @@ -119,15 +120,20 @@ ERL_NIF_TERM convert(ErlNifEnv *env, struct XavDecoder *xav_decoder, AVFrame *fr
return xav_nif_video_frame_to_term(env, frame);
}

AVFrame *dst_frame;
ret = video_converter_convert(frame, &dst_frame, xav_decoder->out_video_fmt);
if (ret <= 0) {
return xav_nif_raise(env, "failed_to_decode");
if (xav_decoder->vc == NULL) {
ret = init_video_converter(xav_decoder, frame);
if (ret < 0) {
return xav_nif_raise(env, "failed_to_init_converter");
}
}

ret = video_converter_convert(xav_decoder->vc, frame);
if (ret < 0) {
return xav_nif_raise(env, "failed_to_convert");
}

frame_term = xav_nif_video_frame_to_term(env, dst_frame);
frame_term = xav_nif_video_frame_to_term(env, xav_decoder->vc->dst_frame);

av_frame_free(&dst_frame);
} else if (xav_decoder->decoder->media_type == AVMEDIA_TYPE_AUDIO) {
XAV_LOG_DEBUG("Converting audio to desired out format");

Expand Down Expand Up @@ -286,6 +292,17 @@ static int init_audio_converter(struct XavDecoder *xav_decoder) {
xav_decoder->out_audio_fmt);
}

static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame) {
xav_decoder->vc = video_converter_alloc();
if (xav_decoder->vc == NULL) {
XAV_LOG_DEBUG("Couldn't allocate video converter");
return -1;
}

return video_converter_init(xav_decoder->vc, frame->width, frame->height,
frame->format, xav_decoder->out_video_fmt);
}

void free_xav_decoder(ErlNifEnv *env, void *obj) {
XAV_LOG_DEBUG("Freeing XavDecoder object");
struct XavDecoder *xav_decoder = (struct XavDecoder *)obj;
Expand All @@ -296,6 +313,10 @@ void free_xav_decoder(ErlNifEnv *env, void *obj) {
if (xav_decoder->ac != NULL) {
audio_converter_free(&xav_decoder->ac);
}

if (xav_decoder->vc != NULL) {
video_converter_free(&xav_decoder->vc);
}
}

static ErlNifFunc xav_funcs[] = {{"new", 4, new},
Expand Down
2 changes: 2 additions & 0 deletions c_src/xav/xav_decoder.h
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
#include "audio_converter.h"
#include "video_converter.h"
#include "decoder.h"

#include <libavutil/pixfmt.h>

struct XavDecoder {
struct Decoder *decoder;
struct AudioConverter *ac;
struct VideoConverter *vc;
enum AVPixelFormat out_video_fmt;
enum AVSampleFormat out_audio_fmt;
int out_sample_rate;
Expand Down
35 changes: 28 additions & 7 deletions c_src/xav/xav_reader.c
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#include "xav_reader.h"
#include "video_converter.h"

static int init_audio_converter(struct XavReader *xav_reader);
static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame);

ErlNifResourceType *xav_reader_resource_type;

ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
Expand Down Expand Up @@ -56,6 +57,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
enif_alloc_resource(xav_reader_resource_type, sizeof(struct XavReader));
xav_reader->reader = NULL;
xav_reader->ac = NULL;
xav_reader->vc = NULL;
xav_reader->out_format = out_format;
xav_reader->out_sample_rate = out_sample_rate;
xav_reader->out_channels = out_channels;
Expand Down Expand Up @@ -111,7 +113,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
} else if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
ERL_NIF_TERM in_format_term =
enif_make_atom(env, av_get_pix_fmt_name(xav_reader->reader->c->pix_fmt));
ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb");
ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb24");
ERL_NIF_TERM framerate_num_term = enif_make_int(env, xav_reader->reader->framerate.num);
ERL_NIF_TERM framerate_den_term = enif_make_int(env, xav_reader->reader->framerate.den);
ERL_NIF_TERM framerate_term = enif_make_tuple(env, 2, framerate_num_term, framerate_den_term);
Expand Down Expand Up @@ -147,15 +149,19 @@ ERL_NIF_TERM next_frame(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
XAV_LOG_DEBUG("Converting video to RGB");

AVFrame *dst_frame;
ret = video_converter_convert(xav_reader->reader->frame, &dst_frame, AV_PIX_FMT_RGB24);
if (xav_reader->vc == NULL) {
ret = init_video_converter(xav_reader, xav_reader->reader->frame);
if (ret < 0) {
return xav_nif_raise(env, "failed_to_init_converter");
}
}

ret = video_converter_convert(xav_reader->vc, xav_reader->reader->frame);
if (ret <= 0) {
return xav_nif_raise(env, "failed_to_read");
}

frame_term = xav_nif_video_frame_to_term(env, dst_frame);

av_frame_free(&dst_frame);
frame_term = xav_nif_video_frame_to_term(env, xav_reader->vc->dst_frame);
} else if (xav_reader->reader->media_type == AVMEDIA_TYPE_AUDIO) {
XAV_LOG_DEBUG("Converting audio to desired out format");

Expand Down Expand Up @@ -277,6 +283,17 @@ static int init_audio_converter(struct XavReader *xav_reader) {
out_sample_fmt);
}

static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame) {
xav_reader->vc = video_converter_alloc();
if (xav_reader->vc == NULL) {
XAV_LOG_DEBUG("Couldn't allocate video converter");
return -1;
}

return video_converter_init(xav_reader->vc, frame->width, frame->height,
frame->format, AV_PIX_FMT_RGB24);
}

void free_xav_reader(ErlNifEnv *env, void *obj) {
XAV_LOG_DEBUG("Freeing XavReader object");
struct XavReader *xav_reader = (struct XavReader *)obj;
Expand All @@ -287,6 +304,10 @@ void free_xav_reader(ErlNifEnv *env, void *obj) {
if (xav_reader->ac != NULL) {
audio_converter_free(&xav_reader->ac);
}

if (xav_reader->vc != NULL) {
video_converter_free(&xav_reader->vc);
}
}

static ErlNifFunc xav_funcs[] = {{"new", 6, new},
Expand Down
2 changes: 2 additions & 0 deletions c_src/xav/xav_reader.h
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
#include "audio_converter.h"
#include "video_converter.h"
#include "reader.h"

struct XavReader {
struct Reader *reader;
struct AudioConverter *ac;
struct VideoConverter *vc;
char *out_format;
int out_sample_rate;
int out_channels;
Expand Down
Loading

0 comments on commit b52115d

Please sign in to comment.