Create Decoders

.formatter.exs

@@ -1,8 +1,15 @@
 [
   inputs: [
-    "{lib,test,config}/**/*.{ex,exs}",
+    "{lib,test,config,c_src}/**/*.{ex,exs}",
     ".formatter.exs",
     "*.exs"
   ],
+  locals_without_parens: [
+    module: 1,
+    spec: 1,
+    state_type: 1,
+    dirty: 2,
+    type: 1
+  ],
   import_deps: [:membrane_core]
 ]

README.md

@@ -1,4 +1,4 @@
-# Membrane Template Plugin
+# Membrane VPx Plugin
 
 [![Hex.pm](https://img.shields.io/hexpm/v/membrane_template_plugin.svg)](https://hex.pm/packages/membrane_template_plugin)
 [![API Docs](https://img.shields.io/badge/api-docs-yellow.svg?style=flat)](https://hexdocs.pm/membrane_template_plugin)

bundlex.exs

@@ -0,0 +1,33 @@
+defmodule Membrane.VPx.BundlexProject do
+  use Bundlex.Project
+
+  def project() do
+    [
+      natives: natives()
+    ]
+  end
+
+  defp natives() do
+    [
+      vpx_decoder: [
+        interface: :nif,
+        sources: ["vpx_decoder.c"],
+        os_deps: [
+          libvpx: [
+            {:precompiled, Membrane.PrecompiledDependencyProvider.get_dependency_url(:libvpx)},
+            {:pkg_config, "vpx"}
+          ]
+        ],
+        preprocessor: Unifex
+      ]
+      # vpx_encoder: [
+      #   interface: :nif,
+      #   sources: ["vpx_encoder.c"],
+      #   os_deps: [
+      #     libvpx: [{:pkg_config, "vpx"}]
+      #   ],
+      #   preprocessor: Unifex
+      # ]
+    ]
+  end
+end

c_src/membrane_vpx_plugin/_generated/.gitignore

@@ -0,0 +1,3 @@
+**/*.h
+**/*.c
+**/*.cpp

c_src/membrane_vpx_plugin/vpx_decoder.c

@@ -0,0 +1,139 @@
+#include "vpx_decoder.h"
+
+void handle_destroy_state(UnifexEnv *env, State *state) {
+  UNIFEX_UNUSED(env);
+
+  vpx_codec_destroy(&state->codec_context);
+}
+
+UNIFEX_TERM create(UnifexEnv *env, Codec codec) {
+  UNIFEX_TERM result;
+  State *state = unifex_alloc_state(env);
+
+  switch (codec) {
+  case CODEC_VP8:
+    state->codec_interface = vpx_codec_vp8_dx();
+    break;
+  case CODEC_VP9:
+    state->codec_interface = vpx_codec_vp9_dx();
+    break;
+  }
+
+  if (vpx_codec_dec_init(&state->codec_context, state->codec_interface, NULL, 0)) {
+    result = create_result_error(env, "Failed to initialize decoder");
+    unifex_release_state(env, state);
+    return result;
+  }
+  result = create_result_ok(env, state);
+  unifex_release_state(env, state);
+  return result;
+}
+
+Dimensions get_plane_dimensions(const vpx_image_t *img, int plane) {
+  const int height =
+      (plane > 0 && img->y_chroma_shift > 0) ? (img->d_h + 1) >> img->y_chroma_shift : img->d_h;
+
+  int width =
+      (plane > 0 && img->x_chroma_shift > 0) ? (img->d_w + 1) >> img->x_chroma_shift : img->d_w;
+
+  // Fixing NV12 chroma width if it is odd
+  if (img->fmt == VPX_IMG_FMT_NV12 && plane == 1)
+    width = (width + 1) & ~1;
+
+  return (Dimensions){width, height};
+}
+size_t get_image_byte_size(const vpx_image_t *img) {
+  const int bytes_per_pixel = (img->fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? 2 : 1;
+  const int number_of_planes = (img->fmt == VPX_IMG_FMT_NV12) ? 2 : 3;
+
+  size_t image_size = 0;
+
+  for (int plane = 0; plane < number_of_planes; ++plane) {
+    Dimensions plane_dimensions = get_plane_dimensions(img, plane);
+    image_size += plane_dimensions.width * plane_dimensions.height * bytes_per_pixel;
+  }
+  return image_size;
+}
+
+void get_output_frame_from_image(const vpx_image_t *img, UnifexPayload *output_frame) {
+  const int bytes_per_pixel = (img->fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? 2 : 1;
+
+  // Assuming that for nv12 we write all chroma data at once
+  const int number_of_planes = (img->fmt == VPX_IMG_FMT_NV12) ? 2 : 3;
+  unsigned char *frame_data = output_frame->data;
+
+  for (int plane = 0; plane < number_of_planes; ++plane) {
+    const unsigned char *buf = img->planes[plane];
+    const int stride = img->stride[plane];
+    Dimensions plane_dimensions = get_plane_dimensions(img, plane);
+
+    for (unsigned int y = 0; y < plane_dimensions.height; ++y) {
+      size_t bytes_to_write = bytes_per_pixel * plane_dimensions.width;
+      memcpy(frame_data, buf, bytes_to_write);
+      buf += stride;
+      frame_data += bytes_to_write;
+    }
+  }
+}
+
+void alloc_output_frame(UnifexEnv *env, const vpx_image_t *img, UnifexPayload **output_frame) {
+  *output_frame = unifex_alloc(sizeof(UnifexPayload));
+  unifex_payload_alloc(env, UNIFEX_PAYLOAD_BINARY, get_image_byte_size(img), *output_frame);
+}
+
+PixelFormat get_pixel_format_from_image(vpx_image_t *img) {
+  switch (img->fmt) {
+  case VPX_IMG_FMT_I422:
+    return PIXEL_FORMAT_I422;
+
+  case VPX_IMG_FMT_I420:
+    return PIXEL_FORMAT_I420;
+
+  case VPX_IMG_FMT_I444:
+    return PIXEL_FORMAT_I444;
+
+  case VPX_IMG_FMT_YV12:
+    return PIXEL_FORMAT_YV12;
+
+  case VPX_IMG_FMT_NV12:
+    return PIXEL_FORMAT_NV12;
+
+  default:
+    return PIXEL_FORMAT_I420;
+  }
+}
+
+UNIFEX_TERM decode_frame(UnifexEnv *env, UnifexPayload *frame, State *state) {
+  vpx_codec_iter_t iter = NULL;
+  vpx_image_t *img = NULL;
+  PixelFormat pixel_format = PIXEL_FORMAT_I420;
+  unsigned int frames_cnt = 0, allocated_frames = 2;
+  UnifexPayload **output_frames = unifex_alloc(allocated_frames * sizeof(*output_frames));
+
+  if (vpx_codec_decode(&state->codec_context, frame->data, frame->size, NULL, 0)) {
+    return decode_frame_result_error(env, "Decoding frame failed");
+  }
+
+  while ((img = vpx_codec_get_frame(&state->codec_context, &iter)) != NULL) {
+    if (frames_cnt >= allocated_frames) {
+      allocated_frames *= 2;
+      output_frames = unifex_realloc(output_frames, allocated_frames * sizeof(*output_frames));
+    }
+
+    alloc_output_frame(env, img, &output_frames[frames_cnt]);
+    get_output_frame_from_image(img, output_frames[frames_cnt]);
+    pixel_format = get_pixel_format_from_image(img);
+    frames_cnt++;
+  }
+
+  UNIFEX_TERM result = decode_frame_result_ok(env, output_frames, frames_cnt, pixel_format);
+  for (unsigned int i = 0; i < frames_cnt; i++) {
+    if (output_frames[i] != NULL) {
+      unifex_payload_release(output_frames[i]);
+      unifex_free(output_frames[i]);
+    }
+  }
+  unifex_free(output_frames);
+
+  return result;
+}

c_src/membrane_vpx_plugin/vpx_decoder.h

@@ -0,0 +1,16 @@
+#pragma once
+#include "vpx/vp8dx.h"
+#include "vpx/vpx_decoder.h"
+#include <erl_nif.h>
+
+typedef struct State {
+  vpx_codec_ctx_t codec_context;
+  vpx_codec_iface_t *codec_interface;
+} State;
+
+typedef struct Dimensions {
+  unsigned int width;
+  unsigned int height;
+} Dimensions;
+
+#include "_generated/vpx_decoder.h"

c_src/membrane_vpx_plugin/vpx_decoder.spec.exs

@@ -0,0 +1,15 @@
+module Membrane.VPx.Decoder.Native
+
+state_type "State"
+
+type codec :: :vp8 | :vp9
+
+type pixel_format :: :I420 | :I422 | :I444 | :NV12 | :YV12
+
+spec create(codec) :: {:ok :: label, state} | {:error :: label, reason :: atom}
+
+spec decode_frame(payload, state) ::
+       {:ok :: label, frames :: [payload], pixel_format :: pixel_format}
+       | {:error :: label, reason :: atom}
+
+dirty :cpu, create: 1, decode_frame: 2

lib/membrane_vpx/decoder/vp8_decoder.ex

@@ -0,0 +1,51 @@
+defmodule Membrane.VP8.Decoder do
+  @moduledoc """
+  Element that decodes a VP8 stream
+  """
+  use Membrane.Filter
+
+  alias Membrane.{VP8, VPx}
+
+  def_options width: [
+                spec: non_neg_integer() | nil,
+                default: nil,
+                description: """
+                Width of a frame, needed if not provided with stream format. If it's not specified either in this option or the stream format, the element will crash.
+                """
+              ],
+              height: [
+                spec: non_neg_integer() | nil,
+                default: nil,
+                description: """
+                Height of a frame, needed if not provided with stream format. If it's not specified either in this option or the stream format, the element will crash.
+                """
+              ],
+              framerate: [
+                spec: {non_neg_integer(), pos_integer()} | nil,
+                default: nil,
+                description: """
+                Framerate, needed if not provided with stream format. If it's not specified either in this option or the stream format, the element will crash.
+                """
+              ]
+
+  def_input_pad :input,
+    accepted_format:
+      any_of(VP8, %Membrane.RemoteStream{content_format: format} when format in [nil, VP8])
+
+  def_output_pad :output,
+    accepted_format: Membrane.RawVideo
+
+  @impl true
+  def handle_init(ctx, opts) do
+    VPx.Decoder.handle_init(ctx, opts, :vp8)
+  end
+
+  @impl true
+  defdelegate handle_setup(ctx, state), to: VPx.Decoder
+
+  @impl true
+  defdelegate handle_stream_format(pad, stream_format, ctx, state), to: VPx.Decoder
+
+  @impl true
+  defdelegate handle_buffer(pad, buffer, ctx, state), to: VPx.Decoder
+end

lib/membrane_vpx/decoder/vp9_decoder.ex

@@ -0,0 +1,51 @@
+defmodule Membrane.VP9.Decoder do
+  @moduledoc """
+  Element that decodes a VP9 stream
+  """
+  use Membrane.Filter
+
+  alias Membrane.{VP9, VPx}
+
+  def_options width: [
+                spec: non_neg_integer() | nil,
+                default: nil,
+                description: """
+                Width of a frame, needed if not provided with stream format. If it's not specified either in this option or the stream format, the element will crash.
+                """
+              ],
+              height: [
+                spec: non_neg_integer() | nil,
+                default: nil,
+                description: """
+                Height of a frame, needed if not provided with stream format. If it's not specified either in this option or the stream format, the element will crash.
+                """
+              ],
+              framerate: [
+                spec: {non_neg_integer(), pos_integer()} | nil,
+                default: nil,
+                description: """
+                Framerate, needed if not provided with stream format. If it's not specified either in this option or the stream format, the element will crash.
+                """
+              ]
+
+  def_input_pad :input,
+    accepted_format:
+      any_of(VP9, %Membrane.RemoteStream{content_format: format} when format in [nil, VP9])
+
+  def_output_pad :output,
+    accepted_format: Membrane.RawVideo
+
+  @impl true
+  def handle_init(ctx, opts) do
+    VPx.Decoder.handle_init(ctx, opts, :vp9)
+  end
+
+  @impl true
+  defdelegate handle_setup(ctx, state), to: VPx.Decoder
+
+  @impl true
+  defdelegate handle_stream_format(pad, stream_format, ctx, state), to: VPx.Decoder
+
+  @impl true
+  defdelegate handle_buffer(pad, buffer, ctx, state), to: VPx.Decoder
+end