Skip to content

Commit

Permalink
Add MJPEG support in GStreamer driver, refs #145
Browse files Browse the repository at this point in the history
  • Loading branch information
@sarxos
sarxos committed Sep 25, 2017
1 parent 524f7ba commit b6e9044
Show file tree
Hide file tree
Showing 4 changed files with 157 additions and 56 deletions.
31 changes: 31 additions & 0 deletions webcam-capture-drivers/driver-gstreamer/src/example/java/GStreamerDriverMjpegExample.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_MJPEG;
import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_RGB;
import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_YUV;

import java.util.Arrays;

import javax.swing.JFrame;

import com.github.sarxos.webcam.Webcam;
import com.github.sarxos.webcam.WebcamPanel;
import com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver;


public class GStreamerDriverMjpegExample {

static {
Webcam.setDriver(new GStreamerDriver(Arrays.asList(FORMAT_MJPEG, FORMAT_RGB, FORMAT_YUV)));
}

public static void main(String[] args) {

WebcamPanel panel = new WebcamPanel(Webcam.getWebcams().get(0));
panel.setFPSDisplayed(true);

JFrame frame = new JFrame("GStreamer Webcam Capture Driver Demo");
frame.add(panel);
frame.pack();
frame.setVisible(true);
frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
}
}
10 changes: 6 additions & 4 deletions webcam-capture-drivers/driver-gstreamer/src/example/java/WebcamPanelExample.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.github.sarxos.webcam.Webcam;
import com.github.sarxos.webcam.WebcamPanel;
import com.github.sarxos.webcam.WebcamPanel.DrawMode;
import com.github.sarxos.webcam.WebcamResolution;
import com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver;

Expand All @@ -14,15 +15,16 @@ public class WebcamPanelExample {

public static void main(String[] args) {

WebcamResolution resolution = WebcamResolution.HD;
Webcam webcam = Webcam.getDefault();
webcam.setViewSize(WebcamResolution.HD720.getSize());
webcam.setViewSize(resolution.getSize());

WebcamPanel panel = new WebcamPanel(webcam);
panel.setDisplayDebugInfo(true);
panel.setFPSDisplayed(true);
panel.setFillArea(true);
panel.setDrawMode(DrawMode.FIT);
panel.setImageSizeDisplayed(true);

JFrame window = new JFrame("Test webcam panel");
JFrame window = new JFrame(webcam + " @ " + resolution);
window.add(panel);
window.setResizable(true);
window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
Expand Down
131 changes: 85 additions & 46 deletions ...driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDevice.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.github.sarxos.webcam.ds.gstreamer;

import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_MJPEG;

import java.awt.Dimension;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferInt;
Expand Down Expand Up @@ -39,16 +41,6 @@ public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, Webc
*/
private static final long LATENESS = 20; // ms

/**
* First formats are better. For example video/x-raw-rgb gives 30 FPS on HD720p where
* video/x-raw-yuv only 10 FPS on the same resolution. The goal is to use these "better" formats
* first, and then fallback to less efficient when not available.
*/
private static final String[] BEST_FORMATS = {
"video/x-raw-rgb",
"video/x-raw-yuv",
};

/**
* Video format to capture.
*/
Expand All @@ -68,11 +60,16 @@ public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, Webc
*/
private final File vfile;

private final GStreamerDriver driver;

/* gstreamer stuff */

private Pipeline pipe = null;
private Element source = null;
private Element filter = null;
private Element jpegpar = null;
private Element jpegdec = null;
private Element[] elements = null;
private RGBDataSink sink = null;

private Caps caps = null;
Expand All @@ -98,12 +95,14 @@ public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, Webc
*
* @param name the name of webcam device
*/
protected GStreamerDevice(String name) {
protected GStreamerDevice(GStreamerDriver driver, String name) {
this.driver = driver;
this.name = name;
this.vfile = null;
}

protected GStreamerDevice(File vfile) {
protected GStreamerDevice(GStreamerDriver driver, File vfile) {
this.driver = driver;
this.name = null;
this.vfile = vfile;
}
Expand All @@ -122,10 +121,10 @@ private synchronized void init() {
pipe = new Pipeline(name);

if (Platform.isWindows()) {
source = ElementFactory.make("dshowvideosrc", "source");
source = ElementFactory.make("dshowvideosrc", "dshowvideosrc");
source.set("device-name", name);
} else if (Platform.isLinux()) {
source = ElementFactory.make("v4l2src", "source");
source = ElementFactory.make("v4l2src", "v4l2src");
source.set("device", vfile.getAbsolutePath());
}

Expand All @@ -134,21 +133,20 @@ private synchronized void init() {
sink.getSinkElement().setMaximumLateness(LATENESS, TimeUnit.MILLISECONDS);
sink.getSinkElement().setQOSEnabled(true);

filter = ElementFactory.make("capsfilter", "filter");
filter = ElementFactory.make("capsfilter", "capsfilter");

if (Platform.isLinux()) {
pipe.addMany(source, filter, sink);
Element.linkMany(source, filter, sink);
pipe.setState(State.READY);
}
jpegpar = ElementFactory.make("jpegparse", "jpegparse");
jpegdec = ElementFactory.make("jpegdec", "jpegdec");

// if (Platform.isLinux()) {
pipelineReady();
// }

resolutions = parseResolutions(source.getPads().get(0));

if (Platform.isLinux()) {
pipe.setState(State.NULL);
Element.unlinkMany(source, filter, sink);
pipe.removeMany(source, filter, sink);
}
// if (Platform.isLinux()) {
pipelineStop();
// }
}

/**
Expand All @@ -161,15 +159,15 @@ private Dimension[] parseResolutions(Pad pad) {

Caps caps = pad.getCaps();

format = findBestFormat(caps);
format = findPreferredFormat(caps);

LOG.debug("Best format is {}", format);

Dimension r = null;
Structure s = null;
String mime = null;

int n = caps.size();
final int n = caps.size();
int i = 0;

Map<String, Dimension> map = new HashMap<String, Dimension>();
Expand All @@ -190,19 +188,19 @@ private Dimension[] parseResolutions(Pad pad) {

} while (i < n);

Dimension[] resolutions = new ArrayList<Dimension>(map.values()).toArray(new Dimension[map.size()]);
final Dimension[] resolutions = new ArrayList<Dimension>(map.values()).toArray(new Dimension[0]);

if (LOG.isDebugEnabled()) {
for (Dimension d : resolutions) {
LOG.debug("Resolution detected {}", d);
LOG.debug("Resolution detected {} with format {}", d, format);
}
}

return resolutions;
}

private static String findBestFormat(Caps caps) {
for (String f : BEST_FORMATS) {
private String findPreferredFormat(Caps caps) {
for (String f : driver.getPreferredFormats()) {
for (int i = 0, n = caps.size(); i < n; i++) {
if (f.equals(caps.getStructure(i).getName())) {
return f;
Expand Down Expand Up @@ -287,19 +285,17 @@ public void open() {
caps.dispose();
}

caps = Caps.fromString(String.format("%s,width=%d,height=%d", format, size.width, size.height));

caps = Caps.fromString(String.format("%s,framerate=30/1,width=%d,height=%d", format, size.width, size.height));
filter.setCaps(caps);

LOG.debug("Link elements");
LOG.debug("Using filter caps: {}", caps);

pipe.addMany(source, filter, sink);
Element.linkMany(source, filter, sink);
pipe.setState(State.PLAYING);
pipelinePlay();

LOG.debug("Wait for device to be ready");

// wait max 20s for image to appear
synchronized (this) {
LOG.debug("Wait for device to be ready");
try {
this.wait(20000);
} catch (InterruptedException e) {
Expand All @@ -308,6 +304,51 @@ public void open() {
}
}

private void pipelineElementsReset() {
elements = null;
}

private Element[] pipelineElementsPrepare() {
if (elements == null) {
if (FORMAT_MJPEG.equals(format)) {
elements = new Element[] { source, filter, jpegpar, jpegdec, sink };
} else {
elements = new Element[] { source, filter, sink };
}
}
return elements;
}

private void pipelineElementsLink() {
final Element[] elements = pipelineElementsPrepare();
pipe.addMany(elements);
if (!Element.linkMany(elements)) {
LOG.warn("Some elements were not successfully linked!");
}
}

private void pipelineElementsUnlink() {
final Element[] elements = pipelineElementsPrepare();
Element.unlinkMany(elements);
pipe.removeMany(elements);
}

private void pipelineReady() {
pipelineElementsLink();
pipe.setState(State.READY);
}

private void pipelinePlay() {
pipelineElementsReset();
pipelineElementsLink();
pipe.setState(State.PLAYING);
}

private void pipelineStop() {
pipe.setState(State.NULL);
pipelineElementsUnlink();
}

@Override
public void close() {

Expand All @@ -317,13 +358,9 @@ public void close() {

LOG.debug("Closing GStreamer device");

image = null;

LOG.debug("Unlink elements");
pipelineStop();

pipe.setState(State.NULL);
Element.unlinkMany(source, filter, sink);
pipe.removeMany(source, filter, sink);
image = null;
}

@Override
Expand All @@ -337,11 +374,13 @@ public void dispose() {

close();

filter.dispose();
source.dispose();
filter.dispose();
jpegpar.dispose();
jpegdec.dispose();
caps.dispose();
sink.dispose();
pipe.dispose();
caps.dispose();
}

@Override
Expand Down
41 changes: 35 additions & 6 deletions ...driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;

Expand Down Expand Up @@ -45,13 +46,20 @@ public void run() {
private static final AtomicBoolean INITIALIZED = new AtomicBoolean(false);

public GStreamerDriver() {
if (INITIALIZED.compareAndSet(false, true)) {
init();
}
init();
}

public GStreamerDriver(final List<String> preferredFormats) {
init();
setPreferredFormats(preferredFormats);
}

private static final void init() {

if (!INITIALIZED.compareAndSet(false, true)) {
return;
}

if (!Platform.isWindows() && !Platform.isLinux()) {
throw new WebcamException(String.format("%s has been designed to work only on Windows and Linux platforms", GStreamerDriver.class.getSimpleName()));
}
Expand Down Expand Up @@ -92,6 +100,27 @@ private static final void init() {
Runtime.getRuntime().addShutdownHook(new GStreamerShutdownHook());
}

public static final String FORMAT_RGB = "video/x-raw-rgb";
public static final String FORMAT_YUV = "video/x-raw-yuv";
public static final String FORMAT_MJPEG = "image/jpeg";

private List<String> preferredFormats = new ArrayList<>(Arrays.asList(FORMAT_RGB, FORMAT_YUV, FORMAT_MJPEG));

/**
* Set preferred video formats for this driver. First formats from the list are better and will
* be selected if available.
*/
public void setPreferredFormats(List<String> preferredFormats) {
if (preferredFormats.isEmpty()) {
throw new IllegalArgumentException("Preferred formats list must not be empty");
}
this.preferredFormats = new ArrayList<>(preferredFormats);
}

public List<String> getPreferredFormats() {
return preferredFormats;
}

@Override
public List<WebcamDevice> getDevices() {

Expand All @@ -106,17 +135,17 @@ public List<WebcamDevice> getDevices() {
srcname = "qtkitvideosrc";
}

Element src = ElementFactory.make(srcname, "source");
final Element src = ElementFactory.make(srcname, "source");

try {
if (Platform.isWindows()) {
PropertyProbe probe = PropertyProbe.wrap(src);
for (Object name : probe.getValues("device-name")) {
devices.add(new GStreamerDevice(name.toString()));
devices.add(new GStreamerDevice(this, name.toString()));
}
} else if (Platform.isLinux()) {
for (File vfile : NixVideoDevUtils.getVideoFiles()) {
devices.add(new GStreamerDevice(vfile));
devices.add(new GStreamerDevice(this, vfile));
}
} else {
throw new RuntimeException("Platform unsupported by GStreamer capture driver");
Expand Down

0 comments on commit b6e9044

Please sign in to comment.