From b6e9044cab2a83b79e1063755ff106883df16a65 Mon Sep 17 00:00:00 2001 From: Bartosz Firyn Date: Mon, 25 Sep 2017 09:35:32 +0200 Subject: [PATCH] Add MJPEG support in GStreamer driver, refs #145 --- .../java/GStreamerDriverMjpegExample.java | 31 +++++ .../src/example/java/WebcamPanelExample.java | 10 +- .../webcam/ds/gstreamer/GStreamerDevice.java | 131 ++++++++++++------ .../webcam/ds/gstreamer/GStreamerDriver.java | 41 +++++- 4 files changed, 157 insertions(+), 56 deletions(-) create mode 100644 webcam-capture-drivers/driver-gstreamer/src/example/java/GStreamerDriverMjpegExample.java diff --git a/webcam-capture-drivers/driver-gstreamer/src/example/java/GStreamerDriverMjpegExample.java b/webcam-capture-drivers/driver-gstreamer/src/example/java/GStreamerDriverMjpegExample.java new file mode 100644 index 00000000..b2b743e3 --- /dev/null +++ b/webcam-capture-drivers/driver-gstreamer/src/example/java/GStreamerDriverMjpegExample.java @@ -0,0 +1,31 @@ +import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_MJPEG; +import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_RGB; +import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_YUV; + +import java.util.Arrays; + +import javax.swing.JFrame; + +import com.github.sarxos.webcam.Webcam; +import com.github.sarxos.webcam.WebcamPanel; +import com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver; + + +public class GStreamerDriverMjpegExample { + + static { + Webcam.setDriver(new GStreamerDriver(Arrays.asList(FORMAT_MJPEG, FORMAT_RGB, FORMAT_YUV))); + } + + public static void main(String[] args) { + + WebcamPanel panel = new WebcamPanel(Webcam.getWebcams().get(0)); + panel.setFPSDisplayed(true); + + JFrame frame = new JFrame("GStreamer Webcam Capture Driver Demo"); + frame.add(panel); + frame.pack(); + frame.setVisible(true); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + } +} diff --git a/webcam-capture-drivers/driver-gstreamer/src/example/java/WebcamPanelExample.java b/webcam-capture-drivers/driver-gstreamer/src/example/java/WebcamPanelExample.java index d7838271..0a634834 100644 --- a/webcam-capture-drivers/driver-gstreamer/src/example/java/WebcamPanelExample.java +++ b/webcam-capture-drivers/driver-gstreamer/src/example/java/WebcamPanelExample.java @@ -2,6 +2,7 @@ import com.github.sarxos.webcam.Webcam; import com.github.sarxos.webcam.WebcamPanel; +import com.github.sarxos.webcam.WebcamPanel.DrawMode; import com.github.sarxos.webcam.WebcamResolution; import com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver; @@ -14,15 +15,16 @@ public class WebcamPanelExample { public static void main(String[] args) { + WebcamResolution resolution = WebcamResolution.HD; Webcam webcam = Webcam.getDefault(); - webcam.setViewSize(WebcamResolution.HD720.getSize()); + webcam.setViewSize(resolution.getSize()); WebcamPanel panel = new WebcamPanel(webcam); - panel.setDisplayDebugInfo(true); panel.setFPSDisplayed(true); - panel.setFillArea(true); + panel.setDrawMode(DrawMode.FIT); + panel.setImageSizeDisplayed(true); - JFrame window = new JFrame("Test webcam panel"); + JFrame window = new JFrame(webcam + " @ " + resolution); window.add(panel); window.setResizable(true); window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); diff --git a/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDevice.java b/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDevice.java index 4bd3c7ad..9a0b3fe1 100644 --- a/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDevice.java +++ b/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDevice.java @@ -1,5 +1,7 @@ package com.github.sarxos.webcam.ds.gstreamer; +import static com.github.sarxos.webcam.ds.gstreamer.GStreamerDriver.FORMAT_MJPEG; + import java.awt.Dimension; import java.awt.image.BufferedImage; import java.awt.image.DataBufferInt; @@ -39,16 +41,6 @@ public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, Webc */ private static final long LATENESS = 20; // ms - /** - * First formats are better. For example video/x-raw-rgb gives 30 FPS on HD720p where - * video/x-raw-yuv only 10 FPS on the same resolution. The goal is to use these "better" formats - * first, and then fallback to less efficient when not available. - */ - private static final String[] BEST_FORMATS = { - "video/x-raw-rgb", - "video/x-raw-yuv", - }; - /** * Video format to capture. */ @@ -68,11 +60,16 @@ public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, Webc */ private final File vfile; + private final GStreamerDriver driver; + /* gstreamer stuff */ private Pipeline pipe = null; private Element source = null; private Element filter = null; + private Element jpegpar = null; + private Element jpegdec = null; + private Element[] elements = null; private RGBDataSink sink = null; private Caps caps = null; @@ -98,12 +95,14 @@ public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, Webc * * @param name the name of webcam device */ - protected GStreamerDevice(String name) { + protected GStreamerDevice(GStreamerDriver driver, String name) { + this.driver = driver; this.name = name; this.vfile = null; } - protected GStreamerDevice(File vfile) { + protected GStreamerDevice(GStreamerDriver driver, File vfile) { + this.driver = driver; this.name = null; this.vfile = vfile; } @@ -122,10 +121,10 @@ private synchronized void init() { pipe = new Pipeline(name); if (Platform.isWindows()) { - source = ElementFactory.make("dshowvideosrc", "source"); + source = ElementFactory.make("dshowvideosrc", "dshowvideosrc"); source.set("device-name", name); } else if (Platform.isLinux()) { - source = ElementFactory.make("v4l2src", "source"); + source = ElementFactory.make("v4l2src", "v4l2src"); source.set("device", vfile.getAbsolutePath()); } @@ -134,21 +133,20 @@ private synchronized void init() { sink.getSinkElement().setMaximumLateness(LATENESS, TimeUnit.MILLISECONDS); sink.getSinkElement().setQOSEnabled(true); - filter = ElementFactory.make("capsfilter", "filter"); + filter = ElementFactory.make("capsfilter", "capsfilter"); - if (Platform.isLinux()) { - pipe.addMany(source, filter, sink); - Element.linkMany(source, filter, sink); - pipe.setState(State.READY); - } + jpegpar = ElementFactory.make("jpegparse", "jpegparse"); + jpegdec = ElementFactory.make("jpegdec", "jpegdec"); + + // if (Platform.isLinux()) { + pipelineReady(); + // } resolutions = parseResolutions(source.getPads().get(0)); - if (Platform.isLinux()) { - pipe.setState(State.NULL); - Element.unlinkMany(source, filter, sink); - pipe.removeMany(source, filter, sink); - } + // if (Platform.isLinux()) { + pipelineStop(); + // } } /** @@ -161,7 +159,7 @@ private Dimension[] parseResolutions(Pad pad) { Caps caps = pad.getCaps(); - format = findBestFormat(caps); + format = findPreferredFormat(caps); LOG.debug("Best format is {}", format); @@ -169,7 +167,7 @@ private Dimension[] parseResolutions(Pad pad) { Structure s = null; String mime = null; - int n = caps.size(); + final int n = caps.size(); int i = 0; Map map = new HashMap(); @@ -190,19 +188,19 @@ private Dimension[] parseResolutions(Pad pad) { } while (i < n); - Dimension[] resolutions = new ArrayList(map.values()).toArray(new Dimension[map.size()]); + final Dimension[] resolutions = new ArrayList(map.values()).toArray(new Dimension[0]); if (LOG.isDebugEnabled()) { for (Dimension d : resolutions) { - LOG.debug("Resolution detected {}", d); + LOG.debug("Resolution detected {} with format {}", d, format); } } return resolutions; } - private static String findBestFormat(Caps caps) { - for (String f : BEST_FORMATS) { + private String findPreferredFormat(Caps caps) { + for (String f : driver.getPreferredFormats()) { for (int i = 0, n = caps.size(); i < n; i++) { if (f.equals(caps.getStructure(i).getName())) { return f; @@ -287,19 +285,17 @@ public void open() { caps.dispose(); } - caps = Caps.fromString(String.format("%s,width=%d,height=%d", format, size.width, size.height)); - + caps = Caps.fromString(String.format("%s,framerate=30/1,width=%d,height=%d", format, size.width, size.height)); filter.setCaps(caps); - LOG.debug("Link elements"); + LOG.debug("Using filter caps: {}", caps); - pipe.addMany(source, filter, sink); - Element.linkMany(source, filter, sink); - pipe.setState(State.PLAYING); + pipelinePlay(); + + LOG.debug("Wait for device to be ready"); // wait max 20s for image to appear synchronized (this) { - LOG.debug("Wait for device to be ready"); try { this.wait(20000); } catch (InterruptedException e) { @@ -308,6 +304,51 @@ public void open() { } } + private void pipelineElementsReset() { + elements = null; + } + + private Element[] pipelineElementsPrepare() { + if (elements == null) { + if (FORMAT_MJPEG.equals(format)) { + elements = new Element[] { source, filter, jpegpar, jpegdec, sink }; + } else { + elements = new Element[] { source, filter, sink }; + } + } + return elements; + } + + private void pipelineElementsLink() { + final Element[] elements = pipelineElementsPrepare(); + pipe.addMany(elements); + if (!Element.linkMany(elements)) { + LOG.warn("Some elements were not successfully linked!"); + } + } + + private void pipelineElementsUnlink() { + final Element[] elements = pipelineElementsPrepare(); + Element.unlinkMany(elements); + pipe.removeMany(elements); + } + + private void pipelineReady() { + pipelineElementsLink(); + pipe.setState(State.READY); + } + + private void pipelinePlay() { + pipelineElementsReset(); + pipelineElementsLink(); + pipe.setState(State.PLAYING); + } + + private void pipelineStop() { + pipe.setState(State.NULL); + pipelineElementsUnlink(); + } + @Override public void close() { @@ -317,13 +358,9 @@ public void close() { LOG.debug("Closing GStreamer device"); - image = null; - - LOG.debug("Unlink elements"); + pipelineStop(); - pipe.setState(State.NULL); - Element.unlinkMany(source, filter, sink); - pipe.removeMany(source, filter, sink); + image = null; } @Override @@ -337,11 +374,13 @@ public void dispose() { close(); - filter.dispose(); source.dispose(); + filter.dispose(); + jpegpar.dispose(); + jpegdec.dispose(); + caps.dispose(); sink.dispose(); pipe.dispose(); - caps.dispose(); } @Override diff --git a/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDriver.java b/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDriver.java index ec7e2137..a45680ce 100644 --- a/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDriver.java +++ b/webcam-capture-drivers/driver-gstreamer/src/main/java/com/github/sarxos/webcam/ds/gstreamer/GStreamerDriver.java @@ -2,6 +2,7 @@ import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -45,13 +46,20 @@ public void run() { private static final AtomicBoolean INITIALIZED = new AtomicBoolean(false); public GStreamerDriver() { - if (INITIALIZED.compareAndSet(false, true)) { - init(); - } + init(); + } + + public GStreamerDriver(final List preferredFormats) { + init(); + setPreferredFormats(preferredFormats); } private static final void init() { + if (!INITIALIZED.compareAndSet(false, true)) { + return; + } + if (!Platform.isWindows() && !Platform.isLinux()) { throw new WebcamException(String.format("%s has been designed to work only on Windows and Linux platforms", GStreamerDriver.class.getSimpleName())); } @@ -92,6 +100,27 @@ private static final void init() { Runtime.getRuntime().addShutdownHook(new GStreamerShutdownHook()); } + public static final String FORMAT_RGB = "video/x-raw-rgb"; + public static final String FORMAT_YUV = "video/x-raw-yuv"; + public static final String FORMAT_MJPEG = "image/jpeg"; + + private List preferredFormats = new ArrayList<>(Arrays.asList(FORMAT_RGB, FORMAT_YUV, FORMAT_MJPEG)); + + /** + * Set preferred video formats for this driver. First formats from the list are better and will + * be selected if available. + */ + public void setPreferredFormats(List preferredFormats) { + if (preferredFormats.isEmpty()) { + throw new IllegalArgumentException("Preferred formats list must not be empty"); + } + this.preferredFormats = new ArrayList<>(preferredFormats); + } + + public List getPreferredFormats() { + return preferredFormats; + } + @Override public List getDevices() { @@ -106,17 +135,17 @@ public List getDevices() { srcname = "qtkitvideosrc"; } - Element src = ElementFactory.make(srcname, "source"); + final Element src = ElementFactory.make(srcname, "source"); try { if (Platform.isWindows()) { PropertyProbe probe = PropertyProbe.wrap(src); for (Object name : probe.getValues("device-name")) { - devices.add(new GStreamerDevice(name.toString())); + devices.add(new GStreamerDevice(this, name.toString())); } } else if (Platform.isLinux()) { for (File vfile : NixVideoDevUtils.getVideoFiles()) { - devices.add(new GStreamerDevice(vfile)); + devices.add(new GStreamerDevice(this, vfile)); } } else { throw new RuntimeException("Platform unsupported by GStreamer capture driver");