[qtAV] Image sequence caching

CMakeLists.txt

@@ -39,6 +39,7 @@ add_definitions(-DQT_ALICEVISIONIMAGEIO_USE_FORMATS_BLACKLIST)
 
 
 # AliceVision dependency
+add_definitions(-DBOOST_USE_WINAPI_VERSION=BOOST_WINAPI_VERSION_WIN7)
 find_package(AliceVision REQUIRED)
 
 

src/qtAliceVision/CMakeLists.txt

@@ -11,6 +11,8 @@ set(PLUGIN_SOURCES
     MSfMDataStats.cpp
     PanoramaViewer.cpp
     Painter.cpp
+    SequenceCache.cpp
+    SingleImageLoader.cpp
     )
 
 set(PLUGIN_HEADERS
@@ -27,6 +29,9 @@ set(PLUGIN_HEADERS
     Surface.hpp
     ShaderImageViewer.hpp
     Painter.hpp
+    ImageServer.hpp
+    SequenceCache.hpp
+    SingleImageLoader.hpp
     )
 
 set(PLUGIN_MOCS
@@ -36,6 +41,8 @@ set(PLUGIN_MOCS
     MViewStats.hpp
     MTracks.hpp
     MSfMDataStats.hpp
+    SequenceCache.hpp
+    SingleImageLoader.hpp
     )
 
 

src/qtAliceVision/FloatImageViewer.cpp

@@ -7,74 +7,14 @@
 #include <QSGTexture>
 #include <QThreadPool>
 
-#include <aliceVision/image/Image.hpp>
-#include <aliceVision/image/convertion.hpp>
-#include <aliceVision/image/io.hpp>
-#include <aliceVision/image/resampling.hpp>
+#include <cmath>
+#include <algorithm>
+#include <vector>
+#include <utility>
 
 namespace qtAliceVision
 {
 
-FloatImageIORunnable::FloatImageIORunnable(const QUrl& path, int downscaleLevel, QObject* parent)
-    : QObject(parent)
-    , _path(path)
-    , _downscaleLevel(downscaleLevel)
-{
-}
-
-void FloatImageIORunnable::run()
-{
-    using namespace aliceVision;
-    QSharedPointer<FloatImage> result;
-    QSize sourceSize(0, 0);
-    QVariantMap qmetadata;
-
-    try
-    {
-        const auto path = _path.toLocalFile().toUtf8().toStdString();
-        FloatImage image;
-        image::readImage(path, image,
-                         image::EImageColorSpace::LINEAR); // linear: sRGB conversion is done in display shader
-
-        sourceSize = QSize(image.Width(), image.Height());
-
-        // ensure it fits in GPU memory
-        if (FloatTexture::maxTextureSize() != -1)
-        {
-            const auto maxTextureSize = FloatTexture::maxTextureSize();
-            while (maxTextureSize != -1 && (image.Width() > maxTextureSize || image.Height() > maxTextureSize))
-            {
-                FloatImage tmp;
-                aliceVision::image::ImageHalfSample(image, tmp);
-                image = std::move(tmp);
-            }
-        }
-
-        // ensure it fits in RAM memory
-        for (int i = 0; i < _downscaleLevel; i++)
-        {
-            FloatImage tmp;
-            aliceVision::image::ImageHalfSample(image, tmp);
-            image = std::move(tmp);
-        }
-
-        // load metadata as well
-        const auto metadata = image::readImageMetadata(path);
-        for (const auto& item : metadata)
-        {
-            qmetadata[QString::fromStdString(item.name().string())] = QString::fromStdString(item.get_string());
-        }
-
-        result = QSharedPointer<FloatImage>(new FloatImage(std::move(image)));
-    }
-    catch (std::exception& e)
-    {
-        qInfo() << "[QtAliceVision] Failed to load image: " << _path << "\n" << e.what();
-    }
-
-    Q_EMIT resultReady(result, sourceSize, qmetadata);
-}
-
 FloatImageViewer::FloatImageViewer(QQuickItem* parent)
     : QQuickItem(parent)
 {
@@ -102,9 +42,15 @@ FloatImageViewer::FloatImageViewer(QQuickItem* parent)
 
     connect(&_surface, &Surface::subdivisionsChanged, this, &FloatImageViewer::update);
     connect(&_surface, &Surface::verticesChanged, this, &FloatImageViewer::update);
+
+    connect(&_sequenceCache, &imgserve::SequenceCache::requestHandled, this, &FloatImageViewer::reload);
+    connect(&_singleImageLoader, &imgserve::SingleImageLoader::requestHandled, this, &FloatImageViewer::reload);
+    connect(this, &FloatImageViewer::sequenceChanged, this, &FloatImageViewer::reload);
+    connect(this, &FloatImageViewer::useSequenceChanged, this, &FloatImageViewer::reload);
 }
 
-FloatImageViewer::~FloatImageViewer() {}
+FloatImageViewer::~FloatImageViewer()
+{}
 
 // LOADING FUNCTIONS
 void FloatImageViewer::setLoading(bool loading)
@@ -117,6 +63,18 @@ void FloatImageViewer::setLoading(bool loading)
     Q_EMIT loadingChanged();
 }
 
+void FloatImageViewer::setSequence(const QVariantList& paths)
+{
+    _sequenceCache.setSequence(paths);
+
+    Q_EMIT sequenceChanged();
+}
+
+QVariantList FloatImageViewer::getCachedFrames() const
+{
+    return _sequenceCache.getCachedFrames();
+}
+
 void FloatImageViewer::reload()
 {
     if (_clearBeforeLoad)
@@ -125,12 +83,12 @@ void FloatImageViewer::reload()
         _imageChanged = true;
         Q_EMIT imageChanged();
     }
+
     _outdated = false;
+    if (_loading) _outdated = true;
 
     if (!_source.isValid())
     {
-        if (_loading)
-            _outdated = true;
         _image.reset();
         _imageChanged = true;
         _surface.clearVertices();
@@ -139,44 +97,35 @@ void FloatImageViewer::reload()
         return;
     }
 
-    if (!_loading)
-    {
-        setLoading(true);
+    // Send request
+    imgserve::RequestData reqData;
+    reqData.path = _source.toLocalFile().toUtf8().toStdString();
+    reqData.downscale = 1 << _downscaleLevel;
 
-        // async load from file
-        auto ioRunnable = new FloatImageIORunnable(_source, _downscaleLevel);
-        connect(ioRunnable, &FloatImageIORunnable::resultReady, this, &FloatImageViewer::onResultReady);
-        QThreadPool::globalInstance()->start(ioRunnable);
-    }
-    else
+    imgserve::ResponseData response = _useSequence ? _sequenceCache.request(reqData) : _singleImageLoader.request(reqData);
+
+    if (response.img)
     {
-        _outdated = true;
-    }
-}
+        setLoading(false);
 
-void FloatImageViewer::onResultReady(QSharedPointer<FloatImage> image, QSize sourceSize, const QVariantMap& metadata)
-{
-    setLoading(false);
+        _surface.setVerticesChanged(true);
+        _surface.setNeedToUseIntrinsic(true);
+        _image = response.img;
+        _imageChanged = true;
+        Q_EMIT imageChanged();
+
+        _sourceSize = response.dim;
+        Q_EMIT sourceSizeChanged();
 
-    if (_outdated)
+        _metadata = response.metadata;
+        Q_EMIT metadataChanged();
+    }
+    else
     {
-        // another request has been made while io thread was working
-        _image.reset();
-        reload();
-        return;
+        setLoading(true);
     }
 
-    _surface.setVerticesChanged(true);
-    _surface.setNeedToUseIntrinsic(true);
-    _image = image;
-    _imageChanged = true;
-    Q_EMIT imageChanged();
-
-    _sourceSize = sourceSize;
-    Q_EMIT sourceSizeChanged();
-
-    _metadata = metadata;
-    Q_EMIT metadataChanged();
+    Q_EMIT cachedFramesChanged();
 }
 
 QVector4D FloatImageViewer::pixelValueAt(int x, int y)
@@ -421,4 +370,4 @@ void FloatImageViewer::updatePaintSurface(QSGGeometryNode* root, QSGSimpleMateri
     root->childAtIndex(0)->markDirty(QSGNode::DirtyGeometry | QSGNode::DirtyMaterial);
 }
 
-} // namespace qtAliceVision
+} // namespace qtAliceVision

src/qtAliceVision/FloatImageViewer.hpp

@@ -1,47 +1,27 @@
 #pragma once
 
 #include "FloatTexture.hpp"
-#include "ShaderImageViewer.hpp"
 #include "Surface.hpp"
+#include "ShaderImageViewer.hpp"
+#include "SequenceCache.hpp"
+#include "SingleImageLoader.hpp"
+
+#include <aliceVision/image/all.hpp>
 
 #include <QQuickItem>
 #include <QRunnable>
 #include <QSGGeometryNode>
 #include <QSGSimpleMaterial>
-#include <QUrl>
-
-#include <QSharedPointer>
 #include <QVariant>
 #include <QVector4D>
+#include <QList>
 
 #include <memory>
 #include <string>
 
 namespace qtAliceVision
 {
 
-/**
- * @brief QRunnable object dedicated to load image using AliceVision.
- */
-class FloatImageIORunnable : public QObject, public QRunnable
-{
-    Q_OBJECT
-
-public:
-    explicit FloatImageIORunnable(const QUrl& path, int downscaleLevel = 0, QObject* parent = nullptr);
-
-    /// Load image at path
-    Q_SLOT void run() override;
-
-    /// Emitted when the image is loaded
-    Q_SIGNAL void resultReady(QSharedPointer<qtAliceVision::FloatImage> image, QSize sourceSize,
-                              const QVariantMap& metadata);
-
-private:
-    QUrl _path;
-    int _downscaleLevel;
-};
-
 /**
  * @brief Load and display image.
  */
@@ -76,6 +56,12 @@ class FloatImageViewer : public QQuickItem
 
     Q_PROPERTY(bool cropFisheye READ getCropFisheye WRITE setCropFisheye NOTIFY isCropFisheyeChanged)
 
+    Q_PROPERTY(QVariantList sequence WRITE setSequence NOTIFY sequenceChanged)
+
+    Q_PROPERTY(QVariantList cachedFrames READ getCachedFrames NOTIFY cachedFramesChanged)
+
+    Q_PROPERTY(bool useSequence MEMBER _useSequence NOTIFY useSequenceChanged)
+
 public:
     explicit FloatImageViewer(QQuickItem* parent = nullptr);
     ~FloatImageViewer() override;
@@ -134,18 +120,23 @@ class FloatImageViewer : public QQuickItem
     Q_SIGNAL void canBeHoveredChanged();
     Q_SIGNAL void sfmRequiredChanged();
     Q_SIGNAL void fisheyeCircleParametersChanged();
+    Q_SIGNAL void sequenceChanged();
+    Q_SIGNAL void cachedFramesChanged();
+    Q_SIGNAL void useSequenceChanged();
 
     // Q_INVOKABLE
     Q_INVOKABLE QVector4D pixelValueAt(int x, int y);
 
     Surface* getSurfacePtr() { return &_surface; }
 
+    void setSequence(const QVariantList& paths);
+
+    QVariantList getCachedFrames() const;
+
 private:
     /// Reload image from source
     void reload();
-    /// Handle result from asynchronous file loading
-    Q_SLOT void onResultReady(QSharedPointer<qtAliceVision::FloatImage> image, QSize sourceSize,
-                              const QVariantMap& metadata);
+
     /// Custom QSGNode update
     QSGNode* updatePaintNode(QSGNode* oldNode, QQuickItem::UpdatePaintNodeData* data) override;
 
@@ -161,7 +152,7 @@ class FloatImageViewer : public QQuickItem
 
     bool _imageChanged = false;
     EChannelMode _channelMode;
-    QSharedPointer<FloatImage> _image;
+    std::shared_ptr<FloatImage> _image;
     QRectF _boundingRect;
     QSize _textureSize;
     QSize _sourceSize = QSize(0, 0);
@@ -177,9 +168,13 @@ class FloatImageViewer : public QQuickItem
     bool _canBeHovered = false;
 
     bool _cropFisheye = false;
+
+    imgserve::SequenceCache _sequenceCache;
+    imgserve::SingleImageLoader _singleImageLoader;
+    bool _useSequence = true;
 };
 
 } // namespace qtAliceVision
 
 Q_DECLARE_METATYPE(qtAliceVision::FloatImage)
-Q_DECLARE_METATYPE(QSharedPointer<qtAliceVision::FloatImage>)
+Q_DECLARE_METATYPE(std::shared_ptr<qtAliceVision::FloatImage>)

src/qtAliceVision/FloatTexture.cpp

@@ -20,7 +20,7 @@ FloatTexture::~FloatTexture()
     }
 }
 
-void FloatTexture::setImage(QSharedPointer<FloatImage>& image)
+void FloatTexture::setImage(std::shared_ptr<FloatImage>& image)
 {
     _srcImage = image;
     _textureSize = {_srcImage->Width(), _srcImage->Height()};

src/qtAliceVision/FloatTexture.hpp

@@ -6,7 +6,7 @@
 
 #include <QSGTexture>
 
-#include <QSharedPointer>
+#include <memory>
 
 namespace qtAliceVision
 {
@@ -30,8 +30,8 @@ class FloatTexture : public QSGTexture
 
     bool hasMipmaps() const override { return mipmapFiltering() != QSGTexture::None; }
 
-    void setImage(QSharedPointer<FloatImage>& image);
-    const FloatImage& image() { return *_srcImage; }
+    void setImage(std::shared_ptr<FloatImage>& image);
+    const FloatImage &image() { return *_srcImage; }
 
     void bind() override;
 
@@ -48,7 +48,7 @@ class FloatTexture : public QSGTexture
     bool isValid() const;
 
 private:
-    QSharedPointer<FloatImage> _srcImage;
+    std::shared_ptr<FloatImage> _srcImage;
 
     unsigned int _textureId = 0;
     QSize _textureSize;