LMMS · lukas-w · Dec 18, 2017 · Jun 17, 2016 · Oct 2, 2016 · Jan 2, 2017
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "src/3rdparty/qt5-x11embed"]
+	path = src/3rdparty/qt5-x11embed
+	url = https://github.com/Lukas-W/qt5-x11embed.git
diff --git a/.travis/linux..install.sh b/.travis/linux..install.sh
@@ -2,11 +2,13 @@
 
 PACKAGES="cmake libsndfile-dev fftw3-dev libvorbis-dev libogg-dev libmp3lame-dev
 	libasound2-dev libjack-dev libsdl-dev libsamplerate0-dev libstk0-dev
-	libfluidsynth-dev portaudio19-dev wine-dev g++-multilib libfltk1.3-dev
+	libfluidsynth-dev portaudio19-dev g++-multilib libfltk1.3-dev
 	libgig-dev libsoundio-dev"
 
+VST_PACKAGES="wine-dev libqt5x11extras5-dev qtbase5-private-dev libxcb-util0-dev libxcb-keysyms1-dev"
+
 # Help with unmet dependencies
-PACKAGES="$PACKAGES libjack0"
+PACKAGES="$PACKAGES $VST_PACKAGES libjack0"
 
 if [ $QT5 ]; then
 	PACKAGES="$PACKAGES qtbase5-dev qttools5-dev-tools qttools5-dev"

diff --git a/include/RemotePlugin.h b/include/RemotePlugin.h
@@ -414,6 +414,7 @@ class shmFifo
 enum RemoteMessageIDs
 {
 	IdUndefined,
+	IdHostInfoGotten,
 	IdInitDone,
 	IdQuit,
 	IdSampleRateInformation,
@@ -426,6 +427,8 @@ enum RemoteMessageIDs
 	IdChangeOutputCount,
 	IdShowUI,
 	IdHideUI,
+	IdToggleUI,
+	IdIsUIVisible,
 	IdSaveSettingsToString,
 	IdSaveSettingsToFile,
 	IdLoadSettingsFromString,
@@ -769,6 +772,12 @@ class EXPORT RemotePlugin : public QObject, public RemotePluginBase
 
 	bool init( const QString &pluginExecutable, bool waitForInitDoneMsg );
 
+	inline void waitForHostInfoGotten()
+	{
+		m_failed = waitForMessage( IdHostInfoGotten ).id
+							!= IdHostInfoGotten;
+	}
+
 	inline void waitForInitDone( bool _busyWaiting = true )
 	{
 		m_failed = waitForMessage( IdInitDone, _busyWaiting ).id != IdInitDone;
@@ -787,18 +796,21 @@ class EXPORT RemotePlugin : public QObject, public RemotePluginBase
 		unlock();
 	}
 
-	void showUI()
+
+	void toggleUI()
 	{
 		lock();
-		sendMessage( IdShowUI );
+		sendMessage( IdToggleUI );
 		unlock();
 	}
 
-	void hideUI()
+	int isUIVisible()
 	{
 		lock();
-		sendMessage( IdHideUI );
+		sendMessage( IdIsUIVisible );
 		unlock();
+		message m = waitForMessage( IdIsUIVisible );
+		return m.id != IdIsUIVisible ? -1 : m.getInt() ? 1 : 0;
 	}
 
 	inline bool failed() const
@@ -816,6 +828,9 @@ class EXPORT RemotePlugin : public QObject, public RemotePluginBase
 		m_commMutex.unlock();
 	}
 
+public slots:
+	void showUI();
+	void hideUI();
 
 protected:
 	inline void setSplittedChannels( bool _on )
@@ -1155,6 +1170,7 @@ RemotePluginClient::RemotePluginClient( const char * socketPath ) :
 		m_vstSyncData = (VstSyncData *) m_shmQtID.data();
 		m_bufferSize = m_vstSyncData->m_bufferSize;
 		m_sampleRate = m_vstSyncData->m_sampleRate;
+		sendMessage( IdHostInfoGotten );
 		return;
 	}
 #else
@@ -1182,6 +1198,7 @@ RemotePluginClient::RemotePluginClient( const char * socketPath ) :
 			{
 				m_bufferSize = m_vstSyncData->m_bufferSize;
 				m_sampleRate = m_vstSyncData->m_sampleRate;
+				sendMessage( IdHostInfoGotten );
 
 				// detach segment
 				if( shmdt(m_vstSyncData) == -1 )
@@ -1197,6 +1214,12 @@ RemotePluginClient::RemotePluginClient( const char * socketPath ) :
 	// if attaching shared memory fails
 	sendMessage( IdSampleRateInformation );
 	sendMessage( IdBufferSizeInformation );
+	if( waitForMessage( IdBufferSizeInformation ).id
+						!= IdBufferSizeInformation )
+	{
+		fprintf( stderr, "Could not get buffer size information\n" );
+	}
+	sendMessage( IdHostInfoGotten );
 }
 
 

diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt
@@ -35,6 +35,22 @@ IF(LMMS_MINIMAL)
 	SET(PLUGIN_LIST ${MINIMAL_LIST} ${PLUGIN_LIST})
 ENDIF()
 
+OPTION(LMMS_EMBED_VST "Turn on to embed VSTs as subwindows. Turn off to open VSTs in separate windows." ON)
+
+IF(WANT_QT5)
+	OPTION(LMMS_EMBED_VST_X11 "Turn on to embed VSTs using the X11Embed protocol. Turn off to use Qt's own embedding, which may be broken with your Qt version." ON)
+ENDIF()
+
+IF(NOT LMMS_EMBED_VST OR NOT LMMS_BUILD_LINUX)
+	SET(LMMS_EMBED_VST_X11 OFF)
+ENDIF()
+
+# Qt4 has no QWindow::createWindowContainer, so we always use QX11EmbedContainer
+IF(LMMS_EMBED_VST AND NOT WANT_QT5)
+	SET(LMMS_EMBED_VST_X11 ON)
+ENDIF()
+
+
 IF("${PLUGIN_LIST}" STREQUAL "")
 	SET(PLUGIN_LIST
 		${MINIMAL_LIST}

diff --git a/plugins/VstEffect/VstEffectControlDialog.cpp b/plugins/VstEffect/VstEffectControlDialog.cpp
@@ -43,7 +43,9 @@
 
 VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 	EffectControlDialog( _ctl ),
+#ifdef LMMS_EMBED_VST
 	m_pluginWidget( NULL ),
+#endif
 	m_plugin( NULL ),
 	tbLabel( NULL )
 {
@@ -56,6 +58,7 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 					_ctl->m_effect->m_plugin != NULL )
 	{
 		m_plugin = _ctl->m_effect->m_plugin;
+#ifdef LMMS_EMBED_VST
 		m_plugin->showEditor( NULL, true );
 		m_pluginWidget = m_plugin->pluginWidget();
 
@@ -66,18 +69,31 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 			m_pluginWidget = m_plugin->pluginWidget( false );
 		}
 #endif
+
+#else // LMMS_EMBED_VST
+		m_plugin->showUI();
+#endif
 	}
 
+#ifdef LMMS_EMBED_VST
 	if( m_pluginWidget )
+#else
+	if( m_plugin )
+#endif
 	{
-		setWindowTitle( m_pluginWidget->windowTitle() );
+		setWindowTitle( m_plugin->name() );
 		setMinimumWidth( 250 );
 
 		QPushButton * btn = new QPushButton( tr( "Show/hide" ) );
+#ifdef LMMS_EMBED_VST
 		btn->setCheckable( true );
 		connect( btn, SIGNAL( toggled( bool ) ),
-					m_pluginWidget, SLOT( setVisible( bool ) ) );
+					SLOT( togglePluginUI( bool ) ) );
 		emit btn->click();
+#else
+		connect( btn, SIGNAL( clicked( bool ) ),
+					SLOT( togglePluginUI( bool ) ) );
+#endif
 
 		btn->setMinimumWidth( 78 );
 		btn->setMaximumWidth( 78 );
@@ -206,8 +222,12 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 		m_savePresetButton->setMinimumHeight( 21 );
 		m_savePresetButton->setMaximumHeight( 21 );
 
+#ifdef LMMS_EMBED_VST
 		int newSize = m_pluginWidget->width() + 20;
 		newSize = (newSize < 250) ? 250 : newSize;
+#else
+		int newSize = 250;
+#endif
 		QWidget* resize = new QWidget(this);
 		resize->resize( newSize, 10 );
 		QWidget* space0 = new QWidget(this);
@@ -219,7 +239,11 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 		l->addItem( new QSpacerItem( newSize - 20, 30, QSizePolicy::Fixed,
 						QSizePolicy::Fixed ), 1, 0 );
 		l->addWidget( resize, 2, 0, 1, 1, Qt::AlignCenter );
+#ifdef LMMS_EMBED_VST
+#if QT_VERSION < 0x050000
 		l->addWidget( m_pluginWidget, 3, 0, 1, 1, Qt::AlignCenter );
+#endif
+#endif
 		l->setRowStretch( 5, 1 );
 		l->setColumnStretch( 1, 1 );
 
@@ -264,3 +288,25 @@ VstEffectControlDialog::~VstEffectControlDialog()
 	//delete m_pluginWidget;
 }
 
+
+
+
+void VstEffectControlDialog::togglePluginUI( bool checked )
+{
+	if( m_plugin )
+	{
+#ifdef LMMS_EMBED_VST
+		if( checked )
+		{
+			m_plugin->showEditor( NULL, true );
+		}
+		else
+		{
+			m_plugin->hideEditor();
+		}
+#else
+		m_plugin->toggleUI();
+#endif
+	}
+}
+
diff --git a/plugins/VstEffect/VstEffectControlDialog.h b/plugins/VstEffect/VstEffectControlDialog.h
@@ -51,7 +51,9 @@ class VstEffectControlDialog : public EffectControlDialog
 	virtual void paintEvent( QPaintEvent * _pe );
 
 private:
+#ifdef LMMS_EMBED_VST
 	QWidget * m_pluginWidget;
+#endif
 
 	PixmapButton * m_openPresetButton;
 	PixmapButton * m_rolLPresetButton;
@@ -62,6 +64,9 @@ class VstEffectControlDialog : public EffectControlDialog
 	VstPlugin * m_plugin;
 
 	QLabel * tbLabel;
+
+private slots:
+	void togglePluginUI( bool checked );
 } ;
 
 #endif
diff --git a/plugins/vestige/CMakeLists.txt b/plugins/vestige/CMakeLists.txt
@@ -1,3 +1,10 @@
+IF(LMMS_EMBED_VST)
+    ADD_DEFINITIONS(-DLMMS_EMBED_VST)
+    IF(LMMS_EMBED_VST_X11)
+        ADD_DEFINITIONS(-DLMMS_EMBED_VST_X11)
+    ENDIF()
+ENDIF()
+
 IF(LMMS_SUPPORT_VST)
    INCLUDE(BuildPlugin)
    INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/../vst_base")

diff --git a/plugins/vestige/vestige.cpp b/plugins/vestige/vestige.cpp
@@ -261,7 +261,11 @@ void vestigeInstrument::loadFile( const QString & _file )
 		return;
 	}
 
+#ifdef LMMS_EMBED_VST
 	m_plugin->showEditor( NULL, false );
+#else
+	m_plugin->showUI();
+#endif
 
 	if( set_ch_name )
 	{
@@ -735,19 +739,11 @@ void VestigeInstrumentView::toggleGUI( void )
 	{
 		return;
 	}
-	QWidget * w = m_vi->m_plugin->pluginWidget();
-	if( w == NULL )
-	{
-		return;
-	}
-	if( w->isHidden() )
-	{
-		w->show();
-	}
-	else
-	{
-		w->hide();
-	}
+#ifdef LMMS_EMBED_VST
+	m_vi->m_plugin->toggleEditor();
+#else
+	m_vi->m_plugin->toggleUI();
+#endif
 }
 
 

diff --git a/plugins/vst_base/CMakeLists.txt b/plugins/vst_base/CMakeLists.txt
@@ -2,6 +2,16 @@ IF(LMMS_SUPPORT_VST)
 
 INCLUDE(BuildPlugin)
 
+IF(LMMS_EMBED_VST)
+    SET(EMBED_FLAGS "-DLMMS_EMBED_VST")
+    IF(LMMS_EMBED_VST_X11)
+        LIST(APPEND EMBED_FLAGS "-DLMMS_EMBED_VST_X11")
+    ENDIF()
+ENDIF()
+
+ADD_DEFINITIONS(${EMBED_FLAGS})
+
+
 IF(LMMS_BUILD_WIN32)
 	ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
 	ADD_EXECUTABLE(RemoteVstPlugin "${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp")
@@ -28,6 +38,7 @@ SET(REMOTE_VST_PLUGIN_FILEPATH "RemoteVstPlugin" CACHE STRING "Relative file pat
 
 ADD_DEFINITIONS(-DREMOTE_VST_PLUGIN_FILEPATH="${REMOTE_VST_PLUGIN_FILEPATH}")
 BUILD_PLUGIN(vstbase vst_base.cpp VstPlugin.cpp VstPlugin.h communication.h MOCFILES VstPlugin.h)
+TARGET_LINK_LIBRARIES(vstbase qx11embedcontainer)
 
 IF(LMMS_BUILD_LINUX AND NOT WANT_VST_NOWINE)
 
@@ -48,15 +59,26 @@ ENDIF(LMMS_HOST_X86_64)
 SET(WINE_CXX_FLAGS "" CACHE STRING "Extra flags passed to wineg++")
 
 STRING(REPLACE " " ";" WINE_BUILD_FLAGS ${CMAKE_CXX_FLAGS} " " ${CMAKE_EXE_LINKER_FLAGS} " " ${WINE_CXX_FLAGS})
+
 ADD_CUSTOM_COMMAND(
-		SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp"
-		COMMAND ${WINE_CXX}
-		ARGS -I${CMAKE_BINARY_DIR} -I${CMAKE_SOURCE_DIR}/include -I${CMAKE_INSTALL_PREFIX}/include/wine/windows -I${CMAKE_INSTALL_PREFIX}/include -I/usr/include/wine/windows ${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp -ansi -mwindows -lpthread ${EXTRA_FLAGS} -fno-omit-frame-pointer ${WINE_BUILD_FLAGS} -o ../RemoteVstPlugin
-		# Ensure correct file extension
-		COMMAND sh -c "mv ../RemoteVstPlugin.exe ../RemoteVstPlugin || true"
-		TARGET vstbase
-		OUTPUTS ../RemoteVstPlugin
-		)
+	SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp"
+	COMMAND ${WINE_CXX}
+	ARGS -I${CMAKE_BINARY_DIR}
+		-I${CMAKE_SOURCE_DIR}/include
+		-I${CMAKE_INSTALL_PREFIX}/include/wine/windows
+		-I${CMAKE_INSTALL_PREFIX}/include
+		-I${WINE_INCLUDE_BASE_DIR}
+		-L${WINE_LIBRARY_DIR}
+		${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp
+		-std=c++0x
+		-mwindows -lpthread ${EXTRA_FLAGS} -fno-omit-frame-pointer
+		${WINE_BUILD_FLAGS}
+		${EMBED_FLAGS}
+		-o ../RemoteVstPlugin
+	COMMAND sh -c "mv ../RemoteVstPlugin.exe ../RemoteVstPlugin || true"
+	TARGET vstbase
+	OUTPUTS ../RemoteVstPlugin
+)
 
 SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ../RemoteVstPlugin.exe.so)
 INSTALL(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/../RemoteVstPlugin" "${CMAKE_CURRENT_BINARY_DIR}/../RemoteVstPlugin.exe.so" DESTINATION "${PLUGIN_DIR}")