wayland/layershell: refactor layer shell surface integration

In addition to the much needed cleanup:

- The bridge/extension type is now directly tied to the QWindow
instead of the WlrLayershell object, and is much smaller.
- Layer requests are now comitted via polish instead of for each
change individually.

Author: outfoxxed

src/wayland/CMakeLists.txt

@@ -85,7 +85,6 @@ add_library(quickshell-wayland-init OBJECT init.cpp)
 set(WAYLAND_MODULES)
 
 if (WAYLAND_WLR_LAYERSHELL)
-	target_sources(quickshell-wayland PRIVATE wlr_layershell.cpp)
 	add_subdirectory(wlr_layershell)
 	target_compile_definitions(quickshell-wayland PRIVATE QS_WAYLAND_WLR_LAYERSHELL)
 	target_compile_definitions(quickshell-wayland-init PRIVATE QS_WAYLAND_WLR_LAYERSHELL)

src/wayland/init.cpp

@@ -7,7 +7,7 @@
 #include "../core/plugin.hpp"
 
 #ifdef QS_WAYLAND_WLR_LAYERSHELL
-#include "wlr_layershell.hpp"
+#include "wlr_layershell/wlr_layershell.hpp"
 #endif
 
 void installPlatformMenuHook(); // NOLINT(misc-use-internal-linkage)
@@ -39,7 +39,12 @@ class WaylandPlugin: public QsEnginePlugin {
 
 	void registerTypes() override {
 #ifdef QS_WAYLAND_WLR_LAYERSHELL
-		qmlRegisterType<WaylandPanelInterface>("Quickshell._WaylandOverlay", 1, 0, "PanelWindow");
+		qmlRegisterType<qs::wayland::layershell::WaylandPanelInterface>(
+		    "Quickshell._WaylandOverlay",
+		    1,
+		    0,
+		    "PanelWindow"
+		);
 
 		// If any types are defined inside a module using QML_ELEMENT then all QML_ELEMENT types
 		// will not be registered. This can be worked around with a module import which makes

src/wayland/module.md

@@ -1,8 +1,7 @@
 name = "Quickshell.Wayland"
 description = "Wayland specific Quickshell types"
 headers = [
-	"wlr_layershell/window.hpp",
-	"wlr_layershell.hpp",
+	"wlr_layershell/wlr_layershell.hpp",
 	"session_lock.hpp",
 	"toplevel_management/qml.hpp",
 	"screencopy/view.hpp",

src/wayland/wlr_layershell/CMakeLists.txt

@@ -1,7 +1,7 @@
 qt_add_library(quickshell-wayland-layershell STATIC
+	wlr_layershell.cpp
 	shell_integration.cpp
 	surface.cpp
-	window.cpp
 )
 
 qt_add_qml_module(quickshell-wayland-layershell

src/wayland/wlr_layershell/shell_integration.cpp

@@ -6,16 +6,20 @@
 
 #include "surface.hpp"
 
-QSWaylandLayerShellIntegration::QSWaylandLayerShellIntegration()
-    : QtWaylandClient::QWaylandShellIntegrationTemplate<QSWaylandLayerShellIntegration>(4) {}
+namespace qs::wayland::layershell {
 
-QSWaylandLayerShellIntegration::~QSWaylandLayerShellIntegration() {
+LayerShellIntegration::LayerShellIntegration()
+    : QtWaylandClient::QWaylandShellIntegrationTemplate<LayerShellIntegration>(4) {}
+
+LayerShellIntegration::~LayerShellIntegration() {
 	if (this->isInitialized()) {
 		this->destroy();
 	}
 }
 
 QtWaylandClient::QWaylandShellSurface*
-QSWaylandLayerShellIntegration::createShellSurface(QtWaylandClient::QWaylandWindow* window) {
-	return new QSWaylandLayerSurface(this, window);
+LayerShellIntegration::createShellSurface(QtWaylandClient::QWaylandWindow* window) {
+	return new LayerSurface(this, window);
 }
+
+} // namespace qs::wayland::layershell

src/wayland/wlr_layershell/shell_integration.hpp

@@ -5,14 +5,18 @@
 #include <qtclasshelpermacros.h>
 #include <qwayland-wlr-layer-shell-unstable-v1.h>
 
-class QSWaylandLayerShellIntegration
-    : public QtWaylandClient::QWaylandShellIntegrationTemplate<QSWaylandLayerShellIntegration>
+namespace qs::wayland::layershell {
+
+class LayerShellIntegration
+    : public QtWaylandClient::QWaylandShellIntegrationTemplate<LayerShellIntegration>
     , public QtWayland::zwlr_layer_shell_v1 {
 public:
-	QSWaylandLayerShellIntegration();
-	~QSWaylandLayerShellIntegration() override;
-	Q_DISABLE_COPY_MOVE(QSWaylandLayerShellIntegration);
+	LayerShellIntegration();
+	~LayerShellIntegration() override;
+	Q_DISABLE_COPY_MOVE(LayerShellIntegration);
 
 	QtWaylandClient::QWaylandShellSurface* createShellSurface(QtWaylandClient::QWaylandWindow* window
 	) override;
 };
+
+} // namespace qs::wayland::layershell

src/wayland/wlr_layershell/surface.cpp

@@ -1,6 +1,7 @@
 #include "surface.hpp"
 #include <algorithm>
 #include <any>
+#include <utility>
 
 #include <private/qhighdpiscaling_p.h>
 #include <private/qwaylanddisplay_p.h>
@@ -13,16 +14,20 @@
 #include <qsize.h>
 #include <qtversionchecks.h>
 #include <qtypes.h>
+#include <qvariant.h>
 #include <qwayland-wlr-layer-shell-unstable-v1.h>
+#include <qwindow.h>
 
 #include "../../window/panelinterface.hpp"
 #include "shell_integration.hpp"
-#include "window.hpp"
+#include "wlr_layershell.hpp"
 
 #if QT_VERSION < QT_VERSION_CHECK(6, 7, 0)
 #include <qpoint.h>
 #endif
 
+namespace qs::wayland::layershell {
+
 namespace {
 
 [[nodiscard]] QtWayland::zwlr_layer_shell_v1::layer toWaylandLayer(const WlrLayer::Enum& layer
@@ -69,21 +74,72 @@ toWaylandKeyboardFocus(const WlrKeyboardFocus::Enum& focus) noexcept {
 
 } // namespace
 
-QSWaylandLayerSurface::QSWaylandLayerSurface(
-    QSWaylandLayerShellIntegration* shell,
-    QtWaylandClient::QWaylandWindow* window
-)
+void LayerSurfaceBridge::commitState() {
+	if (this->surface) this->surface->commit();
+}
+
+LayerSurfaceBridge* LayerSurfaceBridge::get(QWindow* window) {
+	auto v = window->property("layershell_bridge");
+
+	if (v.canConvert<LayerSurfaceBridge*>()) {
+		return v.value<LayerSurfaceBridge*>();
+	}
+
+	return nullptr;
+}
+
+LayerSurfaceBridge* LayerSurfaceBridge::init(QWindow* window, LayerSurfaceState state) {
+	auto* bridge = LayerSurfaceBridge::get(window);
+
+	if (!bridge) {
+		bridge = new LayerSurfaceBridge(window);
+		window->setProperty("layershell_bridge", QVariant::fromValue(bridge));
+	} else if (!bridge->state.isCompatible(state)) {
+		return nullptr;
+	}
+
+	if (!bridge->surface) {
+		// Qt appears to be resetting the window's screen on creation on some systems. This works around it.
+		auto* screen = window->screen();
+		window->create();
+		window->setScreen(screen);
+
+		auto* waylandWindow = dynamic_cast<QtWaylandClient::QWaylandWindow*>(window->handle());
+		if (waylandWindow == nullptr) {
+			qWarning() << window << "is not a wayland window. Cannot create layershell surface.";
+			return nullptr;
+		}
+
+		static LayerShellIntegration* layershellIntegration = nullptr; // NOLINT
+		if (layershellIntegration == nullptr) {
+			layershellIntegration = new LayerShellIntegration();
+			if (!layershellIntegration->initialize(waylandWindow->display())) {
+				delete layershellIntegration;
+				layershellIntegration = nullptr;
+				qWarning() << "Failed to initialize layershell integration";
+			}
+		}
+
+		waylandWindow->setShellIntegration(layershellIntegration);
+	}
+
+	bridge->state = std::move(state);
+	bridge->commitState();
+
+	return bridge;
+}
+
+LayerSurface::LayerSurface(LayerShellIntegration* shell, QtWaylandClient::QWaylandWindow* window)
     : QtWaylandClient::QWaylandShellSurface(window) {
 
 	auto* qwindow = window->window();
-	this->ext = LayershellWindowExtension::get(qwindow);
 
-	if (this->ext == nullptr) {
-		qFatal() << "QSWaylandLayerSurface created with null LayershellWindowExtension";
-	}
+	this->bridge = LayerSurfaceBridge::get(qwindow);
+	if (this->bridge == nullptr) qFatal() << "LayerSurface created with null bridge";
+	const auto& s = this->bridge->state;
 
 	wl_output* output = nullptr; // NOLINT (include)
-	if (this->ext->useWindowScreen) {
+	if (!s.compositorPickesScreen) {
 		auto* waylandScreen =
 		    dynamic_cast<QtWaylandClient::QWaylandScreen*>(qwindow->screen()->handle());
 
@@ -98,36 +154,28 @@ QSWaylandLayerSurface::QSWaylandLayerSurface(
 	this->init(shell->get_layer_surface(
 	    window->waylandSurface()->object(),
 	    output,
-	    toWaylandLayer(this->ext->mLayer),
-	    this->ext->mNamespace
+	    toWaylandLayer(s.layer),
+	    s.mNamespace
 	));
 
-	this->updateAnchors();
-	this->updateLayer();
-	this->updateMargins();
-	this->updateExclusiveZone();
-	this->updateKeyboardFocus();
-
-	// new updates will be sent from the extension
-	this->ext->surface = this;
-
-	auto size = constrainedSize(this->ext->mAnchors, window->surfaceSize());
+	auto size = constrainedSize(s.anchors, QHighDpi::toNativePixels(s.implicitSize, qwindow));
 	this->set_size(size.width(), size.height());
-}
-
-QSWaylandLayerSurface::~QSWaylandLayerSurface() {
-	if (this->ext != nullptr) {
-		this->ext->surface = nullptr;
-	}
+	this->set_anchor(toWaylandAnchors(s.anchors));
+	this->set_margin(
+	    QHighDpi::toNativePixels(s.margins.mTop, qwindow),
+	    QHighDpi::toNativePixels(s.margins.mRight, qwindow),
+	    QHighDpi::toNativePixels(s.margins.mBottom, qwindow),
+	    QHighDpi::toNativePixels(s.margins.mLeft, qwindow)
+	);
+	this->set_exclusive_zone(QHighDpi::toNativePixels(s.exclusiveZone, qwindow));
+	this->set_keyboard_interactivity(toWaylandKeyboardFocus(s.keyboardFocus));
 
-	this->destroy();
+	this->bridge->surface = this;
 }
 
-void QSWaylandLayerSurface::zwlr_layer_surface_v1_configure(
-    quint32 serial,
-    quint32 width,
-    quint32 height
-) {
+LayerSurface::~LayerSurface() { this->destroy(); }
+
+void LayerSurface::zwlr_layer_surface_v1_configure(quint32 serial, quint32 width, quint32 height) {
 	this->ack_configure(serial);
 
 	this->size = QSize(static_cast<qint32>(width), static_cast<qint32>(height));
@@ -146,51 +194,53 @@ void QSWaylandLayerSurface::zwlr_layer_surface_v1_configure(
 	}
 }
 
-void QSWaylandLayerSurface::zwlr_layer_surface_v1_closed() { this->window()->window()->close(); }
+void LayerSurface::zwlr_layer_surface_v1_closed() { this->window()->window()->close(); }
 
-bool QSWaylandLayerSurface::isExposed() const { return this->configured; }
+bool LayerSurface::isExposed() const { return this->configured; }
 
-void QSWaylandLayerSurface::applyConfigure() {
-	this->window()->resizeFromApplyConfigure(this->size);
-}
+void LayerSurface::applyConfigure() { this->window()->resizeFromApplyConfigure(this->size); }
 
-void QSWaylandLayerSurface::setWindowGeometry(const QRect& geometry) {
-	if (this->ext == nullptr) return;
-	auto size = constrainedSize(this->ext->mAnchors, geometry.size());
-	this->set_size(size.width(), size.height());
-}
+QWindow* LayerSurface::qwindow() { return this->window()->window(); }
 
-QWindow* QSWaylandLayerSurface::qwindow() { return this->window()->window(); }
+void LayerSurface::commit() {
+	const auto& p = this->bridge->state;
+	auto& c = this->committed;
 
-void QSWaylandLayerSurface::updateLayer() {
-	this->set_layer(toWaylandLayer(this->ext->mLayer));
-	this->window()->waylandSurface()->commit();
-}
+	if (p.implicitSize != c.implicitSize || p.anchors != c.anchors) {
+		auto size =
+		    constrainedSize(p.anchors, QHighDpi::toNativePixels(p.implicitSize, this->qwindow()));
+		this->set_size(size.width(), size.height());
+	}
 
-void QSWaylandLayerSurface::updateAnchors() {
-	this->set_anchor(toWaylandAnchors(this->ext->mAnchors));
-	this->setWindowGeometry(this->window()->windowContentGeometry());
-	this->window()->waylandSurface()->commit();
-}
+	if (p.anchors != c.anchors) {
+		this->set_anchor(toWaylandAnchors(p.anchors));
+	}
 
-void QSWaylandLayerSurface::updateMargins() {
-	auto& margins = this->ext->mMargins;
-	this->set_margin(margins.mTop, margins.mRight, margins.mBottom, margins.mLeft);
-	this->window()->waylandSurface()->commit();
-}
+	if (p.margins != c.margins) {
+		this->set_margin(
+		    QHighDpi::toNativePixels(p.margins.mTop, this->qwindow()),
+		    QHighDpi::toNativePixels(p.margins.mRight, this->qwindow()),
+		    QHighDpi::toNativePixels(p.margins.mBottom, this->qwindow()),
+		    QHighDpi::toNativePixels(p.margins.mLeft, this->qwindow())
+		);
+	}
 
-void QSWaylandLayerSurface::updateExclusiveZone() {
-	auto nativeZone = QHighDpi::toNativePixels(this->ext->mExclusiveZone, this->window()->window());
-	this->set_exclusive_zone(nativeZone);
-	this->window()->waylandSurface()->commit();
-}
+	if (p.layer != c.layer) {
+		this->set_layer(p.layer);
+	}
 
-void QSWaylandLayerSurface::updateKeyboardFocus() {
-	this->set_keyboard_interactivity(toWaylandKeyboardFocus(this->ext->mKeyboardFocus));
-	this->window()->waylandSurface()->commit();
+	if (p.exclusiveZone != c.exclusiveZone) {
+		this->set_exclusive_zone(QHighDpi::toNativePixels(p.exclusiveZone, this->qwindow()));
+	}
+
+	if (p.keyboardFocus != c.keyboardFocus) {
+		this->set_keyboard_interactivity(toWaylandKeyboardFocus(p.keyboardFocus));
+	}
+
+	c = p;
 }
 
-void QSWaylandLayerSurface::attachPopup(QtWaylandClient::QWaylandShellSurface* popup) {
+void LayerSurface::attachPopup(QtWaylandClient::QWaylandShellSurface* popup) {
 	std::any role = popup->surfaceRole();
 
 	if (auto* popupRole = std::any_cast<::xdg_popup*>(&role)) { // NOLINT
@@ -200,3 +250,5 @@ void QSWaylandLayerSurface::attachPopup(QtWaylandClient::QWaylandShellSurface* p
 		           << "as the popup is not an xdg_popup.";
 	}
 }
+
+} // namespace qs::wayland::layershell