wayland/screencopy: apply output transform to wlr screencopy

Note that this only fixes output copies, and not toplevel copies.
Toplevel copies are harder because a toplevel can be on more than
one output. Hopefully we'll all be using image-copy-capture before
this one comes up.

Fixes #75

Author: outfoxxed

src/wayland/buffer/manager.hpp

@@ -40,6 +40,7 @@ struct WlBufferTransform {
 
 	[[nodiscard]] int degrees() const { return 90 * (this->transform & 0b11111011); }
 	[[nodiscard]] bool flip() const { return this->transform & 0b00000100; }
+	[[nodiscard]] bool flipSize() const { return this->transform & 0b00000001; }
 
 	void apply(QMatrix4x4& matrix) const {
 		matrix.rotate(this->flip() ? 180 : 0, 0, 1, 0);

src/wayland/screencopy/view.cpp

@@ -120,8 +120,14 @@ void ScreencopyView::captureFrame() {
 void ScreencopyView::onFrameCaptured() {
 	this->setFlag(QQuickItem::ItemHasContents);
 	this->update();
+
+	const auto& frontbuffer = this->context->swapchain().frontbuffer();
+
+	auto size = frontbuffer->size();
+	if (frontbuffer->transform.flipSize()) size.transpose();
+
+	this->bSourceSize = size;
 	this->bHasContent = true;
-	this->bSourceSize = this->context->swapchain().frontbuffer()->size();
 }
 
 void ScreencopyView::componentComplete() {

src/wayland/screencopy/wlr_screencopy/wlr_screencopy.cpp

@@ -1,12 +1,14 @@
 #include "wlr_screencopy.hpp"
 #include <cstdint>
 
+#include <private/qwaylanddisplay_p.h>
 #include <private/qwaylandscreen_p.h>
 #include <qlogging.h>
 #include <qloggingcategory.h>
 #include <qobject.h>
 #include <qscreen.h>
 #include <qtmetamacros.h>
+#include <qtypes.h>
 #include <qwaylandclientextension.h>
 #include <wayland-wlr-screencopy-unstable-v1-client-protocol.h>
 
@@ -45,6 +47,7 @@ WlrScreencopyContext::WlrScreencopyContext(
     , screen(dynamic_cast<QtWaylandClient::QWaylandScreen*>(screen->handle()))
     , paintCursors(paintCursors)
     , region(region) {
+	this->transform.setScreen(this->screen);
 	QObject::connect(screen, &QObject::destroyed, this, &WlrScreencopyContext::onScreenDestroyed);
 }
 
@@ -99,9 +102,7 @@ void WlrScreencopyContext::zwlr_screencopy_frame_v1_linux_dmabuf(
 }
 
 void WlrScreencopyContext::zwlr_screencopy_frame_v1_flags(uint32_t flags) {
-	if (flags & ZWLR_SCREENCOPY_FRAME_V1_FLAGS_Y_INVERT) {
-		this->mSwapchain.backbuffer()->transform = buffer::WlBufferTransform::Flipped180;
-	}
+	this->yInvert = flags & ZWLR_SCREENCOPY_FRAME_V1_FLAGS_Y_INVERT;
 }
 
 void WlrScreencopyContext::zwlr_screencopy_frame_v1_buffer_done() {
@@ -119,15 +120,69 @@ void WlrScreencopyContext::zwlr_screencopy_frame_v1_ready(
     uint32_t /*tvSecLo*/,
     uint32_t /*tvNsec*/
 ) {
-	this->destroy();
-	this->copiedFirstFrame = true;
-	this->mSwapchain.swapBuffers();
-	emit this->frameCaptured();
+	this->submitFrame();
 }
 
 void WlrScreencopyContext::zwlr_screencopy_frame_v1_failed() {
 	qCWarning(logScreencopy) << "Ending recording due to screencopy failure for" << this;
 	emit this->stopped();
 }
 
+void WlrScreencopyContext::updateTransform(bool previouslyUnset) {
+	if (previouslyUnset && this->copiedFirstFrame) this->submitFrame();
+}
+
+void WlrScreencopyContext::submitFrame() {
+	this->copiedFirstFrame = true;
+	if (this->transform.transform == -1) return;
+
+	auto flipTransform =
+	    this->yInvert ? buffer::WlBufferTransform::Flipped180 : buffer::WlBufferTransform::Normal0;
+
+	this->mSwapchain.backbuffer()->transform = this->transform.transform ^ flipTransform;
+
+	this->destroy();
+	this->mSwapchain.swapBuffers();
+	emit this->frameCaptured();
+}
+
+WlrScreencopyContext::OutputTransformQuery::OutputTransformQuery(WlrScreencopyContext* context)
+    : context(context) {}
+
+WlrScreencopyContext::OutputTransformQuery::~OutputTransformQuery() {
+	if (this->isInitialized()) this->release();
+}
+
+void WlrScreencopyContext::OutputTransformQuery::setScreen(QtWaylandClient::QWaylandScreen* screen
+) {
+	// cursed hack
+	class QWaylandScreenReflector: public QtWaylandClient::QWaylandScreen {
+	public:
+		[[nodiscard]] int globalId() const { return this->m_outputId; }
+	};
+
+	if (this->isInitialized()) this->release();
+
+	this->init(
+	    screen->display()->wl_registry(),
+	    static_cast<QWaylandScreenReflector*>(screen)->globalId(), // NOLINT
+	    3
+	);
+}
+
+void WlrScreencopyContext::OutputTransformQuery::output_geometry(
+    qint32 /*x*/,
+    qint32 /*y*/,
+    qint32 /*width*/,
+    qint32 /*height*/,
+    qint32 /*subpixel*/,
+    const QString& /*make*/,
+    const QString& /*model*/,
+    qint32 transform
+) {
+	auto newTransform = this->transform == -1;
+	this->transform = transform;
+	this->context->updateTransform(newTransform);
+}
+
 } // namespace qs::wayland::screencopy::wlr

src/wayland/screencopy/wlr_screencopy/wlr_screencopy_p.hpp

@@ -1,7 +1,10 @@
 #pragma once
 
+#include <private/qwayland-wayland.h>
 #include <private/qwaylandscreen_p.h>
+#include <qcontainerfwd.h>
 #include <qtclasshelpermacros.h>
+#include <qtypes.h>
 #include <qwayland-wlr-screencopy-unstable-v1.h>
 
 #include "../manager.hpp"
@@ -24,6 +27,7 @@ class WlrScreencopyContext
 	Q_DISABLE_COPY_MOVE(WlrScreencopyContext);
 
 	void captureFrame() override;
+	void updateTransform(bool previouslyUnset);
 
 protected:
 	// clang-format off
@@ -39,9 +43,38 @@ private slots:
 	void onScreenDestroyed();
 
 private:
+	void submitFrame();
+
+	class OutputTransformQuery: public QtWayland::wl_output {
+	public:
+		OutputTransformQuery(WlrScreencopyContext* context);
+		~OutputTransformQuery() override;
+		Q_DISABLE_COPY_MOVE(OutputTransformQuery);
+
+		qint32 transform = -1;
+		void setScreen(QtWaylandClient::QWaylandScreen* screen);
+
+	protected:
+		void output_geometry(
+		    qint32 x,
+		    qint32 y,
+		    qint32 width,
+		    qint32 height,
+		    qint32 subpixel,
+		    const QString& make,
+		    const QString& model,
+		    qint32 transform
+		) override;
+
+	private:
+		WlrScreencopyContext* context;
+	};
+
 	WlrScreencopyManager* manager;
 	buffer::WlBufferRequest request;
 	bool copiedFirstFrame = false;
+	OutputTransformQuery transform {this};
+	bool yInvert = false;
 
 	QtWaylandClient::QWaylandScreen* screen;
 	bool paintCursors;