show popups in fullscreen Xwayland apps

The dialogs opened by clicking items in Xwyland applications' toolbars
are unmanaged clients, so placing them in LyrFloat drew them below their
source client in LyrFS when it is fullscreened.

Add a new dedicated layer for unamanged clients to fix it. Don't put
them in LyrFS to prevent the regular fullscreen client from hiding
unmanaged ones, e.g. by opening dmenu and then clicking the regular
client.

Fixes https://codeberg.org/dwl/dwl/issues/556

client.h

@@ -15,6 +15,32 @@ client_is_x11(Client *c)
 	return 0;
 }
 
+static inline void
+client_get_size_hints(Client *c, struct wlr_box *max, struct wlr_box *min)
+{
+	struct wlr_xdg_toplevel_state *state;
+
+#ifdef XWAYLAND
+	if (client_is_x11(c)) {
+		xcb_size_hints_t *size_hints = c->surface.xwayland->size_hints;
+		if (size_hints) {
+			max->width = size_hints->max_width;
+			max->height = size_hints->max_height;
+			min->width = size_hints->min_width;
+			min->height = size_hints->min_height;
+		}
+		return;
+	}
+#endif
+
+	state = &c->surface.xdg->toplevel->current;
+	max->width = state->max_width;
+	max->height = state->max_height;
+	min->width = state->min_width;
+	min->height = state->min_height;
+}
+
+
 static inline struct wlr_surface *
 client_surface(Client *c)
 {
@@ -206,13 +232,12 @@ client_get_title(Client *c)
 static inline int
 client_is_float_type(Client *c)
 {
-	struct wlr_xdg_toplevel *toplevel;
-	struct wlr_xdg_toplevel_state state;
+	struct wlr_box min = {0}, max = {0};
+	client_get_size_hints(c, &max, &min);
 
 #ifdef XWAYLAND
 	if (client_is_x11(c)) {
 		struct wlr_xwayland_surface *surface = c->surface.xwayland;
-		xcb_size_hints_t *size_hints = surface->size_hints;
 		if (surface->modal)
 			return 1;
 
@@ -223,17 +248,13 @@ client_is_float_type(Client *c)
 			return 1;
 		}
 
-		return size_hints && size_hints->min_width > 0 && size_hints->min_height > 0
-			&& (size_hints->max_width == size_hints->min_width
-				|| size_hints->max_height == size_hints->min_height);
+		return min.width > 0 && min.height > 0 &&
+			(min.width == max.width || min.height == max.height);
 	}
 #endif
 
-	toplevel = c->surface.xdg->toplevel;
-	state = toplevel->current;
-	return toplevel->parent || (state.min_width != 0 && state.min_height != 0
-		&& (state.min_width == state.max_width
-			|| state.min_height == state.max_height));
+	return c->surface.xdg->toplevel->parent || (min.width > 0 && min.height > 0 &&
+			(min.width == max.width || min.height == max.height));
 }
 
 static inline int
@@ -344,20 +365,19 @@ client_set_scale(struct wlr_surface *s, float scale)
 	wlr_surface_set_preferred_buffer_scale(s, (int32_t)ceilf(scale));
 }
 
-static inline uint32_t
+static inline void
 client_set_size(Client *c, uint32_t width, uint32_t height)
 {
 #ifdef XWAYLAND
 	if (client_is_x11(c)) {
 		wlr_xwayland_surface_configure(c->surface.xwayland,
 				c->geom.x + c->bw, c->geom.y + c->bw, width, height);
-		return 0;
+		return;
 	}
 #endif
-	if ((int32_t)width == c->surface.xdg->toplevel->current.width
-			&& (int32_t)height == c->surface.xdg->toplevel->current.height)
-		return 0;
-	return wlr_xdg_toplevel_set_size(c->surface.xdg->toplevel, (int32_t)width, (int32_t)height);
+	if ((int32_t)width != c->surface.xdg->toplevel->current.width
+			|| (int32_t)height != c->surface.xdg->toplevel->current.height)
+		wlr_xdg_toplevel_set_size(c->surface.xdg->toplevel, (int32_t)width, (int32_t)height);
 }
 
 static inline void

dwl.c

@@ -3,6 +3,7 @@
  */
 #include <getopt.h>
 #include <libinput.h>
+#include <limits.h>
 #include <linux/input-event-codes.h>
 #include <math.h>
 #include <signal.h>
@@ -87,7 +88,8 @@
 /* enums */
 enum { CurNormal, CurPressed, CurMove, CurResize }; /* cursor */
 enum { XDGShell, LayerShell, X11 }; /* client types */
-enum { LyrBg, LyrBottom, LyrTile, LyrFloat, LyrTop, LyrFS, LyrOverlay, LyrIMPopup, LyrBlock, NUM_LAYERS }; /* scene layers */
+enum { LyrBg, LyrBottom, LyrTile, LyrFloat, LyrTop, LyrFS, LyrUnmanaged, LyrOverlay,
+	LyrIMPopup, LyrBlock, NUM_LAYERS }; /* scene layers */
 enum { AxisUp, AxisRight, AxisDown, AxisLeft };
 
 typedef union {
@@ -156,7 +158,6 @@ typedef struct {
 	unsigned int bw;
 	uint32_t tags;
 	int isfloating, isurgent, isfullscreen;
-	uint32_t resize; /* configure serial of a pending resize */
 } Client;
 
 typedef struct {
@@ -484,9 +485,18 @@ static struct wlr_xwayland *xwayland;
 void
 applybounds(Client *c, struct wlr_box *bbox)
 {
-	/* set minimum possible */
-	c->geom.width = MAX(1 + 2 * (int)c->bw, c->geom.width);
-	c->geom.height = MAX(1 + 2 * (int)c->bw, c->geom.height);
+	if (!c->isfullscreen) {
+		struct wlr_box min = {0}, max = {0};
+		client_get_size_hints(c, &max, &min);
+		c->geom.width = MAX(min.width + 2 * (int)c->bw, c->geom.width);
+		c->geom.height = MAX(min.height + 2 * (int)c->bw, c->geom.height);
+		/* Some clients set their max size to INT_MAX, which does not violate the
+		 * protocol but it's unnecesary, as they can set their max size to zero. */
+		if (max.width > 0 && 2 * (int)c->bw <= INT_MAX - max.width) /* Checks for overflow */
+			c->geom.width = MIN(max.width + 2 * (int)c->bw, c->geom.width);
+		if (max.height > 0 && 2 * (int)c->bw <= INT_MAX - max.height) /* Checks for overflow */
+			c->geom.height = MIN(max.height + 2 * (int)c->bw, c->geom.height);
+	}
 
 	if (c->geom.x >= bbox->x + bbox->width)
 		c->geom.x = bbox->x + bbox->width - c->geom.width;
@@ -783,6 +793,7 @@ cleanup(void)
 	destroykeyboardgroup(&kb_group->destroy, NULL);
 
 	input_method_relay_finish(input_method_relay);
+	input_method_relay = NULL;
 
 	/* If it's not destroyed manually, it will cause a use-after-free of wlr_seat.
 	 * Destroy it until it's fixed on the wlroots side */
@@ -959,10 +970,6 @@ commitnotify(struct wl_listener *listener, void *data)
 	}
 
 	resize(c, c->geom, (c->isfloating && !c->isfullscreen));
-
-	/* mark a pending resize as completed */
-	if (c->resize && c->resize <= c->surface.xdg->current.configure_serial)
-		c->resize = 0;
 }
 
 void
@@ -1534,10 +1541,9 @@ focusclient(Client *c, int lift)
 			return;
 		} else if (old_c && old_c == exclusive_focus && client_wants_focus(old_c)) {
 			return;
-		/* Don't deactivate old client if the new one wants focus, as this causes issues with winecfg
-		 * and probably other clients */
-		} else if (old_c && !client_is_unmanaged(old_c) && (!c || !client_wants_focus(c))) {
-			client_set_border_color(old_c, bordercolor);
+		} else if (old_c && !client_is_unmanaged(old_c)) {
+			if (c && !client_is_unmanaged(c))
+				client_set_border_color(old_c, bordercolor);
 
 			client_activate_surface(old, 0);
 		}
@@ -1742,6 +1748,7 @@ keypress(struct wl_listener *listener, void *data)
 				consumed[event->keycode] = 1;
 				key->func(&key->arg);
 				handled = 1;
+				break;
 			}
 		}
 	}
@@ -1794,8 +1801,10 @@ keyrepeat(void *data)
 
 	for (int i = 0; i < 2; i++) {
 		const Key *key = keybinding(group->mods, group->keysyms[i]);
-		if (key)
+		if (key) {
 			key->func(&key->arg);
+			break;
+		}
 	}
 
 	return 0;
@@ -1861,7 +1870,7 @@ mapnotify(struct wl_listener *listener, void *data)
 	/* Handle unmanaged clients first so we can return prior create borders */
 	if (client_is_unmanaged(c)) {
 		/* Unmanaged clients always are floating */
-		wlr_scene_node_reparent(&c->scene->node, layers[LyrFloat]);
+		wlr_scene_node_reparent(&c->scene->node, layers[LyrUnmanaged]);
 		wlr_scene_node_set_position(&c->scene->node, c->geom.x, c->geom.y);
 		client_set_size(c, c->geom.width, c->geom.height);
 		if (client_wants_focus(c)) {
@@ -2362,9 +2371,7 @@ resize(Client *c, struct wlr_box geo, int interact)
 	wlr_scene_node_set_position(&c->border[2]->node, 0, c->bw);
 	wlr_scene_node_set_position(&c->border[3]->node, c->geom.width - c->bw, c->bw);
 
-	/* this is a no-op if size hasn't changed */
-	c->resize = client_set_size(c, c->geom.width - 2 * c->bw,
-			c->geom.height - 2 * c->bw);
+	client_set_size(c, c->geom.width - 2 * c->bw, c->geom.height - 2 * c->bw);
 	client_get_clip(c, &clip);
 	wlr_scene_subsurface_tree_set_clip(&c->scene_surface->node, &clip);
 }

ime.h

@@ -625,7 +625,7 @@ void
 input_method_relay_set_focus(struct input_method_relay *relay,
 		struct wlr_surface *surface)
 {
-	if (relay->focused_surface == surface)
+	if (!relay || relay->focused_surface == surface)
 		return;
 
 	if (relay->focused_surface)