x11 fullscreen fixes and some memory cleanup

qe.c

@@ -9306,9 +9306,9 @@ static void quit_examine_buffers(QuitState *is)
         if (use_session_file)
             do_save_session(qs->active_window, 0);
 #endif
-        qe_free(&is);
         url_exit();
     }
+    qe_free(&is);
 }
 
 static void quit_key(void *opaque, int ch)

x11.c

@@ -1087,58 +1087,113 @@ static void x11_dpy_flush(QEditScreen *s)
     update_reset(xs);
 #endif
 }
-static void set_window_bordered(QEditScreen *s, int border) {
+static void x11_dpy_bordered(QEditScreen *s, int border) {
     X11State *xs = s->priv_data;
 
-    Atom WM_HINTS = XInternAtom(xs->display, "_MOTIF_WM_HINTS", True);
-    if (WM_HINTS != None) {
-        // Hints used by Motif compliant window managers
-        struct
-        {
-            unsigned long flags;
-            unsigned long functions;
-            unsigned long decorations;
-            long input_mode;
-            unsigned long status;
-        } MWMHints = {
-            (1L << 1), 0, border ? 1 : 0, 0, 0
-        };
-
-        XChangeProperty(xs->display, xs->window, WM_HINTS, WM_HINTS, 32,
-                            PropModeReplace, (unsigned char *)&MWMHints,
-                            sizeof(MWMHints) / sizeof(long));
-    } else {
-        XSetTransientForHint(xs->display, xs->window, RootWindow(xs->display, xs->xscreen));
+    struct
+    {
+        unsigned long flags;
+        unsigned long functions;
+        unsigned long decorations;
+        long input_mode;
+        unsigned long status;
+    }  hints;
+    Atom property;
+    hints.flags = 2;
+    hints.functions = 0;
+    hints.decorations = border ? 1 : 0;
+    hints.input_mode = 0;
+    hints.status = 0;
+    property = XInternAtom(xs->display, "_MOTIF_WM_HINTS", True);
+    if (property != None) {
+        XChangeProperty(xs->display, xs->window, property, property, 32, PropModeReplace, (unsigned char *)&hints, 5);
     }
 }
 
+
+static int x11_dpy_full_screen_check(QEditScreen *s) {
+    X11State *xs = s->priv_data;
+    Atom type;
+    Atom property;
+    Atom *atoms = NULL;
+    unsigned long len;
+    unsigned long remaining;
+    int format;
+    int result;
+    int retval = 0;
+
+    property = XInternAtom(xs->display, "_NET_WM_STATE", True);
+
+    if (property == None) {
+        return 0;
+    }
+
+    result = XGetWindowProperty(
+        xs->display,
+        xs->window,
+        property,
+        0,
+        32,
+        False,
+        XA_ATOM,
+        &type,
+        &format,
+        &len,
+        &remaining,
+        (unsigned char **)&atoms);
+
+    if ((result == Success) && atoms) {
+        Atom wm_fullscreen = XInternAtom(xs->display, "_NET_WM_STATE_FULLSCREEN", False);
+
+        for (unsigned int i = 0; i < len; i++) {
+            if (atoms[i] == wm_fullscreen) {
+                retval = 1;
+                break;
+            }
+        }
+        XFree(atoms);
+    }
+    return retval;
+}
+
+
 static void x11_dpy_full_screen(QEditScreen *s, int full_screen)
 {
     X11State *xs = s->priv_data;
-    XWindowAttributes attr1;
-    Window win;
 
-    XGetWindowAttributes(xs->display, xs->window, &attr1);
     if (full_screen) {
-        if (attr1.width != xs->screen_width || attr1.height != xs->screen_height) {
-            /*NOTE: window border hides before resize*/
-            set_window_bordered(s, 0);
-            /* store current window position and size */
-            XTranslateCoordinates(xs->display, xs->window, attr1.root, 0, 0,
-                                  &xs->last_window_x, &xs->last_window_y, &win);
-            xs->last_window_width = attr1.width;
-            xs->last_window_height = attr1.height;
-            XMoveResizeWindow(xs->display, xs->window,
-                              0, 0, xs->screen_width, xs->screen_height);
-        }
-    } else {
-        if (xs->last_window_width) {
-            XMoveResizeWindow(xs->display, xs->window,
-                              xs->last_window_x, xs->last_window_y,
-                              xs->last_window_width, xs->last_window_height);
-            /*NOTE: window border shows after resize*/
-            set_window_bordered(s, 1);
-        }
+        x11_dpy_bordered(s, 0);
+    }
+
+    XEvent xev;
+    Atom wm_state = XInternAtom(xs->display, "_NET_WM_STATE", False);
+    Atom wm_fullscreen = XInternAtom(xs->display, "_NET_WM_STATE_FULLSCREEN", False);
+    memset(&xev, 0, sizeof(xev));
+    xev.type = ClientMessage;
+    xev.xclient.window = xs->window;
+    xev.xclient.message_type = wm_state;
+    xev.xclient.format = 32;
+    xev.xclient.data.l[0] = full_screen ? 1 : 0;
+    xev.xclient.data.l[1] = wm_fullscreen;
+    xev.xclient.data.l[2] = 0;
+
+    XSendEvent(xs->display, DefaultRootWindow(xs->display), False, SubstructureRedirectMask | SubstructureNotifyMask, &xev);
+
+    Atom bypass_compositor = XInternAtom(xs->display, "_NET_WM_BYPASS_COMPOSITOR", False);
+    unsigned long compositing_disable_on = 0;
+
+    if (full_screen) {
+        compositing_disable_on = 2;
+    }
+
+    if (bypass_compositor != None) {
+        XChangeProperty(xs->display, xs->window, bypass_compositor, XA_CARDINAL, 32, PropModeReplace, (unsigned char *)&compositing_disable_on, 1);
+    }
+
+    XFlush(xs->display);
+
+    if (!full_screen) {
+        x11_dpy_bordered(s, 1);
     }
 }
 
@@ -1399,6 +1454,11 @@ static void x11_handle_event(void *opaque)
         case ConfigureNotify:
             if (x11_term_resize(s, xev.xconfigure.width, xev.xconfigure.height)) {
                 qe_expose_set(s, rgn, 0, 0, s->width, s->height);
+                // fullscreen mode may be toggled by user through window manager
+                qs->is_full_screen = x11_dpy_full_screen_check(s);
+                if (!qs->is_full_screen) {
+                    x11_dpy_bordered(s, 1);
+                }
             }
             break;