Fix neighbor DIR selection for overlapping windows

Use center as origin.

geometry.c

@@ -26,11 +26,6 @@
 #include "types.h"
 #include "geometry.h"
 
-double distance(xcb_point_t a, xcb_point_t b)
-{
-	return hypot(a.x - b.x, a.y - b.y);
-}
-
 bool is_inside(xcb_point_t p, xcb_rectangle_t r)
 {
 	return (p.x >= r.x && p.x < (r.x + r.width) &&
@@ -43,45 +38,62 @@ unsigned int area(xcb_rectangle_t r)
 }
 
 
-xcb_point_t center(xcb_rectangle_t r)
+dpoint_t center(xcb_rectangle_t r)
 {
-	return (xcb_point_t) {r.x + r.width/2, r.y + r.height/2};
+	return (dpoint_t) {(double)r.x + ((double)r.width / 2), (double)r.y + ((double)r.height / 2)};
 }
 
-uint32_t rect_dir_dist(xcb_rectangle_t r1, xcb_rectangle_t r2, direction_t dir)
+double distance_center(xcb_rectangle_t r1, xcb_rectangle_t r2)
 {
-	switch (dir) {
+	dpoint_t r1_center = center(r1);
-		case DIR_NORTH:
+	dpoint_t r2_center = center(r2);
-			return r1.y - (r2.y + r2.height);
+	return hypot(r1_center.x - r2_center.x, r1_center.y - r2_center.y);
-			break;
-		case DIR_WEST:
-			return r1.x - (r2.x + r2.width);
-			break;
-		case DIR_SOUTH:
-			return r2.y - (r1.y + r1.height);
-			break;
-		case DIR_EAST:
-			return r2.x - (r1.x + r1.width);
-			break;
-		default:
-			return UINT32_MAX;
-	}
 }
 
 bool on_dir_side(xcb_rectangle_t r1, xcb_rectangle_t r2, direction_t dir)
 {
+	dpoint_t r1_max = {r1.x + r1.width, r1.y + r1.height};
+	dpoint_t r2_max = {r2.x + r2.width, r2.y + r2.height};
+	dpoint_t r1_center = center(r1);
+	dpoint_t r2_center = center(r2);
+
 	switch (dir) {
 		case DIR_NORTH:
-			return (r2.y + r2.height) <= r1.y && r2.x < (r1.x + r1.width) && (r2.x + r2.width) >= r1.x;
+			if (r2_center.y >= r1_center.y)
+				return false;
 			break;
 		case DIR_WEST:
-			return (r2.x + r2.width) <= r1.x && r2.y < (r1.y + r1.height) && (r2.y + r2.height) >= r1.y;
+			if (r2_center.x >= r1_center.x)
+				return false;
 			break;
 		case DIR_SOUTH:
-			return r2.y >= (r1.y + r1.height) && r2.x < (r1.x + r1.width) && (r2.x + r2.width) >= r1.x;
+			if (r1_center.y >= r2_center.y)
+				return false;
 			break;
 		case DIR_EAST:
-			return r2.x >= (r1.x + r1.width) && r2.y < (r1.y + r1.height) && (r2.y + r2.height) >= r1.y;
+			if (r1_center.x >= r2_center.x)
+				return false;
+			break;
+		default:
+			return false;
+	}
+
+	switch (dir) {
+		case DIR_NORTH:
+		case DIR_SOUTH:
+			return
+				(r2.x >= r1.x && r2.x <= r1_max.x) ||
+				(r2_max.x >= r1.x && r2_max.x <= r1_max.x) ||
+				(r1.x >= r2.x && r1.x <= r2_max.x) ||
+				(r1_max.x >= r2.x && r1_max.x <= r2_max.x);
+			break;
+		case DIR_WEST:
+		case DIR_EAST:
+			return
+				(r2.y >= r1.y && r2.y <= r1_max.y) ||
+				(r2_max.y >= r1.y && r2_max.y <= r1_max.y) ||
+				(r1.y >= r2.y && r1.y <= r2_max.y) ||
+				(r1_max.y >= r2.y && r1_max.y <= r2_max.y);
 			break;
 		default:
 			return false;

geometry.h

@@ -28,11 +28,10 @@
 #include <stdbool.h>
 #include <xcb/xcb.h>
 
-double distance(xcb_point_t a, xcb_point_t b);
 bool is_inside(xcb_point_t p, xcb_rectangle_t r);
 unsigned int area(xcb_rectangle_t r);
-xcb_point_t center(xcb_rectangle_t r);
+dpoint_t center(xcb_rectangle_t r);
-uint32_t rect_dir_dist(xcb_rectangle_t r1, xcb_rectangle_t r2, direction_t dir);
+double distance_center(xcb_rectangle_t r1, xcb_rectangle_t r2);
 bool on_dir_side(xcb_rectangle_t r1, xcb_rectangle_t r2, direction_t dir);
 bool rect_eq(xcb_rectangle_t a, xcb_rectangle_t b);
 int rect_cmp(xcb_rectangle_t r1, xcb_rectangle_t r2);

helpers.h

@@ -30,6 +30,7 @@
 #include <stdio.h>
 #include <stdbool.h>
 #include <stdint.h>
+#include <float.h>
 
 #define LENGTH(x)         (sizeof(x) / sizeof(*x))
 #define MAX(A, B)         ((A) > (B) ? (A) : (B))

monitor.c

@@ -401,7 +401,7 @@ monitor_t *monitor_from_client(client_t *c)
 
 monitor_t *nearest_monitor(monitor_t *m, direction_t dir, monitor_select_t sel)
 {
-	uint32_t dmin = UINT32_MAX;
+	double dmin = DBL_MAX;
 	monitor_t *nearest = NULL;
 	xcb_rectangle_t rect = m->rectangle;
 	for (monitor_t *f = mon_head; f != NULL; f = f->next) {
@@ -412,7 +412,7 @@ monitor_t *nearest_monitor(monitor_t *m, direction_t dir, monitor_select_t sel)
 		    !on_dir_side(rect, r, dir)) {
 			continue;
 		}
-		uint32_t d = rect_dir_dist(rect, r, dir);
+		double d = distance_center(rect, r);
 		if (d < dmin) {
 			dmin = d;
 			nearest = f;

tree.c

@@ -957,7 +957,7 @@ void find_nearest_neighbor(coordinates_t *ref, coordinates_t *dst, direction_t d
 	}
 
 	xcb_rectangle_t rect = get_rectangle(ref->desktop, ref->node);
-	uint32_t md = UINT32_MAX, mr = UINT32_MAX;
+	double md = DBL_MAX, mr = UINT32_MAX;
 
 	for (monitor_t *m = mon_head; m != NULL; m = m->next) {
 		desktop_t *d = m->desk;
@@ -971,7 +971,7 @@ void find_nearest_neighbor(coordinates_t *ref, coordinates_t *dst, direction_t d
 			    !on_dir_side(rect, r, dir)) {
 				continue;
 			}
-			uint32_t fd = rect_dir_dist(rect, r, dir);
+			double fd = distance_center(rect, r);
 			uint32_t fr = history_rank(f);
 			if (fd < md || (fd == md && fr < mr)) {
 				md = fd;

types.h

@@ -334,4 +334,9 @@ struct pending_rule_t {
 	pending_rule_t *next;
 };
 
+typedef struct {
+    double x;
+    double y;
+} dpoint_t;
+
 #endif