Calculate logical subpixel order from transformation

Signed-off-by: Mattias Andrée <maandree@kth.se>

2 files changed, 173 insertions, 69 deletions

diff --git a/libfonts.h b/libfonts.h
index 9e7b29d..82a5756 100644
--- a/libfonts.h
+++ b/libfonts.h
@@ -42,19 +42,6 @@ enum libfonts_subpixel_order {
 	 */
 	LIBFONTS_SUBPIXEL_ORDER_OTHER,
 
-	/* For any value V, a rotation of 90⋅R degrees, R integer,
-	 * clockwise, of the physical output, will yeild a subpixel
-	 * order represented with the value (V + R) Mod 4 + 4⋅⌊V / 4⌋,
-	 * if V ≥ 4, otherwise V asis. A rotation that is not
-	 * divisible by 90 degrees, will yeild the subpixel order
-	 * LIBFONTS_SUBPIXEL_ORDER_NONLINEAR if V is greater or equal
-	 * to LIBFONTS_SUBPIXEL_ORDER_NONLINEAR, otherwise V as is */
-
-	/* The reason the subpixel order RG_GB, and similar orders
-	 * are excluded from the list is that the green segment of
-	 * the pixel is disjoint and therefore not suitable for
-	 * subpixel rendering; instead NONLINEAR should is used. */
-
 	LIBFONTS_SUBPIXEL_ORDER_RGB,
 	LIBFONTS_SUBPIXEL_ORDER_R_G_B, /* vertically stacked horizontal stripes */
 	LIBFONTS_SUBPIXEL_ORDER_BGR,
@@ -140,62 +127,86 @@ enum libfonts_orientation {
 
 
 /**
+ * Output transformation structure
+ */
+struct libfonts_transformation {
+	/**
+	 * Transformation matrix, row-major
+	 *
+	 *     ⎛.m[0][0]   .m[0][1]   .m[0][2]⎞
+	 *     ⎜.m[1][0]   .m[1][1]   .m[1][2]⎟
+	 *     ⎝.m[2][0]   .m[2][1]   .m[2][2]⎠
+	 *
+	 * Let all values be zero if it is unknown
+	 */
+	double m[3][3];
+};
+
+
+/**
  * Text rendering settings
  */
 struct libfonts_rendering_settings {
 	/**
 	 * The output device's horizontal pixel density,
 	 * in pixels (not dots) per inch, without output
-	 * rotation applied, zero if not applicable or unknown
+	 * transformations applied, zero if not applicable
+	 * or unknown
 	 */
 	double dpi_x;
 
 	/**
 	 * The output device's vertical pixel density,
 	 * in pixels (not dots) per inch, without output
-	 * rotation applied, zero if not applicable or unknown
+	 * transformations applied, zero if not applicable
+	 * or unknown
 	 */
 	double dpi_y;
 
 	/**
-	 * Antialiasing mode for horizontal, grey text
+	 * Antialiasing mode for horizontal (on unrotated output), grey text
 	 */
 	enum libfonts_antialiasing horizontal_grey_text_antialiasing;
 
 	/**
-	 * Antialiasing mode for vertical, grey text
+	 * Antialiasing mode for vertical (on unrotated output), grey text
 	 */
 	enum libfonts_antialiasing vertical_grey_text_antialiasing;
 
 	/**
-	 * Antialiasing mode for non-horizontal, non-vertical, grey text
+	 * Antialiasing mode for non-horizontal, non-vertical
+	 * (on unrotated output), grey text
 	 */
 	enum libfonts_antialiasing diagonal_grey_text_antialiasing;
 
 	/**
-	 * Antialiasing mode for horizontal, non-grey text
+	 * Antialiasing mode for horizontal (on unrotated output),
+	 * non-grey text
 	 */
 	enum libfonts_antialiasing horizontal_colour_text_antialiasing;
 
 	/**
-	 * Antialiasing mode for vertical, non-grey text
+	 * Antialiasing mode for vertical (on unrotated output),
+	 * non-grey text
 	 */
 	enum libfonts_antialiasing vertical_colour_text_antialiasing;
 
 	/**
-	 * Antialiasing mode for non-horizontal, non-vertical, non-grey text
+	 * Antialiasing mode for non-horizontal, non-vertical (on
+	 * unrotated output), non-grey text
 	 */
 	enum libfonts_antialiasing diagonal_colour_text_antialiasing;
 
 	/**
-	 * TODO
+	 * The output device's physical subpixel order, when it is
+	 * not rotated
 	 */
 	enum libfonts_subpixel_order unrotated_subpixel_order;
 
 	/**
 	 * TODO
 	 */
-	enum libfonts_orientation default_physical_screen_orientation;
+	struct libfonts_transformation default_transformation;
 };
 
 
@@ -486,16 +497,17 @@ int libfonts_get_output_rendering_settings(struct libfonts_rendering_settings *,
  *                  0 otherwise (projector or unsupported EDID)
  */
 int libfonts_get_output_dpi(struct libfonts_output *, const char *);
-/* TODO should take rotation into account */
+/* TODO should take transformation into account */
 
 /**
- * Calculate the subpixel order on an output device after a rotation is applied
+ * Calculate the subpixel order on an output device after
+ * a transformation is applied on its output
  * 
- * @param   unrotated  The device's subpixel order
- * @param   rotation   The applied rotation
- * @return             The device's logical subpixel order
+ * @param   unrotated       The device's subpixel order
+ * @param   transformation  The applied transformation
+ * @return                  The device's logical subpixel order
  */
-enum libfonts_subpixel_order libfonts_calculate_subpixel_order(enum libfonts_subpixel_order, enum libfonts_orientation);
+enum libfonts_subpixel_order libfonts_calculate_subpixel_order(enum libfonts_subpixel_order, const struct libfonts_transformation *);
 
 
 #endif
diff --git a/libfonts_calculate_subpixel_order.c b/libfonts_calculate_subpixel_order.c
index 6d26921..746cece 100644
--- a/libfonts_calculate_subpixel_order.c
+++ b/libfonts_calculate_subpixel_order.c
@@ -2,51 +2,43 @@
 #include "libfonts.h"
 
 
-enum libfonts_subpixel_order
-libfonts_calculate_subpixel_order(enum libfonts_subpixel_order unrotated, enum libfonts_orientation rotation)
-{
-	int rot;
+#define TOLERANCE 0.000001
 
-	switch (rotation) {
-	case LIBFONTS_ORIENTATION_UNKNOWN:
-	default:
-		switch (unrotated) {
-		case LIBFONTS_SUBPIXEL_ORDER_UNKNOWN:
-		case LIBFONTS_SUBPIXEL_ORDER_NONRGB:
-		case LIBFONTS_SUBPIXEL_ORDER_NONLINEAR:
-		case LIBFONTS_SUBPIXEL_ORDER_OTHER:
-			return unrotated;
-		default:
-			return LIBFONTS_SUBPIXEL_ORDER_UNKNOWN;
-		}
-		break;
 
-	case LIBFONTS_ORIENTATION_0_DEGREES_CLOCKWISE:
-		return unrotated;
+static void
+transform(double *x_out, double *y_out, double x, double y, const struct libfonts_transformation *transformation)
+{
+	double a = transformation->m[0][0], b = transformation->m[0][1], c = transformation->m[0][2];
+	double d = transformation->m[1][0], e = transformation->m[1][1], f = transformation->m[1][2];
+	double g = transformation->m[2][0], h = transformation->m[2][1], i = transformation->m[2][2];
+	double w = g*x + h*y + i;
+	*x_out = (a*x + b*y + c) / w;
+	*y_out = (d*x + e*y + f) / w;
+}
 
-	case LIBFONTS_ORIENTATION_90_DEGREES_CLOCKWISE:
-		rot = 1;
-		break;
 
-	case LIBFONTS_ORIENTATION_180_DEGREES_CLOCKWISE:
-		rot = 2;
-		break;
+static int
+eq(double a, double b)
+{
+	return b - TOLERANCE <= a && a <= b + TOLERANCE;
+}
 
-	case LIBFONTS_ORIENTATION_270_DEGREES_CLOCKWISE:
-		rot = 3;
-		break;
 
-	case LIBFONTS_ORIENTATION_OTHER:
-		switch (unrotated) {
-		case LIBFONTS_SUBPIXEL_ORDER_UNKNOWN:
-		case LIBFONTS_SUBPIXEL_ORDER_NONRGB:
-		case LIBFONTS_SUBPIXEL_ORDER_NONLINEAR:
-			return unrotated;
-		default:
-			return LIBFONTS_SUBPIXEL_ORDER_NONLINEAR;
-		}
-		break;
-	}
+enum libfonts_subpixel_order
+libfonts_calculate_subpixel_order(enum libfonts_subpixel_order unrotated, const struct libfonts_transformation *transformation)
+{
+#define ASIS 0
+#define ROTATED_90_DEG_CLOCKWISE 1
+#define ROTATED_180_DEG_CLOCKWISE 2
+#define ROTATED_270_DEG_CLOCKWISE 3
+#define FLIPPED -1
+#define FLOPPED -2
+#define TRANSPOSED -3
+#define ANTITRANSPOSED -4
+#define OTHER -5
+
+	double x[4], y[4], xmin, ymin, xmax, ymax, t1, t2;
+	int trans, i, j;
 
 	switch (unrotated) {
 	case LIBFONTS_SUBPIXEL_ORDER_UNKNOWN:
@@ -55,6 +47,106 @@ libfonts_calculate_subpixel_order(enum libfonts_subpixel_order unrotated, enum l
 	case LIBFONTS_SUBPIXEL_ORDER_OTHER:
 		return unrotated;
 	default:
-		return ((unrotated + rot) & 3) + (unrotated & ~3);
+		break;
+	}
+
+	for (i = 0; i < 3; i++)
+		for (j = 0; j < 3; j++)
+			if (!eq(transformation->m[i][j], 0))
+				goto known;
+	return LIBFONTS_SUBPIXEL_ORDER_UNKNOWN;
+known:
+
+	transform(&x[0], &y[0], 0, 0, transformation);
+	transform(&x[1], &y[1], 1, 0, transformation);
+	transform(&x[2], &y[2], 0, 1, transformation);
+	transform(&x[3], &y[3], 1, 1, transformation);
+
+	t1 = x[0] < x[1] ? x[0] : x[1];
+	t2 = x[2] < x[3] ? x[2] : x[3];
+	xmin = t1 < t2 ? t1 : t2;
+
+	t1 = y[0] < y[1] ? y[0] : y[1];
+	t2 = y[2] < y[3] ? y[2] : y[3];
+	ymin = t1 < t2 ? t1 : t2;
+
+	for (i = 0; i < 4; i++) {
+		x[i] -= xmin;
+		y[i] -= ymin;
+	}
+
+	t1 = x[0] > x[1] ? x[0] : x[1];
+	t2 = x[2] > x[3] ? x[2] : x[3];
+	xmax = t1 > t2 ? t1 : t2;
+
+	t1 = y[0] > y[1] ? y[0] : y[1];
+	t2 = y[2] > y[3] ? y[2] : y[3];
+	ymax = t1 > t2 ? t1 : t2;
+
+	for (i = 0; i < 4; i++) {
+		x[i] /= xmax;
+		y[i] /= ymax;
+	}
+
+	/* Here we select the inverse transformation from what `transformation`
+	 * applies, as `transformation` applies to the image on the output and
+	 * we want to know how the output is physically configured, which is
+	 * the inverse */
+	trans = OTHER;
+	if (eq(x[0], 0) && eq(y[0], 0)  &&  eq(x[3], 1) && eq(y[3], 1)) {
+		if (eq(x[1], 1) && eq(y[1], 0)  &&  eq(x[2], 0) && eq(y[2], 1))
+			trans = ASIS;
+		else if (eq(x[2], 1) && eq(y[2], 0)  &&  eq(x[1], 0) && eq(y[1], 1))
+			trans = TRANSPOSED;
+	} else if (eq(x[1], 0) && eq(y[1], 0)  &&  eq(x[2], 1) && eq(y[2], 1)) {
+		if (eq(x[0], 1) && eq(y[0], 0)  &&  eq(x[3], 0) && eq(y[3], 1))
+			trans = FLOPPED;
+		else if (eq(x[3], 1) && eq(y[3], 0)  &&  eq(x[0], 0) && eq(y[0], 1))
+			trans = ROTATED_90_DEG_CLOCKWISE;
+	} else if (eq(x[2], 0) && eq(y[2], 0)  &&  eq(x[1], 1) && eq(y[1], 1)) {
+		if (eq(x[0], 1) && eq(y[0], 0)  &&  eq(x[3], 0) && eq(y[3], 1))
+			trans = ROTATED_270_DEG_CLOCKWISE;
+		else if (eq(x[3], 1) && eq(y[3], 0)  &&  eq(x[0], 0) && eq(y[0], 1))
+			trans = FLIPPED;
+	} else if (eq(x[3], 0) && eq(y[3], 0)  &&  eq(x[0], 1) && eq(y[0], 1)) {
+		if (eq(x[1], 1) && eq(y[1], 0)  &&  eq(x[2], 0) && eq(y[2], 1))
+			trans = ANTITRANSPOSED;
+		else if (eq(x[2], 1) && eq(y[2], 0)  &&  eq(x[1], 0) && eq(y[1], 1))
+			trans = ROTATED_180_DEG_CLOCKWISE;
+	}
+
+	switch (trans) {
+	case OTHER:
+		return LIBFONTS_SUBPIXEL_ORDER_NONLINEAR;
+
+	case ANTITRANSPOSED:
+		if (unrotated <= LIBFONTS_SUBPIXEL_ORDER_G_R_B)
+			return unrotated ^ 1;
+		else
+			return unrotated ^ 5;
+
+	case TRANSPOSED:
+		if (unrotated <= LIBFONTS_SUBPIXEL_ORDER_G_R_B)
+			return unrotated ^ 1;
+		else
+			return 8 - (unrotated & 7) + (unrotated & ~7);
+
+	case FLOPPED:
+		if (unrotated <= LIBFONTS_SUBPIXEL_ORDER_G_R_B)
+			return unrotated ^ ((~unrotated << 1) & 2);
+		else
+			return unrotated ^ 4 ^ ((unrotated << 1) & 2);
+
+	case FLIPPED:
+		if (unrotated <= LIBFONTS_SUBPIXEL_ORDER_G_R_B)
+			return unrotated ^ ((unrotated << 1) & 2);
+		else
+			return unrotated ^ 4 ^ ((~unrotated << 1) & 2);
+
+	case ASIS:
+		return unrotated;
+
+	default:
+		return ((unrotated + trans) & 3) + (unrotated & ~3);
 	}
 }