Correctly return zero vertices if clipping plane 0 or 2 clip away the

entire polygon.

This fixes a bug that was introduced three years ago in BZ bug 1268854.
What happened was that the final pass over the polygon assumed that the
current polygon was living in plane[0]. But due to the double buffering,
the "current" polygon alternates between plane[0] and plane[1].
The bug had also introduced an early exit so that we could hit the final
pass at a time where the current, now empty, polygon was in plane[1]. So
we would incorrectly treat all 32 points in plane[0] as part of the
final polygon.

This bug was responsible for intermittently unreasonable numbers in
CompositorOGL's fill rate / overdraw overlay.

This fixes a regression caused by the fix for CVE-2016-5252.

2 files changed, 62 insertions, 0 deletions

diff --git a/gfx/tests/gtest/TestMatrix.cpp b/gfx/tests/gtest/TestMatrix.cpp
new file mode 100644
index 0000000000..bc2f9e63cd
--- /dev/null
+++ b/gfx/tests/gtest/TestMatrix.cpp
@@ -0,0 +1,61 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+#include "mozilla/gfx/Matrix.h"
+
+using namespace mozilla;
+using namespace mozilla::gfx;
+
+static Rect NudgedToInt(const Rect& aRect) {
+  Rect r(aRect);
+  r.NudgeToIntegers();
+  return r;
+}
+
+TEST(Matrix, TransformAndClipRect)
+{
+  Rect c(100, 100, 100, 100);
+  Matrix4x4 m;
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(50, 50, 20, 20), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(250, 50, 20, 20), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(250, 250, 20, 20), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(50, 250, 20, 20), c).IsEmpty());
+
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(50, 50, 100, 20), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(150, 50, 100, 20), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(50, 250, 100, 20), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(150, 250, 100, 20), c).IsEmpty());
+
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(50, 50, 20, 100), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(50, 150, 20, 100), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(250, 50, 20, 100), c).IsEmpty());
+  EXPECT_TRUE(m.TransformAndClipBounds(Rect(250, 150, 20, 100), c).IsEmpty());
+
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(50, 50, 100, 100), c))
+                  .IsEqualInterior(Rect(100, 100, 50, 50)));
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(150, 50, 100, 100), c))
+                  .IsEqualInterior(Rect(150, 100, 50, 50)));
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(150, 150, 100, 100), c))
+                  .IsEqualInterior(Rect(150, 150, 50, 50)));
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(50, 150, 100, 100), c))
+                  .IsEqualInterior(Rect(100, 150, 50, 50)));
+
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(110, 110, 80, 80), c))
+                  .IsEqualInterior(Rect(110, 110, 80, 80)));
+
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(50, 50, 200, 200), c))
+                  .IsEqualInterior(Rect(100, 100, 100, 100)));
+
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(50, 50, 200, 100), c))
+                  .IsEqualInterior(Rect(100, 100, 100, 50)));
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(50, 150, 200, 100), c))
+                  .IsEqualInterior(Rect(100, 150, 100, 50)));
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(50, 50, 100, 200), c))
+                  .IsEqualInterior(Rect(100, 100, 50, 100)));
+  EXPECT_TRUE(NudgedToInt(m.TransformAndClipBounds(Rect(150, 50, 100, 200), c))
+                  .IsEqualInterior(Rect(150, 100, 50, 100)));
+}
diff --git a/gfx/tests/gtest/moz.build b/gfx/tests/gtest/moz.build
index 23b019d1be..ea18c1e3b4 100644
--- a/gfx/tests/gtest/moz.build
+++ b/gfx/tests/gtest/moz.build
@@ -17,6 +17,7 @@ UNIFIED_SOURCES += [
     'TestGfxWidgets.cpp',
     'TestJobScheduler.cpp',
     'TestLayers.cpp',
+    'TestMatrix.cpp',
     'TestMoz2D.cpp',
     'TestPolygon.cpp',
     'TestQcms.cpp',