Add size checks to WebGLContext::BufferData()

On MacOS, particularly large allocations within the platform limits
(1.2G+) will fail and crash. This adds a specific size check for that
when working around driver bugs (default).
While there, added a generic size_t limited size check for the
platform, and reporting OOM if too large.

1 files changed, 12 insertions, 0 deletions

diff --git a/dom/canvas/WebGLContextBuffers.cpp b/dom/canvas/WebGLContextBuffers.cpp
index af506c01c1..f53f9d7d75 100644
--- a/dom/canvas/WebGLContextBuffers.cpp
+++ b/dom/canvas/WebGLContextBuffers.cpp
@@ -9,6 +9,8 @@
 #include "WebGLBuffer.h"
 #include "WebGLVertexArray.h"
 
+#include "mozilla/CheckedInt.h"
+
 namespace mozilla {
 
 WebGLRefPtr<WebGLBuffer>*
@@ -345,6 +347,16 @@ WebGLContext::BufferData(GLenum target, WebGLsizeiptr size, GLenum usage)
 
     ////
 
+    const auto checkedSize = CheckedInt<size_t>(size);
+    if (!checkedSize.isValid())
+      return ErrorOutOfMemory("%s: Size too large for platform.", funcName);
+     
+#if defined(XP_MACOSX)
+    if (gl->WorkAroundDriverBugs() && size > 1200000000) {
+      return ErrorOutOfMemory("Allocations larger than 1200000000 fail on MacOS.");
+    }
+#endif
+
     const UniqueBuffer zeroBuffer(calloc(size, 1));
     if (!zeroBuffer)
         return ErrorOutOfMemory("%s: Failed to allocate zeros.", funcName);