Issue #1667 - Part 3: Fix OpenGL load and runtime issues on Big Sur

This fix is included in NSPR 4.27 and Mozilla bug 1652330.
Also put a main thread check in the cocoa draw callback.

2 files changed, 40 insertions, 7 deletions

diff --git a/nsprpub/pr/src/linking/prlink.c b/nsprpub/pr/src/linking/prlink.c
index 1f3430714e..4e73d9df1c 100644
--- a/nsprpub/pr/src/linking/prlink.c
+++ b/nsprpub/pr/src/linking/prlink.c
@@ -776,6 +776,9 @@ pr_LoadLibraryByPathname(const char *name, PRIntn flags)
         int dl_flags = 0;
 #endif
         void *h = NULL;
+#if defined(DARWIN)
+        PRBool okToLoad = PR_FALSE;
+#endif
 
         if (flags & PR_LD_LAZY) {
             dl_flags |= RTLD_LAZY;
@@ -790,12 +793,36 @@ pr_LoadLibraryByPathname(const char *name, PRIntn flags)
             dl_flags |= RTLD_LOCAL;
         }
 #if defined(DARWIN)
-        /* ensure the file exists if it contains a slash character i.e. path */
-        /* DARWIN's dlopen ignores the provided path and checks for the */
-        /* plain filename in DYLD_LIBRARY_PATH */
-        if (strchr(name, PR_DIRECTORY_SEPARATOR) == NULL ||
-            PR_Access(name, PR_ACCESS_EXISTS) == PR_SUCCESS) {
-            h = dlopen(name, dl_flags);
+        /* If the file contains an absolute or relative path (slash)
+         * and the path doesn't look like a System path, then require
+         * the file exists.
+         * The reason is that DARWIN's dlopen ignores the provided path
+         * and checks for the plain filename in DYLD_LIBRARY_PATH,
+         * which could load an unexpected version of a library. */
+        if (strchr(name, PR_DIRECTORY_SEPARATOR) == NULL) {
+          /* no slash, allow to load from any location */
+          okToLoad = PR_TRUE;
+        } else {
+          const char systemPrefix1[] = "/System/";
+          const size_t systemPrefixLen1 = strlen(systemPrefix1);
+          const char systemPrefix2[] = "/usr/lib/";
+          const size_t systemPrefixLen2 = strlen(systemPrefix2);
+          const name_len = strlen(name);
+          if (((name_len > systemPrefixLen1) &&
+               (strncmp(name, systemPrefix1, systemPrefixLen1) == 0)) ||
+             ((name_len > systemPrefixLen2) &&
+               (strncmp(name, systemPrefix2, systemPrefixLen2) == 0))) {
+            /* found at beginning, it's a system library.
+             * Skip filesystem check (required for macOS 11),
+             * allow loading from any location */
+            okToLoad = PR_TRUE;
+          } else if (PR_Access(name, PR_ACCESS_EXISTS) == PR_SUCCESS) {
+            /* file exists, allow to load */
+            okToLoad = PR_TRUE;
+          }
+        }
+        if (okToLoad) {
+          h = dlopen(name, dl_flags);
         }
 #else
         h = dlopen(name, dl_flags);
diff --git a/widget/cocoa/nsChildView.mm b/widget/cocoa/nsChildView.mm
index 95547a13ec..868687fe16 100644
--- a/widget/cocoa/nsChildView.mm
+++ b/widget/cocoa/nsChildView.mm
@@ -3564,10 +3564,16 @@ NSEvent* gLastDragMouseDownEvent = nil;
 // This method is called from mPixelHostingView's drawRect handler.
 - (void)doDrawRect:(NSRect)aRect
 {
-  CGContextRef cgContext = (CGContextRef)[[NSGraphicsContext currentContext] graphicsPort];
+  if (!NS_IsMainThread()) {
+    // In the presence of CoreAnimation, this method can sometimes be called on
+    // a non-main thread. Ignore those calls because Gecko can only react to
+    // them on the main thread.
+    return;
+  }
 
   if (!mGeckoChild || !mGeckoChild->IsVisible())
     return;
+  CGContextRef cgContext = (CGContextRef)[[NSGraphicsContext currentContext] graphicsPort];
 
   if ([self isUsingOpenGL]) {
     // Since this view is usually declared as opaque, the window's pixel