Issue #1829 - Readd a fix for Big Sur not loading libraries such as OpenGL.

This fix was included earlier but got clobbered by a recent NSPR update.

1 files changed, 34 insertions, 7 deletions

diff --git a/nsprpub/pr/src/linking/prlink.c b/nsprpub/pr/src/linking/prlink.c
index 1f3430714e..012af34c43 100644
--- a/nsprpub/pr/src/linking/prlink.c
+++ b/nsprpub/pr/src/linking/prlink.c
@@ -776,6 +776,9 @@ pr_LoadLibraryByPathname(const char *name, PRIntn flags)
         int dl_flags = 0;
 #endif
         void *h = NULL;
+#if defined(DARWIN)
+        PRBool okToLoad = PR_FALSE;
+#endif
 
         if (flags & PR_LD_LAZY) {
             dl_flags |= RTLD_LAZY;
@@ -790,13 +793,37 @@ pr_LoadLibraryByPathname(const char *name, PRIntn flags)
             dl_flags |= RTLD_LOCAL;
         }
 #if defined(DARWIN)
-        /* ensure the file exists if it contains a slash character i.e. path */
-        /* DARWIN's dlopen ignores the provided path and checks for the */
-        /* plain filename in DYLD_LIBRARY_PATH */
-        if (strchr(name, PR_DIRECTORY_SEPARATOR) == NULL ||
-            PR_Access(name, PR_ACCESS_EXISTS) == PR_SUCCESS) {
-            h = dlopen(name, dl_flags);
-        }
+       /* If the file contains an absolute or relative path (slash)
+        * and the path doesn't look like a System path, then require
+        * the file exists.
+        * The reason is that DARWIN's dlopen ignores the provided path
+        * and checks for the plain filename in DYLD_LIBRARY_PATH,
+        * which could load an unexpected version of a library. */
+       if (strchr(name, PR_DIRECTORY_SEPARATOR) == NULL) {
+         /* no slash, allow to load from any location */
+         okToLoad = PR_TRUE;
+       } else {
+         const char systemPrefix1[] = "/System/";
+         const size_t systemPrefixLen1 = strlen(systemPrefix1);
+         const char systemPrefix2[] = "/usr/lib/";
+         const size_t systemPrefixLen2 = strlen(systemPrefix2);
+         const name_len = strlen(name);
+         if (((name_len > systemPrefixLen1) &&
+              (strncmp(name, systemPrefix1, systemPrefixLen1) == 0)) ||
+            ((name_len > systemPrefixLen2) &&
+              (strncmp(name, systemPrefix2, systemPrefixLen2) == 0))) {
+           /* found at beginning, it's a system library.
+            * Skip filesystem check (required for macOS 11),
+            * allow loading from any location */
+           okToLoad = PR_TRUE;
+         } else if (PR_Access(name, PR_ACCESS_EXISTS) == PR_SUCCESS) {
+           /* file exists, allow to load */
+           okToLoad = PR_TRUE;
+         }
+       }
+       if (okToLoad) {
+         h = dlopen(name, dl_flags);
+       }
 #else
         h = dlopen(name, dl_flags);
 #endif