From 3660c07f5ad83e0f1b26a8ccd1e3244189165ba6 Mon Sep 17 00:00:00 2001
From: Matthias Baesken <mbaesken@openjdk.org>
Date: Wed, 23 Oct 2024 08:10:45 +0000
Subject: [PATCH] 8339487: ProcessHandleImpl os_getChildren sysctl call - retry
 in case of ENOMEM and enhance exception message

Backport-of: 4ff72dc57e65e99b129f0ba28196994edf402018
---
 .../native/libjava/ProcessHandleImpl_macosx.c | 44 ++++++++++++-------
 .../native/libjava/ProcessHandleImpl_unix.c   |  2 +-
 2 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/src/java.base/macosx/native/libjava/ProcessHandleImpl_macosx.c b/src/java.base/macosx/native/libjava/ProcessHandleImpl_macosx.c
index 30fa8eea206..b9f3428c1fa 100644
--- a/src/java.base/macosx/native/libjava/ProcessHandleImpl_macosx.c
+++ b/src/java.base/macosx/native/libjava/ProcessHandleImpl_macosx.c
@@ -89,25 +89,35 @@ jint os_getChildren(JNIEnv *env, jlong jpid, jlongArray jarray,
         }
     }
 
-    // Get buffer size needed to read all processes
-    int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0};
-    if (sysctl(mib, 4, NULL, &bufSize, NULL, 0) < 0) {
-        JNU_ThrowByNameWithLastError(env,
-            "java/lang/RuntimeException", "sysctl failed");
-        return -1;
-    }
+    int errsysctl;
+    int maxRetries = 100;
+    void *buffer = NULL;
+    do {
+        int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0};
+        if (buffer != NULL) free(buffer);
+        // Get buffer size needed to read all processes
+        if (sysctl(mib, 4, NULL, &bufSize, NULL, 0) < 0) {
+            JNU_ThrowByNameWithMessageAndLastError(env,
+                "java/lang/RuntimeException", "sysctl failed");
+            return -1;
+        }
 
-    // Allocate buffer big enough for all processes
-    void *buffer = malloc(bufSize);
-    if (buffer == NULL) {
-        JNU_ThrowOutOfMemoryError(env, "malloc failed");
-        return -1;
-    }
+        // Allocate buffer big enough for all processes; add a little
+        // bit of space to be able to hold a few more proc infos
+        // for processes started right after the first sysctl call
+        buffer = malloc(bufSize + 4 * sizeof(struct kinfo_proc));
+        if (buffer == NULL) {
+            JNU_ThrowOutOfMemoryError(env, "malloc failed");
+            return -1;
+        }
 
-    // Read process info for all processes
-    if (sysctl(mib, 4, buffer, &bufSize, NULL, 0) < 0) {
-        JNU_ThrowByNameWithLastError(env,
-            "java/lang/RuntimeException", "sysctl failed");
+        // Read process info for all processes
+        errsysctl = sysctl(mib, 4, buffer, &bufSize, NULL, 0);
+    } while (errsysctl < 0 && errno == ENOMEM && maxRetries-- > 0);
+
+    if (errsysctl < 0) {
+        JNU_ThrowByNameWithMessageAndLastError(env,
+            "java/lang/RuntimeException", "sysctl failed to get info about all processes");
         free(buffer);
         return -1;
     }
diff --git a/src/java.base/unix/native/libjava/ProcessHandleImpl_unix.c b/src/java.base/unix/native/libjava/ProcessHandleImpl_unix.c
index d53e88764c5..8b900dcf6f6 100644
--- a/src/java.base/unix/native/libjava/ProcessHandleImpl_unix.c
+++ b/src/java.base/unix/native/libjava/ProcessHandleImpl_unix.c
@@ -537,7 +537,7 @@ jint unix_getChildren(JNIEnv *env, jlong jpid, jlongArray jarray,
      * position integer as a filename.
      */
     if ((dir = opendir("/proc")) == NULL) {
-        JNU_ThrowByNameWithLastError(env,
+        JNU_ThrowByNameWithMessageAndLastError(env,
             "java/lang/RuntimeException", "Unable to open /proc");
         return -1;
     }
-- 
GitLab