From 0b87fbbe994ab878215514c14fd5f1b4f1cf2790 Mon Sep 17 00:00:00 2001 From: demrom <47463752+demrom@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:30:57 +0300 Subject: [PATCH] Make ending commands and waiting callers sleep instead of spinning Callers that start a command or change the directory looped on cleanup_counter, a command that ends looped on pthread_mutex_trylock(&pid_mtx), and a command that ends in the middle of a pipeline looped on current_command_root_thread with an fflush each time. The counter now changes under a mutex and the callers sleep on a condition variable. The trylock loop is a pthread_mutex_lock. The pipeline wait sleeps on a condition that a command signals when it clears current_command_root_thread, and wakes every 50 ms to see whether the command was cancelled. tests/cleanup-wait.m measures the CPU of a pipeline whose middle command waits for the first one. ios_cleanupWait disables cancellation while it waits, so a command cancelled there does not keep cleanup_mutex. tests/cleanup-cancel.m cancels a thread blocked in chdir while a command ends. --- ios_system.m | 47 +++++++++++++++++++++++--------- libc_replacement.c | 31 +++++++++++++++++++-- tests/README.md | 6 ++++ tests/cleanup-cancel.m | 61 +++++++++++++++++++++++++++++++++++++++++ tests/cleanup-wait.m | 62 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 tests/cleanup-cancel.m create mode 100644 tests/cleanup-wait.m diff --git a/ios_system.m b/ios_system.m index 097a8bea..a7e16e8c 100644 --- a/ios_system.m +++ b/ios_system.m @@ -406,7 +406,20 @@ int ios_getCommandStatus(void) { static functionParameters *runningSignalCommands = NULL; extern pthread_mutex_t pid_mtx; -extern _Atomic(int) cleanup_counter; +extern void ios_cleanupBegin(void); +extern void ios_cleanupEnd(void); +extern void ios_cleanupWait(void); + +// Commands that end in the middle of a pipeline wait here until the first command of the pipeline has ended. +static pthread_mutex_t rootThreadMutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t rootThreadCleared = PTHREAD_COND_INITIALIZER; + +static void clearRootThread(sessionParameters *session) { + pthread_mutex_lock(&rootThreadMutex); + session->current_command_root_thread = 0; + pthread_cond_broadcast(&rootThreadCleared); + pthread_mutex_unlock(&rootThreadMutex); +} extern void ios_releaseBackgroundThread(pthread_t thread); extern int startedPreparingWebAssemblyCommand(void); extern __thread int currentWebAssemblyCommandOrder; @@ -657,8 +670,8 @@ static void cleanup_function(void* parameters) { } // Some programs stop waiting as soon as stdout/stderr close (which makes sense) // This fclose does close the fileno, but I find it re-opened later. - cleanup_counter++; - while (pthread_mutex_trylock(&pid_mtx) != 0) { } // Someone else has the lock, so we wait. + ios_cleanupBegin(); + pthread_mutex_lock(&pid_mtx); // Someone else may have the lock, so we wait. pthread_mutex_unlock(&pid_mtx); if (mustCloseStderr && !callerOwnsStream(p->stderr)) { NSLog(@"Closing stderr (mustCloseStderr): %d \n", fileno(p->stderr)); @@ -696,7 +709,7 @@ static void cleanup_function(void* parameters) { int res = fclose(p->stdin); } // Upstream commands may still need ios_fork/chdir while we wait. - cleanup_counter--; + ios_cleanupEnd(); // Close our streams before waiting: an early reader must unblock upstream writers. // If the command was started as a pipe, we wait for the first command to finish sending data // There is an exception for ssh, which can be started by scp or sftp. They will wait for it. @@ -711,15 +724,23 @@ static void cleanup_function(void* parameters) { if (ownedCommandCancelled(p)) break; fflush(currentSession->stdout); fflush(currentSession->stderr); + // Sleep until the root thread ends; the timeout is for cancellation, which has no condition to signal. + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_nsec += 50 * 1000 * 1000; + if (deadline.tv_nsec >= 1000000000) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000; } + pthread_mutex_lock(&rootThreadMutex); + if (currentSession->current_command_root_thread != 0) pthread_cond_timedwait(&rootThreadCleared, &rootThreadMutex, &deadline); + pthread_mutex_unlock(&rootThreadMutex); } NSLog(@"Thread %x is done waiting for root_thread of currentSession: %x \n", current_thread, currentSession->current_command_root_thread); } else { NSLog(@"Terminating root_thread of currentSession %x \n", current_thread); - currentSession->current_command_root_thread = 0; + clearRootThread(currentSession); } } } - cleanup_counter++; + ios_cleanupBegin(); // Specific to run multiple python3 interpreters: NSString* commandNameString = [NSString stringWithCString: commandName encoding:NSUTF8StringEncoding]; if (strncmp(commandName, "python", 6) == 0) { @@ -832,12 +853,12 @@ static void cleanup_function(void* parameters) { ios_releaseThread(current_thread); } if (p->commandStarted && currentSession->current_command_root_thread == current_thread) { - currentSession->current_command_root_thread = 0; + clearRootThread(currentSession); } if (p->commandStarted && currentSession->mainThreadId == current_thread) { currentSession->mainThreadId = 0; } - cleanup_counter--; + ios_cleanupEnd(); NSLog(@"returning from cleanup_function, session: %s\n", (char*)currentSession->context); leaveOwnedCommand(p); free(parameters); // This was malloc'ed in ios_system @@ -1406,7 +1427,7 @@ void __cd_to_dir(NSString *newDir, NSFileManager *fileManager) { #undef fchdir int ios_fchdir(const int fd) { // NSLog(@"Locking for thread %x in ios_fchdir\n", pthread_self()); - while (cleanup_counter > 0) { } // Don't chdir while a command is ending. + ios_cleanupWait(); // Don't chdir while a command is ending. // We cannot have someone change the current directory while a command is starting or terminating. // hence the mutex_lock here. pthread_mutex_lock(&pid_mtx); @@ -1450,7 +1471,7 @@ int ios_fchdir(const int fd) { int ios_fchdir_nolock(const int fd) { // NSLog(@"fchdir_nolock: %x thread %x\n", fd, pthread_self()); // Same function as fchdir, except it does not lock. To be called when resetting directory after fork(). - while (cleanup_counter > 0) { } // Don't chdir while a command is ending. + ios_cleanupWait(); // Don't chdir while a command is ending. int result = fchdir(fd); if (result < 0) { return result; @@ -1537,7 +1558,7 @@ int chdir_nolock(const char* path) { // For some Unix commands that call chdir: // Is also called at the end of the execution of each command int chdir(const char* path) { - while (cleanup_counter > 0) { } // Don't chdir while a command is ending. + ios_cleanupWait(); // Don't chdir while a command is ending. // NSLog(@"Locking for thread %x in chdir, cd %s\n", pthread_self(), path); // We cannot have someone change the current directory while a command is starting or terminating. // hence the mutex_lock here. @@ -4231,7 +4252,7 @@ int ios_system(const char* inputCmd) { // If there are auxiliary process, also wait for them: if (currentSession->lastThreadId > 0) pthread_join(currentSession->lastThreadId, NULL); currentSession->lastThreadId = 0; - currentSession->current_command_root_thread = 0; + clearRootThread(currentSession); } currentSession->isMainThread = true; } else { @@ -4246,7 +4267,7 @@ int ios_system(const char* inputCmd) { // If there are auxiliary process, also wait for them: if (currentSession->lastThreadId > 0) pthread_join(currentSession->lastThreadId, NULL); currentSession->lastThreadId = 0; - currentSession->current_command_root_thread = 0; + clearRootThread(currentSession); } else { pthread_detach(_tid); // a thread must be either joined or detached } diff --git a/libc_replacement.c b/libc_replacement.c index e4f39f03..ee1eddb0 100644 --- a/libc_replacement.c +++ b/libc_replacement.c @@ -203,9 +203,36 @@ static int pid_overflow = 0; static pid_t current_pid = 0; // We need to lock current_pid during operations pthread_mutex_t pid_mtx = PTHREAD_MUTEX_INITIALIZER; -_Atomic(int) cleanup_counter = 0; +// Number of commands that are ending. Callers that change the directory or take a pid wait until it is zero. +static _Atomic(int) cleanup_counter = 0; +static pthread_mutex_t cleanup_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cleanup_idle = PTHREAD_COND_INITIALIZER; static pid_t last_allocated_pid = 0; +void ios_cleanupBegin(void) { + pthread_mutex_lock(&cleanup_mutex); + cleanup_counter++; + pthread_mutex_unlock(&cleanup_mutex); +} + +void ios_cleanupEnd(void) { + pthread_mutex_lock(&cleanup_mutex); + if (--cleanup_counter == 0) pthread_cond_broadcast(&cleanup_idle); + pthread_mutex_unlock(&cleanup_mutex); +} + +void ios_cleanupWait(void) { + // pthread_cond_wait is a cancellation point and returns with cleanup_mutex held: a cancelled command would run + // cleanup_function, whose ios_cleanupBegin locks cleanup_mutex again and never returns. The spin it replaces + // was not a cancellation point; the cancel now arrives at the next one after the wait. + int cancellation; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancellation); + pthread_mutex_lock(&cleanup_mutex); + while (cleanup_counter > 0) pthread_cond_wait(&cleanup_idle, &cleanup_mutex); + pthread_mutex_unlock(&cleanup_mutex); + pthread_setcancelstate(cancellation, NULL); +} + void makeGlobal(void) { copyEnvironment[current_pid] = environment[current_pid]; @@ -230,7 +257,7 @@ void newPreviousDirectory(void) { // We do not recycle process ids too quickly to avoid collisions. void storeEnvironment(char* envp[]); static inline const pid_t ios_nextAvailablePid(void) { - while (cleanup_counter > 0) { } // Don't start a command while another is ending. + ios_cleanupWait(); // Don't start a command while another is ending. // fprintf(stderr, "Locking in ios_nextAvailablePid\n"); pthread_mutex_lock(&pid_mtx); char** currentEnvironment = environmentVariables(current_pid); diff --git a/tests/README.md b/tests/README.md index ca47a9e7..2568a564 100644 --- a/tests/README.md +++ b/tests/README.md @@ -160,3 +160,9 @@ xcrun simctl spawn "$SIMULATOR_UDID" "$PWD/.build/waitpid-check/waitpid-idle" ``` Expected: exit 0 and `ios_waitpid: wall=1.5s cpu=0.0s PASS`. With the polling wait: exit 1 and `FAIL`. The twenty-second alarm bounds a stalled wait. + +# Waiting without spinning + +`cleanup-wait.m` runs `hold | mid | skip` with `joinMainThread = false`: `hold` runs 2 s, `mid` ends after 300 ms while `hold` is the pipeline root, so `mid` waits for the root before it releases its pid. The test measures the process CPU over 3 s. Build it like `waitpid-idle.m`. Expected: exit 0 and `cleanup-wait: cpu=0.01s in 3 s PASS`. The wait used to loop on the session with `fflush` and used 1.7 s of CPU (exit 1, `FAIL`). The same change replaced the other busy loops: callers that start a command or change the directory sleep on a condition until no command is ending (`ios_cleanupWait`), and a command that ends waits for `pid_mtx` with `pthread_mutex_lock` instead of a `trylock` loop. + +`cleanup-cancel.m` plays an ending command with `ios_cleanupBegin`/`ios_cleanupEnd`, starts a command that blocks in `chdir`, cancels its thread, ends the pretend command and runs one more command. Build it like `waitpid-idle.m`, with `-I .` for `ios_error.h`. Expected: exit 0 and `cleanup-cancel: PASS`. Without the cancellation guard in `ios_cleanupWait`: exit 124 and `FAIL, no end after 20 s`, because the cancelled command keeps `cleanup_mutex` locked. diff --git a/tests/cleanup-cancel.m b/tests/cleanup-cancel.m new file mode 100644 index 00000000..29fc398e --- /dev/null +++ b/tests/cleanup-cancel.m @@ -0,0 +1,61 @@ +// A command that waits in chdir while another command is ending can be cancelled (Ctrl-C, ios_kill). The cancel must +// not leave cleanup_mutex locked: cleanup_function of the cancelled command takes it again, and every later +// command start and end would block for ever. The test plays the ending command with ios_cleanupBegin/ios_cleanupEnd, +// cancels a command blocked in chdir, and then runs another command. A watchdog exits 124 when it never finishes. +#import +#import "ios_system.h" +#import "ios_error.h" +#include +#include +#include +#include + +extern void ios_cleanupBegin(void); +extern void ios_cleanupEnd(void); + +static pthread_t cdThread; +static atomic_int cdStarted; + +int cdwait_main(int argc, char **argv) { + cdThread = pthread_self(); + atomic_store(&cdStarted, 1); + chdir("."); // waits while a command is ending + usleep(10000); // a cancellation point: the cancel arrives here once the wait is over + return 0; +} + +int skip_main(int argc, char **argv) { + return 0; +} + +static void timedOut(int signal) { + fprintf(stderr, "cleanup-cancel: FAIL, no end after 20 s\n"); + _Exit(124); +} + +int main(int argc, char **argv) { + @autoreleasepool { + signal(SIGALRM, timedOut); + alarm(20); + initializeEnvironment(); + ios_switchSession("cleanup-cancel-probe"); + ios_setStreams(stdin, stdout, stderr); + replaceCommand(@"cdwait", @"cdwait_main", false); + replaceCommand(@"skip", @"skip_main", false); + joinMainThread = false; + ios_cleanupBegin(); // a command is ending + ios_system("cdwait"); + while (!atomic_load(&cdStarted)) usleep(1000); + usleep(500 * 1000); // cdwait is in chdir by now + pthread_cancel(cdThread); + usleep(300 * 1000); + ios_cleanupEnd(); // hangs here when the cancelled command kept cleanup_mutex + usleep(300 * 1000); + pid_t pid = ios_fork(); + ios_system("skip"); + ios_waitpid(pid); + fprintf(stderr, "cleanup-cancel: PASS\n"); + alarm(0); + return 0; + } +} diff --git a/tests/cleanup-wait.m b/tests/cleanup-wait.m new file mode 100644 index 00000000..033b6720 --- /dev/null +++ b/tests/cleanup-wait.m @@ -0,0 +1,62 @@ +// A command in the middle of a pipeline that ends while the first command of the pipeline is still running waits +// for it before it releases its pid. That wait must sleep: it used to loop on the session with fflush, using a whole core. +// "hold | mid | skip" runs a first command that holds 2 seconds and a middle command that ends after 300 ms. +// A watchdog exits 124 when the test does not finish in 30 seconds. +#import +#import "ios_system.h" +#import "ios_error.h" +#include +#include +#include +#include +#include + +int hold_main(int argc, char **argv) { + usleep(2000 * 1000); + return 0; +} + +int skip_main(int argc, char **argv) { + return 0; +} + +int mid_main(int argc, char **argv) { + usleep(300 * 1000); // the first command is the pipeline's root by now + return 0; +} + +static void timedOut(int signal) { + fprintf(stderr, "cleanup-wait: FAIL, no end after 30 s\n"); + _Exit(124); +} + +static double cpuSeconds(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec + usage.ru_utime.tv_usec / 1e6 + usage.ru_stime.tv_sec + usage.ru_stime.tv_usec / 1e6; +} + +int main(int argc, char **argv) { + @autoreleasepool { + signal(SIGALRM, timedOut); + alarm(30); + initializeEnvironment(); + ios_switchSession("cleanup-wait-probe"); + ios_setStreams(stdin, stdout, stderr); + replaceCommand(@"hold", @"hold_main", false); + replaceCommand(@"skip", @"skip_main", false); + replaceCommand(@"mid", @"mid_main", false); + joinMainThread = false; + double before = cpuSeconds(); + ios_system("hold | mid | skip"); + sleep(3); // the first command ends after 2 s + double cpu = cpuSeconds() - before; + // A sleeping wait costs a few milliseconds; the loop cost about 2 s, one core for as long as the first command ran. + if (cpu > 0.5) { + fprintf(stderr, "cleanup-wait: FAIL, cpu=%.2fs in 3 s, the ending command spun\n", cpu); + return 1; + } + fprintf(stderr, "cleanup-wait: cpu=%.2fs in 3 s PASS\n", cpu); + return 0; + } +}