Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 34 additions & 13 deletions ios_system.m
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,20 @@ int ios_getCommandStatus(void) {
static functionParameters *runningSignalCommands = NULL;

extern pthread_mutex_t pid_mtx;
extern _Atomic(int) cleanup_counter;
extern void ios_cleanupBegin(void);
extern void ios_cleanupEnd(void);
extern void ios_cleanupWait(void);

// Commands that end in the middle of a pipeline wait here until the first command of the pipeline has ended.
static pthread_mutex_t rootThreadMutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t rootThreadCleared = PTHREAD_COND_INITIALIZER;

static void clearRootThread(sessionParameters *session) {
pthread_mutex_lock(&rootThreadMutex);
session->current_command_root_thread = 0;
pthread_cond_broadcast(&rootThreadCleared);
pthread_mutex_unlock(&rootThreadMutex);
}
extern void ios_releaseBackgroundThread(pthread_t thread);
extern int startedPreparingWebAssemblyCommand(void);
extern __thread int currentWebAssemblyCommandOrder;
Expand Down Expand Up @@ -661,8 +674,8 @@ static void cleanup_function(void* parameters) {
}
// Some programs stop waiting as soon as stdout/stderr close (which makes sense)
// This fclose does close the fileno, but I find it re-opened later.
cleanup_counter++;
while (pthread_mutex_trylock(&pid_mtx) != 0) { } // Someone else has the lock, so we wait.
ios_cleanupBegin();
pthread_mutex_lock(&pid_mtx); // Someone else may have the lock, so we wait.
pthread_mutex_unlock(&pid_mtx);
if (mustCloseStderr && !callerOwnsStream(p->stderr)) {
NSLog(@"Closing stderr (mustCloseStderr): %d \n", fileno(p->stderr));
Expand Down Expand Up @@ -700,7 +713,7 @@ static void cleanup_function(void* parameters) {
int res = fclose(p->stdin);
}
// Upstream commands may still need ios_fork/chdir while we wait.
cleanup_counter--;
ios_cleanupEnd();
// Close our streams before waiting: an early reader must unblock upstream writers.
// If the command was started as a pipe, we wait for the first command to finish sending data
// There is an exception for ssh, which can be started by scp or sftp. They will wait for it.
Expand All @@ -715,15 +728,23 @@ static void cleanup_function(void* parameters) {
if (ownedCommandCancelled(p)) break;
fflush(currentSession->stdout);
fflush(currentSession->stderr);
// Sleep until the root thread ends; the timeout is for cancellation, which has no condition to signal.
struct timespec deadline;
clock_gettime(CLOCK_REALTIME, &deadline);
deadline.tv_nsec += 50 * 1000 * 1000;
if (deadline.tv_nsec >= 1000000000) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000; }
pthread_mutex_lock(&rootThreadMutex);
if (currentSession->current_command_root_thread != 0) pthread_cond_timedwait(&rootThreadCleared, &rootThreadMutex, &deadline);
pthread_mutex_unlock(&rootThreadMutex);
}
NSLog(@"Thread %x is done waiting for root_thread of currentSession: %x \n", current_thread, currentSession->current_command_root_thread);
} else {
NSLog(@"Terminating root_thread of currentSession %x \n", current_thread);
currentSession->current_command_root_thread = 0;
clearRootThread(currentSession);
}
}
}
cleanup_counter++;
ios_cleanupBegin();
// Specific to run multiple python3 interpreters:
NSString* commandNameString = [NSString stringWithCString: commandName encoding:NSUTF8StringEncoding];
if (strncmp(commandName, "python", 6) == 0) {
Expand Down Expand Up @@ -836,12 +857,12 @@ static void cleanup_function(void* parameters) {
ios_releaseThread(current_thread);
}
if (p->commandStarted && currentSession->current_command_root_thread == current_thread) {
currentSession->current_command_root_thread = 0;
clearRootThread(currentSession);
}
if (p->commandStarted && currentSession->mainThreadId == current_thread) {
currentSession->mainThreadId = 0;
}
cleanup_counter--;
ios_cleanupEnd();
NSLog(@"returning from cleanup_function, session: %s\n", (char*)currentSession->context);
leaveOwnedCommand(p);
free(parameters); // Allocated in ios_system
Expand Down Expand Up @@ -1411,7 +1432,7 @@ void __cd_to_dir(NSString *newDir, NSFileManager *fileManager) {
#undef fchdir
int ios_fchdir(const int fd) {
// NSLog(@"Locking for thread %x in ios_fchdir\n", pthread_self());
while (cleanup_counter > 0) { } // Don't chdir while a command is ending.
ios_cleanupWait(); // Don't chdir while a command is ending.
// We cannot have someone change the current directory while a command is starting or terminating.
// hence the mutex_lock here.
pthread_mutex_lock(&pid_mtx);
Expand Down Expand Up @@ -1455,7 +1476,7 @@ int ios_fchdir(const int fd) {
int ios_fchdir_nolock(const int fd) {
// NSLog(@"fchdir_nolock: %x thread %x\n", fd, pthread_self());
// Same function as fchdir, except it does not lock. To be called when resetting directory after fork().
while (cleanup_counter > 0) { } // Don't chdir while a command is ending.
ios_cleanupWait(); // Don't chdir while a command is ending.
int result = fchdir(fd);
if (result < 0) {
return result;
Expand Down Expand Up @@ -1542,7 +1563,7 @@ int chdir_nolock(const char* path) {
// For some Unix commands that call chdir:
// Is also called at the end of the execution of each command
int chdir(const char* path) {
while (cleanup_counter > 0) { } // Don't chdir while a command is ending.
ios_cleanupWait(); // Don't chdir while a command is ending.
// NSLog(@"Locking for thread %x in chdir, cd %s\n", pthread_self(), path);
// We cannot have someone change the current directory while a command is starting or terminating.
// hence the mutex_lock here.
Expand Down Expand Up @@ -4239,7 +4260,7 @@ int ios_system(const char* inputCmd) {
// If there are auxiliary process, also wait for them:
if (currentSession->lastThreadId > 0) pthread_join(currentSession->lastThreadId, NULL);
currentSession->lastThreadId = 0;
currentSession->current_command_root_thread = 0;
clearRootThread(currentSession);
}
currentSession->isMainThread = true;
} else {
Expand All @@ -4254,7 +4275,7 @@ int ios_system(const char* inputCmd) {
// If there are auxiliary process, also wait for them:
if (currentSession->lastThreadId > 0) pthread_join(currentSession->lastThreadId, NULL);
currentSession->lastThreadId = 0;
currentSession->current_command_root_thread = 0;
clearRootThread(currentSession);
} else {
pthread_detach(_tid); // a thread must be either joined or detached
}
Expand Down
31 changes: 29 additions & 2 deletions libc_replacement.c
Original file line number Diff line number Diff line change
Expand Up @@ -225,9 +225,36 @@ static int pid_overflow = 0;
static pid_t current_pid = 0;
// We need to lock current_pid during operations
pthread_mutex_t pid_mtx = PTHREAD_MUTEX_INITIALIZER;
_Atomic(int) cleanup_counter = 0;
// Number of commands that are ending. Callers that change the directory or take a pid wait until it is zero.
static _Atomic(int) cleanup_counter = 0;
static pthread_mutex_t cleanup_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t cleanup_idle = PTHREAD_COND_INITIALIZER;
static pid_t last_allocated_pid = 0;

void ios_cleanupBegin(void) {
pthread_mutex_lock(&cleanup_mutex);
cleanup_counter++;
pthread_mutex_unlock(&cleanup_mutex);
}

void ios_cleanupEnd(void) {
pthread_mutex_lock(&cleanup_mutex);
if (--cleanup_counter == 0) pthread_cond_broadcast(&cleanup_idle);
pthread_mutex_unlock(&cleanup_mutex);
}

void ios_cleanupWait(void) {
// pthread_cond_wait is a cancellation point and returns with cleanup_mutex held: a cancelled command would run
// cleanup_function, whose ios_cleanupBegin locks cleanup_mutex again and never returns. The spin it replaces
// was not a cancellation point; the cancel now arrives at the next one after the wait.
int cancellation;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancellation);
pthread_mutex_lock(&cleanup_mutex);
while (cleanup_counter > 0) pthread_cond_wait(&cleanup_idle, &cleanup_mutex);
pthread_mutex_unlock(&cleanup_mutex);
pthread_setcancelstate(cancellation, NULL);
}


void makeGlobal(void) {
copyEnvironment[current_pid] = environment[current_pid];
Expand All @@ -252,7 +279,7 @@ void newPreviousDirectory(void) {
// We do not recycle process ids too quickly to avoid collisions.
void storeEnvironment(char* envp[]);
static inline const pid_t ios_nextAvailablePid(void) {
while (cleanup_counter > 0) { } // Don't start a command while another is ending.
ios_cleanupWait(); // Don't start a command while another is ending.
// fprintf(stderr, "Locking in ios_nextAvailablePid\n");
pthread_mutex_lock(&pid_mtx);
char** currentEnvironment = environmentVariables(current_pid);
Expand Down
5 changes: 5 additions & 0 deletions tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,11 @@ xcrun simctl spawn "$SIMULATOR_UDID" "$PWD/.build/waitpid-check/waitpid-idle"

Expected: exit 0 and `ios_waitpid: wall=1.5s cpu=0.0s PASS`. With the polling wait: exit 1 and `FAIL`. The twenty-second alarm bounds a stalled wait.

# Waiting without spinning

`cleanup-wait.m` runs `hold | mid | skip` with `joinMainThread = false`: `hold` runs 2 s, `mid` ends after 300 ms while `hold` is the pipeline root, so `mid` waits for the root before it releases its pid. The test measures the process CPU over 3 s. Build it like `waitpid-idle.m`. Expected: exit 0 and `cleanup-wait: cpu=0.01s in 3 s PASS`. The wait used to loop on the session with `fflush` and used 1.7 s of CPU (exit 1, `FAIL`). The same change replaced the other busy loops: callers that start a command or change the directory sleep on a condition until no command is ending (`ios_cleanupWait`), and a command that ends waits for `pid_mtx` with `pthread_mutex_lock` instead of a `trylock` loop.

`cleanup-cancel.m` plays an ending command with `ios_cleanupBegin`/`ios_cleanupEnd`, starts a command that blocks in `chdir`, cancels its thread, ends the pretend command and runs one more command. Build it like `waitpid-idle.m`, with `-I .` for `ios_error.h`. Expected: exit 0 and `cleanup-cancel: PASS`. Without the cancellation guard in `ios_cleanupWait`: exit 124 and `FAIL, no end after 20 s`, because the cancelled command keeps `cleanup_mutex` locked.
# Concurrent forks

`concurrent-fork.m` starts eight threads that each run 400 short commands behind `ios_fork`, `ios_system` and `ios_waitpid`. A command that ends must not move `current_pid` under a thread that has just forked; if it does, the new command's slot stays at "not started" and its `ios_waitpid` never returns.
Expand Down
61 changes: 61 additions & 0 deletions tests/cleanup-cancel.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// A command that waits in chdir while another command is ending can be cancelled (Ctrl-C, ios_kill). The cancel must
// not leave cleanup_mutex locked: cleanup_function of the cancelled command takes it again, and every later
// command start and end would block for ever. The test plays the ending command with ios_cleanupBegin/ios_cleanupEnd,
// cancels a command blocked in chdir, and then runs another command. A watchdog exits 124 when it never finishes.
#import <Foundation/Foundation.h>
#import "ios_system.h"
#import "ios_error.h"
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <unistd.h>

extern void ios_cleanupBegin(void);
extern void ios_cleanupEnd(void);

static pthread_t cdThread;
static atomic_int cdStarted;

int cdwait_main(int argc, char **argv) {
cdThread = pthread_self();
atomic_store(&cdStarted, 1);
chdir("."); // waits while a command is ending
usleep(10000); // a cancellation point: the cancel arrives here once the wait is over
return 0;
}

int skip_main(int argc, char **argv) {
return 0;
}

static void timedOut(int signal) {
fprintf(stderr, "cleanup-cancel: FAIL, no end after 20 s\n");
_Exit(124);
}

int main(int argc, char **argv) {
@autoreleasepool {
signal(SIGALRM, timedOut);
alarm(20);
initializeEnvironment();
ios_switchSession("cleanup-cancel-probe");
ios_setStreams(stdin, stdout, stderr);
replaceCommand(@"cdwait", @"cdwait_main", false);
replaceCommand(@"skip", @"skip_main", false);
joinMainThread = false;
ios_cleanupBegin(); // a command is ending
ios_system("cdwait");
while (!atomic_load(&cdStarted)) usleep(1000);
usleep(500 * 1000); // cdwait is in chdir by now
pthread_cancel(cdThread);
usleep(300 * 1000);
ios_cleanupEnd(); // hangs here when the cancelled command kept cleanup_mutex
usleep(300 * 1000);
pid_t pid = ios_fork();
ios_system("skip");
ios_waitpid(pid);
fprintf(stderr, "cleanup-cancel: PASS\n");
alarm(0);
return 0;
}
}
62 changes: 62 additions & 0 deletions tests/cleanup-wait.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
// A command in the middle of a pipeline that ends while the first command of the pipeline is still running waits
// for it before it releases its pid. That wait must sleep: it used to loop on the session with fflush, using a whole core.
// "hold | mid | skip" runs a first command that holds 2 seconds and a middle command that ends after 300 ms.
// A watchdog exits 124 when the test does not finish in 30 seconds.
#import <Foundation/Foundation.h>
#import "ios_system.h"
#import "ios_error.h"
#include <pthread.h>
#include <signal.h>
#include <sys/resource.h>
#include <sys/time.h>
#include <unistd.h>

int hold_main(int argc, char **argv) {
usleep(2000 * 1000);
return 0;
}

int skip_main(int argc, char **argv) {
return 0;
}

int mid_main(int argc, char **argv) {
usleep(300 * 1000); // the first command is the pipeline's root by now
return 0;
}

static void timedOut(int signal) {
fprintf(stderr, "cleanup-wait: FAIL, no end after 30 s\n");
_Exit(124);
}

static double cpuSeconds(void) {
struct rusage usage;
getrusage(RUSAGE_SELF, &usage);
return usage.ru_utime.tv_sec + usage.ru_utime.tv_usec / 1e6 + usage.ru_stime.tv_sec + usage.ru_stime.tv_usec / 1e6;
}

int main(int argc, char **argv) {
@autoreleasepool {
signal(SIGALRM, timedOut);
alarm(30);
initializeEnvironment();
ios_switchSession("cleanup-wait-probe");
ios_setStreams(stdin, stdout, stderr);
replaceCommand(@"hold", @"hold_main", false);
replaceCommand(@"skip", @"skip_main", false);
replaceCommand(@"mid", @"mid_main", false);
joinMainThread = false;
double before = cpuSeconds();
ios_system("hold | mid | skip");
sleep(3); // the first command ends after 2 s
double cpu = cpuSeconds() - before;
// A sleeping wait costs a few milliseconds; the loop cost about 2 s, one core for as long as the first command ran.
if (cpu > 0.5) {
fprintf(stderr, "cleanup-wait: FAIL, cpu=%.2fs in 3 s, the ending command spun\n", cpu);
return 1;
}
fprintf(stderr, "cleanup-wait: cpu=%.2fs in 3 s PASS\n", cpu);
return 0;
}
}
Loading