Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions ios_system.m
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
// the reverse (if type == "r").

#include <pthread.h>
#include <stdatomic.h>
#include <sys/stat.h>
#include <libgen.h> // for basename()
#include <dlfcn.h> // for dlopen()/dlsym()/dlclose()
Expand Down Expand Up @@ -75,14 +76,14 @@
char localMiniRoot[MAXPATHLEN];
pthread_t current_command_root_thread; // thread ID of first command
pthread_t lastThreadId; // thread ID of last command.
pthread_t mainThreadId; // thread ID of parent command, if any (e.g. vim, which starts "sh -c cd dir && flake8 file")
_Atomic(pthread_t) mainThreadId; // thread ID of parent command, if any (e.g. vim, which starts "sh -c cd dir && flake8 file")
FILE* stdin;
FILE* stdout;
FILE* stderr;
FILE* tty;
bool nativeTTY; // Native jobs use the actual descriptor terminal state.
const void* context;
int global_errno;
_Atomic int global_errno; // written by command threads, read by the caller of ios_system
int numCommandsAllocated;
int numCommand;
char** commandName;
Expand Down Expand Up @@ -304,8 +305,9 @@ void ios_setWindowSize(int width, int height, const void* sessionId) {
if (resizedSession->lastThreadId != NULL) {
pthread_kill(resizedSession->lastThreadId, SIGWINCH);
}
if (resizedSession->mainThreadId != NULL) {
pthread_kill(resizedSession->mainThreadId, SIGWINCH);
pthread_t mainThread = resizedSession->mainThreadId; // one load: cleanup_function can clear it in between
if (mainThread != NULL) {
pthread_kill(mainThread, SIGWINCH);
}
}

Expand Down Expand Up @@ -402,7 +404,7 @@ int ios_getCommandStatus(void) {
void *inputOwner;
struct _functionParameters *ownedNext;
pthread_t ownedThread;
bool ownedRunning, commandStarted;
bool ownedRunning, commandStarted, storeMainThread;
} functionParameters;

static pthread_mutex_t commandSignalMutex = PTHREAD_MUTEX_INITIALIZER;
Expand Down Expand Up @@ -859,8 +861,9 @@ static void cleanup_function(void* parameters) {
if (p->commandStarted && currentSession->current_command_root_thread == current_thread) {
clearRootThread(currentSession);
}
if (p->commandStarted && currentSession->mainThreadId == current_thread) {
currentSession->mainThreadId = 0;
if (p->commandStarted) {
pthread_t expectedMain = current_thread;
atomic_compare_exchange_strong(&currentSession->mainThreadId, &expectedMain, (pthread_t)0);
}
ios_cleanupEnd();
NSLog(@"returning from cleanup_function, session: %s\n", (char*)currentSession->context);
Expand Down Expand Up @@ -900,6 +903,11 @@ void crash_handler(int sig) {
NSLog(@"Storing thread_id: %x as root_thread\n", pthread_self());
p->session->current_command_root_thread = pthread_self();
}
if (p->storeMainThread && (p->session != NULL)) {
// On the command thread, so that a command that ends first is never recorded after it ended.
pthread_t noMainThread = NULL;
atomic_compare_exchange_strong(&p->session->mainThreadId, &noMainThread, pthread_self());
}
// NSLog(@"Starting command: %s thread_id %x", p->argv[0], pthread_self());
// re-initialize for getopt:
// TODO: move to __thread variable for optind too
Expand Down Expand Up @@ -3271,6 +3279,7 @@ int ios_system(const char* inputCmd) {
params->backgroundCommand = isBackgroundCommand(command);
params->numInterpreter = 0;
params->webAssemblyOrder = 0;
params->storeMainThread = false;

params->context = thread_context;

Expand Down Expand Up @@ -4250,9 +4259,9 @@ int ios_system(const char* inputCmd) {
__block volatile pthread_t _tid = NULL;
[fileCoordinator coordinateWritingItemAtURL:currentURL options:0 error:NULL byAccessor:^(NSURL *currentURL) {
currentSession->isMainThread = false;
params->storeMainThread = true;
if (createOwnedCommand(&_tid, params) != 0) return;
// ios_storeThreadId(_tid);
if (currentSession->mainThreadId == NULL) currentSession->mainThreadId = _tid;
if (!joinMainThread) pthread_detach(_tid); // a thread must be either joined or detached
}];
// Wait for this process to finish:
Expand All @@ -4266,10 +4275,10 @@ int ios_system(const char* inputCmd) {
currentSession->isMainThread = true;
} else {
currentSession->isMainThread = false;
params->storeMainThread = true;
volatile pthread_t _tid = NULL;
if (createOwnedCommand(&_tid, params) == 0) {
// ios_storeThreadId(_tid);
if (currentSession->mainThreadId == NULL) currentSession->mainThreadId = _tid;
// Wait for this process to finish:
if (joinMainThread) {
pthread_join(_tid, NULL);
Expand Down
4 changes: 3 additions & 1 deletion libc_replacement.c
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,16 @@ void finishedPreparingWebAssemblyCommand(void) {
}

static void executeWebAssemblyCommandsInOrder(void) {
static void (*function)(void) = NULL;
static _Atomic(void (*)(void)) cachedFunction = NULL;
void (*function)(void) = atomic_load(&cachedFunction);
if (function == NULL) {
function = dlsym(RTLD_MAIN_ONLY, "executeWebAssemblyCommands");
}
if (function == NULL) {
// more extensive search, but more expensive too.
function = dlsym(RTLD_DEFAULT, "executeWebAssemblyCommands");
}
atomic_store(&cachedFunction, function);
if (function != NULL) {
while (preparingWebAssemblyCommands > 0) {
// Empty loops create problems in Release mode.
Expand Down
8 changes: 8 additions & 0 deletions tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,14 @@ Expected: exit 0 and `ios_waitpid: wall=1.5s cpu=0.0s PASS`. With the polling wa

Build and run it like `waitpid-idle.m` (`-F` the framework directory, `tests/concurrent-fork.m`, `xcrun simctl spawn`). Expected: exit 0 and `concurrent-fork: 8 threads x 400 commands PASS`. Without the compare-and-swap in `ios_restoreParentPid`: exit 124 and `FAIL, N of 8 threads finished`, bounded by a two-minute alarm that prints the stacks of all threads before it exits (`hang-dump.h`, via `sample`).

# ThreadSanitizer

Build ios_system with `xcodebuild ... ENABLE_THREAD_SANITIZER=YES`, link a test with `-fsanitize=thread`, copy `libclang_rt.tsan_iossim_dynamic.dylib` next to the frameworks and run it with `SIMCTL_CHILD_TSAN_OPTIONS=suppressions=tests/tsan.supp xcrun simctl spawn`. Expected with the suppressions: `concurrent-fork` exits 0 and prints no `WARNING: ThreadSanitizer`. Without the fixes for the cached `executeWebAssemblyCommands` lookup and the session's `global_errno`, it prints 3 to 4 reports per run.

`tsan.supp` hides the accesses inside the pid-allocation window: `ios_nextAvailablePid` returns with `pid_mtx` locked and `ios_storeThreadId` unlocks it on another thread, which TSan cannot follow, so it reports two threads that both hold the mutex. The `race:ios_currentPid` entry also hides the unlocked reads of `current_pid` against the plain writes in `ios_nextAvailablePid`; those races are not changed here.

`main-thread-winch.m` checks that `ios_setWindowSize` signals the command that is the main thread of its session. It starts a quick command with `pthread_create` slowed down by 100 ms (`main-thread-winch-slowcreate.c`, a dylib that interposes `pthread_create`), so the command has ended before its starter goes on, then starts a long command named `dash` with a `SIGWINCH` handler, resizes the window and expects the handler to run on that command's thread. Build the dylib with `-dynamiclib -o libslowcreate.dylib`, then the test like `concurrent-fork.m`, adding `-L <dir> -lslowcreate`. Expected: exit 0 and `main-thread-winch: PASS`. When a command that has ended stays recorded as the main thread, the stale id names a thread started after the command ended, that thread gets the signal, and the test prints `FAIL, SIGWINCH was handled on another thread`.

# Pid admission

`pid-admission.m` starts 200 threads that each run `ios_fork`, `ios_system("hold")` and `ios_waitpid`, with only 127 pids available. `hold` blocks until the test releases it, so while the table is full nothing ends. The callers beyond the limit must wait for a pid and then run. The test measures the CPU the 73 waiting callers use in one second, and fails above 0.5 s. A watchdog thread prints the stacks of all threads (`hang-dump.h`) and exits 124 when nothing has moved for 10 seconds.
Expand Down
16 changes: 16 additions & 0 deletions tests/main-thread-winch-slowcreate.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Interposes pthread_create for main-thread-winch.m: while delayCreate is set, the caller returns 100 ms after the
// new thread started, so a short command has ended before the caller goes on.
#include <pthread.h>
#include <stdatomic.h>
#include <unistd.h>

atomic_int delayCreate;

static int slowCreate(pthread_t *t, const pthread_attr_t *attr, void *(*fn)(void *), void *arg) {
int result = pthread_create(t, attr, fn, arg);
if (atomic_load(&delayCreate)) usleep(100 * 1000);
return result;
}

__attribute__((used)) static struct { const void *replacement, *original; } interposers[]
__attribute__((section("__DATA,__interpose"))) = { { (const void *)slowCreate, (const void *)pthread_create } };
79 changes: 79 additions & 0 deletions tests/main-thread-winch.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// ios_setWindowSize sends SIGWINCH to the main thread of a session (mainThreadId), which is the only way it reaches a
// command that is not the root command of its session, such as dash. A command that has already ended must never
// remain recorded as the main thread: the next command would not be recorded and would never see the signal.
// The test starts a quick command, then a long command named dash with a SIGWINCH handler, resizes the window and
// expects the handler to run on that command's thread. When the quick command's thread stays recorded, the long command is never recorded.
// main-thread-winch-slowcreate.c is a dylib that interposes pthread_create so that the caller returns 100 ms after the thread started:
// the quick command has ended by then, as happens now and then under load.
#import <Foundation/Foundation.h>
#import "ios_system.h"
#import "ios_error.h"
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <unistd.h>

static atomic_int winch, started;
static pthread_t dashThread, winchThread;
static char sessionTag;

static void onWinch(int sig) {
winchThread = pthread_self();
atomic_store(&winch, 1);
}

int dashlong_main(int argc, char **argv) {
dashThread = pthread_self();
signal(SIGWINCH, onWinch);
atomic_store(&started, 1);
for (int i = 0; i < 300 && !atomic_load(&winch); i++) usleep(10000); // up to 3 s
return 0;
}

int quick_main(int argc, char **argv) {
return 0;
}

extern atomic_int delayCreate; // in main-thread-winch-slowcreate.c

static void *park(void *arg) {
while (1) sleep(1);
return NULL;
}

static void timedOut(int sig) {
fprintf(stderr, "main-thread-winch: FAIL, no end after 120 s\n");
_Exit(124);
}

int main(int argc, char **argv) {
@autoreleasepool {
signal(SIGALRM, timedOut);
alarm(120);
initializeEnvironment();
ios_switchSession(&sessionTag);
ios_setStreams(stdin, stdout, stderr);
replaceCommand(@"dash", @"dashlong_main", false);
replaceCommand(@"quick", @"quick_main", false);
joinMainThread = false;
atomic_store(&delayCreate, 1);
ios_system("quick");
atomic_store(&delayCreate, 0);
usleep(500 * 1000);
// Threads that stay alive, so that the next command thread does not reuse the one that ended
pthread_t parked;
for (int i = 0; i < 8; i++) pthread_create(&parked, NULL, park, NULL);
ios_system("dash");
while (!atomic_load(&started)) usleep(1000);
ios_setWindowSize(80, 24, &sessionTag);
for (int i = 0; i < 100 && !atomic_load(&winch); i++) usleep(10000); // up to 1 s
// The signal must reach the long command's own thread: a stale id names a thread started after the command ended, and that thread gets the signal.
if (!atomic_load(&winch) || winchThread != dashThread) {
fprintf(stderr, "main-thread-winch: FAIL, SIGWINCH %s\n", atomic_load(&winch) ? "was handled on another thread" : "never arrived");
return 1;
}
fprintf(stderr, "main-thread-winch: PASS\n");
alarm(0);
return 0;
}
}
12 changes: 12 additions & 0 deletions tests/tsan.supp
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# ThreadSanitizer suppressions for the ios_system tests (TSAN_OPTIONS=suppressions=tests/tsan.supp).
# ios_nextAvailablePid locks pid_mtx and returns with it locked; ios_storeThreadId unlocks it on the
# command thread. A mutex released by another thread is invisible to TSan, so every access inside the
# fork window is reported as a race between two threads that both "hold" pid_mtx.
race:ios_nextAvailablePid
race:storeEnvironment
race:environmentVariables
race:resetEnvironment
race:ios_restoreParentPid
# ios_currentPid reads current_pid without pid_mtx, against plain writes in ios_nextAvailablePid.
# This hides those reads too; they are not changed here.
race:ios_currentPid
Loading