On macOS 14 and later, a multithreaded program that uses libgit2 on a repository with core.precomposeunicode = true can hang forever if it handles any signal. git and libgit2 both set that option on APFS (git_repository_init does it in repository.c). Every thread that opens or closes an iconv converter ends up blocked in __psynch_rw_wrlock, and no thread holds the lock. Nothing short of killing the process recovers it.

Reproduction steps

The program below runs 64 threads that call git_reference_lookup() in a loop on a fresh repository with core.precomposeunicode = true. One more thread sends SIGUSR1, whose handler does nothing, to a random worker every 200 µs.

cc -o repro repro.c -I<libgit2>/include -L<build> -lgit2 -Wl,-rpath,<build>
./repro /tmp

/*
 * 64 threads look up a reference in a repository with
 * core.precomposeunicode = true while another thread sends SIGUSR1 (a no-op
 * handler) to a random one of them every 200us.  On macOS 14+ the lookups
 * stop within seconds, for good.
 *
 *   cc -o repro repro-minimal.c -I<libgit2>/include -L<build> -lgit2 -Wl,-rpath,<build>
 *   ./repro /tmp        (exit 0: kept going for 20 s; exit 3: wedged)
 */
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <git2.h>

#define THREADS 64

static char path[1024];
static pthread_t workers[THREADS];
static atomic_ulong lookups;

static void check(int error, const char *what)
{
	if (error < 0) {
		fprintf(stderr, "%s: %s\n", what, git_error_last()->message);
		exit(2);
	}
}

static void on_sigusr1(int sig)
{
	(void)sig;
}

static void *worker(void *arg)
{
	git_repository *repo;
	git_reference *ref;

	(void)arg;
	check(git_repository_open(&repo, path), "open");

	for (;;) {
		check(git_reference_lookup(&ref, repo, "refs/heads/main"), "lookup");
		git_reference_free(ref);
		atomic_fetch_add(&lookups, 1);
	}

	return NULL;
}

static void *signaller(void *arg)
{
	(void)arg;

	for (;;) {
		pthread_kill(workers[arc4random_uniform(THREADS)], SIGUSR1);
		usleep(200);
	}

	return NULL;
}

int main(int argc, char **argv)
{
	git_repository *repo;
	git_config *config;
	git_treebuilder *tb;
	git_signature *sig;
	git_tree *tree;
	git_oid id;
	struct sigaction sa;
	unsigned long seen = 0;
	int i, still = 0;
	pthread_t t;

	snprintf(path, sizeof(path), "%s/repro.XXXXXX", argc > 1 ? argv[1] : "/tmp");
	if (!mkdtemp(path))
		return 2;

	check(git_libgit2_init(), "init");
	check(git_repository_init(&repo, path, 0), "repository_init");
	check(git_repository_config(&config, repo), "config");
	check(git_config_set_bool(config, "core.precomposeunicode", 1), "config_set");
	check(git_treebuilder_new(&tb, repo, NULL), "treebuilder");
	check(git_treebuilder_write(&id, tb), "treebuilder_write");
	check(git_tree_lookup(&tree, repo, &id), "tree_lookup");
	check(git_signature_now(&sig, "repro", "repro@example.com"), "signature");
	check(git_commit_create_v(&id, repo, "refs/heads/main", sig, sig, NULL,
	                          "seed", tree, 0), "commit");

	memset(&sa, 0, sizeof(sa));
	sa.sa_handler = on_sigusr1;
	sa.sa_flags = SA_RESTART;
	sigaction(SIGUSR1, &sa, NULL);

	for (i = 0; i < THREADS; i++)
		pthread_create(&workers[i], NULL, worker, NULL);
	pthread_create(&t, NULL, signaller, NULL);

	for (i = 0; i < 200; i++) {
		usleep(100000);

		if (atomic_load(&lookups) != seen) {
			seen = atomic_load(&lookups);
			still = 0;
		} else if (++still == 50) {
			printf("wedged: no lookup for 5 s, after %lu\n", seen);
			fflush(stdout);
			_exit(3);
		}
	}

	printf("ok: %lu lookups in 20 s\n", seen);
	fflush(stdout);
	_exit(0);
}

The signal storm only makes the hang quick to reach. Any signal handler can trigger it. We first hit it in a Rust test suite (git2-rs) that hung for 16 minutes at 0% CPU, with 26 threads in __psynch_rw_wrlock under _citrus_iconv_open. The signal there was SIGCHLD from exiting git child processes, caught by the handler tokio installs for its process reaper (SA_RESTART | SA_SIGINFO).

Expected behavior

The lookups keep going.

Actual behavior

The lookups stop within 0.1 s and never resume. On main (0551dfd), 3 of 3 runs of the program above wedged after 265–3,132 lookups; on v1.9.7, 3 of 3 wedged after 882–1,713. A longer variant of the harness stops the signals once a run wedges: in 5 of 5 runs on main, no lookup completed in the 10 s that followed. sample shows all 64 workers parked: 36–46 in iconv_open, 18–28 in iconv_close, and no thread inside libiconv for any other reason.

git_reference_lookup                 git_reference_lookup
git_reference_lookup_resolved        git_reference_lookup_resolved
reference_normalize_for_repo         reference_normalize_for_repo
git_reference_normalize_name         git_reference_normalize_name
git_reference__normalize_name        git_reference__normalize_name
git_fs_path_iconv_init_precompose    git_fs_path_iconv_clear
__bsd___iconv_open                   __bsd_iconv_close
_citrus_iconv_open                   _citrus_iconv_close
_pthread_rwlock_lock_slow            release_shared
_pthread_rwlock_lock_wait            _pthread_rwlock_lock_slow
__psynch_rw_wrlock                   _pthread_rwlock_lock_wait
                                     __psynch_rw_wrlock

Root cause

Since macOS 14, the system iconv has been FreeBSD's Citrus implementation. macOS 13.5 ships Apple's libiconv-64, which is GNU-based; 14.0 ships libiconv-80.1.1; 26.5 ships libiconv-115.120.2. Citrus keeps one process-wide pthread_rwlock_t ci_lock. Every iconv_open write-locks it twice, in init_cache and get_shared. Every iconv_close write-locks it once, in release_shared.

macOS's rwlock appears to lose the wakeup when a thread that is waiting for it takes a signal. iconv is not the cause: the same storm against a bare pthread_rwlock_t, with 64 threads doing nothing but wrlock/unlock, wedged permanently in 3 of 3 runs. The identical loop on a pthread_mutex_t kept going in 3 of 3 runs, at 1.5–1.9 billion lock/unlock pairs in 20 s. That reproducer is attached at the end.

libgit2 runs into this easily because it opens and closes a converter for every use: git_fs_path_iconv_init_precompose and git_fs_path_iconv_clear. With precomposition on, it does this for every directory it iterates (git_fs_path_diriter_init and git_fs_path_direach), and for every reference name it normalizes (git_reference__normalize_name). Threads doing reference lookups, status, diffs or refdb iteration therefore contend on ci_lock constantly, and one badly timed signal wedges the process.

Proposed fix

#7404 serializes iconv_open and iconv_close behind one git_mutex, set up with the rest of the runtime as git_rand_global_init sets up its lock. Only one thread is then ever inside those calls, so ci_lock never has a waiter and there is no wakeup to lose. iconv() itself never takes ci_lock and is not serialized.

• It fixes the hang. Under the storm, patched main ran 3 of 3 runs of the program above (472k–484k lookups each) and 5 of 5 runs of the longer harness, all for the full 20 s. A 300 s soak completed 6.3M lookups while receiving about 985k signals. The same change, backported to v1.9.7 (GIT_USE_ICONV there), ran 3 of 3.
• Throughput. Single-threaded throughput is unchanged: about 75k lookups/s with or without the patch. With 64 threads on one repository, the patched build does about 22k lookups/s against about 75k unpatched. That is not the mutex's cost. Unpatched threads spend about 95% of their samples parked on ci_lock, which throttles the file I/O they do the rest of the time. Unpatched libgit2 with core.precomposeunicode = false, which never calls iconv, also does about 22k/s. Any fix that removes the iconv contention lands on that number; the patch adds no measurable cost.
• Why not reuse converters? A converter can't be shared between threads unless iconv() is serialized as well. Caching one per thread still opens one at every thread start and closes it at every thread exit, so it needs the same lock to be safe and adds lifetime management besides. Pinning a converter for the life of the process doesn't help either, because every open still takes ci_lock.
• No new test. The failure needs a signal to land on a thread that is waiting in the OS's rwlock. It is timing-dependent and macOS-only, and a regression would hang the suite rather than fail it. The existing iconv, path, refs, iterator and threads tests exercise the new lock, and tests/util/init.c exercises its init and shutdown cycles. The full offline suite passes on patched main. The util suite and the status::renames, refs, repo::init, iterator and threads suites give identical results (pass, skip and fail) with and without the patch, on main and on v1.9.7.

Other notes

• With the legacy -DUSE_ICONV=ON, deps/ntlmclient's iconv backend also opens and closes converters, once per NTLM client. The patch doesn't cover it; in a default build, fs_path.c is the only iconv caller.
• Other iconv users in the same process are beyond libgit2's control. The underlying bug is in macOS.
• Until a release ships the fix, there are two workarounds. Setting core.precomposeunicode = false avoids iconv entirely, at the cost of precomposition. When libgit2 is linked statically, as libgit2-sys does, the application can define its own iconv_open and iconv_close that forward to libiconv's under a lock.
• A downstream report that looks like the same hang: App hangs: libiconv lock convoy from unbounded concurrent git2 diffs across all repos Rymera-Web-Co/Gamut#89. It describes a 387 s stall at near-zero CPU, with nine threads in __psynch_rw_wrlock under _citrus_iconv_open or release_shared.

/*
 * macOS pthread_rwlock_t loses wakeups when waiters are interrupted by
 * signals: no iconv, no libgit2.  64 threads take and release one write
 * lock in a loop while another thread sends SIGUSR1 (no-op handler,
 * SA_RESTART) to a random one every 200us.  `mutex` runs the identical
 * loop on a pthread_mutex_t instead.
 *
 *   cc -O2 -o rwlock rwlock.c && ./rwlock rwlock && ./rwlock mutex
 *
 * Exit status: 0 = kept going for 20 s; 3 = no progress for 5 s and none
 * in the 10 s after the signals stopped (permanent).
 */
#include <errno.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#define THREADS 64

static pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER;
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_t workers[THREADS];
static atomic_ulong ops;
static atomic_int stop;
static int use_mutex;

static void on_sigusr1(int sig) { (void)sig; }

static void *worker(void *arg)
{
	(void)arg;
	for (;;) {
		if (use_mutex) {
			pthread_mutex_lock(&mutex);
			pthread_mutex_unlock(&mutex);
		} else {
			pthread_rwlock_wrlock(&rwlock);
			pthread_rwlock_unlock(&rwlock);
		}
		atomic_fetch_add_explicit(&ops, 1, memory_order_relaxed);
	}
	return NULL;
}

static void *signaller(void *arg)
{
	unsigned long x = 88172645463325252UL;

	(void)arg;
	while (!atomic_load(&stop)) {
		x ^= x << 13; x ^= x >> 7; x ^= x << 17;
		pthread_kill(workers[x % THREADS], SIGUSR1);
		usleep(200);
	}
	return NULL;
}

int main(int argc, char **argv)
{
	struct sigaction sa;
	struct timespec left = { 10, 0 };
	unsigned long seen = 0, n;
	int tick, still = 0, i;
	pthread_t t;

	use_mutex = argc > 1 && !strcmp(argv[1], "mutex");

	memset(&sa, 0, sizeof(sa));
	sa.sa_handler = on_sigusr1;
	sa.sa_flags = SA_RESTART;
	sigaction(SIGUSR1, &sa, NULL);

	for (i = 0; i < THREADS; i++)
		pthread_create(&workers[i], NULL, worker, NULL);
	pthread_create(&t, NULL, signaller, NULL);

	for (tick = 1; tick <= 200; tick++) {
		usleep(100000);
		n = atomic_load(&ops);
		if (n != seen) {
			seen = n;
			still = 0;
		} else if (++still == 50) {
			atomic_store(&stop, 1);
			while (nanosleep(&left, &left) < 0 && errno == EINTR)
				;
			n = atomic_load(&ops);
			printf("%s: WEDGED after %lu ops; %s (%lu ops in the 10 s after the signals stopped)\n",
			       use_mutex ? "pthread_mutex_t" : "pthread_rwlock_t", seen,
			       n == seen ? "PERMANENT" : "RECOVERED", n - seen);
			return 3;
		}
	}

	printf("%s: OK, %lu ops in 20 s\n", use_mutex ? "pthread_mutex_t" : "pthread_rwlock_t", seen);
	return 0;
}

Version of libgit2 (release number or SHA1)

main at 0551dfd, and v1.9.7. Also v1.9.4 as vendored by libgit2-sys 0.18.5, where we first hit it.

Operating system(s) tested

macOS 26.5.1 (25F80), with libiconv-115.120.2, on an Apple M3 Ultra. The Citrus iconv with this lock has shipped since macOS 14.0 (libiconv-80.1.1).