Trying to reproduce a bug in an emulator.

pthread-test.c

// clang pthread-test.c -O3 -o pthread-test -lpthread

#include <stdio.h>
#include <stdint.h>
#include <stdatomic.h>
#include <pthread.h>
#include <unistd.h>
#include <stdlib.h>

#define NUM_THREADS 64
#define true 1
#define false 0

pthread_cond_t cond_start = PTHREAD_COND_INITIALIZER;
pthread_cond_t cond_ready = PTHREAD_COND_INITIALIZER;
pthread_cond_t cond_running = PTHREAD_COND_INITIALIZER;

pthread_mutex_t mutex_start = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t mutex_ready = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t mutex_running = PTHREAD_MUTEX_INITIALIZER;

atomic_bool go = false;
atomic_int num_ready = 0;
atomic_bool finish = false;

atomic_int foo = 0;

void work(uint64_t i) {
    for (i *= 300; i > 0; i--) {
        foo++;
    }
    // usleep(i);
}

void* worker_thread(void* arg) {
    uint64_t thread_id = (int64_t)arg;

    //printf("I'm thread %lu\n", thread_id);

    while (1) {
        // notify parent that we are ready
        pthread_mutex_lock(&mutex_ready);
        num_ready++;
        pthread_cond_signal(&cond_ready);
        pthread_mutex_unlock(&mutex_ready);

        // wait for job
        pthread_mutex_lock(&mutex_start);
        while (go == false) {
            pthread_cond_wait(&cond_start, &mutex_start);
        }
        pthread_mutex_unlock(&mutex_start);

        pthread_mutex_lock(&mutex_running);
        num_ready--;
        pthread_cond_signal(&cond_running);
        pthread_mutex_unlock(&mutex_running);

        if (finish) {
            printf("exiting %lu\n", thread_id);
            pthread_exit(0);
        }

        // do job
        //printf("doing work %lu\n", thread_id);
        work(100000 + (random() % 200000));
        //printf("done %lu\n", thread_id);
    }
}

static pthread_t threads[NUM_THREADS];

int main() {

    srandom(2);

    const int loops = 100;

    for (int i=0; i < NUM_THREADS; i++) {
        pthread_create(&threads[i], 0, worker_thread, (void*)((uint64_t)i));
    }

    for (int i = 1; i<=loops; i++) {
        if (i == loops) finish = true; // If this is the last loop iteration, finish
        pthread_mutex_lock(&mutex_ready);
        while (num_ready < NUM_THREADS) {
            pthread_cond_wait(&cond_ready, &mutex_ready);
        }
        pthread_mutex_unlock(&mutex_ready);
        //printf("All threads ready\n");

        usleep(300000); // Sleep between iterations (like geekbench does)

        // Kickoff all threads
        printf("go! %i\n", i);
        pthread_mutex_lock(&mutex_start);
        go = true;
        pthread_cond_broadcast(&cond_start);
        pthread_mutex_unlock(&mutex_start);

        pthread_mutex_lock(&mutex_running);
        while (num_ready > 0) {
            // <--- num_ready = 0
            // <--- pthread_cond_signal
            pthread_cond_wait(&cond_running, &mutex_running);
        }
        go = false;
        pthread_mutex_unlock(&mutex_running);
        //printf("All threads running\n");
    }

    for (int i=0; i < NUM_THREADS; i++) {
        pthread_join(threads[i], NULL);
    }

    printf("All exited\n");
    return 0;
}