// RUN: %libomp-compile && env LIBOMP_NUM_HIDDEN_HELPER_THREADS=0 OMP_PROC_BIND=close OMP_PLACES=cores KMP_AFFINITY=verbose %libomp-run 8 1 4
// REQUIRES: linux
//
// This test pthread_creates 8 root threads before any OpenMP
// runtime entry is ever called. We have all the root threads
// register with the runtime by calling omp_set_num_threads(),
// but this does not initialize their affinity. The fourth root thread
// then calls a parallel region and we make sure its affinity
// is correct. We also make sure all the other root threads are
// free-floating since they have not called into a parallel region.
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <omp.h>
#include <pthread.h>
#include <unistd.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/syscall.h>
#include "libomp_test_affinity.h"
volatile int entry_flag = 0;
volatile int flag = 0;
volatile int num_roots_arrived = 0;
int num_roots;
int spawner = 0;
pthread_mutex_t lock;
int register_workers = 0; // boolean
affinity_mask_t *full_mask;
int __kmpc_global_thread_num(void*);
int get_os_thread_id() {
return (int)syscall(SYS_gettid);
}
int place_and_affinity_match() {
int i, max_cpu;
char buf[512];
affinity_mask_t *mask = affinity_mask_alloc();
int place = omp_get_place_num();
int num_procs = omp_get_place_num_procs(place);
int *ids = (int*)malloc(sizeof(int) * num_procs);
omp_get_place_proc_ids(place, ids);
get_thread_affinity(mask);
affinity_mask_snprintf(buf, sizeof(buf), mask);
printf("Primary Thread Place: %d\n", place);
printf("Primary Thread mask: %s\n", buf);
for (i = 0; i < num_procs; ++i) {
int cpu = ids[i];
if (!affinity_mask_isset(mask, cpu))
return 0;
}
max_cpu = AFFINITY_MAX_CPUS;
for (i = 0; i < max_cpu; ++i) {
int cpu = i;
if (affinity_mask_isset(mask, cpu)) {
int j, found = 0;
for (j = 0; j < num_procs; ++j) {
if (ids[j] == cpu) {
found = 1;
break;
}
}
if (!found)
return 0;
}
}
affinity_mask_free(mask);
free(ids);
return 1;
}
void* thread_func(void *arg) {
int place, nplaces;
int root_id = *((int*)arg);
int pid = getpid();
int tid = get_os_thread_id();
// Order how the root threads are assigned a gtid in the runtime
// i.e., root_id = gtid
while (1) {
int v = entry_flag;
if (v == root_id)
break;
}
// If main root thread
if (root_id == spawner) {
printf("Initial application thread (pid=%d, tid=%d, spawner=%d) reached thread_func (will call OpenMP)\n", pid, tid, spawner);
omp_set_num_threads(4);
#pragma omp atomic
entry_flag++;
// Wait for the workers to signal their arrival before #pragma omp parallel
while (num_roots_arrived < num_roots - 1) {}
// This will trigger the output for KMP_AFFINITY in this case
#pragma omp parallel
{
int gtid = __kmpc_global_thread_num(NULL);
#pragma omp single
{
printf("Exactly %d threads in the #pragma omp parallel\n",
omp_get_num_threads());
}
#pragma omp critical
{
printf("OpenMP thread %d: gtid=%d\n", omp_get_thread_num(), gtid);
}
}
flag = 1;
if (!place_and_affinity_match()) {
fprintf(stderr, "error: place and affinity mask do not match for primary thread\n");
exit (EXIT_FAILURE);
}
} else { // If worker root thread
// Worker root threads, register with OpenMP through omp_set_num_threads()
// if designated to, signal their arrival and then wait for the main root
// thread to signal them to exit.
printf("New root pthread (pid=%d, tid=%d) reached thread_func\n", pid, tid);
if (register_workers)
omp_set_num_threads(4);
#pragma omp atomic
entry_flag++;
pthread_mutex_lock(&lock);
num_roots_arrived++;
pthread_mutex_unlock(&lock);
while (flag == 0) {}
// Main check whether root threads' mask is equal to the
// initial affinity mask
affinity_mask_t *mask = affinity_mask_alloc();
get_thread_affinity(mask);
if (!affinity_mask_equal(mask, full_mask)) {
char buf[1024];
printf("root thread %d mask: ", root_id);
affinity_mask_snprintf(buf, sizeof(buf), mask);
printf("initial affinity mask: %s\n", buf);
fprintf(stderr, "error: root thread %d affinity mask not equal"
" to initial full mask\n", root_id);
affinity_mask_free(mask);
exit(EXIT_FAILURE);
}
affinity_mask_free(mask);
}
return NULL;
}
int main(int argc, char** argv) {
int i;
if (argc != 3 && argc != 4) {
fprintf(stderr, "usage: %s <num_roots> <register_workers_bool> [<spawn_root_number>]\n", argv[0]);
exit(EXIT_FAILURE);
}
// Initialize pthread mutex
pthread_mutex_init(&lock, NULL);
// Get initial full mask
full_mask = affinity_mask_alloc();
get_thread_affinity(full_mask);
// Get the number of root pthreads to create and allocate resources for them
num_roots = atoi(argv[1]);
pthread_t *roots = (pthread_t*)malloc(sizeof(pthread_t) * num_roots);
int *root_ids = (int*)malloc(sizeof(int) * num_roots);
// Get the flag indicating whether to have root pthreads call omp_set_num_threads() or not
register_workers = atoi(argv[2]);
if (argc == 4)
spawner = atoi(argv[3]);
// Spawn worker root threads
for (i = 1; i < num_roots; ++i) {
*(root_ids + i) = i;
pthread_create(roots + i, NULL, thread_func, root_ids + i);
}
// Have main root thread (root 0) go into thread_func
*root_ids = 0;
thread_func(root_ids);
// Cleanup all resources
for (i = 1; i < num_roots; ++i) {
void *status;
pthread_join(roots[i], &status);
}
free(roots);
free(root_ids);
pthread_mutex_destroy(&lock);
return EXIT_SUCCESS;
}
Codebase Browser
llvm