35#include <sys/sysinfo.h>
77 pthread_t threads[64];
79 size_t count = 0, per = 0, at = 0;
80 long want = 0, got = 0;
83 if (!fn || end <= begin)
return 0;
84 if (grain == 0) grain = 1;
87 want = (long)(count / grain);
88 if (want > 64) want = 64;
104 fn(begin, end, user);
108 per = count / (size_t)n;
112 memset(threads, 0,
sizeof(threads));
113 memset(slices, 0,
sizeof(slices));
114 for (i = 0; i < n; i++) {
115 slices[i].
begin = begin + at;
116 at += per + ((size_t)i < count % (size_t)n ? 1 : 0);
117 slices[i].
end = begin + at;
119 slices[i].
user = user;
124 for (i = 1; i < n; i++)
125 started[i] = pthread_create(&threads[i], NULL,
n_parallel_slice, &slices[i]) == 0;
126 fn(slices[0].begin, slices[0].end, user);
127 for (i = 1; i < n; i++) {
129 pthread_join(threads[i], NULL);
131 fn(slices[i].begin, slices[i].end, user);
140 long int nb_procs = 0;
143 GetSystemInfo(&sysinfo);
144 nb_procs = (
long int)sysinfo.dwNumberOfProcessors;
146 nb_procs = sysconf(_SC_NPROCESSORS_ONLN);
160 n_log(
LOG_ERR,
"Thread fatal error, no valid payload found, exiting thread function !");
165 n_log(
LOG_DEBUG,
"Thread %llu started", (
unsigned long long)(uintptr_t)node->
thr);
167 int thread_state = 0;
174 pthread_mutex_lock(&node->
lock);
176 pthread_mutex_unlock(&node->
lock);
180 void* (*func_to_run)(
void* param) = NULL;
181 void* param_to_run = NULL;
182 pthread_mutex_lock(&node->
lock);
184 func_to_run = node->
func;
185 param_to_run = node->
param;
186 pthread_mutex_unlock(&node->
lock);
189 func_to_run(param_to_run);
193 pthread_mutex_lock(&node->
lock);
202 pthread_mutex_unlock(&node->
lock);
208 n_log(
LOG_DEBUG,
"Thread %llu exiting...", (
unsigned long long)(uintptr_t)node->
thr);
210 pthread_mutex_lock(&node->
lock);
212 pthread_mutex_unlock(&node->
lock);
214 n_log(
LOG_DEBUG,
"Thread %llu exited", (
unsigned long long)(uintptr_t)node->
thr);
255 if (sem_init(&
thread_pool->nb_tasks, 0, 1) == -1) {
257 n_log(
LOG_ERR,
"sem_init failed : %s on &thread_pool -> nb_tasks", strerror(error));
266 for (it = 0; it < nbmaxthr; it++) {
273 if (sem_init(&
thread_pool->thread_list[it]->th_start, 0, 0) == -1) {
275 n_log(
LOG_ERR,
"sem_init failed : %s on &thread_pool -> thread_list[ %zu ] -> th_start", strerror(error), it);
279 if (sem_init(&
thread_pool->thread_list[it]->th_end, 0, 0) == -1) {
281 n_log(
LOG_ERR,
"sem_init failed : %s on &thread_pool -> thread_list[ %zu] -> th_end", strerror(error), it);
282 sem_destroy(&
thread_pool->thread_list[it]->th_start);
290 pthread_mutex_init(&
thread_pool->thread_list[it]->lock, NULL);
293 n_log(
LOG_ERR,
"pthread_create failed : %s for it %zu", strerror(errno), it);
294 pthread_mutex_destroy(&
thread_pool->thread_list[it]->lock);
295 sem_destroy(&
thread_pool->thread_list[it]->th_start);
296 sem_destroy(&
thread_pool->thread_list[it]->th_end);
304 for (
size_t j = 0; j < it; j++) {
305 pthread_mutex_lock(&
thread_pool->thread_list[j]->lock);
308 pthread_mutex_unlock(&
thread_pool->thread_list[j]->lock);
309 pthread_join(
thread_pool->thread_list[j]->thr, NULL);
310 pthread_mutex_destroy(&
thread_pool->thread_list[j]->lock);
311 sem_destroy(&
thread_pool->thread_list[j]->th_start);
333 n_log(
LOG_ERR,
"thread_pool is not allocated, can't add processes to it !");
338 n_log(
LOG_ERR,
"thread_pool thread_list is not allocated, can't add processes to it !");
345 n_log(
LOG_ERR,
"invalid mode %d: exactly one of NORMAL_PROC, SYNCED_PROC, DIRECT_PROC must be set",
mode);
352 while (it < thread_pool->max_threads) {
353 pthread_mutex_lock(&
thread_pool->thread_list[it]->lock);
357 pthread_mutex_unlock(&
thread_pool->thread_list[it]->lock);
361 if (it < thread_pool->max_threads) {
369 pthread_mutex_unlock(&
thread_pool->thread_list[it]->lock);
376 pthread_mutex_unlock(&
thread_pool->thread_list[it]->lock);
377 n_log(
LOG_DEBUG,
"proc %p(%p) added on thread %zu", func_ptr, param, it);
383 int cancel_and_return = FALSE;
385 n_log(
LOG_DEBUG,
"Thread pool active threads are all busy and mode is NO_QUEUE, cannot add %p(%p) to pool %p", func_ptr, param,
thread_pool);
386 cancel_and_return = TRUE;
388 n_log(
LOG_ERR,
"Thread pool active threads are all busy, cannot add SYNCED_PROC %p(%p) to pool %p", func_ptr, param,
thread_pool);
389 cancel_and_return = TRUE;
391 n_log(
LOG_ERR,
"Thread pool active threads are all busy, cannot add DIRECT_PROC %p(%p) to pool %p", func_ptr, param,
thread_pool);
392 cancel_and_return = TRUE;
395 if (cancel_and_return) {
406 n_log(
LOG_ERR,
"Failed to allocate THREAD_WAITING_PROC");
410 proc->
func = func_ptr;
415 n_log(
LOG_ERR,
"proc %p(%p) was dropped from waitlist because waitlist of thread pool %p is full", func_ptr, param,
thread_pool);
422 if (sem_trywait(&
thread_pool->nb_tasks) != 0 && errno != EAGAIN) {
447 for (
size_t it = 0; it <
thread_pool->max_threads; it++) {
449 pthread_mutex_lock(&
thread_pool->thread_list[it]->lock);
453 pthread_mutex_unlock(&
thread_pool->thread_list[it]->lock);
455 if (sem_post(&
thread_pool->thread_list[it]->th_start) != 0) {
457 n_log(
LOG_ERR,
"sem_post th_start error in thread_pool %p , thread_list[ %zu ] : %s",
thread_pool, it, strerror(error));
477 for (
size_t it = 0; it <
thread_pool->max_threads; it++) {
479 pthread_mutex_lock(&
thread_pool->thread_list[it]->lock);
483 pthread_mutex_unlock(&
thread_pool->thread_list[it]->lock);
486 if (sem_wait(&
thread_pool->thread_list[it]->th_end) == -1) {
488 n_log(
LOG_ERR,
"sem_wait th_end error in thread_pool %p , thread_list[ %zu ] : %s",
thread_pool, it, strerror(error));
524 if (errno != EAGAIN && errno != EINTR) {
575 __n_assert((*pool)->thread_list,
return FALSE);
578 int max_retries = 1000;
582 pthread_mutex_lock(&(*pool)->lock);
583 for (
size_t it = 0; it < (*pool)->max_threads; it++) {
584 pthread_mutex_lock(&(*pool)->thread_list[it]->lock);
585 int state = (*pool)->thread_list[it]->state;
586 int thread_state = (*pool)->thread_list[it]->thread_state;
587 pthread_mutex_unlock(&(*pool)->thread_list[it]->lock);
594 pthread_mutex_lock(&(*pool)->thread_list[it]->lock);
596 sem_post(&(*pool)->thread_list[it]->th_start);
597 pthread_mutex_unlock(&(*pool)->thread_list[it]->lock);
602 pthread_mutex_unlock(&(*pool)->lock);
606 if (max_retries <= 0) {
607 pthread_mutex_lock(&(*pool)->lock);
608 for (
size_t it = 0; it < (*pool)->max_threads; it++) {
609 pthread_mutex_lock(&(*pool)->thread_list[it]->lock);
612 sem_post(&(*pool)->thread_list[it]->th_start);
614 pthread_mutex_unlock(&(*pool)->thread_list[it]->lock);
616 pthread_mutex_unlock(&(*pool)->lock);
626 for (
size_t it = 0; it < (*pool)->max_threads; it++) {
627 pthread_join((*pool)->thread_list[it]->thr, NULL);
631 for (
size_t it = 0; it < (*pool)->max_threads; it++) {
632 pthread_mutex_destroy(&(*pool)->thread_list[it]->lock);
633 sem_destroy(&(*pool)->thread_list[it]->th_start);
634 sem_destroy(&(*pool)->thread_list[it]->th_end);
635 Free((*pool)->thread_list[it]);
637 Free((*pool)->thread_list);
643 if ((*pool)->waiting_list->nb_items > 0) {
644 n_log(
LOG_ERR,
"thread pool %p destroyed with %zu queued job(s) never run%s", (
void*)(*
pool),
645 (*pool)->waiting_list->nb_items,
646 (*pool)->discard ?
", handed to its discard function" :
", their params not released");
647 if ((*pool)->discard) {
650 if (proc) (*pool)->discard(proc->
param);
656 sem_destroy(&(*pool)->nb_tasks);
658 pthread_mutex_destroy(&(*pool)->lock);
679 while (push_status == 1) {
681 if (node && node->
ptr) {
692 n_log(
LOG_DEBUG,
"waitlist: cannot add proc %p from waiting list of %p, all active threads are busy !", proc,
thread_pool);
706 for (
size_t it = 0; it <
thread_pool->max_threads; it++) {
707 pthread_mutex_lock(&
thread_pool->thread_list[it]->lock);
710 pthread_mutex_unlock(&
thread_pool->thread_list[it]->lock);
724 if (sem_trywait(&
thread_pool->nb_tasks) != 0 && errno != EAGAIN) {
THREAD_POOL * thread_pool
static NETWORK_POOL * pool
static void type(N_GUI_CTX *gui, const char *s)
Type a whole string into whatever has focus.
#define Malloc(__ptr, __struct, __size)
Malloc Handler to get errors and set to 0.
#define __n_assert(__ptr, __ret)
macro to assert things
#define Free(__ptr)
Free Handler to get errors.
void * ptr
void pointer to store
struct LIST_NODE * next
pointer to the next node
int list_push(LIST *list, void *ptr, void(*destructor)(void *ptr))
Add a pointer to the end of the list.
#define list_foreach(__ITEM_, __LIST_)
ForEach macro helper, safe for node removal during iteration.
#define remove_list_node(__LIST_, __NODE_, __TYPE_)
Remove macro helper for void pointer casting.
int list_destroy(LIST **list)
Empty and Free a list container.
LIST * new_generic_list(size_t max_items)
Initialiaze a generic list container to max_items pointers.
#define MAX_LIST_ITEMS
flag to pass to new_generic_list for the maximum possible number of item in a list
Structure of a generic list node.
#define n_log(__LEVEL__,...)
Logging function wrapper to get line and func.
#define LOG_DEBUG
debug-level messages
#define LOG_ERR
error conditions
void u_sleep(unsigned int usec)
wrapper around usleep for API consistency
int state
state of the proc , RUNNING_PROC when it is busy processing func( param) , IDLE_PROC when it waits fo...
int type
SYNCED or DIRECT process start.
void * param
if not NULL , passed as argument
void *(* func)(void *param)
function to call in the thread
sem_t th_end
thread ending semaphore
pthread_mutex_t lock
mutex to prevent mutual access of node parameters
struct THREAD_POOL * thread_pool
pointer to assigned thread pool
sem_t th_start
thread starting semaphore
void *(* func)(void *param)
function to call in the thread
int thread_state
state of the managing thread , RUNNING_THREAD, EXITING_THREAD, EXITED_THREAD
void * param
if not NULL , passed as argument
#define NORMAL_PROC
processing mode for added func, synced start, can be queued
int start_threaded_pool(THREAD_POOL *thread_pool)
Launch the process waiting for execution in the thread pool.
THREAD_POOL * new_thread_pool(size_t nbmaxthr, size_t nb_max_waiting)
Create a new pool of nbmaxthr threads.
#define EXITED_THREAD
indicate that the pool is off, all jobs have been consumed
#define SYNCED_PROC
processing mode for added func, synced start, not queued
int n_parallel_for(size_t begin, size_t end, size_t grain, N_PARALLEL_FN fn, void *user)
Run a loop over a range on several cores, and return when all of it is done.
int add_threaded_process(THREAD_POOL *thread_pool, void *(*func_ptr)(void *param), void *param, int mode)
add a function and params to a thread pool
#define NO_QUEUE
special processing mode for waiting_list: do not add the work in queue since it' coming from the queu...
int refresh_thread_pool(THREAD_POOL *thread_pool)
try to add some waiting DIRECT_PROCs on some free thread slots, else do nothing
#define IDLE_PROC
status of a thread which is waiting for some proc
int wait_for_threaded_pool(THREAD_POOL *thread_pool)
Wait for the thread pool to become idle (no active threads, empty waiting list), blocking without pol...
void(* N_PARALLEL_FN)(size_t begin, size_t end, void *user)
work done by n_parallel_for() on one slice [begin, end) of a range
int destroy_threaded_pool(THREAD_POOL **pool, unsigned int delay)
Stop every thread of a pool and free it.
int wait_for_synced_threaded_pool(THREAD_POOL *thread_pool)
wait for all the launched process, blocking but light on the CPU as there is no polling
long int get_nb_cpu_cores()
get number of core of current system
#define DIRECT_PROC
processing mode for added func, direct start, not queued
#define RUNNING_PROC
status of a thread which proc is currently running
int set_thread_pool_discard(THREAD_POOL *thread_pool, void(*discard)(void *param))
Set what destroy_threaded_pool() does with the param of a queued job it discards.
#define NO_LOCK
if passed to add_threaded_process, skip main table lock in case we are in a func which is already loc...
#define EXITING_THREAD
indicate that the pool is exiting, unfinished jobs will finish and the pool will exit the threads and...
#define RUNNING_THREAD
indicate that the pool is running and ready to use
#define WAITING_PROC
status of a thread who have proc waiting to be processed
#define N_THREAD_POOL_WAIT_POLL_US
A thread pool node.
Structure of a thread pool.
Structure of a waiting process item.
Common headers and low-level functions & define.
void * user
passed to the work
static pthread_mutex_t parallel_lock
Guards parallel_budget.
size_t end
one past its last
size_t begin
first index of the slice
static long parallel_budget
get number of core of current system
static void * n_parallel_slice(void *arg)
Thread entry of one slice of a parallel loop.
void * thread_pool_processing_function(void *param)
Internal thread pool processing function.
One slice of a parallel loop, as handed to its thread.