blender-archive/source/blender/blenlib/intern/task_pool.cc

/*
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/** \file
 * \ingroup bli
 *
 * A generic task system which can be used for any task based subsystem.
 */

#include <stdlib.h>

#include "MEM_guardedalloc.h"

#include "DNA_listBase.h"

#include "BLI_listbase.h"
#include "BLI_math.h"
#include "BLI_mempool.h"
#include "BLI_task.h"
#include "BLI_threads.h"

#include "atomic_ops.h"

/* Define this to enable some detailed statistic print. */
#undef DEBUG_STATS

/* Types */

/* Number of per-thread pre-allocated tasks.
 *
 * For more details see description of TaskMemPool.
 */
#define MEMPOOL_SIZE 256

/* Number of tasks which are pushed directly to local thread queue.
 *
 * This allows thread to fetch next task without locking the whole queue.
 */
#define LOCAL_QUEUE_SIZE 1

/* Number of tasks which are allowed to be scheduled in a delayed manner.
 *
 * This allows to use less locks per graph node children schedule. More details
 * could be found at TaskThreadLocalStorage::do_delayed_push.
 */
#define DELAYED_QUEUE_SIZE 4096

#ifndef NDEBUG
#  define ASSERT_THREAD_ID(scheduler, thread_id) \
    do { \
      if (!BLI_thread_is_main()) { \
        TaskThread *thread = (TaskThread *)pthread_getspecific(scheduler->tls_id_key); \
        if (thread == NULL) { \
          BLI_assert(thread_id == 0); \
        } \
        else { \
          BLI_assert(thread_id == thread->id); \
        } \
      } \
      else { \
        BLI_assert(thread_id == 0); \
      } \
    } while (false)
#else
#  define ASSERT_THREAD_ID(scheduler, thread_id)
#endif

typedef struct Task {
  struct Task *next, *prev;

  TaskRunFunction run;
  void *taskdata;
  bool free_taskdata;
  TaskFreeFunction freedata;
  TaskPool *pool;
} Task;

/* This is a per-thread storage of pre-allocated tasks.
 *
 * The idea behind this is simple: reduce amount of malloc() calls when pushing
 * new task to the pool. This is done by keeping memory from the tasks which
 * were finished already, so instead of freeing that memory we put it to the
 * pool for the later re-use.
 *
 * The tricky part here is to avoid any inter-thread synchronization, hence no
 * lock must exist around this pool. The pool will become an owner of the pointer
 * from freed task, and only corresponding thread will be able to use this pool
 * (no memory stealing and such).
 *
 * This leads to the following use of the pool:
 *
 * - task_push() should provide proper thread ID from which the task is being
 *   pushed from.
 *
 * - Task allocation function which check corresponding memory pool and if there
 *   is any memory in there it'll mark memory as re-used, remove it from the pool
 *   and use that memory for the new task.
 *
 *   At this moment task queue owns the memory.
 *
 * - When task is done and task_free() is called the memory will be put to the
 *   pool which corresponds to a thread which handled the task.
 */
typedef struct TaskMemPool {
  /* Number of pre-allocated tasks in the pool. */
  int num_tasks;
  /* Pre-allocated task memory pointers. */
  Task *tasks[MEMPOOL_SIZE];
} TaskMemPool;

#ifdef DEBUG_STATS
typedef struct TaskMemPoolStats {
  /* Number of allocations. */
  int num_alloc;
  /* Number of avoided allocations (pointer was re-used from the pool). */
  int num_reuse;
  /* Number of discarded memory due to pool saturation, */
  int num_discard;
} TaskMemPoolStats;
#endif

typedef struct TaskThreadLocalStorage {
  /* Memory pool for faster task allocation.
   * The idea is to re-use memory of finished/discarded tasks by this thread.
   */
  TaskMemPool task_mempool;

  /* Local queue keeps thread alive by keeping small amount of tasks ready
   * to be picked up without causing global thread locks for synchronization.
   */
  int num_local_queue;
  Task *local_queue[LOCAL_QUEUE_SIZE];

  /* Thread can be marked for delayed tasks push. This is helpful when it's
   * know that lots of subsequent task pushed will happen from the same thread
   * without "interrupting" for task execution.
   *
   * We try to accumulate as much tasks as possible in a local queue without
   * any locks first, and then we push all of them into a scheduler's queue
   * from within a single mutex lock.
   */
  bool do_delayed_push;
  int num_delayed_queue;
  Task *delayed_queue[DELAYED_QUEUE_SIZE];
} TaskThreadLocalStorage;

struct TaskPool {
  TaskScheduler *scheduler;

  volatile size_t num;
  ThreadMutex num_mutex;
  ThreadCondition num_cond;

  void *userdata;
  ThreadMutex user_mutex;

  volatile bool do_cancel;
  volatile bool do_work;

  volatile bool is_suspended;
  bool start_suspended;
  ListBase suspended_queue;
  size_t num_suspended;

  TaskPriority priority;

  /* If set, this pool may never be work_and_wait'ed, which means TaskScheduler
   * has to use its special background fallback thread in case we are in
   * single-threaded situation.
   */
  bool run_in_background;

  /* This is a task scheduler's ID of a thread at which pool was constructed.
   * It will be used to access task TLS.
   */
  int thread_id;

  /* For the pools which are created from non-main thread which is not a
   * scheduler worker thread we can't re-use any of scheduler's threads TLS
   * and have to use our own one.
   */
  bool use_local_tls;
  TaskThreadLocalStorage local_tls;
#ifndef NDEBUG
  pthread_t creator_thread_id;
#endif

#ifdef DEBUG_STATS
  TaskMemPoolStats *mempool_stats;
#endif
};

struct TaskScheduler {
  pthread_t *threads;
  struct TaskThread *task_threads;
  int num_threads;
  bool background_thread_only;

  ListBase queue;
  ThreadMutex queue_mutex;
  ThreadCondition queue_cond;

  ThreadMutex startup_mutex;
  ThreadCondition startup_cond;
  volatile int num_thread_started;

  volatile bool do_exit;

  /* NOTE: In pthread's TLS we store the whole TaskThread structure. */
  pthread_key_t tls_id_key;
};

typedef struct TaskThread {
  TaskScheduler *scheduler;
  int id;
  TaskThreadLocalStorage tls;
} TaskThread;

/* Helper */
BLI_INLINE void task_data_free(Task *task, const int UNUSED(thread_id))
{
  if (task->free_taskdata) {
    if (task->freedata) {
      task->freedata(task->pool, task->taskdata);
    }
    else {
      MEM_freeN(task->taskdata);
    }
  }
}

BLI_INLINE void initialize_task_tls(TaskThreadLocalStorage *tls)
{
  memset(tls, 0, sizeof(TaskThreadLocalStorage));
}

BLI_INLINE TaskThreadLocalStorage *get_task_tls(TaskPool *pool, const int thread_id)
{
  TaskScheduler *scheduler = pool->scheduler;
  BLI_assert(thread_id >= 0);
  BLI_assert(thread_id <= scheduler->num_threads);
  if (pool->use_local_tls && thread_id == 0) {
    BLI_assert(pool->thread_id == 0);
    BLI_assert(!BLI_thread_is_main());
    BLI_assert(pthread_equal(pthread_self(), pool->creator_thread_id));
    return &pool->local_tls;
  }
  if (thread_id == 0) {
    BLI_assert(BLI_thread_is_main());
    return &scheduler->task_threads[pool->thread_id].tls;
  }
  return &scheduler->task_threads[thread_id].tls;
}

BLI_INLINE void free_task_tls(TaskThreadLocalStorage *tls)
{
  TaskMemPool *task_mempool = &tls->task_mempool;
  for (int i = 0; i < task_mempool->num_tasks; i++) {
    MEM_freeN(task_mempool->tasks[i]);
  }
}

static Task *task_alloc(TaskPool *pool, const int thread_id)
{
  BLI_assert(thread_id <= pool->scheduler->num_threads);
  if (thread_id != -1) {
    BLI_assert(thread_id >= 0);
    BLI_assert(thread_id <= pool->scheduler->num_threads);
    TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
    TaskMemPool *task_mempool = &tls->task_mempool;
    /* Try to re-use task memory from a thread local storage. */
    if (task_mempool->num_tasks > 0) {
      --task_mempool->num_tasks;
      /* Success! We've just avoided task allocation. */
#ifdef DEBUG_STATS
      pool->mempool_stats[thread_id].num_reuse++;
#endif
      return task_mempool->tasks[task_mempool->num_tasks];
    }
    /* We are doomed to allocate new task data. */
#ifdef DEBUG_STATS
    pool->mempool_stats[thread_id].num_alloc++;
#endif
  }
  return (Task *)MEM_mallocN(sizeof(Task), "New task");
}

static void task_free(TaskPool *pool, Task *task, const int thread_id)
{
  task_data_free(task, thread_id);
  BLI_assert(thread_id >= 0);
  BLI_assert(thread_id <= pool->scheduler->num_threads);
  if (thread_id == 0) {
    BLI_assert(pool->use_local_tls || BLI_thread_is_main());
  }
  TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
  TaskMemPool *task_mempool = &tls->task_mempool;
  if (task_mempool->num_tasks < MEMPOOL_SIZE - 1) {
    /* Successfully allowed the task to be re-used later. */
    task_mempool->tasks[task_mempool->num_tasks] = task;
    ++task_mempool->num_tasks;
  }
  else {
    /* Local storage saturated, no other way than just discard
     * the memory.
     *
     * TODO(sergey): We can perhaps store such pointer in a global
     * scheduler pool, maybe it'll be faster than discarding and
     * allocating again.
     */
    MEM_freeN(task);
#ifdef DEBUG_STATS
    pool->mempool_stats[thread_id].num_discard++;
#endif
  }
}

/* Task Scheduler */

static void task_pool_num_decrease(TaskPool *pool, size_t done)
{
  BLI_mutex_lock(&pool->num_mutex);

  BLI_assert(pool->num >= done);

  pool->num -= done;

  if (pool->num == 0) {
    BLI_condition_notify_all(&pool->num_cond);
  }

  BLI_mutex_unlock(&pool->num_mutex);
}

static void task_pool_num_increase(TaskPool *pool, size_t new_num)
{
  BLI_mutex_lock(&pool->num_mutex);

  pool->num += new_num;
  BLI_condition_notify_all(&pool->num_cond);

  BLI_mutex_unlock(&pool->num_mutex);
}

static bool task_scheduler_thread_wait_pop(TaskScheduler *scheduler, Task **task)
{
  bool found_task = false;
  BLI_mutex_lock(&scheduler->queue_mutex);

  while (!scheduler->queue.first && !scheduler->do_exit) {
    BLI_condition_wait(&scheduler->queue_cond, &scheduler->queue_mutex);
  }

  do {
    Task *current_task;

    /* Assuming we can only have a void queue in 'exit' case here seems logical
     * (we should only be here after our worker thread has been woken up from a
     * condition_wait(), which only happens after a new task was added to the queue),
     * but it is wrong.
     * Waiting on condition may wake up the thread even if condition is not signaled
     * (spurious wake-ups), and some race condition may also empty the queue **after**
     * condition has been signaled, but **before** awoken thread reaches this point...
     * See http://stackoverflow.com/questions/8594591
     *
     * So we only abort here if do_exit is set.
     */
    if (scheduler->do_exit) {
      BLI_mutex_unlock(&scheduler->queue_mutex);
      return false;
    }

    for (current_task = (Task *)scheduler->queue.first; current_task != NULL;
         current_task = current_task->next) {
      TaskPool *pool = current_task->pool;

      if (scheduler->background_thread_only && !pool->run_in_background) {
        continue;
      }

      *task = current_task;
      found_task = true;
      BLI_remlink(&scheduler->queue, *task);
      break;
    }
    if (!found_task) {
      BLI_condition_wait(&scheduler->queue_cond, &scheduler->queue_mutex);
    }
  } while (!found_task);

  BLI_mutex_unlock(&scheduler->queue_mutex);

  return true;
}

BLI_INLINE void handle_local_queue(TaskThreadLocalStorage *tls, const int thread_id)
{
  BLI_assert(!tls->do_delayed_push);
  while (tls->num_local_queue > 0) {
    /* We pop task from queue before handling it so handler of the task can
     * push next job to the local queue.
     */
    tls->num_local_queue--;
    Task *local_task = tls->local_queue[tls->num_local_queue];
    /* TODO(sergey): Double-check work_and_wait() doesn't handle other's
     * pool tasks.
     */
    TaskPool *local_pool = local_task->pool;
    local_task->run(local_pool, local_task->taskdata, thread_id);
    task_free(local_pool, local_task, thread_id);
  }
  BLI_assert(!tls->do_delayed_push);
}

static void *task_scheduler_thread_run(void *thread_p)
{
  TaskThread *thread = (TaskThread *)thread_p;
  TaskThreadLocalStorage *tls = &thread->tls;
  TaskScheduler *scheduler = thread->scheduler;
  int thread_id = thread->id;
  Task *task;

  pthread_setspecific(scheduler->tls_id_key, thread);

  /* signal the main thread when all threads have started */
  BLI_mutex_lock(&scheduler->startup_mutex);
  scheduler->num_thread_started++;
  if (scheduler->num_thread_started == scheduler->num_threads) {
    BLI_condition_notify_one(&scheduler->startup_cond);
  }
  BLI_mutex_unlock(&scheduler->startup_mutex);

  /* keep popping off tasks */
  while (task_scheduler_thread_wait_pop(scheduler, &task)) {
    TaskPool *pool = task->pool;

    /* run task */
    BLI_assert(!tls->do_delayed_push);
    task->run(pool, task->taskdata, thread_id);
    BLI_assert(!tls->do_delayed_push);

    /* delete task */
    task_free(pool, task, thread_id);

    /* Handle all tasks from local queue. */
    handle_local_queue(tls, thread_id);

    /* notify pool task was done */
    task_pool_num_decrease(pool, 1);
  }

  return NULL;
}

TaskScheduler *BLI_task_scheduler_create(int num_threads)
{
  TaskScheduler *scheduler = (TaskScheduler *)MEM_callocN(sizeof(TaskScheduler), "TaskScheduler");

  /* multiple places can use this task scheduler, sharing the same
   * threads, so we keep track of the number of users. */
  scheduler->do_exit = false;

  BLI_listbase_clear(&scheduler->queue);
  BLI_mutex_init(&scheduler->queue_mutex);
  BLI_condition_init(&scheduler->queue_cond);

  BLI_mutex_init(&scheduler->startup_mutex);
  BLI_condition_init(&scheduler->startup_cond);
  scheduler->num_thread_started = 0;

  if (num_threads == 0) {
    /* automatic number of threads will be main thread + num cores */
    num_threads = BLI_system_thread_count();
  }

  /* main thread will also work, so we count it too */
  num_threads -= 1;

  /* Add background-only thread if needed. */
  if (num_threads == 0) {
    scheduler->background_thread_only = true;
    num_threads = 1;
  }

  scheduler->task_threads = (TaskThread *)MEM_mallocN(sizeof(TaskThread) * (num_threads + 1),
                                                      "TaskScheduler task threads");

  /* Initialize TLS for main thread. */
  initialize_task_tls(&scheduler->task_threads[0].tls);

  pthread_key_create(&scheduler->tls_id_key, NULL);

  /* launch threads that will be waiting for work */
  if (num_threads > 0) {
    int i;

    scheduler->num_threads = num_threads;
    scheduler->threads = (pthread_t *)MEM_callocN(sizeof(pthread_t) * num_threads,
                                                  "TaskScheduler threads");

    for (i = 0; i < num_threads; i++) {
      TaskThread *thread = &scheduler->task_threads[i + 1];
      thread->scheduler = scheduler;
      thread->id = i + 1;
      initialize_task_tls(&thread->tls);

      if (pthread_create(&scheduler->threads[i], NULL, task_scheduler_thread_run, thread) != 0) {
        fprintf(stderr, "TaskScheduler failed to launch thread %d/%d\n", i, num_threads);
      }
    }
  }

  /* Wait for all worker threads to start before returning to caller to prevent the case where
   * threads are still starting and pthread_join is called, which causes a deadlock on pthreads4w.
   */
  BLI_mutex_lock(&scheduler->startup_mutex);
  /* NOTE: Use loop here to avoid false-positive everything-is-ready caused by spontaneous thread
   * wake up. */
  while (scheduler->num_thread_started != num_threads) {
    BLI_condition_wait(&scheduler->startup_cond, &scheduler->startup_mutex);
  }
  BLI_mutex_unlock(&scheduler->startup_mutex);

  return scheduler;
}

void BLI_task_scheduler_free(TaskScheduler *scheduler)
{
  Task *task;

  /* stop all waiting threads */
  BLI_mutex_lock(&scheduler->queue_mutex);
  scheduler->do_exit = true;
  BLI_condition_notify_all(&scheduler->queue_cond);
  BLI_mutex_unlock(&scheduler->queue_mutex);

  pthread_key_delete(scheduler->tls_id_key);

  /* delete threads */
  if (scheduler->threads) {
    int i;

    for (i = 0; i < scheduler->num_threads; i++) {
      if (pthread_join(scheduler->threads[i], NULL) != 0) {
        fprintf(stderr, "TaskScheduler failed to join thread %d/%d\n", i, scheduler->num_threads);
      }
    }

    MEM_freeN(scheduler->threads);
  }

  /* Delete task thread data */
  if (scheduler->task_threads) {
    for (int i = 0; i < scheduler->num_threads + 1; i++) {
      TaskThreadLocalStorage *tls = &scheduler->task_threads[i].tls;
      free_task_tls(tls);
    }

    MEM_freeN(scheduler->task_threads);
  }

  /* delete leftover tasks */
  for (task = (Task *)scheduler->queue.first; task; task = task->next) {
    task_data_free(task, 0);
  }
  BLI_freelistN(&scheduler->queue);

  /* delete mutex/condition */
  BLI_mutex_end(&scheduler->queue_mutex);
  BLI_condition_end(&scheduler->queue_cond);
  BLI_mutex_end(&scheduler->startup_mutex);
  BLI_condition_end(&scheduler->startup_cond);

  MEM_freeN(scheduler);
}

int BLI_task_scheduler_num_threads(TaskScheduler *scheduler)
{
  return scheduler->num_threads + 1;
}

static void task_scheduler_push(TaskScheduler *scheduler, Task *task, TaskPriority priority)
{
  task_pool_num_increase(task->pool, 1);

  /* add task to queue */
  BLI_mutex_lock(&scheduler->queue_mutex);

  if (priority == TASK_PRIORITY_HIGH) {
    BLI_addhead(&scheduler->queue, task);
  }
  else {
    BLI_addtail(&scheduler->queue, task);
  }

  BLI_condition_notify_one(&scheduler->queue_cond);
  BLI_mutex_unlock(&scheduler->queue_mutex);
}

static void task_scheduler_push_all(TaskScheduler *scheduler,
                                    TaskPool *pool,
                                    Task **tasks,
                                    int num_tasks)
{
  if (num_tasks == 0) {
    return;
  }

  task_pool_num_increase(pool, num_tasks);

  BLI_mutex_lock(&scheduler->queue_mutex);

  for (int i = 0; i < num_tasks; i++) {
    BLI_addhead(&scheduler->queue, tasks[i]);
  }

  BLI_condition_notify_all(&scheduler->queue_cond);
  BLI_mutex_unlock(&scheduler->queue_mutex);
}

static void task_scheduler_clear(TaskScheduler *scheduler, TaskPool *pool)
{
  Task *task, *nexttask;
  size_t done = 0;

  BLI_mutex_lock(&scheduler->queue_mutex);

  /* free all tasks from this pool from the queue */
  for (task = (Task *)scheduler->queue.first; task; task = nexttask) {
    nexttask = task->next;

    if (task->pool == pool) {
      task_data_free(task, pool->thread_id);
      BLI_freelinkN(&scheduler->queue, task);

      done++;
    }
  }

  BLI_mutex_unlock(&scheduler->queue_mutex);

  /* notify done */
  task_pool_num_decrease(pool, done);
}

/* Task Pool */

static TaskPool *task_pool_create_ex(TaskScheduler *scheduler,
                                     void *userdata,
                                     const bool is_background,
                                     const bool is_suspended,
                                     TaskPriority priority)
{
  TaskPool *pool = (TaskPool *)MEM_mallocN(sizeof(TaskPool), "TaskPool");

#ifndef NDEBUG
  /* Assert we do not try to create a background pool from some parent task -
   * those only work OK from main thread. */
  if (is_background) {
    const pthread_t thread_id = pthread_self();
    int i = scheduler->num_threads;

    while (i--) {
      BLI_assert(!pthread_equal(scheduler->threads[i], thread_id));
    }
  }
#endif

  pool->scheduler = scheduler;
  pool->num = 0;
  pool->do_cancel = false;
  pool->do_work = false;
  pool->is_suspended = is_suspended;
  pool->start_suspended = is_suspended;
  pool->num_suspended = 0;
  pool->suspended_queue.first = pool->suspended_queue.last = NULL;
  pool->priority = priority;
  pool->run_in_background = is_background;
  pool->use_local_tls = false;

  BLI_mutex_init(&pool->num_mutex);
  BLI_condition_init(&pool->num_cond);

  pool->userdata = userdata;
  BLI_mutex_init(&pool->user_mutex);

  if (BLI_thread_is_main()) {
    pool->thread_id = 0;
  }
  else {
    TaskThread *thread = (TaskThread *)pthread_getspecific(scheduler->tls_id_key);
    if (thread == NULL) {
      /* NOTE: Task pool is created from non-main thread which is not
       * managed by the task scheduler. We identify ourselves as thread ID
       * 0 but we do not use scheduler's TLS storage and use our own
       * instead to avoid any possible threading conflicts.
       */
      pool->thread_id = 0;
      pool->use_local_tls = true;
#ifndef NDEBUG
      pool->creator_thread_id = pthread_self();
#endif
      initialize_task_tls(&pool->local_tls);
    }
    else {
      pool->thread_id = thread->id;
    }
  }

#ifdef DEBUG_STATS
  pool->mempool_stats = (TaskMemPoolStats *)MEM_callocN(
      sizeof(*pool->mempool_stats) * (scheduler->num_threads + 1), "per-taskpool mempool stats");
#endif

  /* Ensure malloc will go fine from threads,
   *
   * This is needed because we could be in main thread here
   * and malloc could be non-thread safe at this point because
   * no other jobs are running.
   */
  BLI_threaded_malloc_begin();

  return pool;
}

/**
 * Create a normal task pool. Tasks will be executed as soon as they are added.
 */
TaskPool *BLI_task_pool_create(TaskScheduler *scheduler, void *userdata, TaskPriority priority)
{
  return task_pool_create_ex(scheduler, userdata, false, false, priority);
}

/**
 * Create a background task pool.
 * In multi-threaded context, there is no differences with #BLI_task_pool_create(),
 * but in single-threaded case it is ensured to have at least one worker thread to run on
 * (i.e. you don't have to call #BLI_task_pool_work_and_wait
 * on it to be sure it will be processed).
 *
 * \note Background pools are non-recursive
 * (that is, you should not create other background pools in tasks assigned to a background pool,
 * they could end never being executed, since the 'fallback' background thread is already
 * busy with parent task in single-threaded context).
 */
TaskPool *BLI_task_pool_create_background(TaskScheduler *scheduler,
                                          void *userdata,
                                          TaskPriority priority)
{
  return task_pool_create_ex(scheduler, userdata, true, false, priority);
}

/**
 * Similar to BLI_task_pool_create() but does not schedule any tasks for execution
 * for until BLI_task_pool_work_and_wait() is called. This helps reducing threading
 * overhead when pushing huge amount of small initial tasks from the main thread.
 */
TaskPool *BLI_task_pool_create_suspended(TaskScheduler *scheduler,
                                         void *userdata,
                                         TaskPriority priority)
{
  return task_pool_create_ex(scheduler, userdata, false, true, priority);
}

void BLI_task_pool_free(TaskPool *pool)
{
  BLI_task_pool_cancel(pool);

  BLI_mutex_end(&pool->num_mutex);
  BLI_condition_end(&pool->num_cond);

  BLI_mutex_end(&pool->user_mutex);

#ifdef DEBUG_STATS
  printf("Thread ID    Allocated   Reused   Discarded\n");
  for (int i = 0; i < pool->scheduler->num_threads + 1; i++) {
    printf("%02d           %05d       %05d    %05d\n",
           i,
           pool->mempool_stats[i].num_alloc,
           pool->mempool_stats[i].num_reuse,
           pool->mempool_stats[i].num_discard);
  }
  MEM_freeN(pool->mempool_stats);
#endif

  if (pool->use_local_tls) {
    free_task_tls(&pool->local_tls);
  }

  MEM_freeN(pool);

  BLI_threaded_malloc_end();
}

BLI_INLINE bool task_can_use_local_queues(TaskPool *pool, int thread_id)
{
  return (thread_id != -1 && (thread_id != pool->thread_id || pool->do_work));
}

static void task_pool_push(TaskPool *pool,
                           TaskRunFunction run,
                           void *taskdata,
                           bool free_taskdata,
                           TaskFreeFunction freedata,
                           int thread_id)
{
  /* Allocate task and fill it's properties. */
  Task *task = task_alloc(pool, thread_id);
  task->run = run;
  task->taskdata = taskdata;
  task->free_taskdata = free_taskdata;
  task->freedata = freedata;
  task->pool = pool;
  /* For suspended pools we put everything yo a global queue first
   * and exit as soon as possible.
   *
   * This tasks will be moved to actual execution when pool is
   * activated by work_and_wait().
   */
  if (pool->is_suspended) {
    BLI_addhead(&pool->suspended_queue, task);
    atomic_fetch_and_add_z(&pool->num_suspended, 1);
    return;
  }
  /* Populate to any local queue first, this is cheapest push ever. */
  if (task_can_use_local_queues(pool, thread_id)) {
    ASSERT_THREAD_ID(pool->scheduler, thread_id);
    TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
    /* Try to push to a local execution queue.
     * These tasks will be picked up next.
     */
    if (tls->num_local_queue < LOCAL_QUEUE_SIZE) {
      tls->local_queue[tls->num_local_queue] = task;
      tls->num_local_queue++;
      return;
    }
    /* If we are in the delayed tasks push mode, we push tasks to a
     * temporary local queue first without any locks, and then move them
     * to global execution queue with a single lock.
     */
    if (tls->do_delayed_push && tls->num_delayed_queue < DELAYED_QUEUE_SIZE) {
      tls->delayed_queue[tls->num_delayed_queue] = task;
      tls->num_delayed_queue++;
      return;
    }
  }
  /* Do push to a global execution pool, slowest possible method,
   * causes quite reasonable amount of threading overhead.
   */
  task_scheduler_push(pool->scheduler, task, pool->priority);
}

void BLI_task_pool_push(TaskPool *pool,
                        TaskRunFunction run,
                        void *taskdata,
                        bool free_taskdata,
                        TaskFreeFunction freedata)
{
  task_pool_push(pool, run, taskdata, free_taskdata, freedata, -1);
}

void BLI_task_pool_push_from_thread(TaskPool *pool,
                                    TaskRunFunction run,
                                    void *taskdata,
                                    bool free_taskdata,
                                    TaskFreeFunction freedata,
                                    int thread_id)
{
  task_pool_push(pool, run, taskdata, free_taskdata, freedata, thread_id);
}

void BLI_task_pool_work_and_wait(TaskPool *pool)
{
  TaskThreadLocalStorage *tls = get_task_tls(pool, pool->thread_id);
  TaskScheduler *scheduler = pool->scheduler;

  if (atomic_fetch_and_and_uint8((uint8_t *)&pool->is_suspended, 0)) {
    if (pool->num_suspended) {
      task_pool_num_increase(pool, pool->num_suspended);
      BLI_mutex_lock(&scheduler->queue_mutex);

      BLI_movelisttolist(&scheduler->queue, &pool->suspended_queue);

      BLI_condition_notify_all(&scheduler->queue_cond);
      BLI_mutex_unlock(&scheduler->queue_mutex);

      pool->num_suspended = 0;
    }
  }

  pool->do_work = true;

  ASSERT_THREAD_ID(pool->scheduler, pool->thread_id);

  handle_local_queue(tls, pool->thread_id);

  BLI_mutex_lock(&pool->num_mutex);

  while (pool->num != 0) {
    Task *task, *work_task = NULL;
    bool found_task = false;

    BLI_mutex_unlock(&pool->num_mutex);

    BLI_mutex_lock(&scheduler->queue_mutex);

    /* find task from this pool. if we get a task from another pool,
     * we can get into deadlock */

    for (task = (Task *)scheduler->queue.first; task; task = task->next) {
      if (task->pool == pool) {
        work_task = task;
        found_task = true;
        BLI_remlink(&scheduler->queue, task);
        break;
      }
    }

    BLI_mutex_unlock(&scheduler->queue_mutex);

    /* if found task, do it, otherwise wait until other tasks are done */
    if (found_task) {
      /* run task */
      BLI_assert(!tls->do_delayed_push);
      work_task->run(pool, work_task->taskdata, pool->thread_id);
      BLI_assert(!tls->do_delayed_push);

      /* delete task */
      task_free(pool, task, pool->thread_id);

      /* Handle all tasks from local queue. */
      handle_local_queue(tls, pool->thread_id);

      /* notify pool task was done */
      task_pool_num_decrease(pool, 1);
    }

    BLI_mutex_lock(&pool->num_mutex);
    if (pool->num == 0) {
      break;
    }

    if (!found_task) {
      BLI_condition_wait(&pool->num_cond, &pool->num_mutex);
    }
  }

  BLI_mutex_unlock(&pool->num_mutex);

  BLI_assert(tls->num_local_queue == 0);
}

void BLI_task_pool_work_wait_and_reset(TaskPool *pool)
{
  BLI_task_pool_work_and_wait(pool);

  pool->do_work = false;
  pool->is_suspended = pool->start_suspended;
}

void BLI_task_pool_cancel(TaskPool *pool)
{
  pool->do_cancel = true;

  task_scheduler_clear(pool->scheduler, pool);

  /* wait until all entries are cleared */
  BLI_mutex_lock(&pool->num_mutex);
  while (pool->num) {
    BLI_condition_wait(&pool->num_cond, &pool->num_mutex);
  }
  BLI_mutex_unlock(&pool->num_mutex);

  pool->do_cancel = false;
}

bool BLI_task_pool_canceled(TaskPool *pool)
{
  return pool->do_cancel;
}

void *BLI_task_pool_user_data(TaskPool *pool)
{
  return pool->userdata;
}

ThreadMutex *BLI_task_pool_user_mutex(TaskPool *pool)
{
  return &pool->user_mutex;
}

int BLI_task_pool_creator_thread_id(TaskPool *pool)
{
  return pool->thread_id;
}

void BLI_task_pool_delayed_push_begin(TaskPool *pool, int thread_id)
{
  if (task_can_use_local_queues(pool, thread_id)) {
    ASSERT_THREAD_ID(pool->scheduler, thread_id);
    TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
    tls->do_delayed_push = true;
  }
}

void BLI_task_pool_delayed_push_end(TaskPool *pool, int thread_id)
{
  if (task_can_use_local_queues(pool, thread_id)) {
    ASSERT_THREAD_ID(pool->scheduler, thread_id);
    TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
    BLI_assert(tls->do_delayed_push);
    task_scheduler_push_all(pool->scheduler, pool, tls->delayed_queue, tls->num_delayed_queue);
    tls->do_delayed_push = false;
    tls->num_delayed_queue = 0;
  }
}