58 lines
1.7 KiB
Python
58 lines
1.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
celery.concurrency.threads
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
Pool implementation using threads.
|
|
|
|
"""
|
|
from __future__ import absolute_import
|
|
|
|
from celery.five import UserDict
|
|
|
|
from .base import apply_target, BasePool
|
|
|
|
__all__ = ['TaskPool']
|
|
|
|
|
|
class NullDict(UserDict):
|
|
|
|
def __setitem__(self, key, value):
|
|
pass
|
|
|
|
|
|
class TaskPool(BasePool):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
try:
|
|
import threadpool
|
|
except ImportError:
|
|
raise ImportError(
|
|
'The threaded pool requires the threadpool module.')
|
|
self.WorkRequest = threadpool.WorkRequest
|
|
self.ThreadPool = threadpool.ThreadPool
|
|
super(TaskPool, self).__init__(*args, **kwargs)
|
|
|
|
def on_start(self):
|
|
self._pool = self.ThreadPool(self.limit)
|
|
# threadpool stores all work requests until they are processed
|
|
# we don't need this dict, and it occupies way too much memory.
|
|
self._pool.workRequests = NullDict()
|
|
self._quick_put = self._pool.putRequest
|
|
self._quick_clear = self._pool._results_queue.queue.clear
|
|
|
|
def on_stop(self):
|
|
self._pool.dismissWorkers(self.limit, do_join=True)
|
|
|
|
def on_apply(self, target, args=None, kwargs=None, callback=None,
|
|
accept_callback=None, **_):
|
|
req = self.WorkRequest(apply_target, (target, args, kwargs, callback,
|
|
accept_callback))
|
|
self._quick_put(req)
|
|
# threadpool also has callback support,
|
|
# but for some reason the callback is not triggered
|
|
# before you've collected the results.
|
|
# Clear the results (if any), so it doesn't grow too large.
|
|
self._quick_clear()
|
|
return req
|