summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDan McGee <dan@archlinux.org>2011-11-16 12:49:17 -0600
committerDan McGee <dan@archlinux.org>2011-11-16 12:49:17 -0600
commitb1e406d73844d5a30344ca8ac855fe850c52bc2f (patch)
tree0fd8ec585574a079de6f0a57fca902189a71223e
parent404c4b400b2bd2a14e0363e33d66505c51903fe7 (diff)
downloadarchweb-b1e406d73844d5a30344ca8ac855fe850c52bc2f.tar.gz
archweb-b1e406d73844d5a30344ca8ac855fe850c52bc2f.zip
reporead_inotify: spin up read_repo() in separate thread
This prevents memory usage from ballooning to absolutely huge values, such as when multiple threads kick off at the same time. The bulk of our memory allocation obviously comes in these threads and not the main threads, so being able to isolate them in processes helps a lot. Signed-off-by: Dan McGee <dan@archlinux.org>
-rwxr-xr-xdevel/management/commands/reporead_inotify.py11
1 files changed, 9 insertions, 2 deletions
diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index 4c865ce1..ffd49b8f 100755
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -15,6 +15,7 @@ correctly.
"""
import logging
+import multiprocessing
import os.path
import pyinotify
import sys
@@ -133,6 +134,7 @@ class Database(object):
return
if self.update_thread:
self.update_thread.cancel()
+ self.update_thread = None
self._start_update_countdown()
def update(self):
@@ -142,8 +144,13 @@ class Database(object):
self.updating = True
try:
- # invoke reporead's primary method
- read_repo(self.arch, self.path, {})
+ # invoke reporead's primary method. we do this in a separate
+ # process for memory conservation purposes; these processes grow
+ # rather large so it is best to free up the memory ASAP.
+ process = multiprocessing.Process(target=read_repo,
+ args=[self.arch, self.path, {}])
+ process.start()
+ process.join()
finally:
logger.debug('Done updating database %s.', self.path)
with self.lock: