Reset multiproc values when the pid changes. (#169) · pythonAI/client_python@50d6e0b · GitHub
Skip to content

Commit 50d6e0b

Browse files
brian-brazilbrian-brazil
authored
Reset multiproc values when the pid changes. (prometheus#169)
This puts us at ~2000ns for an operation, which is ~300ns than previously but still slightly better than we were before we switched to a single lock. This is twice as slow as without multiproc mode.
1 parent 0634b77 commit 50d6e0b

3 files changed

Lines changed: 50 additions & 17 deletions

File tree

‎README.md‎

Lines changed: 0 additions & 1 deletion

‎prometheus_client/core.py‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -392,9 +392,9 @@ def close(self):
392392
self._f = None
393393

394394

395-
def _MultiProcessValue(__pid=os.getpid()):
396-
pid = __pid
395+
def _MultiProcessValue(_pidFunc=os.getpid):
397396
files = {}
397+
values = []
398398
# Use a single global lock when in multi-processing mode
399399
# as we presume this means there is no threading going on.
400400
# This avoids the need to also have mutexes in __MmapDict.
@@ -406,31 +406,51 @@ class _MmapedValue(object):
406406
_multiprocess = True
407407

408408
def __init__(self, typ, metric_name, name, labelnames, labelvalues, multiprocess_mode='', **kwargs):
409+
self._params = typ, metric_name, name, labelnames, labelvalues, multiprocess_mode
410+
with lock:
411+
self.__reset()
412+
values.append(self)
413+
414+
415+
def __reset(self):
416+
self._pid = _pidFunc()
417+
typ, metric_name, name, labelnames, labelvalues, multiprocess_mode = self._params
409418
if typ == 'gauge':
410419
file_prefix = typ + '_' + multiprocess_mode
411420
else:
412421
file_prefix = typ
413-
with lock:
414-
if file_prefix not in files:
415-
filename = os.path.join(
416-
os.environ['prometheus_multiproc_dir'], '{0}_{1}.db'.format(file_prefix, pid))
417-
files[file_prefix] = _MmapedDict(filename)
422+
if file_prefix not in files:
423+
filename = os.path.join(
424+
os.environ['prometheus_multiproc_dir'], '{0}_{1}.db'.format(file_prefix, self._pid))
425+
files[file_prefix] = _MmapedDict(filename)
418426
self._file = files[file_prefix]
419427
self._key = json.dumps((metric_name, name, labelnames, labelvalues))
420428
self._value = self._file.read_value(self._key)
421429

430+
def __check_for_pid_change(self):
431+
if self._pid != _pidFunc():
432+
# There has been a fork(), reset all the values.
433+
for f in files.values():
434+
f.close()
435+
files.clear()
436+
for value in values:
437+
value.__reset()
438+
422439
def inc(self, amount):
423440
with lock:
441+
self.__check_for_pid_change()
424442
self._value += amount
425443
self._file.write_value(self._key, self._value)
426444

427445
def set(self, value):
428446
with lock:
447+
self.__check_for_pid_change()
429448
self._value = value
430449
self._file.write_value(self._key, self._value)
431450

432451
def get(self):
433452
with lock:
453+
self.__check_for_pid_change()
434454
return self._value
435455

436456
return _MmapedValue

‎tests/test_multiprocess.py‎

Lines changed: 23 additions & 9 deletions

0 commit comments

Comments
 (0)