"""memit magic run details.

def __init__(self, mem_usage, baseline, repeat, timeout, interval,

include_children):

self.mem_usage = mem_usage

self.baseline = baseline

self.repeat = repeat

self.timeout = timeout

self.interval = interval

self.include_children = include_children

def __str__(self):

max_mem = max(self.mem_usage)

inc = max_mem - self.baseline

return 'peak memory: %.02f MiB, increment: %.02f MiB' % (max_mem, inc)

def _repr_pretty_(self, p, cycle):

msg = str(self)

"""

# Convert a pid to a process

if isinstance(process, int):

if process == -1: process = os.getpid()

process = psutil.Process(process)

if not meminfo_attr:

# Use the psutil 2.0 attr if the older version isn't passed in.

meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') else 'get_memory_info'

# Select the psutil function get the children similar to how we selected

# the memory_info attr (a change from excepting the AttributeError).

children_attr = 'children' if hasattr(process, 'children') else 'get_children'

# Loop over the child processes and yield their memory

try:

for child in getattr(process, children_attr)(recursive=True):

if isinstance(memory_metric, str):

meminfo = getattr(child, meminfo_attr)()

yield child.pid, getattr(meminfo, memory_metric) / _TWO_20

else:

yield child.pid, getattr(child, meminfo_attr)()[memory_metric] / _TWO_20

except (psutil.NoSuchProcess, psutil.AccessDenied):

# https://github.com/fabianp/memory_profiler/issues/71

# .. low function to get memory consumption ..

if pid == -1:

pid = os.getpid()

def tracemalloc_tool():

# .. cross-platform but but requires Python 3.4 or higher ..

stat = next(filter(lambda item: str(item).startswith(filename),

tracemalloc.take_snapshot().statistics('filename')))

mem = stat.size / _TWO_20

if timestamps:

return mem, time.time()

else:

return mem

def ps_util_tool():

# .. cross-platform but but requires psutil ..

process = psutil.Process(pid)

try:

# avoid using get_memory_info since it does not exists

# in psutil > 2.0 and accessing it will cause exception.

meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') \

else 'get_memory_info'

mem = getattr(process, meminfo_attr)()[0] / _TWO_20

if include_children:

mem += sum([mem for (pid, mem) in _get_child_memory(process, meminfo_attr)])

if timestamps:

return mem, time.time()

else:

return mem

except psutil.AccessDenied:

pass

# continue and try to get this from ps

def _ps_util_full_tool(memory_metric):

# .. cross-platform but requires psutil > 4.0.0 ..

process = psutil.Process(pid)

try:

if not hasattr(process, 'memory_full_info'):

raise NotImplementedError("Backend `{}` requires psutil > 4.0.0".format(memory_metric))

meminfo_attr = 'memory_full_info'

meminfo = getattr(process, meminfo_attr)()

if not hasattr(meminfo, memory_metric):

raise NotImplementedError(

"Metric `{}` not available. For details, see:".format(memory_metric) +

"https://psutil.readthedocs.io/en/latest/index.html?highlight=memory_info#psutil.Process.memory_full_info")

mem = getattr(meminfo, memory_metric) / _TWO_20

if include_children:

mem += sum([mem for (pid, mem) in _get_child_memory(process, meminfo_attr, memory_metric)])

if timestamps:

return mem, time.time()

else:

return mem

except psutil.AccessDenied:

pass

# continue and try to get this from ps

def posix_tool():

# .. scary stuff ..

if include_children:

raise NotImplementedError((

"The psutil module is required to monitor the "

"memory usage of child processes."

))

warnings.warn("psutil module not found. memory_profiler will be slow")

# ..

# .. memory usage in MiB ..

# .. this should work on both Mac and Linux ..

# .. subprocess.check_output appeared in 2.7, using Popen ..

# .. for backwards compatibility ..

out = subprocess.Popen(['ps', 'v', '-p', str(pid)],

stdout=subprocess.PIPE

).communicate()[0].split(b'\n')

try:

vsz_index = out[0].split().index(b'RSS')

mem = float(out[1].split()[vsz_index]) / 1024

if timestamps:

return mem, time.time()

else:

return mem

except:

if timestamps:

return -1, time.time()

else:

return -1

if backend == 'tracemalloc' and \

(filename is None or filename == '<unknown>'):

raise RuntimeError(

'There is no access to source file of the profiled function'

)

tools = {'tracemalloc': tracemalloc_tool,

'psutil': ps_util_tool,

'psutil_pss': lambda: _ps_util_full_tool(memory_metric="pss"),

'psutil_uss': lambda: _ps_util_full_tool(memory_metric="uss"),

'posix': posix_tool}

"""

def __init__(self, monitor_pid, interval, pipe, backend, max_usage=False,

*args, **kw):

self.monitor_pid = monitor_pid

self.interval = interval

self.pipe = pipe

self.cont = True

self.backend = backend

self.max_usage = max_usage

self.n_measurements = 1

self.timestamps = kw.pop("timestamps", False)

self.include_children = kw.pop("include_children", False)

# get baseline memory usage

self.mem_usage = [

_get_memory(self.monitor_pid, self.backend, timestamps=self.timestamps,

include_children=self.include_children)]

super(MemTimer, self).__init__(*args, **kw)

def run(self):

self.pipe.send(0) # we're ready

stop = False

while True:

cur_mem = _get_memory(

self.monitor_pid, self.backend, timestamps=self.timestamps,

include_children=self.include_children,)

if not self.max_usage:

self.mem_usage.append(cur_mem)

else:

self.mem_usage[0] = max(cur_mem, self.mem_usage[0])

self.n_measurements += 1

if stop:

break

stop = self.pipe.poll(self.interval)

# do one more iteration

self.pipe.send(self.mem_usage)

include_children=False, multiprocess=False, max_usage=False,

retval=False, stream=None, backend=None, max_iterations=None):

"""

backend = choose_backend(backend)

if stream is not None:

timestamps = True

if not max_usage:

ret = []

else:

ret = -1

if timeout is not None:

max_iter = int(round(timeout / interval))

elif isinstance(proc, int):

# external process and no timeout

max_iter = 1

else:

# for a Python function wait until it finishes

max_iter = float('inf')

if max_iterations is not None:

max_iter = max_iterations

if callable(proc):

proc = (proc, (), {})

if isinstance(proc, (list, tuple)):

if len(proc) == 1:

f, args, kw = (proc[0], (), {})

elif len(proc) == 2:

f, args, kw = (proc[0], proc[1], {})

elif len(proc) == 3:

f, args, kw = (proc[0], proc[1], proc[2])

else:

raise ValueError

current_iter = 0

while True:

current_iter += 1

child_conn, parent_conn = Pipe() # this will store MemTimer's results

p = MemTimer(os.getpid(), interval, child_conn, backend,

timestamps=timestamps,

max_usage=max_usage,

include_children=include_children)

p.start()

parent_conn.recv() # wait until we start getting memory

# When there is an exception in the "proc" - the (spawned) monitoring processes don't get killed.

# Therefore, the whole process hangs indefinitely. Here, we are ensuring that the process gets killed!

try:

returned = f(*args, **kw)

parent_conn.send(0) # finish timing

ret = parent_conn.recv()

n_measurements = parent_conn.recv()

if max_usage:

# Convert the one element list produced by MemTimer to a singular value

ret = ret[0]

if retval:

ret = ret, returned

except Exception:

parent = psutil.Process(os.getpid())

for child in parent.children(recursive=True):

os.kill(child.pid, SIGKILL)

p.join(0)

raise

p.join(5 * interval)

if (n_measurements > 4) or (current_iter == max_iter) or (interval < 1e-6):

break

interval /= 10.

elif isinstance(proc, subprocess.Popen):

# external process, launched from Python

line_count = 0

while True:

if not max_usage:

mem_usage = _get_memory(

proc.pid, backend, timestamps=timestamps,

include_children=include_children)

if mem_usage and stream is not None:

stream.write("MEM {0:.6f} {1:.4f}\n".format(*mem_usage))

# Write children to the stream file

if multiprocess:

for idx, chldmem in _get_child_memory(proc.pid):

stream.write("CHLD {0} {1:.6f} {2:.4f}\n".format(idx, chldmem, time.time()))

else:

# Create a nested list with the child memory

if multiprocess:

mem_usage = [mem_usage]

for _, chldmem in _get_child_memory(proc.pid):

mem_usage.append(chldmem)

# Append the memory usage to the return value

ret.append(mem_usage)

else:

ret = max(ret,

_get_memory(

proc.pid, backend, include_children=include_children))

time.sleep(interval)

line_count += 1

# flush every 50 lines. Make 'tail -f' usable on profile file

if line_count > 50:

line_count = 0

if stream is not None:

stream.flush()

if timeout is not None:

max_iter -= 1

if max_iter == 0:

break

if proc.poll() is not None:

break

else:

# external process

if max_iter == -1:

max_iter = 1

counter = 0

while counter < max_iter:

counter += 1

if not max_usage:

mem_usage = _get_memory(

proc, backend, timestamps=timestamps,

include_children=include_children)

if stream is not None:

stream.write("MEM {0:.6f} {1:.4f}\n".format(*mem_usage))

# Write children to the stream file

if multiprocess:

for idx, chldmem in _get_child_memory(proc):

stream.write("CHLD {0} {1:.6f} {2:.4f}\n".format(idx, chldmem, time.time()))

else:

# Create a nested list with the child memory

if multiprocess:

mem_usage = [mem_usage]

for _, chldmem in _get_child_memory(proc):

mem_usage.append(chldmem)

# Append the memory usage to the return value

ret.append(mem_usage)

else:

ret = max([ret,

_get_memory(proc, backend, include_children=include_children)

])

time.sleep(interval)

# Flush every 50 lines.

if counter % 50 == 0 and stream is not None:

stream.flush()

if stream:

return None

""" Find the script.

if os.path.isfile(script_name):

return script_name

path = os.getenv('PATH', os.defpath).split(os.pathsep)

for folder in path:

if not folder:

continue

fn = os.path.join(folder, script_name)

if os.path.isfile(fn):

return fn

sys.stderr.write('Could not find script {0}\n'.format(script_name))

"""Time-stamping context manager."""

def __init__(self, timestamps, filename, backend, timestamper=None, func=None,

include_children=False):

self.timestamps = timestamps

self.filename = filename

self.backend = backend

self.ts = timestamper

self.func = func

self.include_children = include_children

def __enter__(self):

if self.ts is not None:

self.ts.current_stack_level += 1

self.ts.stack[self.func].append(self.ts.current_stack_level)

self.timestamps.append(

_get_memory(os.getpid(), self.backend, timestamps=True,

include_children=self.include_children, filename=self.filename))

def __exit__(self, *args):

if self.ts is not None:

self.ts.current_stack_level -= 1

self.timestamps.append(

_get_memory(os.getpid(), self.backend, timestamps=True,

""" A profiler that just records start and end execution times for

def __init__(self, backend, include_children=False):

self.functions = {}

self.backend = backend

self.include_children = include_children

self.current_stack_level = -1

self.stack = {}

def __call__(self, func=None, precision=None):

if func is not None:

if not callable(func):

raise ValueError("Value must be callable")

self.add_function(func)

f = self.wrap_function(func)

f.__module__ = func.__module__

f.__name__ = func.__name__

f.__doc__ = func.__doc__

f.__dict__.update(getattr(func, '__dict__', {}))

return f

else:

def inner_partial(f):

return self.__call__(f, precision=precision)

return inner_partial

def timestamp(self, name="<block>"):

"""Returns a context manager for timestamping a block of code."""

# Make a fake function

func = lambda x: x

func.__module__ = ""

func.__name__ = name

self.add_function(func)

timestamps = []

self.functions[func].append(timestamps)

# A new object is required each time, since there can be several

# nested context managers.

try:

filename = inspect.getsourcefile(func)

except TypeError:

filename = '<unknown>'

return _TimeStamperCM(

timestamps,

filename,

self.backend,

timestamper=self,

func=func

)

def add_function(self, func):

if func not in self.functions:

self.functions[func] = []

self.stack[func] = []

def wrap_function(self, func):

""" Wrap a function to timestamp it.

def f(*args, **kwds):

# Start time

try:

filename = inspect.getsourcefile(func)

except TypeError:

filename = '<unknown>'

timestamps = [

_get_memory(os.getpid(), self.backend, timestamps=True,

include_children=self.include_children, filename=filename)]

self.functions[func].append(timestamps)

try:

with self.call_on_stack(func, *args, **kwds) as result:

return result

finally:

# end time

timestamps.append(_get_memory(os.getpid(), self.backend, timestamps=True,

include_children=self.include_children,

filename=filename))

return f

@contextmanager

def call_on_stack(self, func, *args, **kwds):

self.current_stack_level += 1

self.stack[func].append(self.current_stack_level)

yield func(*args, **kwds)

self.current_stack_level -= 1

def show_results(self, stream=None):

if stream is None:

stream = sys.stdout

for func, timestamps in self.functions.items():

function_name = "%s.%s" % (func.__module__, func.__name__)

for ts, level in zip(timestamps, self.stack[func]):

stream.write("FUNC %s %.4f %.4f %.4f %.4f %d\n" % (

def __init__(self, include_children, backend):

self.include_children = include_children

self._toplevel = []

self.backend = backend

def add(self, code, toplevel_code=None):

if code in self:

return

if toplevel_code is None:

filename = code.co_filename

if filename.endswith((".pyc", ".pyo")):

filename = filename[:-1]

if not os.path.exists(filename):

print('ERROR: Could not find file ' + filename)

if filename.startswith(("ipython-input", "<ipython-input")):

print(

"NOTE: %mprun can only be used on functions defined in"

" physical files, and not in the IPython environment.")

return

toplevel_code = code

(sub_lines, start_line) = inspect.getsourcelines(code)

linenos = range(start_line,

start_line + len(sub_lines))

self._toplevel.append((filename, code, linenos))

self[code] = {}

else:

self[code] = self[toplevel_code]

for subcode in filter(inspect.iscode, code.co_consts):

self.add(subcode, toplevel_code=toplevel_code)

def trace(self, code, lineno, prev_lineno):

memory = _get_memory(-1, self.backend, include_children=self.include_children,

filename=code.co_filename)

prev_value = self[code].get(lineno, None)

previous_memory = prev_value[1] if prev_value else 0

previous_inc = prev_value[0] if prev_value else 0

prev_line_value = self[code].get(prev_lineno, None) if prev_lineno else None

prev_line_memory = prev_line_value[1] if prev_line_value else 0

occ_count = self[code][lineno][2] + 1 if lineno in self[code] else 1

self[code][lineno] = (

previous_inc + (memory - prev_line_memory),

max(memory, previous_memory),

occ_count,

)

def items(self):

"""Iterate on the toplevel code blocks."""

for (filename, code, linenos) in self._toplevel:

measures = self[code]

if not measures:

continue # skip if no measurement

line_iterator = ((line, measures.get(line)) for line in linenos)

""" A profiler that records the amount of memory for each line """

def __init__(self, **kw):

include_children = kw.get('include_children', False)

backend = kw.get('backend', 'psutil')

self.code_map = CodeMap(

include_children=include_children, backend=backend)

self.enable_count = 0

self.max_mem = kw.get('max_mem', None)

self.prevlines = []

self.backend = choose_backend(kw.get('backend', None))

self.prev_lineno = None

def __call__(self, func=None, precision=1):

if func is not None:

self.add_function(func)

f = self.wrap_function(func)

f.__module__ = func.__module__

f.__name__ = func.__name__

f.__doc__ = func.__doc__

f.__dict__.update(getattr(func, '__dict__', {}))

return f

else:

def inner_partial(f):

return self.__call__(f, precision=precision)

return inner_partial

def add_function(self, func):

""" Record line profiling information for the given Python function.

try:

# func_code does not exist in Python3

code = func.__code__

except AttributeError:

warnings.warn("Could not extract a code object for the object %r"

% func)

else:

self.code_map.add(code)

@contextmanager

def _count_ctxmgr(self):

self.enable_by_count()

try:

yield

finally:

self.disable_by_count()

def wrap_function(self, func):

""" Wrap a function to profile it.

if iscoroutinefunction(func):

@coroutine

def f(*args, **kwargs):

with self._count_ctxmgr():

res = yield from func(*args, **kwargs)

return res

else:

def f(*args, **kwds):

with self._count_ctxmgr():

return func(*args, **kwds)

return f

def runctx(self, cmd, globals, locals):

""" Profile a single executable statement in the given namespaces.

self.enable_by_count()

try:

exec(cmd, globals, locals)

finally:

self.disable_by_count()

return self

def enable_by_count(self):

""" Enable the profiler if it hasn't been enabled before.

if self.enable_count == 0:

self.enable()

self.enable_count += 1

def disable_by_count(self):

""" Disable the profiler if the number of disable requests matches the

if self.enable_count > 0:

self.enable_count -= 1

if self.enable_count == 0:

self.disable()

def trace_memory_usage(self, frame, event, arg):

"""Callback for sys.settrace"""

if frame.f_code in self.code_map:

if event == 'call':

# "call" event just saves the lineno but not the memory

self.prevlines.append(frame.f_lineno)

elif event == 'line':

# trace needs current line and previous line

self.code_map.trace(frame.f_code, self.prevlines[-1], self.prev_lineno)

# saving previous line

self.prev_lineno = self.prevlines[-1]

self.prevlines[-1] = frame.f_lineno

elif event == 'return':

lineno = self.prevlines.pop()

self.code_map.trace(frame.f_code, lineno, self.prev_lineno)

self.prev_lineno = lineno

if self._original_trace_function is not None:

self._original_trace_function(frame, event, arg)

return self.trace_memory_usage

def trace_max_mem(self, frame, event, arg):

# run into PDB as soon as memory is higher than MAX_MEM

if event in ('line', 'return') and frame.f_code in self.code_map:

c = _get_memory(-1, self.backend, filename=frame.f_code.co_filename)

if c >= self.max_mem:

t = ('Current memory {0:.2f} MiB exceeded the '

'maximum of {1:.2f} MiB\n'.format(c, self.max_mem))

sys.stdout.write(t)

sys.stdout.write('Stepping into the debugger \n')

frame.f_lineno -= 2

p = pdb.Pdb()

p.quitting = False

p.stopframe = frame

p.returnframe = None

p.stoplineno = frame.f_lineno - 3

p.botframe = None

return p.trace_dispatch

if self._original_trace_function is not None:

(self._original_trace_function)(frame, event, arg)

return self.trace_max_mem

def __enter__(self):

self.enable_by_count()

def __exit__(self, exc_type, exc_val, exc_tb):

self.disable_by_count()

def enable(self):

self._original_trace_function = sys.gettrace()

if self.max_mem is not None:

sys.settrace(self.trace_max_mem)

else:

sys.settrace(self.trace_memory_usage)

def disable(self):

if stream is None:

stream = sys.stdout

template = '{0:>6} {1:>12} {2:>12} {3:>10} {4:<}'

for (filename, lines) in prof.code_map.items():

header = template.format('Line #', 'Mem usage', 'Increment', 'Occurrences',

'Line Contents')

stream.write(u'Filename: ' + filename + '\n\n')

stream.write(header + u'\n')

stream.write(u'=' * len(header) + '\n')

all_lines = linecache.getlines(filename)

float_format = u'{0}.{1}f'.format(precision + 4, precision)

template_mem = u'{0:' + float_format + '} MiB'

for (lineno, mem) in lines:

if mem:

inc = mem[0]

total_mem = mem[1]

total_mem = template_mem.format(total_mem)

occurrences = mem[2]

inc = template_mem.format(inc)

else:

total_mem = u''

inc = u''

occurrences = u''

tmp = template.format(lineno, total_mem, inc, occurrences, all_lines[lineno - 1])

stream.write(tmp)

# A lprun-style %mprun magic for IPython.

@line_cell_magic

def mprun(self, parameter_s='', cell=None):

""" Execute a statement under the line-by-line memory profiler from the

from io import StringIO

from memory_profiler import show_results, LineProfiler

# Local imports to avoid hard dependency.

from distutils.version import LooseVersion

import IPython

ipython_version = LooseVersion(IPython.__version__)

if ipython_version < '0.11':

from IPython.genutils import page

from IPython.ipstruct import Struct

from IPython.ipapi import UsageError

else:

from IPython.core.page import page

from IPython.utils.ipstruct import Struct

from IPython.core.error import UsageError

# Escape quote markers.

opts_def = Struct(T=[''], f=[])

parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")

opts, arg_str = self.parse_options(parameter_s, 'rf:T:c',

list_all=True)

opts.merge(opts_def)

global_ns = self.shell.user_global_ns

local_ns = self.shell.user_ns

if cell is not None:

arg_str += '\n' + cell

# Get the requested functions.

funcs = []

for name in opts.f:

try:

funcs.append(eval(name, global_ns, local_ns))

except Exception as e:

raise UsageError('Could not find function %r.\n%s: %s' % (name,

e.__class__.__name__,

e))

include_children = 'c' in opts

profile = LineProfiler(include_children=include_children)

for func in funcs:

profile(func)

# Add the profiler to the builtins for @profile.

if 'profile' in builtins.__dict__:

had_profile = True

old_profile = builtins.__dict__['profile']

else:

had_profile = False

old_profile = None

builtins.__dict__['profile'] = profile

try:

profile.runctx(arg_str, global_ns, local_ns)

message = ''

except SystemExit:

message = "*** SystemExit exception caught in code being profiled."

except KeyboardInterrupt:

message = ("*** KeyboardInterrupt exception caught in code being "

"profiled.")

finally:

if had_profile:

builtins.__dict__['profile'] = old_profile