""" This module contains a "session saver" which saves the state of a
NumPy session to a file. At a later time, a different Python
process can be started and the saved session can be restored using
load().
The session saver relies on the Python pickle protocol to save and
restore objects. Objects which are not themselves picklable (e.g.
modules) can sometimes be saved by "proxy", particularly when they
are global constants of some kind. If it's not known that proxying
will work, a warning is issued at save time. If a proxy fails to
reload properly (e.g. because it's not a global constant), a warning
is issued at reload time and that name is bound to a _ProxyFailure
instance which tries to identify what should have been restored.
First, some unfortunate (probably unnecessary) concessions to doctest
to keep the test run free of warnings.
>>> del _PROXY_ALLOWED
>>> del __builtins__
By default, save() stores every variable in the caller's namespace:
>>> import numpy as na
>>> a = na.arange(10)
>>> save()
Alternately, save() can be passed a comma seperated string of variables:
>>> save("a,na")
Alternately, save() can be passed a dictionary, typically one you already
have lying around somewhere rather than created inline as shown here:
>>> save(dictionary={"a":a,"na":na})
If both variables and a dictionary are specified, the variables to be
saved are taken from the dictionary.
>>> save(variables="a,na",dictionary={"a":a,"na":na})
Remove names from the session namespace
>>> del a, na
By default, load() restores every variable/object in the session file
to the caller's namespace.
>>> load()
load() can be passed a comma seperated string of variables to be
restored from the session file to the caller's namespace:
>>> load("a,na")
load() can also be passed a dictionary to *restore to*:
>>> d = {}
>>> load(dictionary=d)
load can be passed both a list variables of variables to restore and a
dictionary to restore to:
>>> load(variables="a,na", dictionary=d)
>>> na.all(a == na.arange(10))
1
>>> na.__name__
'numpy'
NOTE: session saving is faked for modules using module proxy objects.
Saved modules are re-imported at load time but any "state" in the module
which is not restored by a simple import is lost.
"""
__all__ = ['load', 'save']
import sys
import pickle
SAVEFILE="session.dat"
VERBOSE = False # global import-time override
def _foo(): pass
_PROXY_ALLOWED = (type(sys), # module
type(_foo), # function
type(None)) # None
def _update_proxy_types():
"""Suppress warnings for known un-picklables with working proxies."""
pass
def _unknown(_type):
"""returns True iff _type isn't known as OK to proxy"""
return (_type is not None) and (_type not in _PROXY_ALLOWED)
# caller() from the following article with one extra f_back added.
# from http://www.python.org/search/hypermail/python-1994q1/0506.html
# SUBJECT: import ( how to put a symbol into caller's namespace )
# SENDER: Steven D. Majewski (sdm7g@elvis.med.virginia.edu)
# DATE: Thu, 24 Mar 1994 15:38:53 -0500
def _caller():
"""caller() returns the frame object of the function's caller."""
try:
1 + '' # make an error happen
except: # and return the caller's caller's frame
return sys.exc_traceback.tb_frame.f_back.f_back.f_back
def _callers_globals():
"""callers_globals() returns the global dictionary of the caller."""
frame = _caller()
return frame.f_globals
def _callers_modules():
"""returns a list containing the names of all the modules in the caller's
global namespace."""
g = _callers_globals()
mods = []
for k,v in g.items():
if type(v) == type(sys):
mods.append(getattr(v,"__name__"))
return mods
def _errout(*args):
for a in args:
print >>sys.stderr, a,
print >>sys.stderr
def _verbose(*args):
if VERBOSE:
_errout(*args)
class _ProxyingFailure:
"""Object which is bound to a variable for a proxy pickle which failed to reload"""
def __init__(self, module, name, type=None):
self.module = module
self.name = name
self.type = type
def __repr__(self):
return "ProxyingFailure('%s','%s','%s')" % (self.module, self.name, self.type)
class _ModuleProxy(object):
"""Proxy object which fakes pickling a module"""
def __new__(_type, name, save=False):
if save:
_verbose("proxying module", name)
self = object.__new__(_type)
self.name = name
else:
_verbose("loading module proxy", name)
try:
self = _loadmodule(name)
except ImportError:
_errout("warning: module", name,"import failed.")
return self
def __getnewargs__(self):
return (self.name,)
def __getstate__(self):
return False
def _loadmodule(module):
if module not in sys.modules:
modules = module.split(".")
s = ""
for i in range(len(modules)):
s = ".".join(modules[:i+1])
exec "import " + s
return sys.modules[module]
class _ObjectProxy(object):
"""Proxy object which fakes pickling an arbitrary object. Only global
constants can really be proxied."""
def __new__(_type, module, name, _type2, save=False):
if save:
if _unknown(_type2):
_errout("warning: proxying object", module + "." + name,
"of type", _type2, "because it wouldn't pickle...",
"it may not reload later.")
else:
_verbose("proxying object", module, name)
self = object.__new__(_type)
self.module, self.name, self.type = module, name, str(_type2)
else:
_verbose("loading object proxy", module, name)
try:
m = _loadmodule(module)
except (ImportError, KeyError):
_errout("warning: loading object proxy", module + "." + name,
"module import failed.")
return _ProxyingFailure(module,name,_type2)
try:
self = getattr(m, name)
except AttributeError:
_errout("warning: object proxy", module + "." + name,
"wouldn't reload from", m)
return _ProxyingFailure(module,name,_type2)
return self
def __getnewargs__(self):
return (self.module, self.name, self.type)
def __getstate__(self):
return False
class _SaveSession(object):
"""Tag object which marks the end of a save session and holds the
saved session variable names as a list of strings in the same
order as the session pickles."""
def __new__(_type, keys, save=False):
if save:
_verbose("saving session", keys)
else:
_verbose("loading session", keys)
self = object.__new__(_type)
self.keys = keys
return self
def __getnewargs__(self):
return (self.keys,)
def __getstate__(self):
return False
class ObjectNotFound(RuntimeError):
pass
def _locate(modules, object):
for mname in modules:
m = sys.modules[mname]
if m:
for k,v in m.__dict__.items():
if v is object:
return m.__name__, k
else:
raise ObjectNotFound(k)
def save(variables=None, file=SAVEFILE, dictionary=None, verbose=False):
"""saves variables from a numpy session to a file. Variables
which won't pickle are "proxied" if possible.
'variables' a string of comma seperated variables: e.g. "a,b,c"
Defaults to dictionary.keys().
'file' a filename or file object for the session file.
'dictionary' the dictionary in which to look up the variables.
Defaults to the caller's globals()
'verbose' print additional debug output when True.
"""
global VERBOSE
VERBOSE = verbose
_update_proxy_types()
if isinstance(file, str):
file = open(file, "wb")
if dictionary is None:
dictionary = _callers_globals()
if variables is None:
keys = dictionary.keys()
else:
keys = variables.split(",")
source_modules = _callers_modules() + sys.modules.keys()
p = pickle.Pickler(file, protocol=2)
_verbose("variables:",keys)
for k in keys:
v = dictionary[k]
_verbose("saving", k, type(v))
try: # Try to write an ordinary pickle
p.dump(v)
_verbose("pickled", k)
except (pickle.PicklingError, TypeError, SystemError):
# Use proxies for stuff that won't pickle
if isinstance(v, type(sys)): # module
proxy = _ModuleProxy(v.__name__, save=True)
else:
try:
module, name = _locate(source_modules, v)
except ObjectNotFound:
_errout("warning: couldn't find object",k,
"in any module... skipping.")
continue
else:
proxy = _ObjectProxy(module, name, type(v), save=True)
p.dump(proxy)
o = _SaveSession(keys, save=True)
p.dump(o)
file.close()
def load(variables=None, file=SAVEFILE, dictionary=None, verbose=False):
"""load a numpy session from a file and store the specified
'variables' into 'dictionary'.
'variables' a string of comma seperated variables: e.g. "a,b,c"
Defaults to dictionary.keys().
'file' a filename or file object for the session file.
'dictionary' the dictionary in which to look up the variables.
Defaults to the caller's globals()
'verbose' print additional debug output when True.
"""
global VERBOSE
VERBOSE = verbose
if isinstance(file, str):
file = open(file, "rb")
if dictionary is None:
dictionary = _callers_globals()
values = []
p = pickle.Unpickler(file)
while 1:
o = p.load()
if isinstance(o, _SaveSession):
session = dict(zip(o.keys, values))
_verbose("updating dictionary with session variables.")
if variables is None:
keys = session.keys()
else:
keys = variables.split(",")
for k in keys:
dictionary[k] = session[k]
return None
else:
_verbose("unpickled object", str(o))
values.append(o)
def test():
import doctest, numpy.numarray.session
return doctest.testmod(numpy.numarray.session)
|