# -*- test-case-name: twisted.test.test_popsicle -*-
# Twisted, the Framework of Your Internet
# Copyright (C) 2001-2002 Matthew W. Lefkowitz
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of version 2.1 of the GNU Lesser General Public
# License as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Mailsicle
A Reference Implementation of a Popsicle Back-End.
This is a very simple persistence mechanism that demonstrates various things
about the Popsicle persistence manager. It is very slow and its scalability
depends intimately on the performance characteristics of your filesystem, so
use with caution.
There are a few advantages to using Mailsicle for persistence, however. Its
fileformat is almost completely transparent: it uses an RFC822-inspired (though
by no means compliant!) fileformat for easy inspection and manual repair. It
provides very basic indexes, which is a slight advantage over dirdbm+shelf
persistence.
One slightly strange advantage is that it's somewhat tedious to write
persistence adapters for mailsicle. This provides a useful exercise in
locating the essential information that you want to persist, and can be a
useful prelude to developing an efficient relational schema for a set of
objects, while providing a fallback mechanism in the case where a database is
not installed.
Future Work:
- optionally allow for backending to BSDDB or other single-file approach
instead of sprawling across filesystem, first for indexes, then for
content
"""
# system imports
import os
import re
import weakref
from cStringIO import StringIO
# twisted imports
from twisted.python.components import getAdapter,Interface,Adapter,registerAdapter,getAdapterClassWithInheritance
from twisted.python.reflect import qual,namedClass
from twisted.python.compat import dict
from twisted.internet import defer
# sibling imports
from twisted.popsicle import repos,freezer
class IHeaderSaver(Interface):
"""I am an interface which allows objects to be saved to mail-style headers.
"""
def descriptiveName(self):
"""Return a pretty-printed (non-unique) name describing me.
"""
def getItems(self):
"""Get a list of tuples of strings, [(key, value), ...].
"""
def getIndexes(self):
"""Return a list of tuples of strings, [(index_name, index_value), ...]
"""
def getContinuations(self):
"""Get a list of 'continuation' sections. This is a list of lists of tuples.
"""
def loadItems(self, items, toplevel):
"""Take the result of a getItems() call and populate self.original.
'toplevel' is the top-level object if this is a continuation, otherwise
it is self.original
"""
def loadContinuations(self, cont):
"""Take the result of a getContinuations() call and populate self.original.
"""
def writeHeader(f, k, v):
f.write(k.replace("\\", "\\\\").replace("\n", "\\n").replace(": ", ":\ "))
f.write(": ")
f.write(v.replace("\n", "\n\t"))
f.write("\n")
psl = r'"((?:\"|.)*?)"'
poidl = r'(?:%s |)<([^>]*)>' % psl
psl = re.compile(psl)
poidl = re.compile(poidl)
def parseOIDList(s):
return poidl.findall(s)
def parseStringList(s):
return psl.findall(s)
def makeStringList(l):
return ', '.join(map(quotify, l))
def quotify(s):
return '"'+s.replace('\\','\\\\').replace('"','\\"')+'"'
def dictToHeaders(d):
io = StringIO()
for k, v in d.items():
writeHeader(io,k,v)
return io.getvalue()
def headersToTuples(hdrs):
hdrl = []
protohdr = None
for line in hdrs.split("\n"):
if not line: continue
if line[0] != '\t':
header, valBegin = line.split(': ', 1)
if protohdr:
hdrl.append(tuple(protohdr))
protohdr = [header, valBegin]
else:
protohdr[1] += '\n'+line[1:]
if protohdr:
hdrl.append(tuple(protohdr))
return hdrl
wspr = re.compile("\S")
def whitePrefix(s):
return s[:wspr.search(s).start()]
def getSaver(o,repo):
adapt = getAdapter(o, IHeaderSaver, adapterClassLocator=getAdapterClassWithInheritance)
adapt.repo = repo
return adapt
class QueryResults:
def __init__(self, oidlist, repo):
self.oidlist = oidlist
self.repo = repo
def fetch(self, begin=0, end=None):
return defer.execute(self.fetchNow, begin, end)
def fetchNow(self, begin=0, end=None):
if end is None:
end = len(self.oidlist)
entries = self.oidlist[begin:end]
return map(self.repo.loadNow, entries)
class Mailsicle(repos.DirectoryRepository):
def loadOIDNow(self, oid):
try:
f = open(os.path.join(self.dirname, str(oid)))
except (OSError, IOError):
raise repos.OIDNotFound(str(oid))
llt = f.read().split("\n-\n")
ll = []
for lt in llt:
ll.append(headersToTuples(lt))
items = ll.pop(0)
# produce dummy instance
# ditems = dict(items)
# OID and Class headers --
p_oid = items[0][1]
assert str(oid) == p_oid, "Wrong OID in file."
cl = self.createOID(oid, namedClass(items[1][1]))
# TODO: make instance(...) support new-style classes...
saver = getSaver(cl,self)
saver.loadItems(items, cl)
saver.loadContinuations(ll)
return cl
def cache(self, oid, obj, finished=1):
if finished:
idxs = getSaver(obj,self).getIndexes()
else:
idxs = []
self._cache[oid] = obj
self._revCache[obj] = [oid, idxs]
def getOldIndexes(self, obj):
if self._revCache.has_key(obj):
return self._revCache[obj][1]
return []
def getOID(self, obj):
if self._revCache.has_key(obj):
return self._revCache[obj][0]
else:
# TODO: if OID generation really needs to be async...
return freezer.ref(obj).acquireOID(self)
def saveOID(self, oid, obj):
adapt = getSaver(obj,self)
kvl = adapt.getItems()
f = open(os.path.join(self.dirname, str(oid)), 'w')
writeHeader(f, "OID", str(oid))
writeHeader(f, "Class", qual(obj.__class__))
for key, value in kvl:
writeHeader(f, key, value)
for kvl2 in adapt.getContinuations():
f.write("-\n")
for key, value in kvl2:
writeHeader(f, key, value)
idxs = dict(self.getOldIndexes(obj))
nidxs = adapt.getIndexes()
for idx, value in nidxs:
# store all new indexes
if idxs.has_key(idx):
# if I had it before, update it
if idxs[idx] != value:
# print 'updating index',oid,idx,idxs[idx],'=>',value
self.removeIndex(oid, idx, idxs[idx])
self.storeIndex(oid, idx, value)
# track it's been removed
del idxs[idx]
else:
# print 'storing new index',oid,idx,'=>',value
self.storeIndex(oid, idx, value)
for idx, value in idxs.items():
# clear all remaining indexes
# print 'clearing old index',oid,idx,'=>',value
self.removeIndex(oid, idx, value)
def storeIndex(self, oid, idx, value):
# print 'indexing',oid,idx,value
oid = str(oid)
idxd = "index-"+idx
opj = os.path.join
dirn = opj(self.dirname, idxd, value)
if not os.path.isdir(dirn):
os.makedirs(dirn)
os.symlink(opj('..','..', oid),
opj(self.dirname, idxd, value, oid))
def removeIndex(self, oid, idx, value):
# print 'deindexing', oid, idx, value
oid = str(oid)
idxd = "index-"+idx
opj = os.path.join
dirn = opj(self.dirname, idxd, value)
if not os.path.isdir(dirn):
return
if not os.path.islink(opj(dirn, oid)):
return
os.unlink(opj(dirn,oid))
def queryIndex(self, idx, value):
opj = os.path.join
idxd = "index-" + idx
try:
if value == '*':
entries = []
values = os.listdir(opj(self.dirname, idxd))
for v in values:
entries.extend(os.listdir(opj(self.dirname, idxd, v)))
else:
entries = os.listdir(opj(self.dirname, idxd, value))
except OSError:
return QueryResults([], self)
return QueryResults(entries, self)
def loadOIDList(self, s):
l = []
for descript, oid in parseOIDList(s):
if oid:
l.append(self.loadNow(oid))
else:
l.append(None)
return l
def makeOIDList(self,l):
return ', '.join(map(self.addressOID, l))
def addressOID(self, obj, desc=None):
if obj is not None:
oid = freezer.ref(obj, self).acquireOID()
if desc is not None:
return '%s <%s>' % (quotify(desc), oid)
else:
if desc=='':
return '<%s>' % oid
else:
return ('%s <%s>' %
(quotify(getSaver(obj,self).descriptiveName()),
oid))
else:
return '<>'
from twisted.cred import service,authorizer,identity,perspective
def hexlify(s):
# note: when we drop 2.1 support, we can just
# s.encode('hex') and s.decode('hex')
return ''.join(['%02x' % ord(c) for c in s])
def unhexlify(s):
return ''.join([chr(int(s[i:i+2],16)) for i in range(0,len(s),2)])
class DefaultSaver(Adapter):
__implements__ = IHeaderSaver
def descriptiveName(self):
"""Return a pretty-printed (non-unique) name describing me.
"""
if hasattr(self.original, "name"):
return self.original.name
def getItems(self):
"""Get a list of tuples of strings, [(key, value), ...].
"""
return []
def getContinuations(self):
"""Get a list of 'continuation' sections. This is a list of lists of tuples.
"""
return []
def getIndexes(self):
"""
"""
return []
def loaditem_oid(self, kv, vv):
pass
def loaditem_class(self, kv, vv):
pass
def preLoad(self):
pass
def postLoad(self):
pass
def loadItems(self, items, toplevel):
"""Take the result of a getItems() call and populate self.original.
'toplevel' is the top-level object if this is a continuation, otherwise
it is self.original
"""
self.preLoad()
for k,v in items:
ksplit = k.split("+", 1)
kk = ksplit[0]
if len(ksplit) > 1:
kv = ksplit[1]
else:
kv = None
getattr(self,"loaditem_"+kk.lower())(kv,v)
self.postLoad()
def loadContinuations(self, cont):
"""Take the result of a getContinuations() call and populate self.original.
"""
class IdentitySaver(DefaultSaver):
'''Persistor for cred Identities.
'''
def getItems(self):
itm = [("Name", self.original.name),
("Password", hexlify(self.original.hashedPassword))]
for svcnam, pspnam in self.original.keyring.keys():
itm.append(("Key+"+svcnam, pspnam))
return itm
def preLoad(self):
# XXX currently I want authorizers to be in a weird halfway-in
# relationship with the database, so this is necessary to keep
# identities in sync
self.original.authorizer = self.repo._authorizer
self.original.keyring = {}
def loaditem_key(self, kv, vv):
self.original.keyring[(kv,vv)] = 1
def loaditem_password(self, kv, vv):
self.original.hashedPassword = unhexlify(vv)
def loaditem_name(self, kv, vv):
self.original.name = vv
def getIndexes(self):
return [("identity-name", self.original.name)]
registerAdapter(IdentitySaver, identity.Identity, IHeaderSaver)
class PerspectiveSaver(DefaultSaver):
def getItems(self):
return [('Service', self.original.getService().serviceName),
('Name', self.original.perspectiveName)]
def getIndexes(self):
return [("perspective-name-"+self.original.service.serviceName,
self.original.perspectiveName)]
def loaditem_service(self, kv, vv):
self.original.service = self.repo._services[vv]
def loaditem_name(self, kv, vv):
self.original.perspectiveName = vv
registerAdapter(PerspectiveSaver, perspective.Perspective, IHeaderSaver)
class MailsicleAuthorizer(authorizer.Authorizer):
"""A twisted.cred authorizer that's persistent in a Mailsicle database.
NOTE: only one MailsicleAuthorizer may be present in a given Mailsicle
database.
"""
def __init__(self, msicle, serviceCollection=None):
if hasattr(msicle, "_authorizer"):
raise NotImplementedError("Only one authorizer "
"per mailsicle allowed.")
authorizer.Authorizer.__init__(self, serviceCollection)
self.msicle = msicle
self.msicle._authorizer = self
def addIdentity(self, identity):
freezer.register(identity, self.msicle)
## XXX TODO: removeIdentity
def getIdentityRequest(self, name):
return self.msicle.queryIndex("identity-name",
name).fetch(0,1).addCallback(lambda x: x[0])
class MailsicleService(service.Service):
"""A twisted.cred service that's persistent in a Mailsicle database.
NOTE: such services must have a unique serviceName for the mailsicle they
are persistent in. There is currently no checking.
TODO: this should be generalized to load Perspectives from any data-source
that implements a certain Repository interface.
"""
def __init__(self, msicle, serviceName, serviceParent=None, authorizer=None):
service.Service.__init__(self, serviceName, serviceParent, authorizer)
self.msicle = msicle
# XXX FIXME: same as in IdentitySaver.loadItems
if not hasattr(self.msicle, '_services'):
self.msicle._services = {}
self.msicle._services[serviceName] = self
self.perspectives = weakref.WeakValueDictionary()
def __getstate__(self):
dct = self.__dict__.copy()
del dct['perspectives']
def __setstate__(self, dct):
self.__dict__ = dct
self.perspectives = weakref.WeakValueDictionary()
def stopService(self):
freezer.clean()
def createPerspective(self, name):
if self.perspectives.has_key(name):
return self.perspectives[name]
p = self.perspectiveClass(name)
p.setService(self)
freezer.register(p, self.msicle)
self.perspectives[name] = p
return p
def getPerspectiveNamed(self, name):
if self.perspectives.has_key(name):
return self.perspectives[name]
l = self.msicle.queryIndex("perspective-name-"+self.serviceName,
name).fetchNow(0,1)
try:
return l[0]
except IndexError:
raise KeyError("%s doesn't exist." % name)
|