"""Test script for exercising various catalog features under load
Usage:
cd lib/python
python Products/ZCatalog/regressiontests/loadmail.py command args
where each command has its own command-line arguments that it expects.
Note that all of the commands operate on the Zope database,
typically var/Data.fs.
Note that this script uses the proc file system to get memory size.
Many of the commands output performance statisics on lines that look like::
11.3585170507 0.06 2217781L 7212
where the numbers are:
- clock time in seconds
- cpu time used by the main thread, in seconds,
- Database size growth over the test
- Memory growth during the test (if the proc file system is available).
Commands:
base mbox [max]
Build a base database by:
- Deleting ../../var/Data.fs
- Starting Zope
- Adding a top-level folder names 'mail'
- Reading up to max messages from the Unix mailbox file, mbox
and adding them as documents to the mail folder.
index [threshold]
Index all of the DTML documents in the database, committing
sub-transactions after each threshold objects. threshold defaults to
1000.
If the threshold is less than the number of messages, then the
size of the temporary sub-transaction commit file is output.
inc mbox start end [threads [wait]]
Incrementally index messages start to end in unix mailbox mbox.
If the threads argument is supplied, then it specifies the
number of threads to use. For example, with:
python Products/ZCatalog/tests/loadmail.py inc mbox 0 200 2
One thread indexes messages 0 to 99 and another thread indexes messages
100 to 199.
If wait is specified, then after each document is indexed, the
thread sleeps a random number of seconds between 0 and 2*wait.
The default wait is 0.25 seconds.
For each thread, a line that looks like::
3.41 (0, 1)
is output, containing:
- The cpu time
- The number of ZODB transaction conflicts detected when reading
- The number of ZODB transaction conflicts detected when committing
edit edits deletes inserts threads wait
Incrementally edit edits messages from mail. For each message,
do a random number of word deletes between 0 and deletes * 2
and do a random number of inserts between 0 and inserts * 2.
For each thread, a line that looks like::
3.41 (0, 1)
as described above.
pdebug command args
Run one of the other commands in the Python debugger.
sample suite of tests::
cd lib/python
python Products/ZCatalog/regressiontests/loadmail.py base ~/zope.mbox 1000
python Products/ZCatalog/regressiontests/loadmail.py index 100
python Products/ZCatalog/regressiontests/loadmail.py \
inc ~/python-dev.mbox 0 10 2
python Products/ZCatalog/regressiontests/loadmail.py edit 10 10 10 2
"""
import mailbox, time, sys, os, string
sys.path.insert(0, '.')
import whrandom
whrandom.seed(1,2,3)
from string import strip,find,split,lower,atoi
from urllib import quote
def do(db, f, args, returnf=None):
"""Do something and measure it's impact"""
t = c = size = mem = r = None
try:
size=db.getSize()
mem=VmSize()
t=time.time()
c=time.clock()
r=apply(f, args)
t=time.time() - t
c=time.clock() - c
size=db.getSize()-size
mem=VmSize()-mem
finally:
if returnf is not None:
returnf(t, c, size, mem, r)
else:
return t, c, size, mem, r
def loadmessage(dest, message, i, body=None, headers=None):
if body is None: body=message.fp.read()
if headers is None: headers=message.headers
dest.manage_addDTMLDocument(str(i), file=body)
doc=dest[str(i)]
for h in headers:
h=strip(h)
l=find(h,':')
if l <= 0: continue
name=lower(h[:l])
if name=='subject': name='title'
v=strip(h[l+1:])
type='string'
if 0 and name=='date': type='date'
elif 0:
try: atoi(v)
except: pass
else: type=int
if name=='title':
doc.manage_changeProperties(title=h)
else:
try: doc.manage_addProperty(name, v, type)
except: pass
def loadmail(dest, name, mbox, printstat=0, max=-1):
try:
import Products.BTreeFolder.BTreeFolder
except:
dest.manage_addFolder(name)
else:
Products.BTreeFolder.BTreeFolder.manage_addBTreeFolder(dest, name)
dest=getattr(dest, name)
f=open(mbox)
mb=mailbox.UnixMailbox(f)
i=0
message=mb.next()
while message:
if max >= 0 and i > max:
break
if i%100 == 0 and printstat:
fmt = "\t%s\t%s\t\r"
if os.environ.get('TERM') in ('dumb', 'emacs'):
fmt = "\t%s\t%s\t\n"
sys.stdout.write(fmt % (i, f.tell()))
sys.stdout.flush()
if i and (i%5000 == 0):
get_transaction().commit()
dest._p_jar._cache.minimize()
loadmessage(dest, message, i)
i=i+1
message=mb.next()
dest.number_of_messages=i
print
get_transaction().commit()
def loadinc(name, mb, f, printstat=0, max=99999999, wait=1):
from ZODB.POSException import ConflictError
from time import sleep
from whrandom import uniform
import Zope, sys
rconflicts=wconflicts=0
i=0
message=mb.next()
body=message.fp.read()
headers=list(message.headers)
while i < max:
# sys.stderr.write("%s " % i)
# sys.stdout.flush()
if wait: sleep(uniform(0,wait))
jar=Zope.DB.open()
app=jar.root()['Application']
mdest=getattr(app, name)
if i%100 == 0 and printstat:
sys.stdout.write("\t%s\t%s\t\r" % (i, f.tell()))
sys.stdout.flush()
did=str(i)
try:
loadmessage(mdest, message, did, body, headers)
doc=mdest[did]
app.cat.catalog_object(doc)
except ConflictError, v:
# print v.args
rconflicts=rconflicts+1
get_transaction().abort()
else:
try:
get_transaction().commit()
i=i+1
message=mb.next()
body=message.fp.read()
headers=list(message.headers)
except ConflictError:
wconflicts=wconflicts+1
get_transaction().abort()
doc=app=mdest=0
jar.close()
if printstat: sys.stdout.write("\t%s\t%s\t\n" % (i, f.tell()))
sys.stdout.flush()
return rconflicts, wconflicts
def base():
try: os.unlink('../../var/Data.fs')
except: pass
import Zope
app=Zope.app()
if len(sys.argv) > 3:
max = atoi(sys.argv[3])
else:
max = -1
print do(Zope.DB, loadmail, (app, 'mail', sys.argv[2], 1, max))
Zope.DB.close()
class RE:
def redirect(*args, **kw): pass
def indexf(app):
r=RE()
r.PARENTS=[0, app.mail]
app.cat.manage_catalogFoundItems(r,r,'','',['DTML Document'])
get_transaction().commit()
def index():
os.environ['STUPID_LOG_FILE']=''
os.environ['STUPID_LOG_SEVERITY']='-111'
import Zope, Products.ZCatalog.ZCatalog
import AccessControl.SecurityManagement, AccessControl.SpecialUsers
app=Zope.app()
Products.ZCatalog.ZCatalog.manage_addZCatalog(app, 'cat', '')
try:
app.cat.threshold = atoi(sys.argv[2])
except IndexError:
app.cat.threashold = 1000
from Products.ZCTextIndex.ZCTextIndex import
import PLexicon
from Products.ZCTextIndex.Lexicon import
import Splitter, CaseNormalizer
app.cat._setObject('lex',
PLexicon('lex', '', Splitter(), CaseNormalizer())
)
class extra:
doc_attr = 'PrincipiaSearchSource'
lexicon_id = 'lex'
index_type = 'Okapi BM25 Rank'
app.cat.addIndex('PrincipiaSearchSource', 'ZCTextIndex', extra)
get_transaction().commit()
system = AccessControl.SpecialUsers.system
AccessControl.SecurityManagement.newSecurityManager(None, system)
r=RE()
r.PARENTS=[app.cat, app]
print do(Zope.DB, indexf, (app,))
#hist(sys.argv[2])
Zope.DB.close()
def initmaili(n):
import Zope
app=Zope.app()
try:
import Products.BTreeFolder.BTreeFolder
except:
app.manage_addFolder(n)
else:
Products.BTreeFolder.BTreeFolder.manage_addBTreeFolder(app, n)
get_transaction().commit()
app._p_jar.close()
def hist(n):
import Zope
app=Zope.app()
import cPickle
pickler=cPickle.Pickler(open("h%s.hist" % n, 'w'))
h=app.cat._catalog.indexes['PrincipiaSearchSource'].histogram()
pickler.dump(list(h.items()))
#h=app.cat._catalog.uids.keys()
#pickler.dump(list(h))
def inc():
import Zope, thread
min, max = atoi(sys.argv[3]), atoi(sys.argv[4])
count = max-min
try: threads=atoi(sys.argv[5])
except:
threads=1
wait=0
else:
try: wait=atof(sys.argv[6])
except: wait=0.25
wait = wait * 2
count = count / threads
max = min + count
omin=min
db=Zope.DB
size=db.getSize()
mem=VmSize()
t=time.time()
c=time.clock()
mbox=sys.argv[2]
argss=[]
for i in range(threads):
amin=min+i*count
dest='maili%s' % amin
initmaili(dest)
f = open(mbox)
mb=mailbox.UnixMailbox(f)
j=0
while j < amin:
mb.next()
j=j+1
lock=thread.allocate_lock()
lock.acquire()
def returnf(t, c, size, mem, r, lock=lock):
print c, r
lock.release()
argss.append((lock, (dest, mb, f, 1, count, wait), returnf))
for lock, args, returnf in argss:
thread.start_new_thread(do, (Zope.DB, loadinc, args, returnf))
for lock, args, returnf in argss:
lock.acquire()
t=time.time() - t
c=time.clock() - c
size=db.getSize()-size
mem=VmSize()-mem
print t, c, size, mem
#hist("%s-%s-%s" % (omin, count, threads))
Zope.DB.close()
words=['banishment', 'indirectly', 'imprecise', 'peeks',
'opportunely', 'bribe', 'sufficiently', 'Occidentalized', 'elapsing',
'fermenting', 'listen', 'orphanage', 'younger', 'draperies', 'Ida',
'cuttlefish', 'mastermind', 'Michaels', 'populations', 'lent',
'cater', 'attentional', 'hastiness', 'dragnet', 'mangling',
'scabbards', 'princely', 'star', 'repeat', 'deviation', 'agers',
'fix', 'digital', 'ambitious', 'transit', 'jeeps', 'lighted',
'Prussianizations', 'Kickapoo', 'virtual', 'Andrew', 'generally',
'boatsman', 'amounts', 'promulgation', 'Malay', 'savaging',
'courtesan', 'nursed', 'hungered', 'shiningly', 'ship', 'presides',
'Parke', 'moderns', 'Jonas', 'unenlightening', 'dearth', 'deer',
'domesticates', 'recognize', 'gong', 'penetrating', 'dependents',
'unusually', 'complications', 'Dennis', 'imbalances', 'nightgown',
'attached', 'testaments', 'congresswoman', 'circuits', 'bumpers',
'braver', 'Boreas', 'hauled', 'Howe', 'seethed', 'cult', 'numismatic',
'vitality', 'differences', 'collapsed', 'Sandburg', 'inches', 'head',
'rhythmic', 'opponent', 'blanketer', 'attorneys', 'hen', 'spies',
'indispensably', 'clinical', 'redirection', 'submit', 'catalysts',
'councilwoman', 'kills', 'topologies', 'noxious', 'exactions',
'dashers', 'balanced', 'slider', 'cancerous', 'bathtubs', 'legged',
'respectably', 'crochets', 'absenteeism', 'arcsine', 'facility',
'cleaners', 'bobwhite', 'Hawkins', 'stockade', 'provisional',
'tenants', 'forearms', 'Knowlton', 'commit', 'scornful',
'pediatrician', 'greets', 'clenches', 'trowels', 'accepts',
'Carboloy', 'Glenn', 'Leigh', 'enroll', 'Madison', 'Macon', 'oiling',
'entertainingly', 'super', 'propositional', 'pliers', 'beneficiary',
'hospitable', 'emigration', 'sift', 'sensor', 'reserved',
'colonization', 'shrilled', 'momentously', 'stevedore', 'Shanghaiing',
'schoolmasters', 'shaken', 'biology', 'inclination', 'immoderate',
'stem', 'allegory', 'economical', 'daytime', 'Newell', 'Moscow',
'archeology', 'ported', 'scandals', 'Blackfoot', 'leery', 'kilobit',
'empire', 'obliviousness', 'productions', 'sacrificed', 'ideals',
'enrolling', 'certainties', 'Capsicum', 'Brookdale', 'Markism',
'unkind', 'dyers', 'legislates', 'grotesquely', 'megawords',
'arbitrary', 'laughing', 'wildcats', 'thrower', 'sex', 'devils',
'Wehr', 'ablates', 'consume', 'gossips', 'doorways', 'Shari',
'advanced', 'enumerable', 'existentially', 'stunt', 'auctioneers',
'scheduler', 'blanching', 'petulance', 'perceptibly', 'vapors',
'progressed', 'rains', 'intercom', 'emergency', 'increased',
'fluctuating', 'Krishna', 'silken', 'reformed', 'transformation',
'easter', 'fares', 'comprehensible', 'trespasses', 'hallmark',
'tormenter', 'breastworks', 'brassiere', 'bladders', 'civet', 'death',
'transformer', 'tolerably', 'bugle', 'clergy', 'mantels', 'satin',
'Boswellizes', 'Bloomington', 'notifier', 'Filippo', 'circling',
'unassigned', 'dumbness', 'sentries', 'representativeness', 'souped',
'Klux', 'Kingstown', 'gerund', 'Russell', 'splices', 'bellow',
'bandies', 'beefers', 'cameramen', 'appalled', 'Ionian', 'butterball',
'Portland', 'pleaded', 'admiringly', 'pricks', 'hearty', 'corer',
'deliverable', 'accountably', 'mentors', 'accorded',
'acknowledgement', 'Lawrenceville', 'morphology', 'eucalyptus',
'Rena', 'enchanting', 'tighter', 'scholars', 'graduations', 'edges',
'Latinization', 'proficiency', 'monolithic', 'parenthesizing', 'defy',
'shames', 'enjoyment', 'Purdue', 'disagrees', 'barefoot', 'maims',
'flabbergast', 'dishonorable', 'interpolation', 'fanatics', 'dickens',
'abysses', 'adverse', 'components', 'bowl', 'belong', 'Pipestone',
'trainees', 'paw', 'pigtail', 'feed', 'whore', 'conditioner',
'Volstead', 'voices', 'strain', 'inhabits', 'Edwin', 'discourses',
'deigns', 'cruiser', 'biconvex', 'biking', 'depreciation', 'Harrison',
'Persian', 'stunning', 'agar', 'rope', 'wagoner', 'elections',
'reticulately', 'Cruz', 'pulpits', 'wilt', 'peels', 'plants',
'administerings', 'deepen', 'rubs', 'hence', 'dissension', 'implored',
'bereavement', 'abyss', 'Pennsylvania', 'benevolent', 'corresponding',
'Poseidon', 'inactive', 'butchers', 'Mach', 'woke', 'loading',
'utilizing', 'Hoosier', 'undo', 'Semitization', 'trigger', 'Mouthe',
'mark', 'disgracefully', 'copier', 'futility', 'gondola', 'algebraic',
'lecturers', 'sponged', 'instigators', 'looted', 'ether', 'trust',
'feeblest', 'sequencer', 'disjointness', 'congresses', 'Vicksburg',
'incompatibilities', 'commend', 'Luxembourg', 'reticulation',
'instructively', 'reconstructs', 'bricks', 'attache', 'Englishman',
'provocation', 'roughen', 'cynic', 'plugged', 'scrawls', 'antipode',
'injected', 'Daedalus', 'Burnsides', 'asker', 'confronter',
'merriment', 'disdain', 'thicket', 'stinker', 'great', 'tiers',
'oust', 'antipodes', 'Macintosh', 'tented', 'packages',
'Mediterraneanize', 'hurts', 'orthodontist', 'seeder', 'readying',
'babying', 'Florida', 'Sri', 'buckets', 'complementary',
'cartographer', 'chateaus', 'shaves', 'thinkable', 'Tehran',
'Gordian', 'Angles', 'arguable', 'bureau', 'smallest', 'fans',
'navigated', 'dipole', 'bootleg', 'distinctive', 'minimization',
'absorbed', 'surmised', 'Malawi', 'absorbent', 'close', 'conciseness',
'hopefully', 'declares', 'descent', 'trick', 'portend', 'unable',
'mildly', 'Morse', 'reference', 'scours', 'Caribbean', 'battlers',
'astringency', 'likelier', 'Byronizes', 'econometric', 'grad',
'steak', 'Austrian', 'ban', 'voting', 'Darlington', 'bison', 'Cetus',
'proclaim', 'Gilbertson', 'evictions', 'submittal', 'bearings',
'Gothicizer', 'settings', 'McMahon', 'densities', 'determinants',
'period', 'DeKastere', 'swindle', 'promptness', 'enablers', 'wordy',
'during', 'tables', 'responder', 'baffle', 'phosgene', 'muttering',
'limiters', 'custodian', 'prevented', 'Stouffer', 'waltz', 'Videotex',
'brainstorms', 'alcoholism', 'jab', 'shouldering', 'screening',
'explicitly', 'earner', 'commandment', 'French', 'scrutinizing',
'Gemma', 'capacitive', 'sheriff', 'herbivore', 'Betsey', 'Formosa',
'scorcher', 'font', 'damming', 'soldiers', 'flack', 'Marks',
'unlinking', 'serenely', 'rotating', 'converge', 'celebrities',
'unassailable', 'bawling', 'wording', 'silencing', 'scotch',
'coincided', 'masochists', 'graphs', 'pernicious', 'disease',
'depreciates', 'later', 'torus', 'interject', 'mutated', 'causer',
'messy', 'Bechtel', 'redundantly', 'profoundest', 'autopsy',
'philosophic', 'iterate', 'Poisson', 'horridly', 'silversmith',
'millennium', 'plunder', 'salmon', 'missioner', 'advances', 'provers',
'earthliness', 'manor', 'resurrectors', 'Dahl', 'canto', 'gangrene',
'gabler', 'ashore', 'frictionless', 'expansionism', 'emphasis',
'preservations', 'Duane', 'descend', 'isolated', 'firmware',
'dynamites', 'scrawled', 'cavemen', 'ponder', 'prosperity', 'squaw',
'vulnerable', 'opthalmic', 'Simms', 'unite', 'totallers', 'Waring',
'enforced', 'bridge', 'collecting', 'sublime', 'Moore', 'gobble',
'criticizes', 'daydreams', 'sedate', 'apples', 'Concordia',
'subsequence', 'distill', 'Allan', 'seizure', 'Isadore', 'Lancashire',
'spacings', 'corresponded', 'hobble', 'Boonton', 'genuineness',
'artifact', 'gratuities', 'interviewee', 'Vladimir', 'mailable',
'Bini', 'Kowalewski', 'interprets', 'bereave', 'evacuated', 'friend',
'tourists', 'crunched', 'soothsayer', 'fleetly', 'Romanizations',
'Medicaid', 'persevering', 'flimsy', 'doomsday', 'trillion',
'carcasses', 'guess', 'seersucker', 'ripping', 'affliction',
'wildest', 'spokes', 'sheaths', 'procreate', 'rusticates', 'Schapiro',
'thereafter', 'mistakenly', 'shelf', 'ruination', 'bushel',
'assuredly', 'corrupting', 'federation', 'portmanteau', 'wading',
'incendiary', 'thing', 'wanderers', 'messages', 'Paso', 'reexamined',
'freeings', 'denture', 'potting', 'disturber', 'laborer', 'comrade',
'intercommunicating', 'Pelham', 'reproach', 'Fenton', 'Alva', 'oasis',
'attending', 'cockpit', 'scout', 'Jude', 'gagging', 'jailed',
'crustaceans', 'dirt', 'exquisitely', 'Internet', 'blocker', 'smock',
'Troutman', 'neighboring', 'surprise', 'midscale', 'impart',
'badgering', 'fountain', 'Essen', 'societies', 'redresses',
'afterwards', 'puckering', 'silks', 'Blakey', 'sequel', 'greet',
'basements', 'Aubrey', 'helmsman', 'album', 'wheelers', 'easternmost',
'flock', 'ambassadors', 'astatine', 'supplant', 'gird', 'clockwork',
'foxes', 'rerouting', 'divisional', 'bends', 'spacer',
'physiologically', 'exquisite', 'concerts', 'unbridled', 'crossing',
'rock', 'leatherneck', 'Fortescue', 'reloading', 'Laramie', 'Tim',
'forlorn', 'revert', 'scarcer', 'spigot', 'equality', 'paranormal',
'aggrieves', 'pegs', 'committeewomen', 'documented', 'interrupt',
'emerald', 'Battelle', 'reconverted', 'anticipated', 'prejudices',
'drowsiness', 'trivialities', 'food', 'blackberries', 'Cyclades',
'tourist', 'branching', 'nugget', 'Asilomar', 'repairmen', 'Cowan',
'receptacles', 'nobler', 'Nebraskan', 'territorial', 'chickadee',
'bedbug', 'darted', 'vigilance', 'Octavia', 'summands', 'policemen',
'twirls', 'style', 'outlawing', 'specifiable', 'pang', 'Orpheus',
'epigram', 'Babel', 'butyrate', 'wishing', 'fiendish', 'accentuate',
'much', 'pulsed', 'adorned', 'arbiters', 'counted', 'Afrikaner',
'parameterizes', 'agenda', 'Americanism', 'referenda', 'derived',
'liquidity', 'trembling', 'lordly', 'Agway', 'Dillon', 'propellers',
'statement', 'stickiest', 'thankfully', 'autograph', 'parallel',
'impulse', 'Hamey', 'stylistic', 'disproved', 'inquirer', 'hoisting',
'residues', 'variant', 'colonials', 'dequeued', 'especial', 'Samoa',
'Polaris', 'dismisses', 'surpasses', 'prognosis', 'urinates',
'leaguers', 'ostriches', 'calculative', 'digested', 'divided',
'reconfigurer', 'Lakewood', 'illegalities', 'redundancy',
'approachability', 'masterly', 'cookery', 'crystallized', 'Dunham',
'exclaims', 'mainline', 'Australianizes', 'nationhood', 'pusher',
'ushers', 'paranoia', 'workstations', 'radiance', 'impedes',
'Minotaur', 'cataloging', 'bites', 'fashioning', 'Alsop', 'servants',
'Onondaga', 'paragraph', 'leadings', 'clients', 'Latrobe',
'Cornwallis', 'excitingly', 'calorimetric', 'savior', 'tandem',
'antibiotics', 'excuse', 'brushy', 'selfish', 'naive', 'becomes',
'towers', 'popularizes', 'engender', 'introducing', 'possession',
'slaughtered', 'marginally', 'Packards', 'parabola', 'utopia',
'automata', 'deterrent', 'chocolates', 'objectives', 'clannish',
'aspirin', 'ferociousness', 'primarily', 'armpit', 'handfuls',
'dangle', 'Manila', 'enlivened', 'decrease', 'phylum', 'hardy',
'objectively', 'baskets', 'chaired', 'Sepoy', 'deputy', 'blizzard',
'shootings', 'breathtaking', 'sticking', 'initials', 'epitomized',
'Forrest', 'cellular', 'amatory', 'radioed', 'horrified', 'Neva',
'simultaneous', 'delimiter', 'expulsion', 'Himmler', 'contradiction',
'Remus', 'Franklinizations', 'luggage', 'moisture', 'Jews',
'comptroller', 'brevity', 'contradictions', 'Ohio', 'active',
'babysit', 'China', 'youngest', 'superstition', 'clawing', 'raccoons',
'chose', 'shoreline', 'helmets', 'Jeffersonian', 'papered',
'kindergarten', 'reply', 'succinct', 'split', 'wriggle', 'suitcases',
'nonce', 'grinders', 'anthem', 'showcase', 'maimed', 'blue', 'obeys',
'unreported', 'perusing', 'recalculate', 'rancher', 'demonic',
'Lilliputianize', 'approximation', 'repents', 'yellowness',
'irritates', 'Ferber', 'flashlights', 'booty', 'Neanderthal',
'someday', 'foregoes', 'lingering', 'cloudiness', 'guy', 'consumer',
'Berkowitz', 'relics', 'interpolating', 'reappearing', 'advisements',
'Nolan', 'turrets', 'skeletal', 'skills', 'mammas', 'Winsett',
'wheelings', 'stiffen', 'monkeys', 'plainness', 'braziers', 'Leary',
'advisee', 'jack', 'verb', 'reinterpret', 'geometrical', 'trolleys',
'arboreal', 'overpowered', 'Cuzco', 'poetical', 'admirations',
'Hobbes', 'phonemes', 'Newsweek', 'agitator', 'finally', 'prophets',
'environment', 'easterners', 'precomputed', 'faults', 'rankly',
'swallowing', 'crawl', 'trolley', 'spreading', 'resourceful', 'go',
'demandingly', 'broader', 'spiders', 'Marsha', 'debris', 'operates',
'Dundee', 'alleles', 'crunchier', 'quizzical', 'hanging', 'Fisk']
from ZODB.utils import u64
def incedit(edits, wait, ndel=20, nins=20):
import Zope, whrandom, string, time
from ZODB.POSException import ConflictError
rconflicts=wconflicts=0
did=str(edits.pop())
while edits:
if wait: time.sleep(whrandom.uniform(0,wait))
jar=Zope.DB.open()
app=jar.root()['Application']
doc=getattr(app.mail, did)
text=string.split(doc.raw)
n=whrandom.randint(0,ndel*2)
for j in range(n):
if len(text) < 2: break
j=whrandom.randint(0,len(text)-1)
#del text[j]
n=whrandom.randint(0,nins*2)
for j in range(n):
word=whrandom.choice(words)
text.append(word)
doc.raw=string.join(text)
try: app.cat.catalog_object(doc)
except ConflictError, v:
#print v.args
rconflicts=rconflicts+1
get_transaction().abort()
else:
try:
get_transaction().commit()
did=str(edits.pop())
except ConflictError:
wconflicts=wconflicts+1
get_transaction().abort()
doc=app=0
jar.close()
return rconflicts, wconflicts
def edit():
import Zope, thread
nedit, ndel, nins = atoi(sys.argv[2]), atoi(sys.argv[3]), atoi(sys.argv[4])
try: threads=atoi(sys.argv[5])
except:
threads=1
wait=0
else:
try: wait=atof(sys.argv[6])
except: wait=0.25
wait = wait * 2
if threads==1: start_new_thread=apply
else: start_new_thread=thread.start_new_thread
db=Zope.DB
app=Zope.app()
number_of_messages=app.mail.number_of_messages
app._p_jar.close()
size=db.getSize()
mem=VmSize()
t=time.time()
c=time.clock()
alledits={}
argss=[]
for i in range(threads):
lock=thread.allocate_lock()
if threads > 1:
lock.acquire()
def returnf(t, c, size, mem, r, lock=lock):
print c, r
lock.release()
else:
def returnf(t, c, size, mem, r, lock=lock):
print c, r
edits=[0]
while len(edits) <= nedit:
edit=whrandom.randint(0, number_of_messages)
if not alledits.has_key(edit):
alledits[edit]=1
edits.append(edit)
#print edits
argss.append((lock, (edits, wait, ndel, nins), returnf))
for lock, args, returnf in argss:
start_new_thread(do, (Zope.DB, incedit, args, returnf))
for lock, args, returnf in argss:
lock.acquire()
t=time.time() - t
c=time.clock() - c
size=db.getSize()-size
mem=VmSize()-mem
print t, c, size, mem
#hist("e%s" % (threads))
Zope.DB.close()
def VmSize():
try: f=open('/proc/%s/status' % os.getpid())
except: return 0
else:
l=filter(lambda l: l[:7]=='VmSize:', f.readlines())
if l:
l=string.split(string.strip(l[0][7:]))[0]
return string.atoi(l)
return 0
def pdebug():
import pdb
del sys.argv[1]
pdb.run('globals()[sys.argv[1]]()')
if __name__=='__main__':
try: f=globals()[sys.argv[1]]
except:
print __doc__
sys.exit(1)
else: f()
|