##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Simple script for repairing damaged FileStorage files.
Usage: %s [-f] input output
Recover data from a FileStorage data file, skipping over damaged
data. Any damaged data will be lost. This could lead to useless output
of critical data were lost.
Options:
-f
Force output to putput file even if it exists
-v level
Set the verbosity level:
0 -- Show progress indicator (default)
1 -- Show transaction times and sizes
2 -- Show transaction times and sizes, and
show object (record) ids, versions, and sizes.
-p
Copy partial transactions. If a data record in the middle of a
transaction is bad, the data up to the bad data are packed. The
output record is marked as packed. If this option is not used,
transaction with any bad data are skipped.
-P t
Pack data to t seconds in the past. Note that is the "-p"
option is used, then t should be 0.
Important note: The ZODB package must be imporable. You may need
to adjust the Python path accordingly.
"""
# Algorithm:
#
# position to start of input
# while 1:
# if end of file: break
# try: copy_transaction
# except:
# scan for transaction
# continue
import sys, os
if __name__ == '__main__' and len(sys.argv) < 3:
print __doc__ % sys.argv[0]
def die(mess=''):
if not mess: mess="%s: %s" % sys.exc_info()[:2]
print mess+'\n'
sys.exit(1)
try: import ZODB
except ImportError:
if os.path.exists('ZODB'): sys.path.append('.')
elif os.path.exists('FileStorage.py'): sys.path.append('..')
import ZODB
import getopt, ZODB.FileStorage, struct, time
from struct import unpack
from ZODB.utils import t32,p64,U64
from ZODB.TimeStamp import TimeStamp
from cPickle import loads
from ZODB.FileStorage import RecordIterator
class EOF(Exception): pass
class ErrorFound(Exception): pass
def error(mess, *args):
raise ErrorFound(mess % args)
def read_transaction_header(file, pos, file_size):
# Read the transaction record
seek=file.seek
read=file.read
seek(pos)
h=read(23)
if len(h) < 23: raise EOF
tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
if el < 0: el=t32-el
tl=U64(stl)
if status=='c': raise EOF
if pos+(tl+8) > file_size:
error("bad transaction length at %s", pos)
if status not in ' up':
error('invalid status, %s, at %s', status, pos)
if tl < (23+ul+dl+el):
error('invalid transaction length, %s, at %s', tl, pos)
tpos=pos
tend=tpos+tl
if status=='u':
# Undone transaction, skip it
seek(tend)
h=read(8)
if h != stl: error('inconsistent transaction length at %s', pos)
pos=tend+8
return pos, None
pos=tpos+(23+ul+dl+el)
user=read(ul)
description=read(dl)
if el:
try: e=loads(read(el))
except: e={}
else: e={}
result=RecordIterator(
tid, status, user, description, e,
pos, (tend, file, seek, read,
tpos,
)
)
pos=tend
# Read the (intentionally redundant) transaction length
seek(pos)
h=read(8)
if h != stl:
error("redundant transaction length check failed at %s", pos)
pos=pos+8
return pos, result
def scan(file, pos, file_size):
seek=file.seek
read=file.read
while 1:
seek(pos)
data=read(8096)
if not data: return 0
s=0
while 1:
l=data.find('.', s)
if l < 0:
pos=pos+8096
break
if l > 8080:
pos = pos + l
break
s=l+1
tl=U64(data[s:s+8])
if tl < pos:
return pos + s + 8
def iprogress(i):
if i%2: print '.',
else: print (i/2)%10,
sys.stdout.flush()
def progress(p):
for i in range(p): iprogress(i)
def recover(argv=sys.argv):
try:
opts, (inp, outp) = getopt.getopt(argv[1:], 'fv:pP:')
force = partial = verbose = 0
pack = None
for opt, v in opts:
if opt == '-v': verbose = int(v)
elif opt == '-p': partial=1
elif opt == '-f': force=1
elif opt == '-P': pack=time.time()-float(v)
force = filter(lambda opt: opt[0]=='-f', opts)
partial = filter(lambda opt: opt[0]=='-p', opts)
verbose = filter(lambda opt: opt[0]=='-v', opts)
verbose = verbose and int(verbose[0][1]) or 0
print 'Recovering', inp, 'into', outp
except:
die()
print __doc__ % argv[0]
if os.path.exists(outp) and not force:
die("%s exists" % outp)
file=open(inp, "rb")
seek=file.seek
read=file.read
if read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage")
seek(0,2)
file_size=file.tell()
ofs=ZODB.FileStorage.FileStorage(outp, create=1)
_ts=None
ok=1
prog1=0
preindex={}; preget=preindex.get # waaaa
undone=0
pos=4
while pos:
try:
npos, transaction = read_transaction_header(file, pos, file_size)
except EOF:
break
except:
print "\n%s: %s\n" % sys.exc_info()[:2]
if not verbose: progress(prog1)
pos = scan(file, pos, file_size)
continue
if transaction is None:
undone = undone + npos - pos
pos=npos
continue
else:
pos=npos
tid=transaction.tid
if _ts is None:
_ts=TimeStamp(tid)
else:
t=TimeStamp(tid)
if t <= _ts:
if ok: print ('Time stamps out of order %s, %s' % (_ts, t))
ok=0
_ts=t.laterThan(_ts)
tid=`_ts`
else:
_ts = t
if not ok:
print ('Time stamps back in order %s' % (t))
ok=1
if verbose:
print 'begin',
if verbose > 1: print
sys.stdout.flush()
ofs.tpc_begin(transaction, tid, transaction.status)
if verbose:
print 'begin', pos, _ts,
if verbose > 1: print
sys.stdout.flush()
nrec=0
try:
for r in transaction:
oid=r.oid
if verbose > 1: print U64(oid), r.version, len(r.data)
pre=preget(oid, None)
s=ofs.store(oid, pre, r.data, r.version, transaction)
preindex[oid]=s
nrec=nrec+1
except:
if partial and nrec:
ofs._status='p'
ofs.tpc_vote(transaction)
ofs.tpc_finish(transaction)
if verbose: print 'partial'
else:
ofs.tpc_abort(transaction)
print "\n%s: %s\n" % sys.exc_info()[:2]
if not verbose: progress(prog1)
pos = scan(file, pos, file_size)
else:
ofs.tpc_vote(transaction)
ofs.tpc_finish(transaction)
if verbose:
print 'finish'
sys.stdout.flush()
if not verbose:
prog = pos * 20l / file_size
while prog > prog1:
prog1 = prog1 + 1
iprogress(prog1)
bad = file_size - undone - ofs._pos
print "\n%s bytes removed during recovery" % bad
if undone:
print "%s bytes of undone transaction data were skipped" % undone
if pack is not None:
print "Packing ..."
from ZODB.referencesf import referencesf
ofs.pack(pack, referencesf)
ofs.close()
if __name__=='__main__': recover()
|