import UserList
import os.path,glob,copy,shutil,sys,time
import zipfile,string,re,md5
import locale
import ftplib
from StringIO import StringIO
#This must be adapted in accordance to your local settings.
#For western country, cp1252 is good
localecp="cp1252"
class EmptyObj:
pass
class Set:
"a class to manage sets. Duplicates are not taken into account"
def __init__(self,vals=[]):
self.vals={}
for elem in vals:
self.vals[str(elem)]=elem
def get(self):
ll=self.vals.values()
sortfct=lambda a,b: cmp(str(a),str(b))
ll.sort(sortfct)
return ll
def set(self,newset):
for elem in newset:
self.vals[str(elem)]=elem
def join(self,newset):
result={}
for elem in newset:
try:
self.vals.keys().index(str(elem))
result[str(elem)]=elem
except ValueError:
pass
self.vals=result
def merge(self,newset):
for elem in newset:
try:
self.vals.keys().index(str(elem))
except ValueError:
self.vals[str(elem)]=elem
def reduce(self,newset):
for elem in newset:
try:
del self.vals[str(elem)]
except:
pass
class Log:
def __init__(self,output=None):
self.setoutput(output)
def setoutput(self,output):
if output:
self.output=output
else:
self.output=sys.stdout
def format(self,txt):
tts=time.strftime('%c')
return '%s : %s\n' % (tts,txt)
def write(self,txt):
self.output.write(self.format(txt))
class Pickling:
def __getstate__(self):
dict=self.__dict__.copy()
for elem in dict.keys():
if elem[0]=="_":
del dict[elem]
return dict
def __setstate__(self,dict):
self.__class__.__init__(self,**dict)
class FileObj:
def __init__(self,file):
self.file=file.replace('\\','/') # cannot be update, it's an access point to the file
self.name=file.replace('\\','/') #you can change the name to what ever you want
def setlower(self):
self.name=self.name.lower()
def __str__(self):
return str(self.name)
def _trans2re(rule):
if not rule.strip(): return None
tmp=rule.replace('.','\.')
tmp=tmp.replace('[','\[')
tmp=tmp.replace(']','\]')
tmp=tmp.replace('*','.*')
tmp=tmp.replace('?','.{1}')
return re.compile(tmp)
class CommonDesc:
def __init__(self,dir,include_patt=['.*','*'],include_dirs=[],exclude_patt=[],exclude_dirs=[],info=Log(),error=Log(),options={}):
self.dir=dir
self.include_patt=include_patt
self.include_dirs=include_dirs
self.exclude_patt=exclude_patt
self.exclude_dirs=exclude_dirs
self._info=info
self._error=error
self.options=options
def init(self):
pass
def close(self):
pass
def _filterFiles(self):
"all files must be in the self._allfiles"
filters=EmptyObj()
filters.incpatt=[]
for elem in self.include_patt:
rule=_trans2re("^%s$" % elem)
if rule:
filters.incpatt.append(rule)
filters.incdirs=[]
for elem in self.include_dirs:
rule=_trans2re(elem)
if rule:
filters.incdirs.append(rule)
filters.excpatt=[]
for elem in self.exclude_patt:
rule=_trans2re("^%s$" % elem)
if rule:
filters.excpatt.append(rule)
filters.excdirs=[]
for elem in self.exclude_dirs:
rule=_trans2re(elem)
if rule:
filters.excdirs.append(rule)
res=[]
for file in self._allfiles:
dirname=os.path.dirname(file.name)
filename=os.path.basename(file.name)
to_include=False
to_exclude=False
good_dir=None
for rule in filters.incdirs:
if dirname and rule.search(dirname): good_dir=True
if good_dir==True or (good_dir==None and filters.incdirs==[]):
for rule in filters.incpatt:
if rule.search(filename): to_include=True
good_dir=None
for rule in filters.excdirs:
if dirname and rule.search(dirname): good_dir=True
if good_dir==True or (good_dir==None and filters.excdirs==[]):
for rule in filters.excpatt:
if dirname and rule.search(filename): to_exclude=True
if not to_include or to_exclude: continue
res.append(file)
return res
class DirDesc(Pickling,CommonDesc):
def _get_files(self,fct,dirname,fnames):
"based on include and exclude, return a list of files"
self._info.write('Analyzing directory %s' % dirname)
for file in fnames:
fpname=os.path.join(dirname,file)
#TODO: ONLY accept files, no links, no pipes, ...
if not os.path.isfile(fpname): continue
fpname=fpname.replace(os.path.join(self.dir,''),'')
obj=FileObj(fpname)
if fct:
obj.name=fct(obj.name)
self._allfiles.append(obj)
def getFiles(self,fct=None):
"return the list of files objects that match includes and excludes"
self._allfiles=[]
os.path.walk(self.dir,self._get_files,fct)
res=self._filterFiles()
return res
def getContent(self,file):
"return the bytes content of a specific file"
#TODO: manage the EOL: \n or \r\n
return open(os.path.join(self.dir,file.file),'rb').read()
def setMtime(self,file,mtime):
os.utime(os.path.join(self.dir,file.file),(mtime,mtime))
#cmd start: cmd actions. Must always receive file and content
def copyTo(self,newfile,filedata):
"cmd: copy the content to a new file.\nWrite the file as binary file"
self._info.write("copying %s to %s" % (newfile.file,self.dir))
fpname=os.path.join(self.dir,newfile.file)
try:
os.makedirs(os.path.dirname(fpname))
except:
pass
fid=open(fpname,'wb')
content,mtime=filedata
fid.write(content)
fid.close()
self.setMtime(newfile,mtime)
def deleteFile(self,file,dummy):
"cmd: delete a specific file"
self._info.write("deleting %s from %s" % (file.file,self.dir))
os.remove(os.path.join(self.dir,file.file))
path=os.path.dirname(os.path.join(self.dir,file.file))
prev_path=""
while path!=prev_path:
try:
os.removedirs(path)
except:
pass
prev_path=path
path=os.path.dirname(path)
#TODO:adapt the permissions parameters (user,group,rw...)
#cmd end.
def getMtime(self,file):
"get the modification time of a specific file"
ltime=time.localtime(os.path.getmtime(os.path.join(self.dir,file.file)))
return os.path.getmtime(os.path.join(self.dir,file.file))
def getSize(self,file):
"get the size of a specific file"
return os.path.getsize(os.path.join(self.dir,file.file))
def getMD5(self,file):
"get the md5 check sum of a specific file"
md=md5.new(self.getContent(file))
return md.digest()
class ZipDesc(Pickling,CommonDesc):
def init(self):
if os.path.isfile(self.dir):
self._zip=zipfile.ZipFile(self.dir,'a')
else:
self._zip=zipfile.ZipFile(self.dir,'w')
def close(self):
self._zip.close()
def getFiles(self,fct=None):
"return the list of files that match includes and exclude"
#we don't want duplicate
allfiles={}
for elem in self._zip.namelist():
obj=FileObj(unicode(elem,'cp437').encode(localecp))
if fct:
obj.name=fct(obj.name)
allfiles[obj.name]=obj
self._allfiles=allfiles.values()
res=self._filterFiles()
return res
def getContent(self,file):
"return the bytes content of a specific file"
filename=unicode(file.file,localecp).encode('cp437')
return self._zip.read(filename)
def copyTo(self,newfile,filedata):
"cmd: copy the content to a new file.\nWrite the file as binary file"
self._info.write('Copy in zip file:%s' % newfile)
content,mtime=filedata
zipinfo=zipfile.ZipInfo()
zipinfo.filename=unicode(newfile.file,localecp).encode('cp437')
timet=time.localtime(mtime)
zipinfo.date_time=timet[:6]
zipinfo.compress_type=8
self._zip.writestr(zipinfo,content)
def deleteFile(self,file,dummy):
"cmd: delete a specific file"
#not possible for zip files
pass
def getMtime(self,file):
"get the modification time of a specific file"
filename=unicode(file.file,localecp).encode('cp437')
zipinfo=self._zip.getinfo(filename)
ltime=list(zipinfo.date_time)
ltime.extend([0,0,-1])
return time.mktime(ltime)
def getSize(self,file):
"get the size of a specific file"
filename=unicode(file.file,localecp).encode('cp437')
zipinfo=self._zip.getinfo(filename)
return zipinfo.file_size
def getMD5(self,file):
"get the md5 check sum of a specific file"
md=md5.new(self.getContent(file))
return md.digest()
class FtpDesc(Pickling,CommonDesc):
def __init__(self,dir,include_patt=['.*','*'],include_dirs=[],exclude_patt=[],exclude_dirs=[],info=Log(),error=Log(),options={}):
self.dir=dir
self.include_patt=include_patt
self.include_dirs=include_dirs
self.exclude_patt=exclude_patt
self.exclude_dirs=exclude_dirs
self._info=info
self._error=error
self.options=options
def init(self):
self._host=self.options['host']
self._user=self.options['user']
self._passwd=self.options['passwd']
self._rootdir=self.options['rootdir']
if self.options['rootdir'][-1]=="/":
self._rootdir=self.options['rootdir'][:-1]
self._ftp=ftplib.FTP()
if len(self._host.split(':'))==2:
hostname,port=self._host.split(':')
else:
hostname=self._host
port=21
self._ftp.connect(hostname,port)
self._ftp.login(self._user,self._passwd)
def close(self):
self._ftp.close()
def _get_ftp_content(self,dir,fct,files=[]):
self._ftp.cwd(self._rootdir)
try:
elems=self._ftp.nlst(dir)
except:
elems=[]
self._error.write("error nlst: %s" % dir)
self._info.write("Analyzing directory: %s" % dir)
dirs=[]
for elem in elems:
if not elem.startswith(dir):
fpname=dir+"/"+elem
else:
fpname=elem
try:
self._ftp.cwd(fpname)
type="DIR"
except:
type="FILE"
if type=="FILE":
obj=FileObj(elem)
if fct:
obj.name=fct(obj.name)
files.append(obj)
if type=="DIR":
self._ftp.cwd(self._rootdir)
dirs.append(elem)
for newdir in dirs:
self._get_ftp_content(newdir,fct,files)
def getFiles(self,fct=None):
"return the list of files that match includes and exclude"
self._ftp.cwd(self.dir)
allfiles=[]
self._get_ftp_content(self.dir,fct,allfiles)
self._allfiles=allfiles
res=self._filterFiles()
return res
def getContent(self,file):
"return the bytes content of a specific file"
content=[]
self._ftp.retrbinary('RETR %s' % self._rootdir+"/"+file.file,content.append)
return ''.join(content)
def copyTo(self,newfile,filedata):
"cmd: copy the content to a new file.\nWrite the file as binary file"
self._info.write('Copy on ftp server file:%s' % newfile)
content,mtime=filedata
contentobj=StringIO(content)
self._ftp.storbinary('STOR %s' % self._rootdir+"/"+newfile.file,contentfid)
#FTP does NOT allow us to force mtime
def deleteFile(self,file,dummy):
"cmd: delete a specific file"
self._ftp.delete(self._rootdir+"/"+file.file)
def getMtime(self,file):
"get the modification time of a specific file"
mtime=self._ftp.sendcmd('MDTM %s' % self._rootdir+"/"+file.file).split()[1]
ltime=(int(mtime[0:4]),int(mtime[4:6]),int(mtime[6:8]),int(mtime[8:10]),int(mtime[10:12]),int(mtime[12:14]),0,0,-1)
return time.mktime(ltime)
def getSize(self,file):
"get the size of a specific file"
return self._ftp.size(self._rootdir+"/"+file.file)
def getMD5(self,file):
"get the md5 check sum of a specific file"
md=md5.new(self.getContent(file))
return md.digest()
class Job(Pickling):
def __init__(self,local,remote,options={}):
#local and remote must be DirDesc objects
self.local=local
self.remote=remote
self.options=options
self.set_default()
def set_default(self):
self.options.setdefault('remote2local',0)
self.options.setdefault('local2remote',0)
self.options.setdefault('delete',0)
self.options.setdefault('delta',0)
self.options.setdefault('casesensitive',True)
if self.options['delete']==0:
self.options.setdefault('remote2local',1)
self.options.setdefault('local2remote',1)
if self.options['remote2local'] and self.options['local2remote'] and self.options['delete']:
raise "Error in optons parameters"
def getlocalinputs(self):
return {'dir':self.local.dir,'include_patt':self.local.include_patt,'include_dirs':self.local.include_dirs,'exclude_patt':self.local.exclude_patt,'exclude_dirs':self.local.exclude_dirs}
def getremoteinputs(self):
return {'dir':self.remote.dir,'include_patt':self.remote.include_patt,'include_dirs':self.remote.include_dirs,'exclude_patt':self.remote.exclude_patt,'exclude_dirs':self.remote.exclude_dirs}
class FileComp:
def __init__(self,jobcard,info=Log(),error=Log()):
self.commands={}
self.commands[0]=[]
self.commands[1]=[]
self.commands[2]=[]
self.commands[3]=[]
self.commands[4]=[]
#actionlist is the list of results: [file,todir,cmd to get content,cmd action,text]
self.actionlist=[]
self._info=info
self._error=error
self.jobcard=jobcard
self._analyzed=False
def init(self):
"we just initialize local and remote "
self.jobcard.local.init()
self.jobcard.remote.init()
def close(self):
self.jobcard.local.close()
self.jobcard.remote.close()
def getSets(self):
fct=None
if not self.jobcard.options['casesensitive']:
fct=string.lower
alllocalfiles=self.jobcard.local.getFiles(fct)
allremotefiles=self.jobcard.remote.getFiles(fct)
shared=Set(alllocalfiles)
shared.join(allremotefiles)
purelocalfiles=Set(alllocalfiles)
purelocalfiles.reduce(shared.get())
pureremotefiles=Set(allremotefiles)
pureremotefiles.reduce(shared.get())
self.purelocal=purelocalfiles.get()
self.pureremote=pureremotefiles.get()
self.shared=shared.get()
self._info.write("Comparing directories via the methode : %s" % self.__class__.__name__ )
self._info.write("Pure local: %s " % ", ".join([str(elem.name) for elem in purelocalfiles.get()]))
self._info.write("Pure remote: %s " % ", ".join([str(elem.name) for elem in pureremotefiles.get()]))
self._info.write("shared files: %s " % ", ".join([str(elem.name) for elem in shared.get()]))
def analyze(self):
#must return :
# priority cmd text
#example:
# 2 'copy a, b' 'copy to remote'
# 2 'rm a' 'rmove from remote'
# 3 'rmdir a' 'remove directory from remote'
# 1 'mkdir a' 'make dir on local'
# 0 None ''
raise "Metha class. You should not call this one directly"
def getResults(self):
self.analyze()
self._analyzed=True
self.actionlist=[]
for j in range(5):
if self.commands.has_key(j):
for card in self.commands[j]:
self.actionlist.append(card)
return self.actionlist
def __len__(self):
if self._analyzed:
return len(self.actionlist)
else:
return None
def sync(self,confirmationlist=None):
if not self._analyzed: self.getResults()
if confirmationlist==None:
confirmationlist=[1]*len(self.actionlist)
i=0
results=[]
for rec in self.actionlist:
res="skipped"
if confirmationlist[i]:
try:
res="OK"
if rec[2]:
contentobj,mtime=rec[2]
content=contentobj(rec[0])
rec[3](rec[0],(content,mtime))
else:
rec[3](rec[0],None)
except:
type,value,tracebck=sys.exc_info()
res="%s, %s" % (str(type),str(value))
results.append(res)
i+=1
return results
class DateComp(FileComp):
def analyze(self):
self.getSets()
self.commands[2]=[]
if self.jobcard.options['local2remote']:
local2remotelist=copy.copy(self.purelocal)
for file in self.shared:
try:
loctime=self.jobcard.local.getMtime(file)
except:
self._error.write("ERROR to get local Mtime of %s:%s" % (file.name,sys.exc_info()[1]))
continue
try:
remtime=self.jobcard.remote.getMtime(file)
except:
self._error.write("ERROR to get remote Mtime of %s:%s" % (file.name,sys.exc_info()[1]))
continue
self._info.write('for file %s, Date delta is %s (max is %s)' % (file.name, loctime-remtime,self.jobcard.options['delta']))
if (loctime-remtime) > self.jobcard.options['delta']:
local2remotelist.append(file)
for file in local2remotelist:
self.commands[2].append([file,self.jobcard.remote.dir,[self.jobcard.local.getContent,self.jobcard.local.getMtime(file)],self.jobcard.remote.copyTo,self.jobcard.local.dir,'copy to remote'])
if self.jobcard.options['delete']:
for file in self.pureremote:
self.commands[2].append([file,self.jobcard.remote.dir,None,self.jobcard.remote.deleteFile,self.jobcard.remote.dir, 'delete on remote'])
if self.jobcard.options['remote2local']:
remote2locallist=copy.copy(self.pureremote)
for file in self.shared:
try:
loctime=self.jobcard.local.getMtime(file)
except:
self._error.write("ERROR to get local Mtime of %s:%s" % (file.name,sys.exc_info()[1]))
continue
try:
remtime=self.jobcard.remote.getMtime(file)
except:
self._error.write("ERROR to get remote Mtime of %s:%s" % (file.name,sys.exc_info()[1]))
continue
self._info.write('for file %s, Date delta is %s (max is %s)' % (file.name, remtime-loctime,self.jobcard.options['delta']))
if (remtime-loctime) > self.jobcard.options['delta']:
remote2locallist.append(file)
for file in remote2locallist:
self.commands[2].append([file,self.jobcard.local.dir,[self.jobcard.remote.getContent,self.jobcard.remote.getMtime(file)],self.jobcard.local.copyTo,self.jobcard.remote.dir,'copy to local'])
if self.jobcard.options['delete']:
for file in self.purelocal:
self.commands[2].append([file,self.jobcard.local.dir,None,self.jobcard.local.deleteFile,self.jobcard.local.dir, 'delete on local'])
self._info.write("Analysis done!!")
class SizeComp(FileComp):
def analyze(self):
self.getSets()
self.commands[2]=[]
if self.jobcard.options['local2remote']:
local2remotelist=copy.copy(self.purelocal)
for file in self.shared:
try:
locsize=self.jobcard.local.getSize(file)
except:
self._error.write("ERROR to get locale size of %s:%s" % (file.name,sys.exc_info()[1]))
continue
try:
remsize=self.jobcard.remote.getSize(file)
except:
self._error.write("ERROR to get remote size of %s:%s" % (file.name,sys.exc_info()[1]))
continue
self._info.write('for file %s, Size delta is %s (max is %s)' % (file.name, locsize-remsize,self.jobcard.options['delta']))
if (locsize-remsize) > self.jobcard.options['delta']:
local2remotelist.append(file)
for file in local2remotelist:
self.commands[2].append([file,self.jobcard.remote.dir,[self.jobcard.local.getContent,self.jobcard.local.getMtime(file)],self.jobcard.remote.copyTo,self.jobcard.local.dir,'copy to remote'])
if self.jobcard.options['delete']:
for file in self.pureremote:
self.commands[2].append([file,self.jobcard.remote.dir,None,self.jobcard.remote.deleteFile,self.jobcard.remote.dir, 'delete on remote'])
if self.jobcard.options['remote2local']:
remote2locallist=copy.copy(self.pureremote)
for file in self.shared:
try:
locsize=self.jobcard.local.getSize(file)
except:
self._error.write("ERROR to get locale size of %s:%s" % (file.name,sys.exc_info()[1]))
continue
try:
remsize=self.jobcard.remote.getSize(file)
except:
self._error.write("ERROR to get remote size of %s:%s" % (file.name,sys.exc_info()[1]))
continue
self._info.write('for file %s, Size delta is %s (max is %s)' % (file.name, remsize-locsize,self.jobcard.options['delta']))
if (remsize-locsize) > self.jobcard.options['delta']:
remote2locallist.append(file)
for file in remote2locallist:
self.commands[2].append([file,self.jobcard.local.dir,[self.jobcard.remote.getContent,self.jobcard.remote.getMtime(file)],self.jobcard.local.copyTo,self.jobcard.remote.dir,'copy to local'])
if self.jobcard.options['delete']:
for file in self.purelocal:
self.commands[2].append([file,self.jobcard.local.dir,None,self.jobcard.local.deleteFile,self.jobcard.local.dir, 'delete on local'])
self._info.write("Analysis done!!")
class MD5Comp(FileComp):
def analyze(self):
self.getSets()
self.commands[2]=[]
if self.jobcard.options['local2remote']:
local2remotelist=copy.copy(self.purelocal)
for file in self.shared:
try:
locmd5=self.jobcard.local.getMD5(file)
except:
self._error.write("ERROR to compute local md5 of %s:%s" % (file.name,sys.exc_info()[1]))
continue
try:
remmd5=self.jobcard.remote.getMD5(file)
except:
self._error.write("ERROR to compute remote md5 of %s:%s" % (file.name,sys.exc_info()[1]))
continue
if locmd5!=remmd5:
self._info.write('checking MD5 for file %s: DIFFERENT' % (file.name))
local2remotelist.append(file)
else:
self._info.write('checking MD5 for file %s: equal' % (file.name))
for file in local2remotelist:
self.commands[2].append([file,self.jobcard.remote.dir,[self.jobcard.local.getContent,self.jobcard.local.getMtime(file)],self.jobcard.remote.copyTo,self.jobcard.local.dir,'copy to remote'])
if self.jobcard.options['delete']:
for file in self.pureremote:
self.commands[2].append([file,self.jobcard.remote.dir,None,self.jobcard.remote.deleteFile,self.jobcard.remote.dir, 'delete on remote'])
if self.jobcard.options['remote2local']:
remote2locallist=copy.copy(self.pureremote)
for file in self.shared:
try:
locmd5=self.jobcard.local.getMD5(file)
except:
self._error.write("ERROR to compute local md5 of %s:%s" % (file.name,sys.exc_info()[1]))
continue
try:
remmd5=self.jobcard.remote.getMD5(file)
except:
self._error.write("ERROR to compute remote md5 of %s:%s" % (file.name,sys.exc_info()[1]))
continue
if locmd5!=remmd5:
self._info.write('checking MD5 for file %s: DIFFERENT' % (file.name))
remote2locallist.append(file)
else:
self._info.write('checking MD5 for file %s: equal' % (file.name))
for file in remote2locallist:
self.commands[2].append([file,self.jobcard.local.dir,[self.jobcard.remote.getContent,self.jobcard.remote.getMtime(file)],self.jobcard.local.copyTo,self.jobcard.remote.dir,'copy to local'])
if self.jobcard.options['delete']:
for file in self.purelocal:
self.commands[2].append([file,self.jobcard.local.dir,None,self.jobcard.local.deleteFile,self.jobcard.local.dir, 'delete on local'])
self._info.write("Analysis done!!")
|