# Part of the A-A-P recipe executive: copy and move files (remotely)
# Copyright (C) 2002-2003 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here: http://www.a-a-p.org/COPYING
#
# These are functions to copy and move files. Not only on the local system,
# also to remote systems, using http://, ftp://, etc.
#
import os
import os.path
import string
import re
from Dictlist import str2dictlist
from Error import *
from Process import recipe_error,option_error
from Util import *
from Work import getrpstack
from Message import *
def copy_move(line_nr, recdict, arg, copy):
"""Implementation of ":copy -x from to" and ":move -x from to".
When "copy" is non-zero copying is done, otherwise moving.
"arg" is the whole argument.
"line_nr" is used for error messages."""
# Skip when not actually building.
cmdname = (copy and ':copy') or ':move'
if skip_commands():
msg_skip(line_nr, recdict, cmdname + ' ' + arg)
return
rpstack = getrpstack(recdict, line_nr)
# Get the arguments and check the options.
# TODO: check if all options are handled properly
from Commands import get_args
opt = {"f": "force", "force": "force",
"i": "interactive", "interactive": "interactive",
"e": "exist", "exist": "exist", "exists": "exist",
"m": "mkdir", "mkdir": "mkdir",
"c": "continue", "continue": "continue",
"q": "quiet", "quiet": "quiet"}
if copy:
opt.update({
"u": "unlink", "unlink": "unlink",
"p": "preserve", "preserve": "preserve",
"r": "recursive", "recursive": "recursive",
"k": "keepdir", "keepdir": "keepdir"})
optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg, opt)
if attrdict:
option_error(rpstack, attrdict, cmdname)
if len(argdictlist) < 2:
recipe_error(rpstack, _("%s command requires at least two arguments")
% cmdname)
remote_copy_move(rpstack, recdict, copy, argdictlist[:-1],
argdictlist[-1], optiondict, 1, errmsg = 1)
# Schemes handled by scheme_copy(). When changing this also need to change
# code and comments below!
scheme_copy_names = ["rcp", "scp", "rsync"]
def scheme_copy(rpstack, recdict, scheme, fromfile, tmach, destpath, fcount):
"""
Use rcp, scp or rsync, specified with "scheme" to copy "fromfile" (with
"fcount" file names in quotes) to tmach:destpath.
Return non-zero for success.
"""
# Install rcp/scp/rsync when needed.
from DoInstall import assert_pkg
assert_pkg(rpstack, recdict, scheme)
from Remote import get_progname_rsync,get_progname_rcp,get_progname_scp
if scheme == "rcp":
cmd = "%s %s '%s:%s'" % (get_progname_rcp(recdict),
fromfile, tmach, destpath)
elif scheme == "scp":
cmd = "%s %s '%s:%s'" % (get_progname_scp(recdict),
fromfile, tmach, destpath)
else:
# A bug in rsync: it can't handle "file(with)parenthesis" in the
# destination, but escaping them in the source is not allowed!
d = re.sub("([()])", r"\\\1", destpath)
cmd = "%s %s '%s:%s'" % (get_progname_rsync(recdict),
fromfile, tmach, d)
if os.name == "posix":
# On Unix we can use "tee".
ok, text = redir_system_int(recdict, cmd)
else:
# Scp may ask for a password. When we can't use "tee" we need to
# see the message. Can't catch the text output then...
ok = logged_system(recdict, "{interactive} " + cmd) == 0
text = ''
if text:
msg_log(recdict, text)
# Check for an error connecting, scp doesn't exit with an error value
# then. Usual messages are:
# "Secure connection to vim.sf.net refused."
# "lost connection"
# "Read-only file system"
i = string.find(text, "connection to")
if i > 0:
e = string.find(text, "\n", i)
if e > 0:
i = string.find(text, "refused", i, e)
else:
i = -1
if i > 0 or string.find(text, "lost connection") >= 0:
ok = 0
elif string.find(text, "Permission denied") >= 0:
# Happens when a file exists and permissions don't allow
# overwriting: scp: vim/new/con_cvs.php: Permission denied
# TODO: check the file name and only mark this one as failed.
ok = 0
elif string.find(text, "Read-only file system") >= 0:
# Happens when the file system was mounted read-only.
# TODO: check the file name and only mark this one as failed.
ok = 0
elif string.find(text, "Not a directory") >= 0:
# Happens when copying multiple files into a file.
ok = 0
elif string.find(text, "No such file or directory") >= 0:
ok = 0
msg_info(recdict,
_('looks like the directory does not exist, creating it'))
# TODO: This is just guessing; how do we know if the destination
# is a directory or a file?
if fcount > 1:
dirname = destpath
else:
dirname = os.path.dirname(destpath)
mkdircmd = ('ssh %s mkdir -p %s' % (tmach, dirname))
if logged_system(recdict, mkdircmd) == 0:
# Try copying again.
ok = (logged_system(recdict, cmd) == 0)
return ok
# Cached ftp connections.
ftp_conn = {}
did_set_exitfunc = 0
old_exitfunc = None
def ftpCloseAll():
"""
Close all cached ftp connections. Invoked on exit.
"""
import ftplib
if old_exitfunc:
old_exitfunc() # Invoke previous exit function.
global ftp_conn
for ftp in ftp_conn.values():
try:
ftp.quit()
except ftplib.all_errors, e:
msg_log({}, _("could not quit an ftp connection: %s") % e)
ftp_conn = {}
# Cached passwords for ftp connections.
ftp_pass = {}
def ftpConnect(mach):
"""
Connect to an ftp server "mach".
If "mach" is in the form "user:passwd@ftp.com" use that.
Otherwise consult the netrc file or ask the user.
Returns a tuple. The first item is an ftp object.
If the connection was successful the second item is an empty string.
If the connection failed the second item is an error message.
"""
# Use a cached connection if there is one.
if mach in ftp_conn.keys():
return ftp_conn[mach], ''
passwd = ''
acct = ''
key = None
msg = ''
# For "user:passwd@ftp.com" split at the @
i = string.find(mach, '@')
if i > 0:
user = mach[:i]
machname = mach[i+1:]
i = string.find(user, ':')
if i > 0:
passwd = user[i+1:]
user = user[:i]
else:
key = user + "@" + machname
passwd = ftp_pass.get(key)
if not passwd:
prompt = (_('Enter password for user %s at %s: ')
% (user, machname))
try:
import getpass
passwd = getpass.getpass(prompt)
except:
# TODO: should display stars for typed chars
passwd = raw_input(prompt)
else:
user = ''
machname = mach
import aapnetrc
# obtain the login name and password from the netrc file
try:
n = aapnetrc.netrc()
res = n.authenticators(machname)
if res is None:
user = ''
else:
user, acct, passwd = res
except (IOError, aapnetrc.NetrcParseError), e:
pass
# Try to open the connection to the ftp server.
import ftplib
try:
ftp = ftplib.FTP()
i = string.find(machname, ':')
if i > 0:
# Port number specified.
ftp.connect(machname[0:i], machname[i+1:])
else:
# Use default port number.
ftp.connect(machname)
if user != '':
if passwd != '':
if acct != '':
ftp.login(user, passwd, acct)
else:
ftp.login(user, passwd)
else:
ftp.login(user)
else:
ftp.login()
except ftplib.all_errors, e:
msg = (_('Cannot open connection to "%s": %s') % (mach, str(e)))
else:
if key and passwd:
# Remember the typed password, so that the user doesn't have to
# type it again.
ftp_pass[key] = passwd
# Cache the connection for another time.
ftp_conn[mach] = ftp
# Setup the exit function to close the connection.
global did_set_exitfunc
global old_exitfunc
if not did_set_exitfunc:
did_set_exitfunc = 1
old_exitfunc = getattr(sys, 'exitfunc', None)
sys.exitfunc = ftpCloseAll
return ftp, msg
def ftp_may_mkdir(recdict, error, ftp, destpath):
"""If "error" indicates we failed to upload through ftp because the
directory doesn't exist try creating that directory. This works
recursively.
Return non-zero if the directory was created."""
if string.find(str(error), "No such file or dir") >= 0:
adir = os.path.dirname(destpath)
if len(adir) < len(destpath):
import ftplib
msg_info(recdict, _('Directory "%s" does not appear to exist, creating it...') % adir)
try:
ftp.mkd(adir)
except ftplib.all_errors, e:
# Maybe the upper directory doesn't exist, try creating it.
if ftp_may_mkdir(recdict, e, ftp, adir):
try:
ftp.mkd(adir)
except ftplib.all_errors, e:
msg_info(recdict, _('Could not create directory "%s"') % adir)
return 0
else:
return 0
return 1
return 0
def remote_copy_move(rpstack, recdict, copy, from_items, to_item,
optiondict, argexpand, errmsg = 0):
"""Copy or move command. Copy when "copy" is non-zero.
"from_items" is a dictlist of the files to copy or move from.
"to_item" is the dictionary of the destination.
"optiondict" is a dictionary with options:
recursive: recursive copy.
exist: skip if destination exists
interactive: ask about overwriting.
preserve: keep mode bits and timestamp
quiet: don't report copied/moved files
continue: ignore wildcards without matches
mkdir: create directory when needed
"argexpand" is non-zero when wildcards are to be expanded (called from
":copy" or ":move").
When "errmsg" is non-zero, call recipe_error() for an error.
Returns a list of filenames that failed."""
from Remote import url_split3
import glob
failed = []
# -l: change symlinks to real files
#if optiondict.get("unlink"):
# keepsymlinks = 0
#else:
# keepsymlinks = 1
# Expand and check the "from" arguments.
fromlist = []
for a in from_items:
fname = a["name"]
fscheme, fmach, fpath = url_split3(fname)
if fscheme == '':
# It's a local file, may expand ~user and wildcards.
f = os.path.expanduser(fname)
if argexpand:
fl = glob.glob(f)
if len(fl) == 0:
if optiondict.get("continue") and has_wildcard(f):
continue
recipe_error(rpstack, _('No match for "%s"') % fname)
else:
fl = [ f ]
# For copy without {r} sources can't be a directory.
if copy and not optiondict.get("recursive"):
for l in fl:
if os.path.isdir(l):
recipe_error(rpstack,
_('Copying a directory requires {r} flag: %s') % l)
fromlist.extend(fl)
else:
# It's a URL, append without expanding.
if not copy:
recipe_error(rpstack, _('Cannot move from a URL yet: "%s"')
% fname)
if argexpand:
# Do change [*] to *, we might support expanding later.
f = expand_unescape(fname)
else:
f = fname
fromlist.append(f)
# Expand and check the "to" argument.
tname = to_item["name"]
tscheme, tmach, tpath = url_split3(tname)
if tscheme == '':
# For a local file expand "~/" and "~user/". Don't use "tname", it may
# start with "file:".
tpath = os.path.expanduser(tpath)
if argexpand:
if tscheme == '':
# For a local destination file expand wildcards. If expanding
# results in no matches use the name without expansion.
l = glob.glob(tpath)
if len(l) > 1:
recipe_error(rpstack, _('More than one match for "%s"') % tname)
if len(l) == 1:
tpath = l[0]
else:
# Can't expand wildcards in a URL yet, at least remove escaped
# wildcards.
tpath = expand_unescape(tpath)
if tscheme == '' and optiondict.get("mkdir"):
# Create local target directory when it doesn't exist.
if len(fromlist) > 1:
dirpath = tpath;
else:
dirpath = os.path.dirname(tpath)
may_create_dir(recdict, rpstack, dirpath, optiondict.get("quiet"))
# If there is more than one source, target must be a directory.
if tscheme == '' and len(fromlist) > 1 and not os.path.isdir(tpath):
recipe_error(rpstack, _('Destination must be a directory: "%s"')
% tname)
msg = ''
if tscheme == "ftp":
#
# Prepare for uploading through ftp.
#
ftp, msg = ftpConnect(tmach)
elif tscheme in scheme_copy_names:
#
# TODO: Prepare for uploading through rcp/scp/rsync.
#
pass
elif tscheme != '':
msg = _('Can only upload to rcp://, scp://, rsync:// and ftp://')
# If copy/move isn't possible either give an error message or return the
# whole list of sources.
if msg:
if argexpand:
msg_error(recdict, msg)
return map(lambda x : x["name"], from_items)
recipe_error(rpstack, msg)
# files to be copied with scp or rsync later.
scheme_files = {}
scheme_filelist = {}
for scheme in scheme_copy_names:
scheme_files[scheme] = ''
scheme_filelist[scheme] = []
#
# Loop over all "from" files.
#
for fname in fromlist:
fscheme, fmach, fpath = url_split3(fname)
# If the destination is a directory, append the source file name to the
# destination directory.
# "dest" includes the scheme (may be "file:" when "tscheme" is empty).
if ((tscheme != '' and len(fromlist) > 1)
or (tscheme == '' and os.path.isdir(tpath))):
if optiondict.get("keepdir"):
dest = os.path.join(tname, fpath)
destpath = os.path.join(tpath, fpath)
if tscheme == '' and os.path.dirname(fpath):
# Create the directory if it doesn't exist.
dirpath = os.path.dirname(destpath)
may_create_dir(recdict, rpstack, dirpath,
optiondict.get("quiet"))
else:
dest = os.path.join(tname, os.path.basename(fname))
destpath = os.path.join(tpath, os.path.basename(fname))
else:
dest = tname
destpath = tpath
# If destination is a local file and exists:
# - When {exist} option used, silently ignore.
# - When {interactive} option used, ask for overwriting. Use a special
# string to allow translating the response characters.
if tscheme == '' and os.path.exists(destpath):
if optiondict.get("exist"):
msg = _('file already exists: "%s"') % dest
msg_note(recdict, msg)
failed.append(fname)
continue
if optiondict.get("interactive"):
reply = raw_input(_('"%s" exists, overwrite? (y/n) ') % dest)
if (len(reply) == 0 or not reply[0]
in _("yY up to four chars that mean yes")[:4]):
if copy:
msg_note(recdict, _("file not copied"))
else:
msg_note(recdict, _("file not moved"))
failed.append(fname)
continue
if fscheme == '' and tscheme == '':
#
# local file copy or move
#
if not copy:
try:
os.rename(fpath, destpath)
done = 1
except:
done = 0 # renaming failed, try copying
if copy or not done:
try:
import shutil
# TODO: handle symlinks and "keepsymlinks".
if os.path.isdir(fpath):
# Cannot use shutil.copytree here, it fails when the
# destination already exists.
if not os.path.exists(destpath):
os.mkdir(destpath)
# Call ourselves recursively
from Commands import dir_contents
flist = map(lambda x: {"name": x}, dir_contents(fpath))
if flist:
# Remove the "keepdir" attribute, the directory
# name is already in "destpath" now.
if optiondict.has_key("keepdir"):
opt = optiondict.copy()
del opt["keepdir"]
else:
opt = optiondict
f = remote_copy_move(rpstack, recdict, copy, flist,
{"name": destpath}, opt, 0, errmsg)
if f:
failed.extend(f)
elif not copy or optiondict.get("preserve"):
shutil.copy2(fpath, destpath)
else:
shutil.copy(fpath, destpath)
except (IOError, OSError), e:
msg = (_('Cannot copy "%s" to "%s": %s')
% (fname, dest, str(e)))
if not argexpand and not errmsg:
msg_note(recdict, msg)
failed.append(fname)
else:
recipe_error(rpstack, msg)
continue
if not copy:
if os.path.isdir(fpath):
deltree(fpath)
else:
os.remove(fpath)
if not optiondict.get("quiet"):
f = shorten_name(fpath)
if copy:
msg_info(recdict, _('Copied "%s" to "%s"') % (f, dest))
else:
msg_info(recdict, _('Moved "%s" to "%s"') % (f, dest))
else:
if fscheme != '':
# download to local file
from Remote import url_download
try:
if tscheme != '':
tmpfile, rtime = url_download(recdict, fname, '')
else:
tmpfile, rtime = url_download(recdict, fname, destpath)
except IOError, e:
msg = (_('Cannot download "%s" to "%s": %s')
% (fname, dest, str(e)))
if not argexpand and not errmsg:
msg_note(recdict, msg)
failed.append(fname)
else:
recipe_error(rpstack, msg)
continue
if tscheme != '':
if fscheme != '':
# use temporary file
fromfile = tmpfile
else:
fromfile = fpath
postponed = 0
if tscheme == 'ftp':
# No system command, give a message about what we're doing,
# it may take a while.
if fscheme != '':
msg_info(recdict, _('Uploading to "%s"' % dest))
else:
if not copy:
msg_info(recdict, _('Moving "%s" to "%s"'
% (fname, dest)))
else:
msg_info(recdict, _('Uploading "%s" to "%s"'
% (fname, dest)))
# FTP must use '/' path separator characters,
# os.path.join() may have used something else.
if os.sep != '/':
destpath = string.replace(destpath, os.sep, '/')
def store_file(ftp, fromfile, destpath):
import ftplib # avoid warning message
error = None
try:
f = open(fromfile, "rb")
except IOError, e:
return e
try:
ftp.storbinary("STOR " + destpath, f, 8192)
except ftplib.all_errors, e:
error = e
f.close()
return error
error = store_file(ftp, fromfile, destpath)
# If it looks like the directory doesn't exist, try
# creating it.
if error and ftp_may_mkdir(recdict, error, ftp, destpath):
msg_info(recdict, _('Directory created, attempt uploading again...'))
error = store_file(ftp, fromfile, destpath)
if error:
msg = (_('Cannot upload "%s" to "%s": %s')
% (fname, dest, str(error)))
if not argexpand and not errmsg:
msg_note(recdict, msg)
failed.append(fname)
else:
recipe_error(rpstack, msg)
if fscheme != '':
os.remove(tmpfile) # delete temporary file
continue
elif tscheme in scheme_copy_names:
if fscheme == '' and len(fromlist) > 1:
# do all local files at once below
scheme_files[tscheme] = (scheme_files[tscheme]
+ "'" + fromfile + "' ")
scheme_filelist[tscheme].append(fromfile)
postponed = 1
elif not scheme_copy(rpstack, recdict, tscheme,
"'" + fromfile + "'", tmach, destpath, 1):
msg = _('Cannot upload "%s" to "%s"') % (fname, dest)
if not argexpand and not errmsg:
msg_note(recdict, msg)
failed.append(fname)
else:
recipe_error(rpstack, msg)
if fscheme != '':
os.remove(tmpfile) # delete temporary file
continue
if fscheme != '':
os.remove(tmpfile) # delete temporary file
msg_info(recdict, _('Uploaded to "%s"' % dest))
else:
if not copy:
os.remove(fname)
msg_info(recdict, _('Moved "%s" to "%s"'
% (shorten_name(fname), dest)))
elif not postponed:
msg_info(recdict, _('Uploaded "%s" to "%s"'
% (shorten_name(fname), dest)))
# End of loop over all "from" files.
if fscheme != '':
from Remote import url_cleanup
url_cleanup(fscheme)
# Copy collected files with rcp, scp and rsync.
for scheme in scheme_copy_names:
files = scheme_files[scheme]
if files:
filelist = scheme_filelist[scheme]
# If there is only one file to copy, add its basename to the
# destination path to avoid creating a file by the name of the
# directory of that file.
if len(filelist) == 1:
tpath = os.path.join(tpath,
os.path.basename(filelist[0]))
if scheme_copy(rpstack, recdict, scheme, files, tmach, tpath,
len(filelist)):
dl = shorten_dictlist(str2dictlist(rpstack, files))
flist = map(lambda x: x["name"], dl)
msg_info(recdict, _('Uploaded "%s" to "%s"' % (flist, tname)))
else:
msg = _('Cannot upload "%s" to "%s"') % (files, tname)
if not argexpand and not errmsg:
msg_note(recdict, msg)
failed.extend(filelist)
else:
recipe_error(rpstack, msg)
return failed
def may_create_dir(recdict, rpstack, dir, quiet):
"""
Create directory "dir" if it does not exist.
"""
if not os.path.exists(dir):
try:
os.makedirs(dir)
if not quiet:
msg_info(recdict, _('Created directory "%s"') % dir)
except StandardError, e:
recipe_error(rpstack, _('Destination directory "%s" cannot be created: %s') % (dir, str(e)))
# vim: set sw=4 et sts=4 tw=79 fo+=l:
|