diff --git a/indra/lib/python/indra/ipc/webdav.py b/indra/lib/python/indra/ipc/webdav.py
new file mode 100644
index 0000000000..d50b3f2b26
--- /dev/null
+++ b/indra/lib/python/indra/ipc/webdav.py
@@ -0,0 +1,577 @@
+"""
+@file webdav.py
+@brief Classes to make manipulation of a webdav store easier.
+
+Copyright (c) 2006-2007, Linden Research, Inc.
+$License$
+"""
+
+import sys, os, httplib, urlparse
+import socket, time
+import xml.dom.minidom
+import syslog
+# import signal
+
+__revision__ = '0'
+
+dav_debug = False
+
+
+# def urlsafe_b64decode (enc):
+# return base64.decodestring (enc.replace ('_', '/').replace ('-', '+'))
+
+# def urlsafe_b64encode (str):
+# return base64.encodestring (str).replace ('+', '-').replace ('/', '_')
+
+
+class DAVError (Exception):
+ """ Base class for exceptions in this module. """
+ def __init__ (self, status=0, message='', body='', details=''):
+ self.status = status
+ self.message = message
+ self.body = body
+ self.details = details
+ Exception.__init__ (self, '%d:%s:%s%s' % (self.status, self.message,
+ self.body, self.details))
+
+ def print_to_stderr (self):
+ """ print_to_stderr docstring """
+ print >> sys.stderr, str (self.status) + ' ' + self.message
+ print >> sys.stderr, str (self.details)
+
+
+class Timeout (Exception):
+ """ Timeout docstring """
+ def __init__ (self, arg=''):
+ Exception.__init__ (self, arg)
+
+
+def alarm_handler (signum, frame):
+ """ alarm_handler docstring """
+ raise Timeout ('caught alarm')
+
+
+class WebDAV:
+ """ WebDAV docstring """
+ def __init__ (self, url, proxy=None, retries_before_fail=6):
+ self.init_url = url
+ self.init_proxy = proxy
+ self.retries_before_fail = retries_before_fail
+ url_parsed = urlparse.urlsplit (url)
+
+ self.top_path = url_parsed[ 2 ]
+ # make sure top_path has a trailing /
+ if self.top_path == None or self.top_path == '':
+ self.top_path = '/'
+ elif len (self.top_path) > 1 and self.top_path[-1:] != '/':
+ self.top_path += '/'
+
+ if dav_debug:
+ syslog.syslog ('new WebDAV %s : %s' % (str (url), str (proxy)))
+
+ if proxy:
+ proxy_parsed = urlparse.urlsplit (proxy)
+ self.host_header = url_parsed[ 1 ]
+ host_and_port = proxy_parsed[ 1 ].split (':')
+ self.host = host_and_port[ 0 ]
+ if len (host_and_port) > 1:
+ self.port = int(host_and_port[ 1 ])
+ else:
+ self.port = 80
+ else: # no proxy
+ host_and_port = url_parsed[ 1 ].split (':')
+ self.host_header = None
+ self.host = host_and_port[ 0 ]
+ if len (host_and_port) > 1:
+ self.port = int(host_and_port[ 1 ])
+ else:
+ self.port = 80
+
+ self.connection = False
+ self.connect ()
+
+
+ def log (self, msg, depth=0):
+ """ log docstring """
+ if dav_debug and depth == 0:
+ host = str (self.init_url)
+ if host == 'http://int.tuco.lindenlab.com:80/asset/':
+ host = 'tuco'
+ if host == 'http://harriet.lindenlab.com/asset-keep/':
+ host = 'harriet/asset-keep'
+ if host == 'http://harriet.lindenlab.com/asset-flag/':
+ host = 'harriet/asset-flag'
+ if host == 'http://harriet.lindenlab.com/asset/':
+ host = 'harriet/asset'
+ if host == 'http://ozzy.lindenlab.com/asset/':
+ host = 'ozzy/asset'
+ if host == 'http://station11.lindenlab.com:12041/:':
+ host = 'station11:12041'
+ proxy = str (self.init_proxy)
+ if proxy == 'None':
+ proxy = ''
+ if proxy == 'http://int.tuco.lindenlab.com:3128/':
+ proxy = 'tuco'
+ syslog.syslog ('WebDAV (%s:%s) %s' % (host, proxy, str (msg)))
+
+
+ def connect (self):
+ """ connect docstring """
+ self.log ('connect')
+ self.connection = httplib.HTTPConnection (self.host, self.port)
+
+ def __err (self, response, details):
+ """ __err docstring """
+ raise DAVError (response.status, response.reason, response.read (),
+ str (self.init_url) + ':' + \
+ str (self.init_proxy) + ':' + str (details))
+
+ def request (self, method, path, body=None, headers=None,
+ read_all=True, body_hook = None, recurse=0, allow_cache=True):
+ """ request docstring """
+ # self.log ('request %s %s' % (method, path))
+ if headers == None:
+ headers = {}
+ if not allow_cache:
+ headers['Pragma'] = 'no-cache'
+ headers['cache-control'] = 'no-cache'
+ try:
+ if method.lower () != 'purge':
+ if path.startswith ('/'):
+ path = path[1:]
+ if self.host_header: # use proxy
+ headers[ 'host' ] = self.host_header
+ fullpath = 'http://%s%s%s' % (self.host_header,
+ self.top_path, path)
+ else: # no proxy
+ fullpath = self.top_path + path
+ else:
+ fullpath = path
+
+ self.connection.request (method, fullpath, body, headers)
+ if body_hook:
+ body_hook ()
+
+ # signal.signal (signal.SIGALRM, alarm_handler)
+ # try:
+ # signal.alarm (120)
+ # signal.alarm (0)
+ # except Timeout, e:
+ # if recurse < 6:
+ # return self.retry_request (method, path, body, headers,
+ # read_all, body_hook, recurse)
+ # else:
+ # raise DAVError (0, 'timeout', self.host,
+ # (method, path, body, headers, recurse))
+
+ response = self.connection.getresponse ()
+
+ if read_all:
+ while len (response.read (1024)) > 0:
+ pass
+ if (response.status == 500 or \
+ response.status == 503 or \
+ response.status == 403) and \
+ recurse < self.retries_before_fail:
+ return self.retry_request (method, path, body, headers,
+ read_all, body_hook, recurse)
+ return response
+ except (httplib.ResponseNotReady,
+ httplib.BadStatusLine,
+ socket.error):
+ # if the server hangs up on us (keepalive off, broken pipe),
+ # we need to reconnect and try again.
+ if recurse < self.retries_before_fail:
+ return self.retry_request (method, path, body, headers,
+ read_all, body_hook, recurse)
+ raise DAVError (0, 'reconnect failed', self.host,
+ (method, path, body, headers, recurse))
+
+
+ def retry_request (self, method, path, body, headers,
+ read_all, body_hook, recurse):
+ """ retry_request docstring """
+ time.sleep (10.0 * recurse)
+ self.connect ()
+ return self.request (method, path, body, headers,
+ read_all, body_hook, recurse+1)
+
+
+
+ def propfind (self, path, body=None, depth=1):
+ """ propfind docstring """
+ # self.log ('propfind %s' % path)
+ headers = {'Content-Type':'text/xml; charset="utf-8"',
+ 'Depth':str(depth)}
+ response = self.request ('PROPFIND', path, body, headers, False)
+ if response.status == 207:
+ return response # Multi-Status
+ self.__err (response, ('PROPFIND', path, body, headers, 0))
+
+
+ def purge (self, path):
+ """ issue a squid purge command """
+ headers = {'Accept':'*/*'}
+ response = self.request ('PURGE', path, None, headers)
+ if response.status == 200 or response.status == 404:
+ # 200 if it was purge, 404 if it wasn't there.
+ return response
+ self.__err (response, ('PURGE', path, None, headers))
+
+
+ def get_file_size (self, path):
+ """
+ Use propfind to ask a webdav server what the size of
+ a file is. If used on a directory (collection) return 0
+ """
+ self.log ('get_file_size %s' % path)
+ # "getcontentlength" property
+ # 8.1.1 Example - Retrieving Named Properties
+ # http://docs.python.org/lib/module-xml.dom.html
+ nsurl = 'http://apache.org/dav/props/'
+ doc = xml.dom.minidom.Document ()
+ propfind_element = doc.createElementNS (nsurl, "D:propfind")
+ propfind_element.setAttributeNS (nsurl, 'xmlns:D', 'DAV:')
+ doc.appendChild (propfind_element)
+ prop_element = doc.createElementNS (nsurl, "D:prop")
+ propfind_element.appendChild (prop_element)
+ con_len_element = doc.createElementNS (nsurl, "D:getcontentlength")
+ prop_element.appendChild (con_len_element)
+
+ response = self.propfind (path, doc.toxml ())
+ doc.unlink ()
+
+ resp_doc = xml.dom.minidom.parseString (response.read ())
+ cln = resp_doc.getElementsByTagNameNS ('DAV:','getcontentlength')[ 0 ]
+ try:
+ content_length = int (cln.childNodes[ 0 ].nodeValue)
+ except IndexError:
+ return 0
+ resp_doc.unlink ()
+ return content_length
+
+
+ def file_exists (self, path):
+ """
+ do an http head on the given file. return True if it succeeds
+ """
+ self.log ('file_exists %s' % path)
+ expect_gzip = path.endswith ('.gz')
+ response = self.request ('HEAD', path)
+ got_gzip = response.getheader ('Content-Encoding', '').strip ()
+ if got_gzip.lower () == 'x-gzip' and expect_gzip == False:
+ # the asset server fakes us out if we ask for the non-gzipped
+ # version of an asset, but the server has the gzipped version.
+ return False
+ return response.status == 200
+
+
+ def mkdir (self, path):
+ """ mkdir docstring """
+ self.log ('mkdir %s' % path)
+ headers = {}
+ response = self.request ('MKCOL', path, None, headers)
+ if response.status == 201:
+ return # success
+ if response.status == 405:
+ return # directory already existed?
+ self.__err (response, ('MKCOL', path, None, headers, 0))
+
+
+ def delete (self, path):
+ """ delete docstring """
+ self.log ('delete %s' % path)
+ headers = {'Depth':'infinity'} # collections require infinity
+ response = self.request ('DELETE', path, None, headers)
+ if response.status == 204:
+ return # no content
+ if response.status == 404:
+ return # hmm
+ self.__err (response, ('DELETE', path, None, headers, 0))
+
+
+ def list_directory (self, path, dir_filter=None, allow_cache=True,
+ minimum_cache_time=False):
+ """
+ Request an http directory listing and parse the filenames out of lines
+ like: ' X'. If a filter function is provided,
+ only return filenames that the filter returns True for.
+
+ This is sort of grody, but it seems faster than other ways of getting
+ this information from an isilon.
+ """
+ self.log ('list_directory %s' % path)
+
+ def try_match (lline, before, after):
+ """ try_match docstring """
+ try:
+ blen = len (before)
+ asset_start_index = lline.index (before)
+ asset_end_index = lline.index (after, asset_start_index + blen)
+ asset = line[ asset_start_index + blen : asset_end_index ]
+
+ if not dir_filter or dir_filter (asset):
+ return [ asset ]
+ return []
+ except ValueError:
+ return []
+
+ if len (path) > 0 and path[-1:] != '/':
+ path += '/'
+
+ response = self.request ('GET', path, None, {}, False,
+ allow_cache=allow_cache)
+
+ if allow_cache and minimum_cache_time: # XXX
+ print response.getheader ('Date')
+ # s = "2005-12-06T12:13:14"
+ # from datetime import datetime
+ # from time import strptime
+ # datetime(*strptime(s, "%Y-%m-%dT%H:%M:%S")[0:6])
+ # datetime.datetime(2005, 12, 6, 12, 13, 14)
+
+ if response.status != 200:
+ self.__err (response, ('GET', path, None, {}, 0))
+ assets = []
+ for line in response.read ().split ('\n'):
+ lline = line.lower ()
+ if lline.find ("parent directory") == -1:
+ # isilon file
+ assets += try_match (lline, ' ')
+ # apache dir
+ assets += try_match (lline, 'alt="[dir]"> ')
+ # apache file
+ assets += try_match (lline, 'alt="[ ]"> ')
+ return assets
+
+
+ def __tmp_filename (self, path_and_file):
+ """ __tmp_filename docstring """
+ head, tail = os.path.split (path_and_file)
+ if head != '':
+ return head + '/.' + tail + '.' + str (os.getpid ())
+ else:
+ return head + '.' + tail + '.' + str (os.getpid ())
+
+
+ def __put__ (self, filesize, body_hook, remotefile):
+ """ __put__ docstring """
+ headers = {'Content-Length' : str (filesize)}
+ remotefile_tmp = self.__tmp_filename (remotefile)
+ response = self.request ('PUT', remotefile_tmp, None,
+ headers, True, body_hook)
+ if not response.status in (201, 204): # created, no content
+ self.__err (response, ('PUT', remotefile, None, headers, 0))
+ if filesize != self.get_file_size (remotefile_tmp):
+ try:
+ self.delete (remotefile_tmp)
+ except:
+ pass
+ raise DAVError (0, 'tmp upload error', remotefile_tmp)
+ # move the file to its final location
+ try:
+ self.rename (remotefile_tmp, remotefile)
+ except DAVError, exc:
+ if exc.status == 403: # try to clean up the tmp file
+ try:
+ self.delete (remotefile_tmp)
+ except:
+ pass
+ raise
+ if filesize != self.get_file_size (remotefile):
+ raise DAVError (0, 'file upload error', str (remotefile_tmp))
+
+
+ def put_string (self, strng, remotefile):
+ """ put_string docstring """
+ self.log ('put_string %d -> %s' % (len (strng), remotefile))
+ filesize = len (strng)
+ def body_hook ():
+ """ body_hook docstring """
+ self.connection.send (strng)
+ self.__put__ (filesize, body_hook, remotefile)
+
+
+ def put_file (self, localfile, remotefile):
+ """
+ Send a local file to a remote webdav store. First, upload to
+ a temporary filename. Next make sure the file is the size we
+ expected. Next, move the file to its final location. Next,
+ check the file size at the final location.
+ """
+ self.log ('put_file %s -> %s' % (localfile, remotefile))
+ filesize = os.path.getsize (localfile)
+ def body_hook ():
+ """ body_hook docstring """
+ handle = open (localfile)
+ while True:
+ data = handle.read (1300)
+ if len (data) == 0:
+ break
+ self.connection.send (data)
+ handle.close ()
+ self.__put__ (filesize, body_hook, remotefile)
+
+
+ def create_empty_file (self, remotefile):
+ """ create an empty file """
+ self.log ('touch_file %s' % (remotefile))
+ headers = {'Content-Length' : '0'}
+ response = self.request ('PUT', remotefile, None, headers)
+ if not response.status in (201, 204): # created, no content
+ self.__err (response, ('PUT', remotefile, None, headers, 0))
+ if self.get_file_size (remotefile) != 0:
+ raise DAVError (0, 'file upload error', str (remotefile))
+
+
+ def __get_file_setup (self, remotefile, check_size=True):
+ """ __get_file_setup docstring """
+ if check_size:
+ remotesize = self.get_file_size (remotefile)
+ response = self.request ('GET', remotefile, None, {}, False)
+ if response.status != 200:
+ self.__err (response, ('GET', remotefile, None, {}, 0))
+ try:
+ content_length = int (response.getheader ("Content-Length"))
+ except TypeError:
+ content_length = None
+ if check_size:
+ if content_length != remotesize:
+ raise DAVError (0, 'file DL size error', remotefile)
+ return (response, content_length)
+
+
+ def __get_file_read (self, writehandle, response, content_length):
+ """ __get_file_read docstring """
+ if content_length != None:
+ so_far_length = 0
+ while so_far_length < content_length:
+ data = response.read (content_length - so_far_length)
+ if len (data) == 0:
+ raise DAVError (0, 'short file download')
+ so_far_length += len (data)
+ writehandle.write (data)
+ while len (response.read ()) > 0:
+ pass
+ else:
+ while True:
+ data = response.read ()
+ if (len (data) < 1):
+ break
+ writehandle.write (data)
+
+
+ def get_file (self, remotefile, localfile, check_size=True):
+ """
+ Get a remote file from a webdav server. Download to a local
+ tmp file, then move into place. Sanity check file sizes as
+ we go.
+ """
+ self.log ('get_file %s -> %s' % (remotefile, localfile))
+ (response, content_length) = \
+ self.__get_file_setup (remotefile, check_size)
+ localfile_tmp = self.__tmp_filename (localfile)
+ handle = open (localfile_tmp, 'w')
+ self.__get_file_read (handle, response, content_length)
+ handle.close ()
+ if check_size:
+ if content_length != os.path.getsize (localfile_tmp):
+ raise DAVError (0, 'file DL size error',
+ remotefile+','+localfile)
+ os.rename (localfile_tmp, localfile)
+
+
+ def get_file_as_string (self, remotefile, check_size=True):
+ """
+ download a file from a webdav server and return it as a string.
+ """
+ self.log ('get_file_as_string %s' % remotefile)
+ (response, content_length) = \
+ self.__get_file_setup (remotefile, check_size)
+ # (tmp_handle, tmp_filename) = tempfile.mkstemp ()
+ tmp_handle = os.tmpfile ()
+ self.__get_file_read (tmp_handle, response, content_length)
+ tmp_handle.seek (0)
+ ret = tmp_handle.read ()
+ tmp_handle.close ()
+ # os.unlink (tmp_filename)
+ return ret
+
+
+ def get_post_as_string (self, remotefile, body):
+ """
+ Do an http POST, send body, get response and return it.
+ """
+ self.log ('get_post_as_string %s' % remotefile)
+ # headers = {'Content-Type':'application/x-www-form-urlencoded'}
+ headers = {'Content-Type':'text/xml; charset="utf-8"'}
+ # b64body = urlsafe_b64encode (asset_url)
+ response = self.request ('POST', remotefile, body, headers, False)
+ if response.status != 200:
+ self.__err (response, ('POST', remotefile, body, headers, 0))
+ try:
+ content_length = int (response.getheader ('Content-Length'))
+ except TypeError:
+ content_length = None
+ tmp_handle = os.tmpfile ()
+ self.__get_file_read (tmp_handle, response, content_length)
+ tmp_handle.seek (0)
+ ret = tmp_handle.read ()
+ tmp_handle.close ()
+ return ret
+
+
+ def __destination_command (self, verb, remotesrc, dstdav, remotedst):
+ """
+ self and dstdav should point to the same http server.
+ """
+ if len (remotedst) > 0 and remotedst[ 0 ] == '/':
+ remotedst = remotedst[1:]
+ headers = {'Destination': 'http://%s:%d%s%s' % (dstdav.host,
+ dstdav.port,
+ dstdav.top_path,
+ remotedst)}
+ response = self.request (verb, remotesrc, None, headers)
+ if response.status == 201:
+ return # created
+ if response.status == 204:
+ return # no content
+ self.__err (response, (verb, remotesrc, None, headers, 0))
+
+
+ def rename (self, remotesrc, remotedst):
+ """ rename a file on a webdav server """
+ self.log ('rename %s -> %s' % (remotesrc, remotedst))
+ self.__destination_command ('MOVE', remotesrc, self, remotedst)
+ def xrename (self, remotesrc, dstdav, remotedst):
+ """ rename a file on a webdav server """
+ self.log ('xrename %s -> %s' % (remotesrc, remotedst))
+ self.__destination_command ('MOVE', remotesrc, dstdav, remotedst)
+
+
+ def copy (self, remotesrc, remotedst):
+ """ copy a file on a webdav server """
+ self.log ('copy %s -> %s' % (remotesrc, remotedst))
+ self.__destination_command ('COPY', remotesrc, self, remotedst)
+ def xcopy (self, remotesrc, dstdav, remotedst):
+ """ copy a file on a webdav server """
+ self.log ('xcopy %s -> %s' % (remotesrc, remotedst))
+ self.__destination_command ('COPY', remotesrc, dstdav, remotedst)
+
+
+def put_string (data, url):
+ """
+ upload string s to a url
+ """
+ url_parsed = urlparse.urlsplit (url)
+ dav = WebDAV ('%s://%s/' % (url_parsed[ 0 ], url_parsed[ 1 ]))
+ dav.put_string (data, url_parsed[ 2 ])
+
+
+def get_string (url, check_size=True):
+ """
+ return the contents of a url as a string
+ """
+ url_parsed = urlparse.urlsplit (url)
+ dav = WebDAV ('%s://%s/' % (url_parsed[ 0 ], url_parsed[ 1 ]))
+ return dav.get_file_as_string (url_parsed[ 2 ], check_size)