diff --git a/src/lgo.py b/src/lgo.py index 929453d..c29be72 100755 --- a/src/lgo.py +++ b/src/lgo.py @@ -366,10 +366,6 @@ class Lgo(App): href.startswith('http://ftp.gnome.org/')): continue filename = self.download(href) - # FIXME: there's no support for .xz on the webapps vm used for - # library-web, fallback to the .bz2 archive. - if filename.endswith('.xz'): - filename = os.path.splitext(filename)[0] + '.bz2' if not filename: continue logging.info('extracting module %s (from %s moduleset)' % ( diff --git a/src/utils.py b/src/utils.py index c21f589..9955a4f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -22,6 +22,9 @@ import re import time import urllib2 import sys +import tarfile +import lzma # pyliblzma + class FakeTarFile: ''' @@ -104,3 +107,89 @@ class LogFormatter(logging.Formatter): sys.stdout.flush() return '%c:[%s] %s' % (record.levelname[0], time.strftime('%Y%m%d %H:%M:%S'), record.msg) + +class _LZMAProxy(object): + """Small proxy class that enables external file object + support for "r:lzma" and "w:lzma" modes. This is actually + a workaround for a limitation in lzma module's LZMAFile + class which (unlike gzip.GzipFile) has no support for + a file object argument. + """ + + blocksize = 16 * 1024 + + def __init__(self, fileobj, mode): + self.fileobj = fileobj + self.mode = mode + self.name = getattr(self.fileobj, "name", None) + self.init() + + def init(self): + self.pos = 0 + if self.mode == "r": + self.lzmaobj = lzma.LZMADecompressor() + self.fileobj.seek(0) + self.buf = "" + else: + self.lzmaobj = lzma.LZMACompressor() + + def read(self, size): + b = [self.buf] + x = len(self.buf) + while x < size: + raw = self.fileobj.read(self.blocksize) + if not raw: + break + try: + data = self.lzmaobj.decompress(raw) + except EOFError: + break + b.append(data) + x += len(data) + self.buf = "".join(b) + + buf = self.buf[:size] + self.buf = self.buf[size:] + self.pos += len(buf) + return buf + + def seek(self, pos): + if pos < self.pos: + self.init() + self.read(pos - self.pos) + + +class XzTarFile(tarfile.TarFile): + + OPEN_METH = tarfile.TarFile.OPEN_METH.copy() + OPEN_METH["xz"] = "xzopen" + + @classmethod + def xzopen(cls, name, mode="r", fileobj=None, **kwargs): + """Open gzip compressed tar archive name for reading or writing. + Appending is not allowed. + """ + if len(mode) > 1 or mode not in "rw": + raise ValueError("mode must be 'r' or 'w'") + + if fileobj is not None: + fileobj = _LMZAProxy(fileobj, mode) + else: + fileobj = lzma.LZMAFile(name, mode) + + try: + # lzma doesn't immediately return an error + # try and read a bit of data to determine if it is a valid xz file + fileobj.read(_LZMAProxy.blocksize) + fileobj.seek(0) + t = cls.taropen(name, mode, fileobj, **kwargs) + except IOError: + raise tarfile.ReadError("not a xz file") + except lzma.error: + raise tarfile.ReadError("not a xz file") + t._extfileobj = False + return t + +if not hasattr(tarfile.TarFile, 'xvopen'): + tarfile.open = XzTarFile.open +