1613 lines
66 KiB
Python
Executable File
1613 lines
66 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# libgo - script to build library.gnome.org
|
|
# Copyright (C) 2007 Frederic Peters
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
_svn_id = '$Id$'
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import urllib2
|
|
from cStringIO import StringIO
|
|
from optparse import OptionParser
|
|
import logging
|
|
import elementtree.ElementTree as ET
|
|
import tarfile
|
|
import glob
|
|
import tempfile
|
|
import stat
|
|
import subprocess
|
|
import dbm
|
|
import shutil
|
|
import socket
|
|
|
|
try:
|
|
import html5lib
|
|
except ImportError:
|
|
html5lib = None
|
|
|
|
from config import Config
|
|
import errors
|
|
import utils
|
|
|
|
app = None
|
|
data_dir = os.path.join(os.path.dirname(__file__), '../data')
|
|
|
|
licence_modules = ['fdl', 'gpl', 'lgpl']
|
|
|
|
# timeout for downloads, so it doesn't hang on connecting to sourceforge
|
|
socket.setdefaulttimeout(10)
|
|
|
|
def version_cmp(x, y):
|
|
# returns < 0 if x < y, 0 if x == y, and > 0 if x > y
|
|
try:
|
|
return cmp([int(j) for j in x.split('.')], [int(k) for k in y.split('.')])
|
|
except ValueError:
|
|
logging.warning('failure in version_cmp: %r vs %r' % (x, y))
|
|
return 0
|
|
|
|
def download(href):
|
|
filename = '/'.join(urllib2.urlparse.urlparse(href)[1:3])
|
|
cache_filename = os.path.join(app.config.download_cache_dir, filename)
|
|
cache_dir = os.path.split(cache_filename)[0]
|
|
if not os.path.exists(cache_dir):
|
|
os.makedirs(cache_dir)
|
|
if not os.path.exists(cache_filename):
|
|
logging.info('downloading %s' % href)
|
|
try:
|
|
s = urllib2.urlopen(href).read()
|
|
except urllib2.HTTPError, e:
|
|
logging.warning('error %s downloading %s' % (e.code, href))
|
|
return None
|
|
except urllib2.URLError, e:
|
|
logging.warning('error (URLError) downloading %s' % href)
|
|
return None
|
|
open(cache_filename, 'w').write(s)
|
|
return cache_filename
|
|
|
|
|
|
class Document:
|
|
channel = None # one of ('users', 'devel', 'about', 'admin')
|
|
module = None
|
|
path = None
|
|
category = None
|
|
toc_id = None
|
|
subsection = None
|
|
weight = 0.5
|
|
|
|
title = None # indexed on language, most recent version
|
|
abstract = None # indexed on language, most recent version
|
|
href = None # for external docs, indexed on language
|
|
|
|
languages = None # list of available languages
|
|
versions = None # list of available versions
|
|
version_keywords = None
|
|
version_mapping = None # mapping of two-number version to full-numbers version
|
|
# like {'2.18': '2.18.3', '2.19': '2.19.90'}
|
|
|
|
def __init__(self):
|
|
self.title = {}
|
|
self.abstract = {}
|
|
self.versions = []
|
|
self.version_keywords = {}
|
|
self.version_mapping = {}
|
|
self.keywords = []
|
|
|
|
def create_element(self, parent, language, original_language = None):
|
|
if not language in self.languages:
|
|
return
|
|
doc = ET.SubElement(parent, 'document')
|
|
if language == 'C':
|
|
language = 'en'
|
|
if not original_language:
|
|
original_language = language
|
|
href_language = None
|
|
if self.module:
|
|
doc.set('modulename', self.module)
|
|
if self.path:
|
|
doc.set('path', self.path)
|
|
elif self.href:
|
|
if self.href.get(original_language) and self.href.get(original_language) != '-':
|
|
href_language = original_language
|
|
doc.set('href', self.href.get(original_language))
|
|
else:
|
|
href_language = 'en'
|
|
doc.set('href', self.href.get('en'))
|
|
else:
|
|
logging.error('no path and no href in module %s ' % self.module)
|
|
return
|
|
title = self.title.get(original_language)
|
|
if not title:
|
|
title = self.title.get(language)
|
|
if not title:
|
|
title = self.module
|
|
ET.SubElement(doc, 'title').text = title
|
|
|
|
abstract = self.abstract.get(original_language)
|
|
if not abstract:
|
|
abstract = self.abstract.get(language)
|
|
if abstract:
|
|
ET.SubElement(doc, 'abstract').text = abstract
|
|
|
|
doc.set('channel', self.channel)
|
|
doc.set('weight', str(self.weight))
|
|
|
|
if href_language:
|
|
doc.set('lang', href_language)
|
|
else:
|
|
doc.set('lang', language)
|
|
|
|
#if self.category:
|
|
# doc.set('category', self.category)
|
|
if self.toc_id:
|
|
doc.set('toc_id', self.toc_id)
|
|
|
|
langs = ET.SubElement(doc, 'other-languages')
|
|
for l in self.languages:
|
|
if l == language or l == 'C':
|
|
continue
|
|
ET.SubElement(langs, 'lang').text = l
|
|
|
|
if self.versions:
|
|
versions = ET.SubElement(doc, 'versions')
|
|
for v in sorted(self.versions, version_cmp):
|
|
version = ET.SubElement(versions, 'version')
|
|
version.set('href', v)
|
|
if v in self.version_mapping:
|
|
version.text = self.version_mapping[v]
|
|
else:
|
|
version.text = v
|
|
if v in self.version_keywords:
|
|
version.set('keyword', self.version_keywords[v])
|
|
|
|
if self.keywords:
|
|
keywords = ET.SubElement(doc, 'keywords')
|
|
for k in self.keywords:
|
|
keyword = ET.SubElement(keywords, 'keyword')
|
|
keyword.text = k
|
|
|
|
|
|
class RemoteDocument(Document):
|
|
html2html_xsl_file = os.path.join(data_dir, 'xslt', 'html2html.xsl')
|
|
|
|
def __init__(self, overlay):
|
|
Document.__init__(self)
|
|
self.overlay = overlay
|
|
if 'doc_module' in overlay.attrib:
|
|
self.module = overlay.attrib['doc_module']
|
|
self.channel = overlay.attrib['channel']
|
|
self.category = overlay.attrib['category']
|
|
self.toc_id = self.category
|
|
self.title = {}
|
|
self.href = {}
|
|
self.abstract = {}
|
|
for title in overlay.findall('title'):
|
|
lang = title.attrib.get(
|
|
'{http://www.w3.org/XML/1998/namespace}lang', 'en')
|
|
self.title[lang] = title.text
|
|
for abstract in overlay.findall('abstract'):
|
|
lang = abstract.attrib.get(
|
|
'{http://www.w3.org/XML/1998/namespace}lang', 'en')
|
|
self.abstract[lang] = abstract.text
|
|
for href in overlay.findall('href'):
|
|
if href.text == '-':
|
|
continue
|
|
lang = href.attrib.get(
|
|
'{http://www.w3.org/XML/1998/namespace}lang', 'en')
|
|
self.href[lang] = href.text
|
|
self.languages = self.title.keys()
|
|
|
|
if overlay.find('subsection') is not None:
|
|
self.subsection = overlay.find('subsection').text
|
|
|
|
if overlay.find('local') is not None:
|
|
self.remote_to_local(overlay.find('local').attrib)
|
|
|
|
def remote_to_local(self, attribs):
|
|
web_output_dir = os.path.join(app.config.output_dir, self.channel, self.module)
|
|
mtime_xsl = os.stat(self.html2html_xsl_file)[stat.ST_MTIME]
|
|
|
|
for lang in self.href:
|
|
if self.href[lang] == '-':
|
|
continue
|
|
filename = self.download(self.href[lang])
|
|
if not filename:
|
|
continue
|
|
dst = os.path.join(web_output_dir, 'index.html.%s' % lang)
|
|
if os.path.exists(dst) and (
|
|
os.stat(dst)[stat.ST_MTIME] > max(mtime_xsl, os.stat(filename)[stat.ST_MTIME])):
|
|
continue
|
|
|
|
parser = html5lib.HTMLParser()
|
|
doc = parser.parse(open(filename))
|
|
doc.childNodes[-1].attributes['xmlns'] = 'http://www.w3.org/1999/xhtml'
|
|
cmd = ['xsltproc', '--output', dst,
|
|
'--stringparam', 'libgo.originalhref', self.href[lang],
|
|
'--stringparam', 'libgo.channel', self.channel,
|
|
'--nonet', '--xinclude', self.html2html_xsl_file, '-']
|
|
for k in attribs:
|
|
cmd.insert(-2, '--stringparam')
|
|
cmd.insert(-2, 'libgo.%s' % k)
|
|
cmd.insert(-2, attribs[k])
|
|
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
xsltproc = subprocess.Popen(cmd, stdin = subprocess.PIPE)
|
|
xsltproc.communicate(doc.toxml())
|
|
xsltproc.wait()
|
|
if xsltproc.returncode:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), xsltproc.returncode))
|
|
|
|
self.path = '/' + os.path.join(self.channel, self.module) + '/'
|
|
|
|
def download(self, href):
|
|
# TODO: add some support (think <local update="daily"/>) so the file
|
|
# can be "watched" for changes
|
|
return download(href)
|
|
|
|
|
|
class SubIndex:
|
|
def __init__(self, node):
|
|
self.id = node.attrib.get('id')
|
|
self.weight = node.attrib.get('weight')
|
|
self.sections = node.find('sections').text.split()
|
|
self.title = {}
|
|
self.abstract = {}
|
|
|
|
for title in node.findall('title'):
|
|
lang = title.attrib.get(
|
|
'{http://www.w3.org/XML/1998/namespace}lang', 'en')
|
|
self.title[lang] = title.text
|
|
for abstract in node.findall('abstract'):
|
|
lang = abstract.attrib.get(
|
|
'{http://www.w3.org/XML/1998/namespace}lang', 'en')
|
|
self.abstract[lang] = abstract.text
|
|
|
|
def create_element(self, parent, channel, language):
|
|
index = ET.SubElement(parent, 'index')
|
|
if language == 'C':
|
|
language = 'en'
|
|
index.set('id', self.id)
|
|
index.set('lang', language)
|
|
index.set('channel', channel)
|
|
index.set('weigth', self.weight)
|
|
|
|
title = self.title.get(language)
|
|
if not title:
|
|
title = self.title.get('en')
|
|
if not title:
|
|
title = self.id
|
|
ET.SubElement(index, 'title').text = title
|
|
|
|
abstract = self.abstract.get(language)
|
|
if not abstract:
|
|
abstract = self.abstract.get('en')
|
|
if abstract:
|
|
ET.SubElement(index, 'abstract').text = abstract
|
|
|
|
return index
|
|
|
|
|
|
class DocModule:
|
|
makefile_am = None
|
|
|
|
filename = None
|
|
dirname = None
|
|
modulename = None
|
|
|
|
related_xsl_files = None
|
|
mtime_xslt_files = 0
|
|
|
|
def __init__(self, tar, tarinfo, makefile_am):
|
|
self.dirname = os.path.dirname(tarinfo.name)
|
|
if makefile_am:
|
|
self.makefile_am = makefile_am
|
|
self.modulename = re.findall(r'DOC_MODULE\s?=\s?(.*)', makefile_am)[0].strip()
|
|
self.version = os.path.splitext(tar.name)[0].split('-')[-1]
|
|
self.one_dot_version = re.match(r'\d+\.\d+', self.version).group()
|
|
|
|
if self.related_xsl_files:
|
|
self.mtime_xslt_files = max([os.stat(
|
|
os.path.join(data_dir, 'xslt', x))[stat.ST_MTIME] \
|
|
for x in self.related_xsl_files])
|
|
|
|
def extract(self):
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts')
|
|
if not os.path.exists(ext_dirname):
|
|
os.makedirs(ext_dirname)
|
|
|
|
if not os.path.exists(os.path.join(ext_dirname, self.dirname)):
|
|
logging.debug('extracting %s' % self.dirname)
|
|
tar = tarfile.open(self.filename, 'r')
|
|
for tarinfo in tar.getmembers():
|
|
if not os.path.split(tarinfo.name)[0].startswith(self.dirname):
|
|
continue
|
|
dest = os.path.join(ext_dirname, tarinfo.name)
|
|
if os.path.exists(dest):
|
|
continue
|
|
if tarinfo.isdir() and not os.path.exists(dest):
|
|
os.makedirs(dest)
|
|
elif tarinfo.isreg():
|
|
if not os.path.exists(os.path.dirname(dest)):
|
|
os.makedirs(os.path.dirname(dest))
|
|
open(dest, 'w').write(tar.extractfile(tarinfo).read())
|
|
tar.close()
|
|
|
|
def setup_path(self):
|
|
if self.modulename in licence_modules:
|
|
# special casing the licences, they do not go in a
|
|
# versioned path
|
|
self.path = '/' + os.path.join(self.channel, self.modulename) + '/'
|
|
else:
|
|
self.path = '/' + os.path.join(self.channel, self.modulename,
|
|
self.one_dot_version) + '/'
|
|
|
|
def get_libgo_document(self, doc_linguas):
|
|
try:
|
|
doc = [x for x in app.documents if \
|
|
x.module == self.modulename and x.channel == self.channel][0]
|
|
except IndexError:
|
|
doc = Document()
|
|
doc.filename = self.dirname
|
|
doc.module = self.modulename
|
|
doc.channel = self.channel
|
|
doc.languages = doc_linguas
|
|
doc.path = self.path
|
|
doc._last_version = self.version
|
|
doc.versions = [self.one_dot_version]
|
|
app.documents.append(doc)
|
|
else:
|
|
if doc._last_version == self.version:
|
|
# file was already processed in a previous moduleset
|
|
return None
|
|
doc._last_version = self.version
|
|
if int(self.one_dot_version.split('.')[1]) % 2 == 0:
|
|
# update path to point to the latest version (but no
|
|
# development versions)
|
|
doc.path = self.path
|
|
if not self.one_dot_version in doc.versions:
|
|
doc.versions.append(self.one_dot_version)
|
|
for lang in doc_linguas:
|
|
if not lang in doc.languages:
|
|
doc.languages.append(lang)
|
|
|
|
doc.version_mapping[self.one_dot_version] = self.version
|
|
|
|
# only keep authorised languages
|
|
if app.config.languages:
|
|
for lang in doc.languages[:]:
|
|
if lang not in app.config.languages:
|
|
doc.languages.remove(lang)
|
|
|
|
return doc
|
|
|
|
def install_version_symlinks(self, doc):
|
|
'''Create stable and devel symlinks'''
|
|
|
|
if self.channel == 'about':
|
|
return
|
|
|
|
if not self.one_dot_version in doc.versions:
|
|
# this version doesn't appear in available versions, probably it
|
|
# had been depreciated
|
|
return
|
|
|
|
web_output_dir = os.path.join(app.config.output_dir, self.channel, self.modulename)
|
|
development_release = (int(self.one_dot_version.split('.')[1]) % 2 == 1)
|
|
|
|
if development_release:
|
|
keyword = 'unstable'
|
|
else:
|
|
keyword = 'stable'
|
|
|
|
path = os.path.join(web_output_dir, keyword)
|
|
installed = False
|
|
if os.path.islink(path):
|
|
currently_marked = os.readlink(path)
|
|
if currently_marked == 'stable':
|
|
currently_marked = os.readlink(os.path.join(web_output_dir, 'stable'))
|
|
if version_cmp(self.version, currently_marked) >= 0:
|
|
# install new symlink
|
|
os.unlink(path)
|
|
os.symlink(self.one_dot_version, path)
|
|
installed = True
|
|
else:
|
|
os.symlink(self.one_dot_version, path)
|
|
installed = True
|
|
|
|
if installed and not development_release:
|
|
if doc.path == '/%s/%s/%s/' % (self.channel, self.modulename, self.one_dot_version):
|
|
# set default path to use the keyword
|
|
doc.path = '/%s/%s/stable/' % (self.channel, self.modulename)
|
|
|
|
# if there is no unstable link, create it even for a stable release
|
|
# (or if stable is newer)
|
|
path = os.path.join(web_output_dir, 'unstable')
|
|
if os.path.islink(path):
|
|
currently_unstable = os.readlink(path)
|
|
if currently_unstable == 'stable' or \
|
|
version_cmp(self.version, currently_unstable) >= 0:
|
|
os.unlink(path)
|
|
os.symlink('stable', path)
|
|
else:
|
|
os.symlink('stable', path)
|
|
|
|
if installed:
|
|
for k in doc.version_keywords.keys():
|
|
if doc.version_keywords.get(k) == keyword:
|
|
del doc.version_keywords[k]
|
|
doc.version_keywords[self.one_dot_version] = keyword
|
|
|
|
|
|
class GtkDocModule(DocModule):
|
|
gtkdoc_xsl_file = os.path.join(data_dir, 'xslt', 'gtk-doc.xsl')
|
|
html2html_xsl_file = os.path.join(data_dir, 'xslt', 'html2html.xsl')
|
|
|
|
related_xsl_files = ['gtk-doc.xsl', 'heading.xsl']
|
|
|
|
def setup_channel(self):
|
|
self.channel = app.overlay.get_channel_overlay(self.modulename, 'devel')
|
|
|
|
def __str__(self):
|
|
return 'gtk-doc module at %s' % self.dirname
|
|
|
|
def process(self):
|
|
doc_module = self.modulename
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts')
|
|
|
|
main_sgml_file = re.findall(r'DOC_MAIN_SGML_FILE\s?=\s?(.*)',
|
|
self.makefile_am)[0].strip()
|
|
main_sgml_file = main_sgml_file.replace('$(DOC_MODULE)', doc_module)
|
|
|
|
try:
|
|
html_images = re.findall('HTML_IMAGES\s+=\s+(.*)', self.makefile_am)[0].split()
|
|
except IndexError:
|
|
html_images = []
|
|
|
|
web_output_dir = os.path.join(app.config.output_dir, self.channel,
|
|
doc_module, self.one_dot_version)
|
|
if not os.path.exists(web_output_dir):
|
|
os.makedirs(web_output_dir)
|
|
|
|
if not app.rebuild_all and os.path.exists(
|
|
os.path.join(web_output_dir, '%s.devhelp' % doc_module)):
|
|
mtime = os.stat(os.path.join(web_output_dir, '%s.devhelp' % doc_module))[stat.ST_MTIME]
|
|
else:
|
|
mtime = 0
|
|
|
|
if mtime > max(self.mtime_tarball, self.mtime_xslt_files):
|
|
logging.debug('using already generated doc')
|
|
else:
|
|
logging.info('generating doc in %s' % web_output_dir[len(app.config.output_dir):])
|
|
cmd = ['xsltproc', '--output', web_output_dir + '/',
|
|
'--nonet', '--xinclude',
|
|
'--stringparam', 'libgo.lang', 'en',
|
|
'--stringparam', 'gtkdoc.bookname', doc_module,
|
|
'--stringparam', 'gtkdoc.version', '"1.8 (+lgo)"',
|
|
'--stringparam', 'libgo.channel', self.channel,
|
|
self.gtkdoc_xsl_file,
|
|
os.path.join(ext_dirname, self.dirname, main_sgml_file)]
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
if rc == 6:
|
|
# build failed, probably because it has inline references in
|
|
# documentation and would require a full module build to get
|
|
# them properly. (happens with GTK+)
|
|
|
|
if html5lib:
|
|
# convert files to XML, then process them with xsltproc
|
|
# to get library.gnome.org look
|
|
|
|
logging.debug('transforming files shipped with tarball')
|
|
parser = html5lib.HTMLParser()
|
|
|
|
for filename in os.listdir(os.path.join(
|
|
ext_dirname, self.dirname, 'html')):
|
|
src = os.path.join(
|
|
ext_dirname, self.dirname, 'html', filename)
|
|
dst = os.path.join(web_output_dir, filename)
|
|
if not filename.endswith('.html'):
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
continue
|
|
doc = parser.parse(open(src))
|
|
doc.childNodes[-1].attributes['xmlns'] = 'http://www.w3.org/1999/xhtml'
|
|
temporary = tempfile.NamedTemporaryFile()
|
|
temporary.write(doc.toxml())
|
|
temporary.flush()
|
|
|
|
cmd = ['xsltproc', '--output', dst,
|
|
'--nonet', '--xinclude',
|
|
self.html2html_xsl_file,
|
|
os.path.join(ext_dirname,
|
|
self.dirname, temporary.name)]
|
|
rc = subprocess.call(cmd)
|
|
else:
|
|
# simply copy files shipped in tarball
|
|
logging.debug('copying files shipped with tarball')
|
|
for filename in os.listdir(os.path.join(
|
|
ext_dirname, self.dirname, 'html')):
|
|
src = os.path.join(ext_dirname, self.dirname, 'html', filename)
|
|
dst = os.path.join(web_output_dir, filename)
|
|
if not os.path.exists(os.path.split(dst)[0]):
|
|
os.makedirs(os.path.split(dst)[0])
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
if html_images:
|
|
# and copy images/
|
|
logging.debug('copying images')
|
|
for html_image in html_images:
|
|
src = os.path.join(ext_dirname, self.dirname, html_image)
|
|
if not os.path.exists(src):
|
|
continue
|
|
dst = os.path.join(web_output_dir, html_image)
|
|
if not os.path.exists(os.path.split(dst)[0]):
|
|
os.makedirs(os.path.split(dst)[0])
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
# in any case, copy png files from gtk-doc
|
|
for src in glob.glob('/usr/share/gtk-doc/data/*.png'):
|
|
dst = os.path.join(web_output_dir, os.path.basename(src))
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
doc = self.get_libgo_document(['en'])
|
|
if not doc:
|
|
return
|
|
|
|
doc.category = 'api'
|
|
doc.toc_id = 'api'
|
|
|
|
if os.path.exists(os.path.join(web_output_dir, 'index.xml.en')):
|
|
tree = ET.parse(os.path.join(web_output_dir, 'index.xml.en'))
|
|
if tree.find('title') is not None:
|
|
doc.title['en'] = tree.find('title').text
|
|
elif tree.find('{http://www.w3.org/1999/xhtml}title') is not None:
|
|
doc.title['en'] = tree.find('{http://www.w3.org/1999/xhtml}title').text
|
|
elif os.path.exists(os.path.join(web_output_dir, '%s.devhelp' % doc_module)):
|
|
tree = ET.parse(os.path.join(web_output_dir, '%s.devhelp' % doc_module))
|
|
doc.title['en'] = tree.getroot().attrib['title']
|
|
|
|
self.install_version_symlinks(doc)
|
|
|
|
|
|
class GnomeDocUtilsModule(DocModule):
|
|
db2html_xsl_file = os.path.join(data_dir, 'xslt', 'db2html.xsl')
|
|
category = None
|
|
|
|
related_xsl_files = ['db2html.xsl', 'heading.xsl']
|
|
|
|
def __init__(self, tar, tarinfo, makefile_am):
|
|
DocModule.__init__(self, tar, tarinfo, makefile_am)
|
|
if self.modulename == '@PACKAGE_NAME@':
|
|
# ekiga has this, use another way, looking at omf files
|
|
try:
|
|
omf_file = [x.name for x in tar.getmembers() if \
|
|
x.name.startswith(self.dirname) and x.name.endswith('.omf.in')][0]
|
|
except IndexError:
|
|
logging.error('failed to get DOC_MODULE for %s' % tarinfo.name)
|
|
self.modulename = os.path.split(omf_file)[-1][:-len('.omf.in')]
|
|
|
|
def setup_channel(self):
|
|
# get category from omf file
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts')
|
|
omf_file = glob.glob(os.path.join(ext_dirname, self.dirname) + '/*.omf.in')
|
|
if omf_file:
|
|
try:
|
|
self.category = ET.parse(omf_file[0]).find('resource/subject').attrib['category']
|
|
except (IndexError, KeyError):
|
|
pass
|
|
|
|
channel = 'users'
|
|
if self.category and (self.category.startswith('GNOME|Development') or
|
|
self.category.startswith('GNOME|Applications|Programming')):
|
|
channel = 'devel'
|
|
|
|
self.channel = app.overlay.get_channel_overlay(self.modulename, channel)
|
|
|
|
def __str__(self):
|
|
return 'gnome-doc-utils module at %s' % self.dirname
|
|
|
|
def process(self):
|
|
doc_module = self.modulename
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts')
|
|
|
|
try:
|
|
doc_linguas = re.findall(r'DOC_LINGUAS\s+=[\t ](.*)',
|
|
self.makefile_am)[0].split()
|
|
doc_linguas.append('en')
|
|
except IndexError:
|
|
doc_linguas = ['en']
|
|
|
|
try:
|
|
doc_figures = re.findall('DOC_FIGURES\s+=\s+(.*)',
|
|
self.makefile_am)[0].split()
|
|
except IndexError:
|
|
doc_figures = []
|
|
|
|
doc_linguas.sort()
|
|
|
|
doc = self.get_libgo_document(doc_linguas)
|
|
if not doc:
|
|
return
|
|
|
|
if self.category:
|
|
doc.category = self.category
|
|
doc.toc_id = app.toc_mapping.get(doc.category)
|
|
|
|
web_output_dir = os.path.join(app.config.output_dir, self.channel,
|
|
doc_module, self.one_dot_version)
|
|
|
|
quirks = app.overlay.get_quirks(self)
|
|
|
|
logging.info('generating doc in %s' % web_output_dir[len(app.config.output_dir):])
|
|
if not os.path.exists(web_output_dir):
|
|
os.makedirs(web_output_dir)
|
|
|
|
for lang in doc.languages:
|
|
if lang == 'en':
|
|
lang_dirname = os.path.join(ext_dirname, self.dirname, 'C')
|
|
else:
|
|
lang_dirname = os.path.join(ext_dirname, self.dirname, lang)
|
|
|
|
xml_file = os.path.join(lang_dirname, doc_module + '.xml')
|
|
if not os.path.exists(xml_file):
|
|
# the document had a translation available in a previous
|
|
# version, and it got removed
|
|
continue
|
|
|
|
xml_index_file = os.path.join(web_output_dir, 'index.xml.%s' % lang)
|
|
if not app.rebuild_all and (app.rebuild_language is None or
|
|
lang != app.rebuild_language) and os.path.exists(xml_index_file):
|
|
mtime = os.stat(xml_index_file)[stat.ST_MTIME]
|
|
if mtime > max(self.mtime_tarball, self.mtime_xslt_files):
|
|
logging.debug('using already generated doc in %s' % lang)
|
|
try:
|
|
tree = self.process_xml_index(xml_index_file, doc, lang)
|
|
except errors.DepreciatedDocumentation:
|
|
break
|
|
continue
|
|
|
|
if 'missing-id-on-top-book-element' in quirks:
|
|
# Evolution documentation top element is currently <book
|
|
# lang="en"> but the gnome-doc-utils stylesheets are
|
|
# looking for the id # attribute to get filename.
|
|
# -- http://bugzilla.gnome.org/show_bug.cgi?id=462811
|
|
t = open(xml_file).read()
|
|
open(xml_file + '.fixed', 'w').write(t.replace('\n<book ', '\n<book id="index" '))
|
|
xml_file = xml_file + '.fixed'
|
|
|
|
# format docbook into html files
|
|
cmd = ['xsltproc', '--output', web_output_dir + '/',
|
|
'--nonet', '--xinclude',
|
|
'--stringparam', 'libgo.lang', lang,
|
|
'--stringparam', 'libgo.channel', self.channel,
|
|
self.db2html_xsl_file, xml_file]
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
|
|
if not os.path.exists(xml_index_file):
|
|
logging.warn('no index file were created for %s' % doc_module)
|
|
continue
|
|
|
|
if doc_figures:
|
|
# and copy images/
|
|
logging.debug('copying figures')
|
|
for doc_figure in doc_figures:
|
|
src = os.path.join(lang_dirname, doc_figure)
|
|
if not os.path.exists(src):
|
|
continue
|
|
dst = os.path.join(web_output_dir, doc_figure + '.%s' % lang)
|
|
if not os.path.exists(os.path.split(dst)[0]):
|
|
os.makedirs(os.path.split(dst)[0])
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
try:
|
|
tree = self.process_xml_index(xml_index_file, doc, lang)
|
|
except errors.DepreciatedDocumentation:
|
|
break
|
|
|
|
# most documentation have @id == 'index', which is perfect for web
|
|
# publishing, for others, create a symlink from index.html.@lang.
|
|
html_index_file = tree.getroot().attrib.get('index')
|
|
if not html_index_file:
|
|
logging.warn('empty html index file for module %s' % doc_module)
|
|
elif html_index_file != 'index':
|
|
link_html_index_file = os.path.join(
|
|
web_output_dir, 'index.html.%s' % lang)
|
|
try:
|
|
os.symlink('%s.html.%s' % (html_index_file, lang), link_html_index_file)
|
|
except OSError:
|
|
logging.warn('failed to create symlink to index file for module %s' % doc_module)
|
|
|
|
self.install_version_symlinks(doc)
|
|
|
|
|
|
def process_xml_index(self, xml_index_file, doc, lang):
|
|
tree = ET.parse(xml_index_file)
|
|
|
|
if tree.find('title') is not None and \
|
|
tree.find('title').text == 'Problem showing document':
|
|
# title used in gnome-panel for depreciated documentation (such
|
|
# as window-list applet, moved to user guide); abort now, and
|
|
# remove this version. Note it would be much easier if this
|
|
# could be detected earlier, perhaps with a marker in
|
|
# Makefile.am or the OMF file.
|
|
doc.versions.remove(self.one_dot_version)
|
|
if len(doc.versions) == 0:
|
|
# there were no other version, remove it completely
|
|
app.documents.remove(doc)
|
|
else:
|
|
# there was another version, fix up path to point to that
|
|
# one
|
|
previous_one_dot_version = re.match(r'\d+\.\d+',
|
|
doc.versions[-1]).group()
|
|
if doc.version_keywords.get(previous_one_dot_version) != 'stable':
|
|
doc.path = '/' + os.path.join(self.channel, self.modulename,
|
|
previous_one_dot_version) + '/'
|
|
raise errors.DepreciatedDocumentation()
|
|
|
|
if tree.find('title') is not None and tree.find('title').text:
|
|
doc.title[lang] = tree.find('title').text
|
|
elif tree.find('{http://www.w3.org/1999/xhtml}title') is not None and \
|
|
tree.find('{http://www.w3.org/1999/xhtml}title').text:
|
|
doc.title[lang] = tree.find('{http://www.w3.org/1999/xhtml}title').text
|
|
if tree.find('abstract') is not None and tree.find('abstract').text:
|
|
doc.abstract[lang] = tree.find('abstract').text
|
|
elif tree.find('{http://www.w3.org/1999/xhtml}abstract') is not None and \
|
|
tree.find('{http://www.w3.org/1999/xhtml}abstract').text:
|
|
doc.abstract[lang] = tree.find('{http://www.w3.org/1999/xhtml}abstract').text
|
|
|
|
return tree
|
|
|
|
|
|
class HtmlFilesModule(DocModule):
|
|
transform_mode = None
|
|
|
|
html2html_xsl_file = os.path.join(data_dir, 'xslt', 'html2html.xsl')
|
|
|
|
related_xsl_files = ['html2html.xsl', 'heading.xsl']
|
|
|
|
def __init__(self, tar, tarinfo, tarball_doc_elem):
|
|
DocModule.__init__(self, tar, tarinfo, None)
|
|
self.tarball_doc_elem = tarball_doc_elem
|
|
self.modulename = self.tarball_doc_elem.attrib.get('doc_module')
|
|
if self.tarball_doc_elem.find('transform-mode') is not None:
|
|
self.transform_mode = self.tarball_doc_elem.find('transform-mode').text
|
|
|
|
def setup_channel(self):
|
|
self.channel = self.tarball_doc_elem.attrib.get('channel')
|
|
|
|
def __str__(self):
|
|
return 'HTML files module at %s' % self.dirname
|
|
|
|
def process(self):
|
|
doc_module = self.modulename
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts')
|
|
|
|
web_output_dir = os.path.join(app.config.output_dir, self.channel,
|
|
doc_module, self.one_dot_version)
|
|
if not os.path.exists(web_output_dir):
|
|
os.makedirs(web_output_dir)
|
|
|
|
if not app.rebuild_all and os.path.exists(os.path.join(web_output_dir, 'index.html')):
|
|
mtime = os.stat(os.path.join(web_output_dir, 'index.html'))[stat.ST_MTIME]
|
|
else:
|
|
mtime = 0
|
|
|
|
if mtime > max(self.mtime_tarball, self.mtime_xslt_files):
|
|
logging.debug('using already generated doc')
|
|
else:
|
|
logging.info('generating doc in %s' % web_output_dir[len(app.config.output_dir):])
|
|
|
|
if html5lib:
|
|
# convert files to XML, then process them with xsltproc
|
|
# to get library.gnome.org look
|
|
|
|
logging.debug('transforming files shipped with tarball')
|
|
parser = html5lib.HTMLParser()
|
|
|
|
for filename in os.listdir(os.path.join(ext_dirname, self.dirname)):
|
|
src = os.path.join(ext_dirname, self.dirname, filename)
|
|
dst = os.path.join(web_output_dir, filename)
|
|
if not filename.endswith('.html'):
|
|
continue
|
|
doc = parser.parse(open(src))
|
|
doc.childNodes[-1].attributes['xmlns'] = 'http://www.w3.org/1999/xhtml'
|
|
temporary = tempfile.NamedTemporaryFile()
|
|
temporary.write(doc.toxml())
|
|
temporary.flush()
|
|
|
|
cmd = ['xsltproc', '--output', dst,
|
|
'--nonet', '--xinclude',
|
|
self.html2html_xsl_file,
|
|
os.path.join(ext_dirname,
|
|
self.dirname, temporary.name)]
|
|
if self.transform_mode:
|
|
cmd.insert(-2, '--stringparam')
|
|
cmd.insert(-2, 'libgo.h2hmode')
|
|
cmd.insert(-2, self.transform_mode)
|
|
rc = subprocess.call(cmd)
|
|
else:
|
|
# simply copy HTML files shipped in tarball
|
|
logging.debug('copying files shipped with tarball')
|
|
for filename in os.listdir(os.path.join(ext_dirname, self.dirname)):
|
|
src = os.path.join(ext_dirname, self.dirname, filename)
|
|
dst = os.path.join(web_output_dir, filename)
|
|
if not filename.endswith('.html'):
|
|
continue
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
# copy non-html files
|
|
for filename in os.listdir(os.path.join(ext_dirname, self.dirname)):
|
|
src = os.path.join(ext_dirname, self.dirname, filename)
|
|
dst = os.path.join(web_output_dir, filename)
|
|
if filename.endswith('.html'):
|
|
continue
|
|
if os.path.isdir(src):
|
|
if os.path.exists(dst):
|
|
shutil.rmtree(dst)
|
|
shutil.copytree(src, dst)
|
|
else:
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
doc = self.get_libgo_document(['en'])
|
|
if not doc:
|
|
return
|
|
|
|
doc.category = self.tarball_doc_elem.get('category')
|
|
doc.toc_id = doc.category
|
|
|
|
if self.tarball_doc_elem.find('index') is not None:
|
|
path = os.path.join(web_output_dir, 'index.html')
|
|
if os.path.islink(path):
|
|
os.unlink(path)
|
|
os.symlink(self.tarball_doc_elem.find('index').text, path)
|
|
|
|
self.install_version_symlinks(doc)
|
|
|
|
|
|
class Formatter(logging.Formatter):
|
|
def __init__(self):
|
|
term = os.environ.get('TERM', '')
|
|
self.is_screen = (term == 'screen')
|
|
logging.Formatter.__init__(self)
|
|
|
|
def format(self, record):
|
|
if self.is_screen and record.levelname[0] == 'I':
|
|
sys.stdout.write('\033klgo: %s\033\\' % record.msg)
|
|
sys.stdout.flush()
|
|
return '%c: %s' % (record.levelname[0], record.msg)
|
|
|
|
|
|
class Overlay:
|
|
def __init__(self, overlay_file):
|
|
self.tree = ET.parse(overlay_file)
|
|
self.modified_docs = {}
|
|
self.new_docs = []
|
|
self.more_tarball_docs = {}
|
|
self.quirks = {}
|
|
|
|
for doc in self.tree.findall('/documents/document'):
|
|
if 'doc_module' in doc.attrib:
|
|
# modifying an existing document
|
|
self.modified_docs[(
|
|
doc.attrib['doc_module'], doc.attrib['channel'])] = doc
|
|
|
|
if not 'doc_module' in doc.attrib or (
|
|
doc.find('new') is not None or doc.find('local') is not None):
|
|
# new document
|
|
self.new_docs.append(doc)
|
|
|
|
if 'matching_tarball' in doc.attrib:
|
|
tarball = doc.attrib['matching_tarball']
|
|
if not tarball in self.more_tarball_docs:
|
|
self.more_tarball_docs[tarball] = []
|
|
tarball_docs = self.more_tarball_docs[tarball]
|
|
tarball_docs.append(doc)
|
|
|
|
self.toc_mapping = {}
|
|
for mapping in self.tree.findall('/subsections/map'):
|
|
channel = mapping.attrib.get('channel')
|
|
sectionid = mapping.attrib.get('id')
|
|
subsection = mapping.attrib.get('subsection')
|
|
self.toc_mapping[(channel, sectionid)] = subsection
|
|
|
|
for quirks in self.tree.findall('/quirks'):
|
|
self.quirks[(quirks.attrib['doc_module'], quirks.attrib['channel'])] = quirks
|
|
|
|
def apply(self, document):
|
|
if (document.channel, document.toc_id) in self.toc_mapping:
|
|
document.subsection = self.toc_mapping[(document.channel, document.toc_id)]
|
|
|
|
key = (document.module, document.channel)
|
|
overlay = self.modified_docs.get(key)
|
|
if overlay is None:
|
|
return
|
|
if overlay.find('title') is not None:
|
|
for title in overlay.findall('title'):
|
|
lang = title.attrib.get(
|
|
'{http://www.w3.org/XML/1998/namespace}lang', 'en')
|
|
document.title[lang] = title.text
|
|
if overlay.find('abstract') is not None:
|
|
for abstract in overlay.findall('abstract'):
|
|
lang = abstract.attrib.get(
|
|
'{http://www.w3.org/XML/1998/namespace}lang', 'en')
|
|
document.abstract[lang] = abstract.text
|
|
if overlay.find('subsection') is not None:
|
|
document.subsection = overlay.find('subsection').text
|
|
if overlay.find('category') is not None:
|
|
document.toc_id = overlay.find('category').text
|
|
if overlay.attrib.get('weight'):
|
|
document.weight = overlay.attrib.get('weight')
|
|
|
|
if overlay.find('keywords') is not None:
|
|
for keyword in overlay.findall('keywords/keyword'):
|
|
document.keywords.append(keyword.text)
|
|
|
|
def get_channel_overlay(self, module, current_channel):
|
|
for doc in self.tree.findall('/documents/document'):
|
|
if doc.attrib.get('doc_module') != module:
|
|
continue
|
|
if doc.attrib.get('old-channel') == current_channel:
|
|
return doc.attrib.get('channel')
|
|
return current_channel
|
|
|
|
def get_new_docs(self):
|
|
l = []
|
|
for overlay in self.new_docs:
|
|
doc = RemoteDocument(overlay)
|
|
self.apply(doc)
|
|
l.append(doc)
|
|
return l
|
|
|
|
def get_section_weight(self, section_id):
|
|
for section in self.tree.findall('subsections/subsection'):
|
|
if section.attrib.get('id') == section_id:
|
|
return float(section.attrib.get('weight', 0.5))
|
|
return 0.5
|
|
|
|
def get_subindexes(self, channel):
|
|
for subindexes in self.tree.findall('subsections/subindexes'):
|
|
if subindexes.attrib.get('channel') != channel:
|
|
return
|
|
|
|
return [SubIndex(x) for x in subindexes.findall('subindex')]
|
|
|
|
def get_quirks(self, doc):
|
|
key = (doc.modulename, doc.channel)
|
|
quirks = self.quirks.get(key)
|
|
if quirks is None:
|
|
return []
|
|
q = []
|
|
for quirk in quirks.findall('quirk'):
|
|
min_version = quirk.attrib.get('appears-in')
|
|
max_version = quirk.attrib.get('fixed-in')
|
|
if min_version and version_cmp(min_version, doc.version) > 0:
|
|
continue
|
|
if max_version and version_cmp(max_version, doc.version) <= 0:
|
|
continue
|
|
q.append(quirk.text)
|
|
return q
|
|
|
|
|
|
class FtpDotGnomeDotOrg:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
if self.config.ftp_gnome_org_local_copy:
|
|
self.ftp_gnome_org_local_copy = self.config.ftp_gnome_org_local_copy
|
|
self.download = self.download_local
|
|
self.listdir = self.listdir_local
|
|
else:
|
|
self.ftp_gnome_org_cache_dir = os.path.join(
|
|
config.download_cache_dir, 'ftp.gnome.org')
|
|
if not os.path.exists(self.ftp_gnome_org_cache_dir):
|
|
os.makedirs(self.ftp_gnome_org_cache_dir)
|
|
|
|
def download(self, filename):
|
|
cache_filename = os.path.join(self.ftp_gnome_org_cache_dir, filename)
|
|
cache_dir = os.path.split(cache_filename)[0]
|
|
if not os.path.exists(cache_dir):
|
|
os.makedirs(cache_dir)
|
|
# TODO: support for self.config.use_latest_version
|
|
if os.path.exists(cache_filename) and os.stat(cache_filename)[stat.ST_SIZE]:
|
|
logging.debug('using cache of ftp://ftp.gnome.org/%s' % filename)
|
|
return (cache_filename, open(cache_filename))
|
|
logging.info('downloading ftp://ftp.gnome.org/%s' % filename)
|
|
try:
|
|
open(cache_filename, 'w').write(
|
|
urllib2.urlopen('ftp://ftp.gnome.org/' + filename).read())
|
|
except IOError:
|
|
raise
|
|
return (cache_filename, open(cache_filename))
|
|
|
|
def download_local(self, filename):
|
|
local_filename = os.path.join(self.ftp_gnome_org_local_copy, filename)
|
|
if self.config.use_latest_version:
|
|
dirname, basename = os.path.split(local_filename)
|
|
module_name = basename.split('-')[0]
|
|
try:
|
|
latest_version = [x for x in os.listdir(dirname) if x.startswith('LATEST-IS-')]
|
|
if latest_version:
|
|
latest_base = '%s-%s.' % (module_name, latest_version[0].split('-')[-1])
|
|
new_basename = [x for x in os.listdir(dirname) if \
|
|
x.startswith(latest_base) and (
|
|
x.endswith('.tar.gz') or x.endswith('.tar.bz2'))]
|
|
if new_basename:
|
|
local_filename = os.path.join(dirname, new_basename[0])
|
|
logging.debug('using %s instead of %s' % (new_basename[0], basename))
|
|
except OSError:
|
|
pass
|
|
return (local_filename, open(local_filename))
|
|
|
|
def listdir(self, dirname):
|
|
l = []
|
|
for line in urllib2.urlopen('ftp://ftp.gnome.org/' + dirname):
|
|
l.append(line.strip().split()[-1])
|
|
return l
|
|
|
|
def listdir_local(self, dirname):
|
|
return sorted(os.listdir(os.path.join(self.ftp_gnome_org_local_copy, dirname)))
|
|
|
|
|
|
class Lgo:
|
|
indexes_xsl_file = os.path.join(data_dir, 'xslt', 'indexes.xsl')
|
|
javascript_dir = os.path.join(data_dir, 'js')
|
|
skin_dir = os.path.join(data_dir, 'skin')
|
|
|
|
rebuild_all = False
|
|
debug = False
|
|
|
|
def __init__(self):
|
|
self.documents = []
|
|
|
|
parser = OptionParser()
|
|
parser.add_option('-c', '--config', dest = 'config')
|
|
parser.add_option('-v', '--verbose',
|
|
action = 'count', dest = 'verbose', default = 0,
|
|
help = 'verbosity level (more -v for more verbose)')
|
|
parser.add_option('--rebuild', dest = 'rebuild_module',
|
|
help = 'rebuild documentation from FILENAME', metavar = 'FILENAME')
|
|
parser.add_option('--rebuild-all',
|
|
action = 'store_true', dest = 'rebuild_all',
|
|
help = 'rebuild all documents (even those that were already built)')
|
|
parser.add_option('--rebuild-language', dest = 'rebuild_language',
|
|
help = 'rebuild all documents in LANGUAGE', metavar = 'LANGUAGE')
|
|
parser.add_option('--skip-extra-tarballs',
|
|
action = 'store_false', dest = 'skip_extra_tarballs',
|
|
help = "don't look for documentation extra tarballs")
|
|
self.options, args = parser.parse_args()
|
|
|
|
logging.basicConfig(level = 10 + logging.CRITICAL - self.options.verbose*10,
|
|
formatter = Formatter())
|
|
logging.getLogger().handlers[0].setFormatter(Formatter())
|
|
|
|
self.debug = (self.options.verbose >= 5)
|
|
self.rebuild_all = self.options.rebuild_all
|
|
self.rebuild_language = self.options.rebuild_language
|
|
|
|
if self.options.config:
|
|
self.config = Config(filename = self.options.config)
|
|
else:
|
|
self.config = Config()
|
|
|
|
self.check_sanity()
|
|
|
|
def run(self):
|
|
self.ftp_gnome_org = FtpDotGnomeDotOrg(self.config)
|
|
self.overlay = Overlay(os.path.join(data_dir, 'overlay.xml'))
|
|
self.get_yelp_categories()
|
|
|
|
if self.options.rebuild_module:
|
|
self.rebuild_all = True
|
|
for doc_module in self.extract_modules(self.options.rebuild_module):
|
|
doc_module.process()
|
|
sys.exit(0)
|
|
|
|
self.copy_static_files()
|
|
self.process_releases()
|
|
if not self.options.skip_extra_tarballs:
|
|
self.process_extra_tarballs()
|
|
self.apply_overlay()
|
|
self.generate_indexes()
|
|
self.generate_dbm_symbols_file()
|
|
self.generate_static_pages()
|
|
|
|
def check_sanity(self):
|
|
for filename in [os.path.join(data_dir, 'overlay.xml'),
|
|
os.path.join(data_dir, 'catalog.xml')]:
|
|
if not os.path.exists(filename):
|
|
print >> sys.stderr, '%s is missing, you should run make'
|
|
sys.exit(1)
|
|
|
|
if not self.config.output_dir.endswith(os.path.sep):
|
|
logging.warning('output dir should end with slash')
|
|
self.config.output_dir += os.path.sep
|
|
|
|
def get_yelp_categories(self):
|
|
logging.info('Getting categories from Yelp')
|
|
|
|
scrollkeeper_xml = os.path.join(data_dir, 'externals', 'scrollkeeper.xml')
|
|
toc_xml = os.path.join(data_dir, 'externals', 'toc.xml')
|
|
|
|
if not os.path.exists(scrollkeeper_xml) or not os.path.exists(toc_xml):
|
|
filename = FtpDotGnomeDotOrg(self.config).download(
|
|
'pub/GNOME/sources/yelp/2.18/yelp-2.18.1.tar.bz2')[0]
|
|
|
|
tar = tarfile.open(filename, 'r')
|
|
done = 0
|
|
for tarinfo in tar:
|
|
filename = os.path.basename(tarinfo.name)
|
|
if filename == 'scrollkeeper.xml':
|
|
open(scrollkeeper_xml, 'w').write(tar.extractfile(tarinfo).read())
|
|
done += 1
|
|
elif filename == 'toc.xml':
|
|
open(toc_xml, 'w').write(tar.extractfile(tarinfo).read())
|
|
done += 1
|
|
if done == 2:
|
|
break
|
|
|
|
yelp_toc_tree = ET.fromstring(open(scrollkeeper_xml).read())
|
|
self.toc_mapping = {}
|
|
for subtoc in yelp_toc_tree.findall('toc'):
|
|
sub_id = subtoc.attrib['id']
|
|
for subject in subtoc.findall('subject'):
|
|
self.toc_mapping[subject.attrib['category']] = sub_id
|
|
|
|
def copy_static_files(self):
|
|
if not os.path.exists(os.path.join(self.config.output_dir, 'js')):
|
|
os.makedirs(os.path.join(self.config.output_dir, 'js'))
|
|
if not os.path.exists(os.path.join(self.config.output_dir, 'skin')):
|
|
os.makedirs(os.path.join(self.config.output_dir, 'skin'))
|
|
if not os.path.exists(os.path.join(self.config.output_dir, 'skin/icons')):
|
|
os.makedirs(os.path.join(self.config.output_dir, 'skin/icons'))
|
|
|
|
for src in glob.glob('%s/*.js' % self.javascript_dir):
|
|
dst = os.path.join(self.config.output_dir, 'js', os.path.basename(src))
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
for src in glob.glob('%s/*.css' % self.skin_dir) + \
|
|
glob.glob('%s/*.png' % self.skin_dir) + \
|
|
glob.glob('%s/*.gif' % self.skin_dir):
|
|
dst = os.path.join(self.config.output_dir, 'skin', os.path.basename(src))
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
for src in glob.glob('%s/icons/*.png' % self.skin_dir):
|
|
dst = os.path.join(self.config.output_dir, 'skin/icons', os.path.basename(src))
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
def process_releases(self):
|
|
'''Download GNOME releases'''
|
|
releases = self.ftp_gnome_org.listdir('pub/GNOME/teams/releng/')
|
|
|
|
for i, r in enumerate(releases[:]):
|
|
if self.config.version_min and version_cmp(r, self.config.version_min) < 0:
|
|
continue
|
|
if self.config.version_max and version_cmp(r, self.config.version_max) > 0:
|
|
continue
|
|
|
|
if i < len(releases)-1 and releases[i+1].startswith(re.match(r'\d+\.\d+\.', r).group()):
|
|
# next release has the same major.minor version number, so skip
|
|
# this one and get the newer one later
|
|
logging.debug('skipping release %s, not the last in serie' % r)
|
|
continue
|
|
|
|
if int(r.split('.')[1]) % 2 == 1:
|
|
# odd release, development, skip unless this is the current
|
|
# development serie
|
|
if not releases[-1].startswith(re.match(r'\d+\.\d+\.', r).group()):
|
|
logging.debug('skipping release %s, not the last in serie' % r)
|
|
continue
|
|
|
|
if version_cmp(r, '2.19.0') < 0:
|
|
url = 'pub/GNOME/teams/releng/%(r)s/gnome-%(r)s.modules'
|
|
else:
|
|
# from 2.19.0, the jhbuild moduleset structure changed
|
|
url = 'pub/GNOME/teams/releng/%(r)s/gnome-suites-%(r)s.modules'
|
|
|
|
logging.info('Getting GNOME release: %s' % r)
|
|
moduleset = self.ftp_gnome_org.download(url % {'r': r})[1]
|
|
self.process_moduleset(moduleset, r)
|
|
|
|
def download(self, url):
|
|
if url.startswith('http://download.gnome.org/') or \
|
|
url.startswith('http://ftp.gnome.org/'):
|
|
url = url.replace('http://download.gnome.org/', 'pub/GNOME/')
|
|
url = url.replace('http://ftp.gnome.org/', '')
|
|
try:
|
|
filename = self.ftp_gnome_org.download(url)[0]
|
|
except IOError:
|
|
logging.error('error downloading %s' % url)
|
|
return
|
|
else:
|
|
filename = download(url)
|
|
|
|
return filename
|
|
|
|
def process_moduleset(self, moduleset, version_number):
|
|
'''Download tarballs from a module set'''
|
|
|
|
doc_modules = []
|
|
|
|
tree = ET.parse(moduleset)
|
|
for tarball in tree.findall('tarball'):
|
|
if self.config.modules is not None and not tarball.attrib['id'] in self.config.modules:
|
|
continue
|
|
href = tarball.find('source').attrib['href']
|
|
filename = self.download(href)
|
|
if not filename:
|
|
continue
|
|
logging.info('extracting module %s (from %s moduleset)' % (
|
|
tarball.attrib['id'], version_number))
|
|
doc_modules.extend(self.extract_modules(filename))
|
|
|
|
gduxrefs = ET.Element('gduxrefs')
|
|
for doc_module in doc_modules:
|
|
if not isinstance(doc_module, GnomeDocUtilsModule):
|
|
continue
|
|
element = ET.SubElement(gduxrefs, 'doc')
|
|
element.set('id', doc_module.modulename)
|
|
element.set('path', doc_module.path)
|
|
|
|
# XXX: this is writing in build dir, not really nice, but this allows
|
|
# for a known location relative to the XSL files.
|
|
tmp_dirname = os.path.join(data_dir, 'tmp')
|
|
if not os.path.exists(tmp_dirname):
|
|
os.makedirs(tmp_dirname)
|
|
|
|
tree = ET.ElementTree(gduxrefs)
|
|
tree.write(os.path.join(tmp_dirname, 'gduxrefs.xml'))
|
|
|
|
for doc_module in doc_modules:
|
|
logging.info('processing %s (from %s moduleset)' % (
|
|
doc_module, version_number))
|
|
doc_module.process()
|
|
|
|
def extract_modules(self, filename):
|
|
logging.debug('looking for doc modules in %s' % filename)
|
|
doc_modules = []
|
|
|
|
mtime = os.stat(filename)[stat.ST_MTIME]
|
|
|
|
if self.config.fast_mode:
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts',
|
|
os.path.splitext(os.path.splitext(os.path.basename(filename))[0])[0])
|
|
stamp_file = ext_dirname + '.extract-stamp'
|
|
else:
|
|
stamp_file = None
|
|
|
|
if stamp_file and os.path.exists(stamp_file) and not os.path.exists(ext_dirname):
|
|
# file was extracted once, and no doc module were found inside
|
|
return []
|
|
elif stamp_file and os.path.exists(stamp_file):
|
|
tar = utils.FakeTarFile(ext_dirname)
|
|
else:
|
|
tar = tarfile.open(filename, 'r')
|
|
|
|
doc_version = os.path.splitext(tar.name)[0].split('-')[-1]
|
|
|
|
base_tarball_name = os.path.basename(filename).rsplit('-', 1)[0]
|
|
more_tarball_docs = self.overlay.more_tarball_docs.get(
|
|
base_tarball_name, [])[:]
|
|
|
|
for tarinfo in tar:
|
|
doc = None
|
|
if os.path.split(tarinfo.name)[-1] == 'Makefile.am':
|
|
fd = tar.extractfile(tarinfo)
|
|
makefile_am = fd.read()
|
|
makefile_am = makefile_am.replace('\\\n', ' ')
|
|
if 'DOC_MODULE' in makefile_am and \
|
|
'include $(top_srcdir)/gnome-doc-utils.make' in makefile_am:
|
|
logging.debug('found usage of gnome-doc-utils in %s' % tarinfo.name)
|
|
doc = GnomeDocUtilsModule(tar, tarinfo, makefile_am)
|
|
elif 'include $(top_srcdir)/gtk-doc.make' in makefile_am:
|
|
logging.debug('found usage of gtk-doc in %s' % tarinfo.name)
|
|
doc = GtkDocModule(tar, tarinfo, makefile_am)
|
|
else:
|
|
continue
|
|
|
|
if not doc.modulename or doc.modulename in self.config.blacklist:
|
|
continue
|
|
else:
|
|
for more_doc in more_tarball_docs[:]:
|
|
if 'minimum-version' in more_doc.attrib:
|
|
if version_cmp(doc_version, more_doc.attrib.get('minimum-version')) < 0:
|
|
more_tarball_docs.remove(more_doc)
|
|
continue
|
|
if tarinfo.name.endswith(more_doc.attrib.get('dir')):
|
|
doc = HtmlFilesModule(tar, tarinfo, more_doc)
|
|
more_tarball_docs.remove(more_doc)
|
|
continue
|
|
|
|
if doc:
|
|
doc.filename = filename
|
|
doc.mtime_tarball = mtime
|
|
doc.extract()
|
|
doc.setup_channel()
|
|
doc.setup_path()
|
|
doc_modules.append(doc)
|
|
|
|
tar.close()
|
|
|
|
if stamp_file:
|
|
# touch extract stamp file
|
|
file(stamp_file, 'w').close()
|
|
|
|
return doc_modules
|
|
|
|
def process_extra_tarballs(self):
|
|
if self.config.extra_tarballs:
|
|
for url in self.config.extra_tarballs:
|
|
filename = self.download(url)
|
|
if not filename:
|
|
continue
|
|
for doc_module in self.extract_modules(filename):
|
|
doc_module.process()
|
|
|
|
def apply_overlay(self):
|
|
logging.info('Applying overlay')
|
|
for doc in self.documents:
|
|
self.overlay.apply(doc)
|
|
|
|
self.documents.extend(self.overlay.get_new_docs())
|
|
|
|
def generate_indexes(self):
|
|
logging.info('generating indexes')
|
|
indexes = ET.Element('indexes')
|
|
indexes.set('svnid', _svn_id)
|
|
|
|
# get all possible languages
|
|
languages = {}
|
|
for doc in self.documents:
|
|
for lang in doc.languages:
|
|
if lang == 'C':
|
|
continue # ignore
|
|
if self.config.languages and not lang in self.config.languages:
|
|
continue
|
|
languages[lang] = True
|
|
|
|
for lang in languages.keys():
|
|
home = ET.SubElement(indexes, 'home')
|
|
home.set('lang', lang)
|
|
|
|
for channel in ('users', 'devel', 'admin'):
|
|
docs = [x for x in self.documents if x.channel == channel]
|
|
|
|
if not docs:
|
|
continue
|
|
|
|
for lang in languages.keys():
|
|
logging.debug('generating index for lang %s' % lang)
|
|
|
|
sections = {}
|
|
for x in docs:
|
|
sections[x.toc_id] = True
|
|
sections = sections.keys()
|
|
sections.sort()
|
|
|
|
docs.sort(lambda x,y: cmp(x.title.get(lang), y.title.get(lang)))
|
|
|
|
subindexes = self.overlay.get_subindexes(channel)
|
|
if not subindexes:
|
|
index = ET.SubElement(indexes, 'index')
|
|
index.set('lang', lang)
|
|
index.set('channel', channel)
|
|
for section in sections:
|
|
section_docs = [x for x in docs if x.toc_id == section]
|
|
if not section_docs:
|
|
continue
|
|
self.create_section(index, section, section_docs, lang)
|
|
else:
|
|
remaining_sections = sections[:]
|
|
subindex_index = ET.SubElement(indexes, 'index')
|
|
subindex_index.set('lang', lang)
|
|
subindex_index.set('channel', channel)
|
|
|
|
for subindex in subindexes:
|
|
local_sections = [x for x in sections if x in subindex.sections]
|
|
if not local_sections:
|
|
continue
|
|
|
|
index = subindex.create_element(subindex_index,
|
|
channel, lang)
|
|
|
|
for section in local_sections:
|
|
remaining_sections.remove(section)
|
|
section_docs = [x for x in docs if x.toc_id == section]
|
|
if not section_docs:
|
|
continue
|
|
self.create_section(index, section, section_docs, lang)
|
|
if remaining_sections:
|
|
logging.warn('%s channel is missing some sections: %s' % (
|
|
channel, ', '.join(remaining_sections)))
|
|
|
|
idx_dirname = os.path.join(self.config.private_dir, 'indexes')
|
|
if not os.path.exists(idx_dirname):
|
|
os.makedirs(idx_dirname)
|
|
|
|
tree = ET.ElementTree(indexes)
|
|
tree.write(os.path.join(idx_dirname, 'indexes.xml'))
|
|
|
|
self.generate_html_indexes()
|
|
|
|
def create_section(self, index, section, section_docs, lang):
|
|
section_node = ET.SubElement(index, 'section')
|
|
section_node.set('toc_id', section)
|
|
section_node.set('weight', str(
|
|
self.overlay.get_section_weight(section)))
|
|
|
|
subsections = {}
|
|
for x in section_docs:
|
|
subsections[x.subsection] = True
|
|
subsections = subsections.keys()
|
|
|
|
for subsection in subsections:
|
|
subsection_docs = [x for x in section_docs if x.subsection == subsection]
|
|
|
|
if subsection is None:
|
|
parent_elem = section_node
|
|
else:
|
|
parent_elem = ET.SubElement(section_node, 'section')
|
|
parent_elem.set('title', subsection)
|
|
parent_elem.set('weight', str(
|
|
self.overlay.get_section_weight(subsection)))
|
|
|
|
for doc in subsection_docs:
|
|
logging.debug('generating index for module %s' % doc.module)
|
|
if lang in doc.languages:
|
|
# document is available in the requested
|
|
# language, perfect.
|
|
doc_lang = lang
|
|
elif lang[:2] in doc.languages:
|
|
# mapping to "general" language, for example
|
|
# from en_GB to en, from fr_BE to fr...
|
|
doc_lang = lang[:2]
|
|
elif [x for x in doc.languages if x[:2] == lang]:
|
|
# mapping to "country" language, for
|
|
# example from pt to pt_BR
|
|
doc_lang = [x for x in doc.languages if x[:2] == lang][0]
|
|
else:
|
|
# fallback to English
|
|
doc_lang = 'en'
|
|
|
|
doc.create_element(parent_elem, doc_lang,
|
|
original_language = lang)
|
|
|
|
def generate_html_indexes(self):
|
|
idx_filename = os.path.join(self.config.private_dir, 'indexes', 'indexes.xml')
|
|
|
|
cmd = ['xsltproc', '--output', self.config.output_dir,
|
|
'--nonet', '--xinclude',
|
|
self.indexes_xsl_file, idx_filename]
|
|
|
|
if self.debug:
|
|
cmd.insert(-2, '--param')
|
|
cmd.insert(-2, 'libgo.debug')
|
|
cmd.insert(-2, 'true()')
|
|
if self.config.symbols_dbm_filepath:
|
|
cmd.insert(-2, '--param')
|
|
cmd.insert(-2, 'libgo.dbm_support')
|
|
cmd.insert(-2, 'true()')
|
|
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
|
|
def generate_static_pages(self):
|
|
try:
|
|
doc_linguas = re.findall(r'DOC_LINGUAS\s+=[\t ](.*)',
|
|
file(os.path.join(data_dir, 'pages', 'Makefile.am')).read())[0].split()
|
|
doc_linguas.append('C')
|
|
except IndexError:
|
|
doc_linguas = ['C']
|
|
|
|
if app.config.languages:
|
|
for lang in doc_linguas[:]:
|
|
if lang not in self.config.languages + ['C']:
|
|
doc_linguas.remove(lang)
|
|
|
|
web_output_dir = os.path.join(self.config.output_dir, 'about')
|
|
for lang in doc_linguas:
|
|
xml_file = os.path.join(os.path.join(data_dir, 'pages', lang, 'libgo.xml'))
|
|
if lang == 'C':
|
|
lang = 'en'
|
|
cmd = ['xsltproc', '--output', web_output_dir + '/',
|
|
'--nonet', '--xinclude',
|
|
'--stringparam', 'libgo.lang', lang,
|
|
'--stringparam', 'libgo.channel', 'about',
|
|
'--param', 'db2html.navbar.bottom', 'false()',
|
|
GnomeDocUtilsModule.db2html_xsl_file, xml_file]
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
|
|
def generate_dbm_symbols_file(self):
|
|
if not self.config.symbols_dbm_filepath:
|
|
return
|
|
logging.info('generating dbm symbols file')
|
|
|
|
if self.rebuild_all and os.path.exists(self.config.symbols_dbm_filepath):
|
|
os.unlink(self.config.symbols_dbm_filepath)
|
|
|
|
cmd = [self.config.httxt2dbm_path, '-i', '-', '-o', self.config.symbols_dbm_filepath]
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
try:
|
|
httxt2dbm = subprocess.Popen(cmd, stdin = subprocess.PIPE).stdin
|
|
except OSError:
|
|
logging.error('failed to generate dbm symbols file (OSError)')
|
|
return
|
|
|
|
for doc in self.documents:
|
|
if doc.category != 'api':
|
|
continue
|
|
if not doc.module or not doc.path:
|
|
continue
|
|
|
|
web_dir = os.path.join(app.config.output_dir, doc.path[1:])
|
|
|
|
devhelp_path = os.path.join(web_dir, '%s.devhelp2' % doc.module)
|
|
if os.path.exists(devhelp_path):
|
|
tree = ET.parse(devhelp_path)
|
|
for keyword in tree.findall('//{http://www.devhelp.net/book}keyword'):
|
|
key = keyword.attrib.get('name').replace('()', '').strip()
|
|
if ' ' in key:
|
|
# ignore keys with spaces in their name
|
|
continue
|
|
value = os.path.join(doc.path, keyword.attrib.get('link'))
|
|
print >> httxt2dbm, key, value
|
|
continue
|
|
|
|
devhelp_path = os.path.join(web_dir, '%s.devhelp' % doc.module)
|
|
if os.path.exists(devhelp_path):
|
|
tree = ET.parse(devhelp_path)
|
|
for elem in tree.findall('//{http://www.devhelp.net/book}sub') + \
|
|
tree.findall('//{http://www.devhelp.net/book}function'):
|
|
key = elem.attrib.get('name').replace('()', '').strip()
|
|
if ' ' in key:
|
|
# ignore keys with spaces in their name
|
|
continue
|
|
value = os.path.join(doc.path, elem.attrib.get('link'))
|
|
print >> httxt2dbm, key, value
|
|
|
|
httxt2dbm.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
app = Lgo()
|
|
app.run()
|
|
|