509 lines
21 KiB
Python
Executable File
509 lines
21 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# libgo - script to build library.gnome.org
|
|
# Copyright (C) 2007-2009 Frederic Peters
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
# 02110-1301 USA
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import tempfile
|
|
import urllib2
|
|
from cStringIO import StringIO
|
|
from optparse import OptionParser
|
|
import logging
|
|
try:
|
|
import elementtree.ElementTree as ET
|
|
except ImportError:
|
|
import xml.etree.ElementTree as ET
|
|
import tarfile
|
|
import stat
|
|
import subprocess
|
|
import dbm
|
|
import shutil
|
|
import socket
|
|
import __builtin__
|
|
|
|
data_dir = os.path.join(os.path.dirname(__file__), '../data')
|
|
__builtin__.__dict__['data_dir'] = data_dir
|
|
|
|
import errors
|
|
import utils
|
|
from utils import version_cmp, is_version_number
|
|
|
|
from document import Document
|
|
from overlay import Overlay
|
|
|
|
from modtypes.gnomedocbook import GnomeDocbookModule
|
|
from modtypes.gtkdoc import GtkDocModule
|
|
from modtypes.htmlfiles import HtmlFilesModule
|
|
from modtypes.mallard import MallardModule
|
|
|
|
from app import App
|
|
|
|
|
|
# timeout for downloads, so it doesn't hang on connecting to sourceforge
|
|
socket.setdefaulttimeout(10)
|
|
|
|
class Lgo(App):
|
|
'''Main Application Class'''
|
|
|
|
def run(self):
|
|
self.overlay = Overlay(os.path.join(data_dir, 'eo-overlay.xml'))
|
|
|
|
if self.options.rebuild_module:
|
|
self.rebuild_all = True
|
|
for doc_module in self.extract_modules(self.options.rebuild_module):
|
|
doc_module.process()
|
|
sys.exit(0)
|
|
|
|
if self.options.rebuild_remote:
|
|
self.generate_static_pages()
|
|
self.apply_overlay()
|
|
sys.exit(0)
|
|
|
|
self.copy_static_files()
|
|
if not self.options.skip_extra_tarballs:
|
|
self.process_extra_tarballs()
|
|
if self.config.nightly_tarballs_location:
|
|
self.process_nightly_tarballs()
|
|
self.apply_overlay()
|
|
self.generate_indexes()
|
|
self.generate_html_indexes()
|
|
self.generate_robots()
|
|
self.generate_sitemap()
|
|
self.generate_symbols_files()
|
|
self.generate_static_pages()
|
|
|
|
def process_modules(self):
|
|
'''Download GNOME releases'''
|
|
releases = self.ftp_gnome_org.listdir('pub/GNOME/teams/releng/')
|
|
|
|
releases = [x for x in releases if is_version_number(x)]
|
|
for i, r in enumerate(releases[:]):
|
|
if self.config.version_min and version_cmp(r, self.config.version_min) < 0:
|
|
continue
|
|
if self.config.version_max and version_cmp(r, self.config.version_max) > 0:
|
|
continue
|
|
|
|
if i < len(releases)-1 and releases[i+1].startswith(re.match(r'\d+\.\d+\.', r).group()) and \
|
|
not r == self.config.version_max:
|
|
# next release has the same major.minor version number, so skip
|
|
# this one and get the newer one later
|
|
logging.debug('skipping release %s, not the last in serie' % r)
|
|
continue
|
|
|
|
if int(r.split('.')[1]) % 2 == 1:
|
|
# odd release, development, skip unless this is the current
|
|
# development serie
|
|
if not releases[-1].startswith(re.match(r'\d+\.\d+\.', r).group()) and \
|
|
not r == self.config.version_max:
|
|
logging.debug('skipping release %s, not the last in serie' % r)
|
|
continue
|
|
|
|
if version_cmp(r, '2.19.0') < 0:
|
|
urls = ['pub/GNOME/teams/releng/%(r)s/gnome-%(r)s.modules']
|
|
elif version_cmp(r, '2.91.3') < 0:
|
|
urls = ['pub/GNOME/teams/releng/%(r)s/gnome-suites-%(r)s.modules']
|
|
else:
|
|
urls = ['pub/GNOME/teams/releng/%(r)s/gnome-apps-%(r)s.modules',
|
|
'pub/GNOME/teams/releng/%(r)s/gnome-suites-core-%(r)s.modules',
|
|
'pub/GNOME/teams/releng/%(r)s/gnome-suites-core-deps-%(r)s.modules']
|
|
|
|
logging.info('Getting GNOME release: %s' % r)
|
|
for url in urls:
|
|
try:
|
|
moduleset = self.ftp_gnome_org.download(url % {'r': r})[1]
|
|
except Exception, e:
|
|
logging.error('Failed retrieving %s (%s)' % (url % {'r': r}, str(e)))
|
|
continue
|
|
self.process_moduleset(moduleset, r)
|
|
|
|
if self.config.version_max is None and r == releases[-1]:
|
|
# this is the last release, and no maximum version was set,
|
|
# add all modules to extra tarballs, so development versions
|
|
# not shipped with latest GNOME are made available.
|
|
moduleset.seek(0)
|
|
self.process_latest_moduleset(moduleset)
|
|
|
|
def extract_modules(self, filename, nightly = False):
|
|
logging.debug('looking for doc modules in %s' % filename)
|
|
doc_modules = []
|
|
|
|
mtime = os.stat(filename)[stat.ST_MTIME]
|
|
|
|
if self.config.fast_mode:
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts',
|
|
os.path.splitext(os.path.splitext(os.path.basename(filename))[0])[0])
|
|
stamp_file = ext_dirname + '.extract-stamp'
|
|
else:
|
|
stamp_file = None
|
|
|
|
base_tarball_name = os.path.basename(filename).rsplit('-', 1)[0]
|
|
if nightly:
|
|
stamp_file = None
|
|
|
|
if stamp_file and os.path.exists(stamp_file) and not os.path.exists(ext_dirname):
|
|
# file was extracted once, and no doc module were found inside
|
|
return []
|
|
elif stamp_file and os.path.exists(stamp_file) and os.stat(stamp_file)[stat.ST_MTIME] < mtime:
|
|
# file was extracted but has been modified since then, remove
|
|
# extraction if it exists:
|
|
if os.path.exists(ext_dirname):
|
|
logging.debug('removing old copy of files from %s' % base_tarball_name)
|
|
shutil.rmtree(ext_dirname, ignore_errors = True)
|
|
tar = tarfile.open(filename, 'r')
|
|
elif stamp_file and os.path.exists(stamp_file):
|
|
tar = utils.FakeTarFile(ext_dirname)
|
|
else:
|
|
tar = tarfile.open(filename, 'r')
|
|
|
|
doc_version = os.path.splitext(tar.name)[0].split('-')[-1]
|
|
if doc_version.endswith('.tar'):
|
|
doc_version = doc_version[:-4]
|
|
|
|
more_tarball_docs = self.overlay.more_tarball_docs.get(
|
|
base_tarball_name, [])[:]
|
|
|
|
for more_doc in more_tarball_docs:
|
|
# don't look for docs that require a newer version of the module
|
|
if 'minimum-version' in more_doc.attrib:
|
|
if version_cmp(doc_version, more_doc.attrib.get('minimum-version')) < 0:
|
|
more_tarball_docs.remove(more_doc)
|
|
|
|
extraction_happened = False
|
|
regex_gdu = re.compile(r'include.*gnome-doc-utils.make', re.DOTALL)
|
|
for tarinfo in tar:
|
|
doc = None
|
|
if os.path.split(tarinfo.name)[-1] in ('Makefile.am', 'GNUmakefile.am'):
|
|
fd = tar.extractfile(tarinfo)
|
|
makefile_am = fd.read()
|
|
# merge lines continued with \
|
|
makefile_am = re.sub(r'\\\s*\n', r' ', makefile_am)
|
|
if 'HELP_ID' in makefile_am and '@YELP_HELP_RULES@' in makefile_am:
|
|
if '.page' in makefile_am:
|
|
logging.debug('found usage of mallard (via YELP_HELP_RULES) in %s' % tarinfo.name)
|
|
doc = MallardModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
else:
|
|
logging.debug('found usage of docbook (via YELP_HELP_RULES) in %s' % tarinfo.name)
|
|
doc = GnomeDocbookModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'DOC_ID' in makefile_am and regex_gdu.findall(makefile_am):
|
|
logging.debug('found usage of mallard in %s' % tarinfo.name)
|
|
doc = MallardModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'DOC_MODULE' in makefile_am and regex_gdu.findall(makefile_am):
|
|
logging.debug('found usage of docbook in %s' % tarinfo.name)
|
|
doc = GnomeDocbookModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'include $(top_srcdir)/gtk-doc.make' in makefile_am or \
|
|
'include $(srcdir)/gtk-doc.make' in makefile_am or \
|
|
'include gtk-doc.make' in makefile_am or \
|
|
('gtkdoc-scan' in makefile_am and not 'gtk-doc' in tarinfo.name):
|
|
logging.debug('found usage of gtk-doc in %s' % tarinfo.name)
|
|
doc = GtkDocModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'SUBDIRS = C' in makefile_am and \
|
|
os.path.basename(filename).startswith('gtk-doc-'):
|
|
logging.debug('found gtk-doc almost gnome-doc-utils manual in %s' % tarinfo.name)
|
|
makefile_am += '\nDOC_MODULE = gtk-doc-manual\n'
|
|
doc = GnomeDocbookModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
else:
|
|
continue
|
|
|
|
if '$(' in doc.modulename:
|
|
continue
|
|
|
|
if not doc.modulename or doc.modulename in self.config.blacklist:
|
|
continue
|
|
else:
|
|
for more_doc in more_tarball_docs[:]:
|
|
if not tarinfo.isdir():
|
|
continue
|
|
directory_name = tarinfo.name
|
|
if not directory_name[-1] == '/':
|
|
directory_name += '/'
|
|
if directory_name.endswith(more_doc.attrib.get('dir')):
|
|
doc = HtmlFilesModule.create_from_tar(tar, tarinfo, more_doc)
|
|
more_tarball_docs.remove(more_doc)
|
|
continue
|
|
|
|
if doc:
|
|
doc.filename = filename
|
|
doc.mtime_tarball = mtime
|
|
if extraction_happened:
|
|
doc.extract(force=True)
|
|
else:
|
|
extraction_happened = doc.extract()
|
|
doc.setup_channel()
|
|
doc.path = self.get_module_web_path(doc)
|
|
if self.config.channels is None or doc.channel in self.config.channels:
|
|
doc_modules.append(doc)
|
|
else:
|
|
logging.debug('ignoring %s, not in an appropriate channel' % doc.modulename)
|
|
|
|
if more_tarball_docs:
|
|
for more_doc in more_tarball_docs:
|
|
logging.error('[%s] overlay file mentioned %s but it was not found' % \
|
|
(base_tarball_name, more_doc.attrib.get('dir')))
|
|
|
|
tar.close()
|
|
|
|
if stamp_file:
|
|
# touch extract stamp file
|
|
if not os.path.exists(ext_dirname):
|
|
os.makedirs(ext_dirname)
|
|
file(stamp_file, 'w').close()
|
|
|
|
return doc_modules
|
|
|
|
def process_extra_tarballs(self):
|
|
if self.config.extra_tarballs:
|
|
logging.info('processing extra tarballs')
|
|
for url in self.config.extra_tarballs:
|
|
logging.debug('processing extra tarball: %s' % url)
|
|
filename = self.download(url)
|
|
if not filename:
|
|
continue
|
|
if not type(filename) is list:
|
|
filename = [filename]
|
|
|
|
for fname in filename:
|
|
for doc_module in self.extract_modules(fname):
|
|
doc_module.process()
|
|
|
|
def process_nightly_tarballs(self):
|
|
logging.info('processing nightly tarballs')
|
|
for filename in os.listdir(self.config.nightly_tarballs_location):
|
|
if not (filename.endswith('.tar.gz') or filename.endswith('.tar.bz2') or
|
|
filename.endswith('.tar.xz')):
|
|
continue
|
|
filename = os.path.join(self.config.nightly_tarballs_location, filename)
|
|
for doc_module in self.extract_modules(filename, nightly = True):
|
|
doc_module.process()
|
|
|
|
def generate_indexes(self):
|
|
logging.info('generating indexes')
|
|
indexes = ET.Element('indexes')
|
|
|
|
# get all possible languages
|
|
languages = {}
|
|
for doc in self.documents:
|
|
for lang in doc.languages:
|
|
if lang == 'C':
|
|
continue # ignore
|
|
if self.config.languages and not lang in self.config.languages:
|
|
continue
|
|
languages[lang] = True
|
|
if 'sr@Latn' in languages and 'sr@latin' in languages:
|
|
del languages['sr@Latn']
|
|
|
|
for lang in languages.keys():
|
|
home = ET.SubElement(indexes, 'home')
|
|
home.set('lang', lang)
|
|
|
|
channels = self.config.channels
|
|
if not channels:
|
|
channels = ('users', 'devel', 'admin', 'misc')
|
|
|
|
for channel in channels:
|
|
docs = [x for x in self.documents if x.channel == channel]
|
|
|
|
if not docs:
|
|
continue
|
|
|
|
for lang in languages.keys():
|
|
logging.debug('generating index for lang %s' % lang)
|
|
|
|
sections = {}
|
|
for x in docs:
|
|
if x.toc_id is None:
|
|
logging.warning('doc %s has no toc id -> default to Other' % x.module)
|
|
x.toc_id = 'Other'
|
|
sections[x.toc_id] = True
|
|
sections = sections.keys()
|
|
sections.sort()
|
|
|
|
docs.sort(lambda x,y: cmp(x.title.get(lang), y.title.get(lang)))
|
|
|
|
subindexes = self.overlay.get_subindexes(channel)
|
|
if not subindexes:
|
|
index = ET.SubElement(indexes, 'index')
|
|
index.set('lang', lang)
|
|
index.set('channel', channel)
|
|
for section in sections:
|
|
section_docs = [x for x in docs if x.toc_id == section]
|
|
if not section_docs:
|
|
continue
|
|
self.create_section(index, section, section_docs, lang)
|
|
else:
|
|
remaining_sections = sections[:]
|
|
subindex_index = ET.SubElement(indexes, 'index')
|
|
subindex_index.set('lang', lang)
|
|
subindex_index.set('channel', channel)
|
|
|
|
for subindex in subindexes:
|
|
local_sections = [x for x in sections if x in subindex.sections]
|
|
if not local_sections:
|
|
continue
|
|
|
|
index = subindex.create_element(subindex_index,
|
|
channel, lang)
|
|
|
|
for section in local_sections:
|
|
remaining_sections.remove(section)
|
|
section_docs = [x for x in docs if x.toc_id == section]
|
|
if not section_docs:
|
|
continue
|
|
self.create_section(index, section, section_docs, lang)
|
|
if remaining_sections:
|
|
logging.warn('%s channel is missing some sections: %s' % (
|
|
channel, ', '.join(remaining_sections)))
|
|
|
|
self.indexes_tmp_file = tempfile.NamedTemporaryFile()
|
|
tree = ET.ElementTree(indexes)
|
|
tree.write(self.indexes_tmp_file.name)
|
|
if self.debug:
|
|
tree.write('/tmp/library-web-indexes.xml.%s' % os.getpid())
|
|
|
|
def create_section(self, index, section, section_docs, lang):
|
|
section_node = ET.SubElement(index, 'section')
|
|
section_node.set('toc_id', section)
|
|
section_node.set('weight', str(
|
|
self.overlay.get_section_weight(section)))
|
|
|
|
subsections = {}
|
|
for x in section_docs:
|
|
subsections[x.subsection] = True
|
|
subsections = subsections.keys()
|
|
|
|
for subsection in subsections:
|
|
subsection_docs = [x for x in section_docs if x.subsection == subsection]
|
|
|
|
if subsection is None:
|
|
parent_elem = section_node
|
|
else:
|
|
parent_elem = ET.SubElement(section_node, 'section')
|
|
parent_elem.set('title', subsection)
|
|
parent_elem.set('weight', str(
|
|
self.overlay.get_section_weight(subsection)))
|
|
|
|
for doc in subsection_docs:
|
|
logging.debug('generating index for module %s' % doc.module)
|
|
if lang in doc.languages:
|
|
# document is available in the requested
|
|
# language, perfect.
|
|
doc_lang = lang
|
|
elif lang[:2] in doc.languages:
|
|
# mapping to "general" language, for example
|
|
# from en_GB to en, from fr_BE to fr...
|
|
doc_lang = lang[:2]
|
|
elif [x for x in doc.languages if x[:2] == lang]:
|
|
# mapping to "country" language, for
|
|
# example from pt to pt_BR
|
|
doc_lang = [x for x in doc.languages if x[:2] == lang][0]
|
|
else:
|
|
# fallback to English
|
|
doc_lang = 'en'
|
|
|
|
doc.create_element(parent_elem, doc_lang,
|
|
original_language = lang)
|
|
|
|
def generate_from_indexes(self, xsl_filename):
|
|
idx_filename = self.indexes_tmp_file.name
|
|
|
|
cmd = ['xsltproc', '--output', self.config.output_dir,
|
|
'--nonet', '--xinclude', xsl_filename, idx_filename]
|
|
|
|
if self.debug:
|
|
cmd.insert(-2, '--param')
|
|
cmd.insert(-2, 'libgo.debug')
|
|
cmd.insert(-2, 'true()')
|
|
if self.config.symbols_dbm_filepath:
|
|
cmd.insert(-2, '--param')
|
|
cmd.insert(-2, 'libgo.dbm_support')
|
|
cmd.insert(-2, 'true()')
|
|
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
|
|
def generate_html_indexes(self):
|
|
logging.info('generating index.html files')
|
|
indexes_xsl_file = self.config.indexes_xsl_file or self.default_indexes_xsl_file
|
|
self.generate_from_indexes(indexes_xsl_file)
|
|
|
|
def generate_robots(self):
|
|
logging.info('generating robots.txt file')
|
|
robots_xsl_file = os.path.join(data_dir, 'xslt', 'robots.xsl')
|
|
self.generate_from_indexes(robots_xsl_file)
|
|
|
|
def generate_sitemap(self):
|
|
logging.info('generating sitemap file')
|
|
sitemap_xsl_file = os.path.join(data_dir, 'xslt', 'sitemap.xsl')
|
|
self.generate_from_indexes(sitemap_xsl_file)
|
|
|
|
def generate_static_pages(self):
|
|
try:
|
|
doc_linguas = re.findall(r'DOC_LINGUAS\s+=[\t ](.*)',
|
|
file(os.path.join(data_dir, 'pages', 'Makefile.am')).read())[0].split()
|
|
doc_linguas.append('C')
|
|
except IndexError:
|
|
doc_linguas = ['C']
|
|
|
|
if app.config.languages:
|
|
for lang in doc_linguas[:]:
|
|
if lang not in self.config.languages + ['C']:
|
|
doc_linguas.remove(lang)
|
|
|
|
web_output_dir = os.path.join(self.config.output_dir, 'about')
|
|
for lang in doc_linguas:
|
|
xml_file = os.path.join(os.path.join(data_dir, 'pages', lang, 'libgo.xml'))
|
|
if lang == 'C':
|
|
lang = 'en'
|
|
cmd = ['xsltproc', '--output', web_output_dir + '/',
|
|
'--nonet', '--xinclude',
|
|
'--stringparam', 'libgo.lang', lang,
|
|
'--stringparam', 'libgo.channel', 'about',
|
|
'--param', 'db2html.navbar.bottom', 'false()',
|
|
GnomeDocbookModule.db2html_xsl_file, xml_file]
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
|
|
def get_module_web_path(self, module, versioned=True):
|
|
base_path = self.config.doc_path_template % {
|
|
'channel': module.channel,
|
|
'module': module.modulename }
|
|
|
|
licence_modules = ['fdl', 'gpl', 'lgpl']
|
|
if module.modulename in licence_modules or versioned is False:
|
|
# special casing the licences, they do not go in a
|
|
# versioned path
|
|
return base_path
|
|
else:
|
|
return base_path + module.one_dot_version + '/'
|
|
|
|
def get_module_web_output_dir(self, module, versioned=True):
|
|
return os.path.join(self.config.output_dir,
|
|
self.get_module_web_path(module, versioned=versioned)[1:])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
app = Lgo()
|
|
app.Document = Document
|
|
app.run()
|
|
|