803 lines
34 KiB
Python
Executable File
803 lines
34 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# libgo - script to build library.gnome.org
|
|
# Copyright (C) 2007-2009 Frederic Peters
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
# 02110-1301 USA
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import tempfile
|
|
import urllib2
|
|
from cStringIO import StringIO
|
|
from optparse import OptionParser
|
|
import logging
|
|
try:
|
|
import elementtree.ElementTree as ET
|
|
except ImportError:
|
|
import xml.etree.ElementTree as ET
|
|
import tarfile
|
|
import stat
|
|
import subprocess
|
|
import dbm
|
|
import shutil
|
|
import socket
|
|
import __builtin__
|
|
|
|
data_dir = os.path.join(os.path.dirname(__file__), '../data')
|
|
__builtin__.__dict__['data_dir'] = data_dir
|
|
|
|
import errors
|
|
import utils
|
|
from utils import version_cmp, is_version_number
|
|
|
|
from document import Document
|
|
from overlay import Overlay
|
|
|
|
from modtypes.gnomedocbook import GnomeDocbookModule
|
|
from modtypes.gtkdoc import GtkDocModule
|
|
from modtypes.htmlfiles import HtmlFilesModule
|
|
from modtypes.mallard import MallardModule
|
|
|
|
from app import App
|
|
|
|
|
|
# timeout for downloads, so it doesn't hang on connecting to sourceforge
|
|
socket.setdefaulttimeout(10)
|
|
|
|
|
|
class FtpDotGnomeDotOrg:
|
|
'''Class providing operations on ftp.gnome.org, be it a local copy
|
|
or the real remote one'''
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
if self.config.ftp_gnome_org_local_copy:
|
|
self.ftp_gnome_org_local_copy = self.config.ftp_gnome_org_local_copy
|
|
self.download = self.download_local
|
|
self.listdir = self.listdir_local
|
|
else:
|
|
self.ftp_gnome_org_cache_dir = os.path.join(
|
|
config.download_cache_dir, 'ftp.gnome.org')
|
|
if not os.path.exists(self.ftp_gnome_org_cache_dir):
|
|
os.makedirs(self.ftp_gnome_org_cache_dir)
|
|
|
|
def download(self, filename):
|
|
cache_filename = os.path.join(self.ftp_gnome_org_cache_dir, filename)
|
|
cache_dir = os.path.split(cache_filename)[0]
|
|
if not os.path.exists(cache_dir):
|
|
os.makedirs(cache_dir)
|
|
# TODO: support for self.config.use_latest_version
|
|
if os.path.exists(cache_filename) and os.stat(cache_filename)[stat.ST_SIZE]:
|
|
logging.debug('using cache of ftp://ftp.gnome.org/%s' % filename)
|
|
return (cache_filename, open(cache_filename))
|
|
logging.info('downloading ftp://ftp.gnome.org/%s' % filename)
|
|
try:
|
|
open(cache_filename, 'w').write(
|
|
urllib2.urlopen('ftp://ftp.gnome.org/' + filename).read())
|
|
except IOError:
|
|
raise
|
|
return (cache_filename, open(cache_filename))
|
|
|
|
def download_local(self, filename):
|
|
local_filename = os.path.join(self.ftp_gnome_org_local_copy, filename)
|
|
if self.config.use_latest_version:
|
|
dirname, basename = os.path.split(local_filename)
|
|
module_name = basename.split('-')[0]
|
|
try:
|
|
latest_version = [x for x in os.listdir(dirname) if x.startswith('LATEST-IS-')]
|
|
if latest_version:
|
|
latest_base = '%s-%s.' % (module_name, latest_version[0].split('-')[-1])
|
|
new_basename = [x for x in os.listdir(dirname) if \
|
|
x.startswith(latest_base) and (
|
|
x.endswith('.tar.gz') or x.endswith('.tar.bz2'))]
|
|
if new_basename:
|
|
local_filename = os.path.join(dirname, new_basename[0])
|
|
logging.debug('using %s instead of %s' % (new_basename[0], basename))
|
|
except OSError:
|
|
pass
|
|
return (local_filename, open(local_filename))
|
|
|
|
def listdir(self, dirname):
|
|
l = []
|
|
for line in urllib2.urlopen('ftp://ftp.gnome.org/' + dirname):
|
|
l.append(line.strip().split()[-1])
|
|
return l
|
|
|
|
def listdir_local(self, dirname):
|
|
try:
|
|
return sorted(os.listdir(os.path.join(self.ftp_gnome_org_local_copy, dirname)))
|
|
except OSError:
|
|
return []
|
|
|
|
|
|
class Lgo(App):
|
|
'''Main Application Class'''
|
|
|
|
def run(self):
|
|
self.ftp_gnome_org = FtpDotGnomeDotOrg(self.config)
|
|
self.overlay = Overlay(os.path.join(data_dir, 'overlay.xml'))
|
|
self.get_yelp_categories()
|
|
|
|
if self.options.rebuild_module:
|
|
self.rebuild_all = True
|
|
for doc_module in self.extract_modules(self.options.rebuild_module):
|
|
doc_module.process()
|
|
sys.exit(0)
|
|
|
|
if self.options.rebuild_remote:
|
|
self.generate_static_pages()
|
|
self.apply_overlay()
|
|
sys.exit(0)
|
|
|
|
self.copy_static_files()
|
|
self.process_releases()
|
|
if not self.options.skip_extra_tarballs:
|
|
self.process_extra_tarballs()
|
|
if self.config.nightly_tarballs_location:
|
|
self.process_nightly_tarballs()
|
|
self.apply_overlay()
|
|
self.generate_indexes()
|
|
self.generate_html_indexes()
|
|
self.generate_robots()
|
|
self.generate_sitemap()
|
|
self.generate_symbols_files()
|
|
self.generate_static_pages()
|
|
|
|
def get_yelp_categories(self):
|
|
logging.info('Getting categories from Yelp')
|
|
|
|
scrollkeeper_xml = os.path.join(data_dir, 'externals', 'scrollkeeper.xml')
|
|
toc_xml = os.path.join(data_dir, 'externals', 'toc.xml')
|
|
|
|
if not os.path.exists(os.path.join(data_dir, 'externals')):
|
|
os.mkdir(os.path.join(data_dir, 'externals'))
|
|
|
|
if not os.path.exists(scrollkeeper_xml) or not os.path.exists(toc_xml):
|
|
filename = FtpDotGnomeDotOrg(self.config).download(
|
|
'pub/GNOME/sources/yelp/3.0/yelp-3.0.3.tar.bz2')[0]
|
|
|
|
tar = tarfile.open(filename, 'r')
|
|
done = 0
|
|
for tarinfo in tar:
|
|
filename = os.path.basename(tarinfo.name)
|
|
if filename == 'scrollkeeper.xml':
|
|
open(scrollkeeper_xml, 'w').write(tar.extractfile(tarinfo).read())
|
|
done += 1
|
|
elif filename == 'toc.xml':
|
|
open(toc_xml, 'w').write(tar.extractfile(tarinfo).read())
|
|
done += 1
|
|
if done == 2:
|
|
break
|
|
|
|
yelp_toc_tree = ET.fromstring(open(scrollkeeper_xml).read())
|
|
self.toc_mapping = {}
|
|
for subtoc in yelp_toc_tree.findall('toc'):
|
|
sub_id = subtoc.attrib['id']
|
|
for subject in subtoc.findall('subject'):
|
|
self.toc_mapping[subject.attrib['category']] = sub_id
|
|
|
|
def copy_static_files(self):
|
|
App.copy_static_files(self)
|
|
|
|
src = os.path.join(data_dir, 'gnome-library-search.xml')
|
|
dst = os.path.join(self.config.output_dir, 'gnome-library-search.xml')
|
|
if not os.path.exists(dst) or \
|
|
os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
|
|
open(dst, 'w').write(open(src, 'r').read())
|
|
|
|
def process_releases(self):
|
|
'''Download GNOME releases'''
|
|
releases = self.ftp_gnome_org.listdir('pub/GNOME/teams/releng/')
|
|
|
|
releases = [x for x in releases if is_version_number(x)]
|
|
for i, r in enumerate(releases[:]):
|
|
if self.config.version_min and version_cmp(r, self.config.version_min) < 0:
|
|
continue
|
|
if self.config.version_max and version_cmp(r, self.config.version_max) > 0:
|
|
continue
|
|
|
|
if i < len(releases)-1 and releases[i+1].startswith(re.match(r'\d+\.\d+\.', r).group()) and \
|
|
not r == self.config.version_max:
|
|
# next release has the same major.minor version number, so skip
|
|
# this one and get the newer one later
|
|
logging.debug('skipping release %s, not the last in serie' % r)
|
|
continue
|
|
|
|
if int(r.split('.')[1]) % 2 == 1:
|
|
# odd release, development, skip unless this is the current
|
|
# development serie
|
|
if not releases[-1].startswith(re.match(r'\d+\.\d+\.', r).group()) and \
|
|
not r == self.config.version_max:
|
|
logging.debug('skipping release %s, not the last in serie' % r)
|
|
continue
|
|
|
|
if version_cmp(r, '2.19.0') < 0:
|
|
urls = ['pub/GNOME/teams/releng/%(r)s/gnome-%(r)s.modules']
|
|
elif version_cmp(r, '2.91.3') < 0:
|
|
urls = ['pub/GNOME/teams/releng/%(r)s/gnome-suites-%(r)s.modules']
|
|
else:
|
|
urls = ['pub/GNOME/teams/releng/%(r)s/gnome-apps-%(r)s.modules',
|
|
'pub/GNOME/teams/releng/%(r)s/gnome-suites-core-%(r)s.modules',
|
|
'pub/GNOME/teams/releng/%(r)s/gnome-suites-core-deps-%(r)s.modules']
|
|
|
|
logging.info('Getting GNOME release: %s' % r)
|
|
for url in urls:
|
|
try:
|
|
moduleset = self.ftp_gnome_org.download(url % {'r': r})[1]
|
|
except Exception, e:
|
|
logging.error('Failed retrieving %s (%s)' % (url % {'r': r}, str(e)))
|
|
continue
|
|
self.process_moduleset(moduleset, r)
|
|
|
|
if self.config.version_max is None and r == releases[-1]:
|
|
# this is the last release, and no maximum version was set,
|
|
# add all modules to extra tarballs, so development versions
|
|
# not shipped with latest GNOME are made available.
|
|
moduleset.seek(0)
|
|
self.process_latest_moduleset(moduleset)
|
|
|
|
def download(self, url, use_cache=True):
|
|
if url.startswith('gnome://'):
|
|
# special schema for modules on ftp.gnome.org; URL scheme is
|
|
# gnome://<modulename>?min=<minimum version>
|
|
# (the parameters are optional)
|
|
# it may be used to define extra tarballs and get new versions
|
|
# picked automatically
|
|
if '?' in url:
|
|
module, params = url[8:].split('?')
|
|
params = [x.split('=') for x in params.split('&')]
|
|
try:
|
|
min_version = [x[1] for x in params if x[0] == 'min'][0]
|
|
# number versions used as ftp.gnome.org directories are
|
|
# limited to two numbers
|
|
min_version = '.'.join(min_version.split('.')[:2])
|
|
except IndexError:
|
|
min_version = None
|
|
else:
|
|
module = url[8:]
|
|
min_version = None
|
|
base_module_href = 'pub/GNOME/sources/%s/' % module
|
|
releases = self.ftp_gnome_org.listdir(base_module_href)
|
|
releases = [x for x in releases if is_version_number(x) and
|
|
not (x.endswith('.png') # ignore icons
|
|
or x.endswith('.json'))] # ignore json
|
|
releases.sort(version_cmp)
|
|
extra_filenames = []
|
|
for i, version_dir in enumerate(releases):
|
|
if min_version and version_cmp(version_dir, min_version) < 0:
|
|
continue
|
|
|
|
try:
|
|
# check all parts are numbers
|
|
[int(x) for x in version_dir.split('.')]
|
|
except ValueError:
|
|
continue
|
|
|
|
if int(version_dir.split('.')[1]) % 2 == 1 or int(version_dir.split('.')[0]) == 0:
|
|
# development release, (odd minor version, or major version
|
|
# as 0): skip unless this is the current development serie
|
|
if version_dir.count('.') == 1:
|
|
if not version_dir == releases[-1]:
|
|
continue
|
|
elif not releases[-1].startswith(re.match(r'\d+\.\d+\.', version_dir).group()):
|
|
continue
|
|
|
|
base_version_href = '%s%s/' % (base_module_href, version_dir)
|
|
filenames = self.ftp_gnome_org.listdir(base_version_href)
|
|
filenames = [x for x in filenames if \
|
|
x.startswith(module + '-%s.' % version_dir) and \
|
|
x.endswith('.tar.bz2')]
|
|
def filenames_cmp(x, y):
|
|
return version_cmp(x[len(module)+1:-8], y[len(module)+1:-8])
|
|
filenames.sort(filenames_cmp)
|
|
if not filenames:
|
|
continue
|
|
version_href = '%s%s/%s' % (
|
|
base_module_href, version_dir, filenames[-1])
|
|
extra_filenames.append(self.ftp_gnome_org.download(version_href)[0])
|
|
if len(extra_filenames) == 1:
|
|
return extra_filenames[0]
|
|
return extra_filenames
|
|
elif url.startswith('http://download.gnome.org/') or \
|
|
url.startswith('http://ftp.gnome.org/'):
|
|
url = url.replace('http://download.gnome.org/', 'pub/GNOME/')
|
|
url = url.replace('http://ftp.gnome.org/', '')
|
|
try:
|
|
filename = self.ftp_gnome_org.download(url)[0]
|
|
except IOError:
|
|
logging.error('error downloading %s' % url)
|
|
return
|
|
else:
|
|
filename = App.download(self, url, use_cache=use_cache)
|
|
|
|
return filename
|
|
|
|
def process_moduleset(self, moduleset, version_number):
|
|
'''Download tarballs from a module set'''
|
|
|
|
doc_modules = []
|
|
|
|
tree = ET.parse(moduleset)
|
|
|
|
repositories = {}
|
|
for repository in tree.findall('repository'):
|
|
if repository.attrib['type'] != 'tarball':
|
|
continue
|
|
repositories[repository.attrib['name']] = repository.attrib['href']
|
|
self.repositories = repositories
|
|
|
|
hrefs = []
|
|
|
|
for tarball in tree.findall('tarball'):
|
|
if self.config.modules is not None and not tarball.attrib['id'] in self.config.modules:
|
|
continue
|
|
href = tarball.find('source').attrib['href']
|
|
hrefs.append((tarball, href))
|
|
|
|
for module in tree.findall('autotools'):
|
|
if self.config.modules is not None and not module.attrib['id'] in self.config.modules:
|
|
continue
|
|
branch = module.find('branch')
|
|
if branch is None:
|
|
continue
|
|
repository = repositories.get(branch.attrib.get('repo'))
|
|
if repository is None:
|
|
continue
|
|
href = repository + branch.attrib.get('module')
|
|
hrefs.append((module, href))
|
|
|
|
for module, href in hrefs:
|
|
if not (href.startswith('http://download.gnome.org/') or
|
|
href.startswith('http://ftp.gnome.org/')):
|
|
continue
|
|
filename = self.download(href)
|
|
if not filename:
|
|
continue
|
|
logging.info('extracting module %s (from %s moduleset)' % (
|
|
module.attrib['id'], version_number))
|
|
doc_modules.extend(self.extract_modules(filename))
|
|
|
|
gduxrefs = ET.Element('gduxrefs')
|
|
for doc_module in doc_modules:
|
|
if not isinstance(doc_module, GnomeDocbookModule):
|
|
continue
|
|
element = ET.SubElement(gduxrefs, 'doc')
|
|
element.set('id', doc_module.modulename)
|
|
element.set('path', doc_module.path)
|
|
|
|
# XXX: this is writing in build dir, not really nice, but this allows
|
|
# for a known location relative to the XSL files.
|
|
tmp_dirname = os.path.join(data_dir, 'tmp')
|
|
if not os.path.exists(tmp_dirname):
|
|
os.makedirs(tmp_dirname)
|
|
|
|
tree = ET.ElementTree(gduxrefs)
|
|
tree.write(os.path.join(tmp_dirname, 'gduxrefs.xml'))
|
|
|
|
for doc_module in doc_modules:
|
|
logging.info('processing %s (from %s moduleset)' % (
|
|
doc_module, version_number))
|
|
doc_module.process()
|
|
|
|
def process_latest_moduleset(self, moduleset):
|
|
'''Register all modules from latest modulesets as extra tarballs
|
|
with minimum version set to version+epsilon'''
|
|
tree = ET.parse(moduleset)
|
|
for tarball in tree.findall('tarball'):
|
|
if self.config.modules is not None and not tarball.attrib['id'] in self.config.modules:
|
|
continue
|
|
|
|
url = tarball.find('source').attrib['href']
|
|
if not (url.startswith('http://download.gnome.org/') or
|
|
url.startswith('http://ftp.gnome.org/')):
|
|
continue
|
|
min_version = tarball.attrib.get('version')
|
|
if not min_version:
|
|
continue
|
|
min_version += '.1'
|
|
logging.info('registering module %s for extra tarballs' % tarball.attrib['id'])
|
|
extra_tarball = 'gnome://%s?min=%s' % (tarball.attrib['id'], min_version)
|
|
self.config.extra_tarballs.append(extra_tarball)
|
|
|
|
# XXX: this needs to be refactored, and moduleset processing code to be
|
|
# shared with process_moduleset
|
|
for module in tree.findall('autotools'):
|
|
if self.config.modules is not None and not module.attrib['id'] in self.config.modules:
|
|
continue
|
|
branch = module.find('branch')
|
|
if branch is None:
|
|
continue
|
|
|
|
repository = self.repositories.get(branch.attrib.get('repo'))
|
|
if repository is None:
|
|
continue
|
|
url = repository + branch.attrib.get('module')
|
|
if not (url.startswith('http://download.gnome.org/') or
|
|
url.startswith('http://ftp.gnome.org/')):
|
|
continue
|
|
|
|
min_version = branch.attrib.get('version')
|
|
if not min_version:
|
|
continue
|
|
min_version += '.1'
|
|
logging.info('registering module %s for extra tarballs' % module.attrib['id'])
|
|
extra_tarball = 'gnome://%s?min=%s' % (module.attrib['id'], min_version)
|
|
self.config.extra_tarballs.append(extra_tarball)
|
|
|
|
def extract_modules(self, filename, nightly = False):
|
|
logging.debug('looking for doc modules in %s' % filename)
|
|
doc_modules = []
|
|
|
|
mtime = os.stat(filename)[stat.ST_MTIME]
|
|
|
|
if self.config.fast_mode:
|
|
ext_dirname = os.path.join(app.config.private_dir, 'extracts',
|
|
os.path.splitext(os.path.splitext(os.path.basename(filename))[0])[0])
|
|
stamp_file = ext_dirname + '.extract-stamp'
|
|
else:
|
|
stamp_file = None
|
|
|
|
base_tarball_name = os.path.basename(filename).rsplit('-', 1)[0]
|
|
if nightly:
|
|
stamp_file = None
|
|
|
|
if stamp_file and os.path.exists(stamp_file) and not os.path.exists(ext_dirname):
|
|
# file was extracted once, and no doc module were found inside
|
|
return []
|
|
elif stamp_file and os.path.exists(stamp_file) and os.stat(stamp_file)[stat.ST_MTIME] < mtime:
|
|
# file was extracted but has been modified since then, remove
|
|
# extraction if it exists:
|
|
if os.path.exists(ext_dirname):
|
|
logging.debug('removing old copy of files from %s' % base_tarball_name)
|
|
shutil.rmtree(ext_dirname, ignore_errors = True)
|
|
tar = tarfile.open(filename, 'r')
|
|
elif stamp_file and os.path.exists(stamp_file):
|
|
tar = utils.FakeTarFile(ext_dirname)
|
|
else:
|
|
tar = tarfile.open(filename, 'r')
|
|
|
|
doc_version = os.path.splitext(tar.name)[0].split('-')[-1]
|
|
if doc_version.endswith('.tar'):
|
|
doc_version = doc_version[:-4]
|
|
|
|
more_tarball_docs = self.overlay.more_tarball_docs.get(
|
|
base_tarball_name, [])[:]
|
|
|
|
for more_doc in more_tarball_docs:
|
|
# don't look for docs that require a newer version of the module
|
|
if 'minimum-version' in more_doc.attrib:
|
|
if version_cmp(doc_version, more_doc.attrib.get('minimum-version')) < 0:
|
|
more_tarball_docs.remove(more_doc)
|
|
|
|
extraction_happened = False
|
|
regex_gdu = re.compile(r'include.*gnome-doc-utils.make', re.DOTALL)
|
|
for tarinfo in tar:
|
|
doc = None
|
|
if os.path.split(tarinfo.name)[-1] in ('Makefile.am', 'GNUmakefile.am'):
|
|
fd = tar.extractfile(tarinfo)
|
|
makefile_am = fd.read()
|
|
# merge lines continued with \
|
|
makefile_am = re.sub(r'\\\s*\n', r' ', makefile_am)
|
|
if 'HELP_ID' in makefile_am and '@YELP_HELP_RULES@' in makefile_am:
|
|
logging.debug('found usage of mallard (via YELP_HELP_RULES) in %s' % tarinfo.name)
|
|
doc = MallardModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'DOC_ID' in makefile_am and regex_gdu.findall(makefile_am):
|
|
logging.debug('found usage of mallard in %s' % tarinfo.name)
|
|
doc = MallardModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'DOC_MODULE' in makefile_am and regex_gdu.findall(makefile_am):
|
|
logging.debug('found usage of docbook in %s' % tarinfo.name)
|
|
doc = GnomeDocbookModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'include $(top_srcdir)/gtk-doc.make' in makefile_am or \
|
|
'include $(srcdir)/gtk-doc.make' in makefile_am or \
|
|
'include gtk-doc.make' in makefile_am or \
|
|
('gtkdoc-scan' in makefile_am and not 'gtk-doc' in tarinfo.name):
|
|
logging.debug('found usage of gtk-doc in %s' % tarinfo.name)
|
|
doc = GtkDocModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
elif 'SUBDIRS = C' in makefile_am and \
|
|
os.path.basename(filename).startswith('gtk-doc-'):
|
|
logging.debug('found gtk-doc almost gnome-doc-utils manual in %s' % tarinfo.name)
|
|
makefile_am += '\nDOC_MODULE = gtk-doc-manual\n'
|
|
doc = GnomeDocbookModule.create_from_tar(tar, tarinfo, makefile_am, nightly)
|
|
else:
|
|
continue
|
|
|
|
if '$(' in doc.modulename:
|
|
continue
|
|
|
|
if not doc.modulename or doc.modulename in self.config.blacklist:
|
|
continue
|
|
else:
|
|
for more_doc in more_tarball_docs[:]:
|
|
if not tarinfo.isdir():
|
|
continue
|
|
directory_name = tarinfo.name
|
|
if not directory_name[-1] == '/':
|
|
directory_name += '/'
|
|
if directory_name.endswith(more_doc.attrib.get('dir')):
|
|
doc = HtmlFilesModule.create_from_tar(tar, tarinfo, more_doc)
|
|
more_tarball_docs.remove(more_doc)
|
|
continue
|
|
|
|
if doc:
|
|
doc.filename = filename
|
|
doc.mtime_tarball = mtime
|
|
if extraction_happened:
|
|
doc.extract(force=True)
|
|
else:
|
|
extraction_happened = doc.extract()
|
|
doc.setup_channel()
|
|
doc.path = self.get_module_web_path(doc)
|
|
if self.config.channels is None or doc.channel in self.config.channels:
|
|
doc_modules.append(doc)
|
|
else:
|
|
logging.debug('ignoring %s, not in an appropriate channel' % doc.modulename)
|
|
|
|
if more_tarball_docs:
|
|
for more_doc in more_tarball_docs:
|
|
logging.error('[%s] overlay file mentioned %s but it was not found' % \
|
|
(base_tarball_name, more_doc.attrib.get('dir')))
|
|
|
|
tar.close()
|
|
|
|
if stamp_file:
|
|
# touch extract stamp file
|
|
if not os.path.exists(ext_dirname):
|
|
os.makedirs(ext_dirname)
|
|
file(stamp_file, 'w').close()
|
|
|
|
return doc_modules
|
|
|
|
def process_extra_tarballs(self):
|
|
if self.config.extra_tarballs:
|
|
logging.info('processing extra tarballs')
|
|
for url in self.config.extra_tarballs:
|
|
logging.debug('processing extra tarball: %s' % url)
|
|
filename = self.download(url)
|
|
if not filename:
|
|
continue
|
|
if not type(filename) is list:
|
|
filename = [filename]
|
|
|
|
for fname in filename:
|
|
for doc_module in self.extract_modules(fname):
|
|
doc_module.process()
|
|
|
|
def process_nightly_tarballs(self):
|
|
logging.info('processing nightly tarballs')
|
|
for filename in os.listdir(self.config.nightly_tarballs_location):
|
|
if not (filename.endswith('.tar.gz') or filename.endswith('.tar.bz2')):
|
|
continue
|
|
filename = os.path.join(self.config.nightly_tarballs_location, filename)
|
|
for doc_module in self.extract_modules(filename, nightly = True):
|
|
doc_module.process()
|
|
|
|
def generate_indexes(self):
|
|
logging.info('generating indexes')
|
|
indexes = ET.Element('indexes')
|
|
|
|
# get all possible languages
|
|
languages = {}
|
|
for doc in self.documents:
|
|
for lang in doc.languages:
|
|
if lang == 'C':
|
|
continue # ignore
|
|
if self.config.languages and not lang in self.config.languages:
|
|
continue
|
|
languages[lang] = True
|
|
if 'sr@Latn' in languages and 'sr@latin' in languages:
|
|
del languages['sr@Latn']
|
|
|
|
for lang in languages.keys():
|
|
home = ET.SubElement(indexes, 'home')
|
|
home.set('lang', lang)
|
|
|
|
channels = self.config.channels
|
|
if not channels:
|
|
channels = ('users', 'devel', 'admin', 'misc')
|
|
|
|
for channel in channels:
|
|
docs = [x for x in self.documents if x.channel == channel]
|
|
|
|
if not docs:
|
|
continue
|
|
|
|
for lang in languages.keys():
|
|
logging.debug('generating index for lang %s' % lang)
|
|
|
|
sections = {}
|
|
for x in docs:
|
|
if x.toc_id is None:
|
|
logging.warning('doc %s has no toc id -> default to Other' % x.module)
|
|
x.toc_id = 'Other'
|
|
sections[x.toc_id] = True
|
|
sections = sections.keys()
|
|
sections.sort()
|
|
|
|
docs.sort(lambda x,y: cmp(x.title.get(lang), y.title.get(lang)))
|
|
|
|
subindexes = self.overlay.get_subindexes(channel)
|
|
if not subindexes:
|
|
index = ET.SubElement(indexes, 'index')
|
|
index.set('lang', lang)
|
|
index.set('channel', channel)
|
|
for section in sections:
|
|
section_docs = [x for x in docs if x.toc_id == section]
|
|
if not section_docs:
|
|
continue
|
|
self.create_section(index, section, section_docs, lang)
|
|
else:
|
|
remaining_sections = sections[:]
|
|
subindex_index = ET.SubElement(indexes, 'index')
|
|
subindex_index.set('lang', lang)
|
|
subindex_index.set('channel', channel)
|
|
|
|
for subindex in subindexes:
|
|
local_sections = [x for x in sections if x in subindex.sections]
|
|
if not local_sections:
|
|
continue
|
|
|
|
index = subindex.create_element(subindex_index,
|
|
channel, lang)
|
|
|
|
for section in local_sections:
|
|
remaining_sections.remove(section)
|
|
section_docs = [x for x in docs if x.toc_id == section]
|
|
if not section_docs:
|
|
continue
|
|
self.create_section(index, section, section_docs, lang)
|
|
if remaining_sections:
|
|
logging.warn('%s channel is missing some sections: %s' % (
|
|
channel, ', '.join(remaining_sections)))
|
|
|
|
self.indexes_tmp_file = tempfile.NamedTemporaryFile()
|
|
tree = ET.ElementTree(indexes)
|
|
tree.write(self.indexes_tmp_file.name)
|
|
if self.debug:
|
|
tree.write('/tmp/library-web-indexes.xml.%s' % os.getpid())
|
|
|
|
def create_section(self, index, section, section_docs, lang):
|
|
section_node = ET.SubElement(index, 'section')
|
|
section_node.set('toc_id', section)
|
|
section_node.set('weight', str(
|
|
self.overlay.get_section_weight(section)))
|
|
|
|
subsections = {}
|
|
for x in section_docs:
|
|
subsections[x.subsection] = True
|
|
subsections = subsections.keys()
|
|
|
|
for subsection in subsections:
|
|
subsection_docs = [x for x in section_docs if x.subsection == subsection]
|
|
|
|
if subsection is None:
|
|
parent_elem = section_node
|
|
else:
|
|
parent_elem = ET.SubElement(section_node, 'section')
|
|
parent_elem.set('title', subsection)
|
|
parent_elem.set('weight', str(
|
|
self.overlay.get_section_weight(subsection)))
|
|
|
|
for doc in subsection_docs:
|
|
logging.debug('generating index for module %s' % doc.module)
|
|
if lang in doc.languages:
|
|
# document is available in the requested
|
|
# language, perfect.
|
|
doc_lang = lang
|
|
elif lang[:2] in doc.languages:
|
|
# mapping to "general" language, for example
|
|
# from en_GB to en, from fr_BE to fr...
|
|
doc_lang = lang[:2]
|
|
elif [x for x in doc.languages if x[:2] == lang]:
|
|
# mapping to "country" language, for
|
|
# example from pt to pt_BR
|
|
doc_lang = [x for x in doc.languages if x[:2] == lang][0]
|
|
else:
|
|
# fallback to English
|
|
doc_lang = 'en'
|
|
|
|
doc.create_element(parent_elem, doc_lang,
|
|
original_language = lang)
|
|
|
|
def generate_from_indexes(self, xsl_filename):
|
|
idx_filename = self.indexes_tmp_file.name
|
|
|
|
cmd = ['xsltproc', '--output', self.config.output_dir,
|
|
'--nonet', '--xinclude', xsl_filename, idx_filename]
|
|
|
|
if self.debug:
|
|
cmd.insert(-2, '--param')
|
|
cmd.insert(-2, 'libgo.debug')
|
|
cmd.insert(-2, 'true()')
|
|
if self.config.symbols_dbm_filepath:
|
|
cmd.insert(-2, '--param')
|
|
cmd.insert(-2, 'libgo.dbm_support')
|
|
cmd.insert(-2, 'true()')
|
|
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
|
|
def generate_html_indexes(self):
|
|
logging.info('generating index.html files')
|
|
indexes_xsl_file = self.config.indexes_xsl_file or self.default_indexes_xsl_file
|
|
self.generate_from_indexes(indexes_xsl_file)
|
|
|
|
def generate_robots(self):
|
|
logging.info('generating robots.txt file')
|
|
robots_xsl_file = os.path.join(data_dir, 'xslt', 'robots.xsl')
|
|
self.generate_from_indexes(robots_xsl_file)
|
|
|
|
def generate_sitemap(self):
|
|
logging.info('generating sitemap file')
|
|
sitemap_xsl_file = os.path.join(data_dir, 'xslt', 'sitemap.xsl')
|
|
self.generate_from_indexes(sitemap_xsl_file)
|
|
|
|
def generate_static_pages(self):
|
|
try:
|
|
doc_linguas = re.findall(r'DOC_LINGUAS\s+=[\t ](.*)',
|
|
file(os.path.join(data_dir, 'pages', 'Makefile.am')).read())[0].split()
|
|
doc_linguas.append('C')
|
|
except IndexError:
|
|
doc_linguas = ['C']
|
|
|
|
if app.config.languages:
|
|
for lang in doc_linguas[:]:
|
|
if lang not in self.config.languages + ['C']:
|
|
doc_linguas.remove(lang)
|
|
|
|
web_output_dir = os.path.join(self.config.output_dir, 'about')
|
|
for lang in doc_linguas:
|
|
xml_file = os.path.join(os.path.join(data_dir, 'pages', lang, 'libgo.xml'))
|
|
if lang == 'C':
|
|
lang = 'en'
|
|
cmd = ['xsltproc', '--output', web_output_dir + '/',
|
|
'--nonet', '--xinclude',
|
|
'--stringparam', 'libgo.lang', lang,
|
|
'--stringparam', 'libgo.channel', 'about',
|
|
'--param', 'db2html.navbar.bottom', 'false()',
|
|
GnomeDocbookModule.db2html_xsl_file, xml_file]
|
|
logging.debug('executing %s' % ' '.join(cmd))
|
|
rc = subprocess.call(cmd)
|
|
if rc != 0:
|
|
logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
|
|
|
|
def get_module_web_path(self, module, versioned=True):
|
|
base_path = self.config.doc_path_template % {
|
|
'channel': module.channel,
|
|
'module': module.modulename }
|
|
|
|
licence_modules = ['fdl', 'gpl', 'lgpl']
|
|
if module.modulename in licence_modules or versioned is False:
|
|
# special casing the licences, they do not go in a
|
|
# versioned path
|
|
return base_path
|
|
else:
|
|
return base_path + module.one_dot_version + '/'
|
|
|
|
def get_module_web_output_dir(self, module, versioned=True):
|
|
return os.path.join(self.config.output_dir,
|
|
self.get_module_web_path(module, versioned=versioned)[1:])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
app = Lgo()
|
|
app.Document = Document
|
|
app.run()
|
|
|