193 lines
6.5 KiB
Python
Executable File
193 lines
6.5 KiB
Python
Executable File
#! /usr/bin/env python
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import re
|
|
import urllib2
|
|
import urlparse
|
|
import xml.etree.ElementTree as ET
|
|
|
|
from django.template import Template, Context
|
|
from django.conf import settings
|
|
settings.configure()
|
|
|
|
MAL_NS = 'http://projectmallard.org/1.0/'
|
|
|
|
jhbuildrc = os.path.join(os.getcwd(), 'jhbuildrc')
|
|
checkouts_directory = os.path.join(os.getcwd(), '..', 'src')
|
|
web_directory = os.path.join(os.getcwd(), '..', 'web')
|
|
|
|
if not os.path.exists('jhbuild'):
|
|
os.system('jhbuild -f jhbuildrc -m doceo.modules buildone itstool yelp-xsl yelp-tools')
|
|
|
|
if not os.path.exists(checkouts_directory):
|
|
os.mkdir(checkouts_directory)
|
|
|
|
def checkout(module):
|
|
checkout_dir = os.path.join(checkouts_directory, module)
|
|
|
|
kws = {}
|
|
kws['stdout'] = subprocess.PIPE
|
|
kws['stderr'] = subprocess.STDOUT
|
|
if not os.path.exists(checkout_dir):
|
|
# full clone
|
|
cmd = ['git', 'clone', 'http://repos.entrouvert.org/%s.git' % module]
|
|
kws['cwd'] = checkouts_directory
|
|
else:
|
|
# pull
|
|
cmd = ['git', 'pull']
|
|
kws['cwd'] = checkout_dir
|
|
|
|
subprocess.call(cmd, **kws)
|
|
|
|
def download(url):
|
|
filename = os.path.split(urlparse.urlparse(url)[2])[-1]
|
|
if not os.path.exists(os.path.join(checkouts_directory, filename)):
|
|
fd = file(os.path.join(checkouts_directory, filename), 'w')
|
|
fd.write(urllib2.urlopen(url).read())
|
|
fd.close()
|
|
|
|
|
|
def publish_mallard(module, branch, directory):
|
|
module_name = module.get('name')
|
|
checkout_dir = os.path.join(checkouts_directory, module_name)
|
|
|
|
kws = {}
|
|
kws['stdout'] = subprocess.PIPE
|
|
kws['stderr'] = subprocess.STDOUT
|
|
cmd = ['git', 'checkout', branch]
|
|
subprocess.call(cmd, cwd=checkout_dir, **kws)
|
|
|
|
help_dir = os.path.join(checkout_dir, 'help', 'fr')
|
|
if not os.path.exists(help_dir):
|
|
return
|
|
output_dir = os.path.join(web_directory, module_name, directory)
|
|
if not os.path.exists(output_dir):
|
|
os.makedirs(output_dir)
|
|
index_html = os.path.join(output_dir, 'index.html')
|
|
if os.path.islink(index_html):
|
|
os.unlink(index_html)
|
|
|
|
page_files = [os.path.join(help_dir, x) for x in os.listdir(help_dir) if x.endswith('.page')]
|
|
cmd = ['jhbuild', '-f', jhbuildrc, 'run', 'yelp-build', 'html',
|
|
'-o', output_dir, '-x']
|
|
if module.get('branch') == 'dev' and module.get('private'):
|
|
cmd.append('mallard_extra_dev.xsl')
|
|
else:
|
|
cmd.append('mallard_extra.xsl')
|
|
cmd.extend(page_files)
|
|
subprocess.call(cmd, **kws)
|
|
for static in ('brainstew.js', 'brainstew.css'):
|
|
shutil.copy(static, output_dir)
|
|
|
|
if not os.path.exists(index_html):
|
|
html_files = [x for x in os.listdir(output_dir) if x.endswith('.html')]
|
|
if html_files:
|
|
os.symlink(html_files[0], index_html)
|
|
|
|
def remove_tags(string):
|
|
return re.sub('<.*?>', '', string, flags=re.DOTALL)
|
|
|
|
pages = []
|
|
for page_file in page_files:
|
|
text = open(page_file).read()
|
|
text = re.sub('<title.*type="sort".*</title>', '', text)
|
|
titles = re.findall('<title.*>(.*)<\/title>', text)
|
|
if not titles:
|
|
continue
|
|
pages.append({
|
|
'id': os.path.basename(page_file).rsplit('.', 1)[0],
|
|
'title': remove_tags(titles[0]),
|
|
'other_titles': remove_tags(' '.join(titles[1:])),
|
|
'text': remove_tags(text),
|
|
})
|
|
|
|
json.dump(pages, open(os.path.join(output_dir, 'pages.json'), 'w'), indent=2)
|
|
|
|
if not os.path.exists('../mal2latex'):
|
|
return
|
|
|
|
temp_dir = tempfile.mkdtemp()
|
|
cache_file = os.path.join(temp_dir, '%s.cache' % module_name)
|
|
cmd = ['jhbuild', '-f', jhbuildrc, 'run', 'yelp-build', 'cache', '-o', cache_file]
|
|
cmd.extend(page_files)
|
|
subprocess.call(cmd, **kws)
|
|
|
|
if os.path.exists(cache_file):
|
|
latex_file = os.path.join(temp_dir, 'doc-%s.tex' % module_name)
|
|
cmd = ['xsltproc', '-o', latex_file, '../mal2latex/mal2latex.xsl',
|
|
cache_file]
|
|
subprocess.call(cmd, **kws)
|
|
for subdir in ('media', 'figures'):
|
|
os.symlink(os.path.join(checkout_dir, 'help', 'fr', subdir),
|
|
os.path.join(temp_dir, subdir))
|
|
for admon_file in os.listdir('../mal2latex/admon'):
|
|
if not admon_file.endswith('.pdf'):
|
|
continue
|
|
os.symlink(os.path.join(os.path.abspath('../mal2latex'), 'admon', admon_file),
|
|
os.path.join(temp_dir, admon_file))
|
|
|
|
for i in range(3):
|
|
cmd = ['pdflatex', '--interaction=nonstopmode',
|
|
'-output-directory', temp_dir, latex_file]
|
|
subprocess.call(cmd, cwd=temp_dir, **kws)
|
|
if not os.path.exists(os.path.join(temp_dir, 'doc-%s.pdf' % module_name)):
|
|
break
|
|
else:
|
|
fd = file(os.path.join(output_dir, 'doc-%s.pdf' % module_name), 'w')
|
|
fd.write(file(os.path.join(temp_dir, 'doc-%s.pdf' % module_name)).read())
|
|
fd.close()
|
|
|
|
shutil.rmtree(temp_dir)
|
|
|
|
def publish_tarball(module):
|
|
output_dir = os.path.join(web_directory, module.get('name'), 'stable')
|
|
url = module.get('url')
|
|
directory = module.get('directory')
|
|
filename = os.path.split(urlparse.urlparse(url)[2])[-1]
|
|
tar = tarfile.open(os.path.join(checkouts_directory, filename), 'r')
|
|
for tarinfo in tar:
|
|
if not tarinfo.name.startswith(directory):
|
|
continue
|
|
output_filename = os.path.join(output_dir, tarinfo.name[len(directory):].strip('/'))
|
|
if tarinfo.isdir():
|
|
if not os.path.exists(output_filename):
|
|
os.makedirs(output_filename)
|
|
else:
|
|
fd = tar.extractfile(tarinfo)
|
|
file(output_filename, 'w').write(fd.read())
|
|
|
|
|
|
def create_index(modules):
|
|
index_html = os.path.join(web_directory, 'index.html')
|
|
t = Template(unicode(file('index_template.html').read(), 'utf-8'))
|
|
c = Context({'modules': modules})
|
|
fd = file(index_html, 'w')
|
|
fd.write(t.render(c).encode('utf-8'))
|
|
fd.close()
|
|
|
|
|
|
modules = json.load(file('modules.json'))
|
|
|
|
for module in modules:
|
|
if module.get('type') == 'tarball':
|
|
download(module.get('url'))
|
|
else:
|
|
checkout(module.get('name'))
|
|
|
|
if module.get('directory'):
|
|
module['branch'] = 'stable'
|
|
publish_tarball(module)
|
|
else:
|
|
module['branch'] = 'dev'
|
|
publish_mallard(module, 'master', 'dev')
|
|
|
|
for js_file in os.listdir('js'):
|
|
shutil.copy('js/' + js_file, web_directory)
|
|
|
|
create_index(modules)
|