brainstew/publish.py

#! /usr/bin/env python

import json
import os
import shutil
import subprocess
import tarfile
import tempfile
import urllib2
import urlparse

from django.template import Template, Context
from django.conf import settings
settings.configure()

jhbuildrc = os.path.join(os.getcwd(), 'jhbuildrc')
checkouts_directory = os.path.join(os.getcwd(), '..', 'src')
web_directory = os.path.join(os.getcwd(), '..', 'web')

if not os.path.exists('jhbuild'):
    os.system('jhbuild  -f jhbuildrc -m doceo.modules buildone itstool yelp-xsl yelp-tools')

if not os.path.exists(checkouts_directory):
    os.mkdir(checkouts_directory)

def checkout(module):
    checkout_dir = os.path.join(checkouts_directory, module)

    kws = {}
    kws['stdout'] = subprocess.PIPE
    kws['stderr'] = subprocess.STDOUT
    if not os.path.exists(checkout_dir):
        # full clone
        cmd = ['git', 'clone', 'http://repos.entrouvert.org/%s.git' % module]
        kws['cwd'] = checkouts_directory
    else:
        # pull
        cmd = ['git', 'pull']
        kws['cwd'] = checkout_dir

    subprocess.call(cmd, **kws)

def download(url):
    filename = os.path.split(urlparse.urlparse(url)[2])[-1]
    if not os.path.exists(os.path.join(checkouts_directory, filename)):
        fd = file(os.path.join(checkouts_directory, filename), 'w')
        fd.write(urllib2.urlopen(url).read())
        fd.close()


def publish_mallard(module, branch, directory):
    checkout_dir = os.path.join(checkouts_directory, module)

    kws = {}
    kws['stdout'] = subprocess.PIPE
    kws['stderr'] = subprocess.STDOUT
    cmd = ['git', 'checkout', branch]
    subprocess.call(cmd, cwd=checkout_dir, **kws)

    help_dir = os.path.join(checkout_dir, 'help', 'fr')
    if not os.path.exists(help_dir):
        return
    output_dir = os.path.join(web_directory, module, directory)
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    index_html = os.path.join(output_dir, 'index.html')
    if os.path.islink(index_html):
        os.unlink(index_html)

    cmd = ['jhbuild', '-f', jhbuildrc, 'run', 'yelp-build', 'html', '-o', output_dir]
    cmd.extend([os.path.join(help_dir, x) for x in os.listdir(help_dir) if x.endswith('.page')])
    subprocess.call(cmd, **kws)

    if not os.path.exists(index_html):
        html_files = [x for x in os.listdir(output_dir) if x.endswith('.html')]
        if html_files:
            os.symlink(html_files[0], index_html)

    if not os.path.exists('../mal2latex'):
        return

    temp_dir = tempfile.mkdtemp()
    cache_file = os.path.join(temp_dir, '%s.cache' % module)
    cmd = ['jhbuild', '-f', jhbuildrc, 'run', 'yelp-build', 'cache', '-o', cache_file]
    cmd.extend([os.path.join(help_dir, x) for x in os.listdir(help_dir) if x.endswith('.page')])
    subprocess.call(cmd, **kws)

    if os.path.exists(cache_file):
        latex_file = os.path.join(temp_dir, 'doc-%s.tex' % module)
        cmd = ['xsltproc', '-o', latex_file, '../mal2latex/mal2latex.xsl',
               cache_file]
        subprocess.call(cmd, **kws)
        for subdir in ('media', 'figures'):
            os.symlink(os.path.join(checkout_dir, 'help', 'fr', subdir),
                       os.path.join(temp_dir, subdir))
        for admon_file in os.listdir('../mal2latex/admon'):
            if not admon_file.endswith('.pdf'):
                continue
            os.symlink(os.path.join(os.path.abspath('../mal2latex'), 'admon', admon_file),
                       os.path.join(temp_dir, admon_file))

        for i in range(3):
            cmd = ['pdflatex', '--interaction=nonstopmode',
                   '-output-directory', temp_dir, latex_file]
            subprocess.call(cmd, cwd=temp_dir, **kws)
            if not os.path.exists(os.path.join(temp_dir, 'doc-%s.pdf' % module)):
                break
        else:
            fd = file(os.path.join(output_dir, 'doc-%s.pdf' % module), 'w')
            fd.write(file(os.path.join(temp_dir, 'doc-%s.pdf' % module)).read())
            fd.close()

    shutil.rmtree(temp_dir)

def publish_tarball(module):
    output_dir = os.path.join(web_directory, module.get('name'), 'stable')
    url = module.get('url')
    directory = module.get('directory')
    filename = os.path.split(urlparse.urlparse(url)[2])[-1]
    tar = tarfile.open(os.path.join(checkouts_directory, filename), 'r')
    for tarinfo in tar:
        if not tarinfo.name.startswith(directory):
            continue
        output_filename = os.path.join(output_dir, tarinfo.name[len(directory):].strip('/'))
        if tarinfo.isdir():
            if not os.path.exists(output_filename):
                os.makedirs(output_filename)
        else:
            fd = tar.extractfile(tarinfo)
            file(output_filename, 'w').write(fd.read())


def create_index(modules):
    index_html = os.path.join(web_directory, 'index.html')
    t = Template(unicode(file('index_template.html').read(), 'utf-8'))
    c = Context({'modules': modules})
    fd = file(index_html, 'w')
    fd.write(t.render(c).encode('utf-8'))
    fd.close()


modules = json.load(file('modules.json'))

for module in modules:
    if module.get('type') == 'tarball':
        download(module.get('url'))
    else:
        checkout(module.get('name'))

    if module.get('directory'):
        module['branch'] = 'stable'
        publish_tarball(module)
    else:
        module['branch'] = 'dev'
        publish_mallard(module.get('name'), 'master', 'dev')

create_index(modules)