eobuilder/eobuilder-ctl

#!/usr/bin/env python3

import atexit
import random
import re
import shutil
import subprocess
import sys
import tarfile
import time
import urllib.parse
import os

from eobuilder import settings, VERSION, init
from eobuilder.changelog import changelog_from_git
from eobuilder.cmdline import parse_cmdline, error, cat, touch, call, output, setup_py


# fix urljoin for git+ssh:// URLs
if 'git+ssh' not in urllib.parse.uses_relative:
    urllib.parse.uses_relative.append('git+ssh')


def rm_recursive(path):
    if os.path.exists(path):
        shutil.rmtree(path)


def smart_cleaning(files_path):
    now = time.time()
    project_files = {}
    for file_path in files_path:
        project_name = os.path.basename(file_path).split('_')[0]
        if project_name not in project_files:
            project_files[project_name] = []
        project_files[project_name].append(file_path)

    for project in project_files.iterkeys():
        nb_versions = len(project_files[project])
        if nb_versions > settings.MIN_PACKAGE_VERSIONS:
            project_files[project] = sorted(project_files[project], key=lambda x: os.stat(x).st_mtime)
            for filename in project_files[project]:
                if (
                    nb_versions > settings.MIN_PACKAGE_VERSIONS
                    and os.stat(filename).st_mtime < now - settings.MIN_AGE * 86400
                ):
                    os.remove(filename)
                    nb_versions -= 1


def clean(method):
    print('+ Cleanning %s' % method)
    if method == 'all':
        shutil.rmtree(settings.ORIGIN_PATH)
        shutil.rmtree(settings.PBUILDER_RESULT)
        shutil.rmtree(settings.GIT_PATH)
        shutil.rmtree(settings.LOCK_PATH)
    elif method == 'git':
        shutil.rmtree(settings.GIT_PATH)
    elif method == 'deb':
        shutil.rmtree(settings.PBUILDER_RESULT)
    elif method == 'archives':
        shutil.rmtree(settings.ORIGIN_PATH)
    elif method == 'locks':
        shutil.rmtree(settings.LOCK_PATH)
    elif method == 'smart':
        results_files = []
        origin_files = [os.path.join(settings.ORIGIN_PATH, f) for f in os.listdir(settings.ORIGIN_PATH)]
        for root, dirs, files in os.walk(settings.PBUILDER_RESULT):
            for fname in files:
                results_files.append(os.path.join(root, fname))
        smart_cleaning(results_files)
        smart_cleaning(origin_files)
        now = time.time()
        for root, dirs, files in os.walk(settings.PBUILDER_RESULT):
            for fname in files:
                fname = os.path.join(root, fname)
                ext = os.path.splitext(fname)
                if ext == 'build' and os.stat(fname).st_mtime < now - 365 * 86400:
                    os.remove(fname)
    else:
        error("Cleanning: unknow '%s' option" % method)


def get_project_infos(git_project_path, cmd_options):
    """return a dict with project informations"""
    os.chdir(git_project_path)
    results = {
        'name': '',
        'version': '',
        'fullname': '',
        'ac_version': '',
        'build_dir': '',
        'build_branch': cmd_options.branch,
        'commit_number': '',
        'git_path': git_project_path,
    }
    if os.path.exists('setup.py'):
        # Hack to support setup_requires
        setup_py('--help')
        results['name'] = setup_py('--name 2> /dev/null')[:-1]
        results['version'] = setup_py('--version 2> /dev/null')[:-1]
        results['fullname'] = setup_py('--fullname 2> /dev/null')[:-1]
    elif os.path.exists('configure.ac'):
        call('./autogen.sh')
        call('make all')
        results['name'] = output("./configure --version | head -n1 | sed 's/ configure.*//'")[:-1]
        results['ac_version'] = output("./configure --version | head -n1 | sed 's/.* configure //'")[:-1]
        results['version'] = results['ac_version'].replace('-', '.')
        results['fullname'] = results['name']
    elif os.path.exists('Makefile'):
        results['name'] = output('make name')[:-1]
        results['version'] = output('make version')[:-1]
        results['fullname'] = output('make fullname')[:-1]
    else:
        error('Unsupported project type', exit_code=2)

    results['build_dir'] = os.path.join(
        settings.EOBUILDER_TMP, '%s-%d' % (results['name'], random.randint(0, 1000000))
    )
    atexit.register(rm_recursive, results['build_dir'])
    results['commit_number'] = output('git rev-parse HEAD')[:-1]
    results['lock_path'] = os.path.join(settings.LOCK_PATH, results['name'])
    current_tag = output('git describe --abbrev=0 --tags --match=v*', exit_on_error=False)
    if current_tag:
        results['current_tag'] = current_tag[1:-1]
    else:
        results['current_tag'] = '0.0'

    return results


def prepare_build(dist, project, cmd_options, new):
    """
    Create origin archive, update git and Debian changelog
    """
    package = {
        'repository': settings.DEFAULT_UNSTABLE_REPOSITORIES[dist],
        'version': '',
        'source_name': '',
        'names': [],
    }
    if cmd_options.hotfix:
        package['repository'] = settings.HOTFIX_REPOSITORIES[dist]
    os.chdir(project['git_path'])
    build_branch = cmd_options.branch
    if cmd_options.hotfix and not build_branch.startswith('hotfix/'):
        return error('Invalid name for hotfix branch (must start with hotfix/)', exit_code=2)
    debian_folder = cmd_options.debian_folder
    if os.path.isdir('debian-' + dist) and debian_folder == 'debian':
        debian_folder = 'debian-' + dist
    debian_branch = None
    if not os.path.isdir(debian_folder):
        debian_branch = 'debian'
        debian_folder = 'debian'
        branches = output('git branch -r -l')
        if debian_branch == 'debian' and 'debian-%s' % dist in branches:
            debian_branch = 'debian-' + dist
        if not 'origin/%s' % debian_branch in output('git branch -r -l'):
            print('!!! WARNING: cannot build for dist %s, no debian directory found' % dist)
            return
        print('!!! WARNING obsolete: using a branch for debian/ packaging')
        print('+ Updating Debian branch for %s' % dist)
        call('git checkout --quiet %s' % debian_branch)
        call('git pull')
    else:
        print('+ Building from %s debian folder' % debian_folder)
    for r in cmd_options.repositories:
        repo = r.split(':')
        if repo[0] == dist:
            package['repository'] = repo[1]

    # get package source name
    control_file = os.path.join(debian_folder, 'control')
    package['names'] = re.findall(r'Package\s*:\s*(.*?)\n', cat(control_file))
    package['source_name'] = re.search(r'^Source\s*:\s*(.*?)\n', cat(control_file), re.MULTILINE).group(1)

    # build tarball
    origin_archive = os.path.join(
        settings.ORIGIN_PATH, '%s_%s.orig.tar.bz2' % (package['source_name'], project['version'])
    )
    if not os.path.exists(origin_archive):
        print('+ Generating origin tarball ...')
        os.chdir(project['git_path'])
        call('git checkout --quiet %s' % build_branch)
        if os.path.exists('setup.py'):
            setup_py('clean --all')
            setup_py('sdist --formats=bztar')
            shutil.move('dist/%s.tar.bz2' % project['fullname'], origin_archive)
        elif os.path.exists('./configure.ac'):
            call('make dist-bzip2')
            shutil.move('%s-%s.tar.bz2' % (project['name'], project['ac_version']), origin_archive)
        elif os.path.exists('Makefile'):
            call('make dist-bzip2')
            shutil.move('sdist/%s.tar.bz2' % project['fullname'], origin_archive)
        else:
            error('Unsupported project type', project['build_dir'], exit_code=2)

    last_version_file = os.path.join(
        project['lock_path'],
        '%s_%s_%s.last_version' % (project['name'], package['repository'], build_branch.replace('/', '_')),
    )
    debian_changelog = os.path.join(debian_folder, 'changelog')
    if os.path.exists(last_version_file):
        last_debian_package_version = cat(last_version_file)
    else:
        last_debian_package_version = re.search(
            r'^Version:\s(.*?)$', output('dpkg-parsechangelog -l%s' % debian_changelog), re.MULTILINE
        ).group(1)
    last_version = last_debian_package_version.split('-')[0]
    package['version'] = last_debian_package_version

    if cmd_options.native:
        debian_revision_number = ''
    else:
        debian_revision_number = '-1'

    if last_version == project['version'] and new and '~eob' in last_debian_package_version:
        new_inc = int(last_debian_package_version.rsplit('+', 1)[-1]) + 1
        version_suffix = '%s~eob%s+%s' % (debian_revision_number, settings.DEBIAN_VERSIONS[dist], new_inc)
    else:
        version_suffix = '%s~eob%s+1' % (debian_revision_number, settings.DEBIAN_VERSIONS[dist])
    call('git checkout --quiet %s' % build_branch)
    changelog = '\n'.join(
        changelog_from_git(
            package['source_name'],
            version_suffix,
            project['git_path'],
            package['repository'],
            epoch=cmd_options.epoch,
        )
    )
    if changelog:
        if not os.path.isdir(debian_folder):
            call('git checkout --quiet %s' % debian_branch)
        debian_generated_changelog_filename = debian_changelog + '.generated'
        with open(debian_generated_changelog_filename, 'w+') as f:
            f.write(changelog)
        package['version'] = re.search(
            r'^Version:\s(.*?)$',
            output('dpkg-parsechangelog -l%s' % debian_generated_changelog_filename),
            re.MULTILINE,
        ).group(1)
        os.unlink(debian_generated_changelog_filename)
    else:
        # changelog couldn't be generated, this happens with checkouts from
        # dgit, at least.
        package['version'] = ''

    if os.path.exists(project['build_dir']):
        shutil.rmtree(project['build_dir'])
    os.makedirs(project['build_dir'], 0o755)

    if package['version'].split('-')[0].split(':')[-1] == project['version']:
        # the generated changelog has the right version number, use it.
        good_changelog_contents = changelog
    else:
        # wrong version number, in that case we add an arbitrary new entry
        # to the existing changelog
        package['version'] = '%s%s' % (project['version'], version_suffix)
        call(
            'dch "Eobuilder version" -v %s --distribution %s \
                 --force-bad-version --force-distribution --changelog %s'
            % (
                '%s:%s' % (cmd_options.epoch, package['version'])
                if cmd_options.epoch
                else package['version'],
                package['repository'],
                debian_changelog,
            )
        )
        good_changelog_contents = open(debian_changelog).read()

    if cmd_options.hotfix:
        version_part = build_branch.split('/', 1)[1].lstrip('v')
        if not project['version'].startswith(version_part):
            return error('Invalid name for hotfix branch (must start with version number)', exit_code=2)

    build_file = os.path.join(
        project['lock_path'],
        '%s_%s_%s_%s.build'
        % (project['name'], package['version'], package['repository'], build_branch.replace('/', '_')),
    )
    if os.path.exists(build_file):
        print('+ Already built for %s !' % dist)
        return package

    print('+ Preparing Debian build (%s %s) ...' % (package['source_name'], package['version']))
    if debian_branch:
        call('git checkout --quiet %s' % debian_branch)
    os.chdir(project['build_dir'])
    project_build_path = os.path.join(project['build_dir'], '%s-%s' % (project['name'], project['version']))
    shutil.copy(origin_archive, project['build_dir'])
    tar = tarfile.open('%s_%s.orig.tar.bz2' % (package['source_name'], project['version']), 'r:bz2')
    tar.extractall()
    tar.close()
    if os.path.exists('%s/debian' % project_build_path):
        shutil.rmtree('%s/debian' % project_build_path)

    shutil.copytree(os.path.join(project['git_path'], debian_folder), '%s/debian' % project_build_path)
    with open(os.path.join(project_build_path, 'debian', 'changelog'), 'w') as f:
        f.write(good_changelog_contents)
    return package


def build_project(dist, arch, project, package, new):
    pbuilder_project_result = os.path.join(settings.PBUILDER_RESULT, '%s-%s' % (dist, arch))
    project_build_path = os.path.join(project['build_dir'], '%s-%s' % (project['name'], project['version']))
    if not os.path.exists(pbuilder_project_result):
        os.makedirs(pbuilder_project_result, 0o755)
    os.chdir(project['lock_path'])
    source_build = os.path.join(
        project['lock_path'],
        '%s_%s_%s_%s_source.build'
        % (
            project['name'],
            project['version'],
            package['repository'],
            project['build_branch'].replace('/', '_'),
        ),
    )
    bin_build = os.path.join(
        project['lock_path'],
        '%s_%s_%s_%s_%s.build'
        % (
            project['name'],
            package['version'],
            package['repository'],
            project['build_branch'].replace('/', '_'),
            arch,
        ),
    )
    print('SOURCE_BUILD:', source_build)
    if os.path.exists(source_build):
        source_opt = '-b'
    else:
        source_opt = '-sa'
    if new == 0 and os.path.exists(bin_build):
        print('+ Already build !')
        return
    os.chdir(project_build_path)
    print('+ Building %s %s %s %s' % (project['name'], project['version'], dist, arch))
    call(
        'DIST=%s ARCH=%s pdebuild --use-pdebuild-internal --architecture %s --debbuildopts "%s"'
        % (dist, arch, arch, source_opt)
    )

    print('+ Lock build')
    touch(bin_build)
    if not os.path.exists(source_build):
        touch(source_build)


def send_packages(dist, arch, project, package, last_tag, dput=True):
    stamp_file = os.path.join(
        project['lock_path'],
        '%s_%s_%s_%s_%s.upload'
        % (
            project['name'],
            package['version'],
            package['repository'],
            arch,
            project['build_branch'].replace('/', '_'),
        ),
    )
    if os.path.exists(stamp_file):
        print('+ Already uploaded')
        return

    pbuilder_project_result = os.path.join(settings.PBUILDER_RESULT, '%s-%s' % (dist, arch))

    print('+ Updating local repository...')
    subprocess.check_call(
        'apt-ftparchive packages . | gzip > Packages.gz', cwd=pbuilder_project_result, shell=True
    )

    if dput:
        print('+ Sending package...')
        os.chdir(pbuilder_project_result)
        call(
            'dput -u %s %s_%s_%s.changes'
            % (package['repository'], package['source_name'], package['version'].split(':', 1)[-1], arch)
        )
    else:
        print('+ Package not sent to repository (--no-dput used).')
        return

    open(stamp_file, 'w').close()


def clean_git_on_exit(git_project_path):
    if not os.path.exists(git_project_path):
        return
    os.chdir(git_project_path)
    call('git stash --quiet')
    changelog_tmp = os.path.join(git_project_path, 'debian', 'changelog.git')
    if os.path.exists(changelog_tmp):
        os.remove(changelog_tmp)


def get_git_project_name(project_reference):
    project_name = os.path.basename(project_reference)
    if project_name.endswith('.git'):
        project_name = project_name[:-4]
    return project_name


def get_git_project_path(project_reference):
    return os.path.join(settings.GIT_PATH, get_git_project_name(project_reference))


def get_git_branch_name(project_reference):
    git_project_path = get_git_project_path(project_reference)
    for branch_name in ('main', 'master'):
        try:
            subprocess.check_call(
                ['git', 'rev-parse', branch_name],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
                cwd=git_project_path,
            )
        except subprocess.CalledProcessError:
            continue
        return branch_name
    else:
        raise Exception('failed to determine branch')


def setup_git_tree(project_reference, options):
    git_project_path = get_git_project_path(project_reference)
    if options.branch and options.branch.startswith('wip/'):
        if os.path.exists(git_project_path):
            shutil.rmtree(git_project_path)
    if os.path.exists(git_project_path):
        existing_tree = True
        branch_name = options.branch or get_git_branch_name(project_reference)
        try:
            subprocess.check_call(['git', 'fetch'], cwd=git_project_path)
            subprocess.check_call(['git', 'checkout', '--quiet', branch_name], cwd=git_project_path)
            subprocess.check_call(['git', 'reset', '--hard', 'origin/%s' % branch_name], cwd=git_project_path)
        except subprocess.CalledProcessError as e:
            print(e, file=sys.stderr)
            shutil.rmtree(git_project_path)
            return setup_git_tree(project_reference, options)
    else:
        existing_tree = False
        os.chdir(settings.GIT_PATH)
        if project_reference.startswith('/'):
            call('git clone %s' % project_reference)
        else:
            parsed = urllib.parse.urlparse(project_reference)
            if not parsed.netloc:
                project_url = urllib.parse.urljoin(settings.GIT_REPOSITORY_URL, project_reference) + '.git'
            else:
                project_url = project_reference
            call('git clone %s' % project_url)
        if options.branch:
            subprocess.check_call(['git', 'checkout', '--quiet', options.branch], cwd=git_project_path)
        branch_name = get_git_branch_name(project_reference)

    if not options.branch:
        options.branch = branch_name

    os.chdir(git_project_path)
    try:
        subprocess.check_call(['git', 'checkout', '--quiet', branch_name])
        subprocess.check_call(['git', 'pull'])
        subprocess.check_call(['git', 'submodule', 'init'])
        subprocess.check_call(['git', 'submodule', 'update'])
    except subprocess.CalledProcessError as e:
        if existing_tree:
            print(e, file=sys.stderr)
            shutil.rmtree(git_project_path)
            return setup_git_tree(project_reference, options)
        raise


def main():
    options, args = parse_cmdline()
    for method in options.cleaning:
        clean(method)

    if options.cleaning:
        sys.exit(0)

    init()
    project_reference = args[0]
    git_project_path = get_git_project_path(project_reference)
    atexit.register(clean_git_on_exit, git_project_path)
    if options.branch and options.branch.startswith('origin/'):
        # normalize without origin/
        options.branch = options.branch[len('origin/') :]
    existing_tree = os.path.exists(git_project_path)
    if existing_tree:
        os.chdir(git_project_path)
        last_tag = output('git describe --abbrev=0 --tags --match=v*', exit_on_error=False)
        if last_tag:
            last_tag = last_tag[1:-1]
        else:
            last_tag = '0.0'
    else:
        last_tag = '0.0'
    setup_git_tree(project_reference, options)
    project = get_project_infos(git_project_path, options)

    if not os.path.exists(project['lock_path']):
        os.mkdir(project['lock_path'], 0o755)

    # compare revision between last build and now to determine if something is really new
    new = 1
    current_revision = output('git rev-parse HEAD', True).strip()
    branch_name = get_git_branch_name(project_reference)
    last_branch_revision_file_path = os.path.join(
        project['lock_path'], '%s_%s.last_revision' % (project['name'], branch_name.replace('/', '_'))
    )
    try:
        with open(last_branch_revision_file_path) as f:
            last_branch_revision = f.read().strip()
    except IOError:
        pass
    else:
        if current_revision == last_branch_revision:
            new = 0

    if options.force and not new:
        print('+ Warning force a new build')
        new = 1

    for dist in options.distrib:
        os.chdir(git_project_path)
        call('git checkout --quiet %s' % branch_name)
        package = prepare_build(dist, project, options, new)
        if package:
            for arch in options.architectures:
                build_project(dist, arch, project, package, new)
                send_packages(dist, arch, project, package, last_tag, dput=options.dput)
            print('+ Add a build file to lock new build for %s' % dist)
            touch(
                os.path.join(
                    project['lock_path'],
                    '%s_%s_%s_%s.build'
                    % (
                        project['name'],
                        package['version'],
                        package['repository'],
                        branch_name.replace('/', '_'),
                    ),
                )
            )

            last_version_file = os.path.join(
                project['lock_path'],
                '%s_%s_%s.last_version'
                % (project['name'], package['repository'], branch_name.replace('/', '_')),
            )
            with open(last_version_file, 'w+') as f:
                f.write(package['version'])

    # keep current revision for next build
    with open(last_branch_revision_file_path, 'w+') as f:
        f.write(current_revision)


if __name__ == '__main__':
    main()