566 lines
21 KiB
Python
Executable File
566 lines
21 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import atexit
|
|
import random
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
import urllib.parse
|
|
import os
|
|
|
|
from eobuilder import settings, VERSION, init
|
|
from eobuilder.changelog import changelog_from_git
|
|
from eobuilder.cmdline import parse_cmdline, error, cat, touch, call, output, setup_py
|
|
|
|
|
|
# fix urljoin for git+ssh:// URLs
|
|
if 'git+ssh' not in urllib.parse.uses_relative:
|
|
urllib.parse.uses_relative.append('git+ssh')
|
|
|
|
|
|
def rm_recursive(path):
|
|
if os.path.exists(path):
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def smart_cleaning(files_path):
|
|
now = time.time()
|
|
project_files = {}
|
|
for file_path in files_path:
|
|
project_name = os.path.basename(file_path).split('_')[0]
|
|
if project_name not in project_files:
|
|
project_files[project_name] = []
|
|
project_files[project_name].append(file_path)
|
|
|
|
for project in project_files.iterkeys():
|
|
nb_versions = len(project_files[project])
|
|
if nb_versions > settings.MIN_PACKAGE_VERSIONS:
|
|
project_files[project] = sorted(project_files[project], key=lambda x: os.stat(x).st_mtime)
|
|
for filename in project_files[project]:
|
|
if (
|
|
nb_versions > settings.MIN_PACKAGE_VERSIONS
|
|
and os.stat(filename).st_mtime < now - settings.MIN_AGE * 86400
|
|
):
|
|
os.remove(filename)
|
|
nb_versions -= 1
|
|
|
|
|
|
def clean(method):
|
|
print('+ Cleanning %s' % method)
|
|
if method == 'all':
|
|
shutil.rmtree(settings.ORIGIN_PATH)
|
|
shutil.rmtree(settings.PBUILDER_RESULT)
|
|
shutil.rmtree(settings.GIT_PATH)
|
|
shutil.rmtree(settings.LOCK_PATH)
|
|
elif method == 'git':
|
|
shutil.rmtree(settings.GIT_PATH)
|
|
elif method == 'deb':
|
|
shutil.rmtree(settings.PBUILDER_RESULT)
|
|
elif method == 'archives':
|
|
shutil.rmtree(settings.ORIGIN_PATH)
|
|
elif method == 'locks':
|
|
shutil.rmtree(settings.LOCK_PATH)
|
|
elif method == 'smart':
|
|
results_files = []
|
|
origin_files = [os.path.join(settings.ORIGIN_PATH, f) for f in os.listdir(settings.ORIGIN_PATH)]
|
|
for root, dirs, files in os.walk(settings.PBUILDER_RESULT):
|
|
for fname in files:
|
|
results_files.append(os.path.join(root, fname))
|
|
smart_cleaning(results_files)
|
|
smart_cleaning(origin_files)
|
|
now = time.time()
|
|
for root, dirs, files in os.walk(settings.PBUILDER_RESULT):
|
|
for fname in files:
|
|
fname = os.path.join(root, fname)
|
|
ext = os.path.splitext(fname)
|
|
if ext == 'build' and os.stat(fname).st_mtime < now - 365 * 86400:
|
|
os.remove(fname)
|
|
else:
|
|
error("Cleanning: unknow '%s' option" % method)
|
|
|
|
|
|
def get_project_infos(git_project_path, cmd_options):
|
|
"""return a dict with project informations"""
|
|
os.chdir(git_project_path)
|
|
results = {
|
|
'name': '',
|
|
'version': '',
|
|
'fullname': '',
|
|
'ac_version': '',
|
|
'build_dir': '',
|
|
'build_branch': cmd_options.branch,
|
|
'commit_number': '',
|
|
'git_path': git_project_path,
|
|
}
|
|
if os.path.exists('setup.py'):
|
|
# Hack to support setup_requires
|
|
setup_py('--help')
|
|
results['name'] = setup_py('--name 2> /dev/null')[:-1]
|
|
results['version'] = setup_py('--version 2> /dev/null')[:-1]
|
|
results['fullname'] = setup_py('--fullname 2> /dev/null')[:-1]
|
|
elif os.path.exists('configure.ac'):
|
|
call('./autogen.sh')
|
|
call('make all')
|
|
results['name'] = output("./configure --version | head -n1 | sed 's/ configure.*//'")[:-1]
|
|
results['ac_version'] = output("./configure --version | head -n1 | sed 's/.* configure //'")[:-1]
|
|
results['version'] = results['ac_version'].replace('-', '.')
|
|
results['fullname'] = results['name']
|
|
elif os.path.exists('Makefile'):
|
|
results['name'] = output('make name')[:-1]
|
|
results['version'] = output('make version')[:-1]
|
|
results['fullname'] = output('make fullname')[:-1]
|
|
else:
|
|
error('Unsupported project type', exit_code=2)
|
|
|
|
results['build_dir'] = os.path.join(
|
|
settings.EOBUILDER_TMP, '%s-%d' % (results['name'], random.randint(0, 1000000))
|
|
)
|
|
atexit.register(rm_recursive, results['build_dir'])
|
|
results['commit_number'] = output('git rev-parse HEAD')[:-1]
|
|
results['lock_path'] = os.path.join(settings.LOCK_PATH, results['name'])
|
|
current_tag = output('git describe --abbrev=0 --tags --match=v*', exit_on_error=False)
|
|
if current_tag:
|
|
results['current_tag'] = current_tag[1:-1]
|
|
else:
|
|
results['current_tag'] = '0.0'
|
|
|
|
return results
|
|
|
|
|
|
def prepare_build(dist, project, cmd_options, new):
|
|
"""
|
|
Create origin archive, update git and Debian changelog
|
|
"""
|
|
package = {
|
|
'repository': settings.DEFAULT_UNSTABLE_REPOSITORIES[dist],
|
|
'version': '',
|
|
'source_name': '',
|
|
'names': [],
|
|
}
|
|
if cmd_options.hotfix:
|
|
package['repository'] = settings.HOTFIX_REPOSITORIES[dist]
|
|
os.chdir(project['git_path'])
|
|
build_branch = cmd_options.branch
|
|
if cmd_options.hotfix and not build_branch.startswith('hotfix/'):
|
|
return error('Invalid name for hotfix branch (must start with hotfix/)', exit_code=2)
|
|
debian_folder = cmd_options.debian_folder
|
|
if os.path.isdir('debian-' + dist) and debian_folder == 'debian':
|
|
debian_folder = 'debian-' + dist
|
|
debian_branch = None
|
|
if not os.path.isdir(debian_folder):
|
|
debian_branch = 'debian'
|
|
debian_folder = 'debian'
|
|
branches = output('git branch -r -l')
|
|
if debian_branch == 'debian' and 'debian-%s' % dist in branches:
|
|
debian_branch = 'debian-' + dist
|
|
if not 'origin/%s' % debian_branch in output('git branch -r -l'):
|
|
print('!!! WARNING: cannot build for dist %s, no debian directory found' % dist)
|
|
return
|
|
print('!!! WARNING obsolete: using a branch for debian/ packaging')
|
|
print('+ Updating Debian branch for %s' % dist)
|
|
call('git checkout --quiet %s' % debian_branch)
|
|
call('git pull')
|
|
else:
|
|
print('+ Building from %s debian folder' % debian_folder)
|
|
for r in cmd_options.repositories:
|
|
repo = r.split(':')
|
|
if repo[0] == dist:
|
|
package['repository'] = repo[1]
|
|
|
|
# get package source name
|
|
control_file = os.path.join(debian_folder, 'control')
|
|
package['names'] = re.findall(r'Package\s*:\s*(.*?)\n', cat(control_file))
|
|
package['source_name'] = re.search(r'^Source\s*:\s*(.*?)\n', cat(control_file), re.MULTILINE).group(1)
|
|
|
|
# build tarball
|
|
origin_archive = os.path.join(
|
|
settings.ORIGIN_PATH, '%s_%s.orig.tar.bz2' % (package['source_name'], project['version'])
|
|
)
|
|
if not os.path.exists(origin_archive):
|
|
print('+ Generating origin tarball ...')
|
|
os.chdir(project['git_path'])
|
|
call('git checkout --quiet %s' % build_branch)
|
|
if os.path.exists('setup.py'):
|
|
setup_py('clean --all')
|
|
setup_py('sdist --formats=bztar')
|
|
shutil.move('dist/%s.tar.bz2' % project['fullname'], origin_archive)
|
|
elif os.path.exists('./configure.ac'):
|
|
call('make dist-bzip2')
|
|
shutil.move('%s-%s.tar.bz2' % (project['name'], project['ac_version']), origin_archive)
|
|
elif os.path.exists('Makefile'):
|
|
call('make dist-bzip2')
|
|
shutil.move('sdist/%s.tar.bz2' % project['fullname'], origin_archive)
|
|
else:
|
|
error('Unsupported project type', project['build_dir'], exit_code=2)
|
|
|
|
last_version_file = os.path.join(
|
|
project['lock_path'],
|
|
'%s_%s_%s.last_version' % (project['name'], package['repository'], build_branch.replace('/', '_')),
|
|
)
|
|
debian_changelog = os.path.join(debian_folder, 'changelog')
|
|
if os.path.exists(last_version_file):
|
|
last_debian_package_version = cat(last_version_file)
|
|
else:
|
|
last_debian_package_version = re.search(
|
|
r'^Version:\s(.*?)$', output('dpkg-parsechangelog -l%s' % debian_changelog), re.MULTILINE
|
|
).group(1)
|
|
last_version = last_debian_package_version.split('-')[0]
|
|
package['version'] = last_debian_package_version
|
|
|
|
if cmd_options.native:
|
|
debian_revision_number = ''
|
|
else:
|
|
debian_revision_number = '-1'
|
|
|
|
if last_version == project['version'] and new and '~eob' in last_debian_package_version:
|
|
new_inc = int(last_debian_package_version.rsplit('+', 1)[-1]) + 1
|
|
version_suffix = '%s~eob%s+%s' % (debian_revision_number, settings.DEBIAN_VERSIONS[dist], new_inc)
|
|
else:
|
|
version_suffix = '%s~eob%s+1' % (debian_revision_number, settings.DEBIAN_VERSIONS[dist])
|
|
call('git checkout --quiet %s' % build_branch)
|
|
changelog = '\n'.join(
|
|
changelog_from_git(
|
|
package['source_name'],
|
|
version_suffix,
|
|
project['git_path'],
|
|
package['repository'],
|
|
epoch=cmd_options.epoch,
|
|
)
|
|
)
|
|
if changelog:
|
|
if not os.path.isdir(debian_folder):
|
|
call('git checkout --quiet %s' % debian_branch)
|
|
debian_generated_changelog_filename = debian_changelog + '.generated'
|
|
with open(debian_generated_changelog_filename, 'w+') as f:
|
|
f.write(changelog)
|
|
package['version'] = re.search(
|
|
r'^Version:\s(.*?)$',
|
|
output('dpkg-parsechangelog -l%s' % debian_generated_changelog_filename),
|
|
re.MULTILINE,
|
|
).group(1)
|
|
os.unlink(debian_generated_changelog_filename)
|
|
else:
|
|
# changelog couldn't be generated, this happens with checkouts from
|
|
# dgit, at least.
|
|
package['version'] = ''
|
|
|
|
if os.path.exists(project['build_dir']):
|
|
shutil.rmtree(project['build_dir'])
|
|
os.makedirs(project['build_dir'], 0o755)
|
|
|
|
if package['version'].split('-')[0].split(':')[-1] == project['version']:
|
|
# the generated changelog has the right version number, use it.
|
|
good_changelog_contents = changelog
|
|
else:
|
|
# wrong version number, in that case we add an arbitrary new entry
|
|
# to the existing changelog
|
|
package['version'] = '%s%s' % (project['version'], version_suffix)
|
|
call(
|
|
'dch "Eobuilder version" -v %s --distribution %s \
|
|
--force-bad-version --force-distribution --changelog %s'
|
|
% (
|
|
'%s:%s' % (cmd_options.epoch, package['version'])
|
|
if cmd_options.epoch
|
|
else package['version'],
|
|
package['repository'],
|
|
debian_changelog,
|
|
)
|
|
)
|
|
good_changelog_contents = open(debian_changelog).read()
|
|
|
|
if cmd_options.hotfix:
|
|
version_part = build_branch.split('/', 1)[1].lstrip('v')
|
|
if not project['version'].startswith(version_part):
|
|
return error('Invalid name for hotfix branch (must start with version number)', exit_code=2)
|
|
|
|
build_file = os.path.join(
|
|
project['lock_path'],
|
|
'%s_%s_%s_%s.build'
|
|
% (project['name'], package['version'], package['repository'], build_branch.replace('/', '_')),
|
|
)
|
|
if os.path.exists(build_file):
|
|
print('+ Already built for %s !' % dist)
|
|
return package
|
|
|
|
print('+ Preparing Debian build (%s %s) ...' % (package['source_name'], package['version']))
|
|
if debian_branch:
|
|
call('git checkout --quiet %s' % debian_branch)
|
|
os.chdir(project['build_dir'])
|
|
project_build_path = os.path.join(project['build_dir'], '%s-%s' % (project['name'], project['version']))
|
|
shutil.copy(origin_archive, project['build_dir'])
|
|
tar = tarfile.open('%s_%s.orig.tar.bz2' % (package['source_name'], project['version']), 'r:bz2')
|
|
tar.extractall()
|
|
tar.close()
|
|
if os.path.exists('%s/debian' % project_build_path):
|
|
shutil.rmtree('%s/debian' % project_build_path)
|
|
|
|
shutil.copytree(os.path.join(project['git_path'], debian_folder), '%s/debian' % project_build_path)
|
|
with open(os.path.join(project_build_path, 'debian', 'changelog'), 'w') as f:
|
|
f.write(good_changelog_contents)
|
|
return package
|
|
|
|
|
|
def build_project(dist, arch, project, package, new):
|
|
pbuilder_project_result = os.path.join(settings.PBUILDER_RESULT, '%s-%s' % (dist, arch))
|
|
project_build_path = os.path.join(project['build_dir'], '%s-%s' % (project['name'], project['version']))
|
|
if not os.path.exists(pbuilder_project_result):
|
|
os.makedirs(pbuilder_project_result, 0o755)
|
|
os.chdir(project['lock_path'])
|
|
source_build = os.path.join(
|
|
project['lock_path'],
|
|
'%s_%s_%s_%s_source.build'
|
|
% (
|
|
project['name'],
|
|
project['version'],
|
|
package['repository'],
|
|
project['build_branch'].replace('/', '_'),
|
|
),
|
|
)
|
|
bin_build = os.path.join(
|
|
project['lock_path'],
|
|
'%s_%s_%s_%s_%s.build'
|
|
% (
|
|
project['name'],
|
|
package['version'],
|
|
package['repository'],
|
|
project['build_branch'].replace('/', '_'),
|
|
arch,
|
|
),
|
|
)
|
|
print('SOURCE_BUILD:', source_build)
|
|
if os.path.exists(source_build):
|
|
source_opt = '-b'
|
|
else:
|
|
source_opt = '-sa'
|
|
if new == 0 and os.path.exists(bin_build):
|
|
print('+ Already build !')
|
|
return
|
|
os.chdir(project_build_path)
|
|
print('+ Building %s %s %s %s' % (project['name'], project['version'], dist, arch))
|
|
call(
|
|
'DIST=%s ARCH=%s pdebuild --use-pdebuild-internal --architecture %s --debbuildopts "%s"'
|
|
% (dist, arch, arch, source_opt)
|
|
)
|
|
|
|
print('+ Lock build')
|
|
touch(bin_build)
|
|
if not os.path.exists(source_build):
|
|
touch(source_build)
|
|
|
|
|
|
def send_packages(dist, arch, project, package, last_tag, dput=True):
|
|
stamp_file = os.path.join(
|
|
project['lock_path'],
|
|
'%s_%s_%s_%s_%s.upload'
|
|
% (
|
|
project['name'],
|
|
package['version'],
|
|
package['repository'],
|
|
arch,
|
|
project['build_branch'].replace('/', '_'),
|
|
),
|
|
)
|
|
if os.path.exists(stamp_file):
|
|
print('+ Already uploaded')
|
|
return
|
|
|
|
pbuilder_project_result = os.path.join(settings.PBUILDER_RESULT, '%s-%s' % (dist, arch))
|
|
|
|
print('+ Updating local repository...')
|
|
subprocess.check_call(
|
|
'apt-ftparchive packages . | gzip > Packages.gz', cwd=pbuilder_project_result, shell=True
|
|
)
|
|
|
|
if dput:
|
|
print('+ Sending package...')
|
|
os.chdir(pbuilder_project_result)
|
|
call(
|
|
'dput -u %s %s_%s_%s.changes'
|
|
% (package['repository'], package['source_name'], package['version'].split(':', 1)[-1], arch)
|
|
)
|
|
else:
|
|
print('+ Package not sent to repository (--no-dput used).')
|
|
return
|
|
|
|
open(stamp_file, 'w').close()
|
|
|
|
|
|
def clean_git_on_exit(git_project_path):
|
|
if not os.path.exists(git_project_path):
|
|
return
|
|
os.chdir(git_project_path)
|
|
call('git stash --quiet')
|
|
changelog_tmp = os.path.join(git_project_path, 'debian', 'changelog.git')
|
|
if os.path.exists(changelog_tmp):
|
|
os.remove(changelog_tmp)
|
|
|
|
|
|
def get_git_project_name(project_reference):
|
|
project_name = os.path.basename(project_reference)
|
|
if project_name.endswith('.git'):
|
|
project_name = project_name[:-4]
|
|
return project_name
|
|
|
|
|
|
def get_git_project_path(project_reference):
|
|
return os.path.join(settings.GIT_PATH, get_git_project_name(project_reference))
|
|
|
|
|
|
def get_git_branch_name(project_reference):
|
|
git_project_path = get_git_project_path(project_reference)
|
|
for branch_name in ('main', 'master'):
|
|
try:
|
|
subprocess.check_call(
|
|
['git', 'rev-parse', branch_name],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
cwd=git_project_path,
|
|
)
|
|
except subprocess.CalledProcessError:
|
|
continue
|
|
return branch_name
|
|
else:
|
|
raise Exception('failed to determine branch')
|
|
|
|
|
|
def setup_git_tree(project_reference, options):
|
|
git_project_path = get_git_project_path(project_reference)
|
|
if options.branch and options.branch.startswith('wip/'):
|
|
if os.path.exists(git_project_path):
|
|
shutil.rmtree(git_project_path)
|
|
if os.path.exists(git_project_path):
|
|
existing_tree = True
|
|
branch_name = options.branch or get_git_branch_name(project_reference)
|
|
try:
|
|
subprocess.check_call(['git', 'fetch'], cwd=git_project_path)
|
|
subprocess.check_call(['git', 'checkout', '--quiet', branch_name], cwd=git_project_path)
|
|
subprocess.check_call(['git', 'reset', '--hard', 'origin/%s' % branch_name], cwd=git_project_path)
|
|
except subprocess.CalledProcessError as e:
|
|
print(e, file=sys.stderr)
|
|
shutil.rmtree(git_project_path)
|
|
return setup_git_tree(project_reference, options)
|
|
else:
|
|
existing_tree = False
|
|
os.chdir(settings.GIT_PATH)
|
|
if project_reference.startswith('/'):
|
|
call('git clone %s' % project_reference)
|
|
else:
|
|
parsed = urllib.parse.urlparse(project_reference)
|
|
if not parsed.netloc:
|
|
project_url = urllib.parse.urljoin(settings.GIT_REPOSITORY_URL, project_reference) + '.git'
|
|
else:
|
|
project_url = project_reference
|
|
call('git clone %s' % project_url)
|
|
if options.branch:
|
|
subprocess.check_call(['git', 'checkout', '--quiet', options.branch], cwd=git_project_path)
|
|
branch_name = get_git_branch_name(project_reference)
|
|
|
|
if not options.branch:
|
|
options.branch = branch_name
|
|
|
|
os.chdir(git_project_path)
|
|
try:
|
|
subprocess.check_call(['git', 'checkout', '--quiet', branch_name])
|
|
subprocess.check_call(['git', 'pull'])
|
|
subprocess.check_call(['git', 'submodule', 'init'])
|
|
subprocess.check_call(['git', 'submodule', 'update'])
|
|
except subprocess.CalledProcessError as e:
|
|
if existing_tree:
|
|
print(e, file=sys.stderr)
|
|
shutil.rmtree(git_project_path)
|
|
return setup_git_tree(project_reference, options)
|
|
raise
|
|
|
|
|
|
def main():
|
|
options, args = parse_cmdline()
|
|
for method in options.cleaning:
|
|
clean(method)
|
|
|
|
if options.cleaning:
|
|
sys.exit(0)
|
|
|
|
init()
|
|
project_reference = args[0]
|
|
git_project_path = get_git_project_path(project_reference)
|
|
atexit.register(clean_git_on_exit, git_project_path)
|
|
if options.branch and options.branch.startswith('origin/'):
|
|
# normalize without origin/
|
|
options.branch = options.branch[len('origin/') :]
|
|
existing_tree = os.path.exists(git_project_path)
|
|
if existing_tree:
|
|
os.chdir(git_project_path)
|
|
last_tag = output('git describe --abbrev=0 --tags --match=v*', exit_on_error=False)
|
|
if last_tag:
|
|
last_tag = last_tag[1:-1]
|
|
else:
|
|
last_tag = '0.0'
|
|
else:
|
|
last_tag = '0.0'
|
|
setup_git_tree(project_reference, options)
|
|
project = get_project_infos(git_project_path, options)
|
|
|
|
if not os.path.exists(project['lock_path']):
|
|
os.mkdir(project['lock_path'], 0o755)
|
|
|
|
# compare revision between last build and now to determine if something is really new
|
|
new = 1
|
|
current_revision = output('git rev-parse HEAD', True).strip()
|
|
branch_name = get_git_branch_name(project_reference)
|
|
last_branch_revision_file_path = os.path.join(
|
|
project['lock_path'], '%s_%s.last_revision' % (project['name'], branch_name.replace('/', '_'))
|
|
)
|
|
try:
|
|
with open(last_branch_revision_file_path) as f:
|
|
last_branch_revision = f.read().strip()
|
|
except IOError:
|
|
pass
|
|
else:
|
|
if current_revision == last_branch_revision:
|
|
new = 0
|
|
|
|
if options.force and not new:
|
|
print('+ Warning force a new build')
|
|
new = 1
|
|
|
|
for dist in options.distrib:
|
|
os.chdir(git_project_path)
|
|
call('git checkout --quiet %s' % branch_name)
|
|
package = prepare_build(dist, project, options, new)
|
|
if package:
|
|
for arch in options.architectures:
|
|
build_project(dist, arch, project, package, new)
|
|
send_packages(dist, arch, project, package, last_tag, dput=options.dput)
|
|
print('+ Add a build file to lock new build for %s' % dist)
|
|
touch(
|
|
os.path.join(
|
|
project['lock_path'],
|
|
'%s_%s_%s_%s.build'
|
|
% (
|
|
project['name'],
|
|
package['version'],
|
|
package['repository'],
|
|
branch_name.replace('/', '_'),
|
|
),
|
|
)
|
|
)
|
|
|
|
last_version_file = os.path.join(
|
|
project['lock_path'],
|
|
'%s_%s_%s.last_version'
|
|
% (project['name'], package['repository'], branch_name.replace('/', '_')),
|
|
)
|
|
with open(last_version_file, 'w+') as f:
|
|
f.write(package['version'])
|
|
|
|
# keep current revision for next build
|
|
with open(last_branch_revision_file_path, 'w+') as f:
|
|
f.write(current_revision)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|