570 lines
22 KiB
Python
Executable File
570 lines
22 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import atexit
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
import urllib.parse
|
|
import os
|
|
|
|
from eobuilder import settings, VERSION, init
|
|
from eobuilder.changelog import changelog_from_git
|
|
from eobuilder.cmdline import parse_cmdline, error, cat, touch, call, output, setup_py
|
|
|
|
def rm_recursive(path):
|
|
if os.path.exists(path):
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def smart_cleaning(files_path):
|
|
now = time.time()
|
|
project_files = {}
|
|
for file_path in files_path:
|
|
project_name = os.path.basename(file_path).split('_')[0]
|
|
if project_name not in project_files:
|
|
project_files[project_name] = []
|
|
project_files[project_name].append(file_path)
|
|
|
|
for project in project_files.iterkeys():
|
|
nb_versions = len(project_files[project])
|
|
if nb_versions > settings.MIN_PACKAGE_VERSIONS:
|
|
project_files[project] = \
|
|
sorted(project_files[project],
|
|
key=lambda x: os.stat(x).st_mtime)
|
|
for filename in project_files[project]:
|
|
if nb_versions > settings.MIN_PACKAGE_VERSIONS and \
|
|
os.stat(filename).st_mtime < now - settings.MIN_AGE * 86400:
|
|
os.remove(filename)
|
|
nb_versions -= 1
|
|
|
|
|
|
def clean(method):
|
|
print("+ Cleanning %s" % method)
|
|
if method == "all":
|
|
shutil.rmtree(settings.ORIGIN_PATH)
|
|
shutil.rmtree(settings.PBUILDER_RESULT)
|
|
shutil.rmtree(settings.GIT_PATH)
|
|
shutil.rmtree(settings.LOCK_PATH)
|
|
elif method == "git":
|
|
shutil.rmtree(settings.GIT_PATH)
|
|
elif method == "deb":
|
|
shutil.rmtree(settings.PBUILDER_RESULT)
|
|
elif method == "archives":
|
|
shutil.rmtree(settings.ORIGIN_PATH)
|
|
elif method == "locks":
|
|
shutil.rmtree(settings.LOCK_PATH)
|
|
elif method == "smart":
|
|
results_files = []
|
|
origin_files = [os.path.join(settings.ORIGIN_PATH, f) \
|
|
for f in os.listdir(settings.ORIGIN_PATH)]
|
|
for root, dirs, files in os.walk(settings.PBUILDER_RESULT):
|
|
for fname in files:
|
|
results_files.append(os.path.join(root, fname))
|
|
smart_cleaning(results_files)
|
|
smart_cleaning(origin_files)
|
|
now = time.time()
|
|
for root, dirs, files in os.walk(settings.PBUILDER_RESULT):
|
|
for fname in files:
|
|
fname = os.path.join(root, fname)
|
|
ext = os.path.splitext(fname)
|
|
if ext == "build" and \
|
|
os.stat(fname).st_mtime < now - 365 * 86400:
|
|
os.remove(fname)
|
|
else:
|
|
error("Cleanning: unknow '%s' option" % method)
|
|
|
|
def get_project_infos(git_project_path, cmd_options):
|
|
""" return a dict with project informations
|
|
"""
|
|
os.chdir(git_project_path)
|
|
results = {
|
|
'name': '',
|
|
'version': '',
|
|
'fullname': '',
|
|
'ac_version': '',
|
|
'build_dir': '',
|
|
'build_branch': cmd_options.branch,
|
|
'commit_number': '',
|
|
'git_path': git_project_path,
|
|
}
|
|
if os.path.exists("setup.py"):
|
|
# Hack to support setup_requires
|
|
setup_py("--help")
|
|
results['name'] = setup_py("--name 2> /dev/null")[:-1]
|
|
results['version'] = setup_py("--version 2> /dev/null")[:-1]
|
|
results['fullname'] = setup_py("--fullname 2> /dev/null")[:-1]
|
|
elif os.path.exists("configure.ac"):
|
|
call("./autogen.sh")
|
|
call('make all')
|
|
results['name'] = output(
|
|
"./configure --version | head -n1 | sed 's/ configure.*//'"
|
|
)[:-1]
|
|
results['ac_version'] = output(
|
|
"./configure --version | head -n1 | sed 's/.* configure //'"
|
|
)[:-1]
|
|
results['version'] = results['ac_version'].replace('-', '.')
|
|
results['fullname'] = results['name']
|
|
elif os.path.exists("Makefile"):
|
|
results['name'] = output("make name")[:-1]
|
|
results['version'] = output("make version")[:-1]
|
|
results['fullname'] = output("make fullname")[:-1]
|
|
else:
|
|
error("Unsupported project type", exit_code=2)
|
|
|
|
results['build_dir'] = os.path.join(
|
|
settings.EOBUILDER_TMP, results['name']
|
|
)
|
|
atexit.register(rm_recursive, results['build_dir'])
|
|
results['commit_number'] = output("git rev-parse HEAD")[:-1]
|
|
results['lock_path'] = os.path.join(settings.LOCK_PATH,
|
|
results['name'])
|
|
current_tag = output(
|
|
"git describe --abbrev=0 --tags --match=v*",
|
|
exit_on_error=False)
|
|
if current_tag:
|
|
results['current_tag'] = current_tag[1:-1]
|
|
else:
|
|
results['current_tag'] = "0.0"
|
|
|
|
return results
|
|
|
|
|
|
def prepare_build(dist, project, cmd_options, new):
|
|
"""
|
|
Create origin archive, update git and Debian changelog
|
|
"""
|
|
package = {
|
|
'repository': settings.DEFAULT_UNSTABLE_REPOSITORIES[dist],
|
|
'copy_in_testing': True,
|
|
'version': '',
|
|
'source_name': '',
|
|
'names': []
|
|
}
|
|
if cmd_options.hotfix:
|
|
package['repository'] = settings.HOTFIX_REPOSITORIES[dist]
|
|
os.chdir(project['git_path'])
|
|
build_branch = cmd_options.branch
|
|
if cmd_options.hotfix and not build_branch.startswith('hotfix/'):
|
|
return error("Invalid name for hotfix branch (must start with hotfix/)", exit_code=2)
|
|
debian_folder = cmd_options.debian_folder
|
|
if os.path.isdir('debian-' + dist) and debian_folder == 'debian':
|
|
debian_folder = 'debian-' + dist
|
|
debian_branch = None
|
|
if not os.path.isdir(debian_folder):
|
|
debian_branch = "debian"
|
|
debian_folder = "debian"
|
|
branches = output("git branch -r -l")
|
|
if debian_branch == "debian" and "debian-%s" % dist in branches:
|
|
debian_branch = "debian-" + dist
|
|
if not 'origin/%s' % debian_branch in output("git branch -r -l"):
|
|
print('!!! WARNING: cannot build for dist %s, no debian directory found' % dist)
|
|
return
|
|
print("!!! WARNING obsolete: using a branch for debian/ packaging")
|
|
print("+ Updating Debian branch for %s" % dist)
|
|
call("git checkout --quiet %s" % debian_branch)
|
|
call("git pull")
|
|
else:
|
|
print("+ Building from %s debian folder" % debian_folder)
|
|
for r in cmd_options.repositories:
|
|
repo = r.split(':')
|
|
if repo[0] == dist:
|
|
package["repository"] = repo[1]
|
|
package["copy_in_testing"] = False
|
|
|
|
# get package source name
|
|
control_file = os.path.join(debian_folder, 'control')
|
|
package['names'] = re.findall(r"Package\s*:\s*(.*?)\n", cat(control_file))
|
|
package['source_name'] = re.search(r"^Source\s*:\s*(.*?)\n",
|
|
cat(control_file),
|
|
re.MULTILINE
|
|
).group(1)
|
|
|
|
# build tarball
|
|
origin_archive = os.path.join(settings.ORIGIN_PATH,
|
|
"%s_%s.orig.tar.bz2" % (package['source_name'], project['version']))
|
|
if not os.path.exists(origin_archive):
|
|
print("+ Generating origin tarball ...")
|
|
os.chdir(project['git_path'])
|
|
call("git checkout --quiet %s" % build_branch)
|
|
if os.path.exists('setup.py'):
|
|
setup_py("clean --all")
|
|
setup_py("sdist --formats=bztar")
|
|
shutil.move("dist/%s.tar.bz2" % project['fullname'], origin_archive)
|
|
elif os.path.exists('./configure.ac'):
|
|
call("make dist-bzip2")
|
|
shutil.move("%s-%s.tar.bz2" % \
|
|
(project['name'], project['ac_version']),
|
|
origin_archive)
|
|
elif os.path.exists('Makefile'):
|
|
call("make dist-bzip2")
|
|
shutil.move("sdist/%s.tar.bz2" % project['fullname'], origin_archive)
|
|
else:
|
|
error('Unsupported project type', project['build_dir'], exit_code=2)
|
|
|
|
last_version_file = os.path.join(project['lock_path'],
|
|
"%s_%s_%s.last_version" % (project['name'],
|
|
package['repository'],
|
|
build_branch.replace('/', '_'))
|
|
)
|
|
debian_changelog = os.path.join(debian_folder, 'changelog')
|
|
if os.path.exists(last_version_file):
|
|
last_debian_package_version = cat(last_version_file)
|
|
else:
|
|
last_debian_package_version = re.search(r"^Version:\s(.*?)$",
|
|
output("dpkg-parsechangelog -l%s" % debian_changelog),
|
|
re.MULTILINE).group(1)
|
|
last_version = last_debian_package_version.split('-')[0]
|
|
package['version'] = last_debian_package_version
|
|
|
|
if cmd_options.native:
|
|
debian_revision_number = ''
|
|
else:
|
|
debian_revision_number = '-1'
|
|
|
|
if last_version == project['version'] and new \
|
|
and "~eob" in last_debian_package_version:
|
|
new_inc = int(last_debian_package_version.rsplit('+', 1)[-1]) + 1
|
|
version_suffix = "%s~eob%s+%s" % (debian_revision_number,
|
|
settings.DEBIAN_VERSIONS[dist],
|
|
new_inc)
|
|
else:
|
|
version_suffix = "%s~eob%s+1" % (debian_revision_number,
|
|
settings.DEBIAN_VERSIONS[dist])
|
|
call("git checkout --quiet %s" % build_branch)
|
|
changelog = '\n'.join(changelog_from_git(package['source_name'],
|
|
version_suffix, project['git_path'], package['repository'],
|
|
epoch=cmd_options.epoch))
|
|
if changelog:
|
|
if not os.path.isdir(debian_folder):
|
|
call("git checkout --quiet %s" % debian_branch)
|
|
debian_generated_changelog_filename = debian_changelog + '.generated'
|
|
with open(debian_generated_changelog_filename, 'w+') as f:
|
|
f.write(changelog)
|
|
package['version'] = re.search(r"^Version:\s(.*?)$",
|
|
output("dpkg-parsechangelog -l%s" % debian_generated_changelog_filename),
|
|
re.MULTILINE).group(1)
|
|
os.unlink(debian_generated_changelog_filename)
|
|
else:
|
|
# changelog couldn't be generated, this happens with checkouts from
|
|
# dgit, at least.
|
|
package['version'] = ''
|
|
|
|
if os.path.exists(project['build_dir']):
|
|
shutil.rmtree(project['build_dir'])
|
|
os.makedirs(project['build_dir'], 0o755)
|
|
|
|
if package['version'].split('-')[0].split(':')[-1] == project['version']:
|
|
# the generated changelog has the right version number, use it.
|
|
good_changelog_contents = changelog
|
|
else:
|
|
# wrong version number, in that case we add an arbitrary new entry
|
|
# to the existing changelog
|
|
package['version'] = '%s%s' % (project['version'], version_suffix)
|
|
call('dch "Eobuilder version" -v %s --distribution %s \
|
|
--force-bad-version --force-distribution --changelog %s' % \
|
|
('%s:%s' % (cmd_options.epoch, package['version']) if cmd_options.epoch else package['version'],
|
|
package["repository"],
|
|
debian_changelog))
|
|
good_changelog_contents = open(debian_changelog).read()
|
|
|
|
if cmd_options.hotfix:
|
|
version_part = build_branch.split('/', 1)[1].lstrip('v')
|
|
if not project['version'].startswith(version_part):
|
|
return error("Invalid name for hotfix branch (must start with version number)", exit_code=2)
|
|
|
|
build_file = os.path.join(project['lock_path'],
|
|
"%s_%s_%s_%s.build" % (project['name'],
|
|
package['version'],
|
|
package['repository'],
|
|
build_branch.replace('/', '_'))
|
|
)
|
|
if os.path.exists(build_file):
|
|
print("+ Already built for %s !" % dist)
|
|
return package
|
|
|
|
print("+ Preparing Debian build (%s %s) ..." % (package['source_name'], package['version']))
|
|
if debian_branch:
|
|
call("git checkout --quiet %s" % debian_branch)
|
|
os.chdir(project['build_dir'])
|
|
project_build_path = os.path.join(project['build_dir'],
|
|
"%s-%s" % (project['name'], project['version']))
|
|
shutil.copy(origin_archive, project['build_dir'])
|
|
tar = tarfile.open('%s_%s.orig.tar.bz2' % \
|
|
(package['source_name'], project['version']),
|
|
'r:bz2')
|
|
tar.extractall()
|
|
tar.close()
|
|
if os.path.exists("%s/debian" % project_build_path):
|
|
shutil.rmtree("%s/debian" % project_build_path)
|
|
|
|
shutil.copytree(os.path.join(project['git_path'], debian_folder),
|
|
"%s/debian" % project_build_path)
|
|
with open(os.path.join(project_build_path, 'debian', 'changelog'), 'w') as f:
|
|
f.write(good_changelog_contents)
|
|
return package
|
|
|
|
def build_project(dist, arch, project, package, new):
|
|
pbuilder_project_result = os.path.join(settings.PBUILDER_RESULT,
|
|
'%s-%s' % (dist, arch))
|
|
project_build_path = os.path.join(project['build_dir'],
|
|
"%s-%s" % (project['name'], project['version']))
|
|
if not os.path.exists(pbuilder_project_result):
|
|
os.makedirs(pbuilder_project_result, 0o755)
|
|
os.chdir(project['lock_path'])
|
|
source_build = os.path.join(project['lock_path'],
|
|
"%s_%s_%s_%s_source.build" % (project['name'],
|
|
project['version'],
|
|
package['repository'],
|
|
project['build_branch'].replace('/', '_'))
|
|
)
|
|
bin_build = os.path.join(project['lock_path'],
|
|
"%s_%s_%s_%s_%s.build" % (project['name'],
|
|
package['version'],
|
|
package['repository'],
|
|
project['build_branch'].replace('/', '_'),
|
|
arch)
|
|
)
|
|
print('SOURCE_BUILD:', source_build)
|
|
if os.path.exists(source_build):
|
|
source_opt = '-b'
|
|
else:
|
|
source_opt = '-sa'
|
|
if new == 0 and os.path.exists(bin_build):
|
|
print("+ Already build !")
|
|
return
|
|
os.chdir(project_build_path)
|
|
print("+ Building %s %s %s %s" % \
|
|
(project['name'], project['version'], dist, arch))
|
|
call('DIST=%s ARCH=%s pdebuild --use-pdebuild-internal --architecture %s --debbuildopts "%s"' % \
|
|
(dist, arch, arch, source_opt))
|
|
|
|
print("+ Lock build")
|
|
touch(bin_build)
|
|
if not os.path.exists(source_build):
|
|
touch(source_build)
|
|
|
|
def send_packages(dist, arch, project, package, last_tag):
|
|
stamp_file = os.path.join(project['lock_path'], '%s_%s_%s_%s_%s.upload' % (
|
|
project['name'], package['version'],
|
|
package['repository'], arch, project['build_branch'].replace('/', '_')))
|
|
if os.path.exists(stamp_file):
|
|
print('+ Already uploaded')
|
|
return
|
|
|
|
pbuilder_project_result = os.path.join(settings.PBUILDER_RESULT,
|
|
'%s-%s' % (dist, arch))
|
|
|
|
print("+ Updating local repository...")
|
|
subprocess.check_call('apt-ftparchive packages . | gzip > Packages.gz', cwd=pbuilder_project_result, shell=True)
|
|
|
|
print("+ Sending package...")
|
|
os.chdir(pbuilder_project_result)
|
|
call("dput -u %s %s_%s_%s.changes" % \
|
|
(package['repository'],
|
|
package['source_name'],
|
|
package['version'].split(':', 1)[-1],
|
|
arch)
|
|
)
|
|
|
|
print("+ Updating repository ...")
|
|
call('ssh root@%s "/etc/cron.hourly/process-incoming"' % \
|
|
settings.REPOSITORY_URL)
|
|
old_version = tuple(int(d) for d in last_tag.split('.'))
|
|
new_version = tuple(int(d) for d in project['current_tag'].split('.'))
|
|
if new_version > old_version and \
|
|
project['current_tag'] == project['version']:
|
|
print("New tag detected : %s" % project['current_tag'])
|
|
if package['source_name'] in settings.MANUAL_TESTING_REPOSITORIES:
|
|
package_repos = settings.MANUAL_TESTING_REPOSITORIES[package['source_name']]
|
|
else:
|
|
package_repos = settings.DEFAULT_TESTING_REPOSITORIES
|
|
packages = package['names'] + [package['source_name']]
|
|
packages = " ".join(packages)
|
|
if dist in package_repos and package['copy_in_testing']:
|
|
for repo in package_repos[dist]:
|
|
print("+ Copy %s packages to %s repository (%s)" % (package['source_name'], repo, dist))
|
|
call('ssh root@%s "/usr/bin/reprepro -b /var/vhosts/deb.entrouvert.org copy %s %s %s"'\
|
|
% (settings.REPOSITORY_URL, repo,
|
|
package['repository'], packages))
|
|
call('ssh root@%s /usr/local/bin/update-deb-repo-html' % \
|
|
settings.REPOSITORY_URL)
|
|
|
|
open(stamp_file, 'w').close()
|
|
|
|
def clean_git_on_exit(git_project_path):
|
|
if not os.path.exists(git_project_path):
|
|
return
|
|
os.chdir(git_project_path)
|
|
call("git stash --quiet")
|
|
changelog_tmp = os.path.join(
|
|
git_project_path,
|
|
"debian", "changelog.git"
|
|
)
|
|
if os.path.exists(changelog_tmp):
|
|
os.remove(changelog_tmp)
|
|
|
|
|
|
def get_git_project_name(project_reference):
|
|
project_name = os.path.basename(project_reference)
|
|
if project_name.endswith('.git'):
|
|
project_name = project_name[:-4]
|
|
return project_name
|
|
|
|
|
|
def get_git_project_path(project_reference):
|
|
return os.path.join(settings.GIT_PATH, get_git_project_name(project_reference))
|
|
|
|
|
|
def get_git_branch_name(project_reference):
|
|
git_project_path = get_git_project_path(project_reference)
|
|
for branch_name in ('main', 'master'):
|
|
try:
|
|
subprocess.check_call(['git', 'rev-parse', branch_name],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
cwd=git_project_path)
|
|
except subprocess.CalledProcessError:
|
|
continue
|
|
return branch_name
|
|
else:
|
|
raise Exception('failed to determine branch')
|
|
|
|
|
|
def setup_git_tree(project_reference, options):
|
|
project_name = get_git_project_name(project_reference)
|
|
git_project_path = get_git_project_path(project_reference)
|
|
if options.branch and options.branch.startswith('wip/'):
|
|
if os.path.exists(git_project_path):
|
|
shutil.rmtree(git_project_path)
|
|
if os.path.exists(git_project_path):
|
|
existing_tree = True
|
|
branch_name = get_git_branch_name(project_reference)
|
|
try:
|
|
subprocess.check_call(['git', 'checkout', '--quiet', branch_name],
|
|
cwd=git_project_path)
|
|
subprocess.check_call(['git', 'reset', '--hard', 'origin/%s' % branch_name],
|
|
cwd=git_project_path)
|
|
except subprocess.CalledProcessError as e:
|
|
print(e, file=sys.stderr)
|
|
shutil.rmtree(git_project_path)
|
|
return setup_git_tree(project_reference, options)
|
|
else:
|
|
existing_tree = False
|
|
os.chdir(settings.GIT_PATH)
|
|
if project_reference.startswith('/'):
|
|
call("git clone %s" % project_reference)
|
|
else:
|
|
parsed = urllib.parse.urlparse(project_reference)
|
|
if not parsed.netloc:
|
|
project_url = urllib.parse.urljoin(settings.GIT_REPOSITORY_URL, project_name) + '.git'
|
|
else:
|
|
project_url = project_reference
|
|
call("git clone %s" % project_url)
|
|
branch_name = get_git_branch_name(project_reference)
|
|
|
|
if not options.branch:
|
|
options.branch = branch_name
|
|
|
|
os.chdir(git_project_path)
|
|
try:
|
|
subprocess.check_call(['git', 'checkout', '--quiet', branch_name])
|
|
subprocess.check_call(['git', 'pull'])
|
|
subprocess.check_call(['git', 'submodule', 'init'])
|
|
subprocess.check_call(['git', 'submodule', 'update'])
|
|
except subprocess.CalledProcessError as e:
|
|
if existing_tree:
|
|
print(e, file=sys.stderr)
|
|
shutil.rmtree(git_project_path)
|
|
return setup_git_tree(project_reference, options)
|
|
raise
|
|
|
|
|
|
def main():
|
|
options, args = parse_cmdline()
|
|
for method in options.cleaning:
|
|
clean(method)
|
|
|
|
if options.cleaning:
|
|
sys.exit(0)
|
|
|
|
init()
|
|
project_reference = args[0]
|
|
git_project_path = get_git_project_path(project_reference)
|
|
atexit.register(clean_git_on_exit, git_project_path)
|
|
if options.branch and options.branch.startswith('origin/'):
|
|
# normalize without origin/
|
|
options.branch = options.branch[len('origin/'):]
|
|
existing_tree = os.path.exists(git_project_path)
|
|
if existing_tree:
|
|
os.chdir(git_project_path)
|
|
last_tag = output("git describe --abbrev=0 --tags --match=v*",
|
|
exit_on_error=False)
|
|
if last_tag:
|
|
last_tag = last_tag[1:-1]
|
|
else:
|
|
last_tag = "0.0"
|
|
else:
|
|
last_tag = "0.0"
|
|
setup_git_tree(project_reference, options)
|
|
project = get_project_infos(git_project_path, options)
|
|
|
|
if not os.path.exists(project['lock_path']):
|
|
os.mkdir(project['lock_path'], 0o755)
|
|
|
|
# compare revision between last build and now to determine if something is really new
|
|
new = 1
|
|
current_revision = output("git rev-parse HEAD", True).strip()
|
|
branch_name = get_git_branch_name(project_reference)
|
|
last_branch_revision_file_path = os.path.join(
|
|
project['lock_path'],
|
|
"%s_%s.last_revision" % (
|
|
project['name'],
|
|
branch_name.replace('/', '_')))
|
|
try:
|
|
with open(last_branch_revision_file_path) as f:
|
|
last_branch_revision = f.read().strip()
|
|
except IOError:
|
|
pass
|
|
else:
|
|
if current_revision == last_branch_revision:
|
|
new = 0
|
|
|
|
if options.force and not new:
|
|
print("+ Warning force a new build")
|
|
new = 1
|
|
|
|
for dist in options.distrib:
|
|
os.chdir(git_project_path)
|
|
call("git checkout --quiet %s" % branch_name)
|
|
package = prepare_build(dist, project, options, new)
|
|
if package:
|
|
for arch in options.architectures:
|
|
build_project(dist, arch, project, package, new)
|
|
send_packages(dist, arch, project, package, last_tag)
|
|
print("+ Add a build file to lock new build for %s" % dist)
|
|
touch(os.path.join(project['lock_path'],
|
|
"%s_%s_%s_%s.build" % (project['name'],
|
|
package['version'],
|
|
package['repository'],
|
|
branch_name.replace('/', '_'))
|
|
))
|
|
|
|
last_version_file = os.path.join(project['lock_path'],
|
|
"%s_%s_%s.last_version" % (project['name'],
|
|
package['repository'],
|
|
branch_name.replace('/', '_'))
|
|
)
|
|
with open(last_version_file, 'w+') as f:
|
|
f.write(package['version'])
|
|
|
|
# keep current revision for next build
|
|
with open(last_branch_revision_file_path, 'w+') as f:
|
|
f.write(current_revision)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|