commit 48641cd9c155a54e7beb69895ac486ac5e7177b7 Author: Christophe Siraut Date: Thu Jun 27 13:14:26 2019 +0200 misc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..35ec4ed --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__ +*.pyc +*~ +*.swp diff --git a/ansible/.gitignore b/ansible/.gitignore new file mode 100644 index 0000000..a8b42eb --- /dev/null +++ b/ansible/.gitignore @@ -0,0 +1 @@ +*.retry diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..b889b2a --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,6 @@ +[defaults] + +inventory_plugins = library + +[inventory] +enable_plugins = inventory diff --git a/ansible/library/__init__.py b/ansible/library/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ansible/library/inventory.py b/ansible/library/inventory.py new file mode 100755 index 0000000..bfe5353 --- /dev/null +++ b/ansible/library/inventory.py @@ -0,0 +1,58 @@ +#!/usr/bin/python3 +import os +import sys + +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable + +p = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, p) +from data import serverlist + +class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): + + NAME = 'entrouvert' # used internally by Ansible, it should match the file name but not required + + def verify_file(self, path): + return True + + def parse(self, inventory, loader, path, cache=False): + + #self.loader = loader + #self.inventory = inventory + #self.templar = Templar(loader=loader) + + # call base method to ensure properties are available for use with other helper methods + super(InventoryModule, self).parse(inventory, loader, path, cache) + + + """ + # this method will parse 'common format' inventory sources and + # update any options declared in DOCUMENTATION as needed + config = self._read_config_data(path) + + # if NOT using _read_config_data you should call set_options directly, + # to process any defined configuration for this plugin, + # if you don't define any options you can skip + #self.set_options() + + # example consuming options from inventory source + mysession = apilib.session(user=self.get_option('api_user'), + password=self.get_option('api_pass'), + server=self.get_option('api_server') + ) + """ + + + """ + # make requests to get data to feed into inventory + mydata = mysession.getitall() + + #parse data and create inventory objects: + for colo in mydata: + for server in mydata[colo]['servers']: + self.inventory.add_host(server['name']) + self.inventory.set_variable(server['name'], 'ansible_host', server['external_ip']) + """ + + for s in serverlist: + self.inventory.add_host(s) diff --git a/ansible/playbooks/check_network.yaml b/ansible/playbooks/check_network.yaml new file mode 100644 index 0000000..95cffbf --- /dev/null +++ b/ansible/playbooks/check_network.yaml @@ -0,0 +1,8 @@ +--- +- hosts: node2.dev.saas.entrouvert.org + tasks: + - assert: + that: + - "inventory_hostname == ansible_fqdn" + - "ansible_eno1.macaddress == 'ac:1f:6b:44:55:84'" + - "ansible_default_ipv4 == 'ansible_default_ipv4'" diff --git a/entrouvert-remotes/.gitignore b/entrouvert-remotes/.gitignore new file mode 100644 index 0000000..53752db --- /dev/null +++ b/entrouvert-remotes/.gitignore @@ -0,0 +1 @@ +output diff --git a/entrouvert-remotes/Makefile b/entrouvert-remotes/Makefile new file mode 100644 index 0000000..0c5d706 --- /dev/null +++ b/entrouvert-remotes/Makefile @@ -0,0 +1,43 @@ +.PHONY: clean all collect + +collect: clean output output/dev.saas.entrouvert.org output/test.saas.entrouvert.org output/prod.saas.entrouvert.org + echo 'This files where generated by entrouvert-remotes (maybe triggered by eotasks)' > output/README + echo done + +output/dev.saas.entrouvert.org: + mkdir $@ + scp collect_connections.py authentic.node1.dev.saas.entrouvert.org:/tmp/ + scp collect_connections.py passerelle.node1.dev.saas.entrouvert.org:/tmp/ + ssh authentic.node1.dev.saas.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic + ssh passerelle.node1.dev.saas.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle + # TODO: proper wcs adresses collection + # scp collect_connections.py wcs.node1.dev.saas.entrouvert.org:/tmp/ + # ssh wcs.node1.dev.saas.entrouvert.org sudo -u wcs wcs-manage runscript /tmp/collect_connections.py > output/dev_entries/wcs.node1.dev.saas.entrouvert.org + +output/test.saas.entrouvert.org: + mkdir $@ + scp collect_connections.py authentic.node1.test.saas.entrouvert.org:/tmp/ + scp collect_connections.py passerelle.node1.test.saas.entrouvert.org:/tmp/ + ssh authentic.node1.test.saas.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic + ssh passerelle.node1.test.saas.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle + # TODO: proper wcs adresses collection + +output/prod.saas.entrouvert.org: + mkdir $@ + scp collect_connections.py authentic.rbx.prod.entrouvert.org:/tmp/ + scp collect_connections.py passerelle.rbx.prod.entrouvert.org:/tmp/ + ssh authentic.rbx.prod.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic + ssh passerelle.rbx.prod.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle + +output/chicon.entrouvert.org: + mkdir $@ + scp collect_connections.py authentic.entrouvert.org:/tmp/ + scp collect_connections.py passerelle.entrouvert.org:/tmp/ + ssh authentic.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic + ssh passerelle.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle + +output: + mkdir output + +clean: + -test -d output && rm -r output diff --git a/entrouvert-remotes/collect_connections.py b/entrouvert-remotes/collect_connections.py new file mode 100755 index 0000000..632e06c --- /dev/null +++ b/entrouvert-remotes/collect_connections.py @@ -0,0 +1,100 @@ +# hobo - portal to configure and deploy applications +# Copyright (C) 2016 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import json +import os + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.db import connection + +from hobo.multitenant.middleware import TenantMiddleware + + +class Command(BaseCommand): + + def handle(self, verbosity=0, *args, **options): + entries = {} + for tenant in TenantMiddleware.get_tenants(): + if verbosity > 0: + print(u'* Running command on tenant %s' % tenant.domain_url) + connection.set_tenant(tenant) + key = tenant.domain_url + entries[key] = [] + for entry in self.tenant_entries(): + entries[key].append(entry) + + # hack (see TODO further on) + if os.path.isfile('/usr/bin/passerelle-manage'): + entries['wcs'] = [] + for entry in self.wcs_remotes(): + entries['wcs'].append(entry) + + print(json.dumps(entries)) + + def tenant_entries(self, *args, **options): + entries = [] + if os.path.isfile('/usr/bin/passerelle-manage'): + entries.extend(self.passerelle_remotes()) + ## TODO: proper wcs entries collection + # if os.path.isfile('/usr/bin/wcs-manage'): + # entries.extend(self.wcs_remotes()) + if os.path.isfile('/usr/bin/authentic2-multitenant-manage'): + entries.extend(self.authentic_remotes()) + return entries + + def authentic_remotes(self): + for s in settings.LDAP_AUTH_SETTINGS: + if isinstance(s['url'], list): + for entry in s['url']: + yield entry + else: + yield(s['url']) + + def passerelle_remotes(self): + from passerelle.views import get_all_apps + + for app in get_all_apps(): + for connector in app.objects.all(): + for attr in connector.__dict__.keys(): + if attr.endswith('url'): + url = getattr(connector, attr, None) + if url: + yield(url) + + def wcs_remotes(self): + """ + os.system('grep --color https:// */{formdefs,workflows,datasources,wscalls}/* | grep -v href= | grep -v src= | grep -o "https://.*" | sort | uniq') + """ + + ls = '''https://api.wavebricks.com/#/register?firstname=[form_var_firstname]&lastname=[form_var_lastname]&email=[form_var_courriel]&type_utilisateur=@IF&referrer=EMS2017 + https://sp01.vincennes.fr/webhook-comptecitoyen/ + https://bacasable.atreal.fr/~demowsdia/openads/services/rest_entry.php/dia/ + https://secure2.grandnancy.eu/grcnancywebservices/symfony/web/app_dev.php/eo/test + https://si.metzmetropole.fr/index.php + https://si.metzmetropole.fr/index.php?appid=252&q=gare + https://si-ws-dev.metzmetropole.fr/adp/putAnomaliePublik.php + https://si-ws.metzmetropole.fr/adp/____putAnomaliePublik.php + https://si-ws.metzmetropole.fr/adp/putAnomaliePublik.php + https://si-ws.metzmetropole.fr/sig/getquartierbyadress.php?q=3%20rue%20des%20eglantiers + https://srv-geoserver.mairie-metz.fr/public/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=public:fdp_voi_nume&maxFeatures=50&outputFormat=application/json + https://srv-siapps.mairie-metz.fr/index.php''' + + for l in ls.splitlines(): + yield l.strip() + + +command = Command() +command.handle() diff --git a/entrouvert-remotes/test_connections.py b/entrouvert-remotes/test_connections.py new file mode 100755 index 0000000..3ffc22b --- /dev/null +++ b/entrouvert-remotes/test_connections.py @@ -0,0 +1,101 @@ +#!/usr/bin/python2 +# hobo - portal to configure and deploy applications +# Copyright (C) 2016 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import errno +import json +import socket +import sys +from urlparse import urlparse + +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + + def ad_argument(self, parser): + parser.add_argument('json_filename', type=str) + + def handle(self, json_filename, verbosity=0, *args, **kwargs): + cache = {} + if json_filename == '-': + entries = json.load(sys.stdin) + else: + entries = json.load(open(json_filename)) + for tenant, addresses in entries.iteritems(): + for entry in addresses: + if entry in cache.keys(): + # return cache.get(entry) + continue + if verbosity > 0: + print("testing %s %s" % (tenant, entry)) + host, port, scheme = self.parse_entry(entry) + rs = self.netcat(host, port) + cache[entry] = rs + if not rs: + print("Failed: %s (%s:%s %s) %s" % (tenant, host, port, scheme, rs)) + + def netcat(self, host, port, content=''): + def connect(host, port, content): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(3) + s.connect((host, int(port))) + s.sendall(content.encode()) + s.shutdown(socket.SHUT_WR) + while True: + try: + data = s.recv(4096) + if not data: + break + # print(repr(data)) + except socket.error as e: + if e.errno != errno.ECONNRESET: + raise + s.close() + try: + connect(host, port, content) + return True + # python3 + # except socket.ConnectionResetError: + # return True + # except socket.ConnectionRefusedError: + # return False + except socket.timeout: + return False + except socket.gaierror: + return False + except socket.error: + return False + + def parse_entry(self, entry): + parsed = urlparse(entry) + try: + host, port = parsed.netloc.split(':') + port = int(port) + except ValueError: + host, port = parsed.netloc, 443 + if parsed.scheme == 'ldap': + port = 389 + if parsed.scheme == 'ldaps': + port = 636 + if parsed.scheme == 'http': + port = 80 + if '@' in host: + host = host.split('@')[-1] + return host, port, parsed.scheme + + +command = Command() +command.handle(sys.argv[1]) diff --git a/eoovh/.gitignore b/eoovh/.gitignore new file mode 100644 index 0000000..6c464ad --- /dev/null +++ b/eoovh/.gitignore @@ -0,0 +1,2 @@ +ovh.conf +output diff --git a/eoovh/gen_consumer_key.py b/eoovh/gen_consumer_key.py new file mode 100755 index 0000000..945cb56 --- /dev/null +++ b/eoovh/gen_consumer_key.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + + +import ovh + +# create a client using configuration +client = ovh.Client() + +# Request RO, /me API access +ck = client.new_consumer_key_request() +ck.add_rules(ovh.API_READ_ONLY, "/me/*") +ck.add_rules(ovh.API_READ_ONLY, "/dedicated/*") + +# Request token +validation = ck.request() +print(validation) +print("Please visit %s to authenticate" % validation['validationUrl']) +input("and press Enter to continue...") + +# Print nice welcome message +print("Welcome", client.get('/me')['firstname']) +print("Btw, your 'consumerKey' is '%s'" % validation['consumerKey']) diff --git a/eoovh/ovh_inventory.py b/eoovh/ovh_inventory.py new file mode 100755 index 0000000..f875893 --- /dev/null +++ b/eoovh/ovh_inventory.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import ovh +import socket +import yaml + +client = ovh.Client() + +ovh_servers = {} +servers = {k: {} for k in client.get('/dedicated/server')} +for s, _ in servers.items(): + name, _, addresslist = socket.gethostbyaddr(socket.gethostbyname(s)) + servers[s]['name'] = name + servers[s]['ovh_name'] = s + servers[s]['address'] = addresslist[0] + servers[s]['base'] = client.get('/dedicated/server/%s' % s) + servers[s]['ips'] = client.get('/dedicated/server/%s/ips' % s) + servers[s]['options'] = client.get('/dedicated/server/%s/option' % s) + servers[s]['hardware'] = client.get('/dedicated/server/%s/specifications/hardware' % s) + servers[s]['ip'] = client.get('/dedicated/server/%s/specifications/ip' % s) + servers[s]['network'] = client.get('/dedicated/server/%s/specifications/network' % s) + servers[s]['interfaces'] = {i: {} for i in client.get('/dedicated/server/%s/networkInterfaceController' % s)} + for m, _ in servers[s]['interfaces'].items(): + servers[s]['interfaces'][m]['type'] = client.get('/dedicated/server/%s/networkInterfaceController/%s' % (s, m)) + if servers[s]['interfaces'][m]['type']['linkType'] == 'public': + if 'macaddress' in servers[s].keys(): + print('ERROR: %s has multiple public network interfaces' % s) + servers[s]['macaddress'] = m + + if servers[s]['name'] != servers[s]['base']['reverse'] or '%s.' % servers[s]['name'] != servers[s]['base']['reverse']: + print('ERROR: %s reverse DNS is wrong' % s) + if servers[s]['address'] != servers[s]['base']['ip'] or servers[s]['address'] != servers[s]['network']['routing']['ipv4']['ip']: + print('ERROR: %s ip address is wrong' % s) + + # reindex according to entrouvert key + ovh_servers[servers[s]['name']] = servers[s] + +print(yaml.dump(ovh_servers)) diff --git a/myrepos/mrconfig b/myrepos/mrconfig new file mode 100644 index 0000000..5f7fc9b --- /dev/null +++ b/myrepos/mrconfig @@ -0,0 +1,78 @@ +[DEFAULT] +update = git pull --rebase + +[auquotidien] +checkout = git@git.entrouvert.org:auquotidien.git + +[authentic] +checkout = git@git.entrouvert.org:authentic.git + +[authentic2-auth-fc] +checkout = git clone git@git.entrouvert.org:authentic2-auth-fc.git + +[authentic2-auth-fedict] +checkout = git clone git@git.entrouvert.org:authentic2-auth-fedict.git + +[authentic2-auth-kerberos] +checkout = git clone git@git.entrouvert.org:authentic2-auth-kerberos.git + +[authentic2-auth-msp] +checkout = git clone git@git.entrouvert.org:authentic2-auth-msp.git + +[authentic2-auth-saml2] +checkout = git clone git@git.entrouvert.org:authentic2-auth-saml2.git + +[bidon] +checkout = git@git.entrouvert.org:bidon.git + +[bijoe] +checkout = git@git.entrouvert.org:bijoe.git + +[chrono] +checkout = git@git.entrouvert.org:chrono.git + +[combo] +checkout = git@git.entrouvert.org:combo.git + +[corbo] +checkout = git@git.entrouvert.org:corbo.git + +[django-mellon] +checkout = git@git.entrouvert.org:django-mellon.git + +[django-tenant-schemas] +checkout = git@git.entrouvert.org:django-tenant-schemas.git + +[fargo] +checkout = git@git.entrouvert.org:fargo.git + +[gadjo] +checkout = git@git.entrouvert.org:gadjo.git + +[hobo] +checkout = git@git.entrouvert.org:hobo.git + +[lasso] +checkout = git@git.entrouvert.org:lasso.git + +[lingo] +checkout = git@git.entrouvert.org:lingo.git + +[mandayejs] +checkout = git@git.entrouvert.org:mandayejs.git + +[passerelle] +checkout = git@git.entrouvert.org:passerelle.git + +[publik-base-theme] +checkout = git@git.entrouvert.org:publik-base-theme.git + +[wcs] +checkout = git@git.entrouvert.org:wcs.git + +[wcs-olap] +checkout = git@git.entrouvert.org:wcs-olap.git + +[welco] +checkout = git@git.entrouvert.org:welco.git + diff --git a/publik-builder/Jenkinsfile b/publik-builder/Jenkinsfile new file mode 100644 index 0000000..be21814 --- /dev/null +++ b/publik-builder/Jenkinsfile @@ -0,0 +1,32 @@ +@Library('eo-jenkins-lib@master') import eo.Utils + +pipeline { + agent any + stages { + stage('Unit Tests') { + steps { + sh 'echo "no test implemented"' + } + } + stage('Packaging') { + steps { + script { + if (env.JOB_NAME == 'publik-builder' && env.GIT_BRANCH == 'origin/master') { + sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder publik-builder' + } + } + } + } + } + post { + always { + script { + utils = new Utils() + utils.mail_notify(currentBuild, env, 'admin+adminsys@entrouvert.com') + } + } + success { + cleanWs() + } + } +} diff --git a/publik-builder/Makefile b/publik-builder/Makefile new file mode 100644 index 0000000..29d6db5 --- /dev/null +++ b/publik-builder/Makefile @@ -0,0 +1,31 @@ +.PHONY: clean name version name fullname + +NAME=$(shell basename $(CURDIR)) +VERSION=`git describe | sed 's/^debian\///' | sed 's/v//'` +DIST_FILES = Makefile \ + publik-builder \ + flavors + +version: + @(echo $(VERSION)) + +name: + @(echo $(NAME)) + +fullname: + @(echo $(NAME)-$(VERSION)) + +clean: + rm -rf sdist + +dist-bzip2: dist + cd sdist && tar cfj ../sdist/$(NAME)-$(VERSION).tar.bz2 $(NAME)-$(VERSION) + +dist: clean + -mkdir sdist + rm -rf sdist/$(NAME)-$(VERSION) + mkdir -p sdist/$(NAME)-$(VERSION) + for i in $(DIST_FILES); do \ + cp -R "$$i" sdist/$(NAME)-$(VERSION); \ + done + diff --git a/publik-builder/README b/publik-builder/README new file mode 100644 index 0000000..5fac4f3 --- /dev/null +++ b/publik-builder/README @@ -0,0 +1,11 @@ +publik-builder +============= + +Publik images builder. + +Dependencies +------------ +* debootstrap: Note you can run debootstrap from its source tree without installing it: + + git clone https://salsa.debian.org/installer-team/debootstrap.git + alias debootstrap=$(pwd)/debootstrap/deboostrap diff --git a/publik-builder/debian/changelog b/publik-builder/debian/changelog new file mode 100644 index 0000000..1a428bb --- /dev/null +++ b/publik-builder/debian/changelog @@ -0,0 +1,5 @@ +publik-builder (0.1) stretch; urgency=medium + + * Initial release + + -- Christophe Siraut Tue, 04 Sep 2018 17:15:19 +0200 diff --git a/publik-builder/debian/compat b/publik-builder/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/publik-builder/debian/compat @@ -0,0 +1 @@ +9 diff --git a/publik-builder/debian/control b/publik-builder/debian/control new file mode 100644 index 0000000..876318f --- /dev/null +++ b/publik-builder/debian/control @@ -0,0 +1,11 @@ +Source: publik-builder +Maintainer: Christophe Siraut +Section: python +Priority: optional +Build-Depends: debhelper (>= 9) +Standards-Version: 4.1.3 + +Package: publik-builder +Architecture: all +Depends: ${misc:Depends}, python3:any +Description: Publik images builder diff --git a/publik-builder/debian/copyright b/publik-builder/debian/copyright new file mode 100644 index 0000000..f2ddcbc --- /dev/null +++ b/publik-builder/debian/copyright @@ -0,0 +1,25 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: publik-common +Source: https://www.entrouvert.com + +Files: * +Copyright: 2018 Christophe Siraut + 2018 Entrouvert Admins +License: GPL-3.0+ + +License: GPL-3.0+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian systems, the complete text of the GNU General + Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". diff --git a/publik-builder/debian/gbp.conf b/publik-builder/debian/gbp.conf new file mode 100644 index 0000000..12dcf99 --- /dev/null +++ b/publik-builder/debian/gbp.conf @@ -0,0 +1,7 @@ +[DEFAULT] +cleaner = fakeroot debian/rules clean +builder = debuild -us -uc + +[buildpackage] +export-dir = ../build-area + diff --git a/publik-builder/debian/install b/publik-builder/debian/install new file mode 100644 index 0000000..08a7d5d --- /dev/null +++ b/publik-builder/debian/install @@ -0,0 +1,2 @@ +publik-builder /usr/bin +flavors /var/lib/publik-builder diff --git a/publik-builder/debian/rules b/publik-builder/debian/rules new file mode 100755 index 0000000..301c910 --- /dev/null +++ b/publik-builder/debian/rules @@ -0,0 +1,5 @@ +#!/usr/bin/make -f +export PYBUILD_NAME=publik-common + +%: + dh $@ diff --git a/publik-builder/debian/source/format b/publik-builder/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/publik-builder/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/publik-builder/flavors/server b/publik-builder/flavors/server new file mode 100755 index 0000000..f0fa77e --- /dev/null +++ b/publik-builder/flavors/server @@ -0,0 +1,24 @@ +#!/bin/bash +# publik-bootstrap: start a Publik server +# 2018 Entr'ouvert +set -e + +apt-get update +apt-get install -yqq apt-transport-https ca-certificates wget lsb-release +wget -O - https://deb.entrouvert.org/entrouvert.gpg | apt-key add - +echo "deb http://deb.entrouvert.org/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/entrouvert.list +apt-get update +apt-get install -y publik-common + +publik-create-users +apt install -y postgresql-client bsd-mailx zip # libreoffice +cp /etc/publik/publik.conf.example /etc/publik/publik.conf +public-create-databases + +#apt install hobo hobo-agent +#systemctl restart supervisor +#apt install -y authentic2-multitenant python-authentic2-auth-fc +#apt install -y combo +#apt install -y passerelle +#apt install -y bijoe fargo welco chrono +#apt install -y wcs wcs-au-quotidien diff --git a/publik-builder/publik-builder b/publik-builder/publik-builder new file mode 100755 index 0000000..065e9d2 --- /dev/null +++ b/publik-builder/publik-builder @@ -0,0 +1,91 @@ +#!/usr/bin/python3 +# Publik image builder +# 2018 Entr'ouvert +from contextlib import contextmanager +import argparse +import os +import shutil +import subprocess +import uuid + +parser = argparse.ArgumentParser() +parser.add_argument('--release', type=str, default='stretch') +parser.add_argument('--prefix', type=str, default='publik') +parser.add_argument('--arch', type=str, default='amd64') +parser.add_argument('--clean', action='store_true') +parser.add_argument('--flavor', choices=['server', 'devinst', 'vanilla'], default='server') +parser.add_argument('--form', choices=['tar', 'xz', 'folder'], default='xz') +args = parser.parse_args() + +options = vars(args) +options['mirror'] = 'http://httpredir.debian.org/debian/' +options['tag'] = '{release}-{arch}'.format(**options) +options['cachedir'] = '/var/cache/publik-builder' +options['output_dir'] = os.getcwd() +options['output'] = '{output_dir}/{prefix}-{tag}'.format(**options) + +if os.getuid(): + raise(Exception('please with sudo (or as root)')) + + +def run(cmd): + subprocess.run(cmd, shell=True, check=True) + + +@contextmanager +def mount_chroot(path): + run('mount -t proc /proc {}/proc'.format(path)) + run('mount --bind /sys {}/sys'.format(path)) + run('mount --bind /dev {}/dev'.format(path)) + run('mount --bind /tmp {}/tmp'.format(path)) + try: + yield + finally: + run('umount {}/tmp'.format(path)) + run('umount {}/dev'.format(path)) + run('umount {}/sys'.format(path)) + run('umount {}/proc'.format(path)) + + +rootfs = os.path.join(options['cachedir'], options['tag']) +if args.clean and os.path.isdir(rootfs): + shutil.rmtree(rootfs) +if not os.path.isdir(rootfs): + if not os.path.isdir(options['cachedir']): + os.makedirs(options['cachedir']) + options['rootfs'] = rootfs + run('debootstrap --foreign {release} {rootfs}'.format(**options)) + run('DEBOOTSTRAP_DIR={rootfs}/debootstrap debootstrap --second-stage --arch {arch} --second-stage-target {rootfs} {mirror}'.format(**options)) + + +workdir = os.path.join('/tmp/publik-builder', str(uuid.uuid4())) +if not os.path.isdir('/tmp/publik-builder'): + os.makedirs('/tmp/publik-builder') +options['workdir'] = workdir +print('copying base to workdir {}'.format(workdir)) +run('cp -r {} {}'.format(rootfs, workdir)) + +if args.flavor != 'none': + if args.flavor == 'devinst': + raise(Exception('not implemented')) + script = '/var/lib/publik-builder/flavors/%s' % args.flavor + if not os.path.isfile(script): + script = './flavors/%s' % args.flavor + if not os.path.isfile: + raise(Exception('cannot find {}'.format(script))) + + rscript = '{}/usr/local/bin/publik-bootstrap'.format(workdir) + shutil.copyfile(script, rscript) + run('chmod +x {}'.format(rscript)) + with mount_chroot(workdir): + run('chroot {} publik-bootstrap'.format(workdir)) + run('chroot {} apt-get clean'.format(workdir)) + + +if args.form == 'tar': + run('cd {workdir} && tar cf {output}.tar ./'.format(**options)) +elif args.form == 'xz': + run('cd {workdir} && tar cfJ {output}.tar.xz ./'.format(**options)) +elif args.form == 'folder': + folder = '{output_dir}/{prefix}-{tag}'.format(**options) + shutil.move(workdir, folder) diff --git a/sftp_syncfiles b/sftp_syncfiles new file mode 100755 index 0000000..35569fd --- /dev/null +++ b/sftp_syncfiles @@ -0,0 +1,100 @@ +#!/usr/bin/python3 +import argparse +import cryptography +import glob +import os +import paramiko +import warnings +warnings.simplefilter("ignore", cryptography.utils.CryptographyDeprecationWarning) + + +def connection(hostname, username, password=None, keyfile=None): + if not keyfile: + keyfile = os.path.expanduser('~/.ssh/id_rsa') + key = paramiko.RSAKey.from_private_key_file(keyfile) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.connect(hostname, username='csiraut', password=password, pkey=key) + return paramiko.SFTPClient.from_transport(client.get_transport()) + + +def validate_local_path(path): + if not os.path.exists(path) or not os.path.isdir(path): + raise(Exception('Directory %s does not exist' % path)) + + +def validate_remote_path(sftp, path): + lstat = sftp.lstat(path) + mode = oct(lstat.st_mode)[5] + if mode not in ['5', '7']: + raise(Exception('Remote path is not a folder or not writable')) + + +def parse_location(loc): + hostname, username, path = None, None, loc + if '@' in loc and ':' in loc: + username, remaning = loc.split('@') + hostname, path = remaning.split(':') + return hostname, username, path + + +def get(hostname, username, remotepath, localpath, password=None, keyfile=None): + validate_local_path(localpath) + sftp = connection(hostname, username, password=password, keyfile=keyfile) + validate_remote_path(sftp, remotepath) + + dirlist = sftp.listdir_attr(remotepath) + filelist = [x.filename for x in dirlist if oct(x.st_mode)[5] not in ('5', '7')] + for f in filelist: + if args.verbose: + print('syncing %s to %s' % (remotepath, localpath)) + remotefilepath = '%s/%s' % (remotepath, f) + localfilepath = '%s/%s' % (localpath, f) + sftp.get(remotefilepath, localfilepath) + if args.delete: + if args.verbose: + print('removing %s' % remotefilepath) + sftp.remove(remotefilepath) + + +def put(hostname, username, remotepath, localpath, password=None, keyfile=None): + validate_local_path(localpath) + sftp = connection(hostname, username, password=password, keyfile=keyfile) + validate_remote_path(sftp, remotepath) + + filelist = [f for f in glob.glob('%s/*' % localpath) if not os.path.isdir(f)] + for f in filelist: + remotefilepath = '%s/%s' % (remotepath, f.split('/')[-1]) + if args.verbose: + print('syncing %s to %s' % (f, remotefilepath)) + sftp.put(f, remotefilepath) + if args.delete: + if args.verbose: + print('removing %s' % f) + os.remove(f) + + +def main(args): + orig = parse_location(args.origin) + dest = parse_location(args.destination) + if (orig[0] and dest[0]) or (not orig[1] and not dest[1]) or \ + ((orig[1] and '@' not in orig[1]) and (dest[1] and '@' not in dest[1])): + raise(Exception('Please provide one remote location like this: user@server:/path/to/direction')) + if orig[1]: + action, hostname, username, remotepath, localpath = 'get', orig[0], orig[1], orig[2], dest[2] + else: + action, hostname, username, remotepath, localpath = 'put', dest[0], dest[1], dest[2], orig[2] + + globals()[action](hostname, username, remotepath, localpath, args.password, args.keyfile) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='sftp.py — secure file transfer program') + parser.add_argument('origin', help='directory holding files to sync') + parser.add_argument('destination', help='target directory') + parser.add_argument('--keyfile') + parser.add_argument('--password') + parser.add_argument('--delete', action='store_true', help='remove source file(s) when transfered') + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + main(args)