summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.gitignore4
-rw-r--r--ansible/.gitignore1
-rw-r--r--ansible/ansible.cfg6
-rw-r--r--ansible/library/__init__.py0
-rwxr-xr-xansible/library/inventory.py58
-rw-r--r--ansible/playbooks/check_network.yaml8
-rw-r--r--entrouvert-remotes/.gitignore1
-rw-r--r--entrouvert-remotes/Makefile43
-rwxr-xr-xentrouvert-remotes/collect_connections.py100
-rwxr-xr-xentrouvert-remotes/test_connections.py101
-rw-r--r--eoovh/.gitignore2
-rwxr-xr-xeoovh/gen_consumer_key.py22
-rwxr-xr-xeoovh/ovh_inventory.py38
-rw-r--r--myrepos/mrconfig78
-rw-r--r--publik-builder/Jenkinsfile32
-rw-r--r--publik-builder/Makefile31
-rw-r--r--publik-builder/README11
-rw-r--r--publik-builder/debian/changelog5
-rw-r--r--publik-builder/debian/compat1
-rw-r--r--publik-builder/debian/control11
-rw-r--r--publik-builder/debian/copyright25
-rw-r--r--publik-builder/debian/gbp.conf7
-rw-r--r--publik-builder/debian/install2
-rwxr-xr-xpublik-builder/debian/rules5
-rw-r--r--publik-builder/debian/source/format1
-rwxr-xr-xpublik-builder/flavors/server24
-rwxr-xr-xpublik-builder/publik-builder91
-rwxr-xr-xsftp_syncfiles100
28 files changed, 808 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..35ec4ed
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+__pycache__
+*.pyc
+*~
+*.swp
diff --git a/ansible/.gitignore b/ansible/.gitignore
new file mode 100644
index 0000000..a8b42eb
--- /dev/null
+++ b/ansible/.gitignore
@@ -0,0 +1 @@
+*.retry
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
new file mode 100644
index 0000000..b889b2a
--- /dev/null
+++ b/ansible/ansible.cfg
@@ -0,0 +1,6 @@
+[defaults]
+
+inventory_plugins = library
+
+[inventory]
+enable_plugins = inventory
diff --git a/ansible/library/__init__.py b/ansible/library/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ansible/library/__init__.py
diff --git a/ansible/library/inventory.py b/ansible/library/inventory.py
new file mode 100755
index 0000000..bfe5353
--- /dev/null
+++ b/ansible/library/inventory.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python3
+import os
+import sys
+
+from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
+
+p = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, p)
+from data import serverlist
+
+class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
+
+ NAME = 'entrouvert' # used internally by Ansible, it should match the file name but not required
+
+ def verify_file(self, path):
+ return True
+
+ def parse(self, inventory, loader, path, cache=False):
+
+ #self.loader = loader
+ #self.inventory = inventory
+ #self.templar = Templar(loader=loader)
+
+ # call base method to ensure properties are available for use with other helper methods
+ super(InventoryModule, self).parse(inventory, loader, path, cache)
+
+
+ """
+ # this method will parse 'common format' inventory sources and
+ # update any options declared in DOCUMENTATION as needed
+ config = self._read_config_data(path)
+
+ # if NOT using _read_config_data you should call set_options directly,
+ # to process any defined configuration for this plugin,
+ # if you don't define any options you can skip
+ #self.set_options()
+
+ # example consuming options from inventory source
+ mysession = apilib.session(user=self.get_option('api_user'),
+ password=self.get_option('api_pass'),
+ server=self.get_option('api_server')
+ )
+ """
+
+
+ """
+ # make requests to get data to feed into inventory
+ mydata = mysession.getitall()
+
+ #parse data and create inventory objects:
+ for colo in mydata:
+ for server in mydata[colo]['servers']:
+ self.inventory.add_host(server['name'])
+ self.inventory.set_variable(server['name'], 'ansible_host', server['external_ip'])
+ """
+
+ for s in serverlist:
+ self.inventory.add_host(s)
diff --git a/ansible/playbooks/check_network.yaml b/ansible/playbooks/check_network.yaml
new file mode 100644
index 0000000..95cffbf
--- /dev/null
+++ b/ansible/playbooks/check_network.yaml
@@ -0,0 +1,8 @@
+---
+- hosts: node2.dev.saas.entrouvert.org
+ tasks:
+ - assert:
+ that:
+ - "inventory_hostname == ansible_fqdn"
+ - "ansible_eno1.macaddress == 'ac:1f:6b:44:55:84'"
+ - "ansible_default_ipv4 == 'ansible_default_ipv4'"
diff --git a/entrouvert-remotes/.gitignore b/entrouvert-remotes/.gitignore
new file mode 100644
index 0000000..53752db
--- /dev/null
+++ b/entrouvert-remotes/.gitignore
@@ -0,0 +1 @@
+output
diff --git a/entrouvert-remotes/Makefile b/entrouvert-remotes/Makefile
new file mode 100644
index 0000000..0c5d706
--- /dev/null
+++ b/entrouvert-remotes/Makefile
@@ -0,0 +1,43 @@
+.PHONY: clean all collect
+
+collect: clean output output/dev.saas.entrouvert.org output/test.saas.entrouvert.org output/prod.saas.entrouvert.org
+ echo 'This files where generated by entrouvert-remotes (maybe triggered by eotasks)' > output/README
+ echo done
+
+output/dev.saas.entrouvert.org:
+ mkdir $@
+ scp collect_connections.py authentic.node1.dev.saas.entrouvert.org:/tmp/
+ scp collect_connections.py passerelle.node1.dev.saas.entrouvert.org:/tmp/
+ ssh authentic.node1.dev.saas.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic
+ ssh passerelle.node1.dev.saas.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle
+ # TODO: proper wcs adresses collection
+ # scp collect_connections.py wcs.node1.dev.saas.entrouvert.org:/tmp/
+ # ssh wcs.node1.dev.saas.entrouvert.org sudo -u wcs wcs-manage runscript /tmp/collect_connections.py > output/dev_entries/wcs.node1.dev.saas.entrouvert.org
+
+output/test.saas.entrouvert.org:
+ mkdir $@
+ scp collect_connections.py authentic.node1.test.saas.entrouvert.org:/tmp/
+ scp collect_connections.py passerelle.node1.test.saas.entrouvert.org:/tmp/
+ ssh authentic.node1.test.saas.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic
+ ssh passerelle.node1.test.saas.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle
+ # TODO: proper wcs adresses collection
+
+output/prod.saas.entrouvert.org:
+ mkdir $@
+ scp collect_connections.py authentic.rbx.prod.entrouvert.org:/tmp/
+ scp collect_connections.py passerelle.rbx.prod.entrouvert.org:/tmp/
+ ssh authentic.rbx.prod.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic
+ ssh passerelle.rbx.prod.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle
+
+output/chicon.entrouvert.org:
+ mkdir $@
+ scp collect_connections.py authentic.entrouvert.org:/tmp/
+ scp collect_connections.py passerelle.entrouvert.org:/tmp/
+ ssh authentic.entrouvert.org sudo -u authentic-multitenant authentic2-multitenant-manage runscript /tmp/collect_connections.py > $@/authentic
+ ssh passerelle.entrouvert.org sudo -u passerelle passerelle-manage runscript /tmp/collect_connections.py > $@/passerelle
+
+output:
+ mkdir output
+
+clean:
+ -test -d output && rm -r output
diff --git a/entrouvert-remotes/collect_connections.py b/entrouvert-remotes/collect_connections.py
new file mode 100755
index 0000000..632e06c
--- /dev/null
+++ b/entrouvert-remotes/collect_connections.py
@@ -0,0 +1,100 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2016 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import json
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+from hobo.multitenant.middleware import TenantMiddleware
+
+
+class Command(BaseCommand):
+
+ def handle(self, verbosity=0, *args, **options):
+ entries = {}
+ for tenant in TenantMiddleware.get_tenants():
+ if verbosity > 0:
+ print(u'* Running command on tenant %s' % tenant.domain_url)
+ connection.set_tenant(tenant)
+ key = tenant.domain_url
+ entries[key] = []
+ for entry in self.tenant_entries():
+ entries[key].append(entry)
+
+ # hack (see TODO further on)
+ if os.path.isfile('/usr/bin/passerelle-manage'):
+ entries['wcs'] = []
+ for entry in self.wcs_remotes():
+ entries['wcs'].append(entry)
+
+ print(json.dumps(entries))
+
+ def tenant_entries(self, *args, **options):
+ entries = []
+ if os.path.isfile('/usr/bin/passerelle-manage'):
+ entries.extend(self.passerelle_remotes())
+ ## TODO: proper wcs entries collection
+ # if os.path.isfile('/usr/bin/wcs-manage'):
+ # entries.extend(self.wcs_remotes())
+ if os.path.isfile('/usr/bin/authentic2-multitenant-manage'):
+ entries.extend(self.authentic_remotes())
+ return entries
+
+ def authentic_remotes(self):
+ for s in settings.LDAP_AUTH_SETTINGS:
+ if isinstance(s['url'], list):
+ for entry in s['url']:
+ yield entry
+ else:
+ yield(s['url'])
+
+ def passerelle_remotes(self):
+ from passerelle.views import get_all_apps
+
+ for app in get_all_apps():
+ for connector in app.objects.all():
+ for attr in connector.__dict__.keys():
+ if attr.endswith('url'):
+ url = getattr(connector, attr, None)
+ if url:
+ yield(url)
+
+ def wcs_remotes(self):
+ """
+ os.system('grep --color https:// */{formdefs,workflows,datasources,wscalls}/* | grep -v href= | grep -v src= | grep -o "https://.*" | sort | uniq')
+ """
+
+ ls = '''https://api.wavebricks.com/#/register?firstname=[form_var_firstname]&lastname=[form_var_lastname]&email=[form_var_courriel]&type_utilisateur=@IF&referrer=EMS2017
+ https://sp01.vincennes.fr/webhook-comptecitoyen/
+ https://bacasable.atreal.fr/~demowsdia/openads/services/rest_entry.php/dia/
+ https://secure2.grandnancy.eu/grcnancywebservices/symfony/web/app_dev.php/eo/test
+ https://si.metzmetropole.fr/index.php
+ https://si.metzmetropole.fr/index.php?appid=252&q=gare
+ https://si-ws-dev.metzmetropole.fr/adp/putAnomaliePublik.php
+ https://si-ws.metzmetropole.fr/adp/____putAnomaliePublik.php
+ https://si-ws.metzmetropole.fr/adp/putAnomaliePublik.php
+ https://si-ws.metzmetropole.fr/sig/getquartierbyadress.php?q=3%20rue%20des%20eglantiers
+ https://srv-geoserver.mairie-metz.fr/public/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=public:fdp_voi_nume&maxFeatures=50&outputFormat=application/json
+ https://srv-siapps.mairie-metz.fr/index.php'''
+
+ for l in ls.splitlines():
+ yield l.strip()
+
+
+command = Command()
+command.handle()
diff --git a/entrouvert-remotes/test_connections.py b/entrouvert-remotes/test_connections.py
new file mode 100755
index 0000000..3ffc22b
--- /dev/null
+++ b/entrouvert-remotes/test_connections.py
@@ -0,0 +1,101 @@
+#!/usr/bin/python2
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2016 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import errno
+import json
+import socket
+import sys
+from urlparse import urlparse
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+
+ def ad_argument(self, parser):
+ parser.add_argument('json_filename', type=str)
+
+ def handle(self, json_filename, verbosity=0, *args, **kwargs):
+ cache = {}
+ if json_filename == '-':
+ entries = json.load(sys.stdin)
+ else:
+ entries = json.load(open(json_filename))
+ for tenant, addresses in entries.iteritems():
+ for entry in addresses:
+ if entry in cache.keys():
+ # return cache.get(entry)
+ continue
+ if verbosity > 0:
+ print("testing %s %s" % (tenant, entry))
+ host, port, scheme = self.parse_entry(entry)
+ rs = self.netcat(host, port)
+ cache[entry] = rs
+ if not rs:
+ print("Failed: %s (%s:%s %s) %s" % (tenant, host, port, scheme, rs))
+
+ def netcat(self, host, port, content=''):
+ def connect(host, port, content):
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.settimeout(3)
+ s.connect((host, int(port)))
+ s.sendall(content.encode())
+ s.shutdown(socket.SHUT_WR)
+ while True:
+ try:
+ data = s.recv(4096)
+ if not data:
+ break
+ # print(repr(data))
+ except socket.error as e:
+ if e.errno != errno.ECONNRESET:
+ raise
+ s.close()
+ try:
+ connect(host, port, content)
+ return True
+ # python3
+ # except socket.ConnectionResetError:
+ # return True
+ # except socket.ConnectionRefusedError:
+ # return False
+ except socket.timeout:
+ return False
+ except socket.gaierror:
+ return False
+ except socket.error:
+ return False
+
+ def parse_entry(self, entry):
+ parsed = urlparse(entry)
+ try:
+ host, port = parsed.netloc.split(':')
+ port = int(port)
+ except ValueError:
+ host, port = parsed.netloc, 443
+ if parsed.scheme == 'ldap':
+ port = 389
+ if parsed.scheme == 'ldaps':
+ port = 636
+ if parsed.scheme == 'http':
+ port = 80
+ if '@' in host:
+ host = host.split('@')[-1]
+ return host, port, parsed.scheme
+
+
+command = Command()
+command.handle(sys.argv[1])
diff --git a/eoovh/.gitignore b/eoovh/.gitignore
new file mode 100644
index 0000000..6c464ad
--- /dev/null
+++ b/eoovh/.gitignore
@@ -0,0 +1,2 @@
+ovh.conf
+output
diff --git a/eoovh/gen_consumer_key.py b/eoovh/gen_consumer_key.py
new file mode 100755
index 0000000..945cb56
--- /dev/null
+++ b/eoovh/gen_consumer_key.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+
+import ovh
+
+# create a client using configuration
+client = ovh.Client()
+
+# Request RO, /me API access
+ck = client.new_consumer_key_request()
+ck.add_rules(ovh.API_READ_ONLY, "/me/*")
+ck.add_rules(ovh.API_READ_ONLY, "/dedicated/*")
+
+# Request token
+validation = ck.request()
+print(validation)
+print("Please visit %s to authenticate" % validation['validationUrl'])
+input("and press Enter to continue...")
+
+# Print nice welcome message
+print("Welcome", client.get('/me')['firstname'])
+print("Btw, your 'consumerKey' is '%s'" % validation['consumerKey'])
diff --git a/eoovh/ovh_inventory.py b/eoovh/ovh_inventory.py
new file mode 100755
index 0000000..f875893
--- /dev/null
+++ b/eoovh/ovh_inventory.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import ovh
+import socket
+import yaml
+
+client = ovh.Client()
+
+ovh_servers = {}
+servers = {k: {} for k in client.get('/dedicated/server')}
+for s, _ in servers.items():
+ name, _, addresslist = socket.gethostbyaddr(socket.gethostbyname(s))
+ servers[s]['name'] = name
+ servers[s]['ovh_name'] = s
+ servers[s]['address'] = addresslist[0]
+ servers[s]['base'] = client.get('/dedicated/server/%s' % s)
+ servers[s]['ips'] = client.get('/dedicated/server/%s/ips' % s)
+ servers[s]['options'] = client.get('/dedicated/server/%s/option' % s)
+ servers[s]['hardware'] = client.get('/dedicated/server/%s/specifications/hardware' % s)
+ servers[s]['ip'] = client.get('/dedicated/server/%s/specifications/ip' % s)
+ servers[s]['network'] = client.get('/dedicated/server/%s/specifications/network' % s)
+ servers[s]['interfaces'] = {i: {} for i in client.get('/dedicated/server/%s/networkInterfaceController' % s)}
+ for m, _ in servers[s]['interfaces'].items():
+ servers[s]['interfaces'][m]['type'] = client.get('/dedicated/server/%s/networkInterfaceController/%s' % (s, m))
+ if servers[s]['interfaces'][m]['type']['linkType'] == 'public':
+ if 'macaddress' in servers[s].keys():
+ print('ERROR: %s has multiple public network interfaces' % s)
+ servers[s]['macaddress'] = m
+
+ if servers[s]['name'] != servers[s]['base']['reverse'] or '%s.' % servers[s]['name'] != servers[s]['base']['reverse']:
+ print('ERROR: %s reverse DNS is wrong' % s)
+ if servers[s]['address'] != servers[s]['base']['ip'] or servers[s]['address'] != servers[s]['network']['routing']['ipv4']['ip']:
+ print('ERROR: %s ip address is wrong' % s)
+
+ # reindex according to entrouvert key
+ ovh_servers[servers[s]['name']] = servers[s]
+
+print(yaml.dump(ovh_servers))
diff --git a/myrepos/mrconfig b/myrepos/mrconfig
new file mode 100644
index 0000000..5f7fc9b
--- /dev/null
+++ b/myrepos/mrconfig
@@ -0,0 +1,78 @@
+[DEFAULT]
+update = git pull --rebase
+
+[auquotidien]
+checkout = git@git.entrouvert.org:auquotidien.git
+
+[authentic]
+checkout = git@git.entrouvert.org:authentic.git
+
+[authentic2-auth-fc]
+checkout = git clone git@git.entrouvert.org:authentic2-auth-fc.git
+
+[authentic2-auth-fedict]
+checkout = git clone git@git.entrouvert.org:authentic2-auth-fedict.git
+
+[authentic2-auth-kerberos]
+checkout = git clone git@git.entrouvert.org:authentic2-auth-kerberos.git
+
+[authentic2-auth-msp]
+checkout = git clone git@git.entrouvert.org:authentic2-auth-msp.git
+
+[authentic2-auth-saml2]
+checkout = git clone git@git.entrouvert.org:authentic2-auth-saml2.git
+
+[bidon]
+checkout = git@git.entrouvert.org:bidon.git
+
+[bijoe]
+checkout = git@git.entrouvert.org:bijoe.git
+
+[chrono]
+checkout = git@git.entrouvert.org:chrono.git
+
+[combo]
+checkout = git@git.entrouvert.org:combo.git
+
+[corbo]
+checkout = git@git.entrouvert.org:corbo.git
+
+[django-mellon]
+checkout = git@git.entrouvert.org:django-mellon.git
+
+[django-tenant-schemas]
+checkout = git@git.entrouvert.org:django-tenant-schemas.git
+
+[fargo]
+checkout = git@git.entrouvert.org:fargo.git
+
+[gadjo]
+checkout = git@git.entrouvert.org:gadjo.git
+
+[hobo]
+checkout = git@git.entrouvert.org:hobo.git
+
+[lasso]
+checkout = git@git.entrouvert.org:lasso.git
+
+[lingo]
+checkout = git@git.entrouvert.org:lingo.git
+
+[mandayejs]
+checkout = git@git.entrouvert.org:mandayejs.git
+
+[passerelle]
+checkout = git@git.entrouvert.org:passerelle.git
+
+[publik-base-theme]
+checkout = git@git.entrouvert.org:publik-base-theme.git
+
+[wcs]
+checkout = git@git.entrouvert.org:wcs.git
+
+[wcs-olap]
+checkout = git@git.entrouvert.org:wcs-olap.git
+
+[welco]
+checkout = git@git.entrouvert.org:welco.git
+
diff --git a/publik-builder/Jenkinsfile b/publik-builder/Jenkinsfile
new file mode 100644
index 0000000..be21814
--- /dev/null
+++ b/publik-builder/Jenkinsfile
@@ -0,0 +1,32 @@
+@Library('eo-jenkins-lib@master') import eo.Utils
+
+pipeline {
+ agent any
+ stages {
+ stage('Unit Tests') {
+ steps {
+ sh 'echo "no test implemented"'
+ }
+ }
+ stage('Packaging') {
+ steps {
+ script {
+ if (env.JOB_NAME == 'publik-builder' && env.GIT_BRANCH == 'origin/master') {
+ sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder publik-builder'
+ }
+ }
+ }
+ }
+ }
+ post {
+ always {
+ script {
+ utils = new Utils()
+ utils.mail_notify(currentBuild, env, 'admin+adminsys@entrouvert.com')
+ }
+ }
+ success {
+ cleanWs()
+ }
+ }
+}
diff --git a/publik-builder/Makefile b/publik-builder/Makefile
new file mode 100644
index 0000000..29d6db5
--- /dev/null
+++ b/publik-builder/Makefile
@@ -0,0 +1,31 @@
+.PHONY: clean name version name fullname
+
+NAME=$(shell basename $(CURDIR))
+VERSION=`git describe | sed 's/^debian\///' | sed 's/v//'`
+DIST_FILES = Makefile \
+ publik-builder \
+ flavors
+
+version:
+ @(echo $(VERSION))
+
+name:
+ @(echo $(NAME))
+
+fullname:
+ @(echo $(NAME)-$(VERSION))
+
+clean:
+ rm -rf sdist
+
+dist-bzip2: dist
+ cd sdist && tar cfj ../sdist/$(NAME)-$(VERSION).tar.bz2 $(NAME)-$(VERSION)
+
+dist: clean
+ -mkdir sdist
+ rm -rf sdist/$(NAME)-$(VERSION)
+ mkdir -p sdist/$(NAME)-$(VERSION)
+ for i in $(DIST_FILES); do \
+ cp -R "$$i" sdist/$(NAME)-$(VERSION); \
+ done
+
diff --git a/publik-builder/README b/publik-builder/README
new file mode 100644
index 0000000..5fac4f3
--- /dev/null
+++ b/publik-builder/README
@@ -0,0 +1,11 @@
+publik-builder
+=============
+
+Publik images builder.
+
+Dependencies
+------------
+* debootstrap: Note you can run debootstrap from its source tree without installing it:
+
+ git clone https://salsa.debian.org/installer-team/debootstrap.git
+ alias debootstrap=$(pwd)/debootstrap/deboostrap
diff --git a/publik-builder/debian/changelog b/publik-builder/debian/changelog
new file mode 100644
index 0000000..1a428bb
--- /dev/null
+++ b/publik-builder/debian/changelog
@@ -0,0 +1,5 @@
+publik-builder (0.1) stretch; urgency=medium
+
+ * Initial release
+
+ -- Christophe Siraut <csiraut@entrouvert.com> Tue, 04 Sep 2018 17:15:19 +0200
diff --git a/publik-builder/debian/compat b/publik-builder/debian/compat
new file mode 100644
index 0000000..ec63514
--- /dev/null
+++ b/publik-builder/debian/compat
@@ -0,0 +1 @@
+9
diff --git a/publik-builder/debian/control b/publik-builder/debian/control
new file mode 100644
index 0000000..876318f
--- /dev/null
+++ b/publik-builder/debian/control
@@ -0,0 +1,11 @@
+Source: publik-builder
+Maintainer: Christophe Siraut <csiraut@entrouvert.com>
+Section: python
+Priority: optional
+Build-Depends: debhelper (>= 9)
+Standards-Version: 4.1.3
+
+Package: publik-builder
+Architecture: all
+Depends: ${misc:Depends}, python3:any
+Description: Publik images builder
diff --git a/publik-builder/debian/copyright b/publik-builder/debian/copyright
new file mode 100644
index 0000000..f2ddcbc
--- /dev/null
+++ b/publik-builder/debian/copyright
@@ -0,0 +1,25 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: publik-common
+Source: https://www.entrouvert.com
+
+Files: *
+Copyright: 2018 Christophe Siraut <csiraut@entrouvert.com>
+ 2018 Entrouvert Admins <admin@entrouvert.com>
+License: GPL-3.0+
+
+License: GPL-3.0+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 3 can be found in "/usr/share/common-licenses/GPL-3".
diff --git a/publik-builder/debian/gbp.conf b/publik-builder/debian/gbp.conf
new file mode 100644
index 0000000..12dcf99
--- /dev/null
+++ b/publik-builder/debian/gbp.conf
@@ -0,0 +1,7 @@
+[DEFAULT]
+cleaner = fakeroot debian/rules clean
+builder = debuild -us -uc
+
+[buildpackage]
+export-dir = ../build-area
+
diff --git a/publik-builder/debian/install b/publik-builder/debian/install
new file mode 100644
index 0000000..08a7d5d
--- /dev/null
+++ b/publik-builder/debian/install
@@ -0,0 +1,2 @@
+publik-builder /usr/bin
+flavors /var/lib/publik-builder
diff --git a/publik-builder/debian/rules b/publik-builder/debian/rules
new file mode 100755
index 0000000..301c910
--- /dev/null
+++ b/publik-builder/debian/rules
@@ -0,0 +1,5 @@
+#!/usr/bin/make -f
+export PYBUILD_NAME=publik-common
+
+%:
+ dh $@
diff --git a/publik-builder/debian/source/format b/publik-builder/debian/source/format
new file mode 100644
index 0000000..163aaf8
--- /dev/null
+++ b/publik-builder/debian/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/publik-builder/flavors/server b/publik-builder/flavors/server
new file mode 100755
index 0000000..f0fa77e
--- /dev/null
+++ b/publik-builder/flavors/server
@@ -0,0 +1,24 @@
+#!/bin/bash
+# publik-bootstrap: start a Publik server
+# 2018 Entr'ouvert
+set -e
+
+apt-get update
+apt-get install -yqq apt-transport-https ca-certificates wget lsb-release
+wget -O - https://deb.entrouvert.org/entrouvert.gpg | apt-key add -
+echo "deb http://deb.entrouvert.org/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/entrouvert.list
+apt-get update
+apt-get install -y publik-common
+
+publik-create-users
+apt install -y postgresql-client bsd-mailx zip # libreoffice
+cp /etc/publik/publik.conf.example /etc/publik/publik.conf
+public-create-databases
+
+#apt install hobo hobo-agent
+#systemctl restart supervisor
+#apt install -y authentic2-multitenant python-authentic2-auth-fc
+#apt install -y combo
+#apt install -y passerelle
+#apt install -y bijoe fargo welco chrono
+#apt install -y wcs wcs-au-quotidien
diff --git a/publik-builder/publik-builder b/publik-builder/publik-builder
new file mode 100755
index 0000000..065e9d2
--- /dev/null
+++ b/publik-builder/publik-builder
@@ -0,0 +1,91 @@
+#!/usr/bin/python3
+# Publik image builder
+# 2018 Entr'ouvert
+from contextlib import contextmanager
+import argparse
+import os
+import shutil
+import subprocess
+import uuid
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--release', type=str, default='stretch')
+parser.add_argument('--prefix', type=str, default='publik')
+parser.add_argument('--arch', type=str, default='amd64')
+parser.add_argument('--clean', action='store_true')
+parser.add_argument('--flavor', choices=['server', 'devinst', 'vanilla'], default='server')
+parser.add_argument('--form', choices=['tar', 'xz', 'folder'], default='xz')
+args = parser.parse_args()
+
+options = vars(args)
+options['mirror'] = 'http://httpredir.debian.org/debian/'
+options['tag'] = '{release}-{arch}'.format(**options)
+options['cachedir'] = '/var/cache/publik-builder'
+options['output_dir'] = os.getcwd()
+options['output'] = '{output_dir}/{prefix}-{tag}'.format(**options)
+
+if os.getuid():
+ raise(Exception('please with sudo (or as root)'))
+
+
+def run(cmd):
+ subprocess.run(cmd, shell=True, check=True)
+
+
+@contextmanager
+def mount_chroot(path):
+ run('mount -t proc /proc {}/proc'.format(path))
+ run('mount --bind /sys {}/sys'.format(path))
+ run('mount --bind /dev {}/dev'.format(path))
+ run('mount --bind /tmp {}/tmp'.format(path))
+ try:
+ yield
+ finally:
+ run('umount {}/tmp'.format(path))
+ run('umount {}/dev'.format(path))
+ run('umount {}/sys'.format(path))
+ run('umount {}/proc'.format(path))
+
+
+rootfs = os.path.join(options['cachedir'], options['tag'])
+if args.clean and os.path.isdir(rootfs):
+ shutil.rmtree(rootfs)
+if not os.path.isdir(rootfs):
+ if not os.path.isdir(options['cachedir']):
+ os.makedirs(options['cachedir'])
+ options['rootfs'] = rootfs
+ run('debootstrap --foreign {release} {rootfs}'.format(**options))
+ run('DEBOOTSTRAP_DIR={rootfs}/debootstrap debootstrap --second-stage --arch {arch} --second-stage-target {rootfs} {mirror}'.format(**options))
+
+
+workdir = os.path.join('/tmp/publik-builder', str(uuid.uuid4()))
+if not os.path.isdir('/tmp/publik-builder'):
+ os.makedirs('/tmp/publik-builder')
+options['workdir'] = workdir
+print('copying base to workdir {}'.format(workdir))
+run('cp -r {} {}'.format(rootfs, workdir))
+
+if args.flavor != 'none':
+ if args.flavor == 'devinst':
+ raise(Exception('not implemented'))
+ script = '/var/lib/publik-builder/flavors/%s' % args.flavor
+ if not os.path.isfile(script):
+ script = './flavors/%s' % args.flavor
+ if not os.path.isfile:
+ raise(Exception('cannot find {}'.format(script)))
+
+ rscript = '{}/usr/local/bin/publik-bootstrap'.format(workdir)
+ shutil.copyfile(script, rscript)
+ run('chmod +x {}'.format(rscript))
+ with mount_chroot(workdir):
+ run('chroot {} publik-bootstrap'.format(workdir))
+ run('chroot {} apt-get clean'.format(workdir))
+
+
+if args.form == 'tar':
+ run('cd {workdir} && tar cf {output}.tar ./'.format(**options))
+elif args.form == 'xz':
+ run('cd {workdir} && tar cfJ {output}.tar.xz ./'.format(**options))
+elif args.form == 'folder':
+ folder = '{output_dir}/{prefix}-{tag}'.format(**options)
+ shutil.move(workdir, folder)
diff --git a/sftp_syncfiles b/sftp_syncfiles
new file mode 100755
index 0000000..35569fd
--- /dev/null
+++ b/sftp_syncfiles
@@ -0,0 +1,100 @@
+#!/usr/bin/python3
+import argparse
+import cryptography
+import glob
+import os
+import paramiko
+import warnings
+warnings.simplefilter("ignore", cryptography.utils.CryptographyDeprecationWarning)
+
+
+def connection(hostname, username, password=None, keyfile=None):
+ if not keyfile:
+ keyfile = os.path.expanduser('~/.ssh/id_rsa')
+ key = paramiko.RSAKey.from_private_key_file(keyfile)
+ client = paramiko.SSHClient()
+ client.load_system_host_keys()
+ client.connect(hostname, username='csiraut', password=password, pkey=key)
+ return paramiko.SFTPClient.from_transport(client.get_transport())
+
+
+def validate_local_path(path):
+ if not os.path.exists(path) or not os.path.isdir(path):
+ raise(Exception('Directory %s does not exist' % path))
+
+
+def validate_remote_path(sftp, path):
+ lstat = sftp.lstat(path)
+ mode = oct(lstat.st_mode)[5]
+ if mode not in ['5', '7']:
+ raise(Exception('Remote path is not a folder or not writable'))
+
+
+def parse_location(loc):
+ hostname, username, path = None, None, loc
+ if '@' in loc and ':' in loc:
+ username, remaning = loc.split('@')
+ hostname, path = remaning.split(':')
+ return hostname, username, path
+
+
+def get(hostname, username, remotepath, localpath, password=None, keyfile=None):
+ validate_local_path(localpath)
+ sftp = connection(hostname, username, password=password, keyfile=keyfile)
+ validate_remote_path(sftp, remotepath)
+
+ dirlist = sftp.listdir_attr(remotepath)
+ filelist = [x.filename for x in dirlist if oct(x.st_mode)[5] not in ('5', '7')]
+ for f in filelist:
+ if args.verbose:
+ print('syncing %s to %s' % (remotepath, localpath))
+ remotefilepath = '%s/%s' % (remotepath, f)
+ localfilepath = '%s/%s' % (localpath, f)
+ sftp.get(remotefilepath, localfilepath)
+ if args.delete:
+ if args.verbose:
+ print('removing %s' % remotefilepath)
+ sftp.remove(remotefilepath)
+
+
+def put(hostname, username, remotepath, localpath, password=None, keyfile=None):
+ validate_local_path(localpath)
+ sftp = connection(hostname, username, password=password, keyfile=keyfile)
+ validate_remote_path(sftp, remotepath)
+
+ filelist = [f for f in glob.glob('%s/*' % localpath) if not os.path.isdir(f)]
+ for f in filelist:
+ remotefilepath = '%s/%s' % (remotepath, f.split('/')[-1])
+ if args.verbose:
+ print('syncing %s to %s' % (f, remotefilepath))
+ sftp.put(f, remotefilepath)
+ if args.delete:
+ if args.verbose:
+ print('removing %s' % f)
+ os.remove(f)
+
+
+def main(args):
+ orig = parse_location(args.origin)
+ dest = parse_location(args.destination)
+ if (orig[0] and dest[0]) or (not orig[1] and not dest[1]) or \
+ ((orig[1] and '@' not in orig[1]) and (dest[1] and '@' not in dest[1])):
+ raise(Exception('Please provide one remote location like this: user@server:/path/to/direction'))
+ if orig[1]:
+ action, hostname, username, remotepath, localpath = 'get', orig[0], orig[1], orig[2], dest[2]
+ else:
+ action, hostname, username, remotepath, localpath = 'put', dest[0], dest[1], dest[2], orig[2]
+
+ globals()[action](hostname, username, remotepath, localpath, args.password, args.keyfile)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='sftp.py — secure file transfer program')
+ parser.add_argument('origin', help='directory holding files to sync')
+ parser.add_argument('destination', help='target directory')
+ parser.add_argument('--keyfile')
+ parser.add_argument('--password')
+ parser.add_argument('--delete', action='store_true', help='remove source file(s) when transfered')
+ parser.add_argument('--verbose', action='store_true')
+ args = parser.parse_args()
+ main(args)