remove eoptasks (moved to its own repository)
This commit is contained in:
parent
d2cad4f97a
commit
b025f6b4c2
|
@ -1,422 +0,0 @@
|
|||
#! /usr/bin/python3
|
||||
#
|
||||
# This script provides parallel remote execution of commands, while having
|
||||
# some special knownledge of servers that should *not* be handled in parallel.
|
||||
#
|
||||
# It defers terminal-handling to tmux(1).
|
||||
#
|
||||
# It has some targeting capacities using keywords. Commas for 'OR' and slashes
|
||||
# for 'AND', ex: ext/test,saas/test/passerelle will select all external test
|
||||
# servers + all passerelle servers on the SaaS.
|
||||
#
|
||||
# It takes any shell command and has some builtin shortcuts such as apt.update
|
||||
# and apt.upgrade.
|
||||
#
|
||||
# Requirements: libtmux and pyyaml.
|
||||
#
|
||||
# Configuration: ~/.config/eoptasks.ini
|
||||
# [config]
|
||||
# servers = /path/to/servers.yaml
|
||||
# ignore = server1, server2
|
||||
# stripsuffix = .entrouvert.org
|
||||
#
|
||||
# It is possible to add extra command shortcuts with additional sections, ex:
|
||||
# [command:memcached.restart]
|
||||
# cmd = sudo service memcached restart; /bin/true
|
||||
#
|
||||
# Examples:
|
||||
#
|
||||
# eoptasks -k test apt.upgrade
|
||||
# Run (sudo) apt upgrade on all test servers.
|
||||
#
|
||||
# eoptasks -k test,-database sudo apt install python-gadjo
|
||||
# Run sudo apt install python-gadjo on all test servers except database servers.
|
||||
#
|
||||
# eoptasks -k saas/test/passerelle,ext/test --list-servers
|
||||
# List servers that have saas AND test AND passerelle keywords, OR the
|
||||
# ext AND test keywords.
|
||||
#
|
||||
# eoptasks -k saas/prod -x ^node,^database.node,.*docbow sudo apt install publik-base-theme
|
||||
# Install/upgrade publik-base-theme on all production server of the SaaS excepted
|
||||
# the hypervisor, database, and docbow nodes.
|
||||
|
||||
import argparse
|
||||
import configparser
|
||||
import curses
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
import libtmux
|
||||
import yaml
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, servername, tags=[], display_name=''):
|
||||
self.name = servername
|
||||
self.display_name = display_name or self.name
|
||||
self.keywords = set(re.split(r'[-_ \.]', servername))
|
||||
for tag in tags:
|
||||
self.keywords.add(tag)
|
||||
|
||||
# add all possible hostname parts as keywords,
|
||||
# ex: node1.dev.entrouvert.org will add:
|
||||
# node1.dev, node1.dev.entrouvert, node1.dev.entrouvert.org,
|
||||
# dev.entrouvert, dev.entrouvert.org, entrouvert.org
|
||||
parts = servername.split('.')
|
||||
for i in range(len(parts)-1):
|
||||
for j in range(i, len(parts)):
|
||||
if i != j:
|
||||
self.keywords.add('.'.join(parts[i:j+1]))
|
||||
if i == 0:
|
||||
# add first component without trailing digits, this allows
|
||||
# matching db1.prod.saas.entrouvert.org with the db
|
||||
# keyword.
|
||||
self.keywords.add(re.sub(r'\d+$', '', parts[0]))
|
||||
|
||||
def __repr__(self):
|
||||
return '<Server %s %r>' % (self.name, self.keywords)
|
||||
|
||||
|
||||
def get_servers():
|
||||
servers = []
|
||||
config = configparser.ConfigParser()
|
||||
config.read(os.path.join(os.path.expanduser('~/.config/eoptasks.ini')))
|
||||
serversfile = config.get('config', 'servers', fallback=None)
|
||||
if serversfile is None:
|
||||
print("You need to create ~/.config/eoptasks.ini with such a content:\n"
|
||||
"\n"
|
||||
" [config]\n"
|
||||
" servers = /home/user/src/puppet/data/servers.yaml\n")
|
||||
sys.exit(1)
|
||||
ignorelist = [x.strip() for x in config.get('config', 'ignore', fallback='').split(',')]
|
||||
|
||||
stripsuffixes = [x.strip() for x in config.get('config', 'stripsuffix', fallback='').split(',')]
|
||||
|
||||
def get_display_name(x):
|
||||
for stripsuffix in stripsuffixes:
|
||||
if stripsuffix and x.endswith(stripsuffix):
|
||||
return x[:-len(stripsuffix)]
|
||||
return x
|
||||
|
||||
for s in yaml.safe_load(open(serversfile))['servers']:
|
||||
servername, tags = s.get('name'), s.get('tags', [])
|
||||
if servername in ignorelist:
|
||||
continue
|
||||
servers.append(Server(servername, tags, display_name=get_display_name(servername)))
|
||||
|
||||
return servers
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-l', '--list-servers', action='store_true')
|
||||
parser.add_argument('--session-name', dest='session_name', type=str)
|
||||
parser.add_argument('--status-window', action='store_true')
|
||||
parser.add_argument('--command-window', action='store_true')
|
||||
parser.add_argument('--command-server-name', dest='command_server_name', type=str)
|
||||
parser.add_argument('--noinput', dest='noinput', action='store_true')
|
||||
parser.add_argument('--list-commands', dest='list_commands', action='store_true')
|
||||
parser.add_argument('-k', dest='keywords', type=str)
|
||||
parser.add_argument('-x', dest='exclude', type=str)
|
||||
parser.add_argument('cmd', type=str, nargs='?', default=None)
|
||||
parser.add_argument('args', nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
def filter_servers(servers, args):
|
||||
selected_servers = []
|
||||
if args.keywords:
|
||||
for keyword in args.keywords.split(','):
|
||||
keywords = set(keyword.split('/'))
|
||||
selected_servers.extend([
|
||||
x for x in servers
|
||||
if keywords.issubset(x.keywords) and not x in selected_servers])
|
||||
for keyword in args.keywords.split(','):
|
||||
if keyword.startswith('!') or keyword.startswith('-'):
|
||||
selected_servers = [x for x in selected_servers if keyword[1:] not in x.keywords]
|
||||
else:
|
||||
selected_servers = servers
|
||||
if args.exclude:
|
||||
for exclude in args.exclude.split(','):
|
||||
regex = re.compile(exclude)
|
||||
selected_servers = [x for x in selected_servers if not regex.match(x.name)]
|
||||
return selected_servers
|
||||
|
||||
|
||||
def status_window(args):
|
||||
session_name = args.session_name
|
||||
curses.setupterm()
|
||||
window = curses.initscr()
|
||||
window.addstr(0, 0, 'eoptasks', curses.A_STANDOUT)
|
||||
window.addstr(0, 10, '🙂')
|
||||
curses.curs_set(0)
|
||||
window.refresh()
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
server_address = '/tmp/.eoptasks.%s' % session_name
|
||||
sock.bind(server_address)
|
||||
sock.listen(1)
|
||||
e = None
|
||||
servers_results = {}
|
||||
while True:
|
||||
connection, client_address = sock.accept()
|
||||
try:
|
||||
json_msg = b''
|
||||
while True:
|
||||
data = connection.recv(5000)
|
||||
if not data:
|
||||
break
|
||||
json_msg += data
|
||||
msg = json.loads(json_msg.decode('utf-8'))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
if msg.get('@type') == 'servers-info':
|
||||
servers_info = msg['info']
|
||||
elif msg.get('@type') == 'server-result':
|
||||
servers_results.update(msg['info'])
|
||||
|
||||
try:
|
||||
height, width = window.getmaxyx()
|
||||
max_length = max([len(x['display_name']) for x in servers_info.values()]) + 4
|
||||
nb_columns = (width-4) // max_length
|
||||
|
||||
for i, server_name in enumerate(servers_info):
|
||||
y = 2 + (i//nb_columns)
|
||||
x = 1 + (width//nb_columns) * (i%nb_columns)
|
||||
window.addstr(y, x+3, servers_info[server_name]['display_name'])
|
||||
status_icon = {
|
||||
'running': '⏳',
|
||||
'done': '🆗',
|
||||
}.get(servers_info[server_name]['status'], '💤')
|
||||
if servers_results.get(server_name) == 'error':
|
||||
status_icon = '❗'
|
||||
window.addstr(y, x, status_icon)
|
||||
if y > height-4:
|
||||
break
|
||||
window.refresh()
|
||||
total_servers = len(servers_info.keys())
|
||||
running_servers = len([x for x in servers_info.values() if x['status'] == 'running'])
|
||||
done_servers = len([x for x in servers_info.values() if x['status'] == 'done'])
|
||||
if total_servers == done_servers:
|
||||
break
|
||||
except Exception as e:
|
||||
window.addstr(0, 10, '😡 %r' % e)
|
||||
window.refresh()
|
||||
os.unlink(server_address)
|
||||
window.addstr(0, 10, '😎')
|
||||
window.refresh()
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def send_status_message(tmux_session_name, msg):
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
server_address = '/tmp/.eoptasks.%s' % tmux_session_name
|
||||
try:
|
||||
sock.connect(server_address)
|
||||
except socket.error as e:
|
||||
return
|
||||
sock.sendall(json.dumps(msg).encode('utf-8'))
|
||||
sock.close()
|
||||
|
||||
|
||||
def get_commands():
|
||||
commands = {
|
||||
'apt.update': 'sudo apt update',
|
||||
'apt.upgrade': 'sudo apt update && sudo apt full-upgrade -y',
|
||||
# collectstatic is useful after an upgrade of gadjo.
|
||||
'collectstatic': '''sudo -u authentic-multitenant authentic2-multitenant-manage collectstatic --noinput;
|
||||
sudo -u bijoe bijoe-manage collectstatic --noinput;
|
||||
sudo -u chrono chrono-manage collectstatic --noinput;
|
||||
sudo -u combo combo-manage collectstatic --noinput;
|
||||
sudo -u corbo corbo-manage collectstatic --noinput;
|
||||
sudo -u fargo fargo-manage collectstatic --noinput;
|
||||
sudo -u hobo hobo-manage collectstatic --noinput;
|
||||
sudo -u passerelle passerelle-manage collectstatic --noinput;
|
||||
sudo -u welco welco-manage collectstatic --noinput;
|
||||
sudo -u wcs wcs-manage collectstatic;
|
||||
/bin/true'''.replace('\n', ''),
|
||||
# combo.reload is useful to get a new {% start_timestamp %} after an
|
||||
# upgrade of publik-base-theme.
|
||||
'combo.reload': '''sudo service combo reload; /bin/true''',
|
||||
# hobo-agent.restart is the fastest way to get the number of threads
|
||||
# used by celery under control :/
|
||||
'hobo-agent.restart': '''test -e /etc/hobo-agent/settings.py && sudo supervisorctl reread && sudo supervisorctl restart hobo-agent''',
|
||||
# memcached.restart is useful to force shared theme to be updated.
|
||||
'memcached.restart': '''sudo service memcached restart; /bin/true''',
|
||||
'restart.all': '''sudo systemctl restart authentic2-multitenant bijoe chrono combo corbo fargo hobo passerelle wcs welco; /bin/true''',
|
||||
'passerelle.restart': '''sudo systemctl restart passerelle; /bin/true''',
|
||||
'wcs.restart': '''sudo systemctl restart wcs; /bin/true''',
|
||||
# puppet.update, unfortunately without proper error checking.
|
||||
'puppet.update': '''sudo puppet agent -t || true''',
|
||||
}
|
||||
config = configparser.ConfigParser()
|
||||
config.read(os.path.join(os.path.expanduser('~/.config/eoptasks.ini')))
|
||||
for section in config.sections():
|
||||
if section.startswith('command:'):
|
||||
commands[section[len('command:'):]] = config.get(section, 'cmd')
|
||||
return commands
|
||||
|
||||
|
||||
def command_window(args):
|
||||
tmux_session_name = args.session_name
|
||||
commands = get_commands()
|
||||
if args.cmd in commands:
|
||||
cmd = commands[args.cmd]
|
||||
else:
|
||||
cmd = args.cmd
|
||||
|
||||
if args.args:
|
||||
cmd += ' ' + ' '.join(['"%s"' % x for x in args.args])
|
||||
orig_cmd = cmd
|
||||
while True:
|
||||
# -t: force a tty for interactive commands.
|
||||
rc = subprocess.call(['ssh', '-t', args.command_server_name] + [cmd])
|
||||
if rc == 0:
|
||||
send_status_message(tmux_session_name,
|
||||
{'@type': 'server-result',
|
||||
'info': {args.command_server_name: 'success'}})
|
||||
break
|
||||
send_status_message(tmux_session_name,
|
||||
{'@type': 'server-result',
|
||||
'info': {args.command_server_name: 'error'}})
|
||||
if args.noinput:
|
||||
break
|
||||
choice = None
|
||||
while choice not in ['r', 's', 'q']:
|
||||
choice = input('[R]etry, [S]hell, [Q]uit --> ').lower()
|
||||
if choice == 'r':
|
||||
cmd = orig_cmd
|
||||
elif choice == 's':
|
||||
cmd = 'bash'
|
||||
elif choice == 'q':
|
||||
break
|
||||
|
||||
args = parse_args()
|
||||
|
||||
if args.list_commands:
|
||||
commands = get_commands()
|
||||
for command in sorted(commands):
|
||||
print(command)
|
||||
sys.exit(0)
|
||||
|
||||
if args.status_window:
|
||||
status_window(args)
|
||||
sys.exit(0)
|
||||
|
||||
if args.command_window:
|
||||
command_window(args)
|
||||
sys.exit(0)
|
||||
|
||||
servers = get_servers()
|
||||
selected_servers = filter_servers(servers, args)
|
||||
|
||||
if args.list_servers:
|
||||
for server in sorted(selected_servers, key=lambda x: x.name):
|
||||
print(server.name)
|
||||
sys.exit(0)
|
||||
|
||||
if not selected_servers:
|
||||
sys.stderr.write('No matching servers\n')
|
||||
sys.exit(1)
|
||||
|
||||
if not args.cmd:
|
||||
sys.stderr.write('Missing command\n')
|
||||
sys.exit(1)
|
||||
|
||||
def init_tmux_session():
|
||||
if os.environ.get('TMUX'): # already in a tmux
|
||||
sys.stderr.write('Cannot run embedded in tmux\n')
|
||||
sys.exit(1)
|
||||
|
||||
tmux_session_name = 's%s' % random.randrange(1000)
|
||||
server_address = '/tmp/.eoptasks.%s' % tmux_session_name
|
||||
try:
|
||||
os.unlink(server_address)
|
||||
except OSError:
|
||||
pass
|
||||
os.environ['SHELL'] = '/bin/sh'
|
||||
os.system('tmux new-session -s %s -n 🌑 -d %s --status-window --session-name %s' % (
|
||||
tmux_session_name, sys.argv[0], tmux_session_name))
|
||||
return tmux_session_name
|
||||
|
||||
tmux_session_name = init_tmux_session()
|
||||
|
||||
pid = os.fork()
|
||||
if pid:
|
||||
os.system('tmux attach-session -t %s' % tmux_session_name)
|
||||
else:
|
||||
def cluster_name(server_name):
|
||||
cluster_name = re.sub(r'\d', '', server_name)
|
||||
for location in ('rbx', 'gra', 'sbg'):
|
||||
cluster_name = cluster_name.replace('.%s.' % location, '.loc')
|
||||
return cluster_name
|
||||
|
||||
tmux = libtmux.Server()
|
||||
session = tmux.find_where({'session_name': tmux_session_name})
|
||||
|
||||
status_window = session.attached_window
|
||||
|
||||
all_servers = selected_servers[:]
|
||||
total_number = len(selected_servers)
|
||||
random.shuffle(selected_servers)
|
||||
|
||||
servers_info = {}
|
||||
for server in selected_servers:
|
||||
servers_info[server.name] = {'status': '', 'display_name': server.display_name}
|
||||
|
||||
def send_status():
|
||||
current_windows = [x.name for x in session.list_windows()]
|
||||
for server in all_servers:
|
||||
server_info = servers_info[server.name]
|
||||
if server.name in current_windows:
|
||||
server_info['status'] = 'running'
|
||||
elif server_info['status'] == 'running':
|
||||
server_info['status'] = 'done'
|
||||
send_status_message(tmux_session_name, {'@type': 'servers-info', 'info': servers_info})
|
||||
|
||||
while selected_servers:
|
||||
current_clusters = [cluster_name(x.name) for x in session.list_windows()]
|
||||
for server in selected_servers[:]:
|
||||
if cluster_name(server.name) in current_clusters:
|
||||
continue
|
||||
selected_servers.remove(server)
|
||||
window_cmd = '%s --session-name %s --command-window --command-server-name %s %s "%s" %s' % (
|
||||
sys.argv[0],
|
||||
tmux_session_name,
|
||||
server.name,
|
||||
'--noinput' if args.noinput else '',
|
||||
args.cmd,
|
||||
' '.join(['"%s"' % x for x in args.args]))
|
||||
session.new_window(
|
||||
attach=False,
|
||||
window_name=server.name,
|
||||
window_shell=window_cmd)
|
||||
break
|
||||
else:
|
||||
time.sleep(0.1)
|
||||
while len(session.list_windows()) > 10:
|
||||
send_status()
|
||||
time.sleep(0.1)
|
||||
send_status()
|
||||
|
||||
percentage = (total_number - len(selected_servers)) / total_number
|
||||
if percentage == 1:
|
||||
status_window.rename_window('🌕')
|
||||
elif percentage >= 0.75:
|
||||
status_window.rename_window('🌔')
|
||||
elif percentage >= 0.5:
|
||||
status_window.rename_window('🌓')
|
||||
elif percentage >= 0.25:
|
||||
status_window.rename_window('🌒')
|
||||
|
||||
while len(session.list_windows()) > 1:
|
||||
send_status()
|
||||
time.sleep(0.1)
|
||||
status_window.rename_window('🌕')
|
||||
send_status()
|
Loading…
Reference in New Issue