tooling: webhook handler for gitea -> redmine communication (#70893)

This commit is contained in:
Agate 2022-11-02 10:35:06 +01:00 committed by Gitea
parent e8ef4d3214
commit d6fc582d7d
9 changed files with 690 additions and 31 deletions

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
venv
__pycache__
*.egg-info

16
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,16 @@
@Library('eo-jenkins-lib@main') import eo.Utils
pipeline {
agent any
options {
disableConcurrentBuilds()
timeout(time: 10, unit: 'MINUTES')
}
stages {
stage('Unit Tests') {
steps {
sh 'tox -rv'
}
}
}
}

73
README.md Normal file
View File

@ -0,0 +1,73 @@
Un webservice qui traite les webhooks Gitea pour enricher les tickets correspondants sur Redmine.
## Fonctionnalités
- Ajouter un lien vers une pull request Gitea dans les tickets correspondants sur Redmine
- Changer le statut d'un ticket vers En cours quand la pull request est créée
- Changer le statut d'un ticket vers Résolu quand la pull request est mergée
- Assigner le ticket redmine à l'auteur de la pull request qui est ouverte
- Passer un ticket redmine à En Cours quand une pull request est en WIP
- Informer sur le ticket redmine quand une pull request est relue
- Passer un ticket redmine à validé quand un pull request est validée par une personne
## Environnement de développement
```bash
pip install flask python-redmine
# Variables d'environnement nécessaires
## avec cette valeur, l'URL de webhook à utiliser sera de la forme /incoming-webhook/abcdefgh
export INCOMING_WEBHOOK_SECRET=abcdefgh
## clé d'API redmine à récupérer sur https://dev.entrouvert.org/my/api_key
export REDMINE_API_KEY=xxx
export REDMINE_URL=https://dev.entrouvert.org/
# lancer le serveur en développement
flask --app gitea_redmine --debug run
```
## Configuration Redmine
Côté Redmine, créer un utilisateur gitea dédié et lui donner les droits administrateurs. Les droits administrateurs sont malheureusement requis pour pouvoir faire le mapping entre les utilisateurs gitea et les utilisateurs redmine (cf https://www.redmine.org/projects/redmine/wiki/Rest_Users).
Récurpérer la clé d'API de l'utilisateur qui servira pour le déploiement.
## Configuration Gitea
Côté Gitea, configurer [un webhook au niveau de l'organisation](https://gitea.entrouvert.org/org/entrouvert/settings/hooks)
pour éviter de le faire par projet. Cocher « Custom events » et les évenements « Pull request » et « Pull request reviewed».
## Déploiement
```
git clone gitea@gitea.entrouvert.org:entrouvert/gitea-redmine.git /home/gitea-redmine
cd /home/gitea-redmine
sudo apt install python3-venv python3-pip
python3 -m venv venv
./venv/bin/pip install gunicorn
./venv/bin/pip install -e .
### SystemD
```
[Unit]
Description=Gitea Redmine (Webhook forwarder)
After=syslog.target
After=network.target
[Service]
RestartSec=2s
Type=simple
User=www-data
Group=www-data
WorkingDirectory=/home/gitea-redmine/
ExecStart=/home/gitea-redmine/venv/bin/gunicorn --bind 127.0.0.1:5000 gitea_redmine:app
Restart=always
Environment=REDMINE_URL=https://dev.entrouvert.org/ INCOMING_WEBHOOK_SECRET=abcdefgh REDMINE_API_KEY=xxx
[Install]
WantedBy=emulti-user.target
## Tests
Lancer classiquement via `tox`.

243
gitea_redmine.py Normal file
View File

@ -0,0 +1,243 @@
import functools
import json
import logging
import os
import re
import flask
import redminelib
logging.basicConfig(
level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[logging.StreamHandler()]
)
INCOMING_WEBHOOK_SECRET = os.environ["INCOMING_WEBHOOK_SECRET"]
REDMINE_URL = os.environ["REDMINE_URL"]
REDMINE_API_KEY = os.environ["REDMINE_API_KEY"]
REDMINE_CLIENT = redminelib.Redmine(REDMINE_URL, key=REDMINE_API_KEY)
REDMINE_ASSIGNABLE_GROUP = int(os.environ.get('REDMINE_ASSIGNABLE_GROUP', 21))
REDMINE_STATUSES = {
'En cours': 2,
'Résolu': 3,
'Solution proposée': 12,
'Solution validée': 13,
}
app = flask.Flask(__name__)
@app.route("/incoming-webhook/<token>", methods=['POST'])
def incoming_webhook(token):
if token != INCOMING_WEBHOOK_SECRET:
logging.warning('Received invalid token')
return {'status': 'error', 'detail': 'Invalid token'}, 403
payload = flask.request.json
handler, event = get_handler(payload)
if not handler:
logging.info('Skipping, no handler found for this webhook')
return {'status': 'success', 'detail': 'Skipped, unhandled webhook'}
haystack = f'{payload["pull_request"]["body"]} {payload["pull_request"]["title"]}'
issues_ids = get_issues_ids(haystack)
issues = []
for id in issues_ids:
try:
issues.append(get_redmine_issue(id))
except redminelib.exceptions.ResourceNotFoundError:
logging.warning(f'Unknown redmine issue {id}')
if not issues:
return {'status': 'success', 'detail': 'Skipped, no valid redmine issues linked'}
for issue in issues:
logging.info(f'Calling handler {event} for issue {issue.id}')
handler(issue, payload)
logging.info(f' Done')
return {'status': 'success', 'detail': 'Event processed and forwarded to redmine'}
def get_handler(payload):
if payload.get('action') == 'opened' and payload.get('pull_request'):
return handle_pull_request_opened, 'pull_request.opened'
if payload.get('action') == 'reviewed':
if payload['review']['type'] == 'pull_request_review_rejected':
return handle_pull_request_rejected, 'pull_request.rejected'
if payload['review']['type'] == 'pull_request_review_approved':
return handle_pull_request_approved, 'pull_request.approved'
return None, None
if payload.get('action') == 'edited' and payload.get('pull_request'):
if payload['pull_request']['title'].lower().startswith('wip:'):
return handle_pull_request_draft, 'pull_request.draft'
return handle_pull_request_edited, 'pull_request.edited'
if (
payload.get('action') == 'closed'
and payload.get('pull_request')
and payload['pull_request'].get('merged')
):
return handle_pull_request_merged, 'pull_request.merged'
return None, None
ISSUE_REGEX = re.compile(r'#(\d+)')
def get_issues_ids(text):
"""Extract issues number from their #1234 notation"""
return [int(i) for i in sorted(set(ISSUE_REGEX.findall(text)))]
def noop(payload):
pass
def get_redmine_user(username):
for user in REDMINE_CLIENT.user.filter(group_id=REDMINE_ASSIGNABLE_GROUP):
# 21 is the Entrouvert group
# We cannot easily find a redmine user through the API using
# only its username. We need to loop on possible candida
if user.login == username:
return user
raise Exception(f'No redmine user found for username {username} in group {REDMINE_ASSIGNABLE_GROUP}')
def get_redmine_issue(id):
return REDMINE_CLIENT.issue.get(id, includes=['journals'])
class Abort(Exception):
pass
def make_handler(*actions):
def inner(issue, payload):
flat_payload = flatten(payload)
for action in actions:
try:
action(issue, flat_payload)
except Abort:
return
return inner
def flatten(d, parent_key='', sep='_'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, dict):
items.extend(flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
def assign_to(user_field):
def inner(issue, payload):
redmine_user = get_redmine_user(payload[user_field])
issue.assigned_to_id = redmine_user.id
return inner
def set_status(status_id):
def inner(issue, payload):
issue.status_id = status_id
return inner
def add_note(template):
def inner(issue, payload):
issue.notes = template.format(**payload)
return inner
def save():
def inner(issue, payload):
issue.save()
return inner
def skip_if_not_draft(issue, payload):
if not payload['pull_request_title'].lower().startswith('wip:'):
raise Abort("Not a WIP: PR")
def skip_if_notes_include(field, reason):
def inner(issue, payload):
for entry in issue.journals:
if payload[field] in entry.notes:
raise Abort(reason)
return inner
def skip_if_status(status):
def inner(issue, payload):
if issue.status.id == status:
raise Abort(f'Issue already in status {status}')
return inner
handle_pull_request_opened = make_handler(
assign_to('pull_request_user_username'),
set_status(REDMINE_STATUSES['En cours']),
add_note(
'{pull_request_user_full_name} ({pull_request_user_username}) a ouvert une pull request sur Gitea concernant cette demande :\n\n* URL : {pull_request_url}\n* Titre : {pull_request_title}'
),
save(),
)
handle_pull_request_edited = make_handler(
skip_if_notes_include('pull_request_url', 'Pull request already linked in issue'),
assign_to('pull_request_user_username'),
set_status(REDMINE_STATUSES['En cours']),
add_note(
'{pull_request_user_full_name} ({pull_request_user_username}) a lié une pull request sur Gitea concernant cette demande :\n\n* URL : {pull_request_url}\n* Titre : {pull_request_title}'
),
save(),
)
handle_pull_request_draft = make_handler(
skip_if_status(REDMINE_STATUSES['En cours']),
skip_if_not_draft,
set_status(REDMINE_STATUSES['En cours']),
add_note(
'{pull_request_user_full_name} ({pull_request_user_username}) a commencé à travailler sur une pull request sur Gitea concernant cette demande :\n\n* URL : {pull_request_url}\n* Titre : {pull_request_title}'
),
save(),
)
handle_pull_request_merged = make_handler(
set_status(REDMINE_STATUSES['Résolu']),
add_note(
'{pull_request_user_full_name} ({pull_request_user_username}) a mergé une pull request sur Gitea concernant cette demande :\n\n* URL : {pull_request_url}\n* Titre : {pull_request_title}'
),
save(),
)
handle_pull_request_approved = make_handler(
set_status(REDMINE_STATUSES['Solution proposée']),
save(),
set_status(REDMINE_STATUSES['Solution validée']),
add_note(
'{sender_full_name} ({sender_username}) a approuvé une pull request sur Gitea concernant cette demande :\n\n* URL : {pull_request_url}\n* Commentaire :\n\n{review_content}'
),
save(),
)
handle_pull_request_rejected = make_handler(
set_status(REDMINE_STATUSES['En cours']),
add_note(
'{sender_full_name} ({sender_username}) a relu et demandé des modifications sur une pull request sur Gitea concernant cette demande :\n\n* URL : {pull_request_url}\n* Commentaire :\n\n{review_content}'
),
save(),
)

View File

@ -1,19 +0,0 @@
import os
import json
import requests
import flask
INCOMING_WEBHOOK_SECRET = os.environ["INCOMING_WEBHOOK_SECRET"]
REDMINE_URL = os.environ["REDMINE_URL"]
REDMINE_API_KEY = os.environ["REDMINE_API_KEY"]
from flask import Flask
app = Flask(__name__)
@app.route("/incoming-webhook/<token>", methods=['POST'])
def incoming_webhook(token):
if token != INCOMING_WEBHOOK_SECRET:
return {'status': 'error', 'detail': 'Invalid token'}, 403
return {'status': 'success'}

View File

@ -1,9 +1,7 @@
#! /usr/bin/env python
from setuptools import find_packages, setup
setup(
name='gitea-redmine',
name='gitea_redmine',
version="0.1",
description='Micro serveur HTTP pour intercepter les webhooks Gitea et mettre à jour les tickets redmine correspondants avec les bons status / liens vers pull requests.',
author='Agate Berriot',
@ -19,9 +17,6 @@ setup(
'Programming Language :: Python',
'Programming Language :: Python :: 3',
],
install_requires=[
'python-redmine',
'flask'
],
install_requires=['python-redmine', 'flask'],
zip_safe=False,
)

344
test_app.py Normal file
View File

@ -0,0 +1,344 @@
import pytest
import gitea_redmine
@pytest.fixture
def client():
return gitea_redmine.app.test_client()
@pytest.mark.parametrize(
'payload, expected_handler, expected_event',
[
(
{'action': 'opened', 'pull_request': {'title': 'foo'}},
gitea_redmine.handle_pull_request_opened,
'pull_request.opened',
),
(
{'action': 'edited', 'pull_request': {'title': 'foo'}},
gitea_redmine.handle_pull_request_edited,
'pull_request.edited',
),
(
{'action': 'edited', 'pull_request': {'title': 'WIP: something'}},
gitea_redmine.handle_pull_request_draft,
'pull_request.draft',
),
(
{'action': 'reviewed', 'review': {"type": "pull_request_review_approved"}},
gitea_redmine.handle_pull_request_approved,
'pull_request.approved',
),
(
{'action': 'reviewed', 'review': {"type": "pull_request_review_rejected"}},
gitea_redmine.handle_pull_request_rejected,
'pull_request.rejected',
),
(
{'action': 'closed', 'pull_request': {'merged': True}},
gitea_redmine.handle_pull_request_merged,
'pull_request.merged',
),
({'action': 'unknown', 'pull_request': 'foo'}, None, None),
],
)
def test_get_handler(payload, expected_handler, expected_event):
assert gitea_redmine.get_handler(payload) == (expected_handler, expected_event)
@pytest.mark.parametrize(
'body, expected',
[
('None', []),
('#1, #2', [1, 2]),
('#none, #2', [2]),
('# Header\n, #2', [2]),
],
)
def test_get_issues_ids(body, expected):
assert gitea_redmine.get_issues_ids(body) == expected
@pytest.mark.parametrize(
'dict, expected',
[
({'a': 'foo', 'b': 'bar'}, {'a': 'foo', 'b': 'bar'}),
({'a': {'b': 'bar'}}, {'a_b': 'bar'}),
],
)
def test_flatten(dict, expected):
assert gitea_redmine.flatten(dict) == expected
def test_handle_pull_request_opened(mocker):
redmine_user = mocker.Mock(id=42)
get_redmine_user = mocker.patch.object(gitea_redmine, 'get_redmine_user', return_value=redmine_user)
payload = {
"action": "opened",
"number": 2,
"pull_request": {
"id": 7,
"url": "https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2",
"number": 2,
"user": {
"id": 7,
"login": "testuser",
"full_name": "Test User",
"email": "test_user@noreply.gitea.entrouvert.org",
"username": "testuser",
},
"title": "Foo",
"body": "See #70893",
},
}
issue = mocker.Mock()
gitea_redmine.handle_pull_request_opened(issue, payload)
get_redmine_user.assert_called_once_with('testuser')
assert issue.assigned_to_id == redmine_user.id
assert issue.status_id == gitea_redmine.REDMINE_STATUSES['En cours']
assert issue.notes == (
'Test User (testuser) a ouvert une pull request sur Gitea concernant cette demande :\n\n'
'* URL : https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2\n'
'* Titre : Foo'
)
issue.save.assert_called_once()
def test_handle_pull_request_edited(mocker):
redmine_user = mocker.Mock(id=42)
get_redmine_user = mocker.patch.object(gitea_redmine, 'get_redmine_user', return_value=redmine_user)
payload = {
"action": "edited",
"number": 2,
"pull_request": {
"id": 7,
"url": "https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2",
"number": 2,
"user": {
"id": 7,
"login": "testuser",
"full_name": "Test User",
"email": "test_user@noreply.gitea.entrouvert.org",
"username": "testuser",
},
"title": "Foo",
"body": "See #70893",
},
}
issue = mocker.Mock(journals=[])
gitea_redmine.handle_pull_request_edited(issue, payload)
get_redmine_user.assert_called_once_with('testuser')
assert issue.assigned_to_id == redmine_user.id
assert issue.status_id == gitea_redmine.REDMINE_STATUSES['En cours']
assert issue.notes == (
'Test User (testuser) a lié une pull request sur Gitea concernant cette demande :\n\n'
'* URL : https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2\n'
'* Titre : Foo'
)
issue.save.assert_called_once()
def test_handle_pull_request_draft(mocker):
payload = {
"action": "edited",
"number": 2,
"pull_request": {
"id": 7,
"url": "https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2",
"number": 2,
"user": {
"id": 7,
"login": "testuser",
"full_name": "Test User",
"email": "test_user@noreply.gitea.entrouvert.org",
"username": "testuser",
},
"title": "WIP: Foo",
"body": "See #70893",
},
}
issue = mocker.Mock(journals=[])
gitea_redmine.handle_pull_request_draft(issue, payload)
assert issue.status_id == gitea_redmine.REDMINE_STATUSES['En cours']
assert issue.notes == (
'Test User (testuser) a commencé à travailler sur une pull request sur Gitea concernant cette demande :\n\n'
'* URL : https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2\n'
'* Titre : WIP: Foo'
)
issue.save.assert_called_once()
def test_handle_pull_request_edited_already_linked_does_nothing(mocker):
payload = {
"action": "edited",
"pull_request": {
"url": "https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2",
},
}
issue = mocker.Mock(
journals=[
mocker.Mock(notes='Linked to https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2')
]
)
gitea_redmine.handle_pull_request_edited(issue, payload)
issue.save.assert_not_called()
def test_handle_pull_request_reviewed_approved(mocker):
payload = {
"action": "reviewed",
"number": 2,
"pull_request": {
"id": 7,
"url": "https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2",
"number": 2,
"title": "Foo",
"body": "See #70893",
"merged": True,
},
"sender": {
"id": 7,
"login": "testuser",
"full_name": "Test User",
"email": "test_user@noreply.gitea.entrouvert.org",
"username": "testuser",
},
"review": {"type": "pull_request_review_approved", "content": "Okay pour moi"},
}
issue = mocker.Mock()
gitea_redmine.handle_pull_request_approved(issue, payload)
assert issue.status_id == gitea_redmine.REDMINE_STATUSES['Solution validée']
assert issue.notes == (
'Test User (testuser) a approuvé une pull request sur Gitea concernant cette demande :\n\n'
'* URL : https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2\n'
'* Commentaire :\n\nOkay pour moi'
)
issue.save.assert_called()
def test_handle_pull_request_reviewed_rejected(mocker):
payload = {
"action": "reviewed",
"number": 2,
"pull_request": {
"id": 7,
"url": "https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2",
"number": 2,
"title": "Foo",
"body": "See #70893",
"merged": True,
},
"sender": {
"id": 7,
"login": "testuser",
"full_name": "Test User",
"email": "test_user@noreply.gitea.entrouvert.org",
"username": "testuser",
},
"review": {"type": "pull_request_review_rejected", "content": "Des choses à changer"},
}
issue = mocker.Mock()
gitea_redmine.handle_pull_request_rejected(issue, payload)
assert issue.status_id == gitea_redmine.REDMINE_STATUSES['En cours']
assert issue.notes == (
'Test User (testuser) a relu et demandé des modifications sur une pull request sur Gitea concernant cette demande :\n\n'
'* URL : https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2\n'
'* Commentaire :\n\nDes choses à changer'
)
issue.save.assert_called_once()
def test_handle_pull_request_merged(mocker):
payload = {
"action": "closed",
"number": 2,
"pull_request": {
"id": 7,
"url": "https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2",
"number": 2,
"user": {
"id": 7,
"login": "testuser",
"full_name": "Test User",
"email": "test_user@noreply.gitea.entrouvert.org",
"username": "testuser",
},
"title": "Foo",
"body": "See #70893",
"merged": True,
},
}
issue = mocker.Mock()
gitea_redmine.handle_pull_request_merged(issue, payload)
assert issue.status_id == gitea_redmine.REDMINE_STATUSES['Résolu']
assert issue.notes == (
'Test User (testuser) a mergé une pull request sur Gitea concernant cette demande :\n\n'
'* URL : https://gitea.entrouvert.org/entrouvert/gitea-redmine/pulls/2\n'
'* Titre : Foo'
)
issue.save.assert_called_once()
def test_incoming_webhook_requires_secret(client):
response = client.post(f'/incoming-webhook/invalid', json={})
assert response.status_code == 403
assert response.json == {'status': 'error', 'detail': 'Invalid token'}
response = client.post(
f'/incoming-webhook/{gitea_redmine.INCOMING_WEBHOOK_SECRET}',
json={'action': 'noop'},
)
assert response.status_code == 200
assert response.json == {'status': 'success', 'detail': 'Skipped, unhandled webhook'}
def test_incoming_webhook_calls_proper_handler(client, mocker):
issue1 = mocker.Mock(journals=[])
issue2 = mocker.Mock(journals=[])
get_redmine_issue = mocker.patch.object(gitea_redmine, 'get_redmine_issue', side_effect=[issue1, issue2])
get_handler = mocker.patch.object(gitea_redmine, 'get_handler', return_value=[mocker.Mock(), 'foo'])
payload = {
"action": "foo",
"pull_request": {
"title": "Fix #1234",
"body": "And #5678",
},
}
response = client.post(
f'/incoming-webhook/{gitea_redmine.INCOMING_WEBHOOK_SECRET}',
json=payload,
)
get_handler.assert_called_once_with(payload)
handler = get_handler.return_value[0]
assert handler.call_count == 2
get_redmine_issue.assert_any_call(1234)
get_redmine_issue.assert_any_call(5678)
handler.assert_any_call(issue1, payload)
handler.assert_any_call(issue2, payload)
assert response.status_code == 200
assert response.json == {'status': 'success', 'detail': 'Event processed and forwarded to redmine'}

View File

14
tox.ini
View File

@ -1,13 +1,17 @@
[tox]
toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/gitea-redmine/{env:BRANCH_NAME:}
envlist = py3
skipsdist = True
[testenv]
usedevelop = True
setenv =
TOX_WORK_DIR={toxworkdir}
INCOMING_WEBHOOK_SECRET=noop
REDMINE_URL=noop
REDMINE_API_KEY=noop
SETUPTOOLS_USE_DISTUTILS=stdlib
deps =
requests
python-redmine
flask
pytest
pytest-mock
commands =
pytest