entrouvert
/
petale
17
0
Fork 0
Code Issues Pull Requests 1 Releases Activity

clean: remove files and take partner's info on command line

This commit is contained in:
Benjamin Dauvergne 2022-04-29 19:24:07 +02:00
parent aed59546fc
commit 9f183d7e0a
7 changed files with 108 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
petale/api_views.py
View File

@ -55,21 +55,6 @@ logger = logging.getLogger(__name__)
SENTINEL = object()
def check_unknown_cuts(uuids, creds=SENTINEL):
authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None)
if not authentic_url:
raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined')
authentic_creds = creds if creds is not SENTINEL else getattr(settings, 'PETALE_AUTHENTIC_AUTH', None)
if not authentic_creds:
raise ValueError('missing credentials for authentic, configure PETALE_AUTHENTIC_AUTH')
url = urlparse.urljoin(authentic_url, 'api/users/synchronization/')
response = requests.post(url, json={"known_uuids": list(uuids)}, auth=authentic_creds, verify=False)
response.raise_for_status()
data = response.json()
return data.get("unknown_uuids") or []
def cut_exists(request, cut_uuid):
if not getattr(settings, 'PETALE_CHECK_CUT_UUID', True):
CUT.objects.get_or_create(uuid=cut_uuid)

62
petale/management/commands/clean.py
View File

@ -14,12 +14,29 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import collections
import itertools
import shutil
from urllib.parse import urljoin
from django.core.management.base import BaseCommand
import requests
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from petale import models
from petale.api_views import check_unknown_cuts
from petale.models import CUT, Partner, cut_path
def check_unknown_cuts(uuids, creds):
authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None)
if not authentic_url:
raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined')
url = urljoin(authentic_url, 'api/users/synchronization/')
response = requests.post(url, json={"known_uuids": list(uuids)}, auth=creds, verify=False)
response.raise_for_status()
data = response.json()
return data.get("unknown_uuids") or []
def grouper(n, iterable, fillvalue=None):
@ -33,19 +50,38 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--delete', action='store_true', help='Delete dead petals.')
parser.add_argument('partner_name')
parser.add_argument('partner_client_id')
parser.add_argument('partner_client_secret')
def handle(self, *args, delete=False, **options):
def handle(self, partner_name, partner_client_id, partner_client_secret, delete=False, **options):
try:
partner = Partner.objects.get(name=partner_name)
except Partner.DoesNotExist:
raise CommandError('partner %r does not exist' % partner_name)
cuts = CUT.objects.filter(petal__partner=partner).distinct()
zombie_uuids = set()
for cuts in grouper(500, models.CUT.objects.iterator()):
zombie_uuids.update(check_unknown_cuts({cut.uuid for cut in cuts if cut is not None}))
if options['verbosity'] > 1:
print('Found %d dead cuts on %d known cuts.' % (len(zombie_uuids), models.CUT.objects.count()))
for uuids in grouper(500, cuts.values_list('uuid', flat=True).iterator()):
# remove None
uuids = {uuid for uuid in uuids if uuid}
zombie_uuids.update(check_unknown_cuts(uuids, (partner_client_id, partner_client_secret)))
if options['verbosity'] > 1 or delete:
print('Found %d dead cuts on %d known cuts.' % (len(zombie_uuids), cuts.count()))
if delete:
count, count_by_models = models.CUT.objects.filter(uuid__in=zombie_uuids).delete()
if count and options['verbosity'] > 0:
counts = collections.Counter()
try:
qs = CUT.objects.filter(uuid__in=zombie_uuids)
total = qs.count()
for i, cut in enumerate(qs):
print('Deleting %06d on %06d cuts.' % (i, total), end='\r')
with transaction.atomic():
count, count_by_models = cut.delete()
shutil.rmtree(cut_path(partner, cut.uuid))
counts += collections.Counter(count_by_models)
finally:
print()
print(
'Deleted ',
', '.join(
'%d %s' % (count, model.split('.')[-1]) for model, count in count_by_models.items()
),
', '.join('%d %s' % (count, model.split('.')[-1]) for model, count in counts.items()),
)

15
petale/models.py
View File

@ -18,9 +18,10 @@
import hashlib
from django.contrib.auth.models import User
from django.core.files.storage import default_storage
from django.core.mail import send_mail
from django.core.validators import RegexValidator
from django.db import models
from django.db import models, transaction
from django.utils.translation import ugettext_lazy as _
from . import utils
@ -83,6 +84,12 @@ class Partner(models.Model):
ordering = ['name']
def cut_path(partner, uuid):
return default_storage.path(
'data/{}/{}/{}/'.format(partner.name, hashlib.md5(uuid.encode('ascii')).hexdigest()[:3], uuid)
)
class CUT(models.Model):
uuid = models.CharField(max_length=255, validators=[id_validator], unique=True)
@ -155,6 +162,12 @@ class Petal(models.Model):
key=self.name,
)
@transaction.atomic(savepoint=False)
def delete(self, *args, **kwargs):
result = super().delete(*args, **kwargs)
self.data.delete(save=False)
return result
class Meta:
unique_together = ('name', 'partner', 'cut')
verbose_name = _('Petal')

19
tests/conftest.py
View File

@ -1,6 +1,3 @@
import shutil
from tempfile import mkdtemp
import django_webtest
import pytest
from utils import (
@ -14,17 +11,19 @@ from utils import (
)
@pytest.fixture(autouse=True)
def media(settings, tmp_path):
media_path = tmp_path / 'media'
media_path.mkdir()
settings.MEDIA_ROOT = str(media_path)
return media_path
@pytest.fixture
def app(request, settings):
def app(request, media):
wtm = django_webtest.WebTestMixin()
wtm._patch_settings()
request.addfinalizer(wtm._unpatch_settings)
settings.MEDIA_ROOT = mkdtemp()
def fin():
shutil.rmtree(settings.MEDIA_ROOT)
request.addfinalizer(fin)
return django_webtest.DjangoTestApp()

1
tests/settings.py
View File

@ -14,3 +14,4 @@ if 'postgres' in DATABASES['default']['ENGINE']:
PETALE_AUTHENTIC_URL = 'http://example.net/idp/'
PETALE_AUTHENTIC_AUTH = ('foo', 'bar')
MEDIA_ROOT = '/does-not-exist/'

4
tests/test_api.py
View File

@ -15,7 +15,7 @@ pytestmark = pytest.mark.django_db
def test_authentication_failure(app):
resp = app.get('/api/partner/12345abcde12345abcde12345abcde12/', status=401)
resp = app.get('/api/partner/12345abcde12345abcde12345abcde12/', status=403)
json.loads(resp.text)
@ -363,7 +363,7 @@ def test_idp_based_partner_authentication(mocked_post, app, cut_kevin_uuid, acl)
# failure
response = {"result": 0, "errors": ["Invalid username/password."]}
mocked_post.return_value = FakedResponse(content=json.dumps(response))
app.put_json(url, params=payload, status=401, headers={'If-None-Match': '*'})
app.put_json(url, params=payload, status=403, headers={'If-None-Match': '*'})
response = {"result": 1, "errors": []}
# test with RP
client_id = client_secret = 'a1b2' * 8

51
tests/test_commands.py
View File

@ -4,16 +4,10 @@ import pytest
from django.core.files.base import ContentFile
from django.core.management import call_command
from petale.api_views import check_unknown_cuts
from petale.management.commands.clean import check_unknown_cuts
from petale.models import CUT, Partner, Petal
@pytest.fixture(autouse=True)
def settings(settings):
settings.PETALE_AUTHENTIC_URL = 'https://authentic.example.com/'
settings.PETALE_AUTHENTIC_AUTH = 'admin:admin'
@pytest.fixture
def requests_post():
post_return_value = mock.Mock()
@ -25,10 +19,10 @@ def requests_post():
def test_check_unknown_cuts(requests_post):
requests_post.return_value.json.return_value = {'unknown_uuids': ['1']}
assert check_unknown_cuts(['1']) == ['1']
assert requests_post.call_args[0] == ('https://authentic.example.com/api/users/synchronization/',)
assert check_unknown_cuts(['1'], ('admin', 'admin')) == ['1']
assert requests_post.call_args[0] == ('http://example.net/idp/api/users/synchronization/',)
assert requests_post.call_args[1] == {
'auth': 'admin:admin',
'auth': ('admin', 'admin'),
'json': {'known_uuids': ['1']},
'verify': False,
}
@ -39,31 +33,52 @@ class TestClean:
uuids = {str(i) for i in range(10)}
unknown_uuids = {str(i) for i in range(5)}
@pytest.fixture
def other_partner(self, db):
return Partner.objects.create(name='other')
@pytest.fixture
def partner(self, db):
return Partner.objects.create(name='partner')
@pytest.fixture
def cuts(self, partner):
def cuts(self, partner, other_partner, media):
for uuid in ['a', 'b', 'c']:
cut = CUT.objects.create(uuid=uuid)
petal = Petal.objects.create(name='petal', partner=other_partner, cut=cut, size=0)
petal.data.save('petal.dat', ContentFile(b'1234'))
for uuid in self.uuids:
cut = CUT.objects.create(uuid=uuid)
petal = Petal.objects.create(name='petal', partner=partner, cut=cut, size=0)
petal.data.save('petal.dat', ContentFile(b'1234'))
assert len(list(media.rglob('petal'))) == 13
assert len(list(media.glob('*/*/*/*'))) == 13
@pytest.fixture
def requests_post(self, requests_post):
requests_post.return_value.json.return_value = {'unknown_uuids': self.unknown_uuids}
return requests_post
def test_dry(self, cuts, partner, requests_post):
call_command('clean')
def test_dry(self, cuts, partner, requests_post, media):
call_command('clean', 'partner', '1234', '5678')
assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids
assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids
assert set(CUT.objects.filter(petal__partner=partner).values_list('uuid', flat=True)) == self.uuids
assert len(list(media.rglob('petal'))) == 13
assert len(list(media.glob('*/*/*/*'))) == 13
def test_delete(self, cuts, partner, requests_post):
call_command('clean', '--delete')
def test_delete(self, cuts, partner, requests_post, media):
call_command('clean', '--delete', 'partner', '1234', '5678')
assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids
assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids - self.unknown_uuids
assert (
set(CUT.objects.filter(petal__partner=partner).values_list('uuid', flat=True))
== self.uuids - self.unknown_uuids
)
assert len(list(media.rglob('petal'))) == 8
assert len(list(media.glob('*/*/*/*'))) == 8
call_command('clean', '--delete')
call_command('clean', '--delete', 'partner', '1234', '5678')
assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids - self.unknown_uuids
assert len(list(media.rglob('*/partner/**/petal'))) == 5
assert len(list(media.glob('*/partner/*/*'))) == 5
assert len(list(media.rglob('*/*/**/petal'))) == 8
assert len(list(media.glob('*/*/*/*'))) == 8