88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
# Petale - Simple App as Key/Value Storage Interface
|
|
# Copyright (C) 2017 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import collections
|
|
import itertools
|
|
import shutil
|
|
from urllib.parse import urljoin
|
|
|
|
import requests
|
|
from django.conf import settings
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
from django.db import transaction
|
|
|
|
from petale.models import CUT, Partner, cut_path
|
|
|
|
|
|
def check_unknown_cuts(uuids, creds):
|
|
authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None)
|
|
if not authentic_url:
|
|
raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined')
|
|
|
|
url = urljoin(authentic_url, 'api/users/synchronization/')
|
|
response = requests.post(url, json={"known_uuids": list(uuids)}, auth=creds, verify=False)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
return data.get("unknown_uuids") or []
|
|
|
|
|
|
def grouper(n, iterable, fillvalue=None):
|
|
"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
|
|
args = [iter(iterable)] * n
|
|
return itertools.zip_longest(fillvalue=fillvalue, *args)
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = 'Clean petale data'
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument('--delete', action='store_true', help='Delete dead petals.')
|
|
parser.add_argument('partner_name')
|
|
parser.add_argument('partner_client_id')
|
|
parser.add_argument('partner_client_secret')
|
|
|
|
def handle(self, partner_name, partner_client_id, partner_client_secret, delete=False, **options):
|
|
try:
|
|
partner = Partner.objects.get(name=partner_name)
|
|
except Partner.DoesNotExist:
|
|
raise CommandError('partner %r does not exist' % partner_name)
|
|
|
|
cuts = CUT.objects.filter(petal__partner=partner).distinct()
|
|
zombie_uuids = set()
|
|
for uuids in grouper(500, cuts.values_list('uuid', flat=True).iterator()):
|
|
# remove None
|
|
uuids = {uuid for uuid in uuids if uuid}
|
|
zombie_uuids.update(check_unknown_cuts(uuids, (partner_client_id, partner_client_secret)))
|
|
if options['verbosity'] > 1 or delete:
|
|
print('Found %d dead cuts on %d known cuts.' % (len(zombie_uuids), cuts.count()))
|
|
if delete:
|
|
counts = collections.Counter()
|
|
try:
|
|
qs = CUT.objects.filter(uuid__in=zombie_uuids)
|
|
total = qs.count()
|
|
for i, cut in enumerate(qs):
|
|
print('Deleting %06d on %06d cuts.' % (i, total), end='\r')
|
|
with transaction.atomic():
|
|
count, count_by_models = cut.delete()
|
|
shutil.rmtree(cut_path(partner, cut.uuid))
|
|
counts += collections.Counter(count_by_models)
|
|
finally:
|
|
print()
|
|
print(
|
|
'Deleted ',
|
|
', '.join('%d %s' % (count, model.split('.')[-1]) for model, count in counts.items()),
|
|
)
|