diff --git a/passerelle/apps/base_adresse/migrations/0015_auto_20191206_1244.py b/passerelle/apps/base_adresse/migrations/0015_auto_20191206_1244.py new file mode 100644 index 00000000..8061b976 --- /dev/null +++ b/passerelle/apps/base_adresse/migrations/0015_auto_20191206_1244.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2019-12-06 11:44 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import passerelle.apps.base_adresse.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base_adresse', '0014_auto_20190207_0456'), + ] + + operations = [ + migrations.CreateModel( + name='CityModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=150, verbose_name='City name')), + ('unaccent_name', models.CharField(max_length=150, null=True, verbose_name='City name ascii char')), + ('code', models.CharField(max_length=5, verbose_name='INSEE code')), + ('zipcode', models.CharField(max_length=5, verbose_name='Postal code')), + ('population', models.PositiveIntegerField(verbose_name='Population')), + ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')), + ], + options={ + 'ordering': ['-population', 'zipcode', 'unaccent_name', 'name'], + }, + bases=(passerelle.apps.base_adresse.models.UnaccentNameMixin, models.Model), + ), + migrations.CreateModel( + name='DepartmentModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, verbose_name='Department name')), + ('unaccent_name', models.CharField(max_length=150, null=True, verbose_name='Department name ascii char')), + ('code', models.CharField(max_length=3, unique=True, verbose_name='Department code')), + ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')), + ], + options={ + 'ordering': ['code'], + }, + bases=(passerelle.apps.base_adresse.models.UnaccentNameMixin, models.Model), + ), + migrations.CreateModel( + name='RegionModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=150, verbose_name='Region name')), + ('unaccent_name', models.CharField(max_length=150, null=True, verbose_name='Region name ascii char')), + ('code', models.CharField(max_length=2, unique=True, verbose_name='Region code')), + ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')), + ], + options={ + 'ordering': ['code'], + }, + bases=(passerelle.apps.base_adresse.models.UnaccentNameMixin, models.Model), + ), + migrations.AddField( + model_name='baseadresse', + name='api_geo_url', + field=models.CharField(default=b'https://geo.api.gouv.fr/', help_text='Base Adresse API Geo URL', max_length=128, verbose_name='API Geo URL'), + ), + migrations.AlterField( + model_name='baseadresse', + name='zipcode', + field=models.CharField(blank=True, max_length=600, verbose_name='Postal codes or department number to get streets, separated with commas'), + ), + migrations.AlterField( + model_name='streetmodel', + name='city', + field=models.CharField(max_length=150, verbose_name='City'), + ), + migrations.AddField( + model_name='departmentmodel', + name='region', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base_adresse.RegionModel'), + ), + migrations.AddField( + model_name='citymodel', + name='department', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base_adresse.DepartmentModel'), + ), + migrations.AddField( + model_name='citymodel', + name='region', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base_adresse.RegionModel'), + ), + migrations.AlterUniqueTogether( + name='citymodel', + unique_together=set([('code', 'zipcode')]), + ), + ] diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py index 30068f62..1f3c6be2 100644 --- a/passerelle/apps/base_adresse/models.py +++ b/passerelle/apps/base_adresse/models.py @@ -1,9 +1,9 @@ import bz2 import json -import os import urlparse import unicodedata +from requests import RequestException from django.db import connection, models from django.db.models import Q @@ -15,6 +15,8 @@ from django.utils.six.moves.urllib.parse import urljoin from passerelle.base.models import BaseResource from passerelle.utils.api import endpoint +from passerelle.utils.conversion import simplify +from passerelle.utils.jsonresponse import APIError class BaseAdresse(BaseResource): @@ -24,16 +26,23 @@ class BaseAdresse(BaseResource): verbose_name=_('Service URL'), help_text=_('Base Adresse Web Service URL')) + api_geo_url = models.CharField( + max_length=128, blank=False, + default='https://geo.api.gouv.fr/', + verbose_name=_('API Geo URL'), + help_text=_('Base Adresse API Geo URL')) + category = _('Geographic information system') - api_description = _("The API is a partial view of OpenStreetMap's Nominatim " - "own API; it currently doesn't support all parameters and " - "is limited to the JSON format.") + api_description = _("The geocoding endpoints are a partial view of OpenStreetMap's " + "Nominatim own API; it currently doesn't support all parameters and " + "is limited to the JSON format. The cities, departments and regions " + "endpoints source data from French API Geo.") zipcode = models.CharField( max_length=600, blank=True, - verbose_name=_('Postal codes or county number to get streets, separated with commas')) + verbose_name=_('Postal codes or department number to get streets, separated with commas')) class Meta: verbose_name = _('Base Adresse Web Service') @@ -135,8 +144,7 @@ class BaseAdresse(BaseResource): else: streets = StreetModel.objects.all() if q: - unaccented_q = unicodedata.normalize('NFKD', q).encode('ascii', 'ignore').lower() - streets = streets.filter(unaccent_name__icontains=unaccented_q) + streets = streets.filter(unaccent_name__icontains=simplify(q)) if zipcode: streets = streets.filter(zipcode__startswith=zipcode) @@ -158,6 +166,82 @@ class BaseAdresse(BaseResource): return {'data': result} + @endpoint(description=_('Cities list'), + parameters={ + 'id': {'description': _('Get exactly one city using its code and postal code ' + 'separated with a dot'), + 'example_value': '75056.75014'}, + 'q': {'description': _("Search text in name or postal code"), + 'example_value': 'Paris'}, + 'code': {'description': _('INSEE code'), 'example_value': '75056'}, + 'region_code': {'description': _('Region code'), 'example_value': '11'}, + 'department_code': {'description': _('Department code'), 'example_value': '75'}, + }) + def cities(self, request, id=None, q=None, code=None, region_code=None, + department_code=None): + cities = CityModel.objects.all() + + if id is not None: + try: + code, zipcode = id.split('.') + except ValueError: + raise APIError('Invalid id') + cities = cities.filter(code=code, zipcode=zipcode) + if q: + unaccented_q = simplify(q) + cities = cities.filter(Q(unaccent_name__istartswith=unaccented_q) | + Q(zipcode__istartswith=unaccented_q)) + if code: + cities = cities.filter(code=code) + if region_code: + cities = cities.filter(region__code=region_code) + if department_code: + cities = cities.filter(department__code=department_code) + + cities = cities.select_related('department', 'region') + return {'data': [city.to_json() for city in cities]} + + @endpoint(description=_('Departments list'), + parameters={ + 'id': {'description': _('Get exactly one department using its code'), + 'example_value': '59'}, + 'q': {'description': _('Search text in name or code'), 'example_value': 'Nord'}, + 'region_code': {'description': _('Region code'), 'example_value': '32'}, + }) + def departments(self, request, id=None, q=None, region_code=None): + departments = DepartmentModel.objects.all() + + if id is not None: + departments = departments.filter(code=id) + if q: + unaccented_q = simplify(q) + departments = departments.filter(Q(unaccent_name__istartswith=unaccented_q) | + Q(code__istartswith=unaccented_q)) + if region_code: + departments = departments.filter(region__code=region_code) + + departments = departments.select_related('region') + return {'data': [department.to_json() for department in departments]} + + @endpoint(description=_('Regions list'), + parameters={ + 'id': {'description': _('Get exactly one region using its code'), + 'example_value': '32'}, + 'q': {'description': _('Search text in name or code'), + 'example_value': 'Hauts-de-France'}, + }) + def regions(self, request, id=None, q=None): + regions = RegionModel.objects.all() + + if id is not None: + regions = regions.filter(code=id) + if q: + unaccented_q = simplify(q) + regions = regions.filter(Q(unaccent_name__istartswith=unaccented_q) | + Q(code__istartswith=unaccented_q)) + + return {'data': [region.to_json() for region in regions]} + def check_status(self): if self.service_url == 'https://api-adresse.data.gouv.fr/': result = self.search(None, '169 rue du chateau, paris') @@ -174,6 +258,9 @@ class BaseAdresse(BaseResource): criteria |= Q(zipcode__startswith=zipcode) return StreetModel.objects.filter(criteria) + def cities_exist(self): + return CityModel.objects.exists() + def update_streets_data(self): if not self.get_zipcodes(): return @@ -218,20 +305,91 @@ class BaseAdresse(BaseResource): self.get_streets_queryset().filter(last_update__lt=start_update).delete() + def get_api_geo_endpoint(self, endpoint): + if not self.api_geo_url: + return + error = None + try: + response = self.requests.get(urljoin(self.api_geo_url, endpoint)) + except RequestException as e: + error = e + else: + if response.status_code != 200: + error = 'bad status code (%s)' % response.status_code + else: + try: + result = response.json() + except ValueError: + error = 'invalid json, got: %s' % response.text + if error: + self.logger.error('failed to update api geo data for endpoint %s: %s', + endpoint, error) + return + if not result: + raise Exception('api geo returns empty json') + return result + + def update_api_geo_data(self): + regions_json = self.get_api_geo_endpoint('regions') + departments_json = self.get_api_geo_endpoint('departements') + cities_json = self.get_api_geo_endpoint('communes') + if not (regions_json and departments_json and cities_json): + return + start_update = timezone.now() + + for data in regions_json: + defaults = { + 'name': data['nom'], + } + RegionModel.objects.update_or_create(code=data['code'], defaults=defaults) + RegionModel.objects.filter(last_update__lt=start_update).delete() + + for data in departments_json: + defaults = { + 'name': data['nom'], + 'region': RegionModel.objects.get(code=data['codeRegion']), + } + DepartmentModel.objects.update_or_create(code=data['code'], defaults=defaults) + DepartmentModel.objects.filter(last_update__lt=start_update).delete() + + for data in cities_json: + for zipcode in data['codesPostaux']: + defaults = { + 'name': data['nom'], + 'population': data.get('population', 0), + } + if data.get('codeDepartement'): + defaults['department'] = DepartmentModel.objects.get(code=data['codeDepartement']) + if data.get('codeRegion'): + defaults['region'] = RegionModel.objects.get(code=data['codeRegion']) + CityModel.objects.update_or_create( + code=data['code'], zipcode=zipcode, defaults=defaults) + CityModel.objects.filter(last_update__lt=start_update).delete() + def hourly(self): super(BaseAdresse, self).hourly() + # don't wait for daily job to grab data if self.get_zipcodes() and not self.get_streets_queryset().exists(): - # don't wait for daily job to grab streets self.update_streets_data() + if not CityModel.objects.exists(): + self.update_api_geo_data() def daily(self): super(BaseAdresse, self).daily() self.update_streets_data() + self.update_api_geo_data() -class StreetModel(models.Model): +class UnaccentNameMixin(object): - city = models.CharField(_('City'), max_length=100) + def save(self, *args, **kwargs): + self.unaccent_name = unicodedata.normalize('NFKD', self.name).encode('ascii', 'ignore').lower() + super(UnaccentNameMixin, self).save(*args, **kwargs) + + +class StreetModel(UnaccentNameMixin, models.Model): + + city = models.CharField(_('City'), max_length=150) name = models.CharField(_('Street name'), max_length=150) unaccent_name = models.CharField(_('Street name ascii char'), max_length=150, null=True) zipcode = models.CharField(_('Postal code'), max_length=5) @@ -245,6 +403,86 @@ class StreetModel(models.Model): def __unicode__(self): return self.name - def save(self, *args, **kwargs): - self.unaccent_name = unicodedata.normalize('NFKD', self.name).encode('ascii', 'ignore') - super(StreetModel, self).save(*args, **kwargs) + +@six.python_2_unicode_compatible +class RegionModel(UnaccentNameMixin, models.Model): + + name = models.CharField(_('Region name'), max_length=150) + unaccent_name = models.CharField(_('Region name ascii char'), max_length=150, null=True) + code = models.CharField(_('Region code'), max_length=2, unique=True) + last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True) + + def to_json(self): + return { + 'text': str(self), + 'id': self.code, + 'code': self.code, + 'name': self.name, + } + + class Meta: + ordering = ['code'] + + def __str__(self): + return '%s %s' % (self.code, self.name) + + +@six.python_2_unicode_compatible +class DepartmentModel(UnaccentNameMixin, models.Model): + + name = models.CharField(_('Department name'), max_length=100) + unaccent_name = models.CharField(_('Department name ascii char'), max_length=150, null=True) + code = models.CharField(_('Department code'), max_length=3, unique=True) + region = models.ForeignKey(RegionModel, on_delete=models.CASCADE) + last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True) + + def to_json(self): + return { + 'text': str(self), + 'id': self.code, + 'code': self.code, + 'name': self.name, + 'region_code': self.region.code, + 'region_name': self.region.name, + } + + class Meta: + ordering = ['code'] + + def __str__(self): + return '%s %s' % (self.code, self.name) + + +@six.python_2_unicode_compatible +class CityModel(UnaccentNameMixin, models.Model): + + name = models.CharField(_('City name'), max_length=150) + unaccent_name = models.CharField(_('City name ascii char'), max_length=150, null=True) + code = models.CharField(_('INSEE code'), max_length=5) + zipcode = models.CharField(_('Postal code'), max_length=5) + population = models.PositiveIntegerField(_('Population')) + department = models.ForeignKey(DepartmentModel, on_delete=models.CASCADE, blank=True, null=True) + region = models.ForeignKey(RegionModel, on_delete=models.CASCADE, blank=True, null=True) + last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True) + + def to_json(self): + data = { + 'text': str(self), + 'id': '%s.%s' % (self.code, self.zipcode), + 'code': self.code, + 'name': self.name, + 'zipcode': self.zipcode, + 'population': self.population, + 'department_code': self.department.code if self.department else None, + 'department_name': self.department.name if self.department else None, + 'region_code': self.region.code if self.region else None, + 'region_name': self.region.name if self.region else None, + } + return data + + class Meta: + ordering = ['-population', 'zipcode', 'unaccent_name', 'name'] + unique_together = ('code', 'zipcode') + + def __str__(self): + return '%s %s' % (self.zipcode, self.name) diff --git a/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html b/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html index 1a36bfda..8b955d89 100644 --- a/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html +++ b/passerelle/apps/base_adresse/templates/base_adresse/baseadresse_detail.html @@ -8,6 +8,11 @@ {% trans "Street data is not available yet, it should soon be downloaded." %} {% endif %} +{% if object.api_geo_url and not object.cities_exist %} +