base adresse: adapt to postcode and city now being a list in streets dumps (#42608)

passerelle/apps/base_adresse/models.py

@@ -1,5 +1,5 @@
-import bz2
 import datetime
+import gzip
 
 from jsonfield import JSONField
 from requests import RequestException
@@ -10,6 +10,7 @@ from django.utils.http import urlencode
 from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
 from django.utils import six
+from django.utils.six import StringIO
 from django.utils.six.moves.urllib import parse as urlparse
 
 from passerelle.base.models import BaseResource
@@ -354,27 +355,32 @@ class BaseAdresse(BaseResource):
                 departments.add(zipcode[:2])
 
         for department in departments:
-            ban_file = self.requests.get(
-                'http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_{}-json.bz2'.format(department))
-            if ban_file.status_code != 200:
+            ban_gz = self.requests.get(
+                'https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-{}.ndjson.gz'.format(department))
+            if ban_gz.status_code != 200:
                 continue
 
+            if six.PY3:
+                ban_file = StringIO(gzip.decompress(ban_gz.content).decode('utf-8'))
+            else:
+                ban_file = gzip.GzipFile(fileobj=StringIO(ban_gz.content))
             line = _not_found = object()
-            for line in bz2.decompress(ban_file.content).splitlines():
+            for line in ban_file:
                 street_info = json_loads(line)
+                if isinstance(street_info['postcode'], list):
+                    street_info['postcode'] = six.text_type(street_info['postcode'][0])
                 if street_info['type'] == 'street' and street_info['postcode'].startswith(zipcodes):
-                    if type(street_info['citycode']) is list:
-                        street_info['citycode'] = six.text_type(street_info['citycode'][0])
-                    if type(street_info['name']) is list:
-                        street_info['name'] = six.text_type(street_info['name'][0])
-                    street = StreetModel.objects.get_or_create(citycode=street_info['citycode'],
-                                                               name=street_info['name'][:150])
-                    street[0].city = street_info['city']
-                    street[0].name = street_info['name'][:150]
-                    street[0].zipcode = street_info['postcode']
-                    street[0].type = street_info['type']
-                    street[0].citycode = street_info['citycode']
-                    street[0].save()
+                    for key in ('citycode', 'name', 'city'):
+                        if isinstance(street_info[key], list):
+                            street_info[key] = six.text_type(street_info[key][0])
+                    StreetModel.objects.update_or_create(
+                        citycode=street_info['citycode'],
+                        name=street_info['name'][:150],
+                        defaults={
+                            'city': street_info['city'],
+                            'zipcode': street_info['postcode'],
+                            'type': street_info['type'],
+                        })
             if line is _not_found:
                 raise Exception('bano file is empty')
 

tests/test_base_adresse.py

@@ -348,13 +348,15 @@ def test_base_adresse_streets_get_by_codes(app, base_adresse, street):
     assert resp.json['err'] == 0
     assert len(resp.json['data']) == 0
 
+
 @pytest.mark.usefixtures('mock_update_api_geo')
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update(mocked_get, db, base_adresse):
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
-    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')
+    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-73.ndjson.gz')
     streets = StreetModel.objects.all()
     assert len(streets) == 3
     street = StreetModel.objects.order_by('id').first()
@@ -372,11 +374,12 @@ def test_base_adresse_command_update(mocked_get, db, base_adresse):
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_hourly_update(mocked_get, db, base_adresse):
     base_adresse.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     # check the first hourly job downloads streets
     call_command('cron', 'hourly')
-    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')
+    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-73.ndjson.gz')
     assert StreetModel.objects.all().count() == 3
     # check a second call doesn't download anything
     call_command('cron', 'hourly')
@@ -387,10 +390,11 @@ def test_base_adresse_command_hourly_update(mocked_get, db, base_adresse):
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_97x(mocked_get, db, base_adresse_97x):
     base_adresse_97x.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
-    mocked_get.assert_called_once_with('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_974-json.bz2')
+    mocked_get.assert_called_once_with('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-974.ndjson.gz')
     assert StreetModel.objects.count() == 2
 
 
@@ -398,12 +402,13 @@ def test_base_adresse_command_update_97x(mocked_get, db, base_adresse_97x):
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_corsica(mocked_get, db, base_adresse_corsica):
     base_adresse_corsica.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
     assert mocked_get.call_count == 2
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2A-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2B-json.bz2')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2A.ndjson.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2B.ndjson.gz')
     assert StreetModel.objects.count() == 0
 
 
@@ -411,14 +416,15 @@ def test_base_adresse_command_update_corsica(mocked_get, db, base_adresse_corsic
 @mock.patch('passerelle.utils.Request.get')
 def test_base_adresse_command_update_multiple(mocked_get, db, base_adresse_multiple):
     base_adresse_multiple.update_api_geo_data = lambda: None
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
     call_command('cron', 'daily')
     assert mocked_get.call_count == 4
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_73-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_974-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2A-json.bz2')
-    mocked_get.assert_any_call('http://bano.openstreetmap.fr/BAN_odbl/BAN_odbl_2B-json.bz2')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-73.ndjson.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-974.ndjson.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2A.ndjson.gz')
+    mocked_get.assert_any_call('https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-2B.ndjson.gz')
     assert StreetModel.objects.count() == 5
 
 

tests/test_jobs.py

@@ -18,8 +18,9 @@ from test_base_adresse import base_adresse, StreetModel
 
 @mock.patch('passerelle.utils.Request.get')
 def test_jobs(mocked_get, app, base_adresse, freezer):
-    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.bz2')
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'update_streets_test.gz')
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
 
     freezer.move_to('2019-01-01 00:00:00')
     job = base_adresse.add_job('update_streets_data')
@@ -30,7 +31,7 @@ def test_jobs(mocked_get, app, base_adresse, freezer):
     assert StreetModel.objects.count() == 3
 
     # don't delete streets if bano file is empty
-    mocked_get.return_value = utils.FakedResponse(content='', status_code=200)
+    mocked_get.return_value = utils.FakedResponse(content=b'', status_code=200)
     freezer.move_to('2019-01-01 12:00:00')
     job = base_adresse.add_job('update_streets_data')
     assert job.status == 'registered'
@@ -43,7 +44,8 @@ def test_jobs(mocked_get, app, base_adresse, freezer):
         'error running update_streets_data job (bano file is empty)')
     assert StreetModel.objects.count() == 3
 
-    mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+    with open(filepath, 'rb') as ban_file:
+        mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
 
     StreetModel.objects.all().delete()
 
@@ -133,7 +135,8 @@ def test_jobs(mocked_get, app, base_adresse, freezer):
     StreetModel.objects.all().delete()
     with mock.patch('passerelle.apps.base_adresse.models.BaseAdresse.down') as down:
         down.side_effect = lambda: True
-        mocked_get.return_value = utils.FakedResponse(content=open(filepath, 'rb').read(), status_code=200)
+        with open(filepath, 'rb') as ban_file:
+            mocked_get.return_value = utils.FakedResponse(content=ban_file.read(), status_code=200)
         job = base_adresse.add_job('update_streets_data')
         assert job.status == 'registered'