mdel: intercept exceptions during response retrieval (#86379)

passerelle/apps/mdel/models.py

@@ -181,6 +181,7 @@ class MDEL(BaseResource):
                     continue
                 with atomic_write(os.path.join(self.output_dir, filename)) as fd:
                     client.getfo(filename, fd)
+                yield filename
 
     def hourly(self):
         super().hourly()
@@ -188,7 +189,14 @@ class MDEL(BaseResource):
             for demand in self.demand_set.filter(sent=False):
                 demand.send()
         if self.incoming_sftp:
-            self.get_response_files()
+            filenames = []
+            try:
+                for filename in self.get_response_files():
+                    filenames.append(filename)
+            except Exception as e:
+                self.logger.warning('Could not retrieve all responses: %s', e, exc_info=True)
+            if filenames:
+                self.logger.info('Retrieved new responses: %s', ' ,'.join(filenames))
 
     def daily(self):
         super().daily()

tests/test_mdel.py

@@ -340,14 +340,14 @@ def test_create_aec_demand_with_output_sftp(app, setup, aec_payload, sftpserver,
     assert caplog.messages[-1:] == [f'Demand {demand.name} (pk {demand.num}) sent.']
 
 
-def test_create_aec_demand_with_input_sftp(app, setup, aec_payload, sftpserver):
+def test_create_aec_demand_with_input_sftp(app, setup, aec_payload, sftpserver, caplog):
     setup.incoming_sftp = SFTP('sftp://foo:bar@{server.host}:{server.port}/input/'.format(server=sftpserver))
     setup.save()
-    resp = app.post_json('/mdel/test/create', params=aec_payload, status=200)
-    expected_filename = '%s-1.zip' % resp.json['data']['demand_id']
-    with sftpserver.serve_content({'input': {expected_filename: 'content'}}):
+
+    with sftpserver.serve_content({'input': {'whatever': 'content'}}):
         setup.hourly()
-    assert expected_filename in os.listdir(setup.output_dir)
+    assert (pathlib.Path(setup.output_dir) / 'whatever').read_text() == 'content'
+    assert caplog.messages[-1:] == ['Retrieved new responses: whatever']
 
 
 def test_create_aec_demand_type_without_date_acte(app, setup):
@@ -744,3 +744,14 @@ def test_create_aec_demand_with_output_sftp_error(app, setup, aec_payload, sftps
     demand = setup.demand_set.get()
     assert not demand.sent
     assert resource_logs(caplog) == ['Demand %s (pk %s) could not be sent: %s']
+
+
+def test_create_aec_demand_with_input_sftp_error(app, setup, aec_payload, sftpserver, caplog):
+    caplog.set_level('WARNING')
+    setup.incoming_sftp = SFTP('sftp://foo:bar@{server.host}:{server.port}/input/'.format(server=sftpserver))
+    setup.save()
+
+    app.post_json('/mdel/test/create', params=aec_payload, status=200)
+    setup.hourly()
+
+    assert resource_logs(caplog) == ['Could not retrieve all responses: %s']