sp_fr: accepts 200 *new* requests per run (#32852)

passerelle/apps/sp_fr/models.py

@@ -43,6 +43,7 @@ from passerelle.utils.xml import text_content
 
 from .xsd import Schema
 
+MAX_REQUESTS_PER_ITERATION = 200
 
 PROCEDURE_DOC = 'DOC'
 PROCEDURE_RCO = 'recensementCitoyen'
@@ -114,7 +115,9 @@ class Resource(BaseResource):
     def hourly(self):
         self.run_loop()
 
-    def run_loop(self, count=1):
+    def run_loop(self, count=0):
+        if count == 0:
+            count = MAX_REQUESTS_PER_ITERATION
         with transaction.atomic():
             # lock resource
             r = Resource.objects.select_for_update(skip_locked=True).filter(pk=self.pk)
@@ -139,7 +142,7 @@ class Resource(BaseResource):
                             continue
                         yield file_stat.filename
 
-                for filename, i in zip(helper(), range(count)):
+                for filename in helper():
                     m = FILE_PATTERN.match(filename)
                     if not m:
                         self.logger.info('file "%s" did not match pattern %s, moving to FAILED/',
@@ -162,6 +165,8 @@ class Resource(BaseResource):
                         procedure=procedure,
                         sequence=m.group('sequence'),
                         mapping=mapping)
+                    if not handler.request:
+                        count -= 1
                     try:
                         move, error = handler()
                     except Exception:
@@ -176,6 +181,8 @@ class Resource(BaseResource):
                                 self.logger.warning('handling of file "%s" failed: %s', filename, error)
                             elif move:
                                 sftp.rename(filename, 'DONE/' + filename)
+                    if not count:
+                        break
 
     class FileHandler(object):
         def __init__(self, resource, sftp, filename, identifier, procedure, sequence, mapping):

passerelle/apps/sp_fr/views.py

@@ -63,5 +63,5 @@ class MappingDelete(ResourceChildViewMixin, DeleteView):
 
 def run(request, connector, slug):
     resource = get_object_or_404(models.Resource, slug=slug)
-    resource.run_loop(10)
+    resource.run_loop()
     return HttpResponseRedirect(resource.get_absolute_url())