rewrite the code to get modules and versions of a service (#64667)

This page was iterating on Python side, and was spending a huge amount
of time sending tiny queries and building objects from the results.
Instead, use a single query per service/platform, and fetch the version
at the same time.

Benefits: we divide by more than 100 the number of queries of the project
page, and we speed it up by a factor 10 or more...

scrutiny/projects/models.py

@@ -37,27 +37,42 @@ class Service(models.Model):
         return '%s / %s' % (self.project.title, self.title)
 
     def get_modules(self, platforms=None):
-        modules = {}
+        return [x[0] for x in self.get_modules_with_version(platforms)]
+
+    def get_modules_with_version(self, platforms=None):
+        modules = {}  # name => (module, {platform.id: version})
         if platforms is None:
             platforms = self.project.platform_set.all()
+
         for platform in platforms:
-            try:
-                installed_service = InstalledService.objects.get(service=self, platform=platform)
-            except InstalledService.DoesNotExist:
-                continue
-            seen_modules = []
-            uninstalled_modules = []
-            for installed_version in InstalledVersion.objects.filter(service=installed_service).order_by(
-                '-timestamp'
-            ):
-                if installed_version.version.module in seen_modules:
-                    continue
-                seen_modules.append(installed_version.version.module)
-                if not installed_version.version.version:
-                    uninstalled_modules.append(installed_version.version.module)
-                    continue
-                modules[installed_version.version.module.name] = installed_version.version.module
-        return sorted(modules.values(), key=lambda x: x.name)
+            platform_modules = Module.objects.raw(
+                """
+                WITH all_modules AS (
+                    SELECT
+                        pm.id,
+                        pm.name,
+                        (SELECT
+                            version
+                        FROM projects_version pv
+                        JOIN projects_installedversion piv on piv.version_id = pv.id
+                        JOIN projects_installedservice pis on pis.id = piv.service_id and pis.platform_id = %s and pis.service_id = %s
+                        WHERE pv.module_id = pm.id
+                        ORDER BY timestamp desc LIMIT 1) as version
+                    FROM projects_module pm
+                )
+                SELECT id, name, version AS platform_service_version
+                FROM all_modules
+                WHERE version <> '';
+                """,
+                [platform.id, self.id],
+            )
+            for module in platform_modules:
+                if not (module.name in modules):
+                    modules[module.name] = (module, {platform.id: module.platform_service_version})
+                else:
+                    modules[module.name][1][platform.id] = module.platform_service_version
+
+        return sorted(modules.values(), key=lambda x: x[0].name)
 
     def get_installed_service(self, platform):
         try:

scrutiny/projects/templates/projects/project_detail.html

@@ -24,11 +24,11 @@
                       class="icon-external-link"></a></td>
       {% endfor %}
     </tr>
-    {% for module in service.get_modules %}
+    {% for module in service.get_modules_with_version %}
     <tr class="module">
-      <th>{{ module.name }}</th>
+      <th>{{ module.0.name }}</th>
       {% for platform in platforms %}
-      <td class="version">{% installed_version module=module platform=platform service=service %}</td>
+      <td class="version">{{ module.1|get:platform.id|default:"" }}</td>
       {% endfor %}
     </tr>
     {% endfor %}

scrutiny/projects/templatetags/tags.py

@@ -11,6 +11,17 @@ def installed_version(module, platform, service):
     return v.version.version
 
 
+@register.filter(name='get')
+def get(obj, key):
+    try:
+        return obj.get(key)
+    except AttributeError:
+        try:
+            return obj[key]
+        except (IndexError, KeyError, TypeError):
+            return None
+
+
 @register.simple_tag
 def service_url(platform, service):
     installed_service = service.get_installed_service(platform=platform)