workflows: retry PDF generation if it fails for some unknown reason (#44787)

wcs/wf/export_to_model.py

@@ -18,9 +18,11 @@ import base64
 import collections
 from xml.etree import ElementTree as ET
 import zipfile
+import os
 import random
 import subprocess
 import tempfile
+import time
 import shutil
 
 from django.utils import six
@@ -76,10 +78,19 @@ try:
                         break
                     infile.write(chunk)
                 infile.flush()
-                subprocess.check_call(['libreoffice', '--headless', '--convert-to', 'pdf',
-                                       infile.name, '--outdir', temp_dir],
-                                      stdout=subprocess.DEVNULL,
-                                      stderr=subprocess.DEVNULL)
+                for i in range(3):
+                    lo_output = subprocess.run(
+                            ['libreoffice', '--headless', '--convert-to', 'pdf',
+                                infile.name, '--outdir', temp_dir],
+                            check=True, capture_output=True)
+                    if os.path.exists(infile.name + '.pdf'):
+                        break
+                    # sometimes libreoffice fails and sometimes it's ok
+                    # afterwards.
+                    time.sleep(0.5)
+                if not os.path.exists(infile.name + '.pdf'):
+                    raise Exception('libreoffice failed to produce pdf (stdout: %r, stderr: %r)' % (
+                        lo_output.stdout, lo_output.stderr))
             return open(infile.name + '.pdf', 'rb')
         except subprocess.CalledProcessError:
             raise Exception('libreoffice is failing')