misc: add basic interface to view connector jobs (#36186)

passerelle/base/models.py

@@ -484,6 +484,12 @@ class BaseResource(models.Model):
     def monthly(self):
         pass
 
+    def jobs_set(self):
+        resource_type = ContentType.objects.get_for_model(self)
+        return Job.objects.filter(
+            resource_type=resource_type,
+            resource_pk=self.pk)
+
     def jobs(self):
         # "jobs" cron job to run asynchronous tasks
         if self.down():
@@ -497,11 +503,9 @@ class BaseResource(models.Model):
         while True:
             with transaction.atomic():
                 # lock a job
-                job = Job.objects.exclude(
+                job = self.jobs_set().exclude(
                         pk__in=skipped_jobs
                         ).filter(
-                        resource_type=resource_type,
-                        resource_pk=self.pk,
                         status='registered'
                         ).select_for_update(**skip_locked).first()
                 if not job:

passerelle/base/templatetags/passerelle.py

@@ -51,6 +51,25 @@ def resource_logs_table(context, resource):
     return context
 
 
+@register.inclusion_tag('passerelle/includes/resource-jobs-table.html', takes_context=True)
+def resource_jobs_table(context, resource):
+    request = context.get('request')
+    page = request.GET.get('page', 1)
+
+    qs = resource.jobs_set().order_by('-creation_timestamp')
+
+    paginator = Paginator(qs, 10)
+    try:
+        jobs = paginator.page(page)
+    except PageNotAnInteger:
+        jobs = paginator.page(1)
+    except (EmptyPage,):
+        jobs = paginator.page(paginator.num_pages)
+
+    context['jobs'] = jobs
+    return context
+
+
 @register.filter
 def can_edit(obj, user):
     return user.has_perm(get_permission_codename('change', obj._meta), obj=obj)

passerelle/base/views.py

@@ -14,18 +14,25 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
+
+from dateutil import parser as date_parser
+
 from django.contrib.contenttypes.models import ContentType
 from django.contrib import messages
 from django.core.urlresolvers import reverse
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
+from django.db.models import Q
 from django.forms import models as model_forms
 from django.views.generic import (
     DetailView, ListView, CreateView, UpdateView, DeleteView, FormView)
 from django.http import Http404
+from django.utils.timezone import make_aware
 from django.utils.translation import ugettext_lazy as _
 
-from .models import ApiUser, AccessRight, LoggingParameters, ResourceStatus
+from .models import ApiUser, AccessRight, LoggingParameters, ResourceStatus, Job
 from .forms import ApiUserForm, AccessRightForm, AvailabilityParametersForm
+from ..views import GenericConnectorMixin
 from ..utils import get_trusted_services
 
 
@@ -202,3 +209,66 @@ class ManageAvailabilityView(UpdateView):
                 resource.availability()
 
         return super(ManageAvailabilityView, self).form_valid(form)
+
+
+class GenericViewJobsConnectorView(GenericConnectorMixin, ListView):
+    template_name = 'passerelle/manage/service_jobs.html'
+    paginate_by = 25
+
+    def get_context_data(self, **kwargs):
+        context = super(GenericViewJobsConnectorView, self).get_context_data(**kwargs)
+        context['object'] = self.get_object()
+        context['query'] = self.request.GET.get('q') or ''
+        return context
+
+    def get_object(self):
+        return self.model.objects.get(slug=self.kwargs['slug'])
+
+    def get_queryset(self):
+        connector = self.get_object()
+        resource_type = ContentType.objects.get_for_model(connector)
+        qs = connector.jobs_set().order_by('-creation_timestamp')
+        query = self.request.GET.get('q')
+        if query:
+            try:
+                date = date_parser.parse(query, dayfirst=True)
+            except Exception:
+                qs = qs.filter(method_name__icontains=query)
+            else:
+                date = make_aware(date)
+                if date.hour == 0 and date.minute == 0 and date.second == 0:
+                    # just a date: display all jobs for that date
+                    max_date = date + datetime.timedelta(days=1)
+                    qs = qs.filter(Q(creation_timestamp__gte=date,
+                                     creation_timestamp__lte=date + datetime.timedelta(days=1)) |
+                                   Q(update_timestamp__gte=date,
+                                     update_timestamp__lte=date + datetime.timedelta(days=1)) |
+                                   Q(done_timestamp__gte=date,
+                                     done_timestamp__lte=date + datetime.timedelta(days=1)))
+                elif date.second == 0:
+                    # without seconds: display all jobs in this minute
+                    max_date = date + datetime.timedelta(seconds=60)
+                else:
+                    # display all jobs in the same second
+                    max_date = date + datetime.timedelta(seconds=1)
+
+                qs = qs.filter(Q(creation_timestamp__gte=date,
+                                 creation_timestamp__lte=max_date) |
+                               Q(update_timestamp__gte=date,
+                                 update_timestamp__lte=max_date) |
+                               Q(update_timestamp__gte=date,
+                                 update_timestamp__lte=max_date))
+
+        return qs
+
+
+class GenericJobView(GenericConnectorMixin, DetailView):
+    template_name = 'passerelle/manage/job.html'
+
+    def get_context_data(self, **kwargs):
+        context = super(GenericJobView, self).get_context_data(**kwargs)
+        try:
+            context['job'] = Job.objects.get(pk=self.kwargs['job_pk'])
+        except Job.DoesNotExist:
+            raise Http404()
+        return context

passerelle/static/css/style.css

@@ -39,12 +39,14 @@ div#logs table tr.level-debug {
 	color: #666;
 }
 
+div#jobs table tr.error,
 div#logs table tr.level-warning,
 div#logs table tr.level-error,
 div#logs table tr.level-critical {
 	color: #c33;
 }
 
+div#jobs table tr.error,
 div#logs table tr.level-error,
 div#logs table tr.level-critical {
 	font-weight: bold;

passerelle/static/js/passerelle.js

@@ -8,4 +8,13 @@ $(function() {
           var $dialog = $(response).dialog({modal: true, width: 'auto'});
         });
   });
+  $('#jobs tbody tr').on('click', function() {
+    var base_url = $(this).parents('table.main').data('job-base-url');
+    var job_pk = $(this).data('pk');
+    var url = base_url + job_pk + '/';
+    $.get(url,
+        function(response) {
+          var $dialog = $(response).dialog({modal: true, width: 'auto'});
+        });
+  });
 });

passerelle/templates/passerelle/includes/resource-jobs-table.html

@@ -0,0 +1,30 @@
+{% load i18n passerelle %}
+{% load tz %}
+
+{% if jobs %}
+<table class="main" data-job-base-url="{% url 'view-jobs-connector' connector=object.get_connector_slug slug=object.slug %}" >
+    <thead>
+        <th>{% trans 'Creation Timestamp' %}</th>
+        <th>{% trans 'Update Timestamp' %}</th>
+        <th>{% trans 'Done Timestamp' %}</th>
+        <th>{% trans 'Type' %}</th>
+        <th>{% trans 'Status' %}</th>
+    </thead>
+    <tbody>
+    {% for job in jobs %}
+    <tr data-pk="{{ job.pk }}" {% if job.status == 'failed' %}class="error"{% endif %}>
+        <td class="timestamp">{{ job.creation_timestamp|localtime }}</td>
+        <td class="timestamp">{{ job.update_timestamp|localtime }}</td>
+        <td class="timestamp">{{ job.done_timestamp|localtime }}</td>
+        <td>{{ job.method_name }}</td>
+        <td class="message">{{ job.get_status_display }}</td>
+    </tr>
+    {% endfor %}
+    </tbody>
+</table>
+
+{% with page_obj=logrecords %}
+  {% include "gadjo/pagination.html" with anchor="#jobs" %}
+{% endwith %}
+
+{% endif %}

passerelle/templates/passerelle/manage/job.html

@@ -0,0 +1,19 @@
+{% load i18n %}
+<div class="job-dialog">
+<table>
+<tr>
+  <td>{% trans "Parameters" %}</td>
+  <td>{{ job.parameters }}</td>
+</tr>
+{% for key, value in job.status_details.items %}
+<tr>
+  <td>{{key}}</td>
+  <td>{% for key2, value2 in value.items %}
+          {{key2}}: {{value2}}<br>
+      {% empty %}
+      {{value|linebreaksbr}}
+      {% endfor %}</td>
+</tr>
+{% endfor %}
+</table>
+</div>

passerelle/templates/passerelle/manage/service_jobs.html

@@ -0,0 +1,27 @@
+{% extends "passerelle/manage.html" %}
+{% load i18n passerelle %}
+
+{% block breadcrumb %}
+{{ block.super }}
+<a href="{{object.get_absolute_url}}">{{ object.title }}</a>
+<a href="#">{% trans "Jobs" %}</a>
+{% endblock %}
+
+{% block appbar %}
+<h2>{% trans "Jobs" %}</h2>
+{% endblock %}
+
+{% block content %}
+
+<div id="jobs">
+
+<form>
+  <p><input name="q" type="search" value="{{query}}"> <button>{% trans 'Search' %}</button>
+  <span class="help_text">{% trans "(supports text search in job name or dates)" %}</span>
+  </p>
+</form>
+
+{% include "passerelle/includes/resource-jobs-table.html" with jobs=page_obj %}
+</div>
+
+{% endblock %}

passerelle/templates/passerelle/manage/service_view.html

@@ -109,4 +109,17 @@
 </div>
 {% endif %}
 
+{% if perms.base.view_job and object.jobs_set.all %}
+<div id="jobs" class="section">
+  <h3>{% trans "Jobs" %}
+          <a href="{% url 'view-jobs-connector' connector=object.get_connector_slug slug=object.slug %}">({% trans "full page & filter" %})</a>
+  </h3>
+  <div>
+  {% block jobs %}
+    {% resource_jobs_table resource=object %}
+  {% endblock %}
+  </div>
+</div>
+{% endif %}
+
 {% endblock %}

passerelle/urls.py

@@ -11,6 +11,7 @@ from .views import (HomePageView, ManageView, ManageAddView,
         GenericEditConnectorView, GenericEndpointView, GenericConnectorView,
         GenericViewLogsConnectorView, GenericLogView,
         login, logout, menu_json)
+from .base.views import GenericViewJobsConnectorView, GenericJobView
 from .urls_utils import decorated_includes, required, app_enabled, manager_required
 from .base.urls import access_urlpatterns
 from .plugins import register_apps_urls
@@ -68,6 +69,10 @@ urlpatterns += [
                 GenericViewLogsConnectorView.as_view(), name='view-logs-connector'),
             url(r'^(?P<slug>[\w,-]+)/logs/(?P<log_pk>\d+)/$',
                 GenericLogView.as_view(), name='view-log'),
+            url(r'^(?P<slug>[\w,-]+)/jobs/$',
+                GenericViewJobsConnectorView.as_view(), name='view-jobs-connector'),
+            url(r'^(?P<slug>[\w,-]+)/jobs/(?P<job_pk>\d+)/$',
+                GenericJobView.as_view(), name='view-job'),
         ])))
 ]
 

tests/test_manager.py

@@ -263,3 +263,56 @@ def test_availability_parameters(app, admin_user, monkeypatch):
     resp.form['notification_delays'] = 'x'
     resp = resp.form.submit()
     assert len(resp.pyquery('#id_notification_delays_p .error'))
+
+def test_jobs(app, admin_user):
+    data = StringIO('1;Foo\n2;Bar\n3;Baz')
+    csv = CsvDataSource.objects.create(csv_file=File(data, 't.csv'),
+           columns_keynames='id, text', slug='test', title='a title', description='a description')
+
+    api = ApiUser.objects.create(username='public',
+                    fullname='public',
+                    description='access for all',
+                    keytype='', key='')
+    obj_type = ContentType.objects.get_for_model(csv)
+    AccessRight.objects.create(codename='can_access',
+                    apiuser=api,
+                    resource_type=obj_type,
+                    resource_pk=csv.pk,
+    )
+
+    app = login(app)
+    resp = app.get(csv.get_absolute_url())
+    assert not 'jobs' in resp.text
+
+    csv.add_job('sample_job')
+    resp = app.get(csv.get_absolute_url())
+    assert 'jobs' in resp.text
+    assert 'sample_job' in resp.text
+
+    resp = resp.click('full page', index=1)
+    assert resp.text.count('<tr data-pk') == 1
+    assert resp.text.count('sample_job') == 1
+
+    resp.form['q'] = 'sample'
+    resp = resp.form.submit()
+    assert resp.text.count('<tr data-pk') == 1
+
+    resp.form['q'] = 'blah'
+    resp = resp.form.submit()
+    assert resp.text.count('<tr data-pk') == 0
+
+    resp.form['q'] = datetime.date.today().strftime('%d/%m/%Y')
+    resp = resp.form.submit()
+    assert resp.text.count('<tr data-pk') == 1
+
+    resp.form['q'] = datetime.date.today().strftime('%d/%m/2010')
+    resp = resp.form.submit()
+    assert resp.text.count('<tr data-pk') == 0
+
+    resp.form['q'] = ''
+    resp = resp.form.submit()
+    assert resp.text.count('<tr data-pk') == 1
+    job_pk = re.findall(r'data-pk="(.*)"', resp.text)[0]
+    base_url = re.findall(r'data-job-base-url="(.*)"', resp.text)[0]
+    resp = app.get(base_url + job_pk + '/')
+    resp = app.get(base_url + '12345' + '/', status=404)