passerelle/passerelle/contrib/toulouse_foederis/models.py

779 lines
29 KiB
Python

# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2022 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import requests
from django.core.files.base import ContentFile
from django.db import models, transaction
from django.db.models import JSONField, Q
from django.http import Http404, HttpResponse
from django.urls import reverse
from django.utils.timezone import localtime, now
from django.utils.translation import gettext_lazy as _
from passerelle.base.models import BaseResource, HTTPResource
from passerelle.utils.api import endpoint
from passerelle.utils.json import datasource_array_schema, datasource_schema, response_schema
ATTACHMENT_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'title': _('Attachment data.'),
'description': '',
'required': ['application_id', 'name', 'file'],
'type': 'object',
'properties': {
'application_id': {
'description': _('ID of the application to which to attach the file.'),
'oneOf': [
{
'type': 'string',
'pattern': '^[0-9]+$',
},
{
'type': 'integer',
},
],
},
'name': {
'description': _('Name of the attachment.'),
'type': 'string',
},
'file': {
'description': _('File to attach.'),
'type': 'object',
'required': ['filename', 'content_type', 'content'],
'properties': {
'filename': {
'description': _("File name"),
'type': 'string',
},
'content_type': {
'description': _("MIME type"),
'type': 'string',
},
'content': {
'description': _("Content"),
'type': 'string',
},
},
},
},
}
def boolean_field(description):
return {
'description': description,
'oneOf': [
{'type': 'boolean'},
{
'type': 'string',
'pattern': '[Oo]|[Nn]|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]|1|0',
'pattern_description': _(
'Values "0", "1", "O", "N", "true" or "false" are allowed (case insensitive).'
),
},
],
}
def get_bool(obj):
if obj is True or str(obj).lower() in ['true', 'o', '1']:
return True
if obj is False or str(obj).lower() in ['false', 'n', '0']:
return False
return obj
APPLICATION_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'title': 'Toulouse Foederis application',
'description': '',
'type': 'object',
'properties': {
'type': {
'description': _('Application Type (External or Internal).'),
'type': 'string',
},
'announce_id': {
'description': _('ID of the concerned job offer.'),
'type': 'string',
'pattern': '^[0-9]*$',
},
'civility': {
'description': _("ID of an element of the data source 'civilite'."),
'type': 'string',
'pattern': '^[0-9]+$',
},
'first_name': {
'description': _('Applicant first name.'),
'type': 'string',
},
'last_name': {
'description': _('Applicant last name.'),
'type': 'string',
},
'gender': {
'description': _('Applicant gender.'),
'type': 'string',
'enum': ['H', 'F', ''],
},
'birth_date': {
'description': _('Applicant birth date.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'nationality': {
'description': _("ID of an element of the data source 'nationalite'."),
'type': 'string',
'pattern': '^[0-9]+$',
},
'work_authorization_end_date': {
'description': _("Applicant end of working authorization, if nationality is 'other'."),
'example_value': '2023-04-05',
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'rqth': boolean_field(_('RQTH.')),
'rqth_end_date': {
'description': _('End of RQTH, or none if not applicable.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'driving_license': {
'description': _('Driving license.'),
'type': 'string',
},
'fimo': boolean_field(_('FIMO licence.')),
'fimo_delivrance_date': {
'description': _('FIMO licence delivrance date.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'fimo_end_validity_date': {
'description': _('FIMO licence end validity date.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'current_situation': {
'description': _("ID of an element of the data source 'situation-actuelle'."),
'type': 'string',
'pattern': '^[0-9]+$',
},
'agent_collectivity': {
'description': _("Agent's collectivity"),
'type': 'string',
},
'availability_start_date': {
'description': _('Applicant availability start date.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'availability_end_date': {
'description': _('Applicant availability end date.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'salary_expectations': {
'description': _('Applicant salary expectations.'),
'type': 'string',
},
'address': {
'description': _('Applicant address.'),
'type': 'string',
},
'address_complement': {
'description': _('Applicant address complement.'),
'type': 'string',
},
'zip': {
'description': _('Applicant zip code.'),
'type': 'string',
},
'city': {
'description': _('Applicant city.'),
'type': 'string',
},
'phone': {
'description': _('Applicant phone number.'),
'type': 'string',
},
'email': {
'description': _('Applicant email.'),
'type': 'string',
},
'contract_start_date': {
'description': _('Applicant contract start date.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'contract_end_date': {
'description': _('Applicant contract end date.'),
'type': 'string',
'pattern': '^([0-9]{4}-[0-9]{2}-[0-9]{2})?$',
},
'additional_informations': {
'description': _('Application information complement.'),
'type': 'string',
},
'origin': {
'description': _("ID of an element of the data source 'origine-candidature'."),
'pattern': '^[0-9]*$',
'type': 'string',
},
'origin_precisions': {
'description': _("Precisions if 'origine' is 'other'."),
'type': 'string',
},
'rgpd_agreement': boolean_field(_('RGPD agreement.')),
'job_types': {
'description': _("IDs of elements of the data source 'type-emploi'."),
'type': 'array',
'items': {'type': 'string', 'pattern': '^[0-9]*$'},
},
'job_realms': {
'description': _("IDs of elements of the data source 'domaine-emploi'."),
'type': 'array',
'items': {'type': 'string', 'pattern': '^[0-9]*$'},
},
'job_families': {
'description': _("IDs of elements of the data source 'sous-domaine-emploi'."),
'type': 'array',
'items': {'type': 'string', 'pattern': '^[0-9]*$'},
},
'jobs': {
'description': _("IDs of elements of the data source 'emploi'."),
'type': 'array',
'items': {'type': 'string', 'pattern': '^[0-9]*$'},
},
'desired_work_time': {
'description': _('TC / TNC.'),
'type': 'string',
'enum': ['TC', 'TNC'],
},
'internship_duration': {
'description': _('Duration of the desired internship.'),
'type': 'string',
},
'school_name': {
'description': _("Candidate trainee's school name."),
'type': 'string',
},
'diploma_name': {
'description': _("Candidate trainee's diploma name."),
'type': 'string',
},
'diploma_speciality': {
'description': _("Candidate trainee's diploma speciality."),
'type': 'string',
},
'aimed_diploma_level': {
'description': _("ID of an element of the data source 'niveau-diplome'."),
'type': 'string',
'pattern': '^[0-9]*$',
},
'last_obtained_diploma': {
'description': _("Candidate trainee's last obtained diploma."),
'type': 'string',
},
'last_course_taken': {
'description': _("Candidate trainee's last taken course."),
'type': 'string',
},
},
}
class UpdateError(Exception):
pass
class Resource(BaseResource, HTTPResource):
category = _('Business Process Connectors')
url = models.URLField(_('Webservice Base URL'))
api_key = models.CharField(_('API Key'), max_length=512)
log_requests_errors = False
class Meta:
verbose_name = _('Foederis connector')
verbose_name_plural = _('Foederis connectors')
@property
def referentiels_documents(self):
return self.documents.filter(
external_id__in=[document_id for document_id, _, __ in self.REFERENTIELS]
)
@property
def announces_documents(self):
return self.documents.filter(external_id__startswith='announce-')
@property
def last_update_referentiels(self):
return self.referentiels_documents.aggregate(last_update=models.Min('updated'))['last_update']
@property
def last_update_announces(self):
return self.announces_documents.aggregate(last_update=models.Max('updated'))['last_update']
def http_request(self, method, path, **kwargs):
kwargs.setdefault('headers', {})['API-Key'] = self.api_key
url = self.url + path
response = self.requests.request(method, url, **kwargs)
response.raise_for_status()
try:
data = response.json()
except ValueError:
raise requests.RequestException('content is not JSON')
if data.get('code') != 200:
raise requests.RequestException('code field is not 200, message=%s' % data.get('message'))
return data.get('results', [])
def update_referentiel(self, document_id, path, parent):
try:
results = self.http_request('GET', f'data/{path}?viewIntegrationName=api_publik')
except requests.RequestException:
raise UpdateError(_('Service is unavailable'))
if not results:
return
if parent is None:
data = [{'id': r['id'], 'text': r['name']} for r in results]
else:
parent_fkey = self.REFERENTIELS_FKEYS[parent]
data = [{'id': r['id'], 'text': r['name'], 'parents': r[parent_fkey]} for r in results]
self.documents.update_or_create(defaults={'data': data}, external_id=document_id)
ANNOUNCES_FIELDS = [
# response_field, document_field
('id', 'id'),
('date_publication', 'date'),
('date_fin_publication', 'date_fin_publication'),
('collectivite', 'collectivite'),
('direction', 'direction'),
('intitule_annonce', 'intitule'),
('orientation_recrutement_dgrh', 'orientation_recrutement'),
('cadre_emploi_depuis_ddr', 'cadre_emploi'),
('type_emploi_ddr', 'type_emploi'),
('categorie_ddr', 'categorie'),
('filiere_ddr', 'filiere'),
('intitule_structure_pour_offre', 'intitule_direction'),
('contenu_du_chapeau', 'chapeau'),
('missions_activites', 'description'),
('profil_recherche', 'profil'),
('informations_complmentaires', 'informations_complementaires'),
('reference_offre', 'reference_offre'),
]
DEMANDE_DE_PERSONNEL_FIELDS = [
# response_field, document_field
('missions', 'description'),
('profil_requis', 'profil'),
]
ANNOUNCE_SCHEMA = {
'type': 'object',
'properties': {field: {'type': 'string'} for dummy, field in ANNOUNCES_FIELDS},
}
ANNOUNCE_SCHEMA['properties']['pdf_url'] = {
'type': 'string',
'description': _('Public URL of the PDF announce'),
}
FIELD_ANNOUNCE_FKEY_DEMANDE_DE_PERSONNEL = 'R14848258'
def update_announce(self, response_announce):
document_data = {
document_field: response_announce.get(response_field)
for response_field, document_field in self.ANNOUNCES_FIELDS
}
file_content = None
# retrieve HTML content fields
if len(response_announce[self.FIELD_ANNOUNCE_FKEY_DEMANDE_DE_PERSONNEL]):
recrut_id = response_announce[self.FIELD_ANNOUNCE_FKEY_DEMANDE_DE_PERSONNEL][0]
try:
fields = ','.join([x[0] for x in self.DEMANDE_DE_PERSONNEL_FIELDS])
params = {
'filterName': 'id',
'filterValue': recrut_id,
'fieldList': fields,
'viewIntegrationName': 'api_publik',
}
results = self.http_request('GET', 'data/demande_de_personnel', params=params)
except requests.RequestException:
raise UpdateError(_('Service is unavailable'))
if len(results):
for response_field, document_field in self.DEMANDE_DE_PERSONNEL_FIELDS:
if response_field in results[0]:
document_data[document_field] = results[0][response_field]
document_data['id'] = announce_id = str(response_announce['id'])
external_id = f'announce-{announce_id}'
text = document_data['text'] = document_data['intitule']
if response_announce.get('pdf_ddr'):
try:
path = f'data/annonce/{announce_id}/fields/pdf_ddr?viewIntegrationName=api_publik'
results = self.http_request('GET', path)
except requests.RequestException:
raise UpdateError(_('Service is unavailable'))
if results:
file_content = base64.b64decode(results[0]['pdf_ddr']['fileData'])
with transaction.atomic(savepoint=False):
document, created = self.documents.get_or_create(external_id=external_id)
if document.data == document_data and document.text == text:
return
document.data = document_data
document.text = text
if document.pdf:
document.pdf.delete(save=False)
if file_content:
document.pdf.save(f'annonce-{announce_id}.pdf', ContentFile(file_content), save=False)
else:
document.pdf = None
document.save()
if created:
self.logger.info(_('Created announce %s') % announce_id)
else:
self.logger.info(_('Updated announce %s') % announce_id)
def update_announces(self):
try:
results = self.http_request('GET', 'data/annonce?viewIntegrationName=api_publik')
except requests.RequestException:
raise UpdateError(_('Service is unavailable'))
announces = []
for response_announce in results:
self.update_announce(response_announce)
announces.append('announce-%s' % response_announce['id'])
self.announces_documents.exclude(external_id__in=announces).delete()
REFERENTIELS = [
# document_id, path, parent fkey
('origine_candidature', 'origine_candidature', None),
('civilite', 'civilite', None),
('nationalite', 'nationalite1', None),
('situation_actuelle', 'situation_actuelle', None),
('type_emploi', 'type_emploi', None),
('domaine_emploi', 'domaine_emploi', None),
('sous_domaine_emploi', 'sous_domaine_emploi', 'domaine_emploi'),
('emploi', 'emploi', 'sous_domaine_emploi'),
('niveau_diplome', 'niveau_diplome1', None),
('habilitation', 'habilitation', None),
]
REFERENTIELS_FKEYS = {
'origine_candidature': 'R1261279',
'civilite': 'R60284409',
'nationalite': 'R1249730',
'situation_actuelle': 'R1258320',
'annonce': 'R14848305',
'type_emploi': 'R1249707',
'domaine_emploi': 'R60845221',
'sous_domaine_emploi': 'R60845244',
'emploi': 'R15017962',
'niveau_diplome': 'R1249737',
'habilitation': 'R1276043',
}
def update_referentiels(self):
for document_id, path, parent in self.REFERENTIELS:
self.update_referentiel(document_id, path, parent)
self.update_announces()
def hourly(self):
try:
self.update_referentiels()
except UpdateError as e:
self.logger.warning(_('Update failed: %s') % e)
else:
self.logger.info(_('Referentials updated.'))
@endpoint(
description=_('Get data source'),
long_description=_('Available datasources: %s')
% ', '.join(document_id.replace('_', '-') for document_id, _, __ in REFERENTIELS),
name='ds',
pattern=r'^(?P<name>[a-z_-]+)/$',
example_pattern='{name}/',
perm='can_access',
parameters={
'name': {'description': _('Data source name'), 'example_value': 'domaine_emploi'},
'parent': {'description': _('Parent data source id'), 'example_value': '5776388'},
},
json_schema_response=datasource_schema(),
)
def datasource(self, request, name, parent=None):
name = name.replace('-', '_')
for document_id, _, __ in self.REFERENTIELS:
if document_id == name:
break
else:
raise Http404
document = self.documents.filter(external_id=name).first()
if not document:
return {
'err': 0,
'data': [],
'last_update': None,
}
data = document.data
if parent is not None:
data = [item for item in data if int(parent) in item.get('parents', [])]
return {
'err': 0,
'data': data,
'last_update': localtime(document.updated).strftime('%F %T'),
}
@endpoint(
description=_('Retrieve announce\'s PDF'),
long_description=_('Do not use directly, use the pdf_url field of announces instead.'),
name='announce',
pattern=r'^(?P<announce_id>[0-9]+)/pdf/$',
perm='OPEN',
example_pattern='{announce_id}/pdf/',
parameters={
'announce_id': {'description': _('Announce id'), 'example_value': '12345'},
},
)
def announce_pdf(self, request, announce_id):
# passerelle catch DoesNotExist and converts it to 404
document = self.documents.get(external_id=f'announce-{announce_id}')
with document.pdf.open() as fd:
return HttpResponse(fd, content_type='application/pdf')
@endpoint(
perm="can_access",
name="create-application",
post={
"description": _("Creates an application"),
"request_body": {"schema": {"application/json": APPLICATION_SCHEMA}},
},
)
def create_application(self, request, post_data):
def _get_id(field_name):
id = post_data.get(field_name, None)
if id is None or id == '':
return None
return int(id)
request_data = {
'type_de_candidature': post_data.get('type', 'E'),
'annonce': _get_id('announce_id'),
self.REFERENTIELS_FKEYS['civilite']: _get_id('civility'),
'firstName': post_data.get('first_name', None),
'lastName': post_data.get('last_name', None),
'sexe': post_data.get('gender', None),
'date_de_naissance': post_data.get('birth_date', None),
self.REFERENTIELS_FKEYS['nationalite']: _get_id('nationality'),
'date_fin_autorisation_de_travail': post_data.get('work_authorization_end_date', None),
'rqth': 'O' if get_bool(post_data.get('rqth', False)) else 'N',
'date_fin_rqth': post_data.get('rqth_end_date', None),
'permis_de_conduire': post_data.get('driving_license', None),
'fimo': 'O' if get_bool(post_data.get('fimo', False)) else 'N',
'Date_delivrance_fimo': post_data.get('fimo_delivrance_date', None),
'date_fin_validite_fimo': post_data.get('fimo_end_validity_date', None),
self.REFERENTIELS_FKEYS['situation_actuelle']: _get_id('current_situation'),
'collectivite_agent': post_data.get('agent_collectivity', None),
'date_debut_disponibilite': post_data.get('availability_start_date', None),
'date_fin_disponibilite': post_data.get('availability_end_date', None),
'pretentions_salariales': post_data.get('salary_expectations', None),
'adresse': post_data.get('address', None),
'adresse_ligne_2': post_data.get('address_complement', None),
'code_postal': post_data.get('zip', None),
'ville': post_data.get('city', None),
'telephone': post_data.get('phone', None),
'email': post_data.get('email', None),
'date_debut_contrat': post_data.get('contract_start_date', None),
'date_fin_contrat': post_data.get('contract_end_date', None),
'complement_information_candidature': post_data.get('additional_informations', None),
self.REFERENTIELS_FKEYS['origine_candidature']: _get_id('origin'),
'precision_origine_candidature': post_data.get('origin_precisions', None),
'accord_RGPD': get_bool(post_data.get('rgpd_agreement', False)),
self.REFERENTIELS_FKEYS['type_emploi']: [int(id) for id in post_data.get('job_types', [])],
self.REFERENTIELS_FKEYS['domaine_emploi']: [int(id) for id in post_data.get('job_realms', [])],
self.REFERENTIELS_FKEYS['sous_domaine_emploi']: [
int(id) for id in post_data.get('job_families', [])
],
self.REFERENTIELS_FKEYS['emploi']: [int(id) for id in post_data.get('jobs', [])],
'temps_de_travail_souhaite': post_data.get('desired_work_time', None),
'duree_du_contrat_de_stage_apprentissage': post_data.get('internship_duration', None),
'ecole_centre_de_formation_mission_loc': post_data.get('school_name', None),
'intitule_diplome_vise': post_data.get('diploma_name', None),
'specialite_diplome': post_data.get('diploma_speciality', None),
self.REFERENTIELS_FKEYS['niveau_diplome']: _get_id('aimed_diploma_level'),
'dernier_diplome_obtenu': post_data.get('last_obtained_diploma', None),
'derniere_classe_suivie': post_data.get('last_course_taken', None),
}
request_data = {k: v for k, v in request_data.items() if v is not None and v != ''}
results = self.http_request(
'POST', 'data/candidature?viewIntegrationName=api_publik', json=request_data
)
return {
'err': 0,
'data': {'application_id': results[0]['id']},
}
@endpoint(
name='attach-file',
perm='can_access',
post={
"description": _("Attach a file to an application."),
"request_body": {"schema": {"application/json": ATTACHMENT_SCHEMA}},
},
)
def attach_file(self, request, post_data):
application_id = post_data["application_id"]
attachment_name = post_data["name"]
file = post_data["file"]
self.http_request(
'POST',
f'data/candidature/{application_id}/fields/{attachment_name}?viewIntegrationName=api_publik',
json={
"contentType": file["content_type"],
"value": file["content"],
"fileName": file["filename"],
},
)
return {'err': 0}
@endpoint(
description=_('List announces'),
long_description=_(
'List published announces. Use unpublished=1 parameter to see all announces. When using id to retrieve a specific announce, filters are ignored.'
),
name='announce',
perm='can_access',
parameters={
'q': {'description': _('Free text search')},
'id': {'description': _('Get a specific announce')},
'type_emploi': {'description': _('Filter by job type')},
'categorie': {'description': _('Filter by job category')},
'filiere': {'description': _('Filter by job sector')},
'collectivite': {'description': _('Filter by collectivite')},
'unpublished': {'description': _('Add unpublished announces to the list')},
},
json_schema_response=response_schema(
{'type': 'array', 'items': ANNOUNCE_SCHEMA},
toplevel_properties={
'data_sources': {
'type': 'object',
'patternProperties': {
'': datasource_array_schema(),
},
}
},
),
)
def announce(
self,
request,
q=None,
id=None,
type_emploi=None,
categorie=None,
filiere=None,
collectivite=None,
unpublished=None,
):
unpublished = bool(unpublished and unpublished.lower() in ['1', 'true', 'on'])
qs = self.announces_documents
qs = qs.order_by('-data__date')
if id:
qs = qs.filter(external_id=f'announce-{id}')
else:
today = now().date().strftime('%Y-%m-%d')
if not unpublished:
qs = qs.filter(Q(data__date__isnull=True) | Q(data__date__lte=today))
qs = qs.filter(
Q(data__date_fin_publication__isnull=True) | Q(data__date_fin_publication__gte=today)
)
if q:
qs = qs.filter(data__intitule__icontains=q)
if type_emploi:
qs = qs.filter(data__type_emploi=type_emploi)
if categorie:
qs = qs.filter(data__categorie=categorie)
if filiere:
qs = qs.filter(data__filiere=filiere)
if collectivite:
qs = qs.filter(data__collectivite=collectivite)
data_sources = {document.external_id: document.data for document in self.referentiels_documents}
def pdf_url(request, document):
doc_id = document.external_id.split('-')[-1]
return request.build_absolute_uri(
reverse(
'generic-endpoint',
kwargs={
'connector': self.get_connector_slug(),
'slug': self.slug,
'endpoint': 'announce',
'rest': f'{doc_id}/pdf/',
},
)
)
return {
'err': 0,
'data': [dict(document.data, pdf_url=pdf_url(request, document)) for document in qs],
'data_sources': data_sources,
}
def upload_to(instance, filename):
return f'toulouse_foederis/{instance.resource.slug}/{filename}'
class Document(models.Model):
resource = models.ForeignKey(
verbose_name=_('Resource'),
to=Resource,
on_delete=models.CASCADE,
related_name='documents',
)
external_id = models.TextField(_('Key'), unique=True)
text = models.TextField(_('Text'), blank=True)
data = JSONField(_('Data'), blank=True, default=dict)
pdf = models.FileField(_('PDF file'), blank=True, upload_to=upload_to)
created = models.DateTimeField(_('Created'), auto_now_add=True)
updated = models.DateTimeField(_('Updated'), auto_now=True)
def __repr__(self):
return f'<Document "{self.external_id}">'
def delete(self, *args, **kwargs):
if self.pdf:
self.pdf.delete(save=False)
return super().delete(*args, **kwargs)
class Meta:
verbose_name = _('Foederis data')
verbose_name_plural = _('Foederis datas')