# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2020  Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json
from unittest import mock

import pytest
from requests.exceptions import ConnectionError

import tests.utils
from passerelle.apps.opendatasoft.models import OpenDataSoft, Query
from passerelle.utils import import_site
from tests.test_manager import login

pytestmark = pytest.mark.django_db


FAKED_CONTENT_Q_SEARCH = json.dumps(
    {
        "nhits": 76,
        "parameters": {
            "dataset": "referentiel-adresse-test",
            "format": "json",
            "q": "rue de l'aubepine",
            "rows": 3,
            "timezone": "UTC",
        },
        "records": [
            {
                "datasetid": "referentiel-adresse-test",
                "fields": {
                    "adresse_complete": "33 RUE DE L'AUBEPINE STRASBOURG",
                    "date_exprt": "2019-10-23",
                    "geo_point": [48.6060963542, 7.76978279836],
                    "nom_commun": "Strasbourg",
                    "nom_rue": "RUE DE L'AUBEPINE",
                    "num_com": 482,
                    "numero": "33",
                    "source": "Ville et Eurométropole de Strasbourg",
                },
                "geometry": {"coordinates": [7.76978279836, 48.6060963542], "type": "Point"},
                "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                "recordid": "e00cf6161e52a4c8fe510b2b74d4952036cb3473",
            },
            {
                "datasetid": "referentiel-adresse-test",
                "fields": {
                    "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
                    "date_exprt": "2019-10-23",
                    "geo_point": [48.4920620548, 7.66177412454],
                    "nom_commun": "Lipsheim",
                    "nom_rue": "RUE DE L'AUBEPINE",
                    "num_com": 268,
                    "numero": "19",
                    "source": "Ville et Eurométropole de Strasbourg",
                },
                "geometry": {"coordinates": [7.66177412454, 48.4920620548], "type": "Point"},
                "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8",
            },
            {
                "datasetid": "referentiel-adresse-test",
                "fields": {
                    "adresse_complete": "29 RUE DE L'AUBEPINE STRASBOURG",
                    "date_exprt": "2019-10-23",
                    "geo_point": [48.6056497224, 7.76988497729],
                    "nom_commun": "Strasbourg",
                    "nom_rue": "RUE DE L'AUBEPINE",
                    "num_com": 482,
                    "numero": "29",
                    "source": "Ville et Eurométropole de Strasbourg",
                },
                "geometry": {"coordinates": [7.76988497729, 48.6056497224], "type": "Point"},
                "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                "recordid": "0984a5e1745701f71c91af73ce764e1f7132e0ff",
            },
        ],
    }
)

FAKED_CONTENT_ID_SEARCH = json.dumps(
    {
        "nhits": 1,
        "parameters": {
            "dataset": "referentiel-adresse-test",
            "format": "json",
            "q": "recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8",
            "rows": 1,
            "timezone": "UTC",
        },
        "records": [
            {
                "datasetid": "referentiel-adresse-test",
                "fields": {
                    "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
                    "date_exprt": "2019-10-23",
                    "geo_point": [48.4920620548, 7.66177412454],
                    "nom_commun": "Lipsheim",
                    "nom_rue": "RUE DE L'AUBEPINE",
                    "num_com": 268,
                    "numero": "19",
                    "source": "Ville et Eurométropole de Strasbourg",
                },
                "geometry": {"coordinates": [7.66177412454, 48.4920620548], "type": "Point"},
                "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8",
            }
        ],
    }
)


@pytest.fixture
def connector():
    return tests.utils.setup_access_rights(
        OpenDataSoft.objects.create(
            slug='my_connector',
            service_url='http://www.example.net',
            api_key='my_secret',
        )
    )


@pytest.fixture
def query(connector):
    return Query.objects.create(
        resource=connector,
        name='Référenciel adresses de test',
        slug='my_query',
        description='Rechercher une adresse',
        dataset='referentiel-adresse-test',
        text_template='{{numero}} {{nom_rue}} {{nom_commun}}',
        filter_expression='''
refine.source=Ville et Eurométropole de Strasbourg
exclude.numero=42
exclude.numero=43
''',
        sort='-nom_rue',
        limit=3,
    )


def test_views(db, admin_user, app, connector):
    app = login(app)
    resp = app.get('/opendatasoft/my_connector/', status=200)
    resp = resp.click('New Query')
    resp.form['name'] = 'my query'
    resp.form['slug'] = 'my-query'
    resp.form['dataset'] = 'my-dataset'
    resp = resp.form.submit()
    resp = resp.follow()
    assert resp.html.find('div', {'id': 'panel-queries'}).ul.li.a.text == 'my query'


def test_export_import(query):
    assert OpenDataSoft.objects.count() == 1
    assert Query.objects.count() == 1
    serialization = {'resources': [query.resource.export_json()]}
    OpenDataSoft.objects.all().delete()
    assert OpenDataSoft.objects.count() == 0
    assert Query.objects.count() == 0
    import_site(serialization)
    assert OpenDataSoft.objects.count() == 1
    assert Query.objects.count() == 1


@mock.patch('passerelle.utils.Request.get')
def test_search_empty_contents(mocked_get, app, connector):
    endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
    assert endpoint == '/opendatasoft/my_connector/search'

    # error returned by opendadasoft server
    json_response = json.dumps({'error': "The query is invalid : Field 00 doesn't exist"})
    mocked_get.return_value = tests.utils.FakedResponse(content=json_response, status_code=200)
    resp = app.get(endpoint, status=200)
    assert resp.json['err']
    assert resp.json['err_desc'] == "The query is invalid : Field 00 doesn't exist"


@mock.patch('passerelle.utils.Request.get')
def test_search(mocked_get, app, connector):
    endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
    assert endpoint == '/opendatasoft/my_connector/search'
    params = {
        'dataset': 'referentiel-adresse-test',
        'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
        'sort': '-nom_rue',
        'limit': 3,
    }
    mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
    resp = app.get(endpoint, params=params, status=200)
    assert mocked_get.call_args[1]['params'] == {
        'apikey': 'my_secret',
        'dataset': 'referentiel-adresse-test',
        'sort': '-nom_rue',
        'rows': '3',
    }
    assert not resp.json['err']
    assert len(resp.json['data']) == 3
    # check order is kept
    assert [x['id'] for x in resp.json['data']] == [
        'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
        '7cafcd5c692773e8b863587b2d38d6be82e023d8',
        '0984a5e1745701f71c91af73ce764e1f7132e0ff',
    ]
    # check text results
    assert [x['text'] for x in resp.json['data']] == [
        "33 RUE DE L'AUBEPINE Strasbourg",
        "19 RUE DE L'AUBEPINE Lipsheim",
        "29 RUE DE L'AUBEPINE Strasbourg",
    ]
    # check additional attributes
    assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']


@mock.patch('passerelle.utils.Request.get')
def test_search_using_q(mocked_get, app, connector):
    endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
    assert endpoint == '/opendatasoft/my_connector/search'
    params = {
        'dataset': 'referentiel-adresse-test',
        'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
        'sort': '-nom_rue',
        'limit': '3',
        'q': "rue de l'aubepine",
    }
    mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
    resp = app.get(endpoint, params=params, status=200)
    assert mocked_get.call_args[1]['params'] == {
        'apikey': 'my_secret',
        'dataset': 'referentiel-adresse-test',
        'rows': '3',
        'q': "rue de aubepine",
    }
    assert not resp.json['err']
    assert len(resp.json['data']) == 3
    # check order is kept
    assert [x['id'] for x in resp.json['data']] == [
        'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
        '7cafcd5c692773e8b863587b2d38d6be82e023d8',
        '0984a5e1745701f71c91af73ce764e1f7132e0ff',
    ]
    # check text results
    assert [x['text'] for x in resp.json['data']] == [
        "33 RUE DE L'AUBEPINE Strasbourg",
        "19 RUE DE L'AUBEPINE Lipsheim",
        "29 RUE DE L'AUBEPINE Strasbourg",
    ]
    # check additional attributes
    assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']

    # check operators are removed
    params['q'] = 'please, do NOT send boolean operators like And, OR and nOt'
    resp = app.get(endpoint, params=params, status=200)
    assert mocked_get.call_args[1]['params']['q'] == 'please do send boolean operators like'

    params['q'] = 'field operators are almost ignored too:, -, ==, >, <, >=, <=, [start_date TO end_date]'
    resp = app.get(endpoint, params=params, status=200)
    assert (
        mocked_get.call_args[1]['params']['q']
        == 'field operators are almost ignored too start_date TO end_date'
    )


@mock.patch('passerelle.utils.Request.get')
def test_search_using_id(mocked_get, app, connector):
    endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
    assert endpoint == '/opendatasoft/my_connector/search'
    params = {
        'dataset': 'referentiel-adresse-test',
        'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
    }
    mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
    resp = app.get(endpoint, params=params, status=200)
    assert mocked_get.call_args[1]['params'] == {
        'apikey': 'my_secret',
        'dataset': 'referentiel-adresse-test',
        'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
    }
    assert len(resp.json['data']) == 1
    assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"


@mock.patch('passerelle.utils.Request.get')
def test_query_q(mocked_get, app, query):
    endpoint = '/opendatasoft/my_connector/q/my_query/'
    params = {
        'limit': 3,
    }
    mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
    resp = app.get(endpoint, params=params, status=200)
    assert mocked_get.call_args[1]['params'] == {
        'apikey': 'my_secret',
        'dataset': 'referentiel-adresse-test',
        'refine.source': ['Ville et Eurométropole de Strasbourg'],
        'exclude.numero': ['42', '43'],
        'sort': '-nom_rue',
        'rows': 3,
    }
    assert not resp.json['err']
    assert len(resp.json['data']) == 3
    # check order is kept
    assert [x['id'] for x in resp.json['data']] == [
        'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
        '7cafcd5c692773e8b863587b2d38d6be82e023d8',
        '0984a5e1745701f71c91af73ce764e1f7132e0ff',
    ]
    # check text results
    assert [x['text'] for x in resp.json['data']] == [
        "33 RUE DE L'AUBEPINE Strasbourg",
        "19 RUE DE L'AUBEPINE Lipsheim",
        "29 RUE DE L'AUBEPINE Strasbourg",
    ]
    # check additional attributes
    assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']


@mock.patch('passerelle.utils.Request.get')
def test_query_q_using_q(mocked_get, app, query):
    endpoint = '/opendatasoft/my_connector/q/my_query/'
    params = {
        'q': "rue de l'aubepine",
    }
    mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
    resp = app.get(endpoint, params=params, status=200)
    assert mocked_get.call_args[1]['params'] == {
        'apikey': 'my_secret',
        'dataset': 'referentiel-adresse-test',
        'refine.source': ['Ville et Eurométropole de Strasbourg'],
        'exclude.numero': ['42', '43'],
        'rows': 3,
        'q': "rue de aubepine",
    }
    assert not resp.json['err']
    assert len(resp.json['data']) == 3
    # check order is kept
    assert [x['id'] for x in resp.json['data']] == [
        'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
        '7cafcd5c692773e8b863587b2d38d6be82e023d8',
        '0984a5e1745701f71c91af73ce764e1f7132e0ff',
    ]
    # check text results
    assert [x['text'] for x in resp.json['data']] == [
        "33 RUE DE L'AUBEPINE Strasbourg",
        "19 RUE DE L'AUBEPINE Lipsheim",
        "29 RUE DE L'AUBEPINE Strasbourg",
    ]
    # check additional attributes
    assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']


@mock.patch('passerelle.utils.Request.get')
def test_query_q_using_id(mocked_get, app, query):
    endpoint = '/opendatasoft/my_connector/q/my_query/'
    params = {
        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
    }
    mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
    resp = app.get(endpoint, params=params, status=200)
    assert mocked_get.call_args[1]['params'] == {
        'apikey': 'my_secret',
        'dataset': 'referentiel-adresse-test',
        'refine.source': ['Ville et Eurométropole de Strasbourg'],
        'exclude.numero': ['42', '43'],
        'rows': 3,
        'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
    }
    assert len(resp.json['data']) == 1
    assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"


def test_opendatasoft_query_unicity(admin_user, app, connector, query):
    connector2 = OpenDataSoft.objects.create(
        slug='my_connector2',
        api_key='my_secret',
    )
    Query.objects.create(
        resource=connector2,
        name='Foo Bar',
        slug='foo-bar',
    )

    app = login(app)
    resp = app.get('/manage/opendatasoft/%s/query/new/' % connector.slug)
    resp.form['slug'] = query.slug
    resp.form['name'] = 'Foo Bar'
    resp.form['dataset'] = 'my-dataset'
    resp = resp.form.submit()
    assert resp.status_code == 200
    assert Query.objects.filter(resource=connector).count() == 1
    assert 'A query with this slug already exists' in resp.text
    resp.form['slug'] = 'foo-bar'
    resp.form['name'] = query.name
    resp.form['dataset'] = 'my-dataset'
    resp = resp.form.submit()
    assert Query.objects.filter(resource=connector).count() == 1
    assert resp.status_code == 200
    assert 'A query with this name already exists' in resp.text
    resp.form['slug'] = 'foo-bar'
    resp.form['name'] = 'Foo Bar'
    resp.form['dataset'] = 'my-dataset'
    resp = resp.form.submit()
    assert resp.status_code == 302
    assert Query.objects.filter(resource=connector).count() == 2
    new_query = Query.objects.latest('pk')
    assert new_query.resource == connector

    resp = app.get('/manage/opendatasoft/%s/query/%s/' % (connector.slug, new_query.pk))
    resp.form['slug'] = query.slug
    resp.form['name'] = 'Foo Bar'
    resp = resp.form.submit()
    assert resp.status_code == 200
    assert 'A query with this slug already exists' in resp.text
    resp.form['slug'] = 'foo-bar'
    resp.form['name'] = query.name
    resp = resp.form.submit()
    assert resp.status_code == 200
    assert 'A query with this name already exists' in resp.text
    resp.form['slug'] = 'foo-bar'
    resp.form['name'] = 'Foo Bar'
    resp = resp.form.submit()
    assert resp.status_code == 302


@mock.patch('passerelle.utils.Request.get')
def test_query_q_having_original_fields(mocked_get, app, query):
    endpoint = '/opendatasoft/my_connector/q/my_query/'
    params = {
        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
    }
    content = json.loads(FAKED_CONTENT_ID_SEARCH)
    content['records'][0]['fields']['id'] = 'original id'
    content['records'][0]['fields']['text'] = 'original text'
    query.text_template = '{{id}} - {{original_id}} - {{original_text}}'
    query.save()
    mocked_get.return_value = tests.utils.FakedResponse(content=json.dumps(content), status_code=200)
    resp = app.get(endpoint, params=params, status=200)
    assert resp.json['data'][0]['original_id'] == 'original id'
    assert resp.json['data'][0]['original_text'] == 'original text'
    assert (
        resp.json['data'][0]['text']
        == "7cafcd5c692773e8b863587b2d38d6be82e023d8 - original id - original text"
    )


def test_call_search_errors(app, connector):
    endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
    assert endpoint == '/opendatasoft/my_connector/search'
    url = connector.service_url + '/api/records/1.0/search/'

    # Connection error
    exception = ConnectionError('Remote end closed connection without response')
    with tests.utils.mock_url(url=url, exception=exception):
        resp = app.get(endpoint)
    assert resp.json['err']
    assert resp.json['err_desc'] == 'OpenDataSoft error: Remote end closed connection without response'

    # API error, provides HTTP status code
    with tests.utils.mock_url(url=url, response='{"error": "Unknown dataset: foo"}', status_code=404):
        resp = app.get(endpoint)
    assert resp.json['err']
    assert resp.json['err_desc'] == 'Unknown dataset: foo'

    # HTTP error
    with tests.utils.mock_url(url=url, response='not a json content', reason='Not Found', status_code=404):
        resp = app.get(endpoint)
    assert resp.json['err']
    assert 'OpenDataSoft error: 404 Client Error: Not Found' in resp.json['err_desc']

    # bad JSON response
    with tests.utils.mock_url(url=url, response='not a json content', status_code=200):
        resp = app.get(endpoint)
    assert resp.json['err']
    assert resp.json['err_desc'] == 'OpenDataSoft error: bad JSON response'
