import datetime
import json
import os
from unittest import mock

import pytest
from django.conf import settings
from django.contrib.auth.models import AnonymousUser, Group, User
from django.contrib.contenttypes.models import ContentType
from django.db import connection
from django.test import override_settings
from django.test.client import RequestFactory
from django.test.utils import CaptureQueriesContext
from django.urls import reverse
from django.utils.http import urlencode
from django.utils.timezone import now
from requests.exceptions import ConnectionError
from requests.models import Response

from combo.apps.search.engines import engines
from combo.apps.search.models import IndexedCell, SearchCell
from combo.apps.search.utils import index_site, search_site
from combo.data.models import JsonCell, LinkCell, LinkListCell, MenuCell, Page, PageSnapshot, TextCell

from .test_manager import login

pytestmark = pytest.mark.django_db


SEARCH_SERVICES = {
    'search1': {
        'label': 'Search 1',
        'url': 'http://www.example.net/search/?q=%(q)s',
    },
    'search_tmpl': {
        'label': 'Search with template',
        'url': '[search_url]?q=%(q)s',
    },
    'search_alternate_key': {
        'label': 'Search with alternate key',
        'url': 'http://www.example.net/search/?q=%(q)s',
        'data_key': 'results',
    },
}

TEMPLATE_VARS = {'search_url': 'http://search.example.net/'}


class SearchServices:
    def __init__(self, search_services):
        self.search_services = search_services

    def __enter__(self):
        settings.COMBO_SEARCH_SERVICES = self.search_services

    def __exit__(self, *args, **kwargs):
        settings.COMBO_SEARCH_SERVICES = {}


def test_search_cell(settings, app):
    settings.COMBO_SEARCH_SERVICES = SEARCH_SERVICES
    page = Page(title='Search', slug='search_page', template_name='standard')
    page.save()

    cell = SearchCell(page=page, placeholder='content', order=0)
    cell._search_services = {'data': ['search1']}
    cell.input_placeholder = 'my placeholder'
    cell.save()

    # unknown cell pk
    resp = app.get('/ajax/search/0/search1/?q=foo', status=404)

    resp = cell.render({})
    assert 'input' in resp
    assert 'id="combo-search-input-%s"' % cell.pk in resp
    assert 'autofocus' not in resp
    assert 'placeholder="my placeholder"' in resp

    cell.autofocus = True
    cell.save()
    resp = cell.render({})
    assert 'h2' not in resp
    assert 'autofocus' in resp

    cell.slug = 'var-name'
    cell.title = 'Custom Title'
    context = {'request': RequestFactory().get('/?q_var_name=searchme')}
    resp = cell.render(context)
    assert '<h2>Custom Title</h2>' in resp
    assert "$input.val('searchme');" in resp

    with mock.patch('combo.apps.search.models.requests.get') as requests_get:

        response = {'err': 0, 'data': []}
        mock_json = mock.Mock()
        mock_json.json.return_value = response
        requests_get.return_value = mock_json
        resp = app.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
        assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=foo'
        assert '<li>' not in resp.text
        assert 'no result found' in resp.text

        resp = app.get('/ajax/search/%s/search1/?q=foo%%23bar' % cell.pk, status=200)
        assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=foo%23bar'
        assert '<li>' not in resp.text
        assert 'no result found' in resp.text

        response['data'] = [{'url': 'http://test', 'text': 'barbarbar'}]
        resp = app.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
        assert resp.text.count('<li>') == 1
        assert '<li><a href="http://test">barbarbar</a>' in resp.text
        assert 'no result found' not in resp.text

        response['data'] = [{'url': 'http://test', 'text': 'barbarbar', 'description': 'this is <b>html</b>'}]
        resp = app.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
        assert resp.text.count('<li>') == 1
        assert '<li><a href="http://test">barbarbar</a>' in resp.text
        assert 'this is <b>html</b>' in resp.text
        assert 'no result found' not in resp.text

        resp = app.get('/ajax/search/%s/search1/?q=' % cell.pk, status=200)
        assert '<li>' not in resp.text
        assert 'no result found' not in resp.text

        cell._search_services = {'data': ['search_alternate_key']}
        cell.save()
        response = {'results': [{'url': 'http://test', 'text': 'barbarbar'}]}
        mock_json.json.return_value = response
        resp = app.get('/ajax/search/%s/search_alternate_key/?q=foo' % cell.pk, status=200)
        assert resp.text.count('<li>') == 1
        assert '<li><a href="http://test">barbarbar</a>' in resp.text

        # search engine does not return valid JSON
        class FakedResponse(mock.Mock):
            def json(self):
                return json.loads(self.content)

        requests_get.return_value = FakedResponse(content='notjson', status_code=200)
        resp = app.get('/ajax/search/%s/search_alternate_key/?q=bar' % cell.pk, status=200)
        assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=bar'
        assert '<li>' not in resp.text
        assert 'no result found' in resp.text
        requests_get.return_value = FakedResponse(content='500withbadjson', status_code=500)
        resp = app.get('/ajax/search/%s/search_alternate_key/?q=foo' % cell.pk, status=200)
        assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=foo'
        assert '<li>' not in resp.text
        assert 'no result found' in resp.text

    settings.TEMPLATE_VARS = TEMPLATE_VARS
    cell._search_services = {'data': ['search_tmpl']}
    cell.save()
    with mock.patch('combo.apps.search.models.requests.get') as requests_get:
        response = {'err': 0, 'data': []}
        mock_json = mock.Mock()
        mock_json.json.return_value = response
        requests_get.return_value = mock_json
        resp = app.get('/ajax/search/%s/search_tmpl/?q=foo' % cell.pk, status=200)
        assert requests_get.call_args[0][0] == 'http://search.example.net/?q=foo'

        # TEMPLATE_VARS are accessible in template
        cell.slug = 'searchfoo'
        cell.save()
        templates_settings = [settings.TEMPLATES[0].copy()]
        templates_settings[0]['DIRS'] = ['%s/templates-1' % os.path.abspath(os.path.dirname(__file__))]
        with override_settings(TEMPLATES=templates_settings):
            resp = app.get('/ajax/search/%s/search_tmpl/?q=bar' % cell.pk, status=200)
            assert requests_get.call_args[0][0] == 'http://search.example.net/?q=bar'
            assert 'searchfoo results.data=[]' in resp.text
            assert 'search_url=http://search.example.net/' in resp.text


def test_search_cell_urlsplit(settings, app):
    settings.KNOWN_SERVICES = {
        'passerelle': {
            'default': {
                'title': 'test',
                'url': 'http://example.org',
                'secret': 'combo',
                'orig': 'combo',
                'backoffice-menu-url': 'http://example.org/manage/',
            }
        }
    }
    settings.COMBO_SEARCH_SERVICES = {
        'search1': {
            'label': 'Search 1',
            'url': 'http://example.org/search/?q={{ q }}',  # template var
            'signature': 'foo',
        },
    }

    page = Page.objects.create(title='Search', slug='search_page', template_name='standard')

    cell = SearchCell.objects.create(
        page=page, placeholder='content', order=0, _search_services={'data': ['search1']}
    )

    with mock.patch('combo.apps.search.models.requests.get') as requests_get:
        response = {'err': 0, 'data': []}
        mock_json = mock.Mock()
        mock_json.json.return_value = response
        requests_get.return_value = mock_json
        resp = app.get('/ajax/search/%s/search1/?q=foo?' % cell.pk, status=200)
    assert requests_get.call_args[0][0] == 'http://example.org/search/?q=foo?'
    assert requests_get.call_args[1]['remote_service'] == 'auto'

    with mock.patch('requests.Session.request') as mock_request:
        response = {'err': 0, 'data': []}
        mock_json = mock.Mock(status_code=200)
        mock_json.json.return_value = response
        mock_request.return_value = mock_json
        resp = app.get('/ajax/search/%s/search1/?q=foo?' % cell.pk, status=200)
    assert mock_request.call_args[0][1] == 'http://example.org/search/?orig=combo&q=foo?'
    assert '<li>' not in resp.text
    assert 'no result found' in resp.text


def test_search_global_context(app):
    with SearchServices(SEARCH_SERVICES):
        page = Page(title='Search', slug='search_page', template_name='standard')
        page.save()
        cell = SearchCell(page=page, placeholder='content', order=0)
        cell._search_services = {'data': ['search1']}
        cell.save()
        assert cell.varname == ''

        cell.slug = 'search-item'
        cell.save()
        assert cell.varname == 'search_item'

        jsoncell = JsonCell(page=page, placeholder='content', order=0)
        jsoncell.url = 'http://www.example.net/search/[search_item]/'
        jsoncell.save()
        url = (
            reverse(
                'combo-public-ajax-page-cell',
                kwargs={'page_pk': page.id, 'cell_reference': jsoncell.get_reference()},
            )
            + '?search_item=foo'
        )

        with mock.patch('combo.utils.requests.get') as requests_get:
            data = {'data': []}
            requests_get.return_value = mock.Mock(json=lambda: data, status_code=200)
            app.get(url)
            assert requests_get.call_args[0][0] == 'http://www.example.net/search/foo/'


def test_search_percent_sign(app):
    services = {
        'search2': {
            'label': 'Search 2',
            'url': 'http://www.example.net/search/?email=test%40example.net&q=%(q)s',
            'hit_url_template': 'http://example.net/{{id}}/',
            'hit_label_template': '{{a}} {{b}}',
            'hit_description_template': 'description {{a}}',
        }
    }
    with SearchServices(services):
        page = Page(title='Search', slug='search_page', template_name='standard')
        page.save()

        cell = SearchCell(page=page, placeholder='content', order=0)
        cell._search_services = {'data': ['search2']}
        cell.save()

        with mock.patch('combo.apps.search.models.requests.get') as requests_get:
            response = {
                'err': 0,
                'data': [{'id': '123', 'a': 'A', 'b': 'B'}],
            }
            mock_json = mock.Mock()
            mock_json.json.return_value = response
            requests_get.return_value = mock_json

            resp = app.get('/ajax/search/%s/search2/?q=foo' % cell.pk, status=200)
            assert resp.text.count('<li>') == 1
            assert '<li><a href="http://example.net/123/">A B</a>' in resp.text
            assert '<div>description A</div>' in resp.text


def test_search_custom_templates(app):
    services = {
        'search2': {
            'label': 'Search 2',
            'url': 'http://www.example.net/search/?q=%(q)s',
            'hit_url_template': 'http://example.net/{{id}}/',
            'hit_label_template': '{{a}} {{b}}',
            'hit_description_template': 'description {{a}}',
        }
    }
    with SearchServices(services):
        page = Page(title='Search', slug='search_page', template_name='standard')
        page.save()

        cell = SearchCell(page=page, placeholder='content', order=0)
        cell._search_services = {'data': ['search2']}
        cell.save()

        with mock.patch('combo.apps.search.models.requests.get') as requests_get:
            response = {
                'err': 0,
                'data': [{'id': '123', 'a': 'A', 'b': 'B'}],
            }
            mock_json = mock.Mock()
            mock_json.json.return_value = response
            requests_get.return_value = mock_json

            resp = app.get('/ajax/search/%s/search2/?q=foo' % cell.pk, status=200)
            assert resp.text.count('<li>') == 1
            assert '<li><a href="http://example.net/123/">A B</a>' in resp.text
            assert '<div>description A</div>' in resp.text


def test_search_cell_visibility(settings, app):
    request = RequestFactory().get('/')
    page = Page.objects.create(title='example page', slug='example-page')
    settings.COMBO_SEARCH_SERVICES = SEARCH_SERVICES

    cell = SearchCell(page=page, order=0)
    assert not cell.is_visible(request)

    cell._search_services = {'data': ['_text']}
    assert cell.is_visible(request)


def test_search_contents():
    page = Page(title='example page', slug='example-page')
    page.save()

    # private cells are indexed
    cell = TextCell(page=page, text='foobar', public=False, order=0)
    assert cell.render_for_search().strip() == 'foobar'

    # no indexation of empty cells (is_relevant check)
    cell = TextCell(page=page, text='', order=0)
    assert cell.render_for_search() == ''

    # indexation
    cell = TextCell(page=page, text='<p>foobar</p>', order=0)
    assert cell.render_for_search().strip() == 'foobar'

    # no indexation of menu cells
    cell = MenuCell(page=page, order=0)
    assert cell.exclude_from_search is True


def test_search_contents_index():
    page = Page(title='example page', slug='example-page')
    page.public = True
    page.save()

    cell = TextCell(page=page, placeholder='content', text='<p>foobar</p>', order=0)
    cell.save()

    request = RequestFactory().get('/')
    request.user = AnonymousUser()
    hits = search_site(request, 'foobar')
    assert len(hits) == 0
    index_site()
    hits = search_site(request, 'foobar')
    assert len(hits) == 1


def test_search_contents_technical_placeholder():
    page = Page(title='example page', slug='example-page')
    page.save()

    TextCell(page=page, text='<p>foobar</p>', order=0, placeholder='_off').save()
    TextCell(page=page, text='<p>barfoo</p>', order=0, placeholder='content').save()

    request = RequestFactory().get('/')
    request.user = AnonymousUser()
    index_site()
    hits = search_site(request, 'foobar')
    assert len(hits) == 0
    hits = search_site(request, 'barfoo')
    assert len(hits) == 1


def test_search_api(app):
    page = Page.objects.create(title='example page', slug='example-page')
    TextCell.objects.create(page=page, placeholder='content', text='<p>foobar baz</p>', order=0)
    second_page = Page.objects.create(
        title='second page', slug='second-page', description='Foo Bar Description'
    )
    TextCell.objects.create(page=second_page, placeholder='content', text='<p>other baz</p>', order=0)
    index_site()

    cell = SearchCell.objects.create(
        page=page, placeholder='content', _search_services={'data': ['_text']}, order=0
    )

    resp = app.get('/ajax/search/%s/_text/?q=foobar' % cell.id, status=200)
    assert resp.text.count('<li') == 1
    assert 'example page' in resp.text

    resp = app.get('/ajax/search/%s/_text/?q=other' % cell.id, status=200)
    assert resp.text.count('<li') == 1
    result = resp.text
    result = result.replace('  ', '')
    result = result.replace('\n', '')
    assert '<li><a href="/second-page/">second page</a></li>' in result

    # indexed cell without page
    cell_type = ContentType.objects.get_for_model(cell)
    IndexedCell.objects.create(
        title='other', cell_type=cell_type, cell_pk=cell.pk, public_access=True, url='fake'
    )
    # enable description
    cell._search_services['options'] = {'_text': {'with_description': True}}
    cell.save()
    resp = app.get('/ajax/search/%s/_text/?q=other' % cell.id, status=200)
    assert resp.text.count('<li') == 2
    result = resp.text
    result = result.replace('  ', '')
    result = result.replace('\n', '')
    assert '<li><a href="fake">other</a></li>' in result
    assert '<li><a href="/second-page/">second page</a><div>Foo Bar Description</div></li>' in result

    resp = app.get('/ajax/search/%s/_text/?q=baz' % cell.id, status=200)
    assert resp.text.count('<li') == 2

    resp = app.get('/ajax/search/%s/_text/?q=quux' % cell.id, status=200)
    assert resp.text.count('<li') == 0


def test_search_on_root_page_api(settings, app):
    settings.KNOWN_SERVICES = {}

    # not indexed: with sub_slug
    page = Page.objects.create(title='example page', slug='example-page', sub_slug='foo')
    TextCell.objects.create(page=page, placeholder='content', text='<p>foobar baz</p>', order=0)

    second_page = Page.objects.create(title='second page', slug='second-page')
    TextCell.objects.create(page=second_page, placeholder='content', text='<p>other baz</p>', order=0)
    sub_second_page = Page.objects.create(parent=second_page, title='sub second page', slug='sub-second-page')
    TextCell.objects.create(page=sub_second_page, placeholder='content', text='<p>other baz</p>', order=0)
    # not indexed: with snapshot
    third_page = Page.objects.create(title='second page', slug='third-page')
    TextCell.objects.create(page=third_page, placeholder='content', text='<p>other baz again</p>', order=0)
    third_page.snapshot = PageSnapshot.objects.create(page=third_page)
    third_page.save()

    index_site()

    cell = SearchCell.objects.create(
        page=page, placeholder='content', _search_services={'data': ['_text']}, order=1
    )

    resp = app.get('/ajax/search/%s/_text/?q=baz' % cell.pk, status=200)
    assert resp.text.count('<li') == 2
    cell._search_services = {'data': ['_text_page_second-page']}
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_second-page/?q=baz' % cell.pk, status=200)
    assert resp.text.count('<li') == 2
    cell._search_services = {'data': ['_text_page_sub-second-page']}
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_sub-second-page/?q=baz' % cell.pk, status=200)
    assert resp.text.count('<li') == 1
    # invalid page, search everywhere
    # with sub_slug
    cell._search_services = {'data': ['_text_page_example-page']}
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_example-page/?q=baz' % cell.pk, status=200)
    assert resp.text.count('<li') == 2
    # with snapshot
    cell._search_services = {'data': ['_text_page_third-page']}
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_third-page/?q=baz' % cell.pk, status=200)
    assert resp.text.count('<li') == 2
    # page does not exists, search everywhere
    cell._search_services = {'data': ['_text_page_foo']}
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_foo/?q=baz' % cell.pk, status=200)
    assert resp.text.count('<li') == 2
    # slug is not unique, search everywhere
    page.slug = 'sub-second-page'
    page.sub_slug = ''
    page.save()
    cell._search_services = {'data': ['_text_page_sub-second-page']}
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_sub-second-page/?q=baz' % cell.pk, status=200)
    assert resp.text.count('<li') == 2
    # search with custom title
    cell._search_services = {
        'data': ['_text', '_text_page_sub-second-page'],
        'options': {'_text_page_sub-second-page': {'title': 'Custom Title'}},
    }
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_sub-second-page/?q=baz' % cell.pk, status=200)
    assert 'Custom Title' in resp.text
    assert resp.text.count('<li') == 2


def test_search_external_links(app):
    page = Page(title='example page', slug='example-page')
    page.save()

    cell = SearchCell(page=page, placeholder='content', _search_services={'data': ['_text']}, order=0)
    cell.save()

    index_site()
    request = RequestFactory().get('/')
    request.user = AnonymousUser()
    hits = search_site(request, 'foobar')
    assert len(hits) == 0

    LinkCell(title='foobar', url='http://example.net', page=page, placeholder='content', order=0).save()
    index_site()

    hits = search_site(request, 'foobar')
    assert len(hits) == 1
    assert hits[0]['text'] == 'foobar'
    assert hits[0]['url'] == 'http://example.net'

    # second link with same target
    LinkCell(title='baz', url='http://example.net', page=page, placeholder='content', order=0).save()
    index_site()

    # add a second link with the same target
    hits = search_site(request, 'baz')
    assert len(hits) == 1
    assert hits[0]['text'] in ('foobar', 'baz')
    assert hits[0]['url'] == 'http://example.net'
    hits = search_site(request, 'foobar')
    assert len(hits) == 1
    assert hits[0]['text'] in ('foobar', 'baz')
    assert hits[0]['url'] == 'http://example.net'

    # update cell to search content only on its page
    cell._search_services = {'data': ['_text_page_%s' % page.slug]}
    cell.save()

    hits = search_site(request, 'foobar', pages=[page])
    assert len(hits) == 1


def test_search_external_links_in_list_of_links(app):
    page = Page(title='example page', slug='example-page')
    page.save()

    cell = SearchCell(page=page, placeholder='content', _search_services={'data': ['_text']}, order=0)
    cell.save()

    index_site()
    request = RequestFactory().get('/')
    request.user = AnonymousUser()
    hits = search_site(request, 'foobar')
    assert len(hits) == 0

    cell = LinkListCell.objects.create(order=0, page=page, placeholder='content')
    LinkCell.objects.create(
        page=page,
        placeholder=cell.link_placeholder,
        title='foobar example',
        url='http://example.net',
        order=0,
    )
    LinkCell.objects.create(
        page=page,
        placeholder=cell.link_placeholder,
        title='Example 2 Site',
        url='http://example.org/',
        order=1,
    )

    index_site()

    hits = search_site(request, 'foobar')
    assert len(hits) == 1
    assert hits[0]['text'] == 'foobar example'
    assert hits[0]['url'] == 'http://example.net'

    hits = search_site(request, 'example')
    assert len(hits) == 2


def test_manager_search_cell(settings, app, admin_user):
    page = Page.objects.create(title='One', slug='one', template_name='standard')
    cell = SearchCell.objects.create(page=page, placeholder='content', order=0)
    app = login(app)

    settings.KNOWN_SERVICES = {}
    assert cell._search_services == {}
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/delete/' % (page.pk, cell.pk)
        not in resp.text
    )

    settings.COMBO_SEARCH_SERVICES = SEARCH_SERVICES
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/add/'
        % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/edit/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/edit/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/edit/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/edit/'
        % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/delete/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/delete/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/delete/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/delete/'
        % (page.pk, cell.pk)
        not in resp.text
    )

    # add engines
    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    resp = app.get('/manage/pages/%s/' % page.pk)
    resp = resp.click(href='.*/search_searchcell-%s/engine/search1/add/' % cell.pk)
    resp = app.get('/manage/pages/%s/' % page.pk)
    resp = resp.click(href='.*/search_searchcell-%s/engine/search_tmpl/add/' % cell.pk)
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['_text', 'search1', 'search_tmpl']
    resp = app.get('/manage/pages/%s/' % page.pk)
    # '_text' is always available
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/add/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/add/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/add/'
        % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/edit/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/edit/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/edit/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/edit/'
        % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/delete/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/delete/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/delete/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/delete/'
        % (page.pk, cell.pk)
        not in resp.text
    )

    # edit engines
    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/edit/' % cell.pk)
    resp.form['title'] = 'Custom Title'
    resp.form['with_description'] = True
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['options']['_text'] == {
        'title': 'Custom Title',
        'with_description': True,
    }

    # delete engines
    resp = app.get('/manage/pages/%s/' % page.pk)
    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/delete/' % cell.pk)
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['search1', 'search_tmpl']

    settings.COMBO_SEARCH_SERVICES = {}
    # check there's no crash if search engines are removed from config
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/add/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/add/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/add/'
        % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/delete/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/delete/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/delete/'
        % (page.pk, cell.pk)
        not in resp.text
    )

    # add engines on page and sub pages
    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
    assert list(resp.context['form']['selected_page'].field.queryset) == [page]
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['search1', 'search_tmpl', '_text']
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/delete/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text_page_one/delete/' % (page.pk, cell.pk)
        not in resp.text
    )
    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
    resp.form['selected_page'] = page.pk
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['search1', 'search_tmpl', '_text', '_text_page_one']
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/delete/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text_page_one/delete/' % (page.pk, cell.pk)
        in resp.text
    )

    # remove engine
    resp = resp.click(href='.*/search_searchcell-%s/engine/_text_page_one/delete/' % cell.pk)
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()

    # add engine on page and sub pages, with a custom title and description
    resp = app.get('/manage/pages/%s/' % page.pk)
    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
    resp.form['selected_page'] = page.pk
    resp.form['title'] = 'Custom Title'
    resp.form['with_description'] = True
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['search1', 'search_tmpl', '_text', '_text_page_one']
    assert cell._search_services['options']['_text_page_one'] == {
        'title': 'Custom Title',
        'with_description': True,
    }


def test_manager_search_cell_order(settings, app, admin_user):
    settings.COMBO_SEARCH_SERVICES = SEARCH_SERVICES
    page = Page.objects.create(title='One', slug='one', template_name='standard')
    cell = SearchCell.objects.create(
        page=page,
        placeholder='content',
        order=0,
        _search_services={'data': ['_text', 'search1', 'search_tmpl']},
    )

    params = []
    new_order = [2, 3, 1]
    for service_slug, new_pos in zip(cell._search_services['data'], new_order):
        params.append(('pos_%s' % service_slug, str(new_pos)))

    app = login(app)
    resp = app.get(
        '/manage/search/pages/%s/cell/%s/engine/order?%s' % (page.pk, cell.get_reference(), urlencode(params))
    )
    assert resp.status_code == 204
    cell.refresh_from_db()
    assert cell._search_services == {'data': ['search_tmpl', '_text', 'search1']}


def test_manager_search_cell_tabs(app, admin_user):
    page = Page.objects.create(title='One', slug='one', template_name='standard')
    SearchCell.objects.create(page=page, placeholder='content', order=0)
    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.pk)

    assert not resp.pyquery('[data-tab-slug="general"] input[name$="title"]')
    assert resp.pyquery('[data-tab-slug="appearance"] input[name$="title"]')


def test_manager_waiting_index_message(app, admin_user):
    page = Page.objects.create(title='One', slug='one', template_name='standard')
    cell = SearchCell.objects.create(page=page, placeholder='content', order=0)
    TextCell.objects.create(page=page, placeholder='content', text='<p>foobar</p>', order=0)
    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert 'Content indexing has been scheduled' not in resp.text

    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
    resp = resp.form.submit('submit')
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert 'Content indexing has been scheduled' in resp.text

    index_site()
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert 'Content indexing has been scheduled' not in resp.text


def test_wcs_search_engines(settings, app):
    settings.KNOWN_SERVICES = {}
    search_engines = engines.get_engines()
    assert 'tracking-code' not in search_engines
    assert len([x for x in search_engines if x.startswith('formdata:')]) == 0
    assert len([x for x in search_engines if x.startswith('cards:')]) == 0

    settings.KNOWN_SERVICES = {'wcs': {'default': {'title': 'test', 'url': 'http://127.0.0.1:8999/'}}}

    settings.TEMPLATE_VARS['is_portal_agent'] = False
    search_engines = engines.get_engines()
    assert len([x for x in search_engines if x.startswith('formdata:')]) == 0

    settings.TEMPLATE_VARS['is_portal_agent'] = True
    search_engines = engines.get_engines()
    assert len([x for x in search_engines if x.startswith('formdata:')]) == 1
    for key, engine in search_engines.items():
        if key.startswith('formdata:'):
            assert '&include-anonymised=off' in engine['url']

    # create a page with sub_slug to enable card engines
    Page.objects.create(slug='foo', title='Foo', sub_slug='(?P<foo_id>[a-z0-9]+)')
    with mock.patch('combo.apps.wcs.utils.get_wcs_json') as mock_wcs:
        # no card model found
        mock_wcs.return_value = {}
        search_engines = engines.get_engines()
        assert len([x for x in search_engines if x.startswith('cards:')]) == 0
        assert mock_wcs.call_args_list[0][0][1] == 'api/cards/@list'
        mock_wcs.return_value = {'data': []}
        search_engines = engines.get_engines()
        assert len([x for x in search_engines if x.startswith('cards:')]) == 0

        # card model found, but related page does not exist
        mock_wcs.return_value = {'data': [{'id': 'card-bar', 'text': 'Card Bar'}]}
        search_engines = engines.get_engines()
        assert len([x for x in search_engines if x.startswith('cards:')]) == 0

        # related page exists
        page = Page.objects.create(slug='bar', title='Bar')
        for sub_slug in ['(?P<card-bar_id>[a-z0-9]+)', 'card-bar_id']:
            page.sub_slug = sub_slug
            page.save()
            search_engines = engines.get_engines()
            assert len([x for x in search_engines if x.startswith('cards:')]) == 1
            assert 'cards:c21f969b:card-bar' in search_engines
            card_engine = search_engines['cards:c21f969b:card-bar']
            assert card_engine['url'] == (
                'http://127.0.0.1:8999/api/cards/card-bar/list/'
                '{% if search_service.selected_custom_view %}{{ search_service.selected_custom_view }}{% endif %}'
                '?{% if not search_service.without_user %}NameID={{ user_nameid }}&{% endif %}q=%(q)s'
            )
            assert card_engine['hit_url_template'] == '/bar/{{ id }}'


def test_wcs_errors(settings, app):
    settings.KNOWN_SERVICES = {'wcs': {'default': {'title': 'test', 'url': 'http://127.0.0.1:8999/'}}}
    settings.TEMPLATE_VARS['is_portal_agent'] = True
    Page.objects.create(slug='foo', title='Foo', sub_slug='(?P<foo_id>[a-z0-9]+)')

    with mock.patch('combo.apps.wcs.utils.requests.get') as requests_get:
        mock_resp = Response()
        mock_resp.status_code = 500
        requests_get.return_value = mock_resp
        assert set(engines.get_engines().keys()) == {'tracking-code', 'formdata:c21f969b', '_text'}
    with mock.patch('combo.apps.wcs.utils.requests.get') as requests_get:
        requests_get.side_effect = ConnectionError()
        assert set(engines.get_engines().keys()) == {'tracking-code', 'formdata:c21f969b', '_text'}
    with mock.patch('combo.apps.wcs.utils.requests.get') as requests_get:
        mock_resp = Response()
        mock_resp.status_code = 404
        requests_get.return_value = mock_resp
        assert set(engines.get_engines().keys()) == {'tracking-code', 'formdata:c21f969b', '_text'}


@mock.patch('combo.apps.wcs.utils.get_wcs_json')
def test_wcs_add_search_engines(mock_wcs, settings, app, admin_user):
    settings.KNOWN_SERVICES = {'wcs': {'default': {'title': 'test', 'url': 'http://127.0.0.1:8999/'}}}
    settings.TEMPLATE_VARS['is_portal_agent'] = True
    Page.objects.create(slug='bar', title='Bar', sub_slug='card-bar_id')
    mock_wcs.return_value = {'data': [{'id': 'card-bar', 'text': 'Card Bar'}]}

    page = Page.objects.create(title='One', slug='one', template_name='standard')
    cell = SearchCell.objects.create(page=page, placeholder='content', order=0)
    app = login(app)

    resp = app.get('/manage/pages/%s/' % page.pk)
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/cards:c21f969b:card-bar/add/'
        % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/cards:c21f969b:card-bar/edit/'
        % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/cards:c21f969b:card-bar/delete/'
        % (page.pk, cell.pk)
        not in resp.text
    )

    resp = resp.click(href='.*/search_searchcell-%s/engine/cards:c21f969b:card-bar/add/' % cell.pk)
    assert resp.context['form'].fields['selected_view'].choices == [(None, '- All cards -')]
    resp.form['title'] = 'Custom Title'
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['cards:c21f969b:card-bar']
    assert cell._search_services['options']['cards:c21f969b:card-bar'] == {'title': 'Custom Title'}

    mock_wcs.return_value = {
        'data': [
            {
                'id': 'card-bar',
                'text': 'Card Bar',
                'custom_views': [{'id': 'foo', 'text': 'Foo'}, {'id': 'baz', 'text': 'Baz'}],
            }
        ]
    }
    resp = resp.follow()
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/cards:c21f969b:card-bar/add/'
        % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/cards:c21f969b:card-bar/edit/'
        % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/cards:c21f969b:card-bar/delete/'
        % (page.pk, cell.pk)
        in resp.text
    )
    resp = resp.click(href='.*/search_searchcell-%s/engine/cards:c21f969b:card-bar/add/' % cell.pk)
    assert resp.context['form'].fields['selected_view'].choices == [
        (None, '- All cards -'),
        ('foo', 'Foo'),
        ('baz', 'Baz'),
    ]
    resp.form['selected_view'] = 'foo'
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['cards:c21f969b:card-bar', 'cards:c21f969b:card-bar:foo']

    # edit engine
    resp = resp.follow()
    resp = resp.click(href='.*/search_searchcell-%s/engine/cards:c21f969b:card-bar/edit/' % cell.pk)
    resp.form['title'] = 'Custom Title Foo Bar'
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['options']['cards:c21f969b:card-bar'] == {'title': 'Custom Title Foo Bar'}

    # remove engine
    resp = resp.follow()
    resp = resp.click(href='.*/search_searchcell-%s/engine/cards:c21f969b:card-bar:foo/delete/' % cell.pk)
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['cards:c21f969b:card-bar']

    # without_user ?
    resp = resp.follow()
    resp = resp.click(href='.*/search_searchcell-%s/engine/cards:c21f969b:card-bar/add/' % cell.pk)
    resp.form['without_user'] = True
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == [
        'cards:c21f969b:card-bar',
        'cards:c21f969b:card-bar__without-user__',
    ]

    resp = resp.follow()
    resp = resp.click(href='.*/search_searchcell-%s/engine/cards:c21f969b:card-bar/add/' % cell.pk)
    resp.form['selected_view'] = 'foo'
    resp.form['without_user'] = True
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == [
        'cards:c21f969b:card-bar',
        'cards:c21f969b:card-bar__without-user__',
        'cards:c21f969b:card-bar__without-user__:foo',
    ]


@mock.patch('combo.apps.wcs.utils.get_wcs_json')
@mock.patch('combo.apps.search.models.requests.get')
def test_wcs_search_cell(requests_get, mock_wcs, settings, app):
    settings.KNOWN_SERVICES = {'wcs': {'default': {'title': 'test', 'url': 'http://127.0.0.1:8999/'}}}
    settings.TEMPLATE_VARS['is_portal_agent'] = True
    Page.objects.create(slug='bar', title='Bar', sub_slug='(?P<card-bar_id>[a-z0-9]+)')
    mock_wcs.return_value = {
        'data': [{'id': 'card-bar', 'text': 'Card Bar', 'custom_views': [{'id': 'foo', 'text': 'Foo'}]}]
    }
    response = {'err': 0, 'data': []}
    mock_json = mock.Mock()
    mock_json.json.return_value = response
    requests_get.return_value = mock_json

    page = Page.objects.create(title='One', slug='one', template_name='standard')
    cell = SearchCell.objects.create(page=page, placeholder='content', order=0)
    cell._search_services = {
        'data': [
            'cards:c21f969b:card-bar',
            'cards:c21f969b:card-bar:foo',
            'cards:c21f969b:card-bar__without-user__',
            'cards:c21f969b:card-bar__without-user__:foo',
        ]
    }
    cell.save()
    assert cell.search_services[0]['custom_views'] == [{'id': 'foo', 'text': 'Foo'}]
    assert cell.search_services[0]['label'] == 'Card Bar'
    assert cell.search_services[0]['slug'] == 'cards:c21f969b:card-bar'
    assert 'selected_custom_view' not in cell.search_services[0]
    assert cell.search_services[1]['custom_views'] == [{'id': 'foo', 'text': 'Foo'}]
    assert cell.search_services[1]['label'] == 'Card Bar - Foo'
    assert cell.search_services[1]['slug'] == 'cards:c21f969b:card-bar:foo'
    assert cell.search_services[1]['selected_custom_view'] == 'foo'

    app.get('/ajax/search/%s/cards:c21f969b:card-bar/?q=foobar' % cell.pk)
    assert (
        requests_get.call_args_list[0][0][0]
        == 'http://127.0.0.1:8999/api/cards/card-bar/list/?NameID=&q=foobar'
    )
    requests_get.reset_mock()
    app.get('/ajax/search/%s/cards:c21f969b:card-bar:foo/?q=foobar' % cell.pk)
    assert (
        requests_get.call_args_list[0][0][0]
        == 'http://127.0.0.1:8999/api/cards/card-bar/list/foo?NameID=&q=foobar'
    )
    requests_get.reset_mock()
    app.get('/ajax/search/%s/cards:c21f969b:card-bar__without-user__/?q=foobar' % cell.pk)
    assert requests_get.call_args_list[0][0][0] == 'http://127.0.0.1:8999/api/cards/card-bar/list/?q=foobar'
    requests_get.reset_mock()
    app.get('/ajax/search/%s/cards:c21f969b:card-bar__without-user__:foo/?q=foobar' % cell.pk)
    assert (
        requests_get.call_args_list[0][0][0] == 'http://127.0.0.1:8999/api/cards/card-bar/list/foo?q=foobar'
    )

    # test with unknown view selected
    mock_wcs.return_value = {'data': [{'id': 'card-bar', 'text': 'Card Bar', 'custom_views': []}]}
    del cell.search_services  # clear cache
    assert cell.search_services[0]['custom_views'] == []
    assert cell.search_services[0]['label'] == 'Card Bar'
    assert cell.search_services[0]['slug'] == 'cards:c21f969b:card-bar'
    assert 'selected_custom_view' not in cell.search_services[0]
    assert cell.search_services[1]['custom_views'] == []
    assert cell.search_services[1]['label'] == 'Card Bar - None'  # unknown view
    assert cell.search_services[1]['slug'] == 'cards:c21f969b:card-bar:foo'
    assert cell.search_services[1]['selected_custom_view'] == 'foo'

    requests_get.reset_mock()
    app.get('/ajax/search/%s/cards:c21f969b:card-bar:foo/?q=foobar' % cell.pk)
    assert (
        requests_get.call_args_list[0][0][0]
        == 'http://127.0.0.1:8999/api/cards/card-bar/list/foo?NameID=&q=foobar'
    )

    # test with unknown card model selected
    mock_wcs.return_value = {'data': []}
    del cell.search_services  # clear cache
    assert cell.search_services == []

    requests_get.reset_mock()
    app.get('/ajax/search/%s/cards:c21f969b:card-bar/?q=foobar' % cell.pk)
    assert requests_get.call_args_list == []


def test_profile_search_engines(settings, app):
    settings.KNOWN_SERVICES = {}
    search_engines = engines.get_engines()
    assert 'users' not in search_engines

    settings.KNOWN_SERVICES = {'authentic': {'default': {'title': 'authentic', 'url': 'https://authentic/'}}}
    search_engines = engines.get_engines()
    assert 'users' not in search_engines

    page = Page.objects.create(slug='users', title='Users', sub_slug='(?P<name_id>[a-z0-9]+)')
    search_engines = engines.get_engines()
    assert 'users' in search_engines

    page.sub_slug = 'name_id'
    page.save()
    search_engines = engines.get_engines()
    assert 'users' in search_engines


def test_profile_add_search_engines(settings, app, admin_user):
    settings.KNOWN_SERVICES = {'authentic': {'default': {'title': 'authentic', 'url': 'https://authentic/'}}}
    Page.objects.create(slug='users', title='Users', sub_slug='(?P<name_id>[a-z0-9]+)')

    page = Page.objects.create(title='One', slug='one', template_name='standard')
    cell = SearchCell.objects.create(page=page, placeholder='content', order=0)

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.pk)
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/users/add/' % (page.pk, cell.pk)
        in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/users/edit/' % (page.pk, cell.pk)
        not in resp.text
    )
    assert (
        '/manage/search/pages/%s/cell/search_searchcell-%s/engine/users/delete/' % (page.pk, cell.pk)
        not in resp.text
    )

    # check default value from settings is retained in configuration form
    users_engine = engines.get('users').copy()
    settings.COMBO_SEARCH_SERVICES = {'users': users_engine}
    with mock.patch.dict(
        users_engine, {'hit_description_template': users_engine['hit_description_template'] + ' xxx'}
    ):
        form_resp = resp.click(href='.*/search_searchcell-%s/engine/users/add/' % cell.pk)
        assert form_resp.form['description_template'].value.endswith(' xxx')

    resp = resp.click(href='.*/search_searchcell-%s/engine/users/add/' % cell.pk)
    resp.form['title'] = 'Custom Title'
    assert not resp.form['description_template'].value.endswith(' xxx')
    resp.form['description_template'] = 'Foo Bar'
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == ['users']
    assert cell._search_services['options']['users'] == {
        'title': 'Custom Title',
        'description_template': 'Foo Bar',
    }

    # edit engine
    resp = app.get('/manage/pages/%s/' % page.pk)
    resp = resp.click(href='.*/search_searchcell-%s/engine/users/edit/' % cell.pk)
    resp.form['title'] = 'Custom Title Foo Bar'
    resp.form['description_template'] = ''
    resp.form['description_template'] = 'Foo Bar Baz'
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['options']['users'] == {
        'title': 'Custom Title Foo Bar',
        'description_template': 'Foo Bar Baz',
    }

    # remove engine
    resp = resp.follow()
    resp = resp.click(href='.*/search_searchcell-%s/engine/users/delete/' % cell.pk)
    assert resp.status_int == 302
    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
    cell.refresh_from_db()
    assert cell._search_services['data'] == []


def test_search_profile(settings, app):
    settings.KNOWN_SERVICES = {'authentic': {'default': {'title': 'authentic', 'url': 'https://authentic/'}}}
    Page.objects.create(slug='users', title='Users', sub_slug='(?P<name_id>[a-z0-9]+)')

    page = Page.objects.create(title='One', slug='one', template_name='standard')
    cell = SearchCell.objects.create(page=page, placeholder='content', order=0)

    cell._search_services = {'data': ['users']}
    cell.save()
    with mock.patch('combo.apps.search.models.requests.get') as requests_get:
        response = {
            'results': [
                {
                    'id': '1',
                    'uuid': '982a7c65f6c14a2c80307afd9c4bb368',
                    'first_name': 'Foo',
                    'last_name': 'Bar',
                    'email': 'foo@bar.com',
                    'phone': '0505050505',
                    'mobile': '0606060606',
                    'address': 'There',
                    'zipcode': '75000',
                    'city': 'Paris',
                }
            ],
        }
        mock_json = mock.Mock()
        mock_json.json.return_value = response
        requests_get.return_value = mock_json

        resp = app.get('/ajax/search/%s/users/?q=foo' % cell.pk, status=200)
        assert resp.text.count('<li>') == 1
        assert '<li><a href="/users/982a7c65f6c14a2c80307afd9c4bb368/">Foo Bar</a>' in resp.text
        assert '<div>foo@bar.com 📞 0505050505 📱 0606060606 📨 There 75000 Paris</div>' in resp.text.replace(
            '\n', ''
        )

        cell._search_services['options'] = {'users': {'description_template': '{{ email }} Foo Bar'}}
        cell.save()
        resp = app.get('/ajax/search/%s/users/?q=foo' % cell.pk, status=200)
        assert resp.text.count('<li>') == 1
        assert '<li><a href="/users/982a7c65f6c14a2c80307afd9c4bb368/">Foo Bar</a>' in resp.text
        assert '<div>foo@bar.com Foo Bar</div>' in resp.text


def test_private_search(app):
    page = Page(title='example page', slug='example-page')
    page.save()

    TextCell(page=page, placeholder='content', text='<p>foobar</p>', order=0, public=False).save()
    TextCell(page=page, placeholder='content', text='<p>barfoo</p>', order=0, public=True).save()

    request = RequestFactory().get('/')
    request.user = AnonymousUser()
    index_site()
    hits = search_site(request, 'foobar')
    assert len(hits) == 0
    hits = search_site(request, 'barfoo')
    assert len(hits) == 1

    request.user = User.objects.create_user(username='normal-user')
    hits = search_site(request, 'foobar')
    assert len(hits) == 1
    hits = search_site(request, 'barfoo')
    assert len(hits) == 1


def test_restricted_search(app):
    group = Group(name='plop')
    group.save()

    page = Page(title='example page', slug='example-page')
    page.save()

    cell = TextCell(page=page, placeholder='content', text='<p>foobar</p>', order=0, public=False)
    cell.save()
    cell.groups.set([group])
    TextCell(page=page, placeholder='content', text='<p>barfoo</p>', order=0, public=False).save()
    index_site()

    # first cell is restricted, it's not found
    request = RequestFactory().get('/')
    request.user = User.objects.create_user(username='normal-user')
    hits = search_site(request, 'foobar')
    assert len(hits) == 0
    hits = search_site(request, 'barfoo')
    assert len(hits) == 1

    page.groups.set([group])
    index_site()

    # page is restricted, no cell is found
    hits = search_site(request, 'foobar')
    assert len(hits) == 0
    hits = search_site(request, 'barfoo')
    assert len(hits) == 0

    # user is in group, gets a result
    request.user.groups.set([group])
    hits = search_site(request, 'foobar')
    assert len(hits) == 1
    hits = search_site(request, 'barfoo')
    assert len(hits) == 1

    # cell is excluded from group view
    cell.restricted_to_unlogged = True
    cell.save()
    index_site()

    hits = search_site(request, 'foobar')
    assert len(hits) == 0
    hits = search_site(request, 'barfoo')
    assert len(hits) == 1


def test_no_sub_slug_search(app):
    page = Page(title='example page', slug='example-page')
    page.save()
    TextCell(page=page, placeholder='content', text='<p>foobar</p>', order=0, public=True).save()
    page = Page(title='example page with sub_slug', slug='sub-slugged-page', sub_slug=r'(?P<foo>\d+)')
    page.save()
    TextCell(page=page, placeholder='content', text='<p>barfoo</p>', order=0, public=True).save()

    request = RequestFactory().get('/')
    request.user = AnonymousUser()
    index_site()
    hits = search_site(request, 'foobar')
    assert len(hits) == 1
    hits = search_site(request, 'barfoo')
    assert len(hits) == 0


def test_index_site_inactive_placeholder(app):
    page = Page.objects.create(title='page', slug='example-page')
    cell = TextCell.objects.create(page=page, placeholder='content', text='<p>foobar</p>', order=0)

    assert cell.is_placeholder_active() is True
    index_site()
    assert IndexedCell.objects.count() == 1

    cell.placeholder = ''
    cell.save()
    assert cell.is_placeholder_active() is False
    index_site()
    assert IndexedCell.objects.count() == 0


def test_index_site_invalid_cell(app):
    request = RequestFactory().get('/')
    page = Page.objects.create(title='page', slug='example-page')
    cell = TextCell.objects.create(page=page, placeholder='content', text='<p>foobar</p>', order=0)

    index_site()
    assert IndexedCell.objects.count() == 1

    # invalid cell since just now
    cell.mark_as_invalid(reason_code='foobar')
    assert cell.is_visible(request) is True
    index_site()
    assert IndexedCell.objects.count() == 1

    # invalid cell since two days
    validity_info = cell.get_validity_info()
    validity_info.invalid_since = now() - datetime.timedelta(days=2)
    validity_info.save()
    assert cell.is_visible(request) is False
    index_site()
    assert IndexedCell.objects.count() == 0


def test_index_site_num_queries(settings, app):
    group = Group.objects.create(name='plop')
    for i in range(0, 10):
        page = Page.objects.create(title='page %s' % i, slug='example-page-%s' % i)
        page.groups.set([group])
        for j in range(0, 5):
            cell = TextCell.objects.create(
                page=page, placeholder='content', text='<p>foobar %s</p>' % j, order=j, public=False
            )
            cell.groups.set([group])

    index_site()  # populate cache
    assert IndexedCell.objects.count() == 50
    with CaptureQueriesContext(connection) as ctx:
        index_site()
        assert len(ctx.captured_queries) == 175

    SearchCell.objects.create(
        page=page, placeholder='content', order=0, _search_services={'data': ['search1']}
    )
    # search cells are not indexed
    index_site()
    assert IndexedCell.objects.count() == 50


@mock.patch('combo.apps.search.engines.get_engines')
def test_index_site_search_engines_load(get_engines_mock, settings, app):
    # be sure that get_engines is not called during page indexation
    get_engines_mock.side_effect = Exception

    settings.COMBO_SEARCH_SERVICES = {}

    page = Page.objects.create(title='page', slug='example-page')
    SearchCell.objects.create(
        page=page, placeholder='content', order=0, _search_services={'data': ['search1']}
    )
    TextCell.objects.create(page=page, placeholder='content', text='<p>foobar</p>', order=0)

    # no exception raised
    index_site()


@pytest.mark.skipif(connection.vendor != 'postgresql', reason='only postgresql is supported')
def test_search_by_page_title(app):
    query = 'nanoparticle electrochemistry'
    # First page containing a cell whose text matches the query
    page = Page(title='example page', slug='example-page')
    page.save()
    TextCell(
        page=page,
        placeholder='content',
        text='<p>Some nanoparticle electrochemistry content here</p>',
        order=0,
        public=True,
    ).save()
    # Second page whose title matches the search query
    page_of_interest = Page(title='Nanoparticle electrochemistry', slug='page-of-interest')
    page_of_interest.save()
    TextCell(
        page=page_of_interest,
        placeholder='content',
        text='<p>Some random text here</p>',
        order=0,
        public=True,
    ).save()

    request = RequestFactory().get('/')
    request.user = AnonymousUser()
    index_site()
    hits = search_site(request, query)
    # Check that title matching gets precedence over content matching
    assert len(hits) == 2
    assert hits[0]['text'] == page_of_interest.title
    assert hits[0]['url'] == f'/{page_of_interest.slug}/'
    assert hits[0]['rank'] > hits[1]['rank']


def test_search_same_link_multiple_pages(settings, app):
    settings.KNOWN_SERVICES = {}

    page = Page.objects.create(title='first page', slug='one')
    LinkCell.objects.create(
        title='foobar', url='http://example.net', page=page, placeholder='content', order=0
    )
    second_page = Page.objects.create(title='second page', slug='two')
    LinkCell.objects.create(
        title='barfoo', url='http://example.net', page=second_page, placeholder='content', order=0
    )
    index_site()

    cell = SearchCell.objects.create(
        page=page, placeholder='content', _search_services={'data': ['_text_page_one']}, order=1
    )
    resp = app.get('/ajax/search/%s/_text_page_one/?q=foobar' % cell.pk, status=200)
    assert resp.text.count('<li') == 1
    resp = app.get('/ajax/search/%s/_text_page_one/?q=barfoo' % cell.pk, status=200)
    assert resp.text.count('<li') == 0

    cell._search_services = {'data': ['_text_page_two']}
    cell.save()
    resp = app.get('/ajax/search/%s/_text_page_two/?q=foobar' % cell.pk, status=200)
    assert resp.text.count('<li') == 0
    resp = app.get('/ajax/search/%s/_text_page_two/?q=barfoo' % cell.pk, status=200)
    assert resp.text.count('<li') == 1