hobo/tests/test_matomo_utils.py

import json
from unittest import mock

import pytest
from django.test import override_settings
from django.utils.encoding import force_text
from requests import Response

from hobo.environment.models import Combo, Fargo, Hobo, Variable, Wcs
from hobo.matomo.utils import (
    MatomoError,
    MatomoException,
    MatomoWS,
    auto_configure_matomo,
    compute_cnil_acknowledgment_level,
    get_tenant_name_and_public_urls,
    get_tracking_js,
    get_variable,
    get_variable_value,
    put_tracking_js,
    upgrade_javascript_tag,
    upgrade_site,
    upgrade_user,
)

pytestmark = pytest.mark.django_db

CONFIG = {
    'URL': 'https://matomo.test',
    'TOKEN_AUTH': '1234',
    'EMAIL_TEMPLATE': 'noreply+%(user_login)s@entrouvert.test',
}

MATOMO_SUCCESS = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <success message="ok" />
</result>
"""

MATOMO_ERROR = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <error message="here is the error message" />
</result>
"""

MATOMO_BAD_RESPONSE_1 = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <success message="KO" />
</result>
"""

MATOMO_BAD_RESPONSE_2 = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <success>no message attribute</success>
    <not_success>no success tag</not_success>
</result>
"""

GET_SITE_42_FROM_URL = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <row>
        <idsite>42</idsite>
        <moretags>...</moretags>
    </row>
</result>
"""

GET_NO_SITE_FROM_URL = b"""<?xml version="1.0" encoding="utf-8" ?>
<result />
"""

GET_SITE_BAD_QUERY = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
        <error message="Please specify a value for 'url'." />
</result>
"""

GET_SITE_BAD_RESPONSE = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <row>
        <not_idsite>there is no idsite tag</not_idsite>
        <moretags>...</moretags>
    </row>
</result>
"""

ADD_SITE_SUCCESS = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>42</result>
"""

ADD_SITE_ERROR = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
        <error message="Please specify a value for 'siteName'." />
</result>
"""

ADD_SITE_BAD_RESPONSE = b"""<?xml version="1.0" encoding="utf-8" ?>
<not_result>no result tag</not_result>
"""

ADD_SITE_ALIAS_URLS_SUCCESS = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>1</result>
"""

ADD_SITE_ALIAS_URLS_ERROR = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
	<error message="Please specify a value for 'idSite'." />
</result>
"""

USER_ALREADY_THERE = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <error message="Username 'hobo.dev.publik.love' already exists." />
</result>"""

MAIL_ALREADY_THERE = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <error message="User with email 'hobo.dev.publik.love@testor.org' already exists." />
</result>"""

BAD_CREDENTIAL = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <error message="You can\'t access this resource as it requires a \'superuser\' access." />
</result>"""

DEL_UNKNOWN_USER = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
        <error message="User 'hobo.dev.publik.love' doesn't exist." />
</result>
"""

JAVASCRIPT_TAG = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>&lt;!-- Matomo --&gt;
&lt;script type=&quot;text/javascript&quot;&gt;
  var _paq = window._paq || [];
  /* tracker methods like &quot;setCustomDimension&quot; should be called before &quot;trackPageView&quot; */
  _paq.push(['trackPageView']);
  _paq.push(['enableLinkTracking']);
  (function() {
    var u=&quot;//matomo-test.entrouvert.org/&quot;;
    _paq.push(['setTrackerUrl', u+'piwik.php']);
    _paq.push(['setSiteId', '7']);
    var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
    g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s);
  })();
&lt;/script&gt;
&lt;!-- End Matomo Code --&gt;
</result>
"""

ENHANCED_JAVASCRIPT_TAG = """// Matomo
  var _paq = window._paq || [];

  // disallow cookie's time extension
  _paq.push([function() {
    var self = this;
    function getOriginalVisitorCookieTimeout() {
      var now = new Date(),
      nowTs = Math.round(now.getTime() / 1000),
      visitorInfo = self.getVisitorInfo();
      var createTs = parseInt(visitorInfo[2]);
      var cookieTimeout = 33696000; // 13 months in seconds
      var originalTimeout = createTs + cookieTimeout - nowTs;
      return originalTimeout;
    }
    this.setVisitorCookieTimeout( getOriginalVisitorCookieTimeout() );
  }]);

  /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
  _paq.push(['trackPageView']);
  _paq.push(['enableLinkTracking']);
  (function() {
    var u="//matomo-test.entrouvert.org/";
    _paq.push(['setTrackerUrl', u+'piwik.php']);
    _paq.push(['setSiteId', '7']);
    var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
    g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s);
  })();
// End Matomo Code
"""

JAVASCRIPT_TAG_BAD_RESPONSE = b"""<?xml version="1.0" encoding="utf-8" ?>
<no_result_tag/>
"""

PING_SUCCESS = '{"status":"success","tracked":1,"invalid":0}'
PING_ERROR = '{"status":"not success"}'
PING_NOSTATUS_ERROR = '{}'
PING_NODICT_ERROR = '"json but not a dict"'


def requests_post_mocked_replies(contents):
    """buid an iterator for mock's side_effect parameter"""
    responses = []
    for content in contents:
        response = Response()

        # response may be XML or JSON
        if content[0] == '{':
            response.json = mock.MagicMock(return_value=json.loads(content))
        response._content = content

        response.status_code = 200
        responses.append(response)
    return responses


def test_get_variable():
    """hobo variables from"""

    # create the variable with '' value if not there
    id_site_var = get_variable('name1')
    assert id_site_var.value == ''

    # retrieve the variable if already there
    Variable.objects.create(name='name2', value='42')
    id_site_var = get_variable('name2')
    assert id_site_var.value == '42'


def test_get_variable_value():
    """hobo variables from DB"""

    # variable not there: return default value
    assert get_variable_value('name1') == ''
    assert get_variable_value('name2', default='42') == '42'

    # variable already there
    get_variable('name3', '42')
    assert get_variable_value('name3') == '42'


def test_get_tenant_name_and_public_urls():
    Combo.objects.create(base_url='https://combo.dev.publik.love', template_name='...portal-user...')
    Combo.objects.create(base_url='https://agent-combo.dev.publik.love', template_name='...portal-agent...')
    Combo.objects.create(base_url='https://no-template-combo.dev.publik.love')
    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
    Fargo.objects.create(base_url='https://fargo.dev.publik.love')
    Hobo.objects.create(base_url='https://hobo.dev.publik.love')
    tenant_name, site_urls = get_tenant_name_and_public_urls()
    assert tenant_name == 'combo.dev.publik.love'
    assert site_urls == [
        'https://combo.dev.publik.love/',
        'https://wcs.dev.publik.love/',
        'https://fargo.dev.publik.love/',
    ]


def test_get_tenant_name_and_public_urls_on_primary():
    Hobo.objects.create(base_url='https://hobo.ville1.love', slug='hobo-ville1')
    Hobo.objects.create(base_url='https://hobo.ville2.love', slug='hobo-ville2')
    Combo.objects.create(
        base_url='https://combo.agglo.love', template_name='...portal-user...', slug='portal'
    )
    Combo.objects.create(
        base_url='https://combo.ville1.love',
        template_name='...portal-user...',
        slug='_hobo-ville1_portal',
        secondary=True,
    )
    Combo.objects.create(
        base_url='https://combo.ville2.love',
        template_name='...portal-user...',
        slug='_hobo-ville2_portal',
        secondary=True,
    )
    Combo.objects.create(
        base_url='https://agent-combo.agglo.love', template_name='...portal-agent...', slug='portal-agent'
    )
    Combo.objects.create(
        base_url='https://agent-combo.ville1.love',
        template_name='...portal-agent...',
        slug='_hobo-ville1_portal-agent',
        secondary=True,
    )
    Combo.objects.create(
        base_url='https://agent-combo.ville2.love',
        template_name='...portal-agent...',
        slug='_hobo-ville2_portal-agent',
        secondary=True,
    )
    Wcs.objects.create(base_url='https://wcs.agglo.love', slug='eservices')
    Wcs.objects.create(base_url='https://wcs.ville1.love', slug='_hobo-ville1_eservices', secondary=True)
    Wcs.objects.create(base_url='https://wcs.ville2.love', slug='_hobo-ville2_eservices', secondary=True)
    Fargo.objects.create(base_url='https://fargo.agglo.love', slug='porte-doc')
    Fargo.objects.create(base_url='https://fargo.ville1.love', slug='_hobo-ville1_porte-doc', secondary=True)
    Fargo.objects.create(base_url='https://fargo.ville2.love', slug='_hobo-ville2_porte-doc', secondary=True)

    tenant_name, site_urls = get_tenant_name_and_public_urls()
    assert tenant_name == 'combo.agglo.love'
    assert site_urls == ['https://combo.agglo.love/', 'https://wcs.agglo.love/', 'https://fargo.agglo.love/']


def test_get_tenant_name_and_public_urls_on_secondary():
    Hobo.objects.create(base_url='https://hobo.agglo.love', slug='_interco_hobo')
    Hobo.objects.create(base_url='https://hobo.ville1.love', slug='_interco_hobo-ville1')
    Hobo.objects.create(base_url='https://hobo.ville2.love', slug='_interco_hobo-ville2')
    Combo.objects.create(
        base_url='https://combo.agglo.love',
        template_name='...portal-user...',
        slug='_interco_portal',
        secondary=True,
    )
    Combo.objects.create(
        base_url='https://combo.ville2.love', template_name='...portal-user...', slug='portal'
    )
    Combo.objects.create(
        base_url='https://agent-combo.agglo.love',
        template_name='...portal-agent...',
        slug='_interco_portal-agent',
        secondary=True,
    )
    Combo.objects.create(
        base_url='https://agent-combo.ville2.love', template_name='...portal-agent...', slug='portal-agent'
    )
    Wcs.objects.create(base_url='https://wcs.agglo.love', slug='_interco_eservices', secondary=True)
    Wcs.objects.create(base_url='https://wcs.ville2.love', slug='eservices')
    Fargo.objects.create(base_url='https://fargo.agglo.love', slug='_interco_porte-doc', secondary=True)
    Fargo.objects.create(base_url='https://fargo.ville2.love', slug='porte-doc')

    tenant_name, site_urls = get_tenant_name_and_public_urls()
    assert tenant_name == 'combo.ville2.love'
    assert site_urls == [
        'https://combo.ville2.love/',
        'https://wcs.ville2.love/',
        'https://fargo.ville2.love/',
    ]


def test_matomo_constructor():
    """build the matomo webservice object"""
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()
        assert matomo.url_ws_base == 'https://matomo.test'
        assert matomo.token_auth == '1234'

    with override_settings(MATOMO_SERVER={}):
        with pytest.raises(MatomoException, match="no settings for matomo: 'URL'"):
            matomo = MatomoWS()


def test_parse_response():
    """parser used by all matomo webservice calls"""
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # no error (expected format)
        content = b"""<?xml version="1.0" encoding="utf-8" ?><ok/>"""
        tree = matomo.parse_response(content)
        assert tree.tag == 'ok'

        # error (not XML format)
        content = """this is not XML"""
        with pytest.raises(MatomoException, match="XMLSyntaxError: Start tag expected"):
            tree = matomo.parse_response(content)


def test_parse_error_message():
    """error handler used by all matomo webservice calls"""
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # no error (expected format)
        content = b"""<?xml version="1.0" encoding="utf-8" ?><ok/>"""
        tree = matomo.parse_response(content)
        matomo.raise_on_error(tree)
        assert tree.tag == 'ok'

        # error (expected format)
        content = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <error message="here is the error message" />
</result>
"""
        tree = matomo.parse_response(content)
        with pytest.raises(MatomoError, match='here is the error message'):
            matomo.raise_on_error(tree)

        # error (unexpected format)
        content = b"""<?xml version="1.0" encoding="utf-8" ?>
<result>
    <error>no 'message' attribute here</error>
</result>
"""
        tree = matomo.parse_response(content)
        with pytest.raises(MatomoException, match='internal error'):
            matomo.raise_on_error(tree)


def test_assert_success():
    """webservice to add new user"""
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success
        tree = matomo.parse_response(MATOMO_SUCCESS)
        matomo.raise_on_error(tree)
        assert matomo.assert_success(tree, 'me') is True

        # error (KO instead of ok)
        tree = matomo.parse_response(MATOMO_BAD_RESPONSE_1)
        matomo.raise_on_error(tree)
        with pytest.raises(MatomoException, match='me fails'):
            matomo.assert_success(tree, 'me')

        # error (no message attribute)
        tree = matomo.parse_response(MATOMO_BAD_RESPONSE_2)
        matomo.raise_on_error(tree)
        with pytest.raises(MatomoException, match='me fails'):
            matomo.assert_success(tree, 'me')


@mock.patch('requests.post')
def test_call(mocked_post):
    """test status_code"""
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success (status 200)
        content = MATOMO_SUCCESS
        mocked_post.return_value.status_code = 200
        mocked_post.return_value.content = content
        matomo.call({})

        # failure (status 500)
        mocked_post.return_value.status_code = 500
        with pytest.raises(MatomoException, match='unexpected status code: 500'):
            matomo.call({})


@mock.patch('requests.post')
def test_get_site_from_site_url(mocked_post):
    """webservice to test if the site is already registered"""
    mocked_post.return_value.status_code = 200
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # site already here
        content = GET_SITE_42_FROM_URL
        mocked_post.return_value.content = content
        assert matomo.get_site_id_from_site_url('combo.dev.publik.love') == '42'

        # no such url
        content = GET_NO_SITE_FROM_URL
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match='url not found'):
            matomo.get_site_id_from_site_url('combo.dev.publik.love')

        # error on empty id
        content = GET_SITE_BAD_QUERY
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="Please specify a value for 'url'."):
            matomo.get_site_id_from_site_url('combo.dev.publik.love')

        # bad response (error on success response)
        content = GET_SITE_BAD_RESPONSE
        mocked_post.return_value.content = content
        with pytest.raises(MatomoException, match='get_site_id_from_site_url fails'):
            matomo.get_site_id_from_site_url('combo.dev.publik.love')


@mock.patch('requests.post')
def test_add_site(mocked_post):
    """webservice to add a new site"""
    mocked_post.return_value.status_code = 200
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success
        content = ADD_SITE_SUCCESS
        mocked_post.return_value.content = content
        site_id = matomo.add_site("hobo.dev.publik.love")
        assert site_id == '42'

        # error
        content = ADD_SITE_ERROR
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="Please specify a value for 'siteName'."):
            site_id = matomo.add_site("hobo.dev.publik.love")

        # strange message
        content = ADD_SITE_BAD_RESPONSE
        mocked_post.return_value.content = content
        with pytest.raises(MatomoException, match='add_site fails'):
            site_id = matomo.add_site("hobo.dev.publik.love")


@mock.patch('requests.post')
def test_add_site_alias_url(mocked_post):
    """webservice to add a new site"""
    mocked_post.return_value.status_code = 200
    urls = ['https://combo.dev.publik.love', 'https://wcs.dev.publik.love']
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success
        content = ADD_SITE_ALIAS_URLS_SUCCESS
        mocked_post.return_value.content = content
        site_id = matomo.add_site_alias_urls('42', urls)
        assert site_id == '1'

        # error
        content = ADD_SITE_ALIAS_URLS_ERROR
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="Please specify a value for 'idSite'."):
            site_id = matomo.add_site_alias_urls(None, urls)

        # strange message
        content = ADD_SITE_BAD_RESPONSE
        mocked_post.return_value.content = content
        with pytest.raises(MatomoException, match='add_site_alias_urls fails'):
            site_id = matomo.add_site_alias_urls('42', urls)


@mock.patch('requests.post')
def test_add_user(mocked_post):
    """webservice to add new user"""
    mocked_post.return_value.status_code = 200
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success
        content = MATOMO_SUCCESS
        mocked_post.return_value.content = content
        matomo.add_user('hobo.dev.publik.love', 'xxx', '42')
        assert True

        # error (user already here)
        content = USER_ALREADY_THERE
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="Username 'hobo.dev.publik.love' already"):
            matomo.add_user('hobo.dev.publik.love', 'xxx', '42')

        # error (mail already registered)
        content = MAIL_ALREADY_THERE
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="email 'hobo.dev.publik.love@testor.org'"):
            matomo.add_user('hobo.dev.publik.love', 'xxx', '42')

        # error (bad credentials)
        content = BAD_CREDENTIAL
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="You can\'t access this resource"):
            matomo.add_user('hobo.dev.publik.love', 'xxx', '42')

        # bad success message (wrong attribute value)
        content = MATOMO_BAD_RESPONSE_1
        mocked_post.return_value.content = content
        with pytest.raises(MatomoException, match='add_user fails'):
            matomo.add_user('hobo.dev.publik.love', 'xxx', '42')

        # bad success message (no message attribute)
        content = MATOMO_BAD_RESPONSE_2
        mocked_post.return_value.content = content
        with pytest.raises(MatomoException, match='add_user fails'):
            matomo.add_user('hobo.dev.publik.love', 'xxx', '42')


@mock.patch('requests.post')
def test_del_user(mocked_post):
    """webservice to del an existing user"""
    mocked_post.return_value.status_code = 200
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success
        content = MATOMO_SUCCESS
        mocked_post.return_value.content = content
        matomo.del_user('hobo.dev.publik.love')
        assert True

        # error (unknown user)
        content = DEL_UNKNOWN_USER
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="User 'hobo.dev.publik.love' doesn't exist."):
            matomo.del_user('hobo.dev.publik.love')


@mock.patch('requests.post')
def test_get_javascript_tag(mocked_post):
    """webservice to get matomo JS tag"""
    mocked_post.return_value.status_code = 200
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success
        content = JAVASCRIPT_TAG
        mocked_post.return_value.content = content
        javascript_tag = matomo.get_javascript_tag('42')
        assert javascript_tag.find('(function() {') != -1

        # error (bad credentials)
        content = BAD_CREDENTIAL
        mocked_post.return_value.content = content
        with pytest.raises(MatomoError, match="You can't access this resource"):
            javascript_tag = matomo.get_javascript_tag('42')

        # bad response (no result tag)
        content = JAVASCRIPT_TAG_BAD_RESPONSE
        mocked_post.return_value.content = content
        with pytest.raises(MatomoException, match='get_javascript_tag fails'):
            javascript_tag = matomo.get_javascript_tag('42')


@mock.patch('requests.post')
def test_create_fake_first_tracking_visit(mocked_post):
    """webservice to create a fake first tracking call"""
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()
        response = Response()
        response.status_code = 200

        # success
        content = PING_SUCCESS
        response.json = mock.MagicMock(return_value=json.loads(content))
        mocked_post.return_value = response
        matomo.create_fake_first_tracking_visit('42')
        assert True

        # error
        content = PING_ERROR
        response.json = mock.MagicMock(return_value=json.loads(content))
        mocked_post.return_value = response
        with pytest.raises(MatomoError, match='ping fails'):
            matomo.create_fake_first_tracking_visit('42')

        # failure (no status)
        content = PING_NOSTATUS_ERROR
        response.json = mock.MagicMock(return_value=json.loads(content))
        mocked_post.return_value = response
        with pytest.raises(MatomoException, match=r'internal error on ping \(status expected\)'):
            matomo.create_fake_first_tracking_visit('42')

        # failure (no dict)
        content = PING_NODICT_ERROR
        response.json = mock.MagicMock(return_value=content)
        mocked_post.return_value = response
        with pytest.raises(MatomoException, match=r'internal error on ping \(dict expected\)'):
            matomo.create_fake_first_tracking_visit('42')

        # failure (no JSON)
        response.json = mock.MagicMock(side_effect=ValueError('not a JSON'))
        mocked_post.return_value = response
        with pytest.raises(MatomoException, match=r'internal error on ping \(JSON expected\)'):
            matomo.create_fake_first_tracking_visit('42')

        # failure (status 500)
        mocked_post.return_value.status_code = 500
        with pytest.raises(MatomoException, match='unexpected status code: 500'):
            matomo.create_fake_first_tracking_visit('42')


@mock.patch('requests.post')
def test_upgrade_site(mocked_post):
    """function to test if the site is already regisered"""
    urls = ['https://combo.dev.publik.love', 'https://wcs.dev.publik.love']
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # site not already here
        contents = [GET_NO_SITE_FROM_URL, ADD_SITE_SUCCESS, ADD_SITE_ALIAS_URLS_SUCCESS]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        site_id = upgrade_site(matomo, "hobo.dev.publik.love", urls)
        assert site_id == '42'

        # site already here
        contents = [GET_SITE_42_FROM_URL, ADD_SITE_ALIAS_URLS_SUCCESS]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        site_id = upgrade_site(matomo, "hobo.dev.publik.love", urls)
        assert site_id == '42'

        # error while updating urls
        contents = [GET_SITE_42_FROM_URL, ADD_SITE_ALIAS_URLS_ERROR]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        with pytest.raises(MatomoException):
            upgrade_site(matomo, "hobo.dev.publik.love", urls)

        # error while adding new site
        contents = [GET_NO_SITE_FROM_URL, MATOMO_ERROR]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        with pytest.raises(MatomoException):
            upgrade_site(matomo, "hobo.dev.publik.love", urls)

        # error while looking for site already there
        contents = [MATOMO_ERROR]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        with pytest.raises(MatomoException, match='here is the error message'):
            upgrade_site(matomo, "hobo.dev.publik.love", urls)


@mock.patch('requests.post')
def test_upgrade_user(mocked_post):
    """function to assert we have a user"""
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success (add a new user)
        contents = [DEL_UNKNOWN_USER, MATOMO_SUCCESS]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        logme_url = upgrade_user(matomo, 'hobo.dev.publik.love', '42')
        assert logme_url.find('action=logme&login=hobo.dev.publik.love') != -1

        # success (user already here)
        contents = [MATOMO_SUCCESS, MATOMO_SUCCESS]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        logme_url = upgrade_user(matomo, 'hobo.dev.publik.love', '42')
        assert logme_url.find('action=logme&login=hobo.dev.publik.love') != -1

        # recover on error (del user fails)
        contents = [MATOMO_ERROR, MATOMO_SUCCESS]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        logme_url = upgrade_user(matomo, 'hobo.dev.publik.love', '42')
        assert logme_url.find('action=logme&login=hobo.dev.publik.love') != -1

        # error (add user fails)
        contents = [MATOMO_SUCCESS, MATOMO_ERROR]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        with pytest.raises(MatomoError):
            upgrade_user(matomo, 'hobo.dev.publik.love', '42')


def test_compute_cnil_acknowledgment_level():
    """function use to inspect javascript content"""
    warning_content = force_text(JAVASCRIPT_TAG)

    # can't find cookie's life time extension prevention
    assert compute_cnil_acknowledgment_level(warning_content) == 'good'

    # ok
    success_content = warning_content + '\npaq.push(...) ...getOriginalVisitorCookieTimeout...'
    assert compute_cnil_acknowledgment_level(success_content) == 'excellent'

    # google reference found into javascript
    error_content = success_content + '\n...google...'
    assert compute_cnil_acknowledgment_level(error_content) == 'bad'


def test_get_tracking_js():
    """read previous tracking JS from hobo variables"""
    var1 = get_variable('cnil_compliant_visits_tracking_js', 'content1')
    assert get_tracking_js() == 'content1'

    var1.delete()
    var1 = get_variable('cnil_compliant_visits_tracking_js', '')
    var2 = get_variable('visits_tracking_js', 'content2')
    assert get_tracking_js() == 'content2'

    var1.delete()
    var2.delete()
    get_variable('cnil_compliant_visits_tracking_js', 'content1')
    get_variable('visits_tracking_js', 'content2')
    assert get_tracking_js() == "content1content2"


def test_put_tracking_js():
    """write tracking js into hobo variables:
    - visits_tracking_js: a banner will be displayed (javascript may be not removed)
    - cnil_compliant_visits_tracking_js: javascript is dislayed normally
    """
    # JS is stored into 'cnil_compliant_visits_tracking_js'
    put_tracking_js('/* no gafa => no banner */')
    value1 = get_variable_value('cnil_compliant_visits_tracking_js', 'undefined')
    value2 = get_variable_value('visits_tracking_js', 'undefined')
    assert value1 == '/* no gafa => no banner */'
    assert value2 == 'undefined'

    # JS is stord into 'visits_tracking_js'
    put_tracking_js('/* google => banner */')
    value1 = get_variable_value('cnil_compliant_visits_tracking_js', 'undefined')
    value2 = get_variable_value('visits_tracking_js', 'undefined')
    assert value1 == 'undefined'
    assert value2 == '/* google => banner */'

    # test we remove variables when no more used
    put_tracking_js('')
    value1 = get_variable_value('cnil_compliant_visits_tracking_js', 'undefined')
    value2 = get_variable_value('visits_tracking_js', 'undefined')
    assert value1 == 'undefined'
    assert value2 == 'undefined'


@mock.patch('requests.post')
def test_upgrade_javascript_tag(mocked_post):
    """function to get matomo JS tag"""
    mocked_post.return_value.status_code = 200
    with override_settings(MATOMO_SERVER=CONFIG):
        matomo = MatomoWS()

        # success
        content = JAVASCRIPT_TAG
        mocked_post.return_value.content = content
        javascript_tag = upgrade_javascript_tag(matomo, '42')
        assert javascript_tag.find('(function() {') != -1
        assert javascript_tag.find('&lt;script') == -1
        assert javascript_tag.find('script&gt;') == -1
        assert javascript_tag == ENHANCED_JAVASCRIPT_TAG
        assert compute_cnil_acknowledgment_level(javascript_tag) == 'excellent'


@mock.patch('requests.post')
def test_auto_configure_matomo(mocked_post):
    tracking_js_var = get_variable('visits_tracking_js', 'js_code')
    logme_url_var = get_variable('matomo_logme_url', '')

    Combo.objects.create(base_url='https://combo.dev.publik.love', template_name='...portal-user...')
    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
    Fargo.objects.create(base_url='https://fargo.dev.publik.love')

    with override_settings(MATOMO_SERVER=CONFIG):
        contents = [
            GET_NO_SITE_FROM_URL,
            ADD_SITE_SUCCESS,
            ADD_SITE_ALIAS_URLS_SUCCESS,
            DEL_UNKNOWN_USER,
            MATOMO_SUCCESS,
            JAVASCRIPT_TAG,
            PING_SUCCESS,
        ]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        matomo = MatomoWS()
        assert auto_configure_matomo(matomo) == '42'
        logme_url_var = get_variable('matomo_logme_url')
        assert logme_url_var.value != ''
        tracking_js_var = get_variable('visits_tracking_js')
        assert tracking_js_var.value == ''
        tracking_js2_var = get_variable('cnil_compliant_visits_tracking_js')
        assert tracking_js2_var.value != ''


def test_auto_configure_matomo_no_url():
    # no Combo url so as to raise
    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
    Fargo.objects.create(base_url='https://fargo.dev.publik.love')

    with override_settings(MATOMO_SERVER=CONFIG):
        with pytest.raises(MatomoException, match="no portal-user's url available"):
            matomo = MatomoWS()
            auto_configure_matomo(matomo)


@mock.patch('requests.post')
def test_auto_configure_matomo_error(mocked_post):
    tracking_js_var = get_variable('visits_tracking_js', 'js_code')

    Combo.objects.create(base_url='https://combo.dev.publik.love', template_name='...portal-user...')
    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
    Fargo.objects.create(base_url='https://fargo.dev.publik.love')

    with override_settings(MATOMO_SERVER=CONFIG):
        contents = [
            GET_NO_SITE_FROM_URL,
            ADD_SITE_SUCCESS,
            ADD_SITE_ALIAS_URLS_SUCCESS,
            DEL_UNKNOWN_USER,
            MATOMO_SUCCESS,
            JAVASCRIPT_TAG_BAD_RESPONSE,
        ]
        mocked_post.side_effect = requests_post_mocked_replies(contents)
        with pytest.raises(MatomoException, match="get_javascript_tag fails"):
            matomo = MatomoWS()
            auto_configure_matomo(matomo)
        tracking_js_var = get_variable('visits_tracking_js')
        assert tracking_js_var.value == 'js_code'


def test_legacy_email_template_substitution():
    with override_settings(
        MATOMO_SERVER={
            'URL': 'https://matomo.test',
            'TOKEN_AUTH': '1234',
            'EMAIL_TEMPLATE': 'noreply+%s@example.net',
        }
    ):
        matomo = MatomoWS()
        assert matomo.email_template == 'noreply+%(user_login)s@example.net'


def test_email_template_no_substitution():
    with override_settings(
        MATOMO_SERVER={
            'URL': 'https://matomo.test',
            'TOKEN_AUTH': '1234',
            'EMAIL_TEMPLATE': 'noreply@example.net',
        }
    ):
        matomo = MatomoWS()
        assert matomo.email_template == 'noreply@example.net'