combo/tests/test_dataviz.py

import datetime
import json
import urllib.parse
from unittest import mock

import pytest
from django.apps import apps
from django.contrib.auth.models import Group
from django.test import override_settings
from httmock import HTTMock, remember_called, urlmatch, with_httmock
from requests.exceptions import HTTPError

from combo.apps.dataviz.models import ChartFiltersCell, ChartNgCell, Gauge, Statistic, UnsupportedDataSet
from combo.data.models import Page, ValidityInfo
from combo.utils.spooler import refresh_statistics_data

from .test_public import login
from .utils import manager_submit_cell

pytestmark = pytest.mark.django_db


@pytest.fixture
def cell():
    page = Page(title='One', slug='index')
    page.save()

    cell = Gauge(page=page, order=0, placeholder='content')
    cell.data_source = '[test_url]/XXX'
    cell.save()

    return cell


def test_jsonp_gauge(app, cell):
    with override_settings(TEMPLATE_VARS={'test_url': 'http://www.example.net'}):
        resp = app.get('/')
        assert 'data-gauge-count-jsonp-url="http://www.example.net/XXX"' in resp.text


def test_json_gauge(app, cell):
    cell.jsonp_data_source = False
    cell.save()
    with override_settings(TEMPLATE_VARS={'test_url': 'http://www.example.net'}):
        with mock.patch('combo.apps.dataviz.views.requests.get') as requests_get:
            requests_get.return_value = mock.Mock(content='xxx', status_code=200)
            resp = app.get('/')
            assert 'data-gauge-count-url="/ajax/gauge-count/%s/"' % cell.id in resp.text
            resp = app.get('/ajax/gauge-count/%s/' % cell.id)
            assert resp.text == 'xxx'
            assert requests_get.call_args[0][0] == 'http://www.example.net/XXX'


VISUALIZATION_JSON = [
    {
        'data-url': 'https://bijoe.example.com/visualization/1/json/',
        'path': 'https://bijoe.example.com/visualization/1/iframe/?signature=123',
        'name': 'example visualization (X)',
        'slug': 'example',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/2/json/',
        'path': 'https://bijoe.example.com/visualization/2/iframe/?signature=123',
        'name': 'second visualization (Y)',
        'slug': 'second',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/3/json/',
        'path': 'https://bijoe.example.com/visualization/3/iframe/?signature=123',
        'name': 'third visualization (X/Y)',
        'slug': 'third',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/4/json/',
        'path': 'https://bijoe.example.com/visualization/4/iframe/?signature=123',
        'name': 'fourth visualization (no axis)',
        'slug': 'fourth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/5/json/',
        'path': 'https://bijoe.example.com/visualization/5/iframe/?signature=123',
        'name': 'fifth visualization (loop/X)',
        'slug': 'fifth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/6/json/',
        'path': 'https://bijoe.example.com/visualization/6/iframe/?signature=123',
        'name': 'sixth visualization (loop/Y)',
        'slug': 'sixth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/7/json/',
        'path': 'https://bijoe.example.com/visualization/7/iframe/?signature=123',
        'name': 'seventh visualization (loop/X/Y)',
        'slug': 'seventh',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/8/json/',
        'path': 'https://bijoe.example.com/visualization/8/iframe/?signature=123',
        'name': 'eighth visualization (duration)',
        'slug': 'eighth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/9/json/',
        'path': 'https://bijoe.example.com/visualization/9/iframe/?signature=123',
        'name': 'nineth visualization (loop over varying dimensions)',
        'slug': 'nineth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/10/json/',
        'path': 'https://bijoe.example.com/visualization/10/iframe/?signature=123',
        'name': 'tenth visualization (percents)',
        'slug': 'tenth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/11/json/',
        'path': 'https://bijoe.example.com/visualization/11/iframe/?signature=123',
        'name': 'eleventh visualization (not found)',
        'slug': 'eleventh',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/12/json/',
        'path': 'https://bijoe.example.com/visualization/12/iframe/?signature=123',
        'name': 'twelth visualization (all null)',
        'slug': 'twelth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/13/json/',
        'path': 'https://bijoe.example.com/visualization/13/iframe/?signature=123',
        'name': 'thirteenth visualization (loop with empty x_labels)',
        'slug': 'thirteenth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/14/json/',
        'path': 'https://bijoe.example.com/visualization/14/iframe/?signature=123',
        'name': 'fourteenth visualization (empty data)',
        'slug': 'fourteenth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/15/json/',
        'path': 'https://bijoe.example.com/visualization/15/iframe/?signature=123',
        'name': 'fifteenth visualization (only loop labels)',
        'slug': 'fifteenth',
    },
    {
        'data-url': 'https://bijoe.example.com/visualization/16/json/',
        'path': 'https://bijoe.example.com/visualization/16/iframe/?signature=123',
        'name': 'sixteenth visualization (numerical labels)',
        'slug': 'sixteenth',
    },
]


@remember_called
def bijoe_mock(url, request):
    if url.path == '/visualization/json/':
        return {'content': json.dumps(VISUALIZATION_JSON), 'request': request, 'status_code': 200}
    if url.path == '/visualization/1/json/':
        response = {
            'format': '1',
            'data': [222, 134, 0, 53],
            'axis': {'x_labels': ['web', 'mail', 'phone', 'email']},
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/2/json/':
        response = {
            'format': '1',
            'data': [222, 134, 0, 53],
            'axis': {'y_labels': ['web', 'mail', 'phone', 'email']},
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/3/json/':
        response = {
            'format': '1',
            'data': [
                [222, 134, 0, 53],
                [122, 114, 2, 33],
            ],
            'axis': {
                'x_labels': ['web', 'mail', 'phone', 'email'],
                'y_labels': ['foo', 'bar'],
            },
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/4/json/':
        response = {
            'format': '1',
            'data': 222,
            'axis': {},
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/5/json/':
        response = {
            'format': '1',
            'data': [
                [222, 134, 0, 53],
                [122, 114, 2, 33],
            ],
            'axis': {
                'x_labels': ['web', 'mail', 'phone', 'email'],
                'loop': ['foo', 'bar'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/6/json/':
        response = {
            'format': '1',
            'data': [
                [222, 134, 0, 53],
                [122, 114, 2, 33],
            ],
            'axis': {
                'y_labels': ['web', 'mail', 'phone', 'email'],
                'loop': ['foo', 'bar'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/7/json/':
        response = {
            'format': '1',
            'data': [
                [[222, 134, 0, 53], [122, 114, 2, 33]],
                [[222, 134, 0, 53], [122, 114, 2, 33]],
                [[222, 134, 0, 53], [122, 114, 2, 33]],
                [[222, 134, 0, 53], [122, 114, 2, 33]],
            ],
            'axis': {
                'x_labels': ['foo', 'bar'],
                'y_labels': ['web', 'mail', 'phone', 'email'],
                'loop': ['a', 'b', 'c', 'd'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/8/json/':
        response = {
            'format': '1',
            'data': [1000, 123000, 8600, 86400],
            'axis': {'y_labels': ['web', 'mail', 'email', 'fax']},
            'unit': 'seconds',
            'measure': 'duration',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/9/json/':
        response = {
            'format': '1',
            'data': [
                [1, 1, 1, 1],
                [1],
                [1, 1],
            ],
            'axis': {
                'y_labels': ['web', 'mail', 'email'],
                'loop': ['a', 'b', 'c', 'd'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/10/json/':
        response = {
            'format': '1',
            'data': [10, 20, 30, 40, 0],
            'axis': {'y_labels': ['web', 'mail', 'email', 'fax', 'phone']},
            'unit': None,
            'measure': 'percent',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/11/json/':
        response = {
            'detail': 'not found',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 404}
    if url.path == '/visualization/12/json/':
        response = {
            'format': '1',
            'data': [None, None],
            'axis': {'x_labels': ['web', 'mail']},
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/13/json/':
        response = {
            'format': '1',
            'data': [[[], []], [[], []], [[], []]],
            'axis': {
                'x_labels': [],
                'y_labels': ['web', 'mail'],
                'loop': ['a', 'b', 'c'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/14/json/':
        response = {
            'format': '1',
            'data': [],
            'axis': {
                'x_labels': ['a', 'b', 'c'],
                'y_labels': [],
            },
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/15/json/':
        response = {
            'format': '1',
            'data': [1, 2, 3],
            'axis': {
                'loop': ['a', 'b', 'c'],
            },
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/visualization/16/json/':
        response = {
            'format': '1',
            'data': [1, 2, 3, 4, 5, 6, 7, 8, 9],
            'axis': {
                'x_labels': ['item2', 'foo', 'item20', 'item10', 'foo2', 'Item3', '10', '4', 'item1'],
                'y_labels': [],
            },
            'measure': 'integer',
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}


STATISTICS_LIST = {
    'data': [
        {
            'url': 'https://authentic.example.com/api/statistics/one-serie/',
            'name': 'One serie stat',
            'id': 'one-serie',
            "filters": [
                {
                    "default": "month",
                    "id": "time_interval",
                    "label": "Time interval",
                    "options": [
                        {"id": "day", "label": "Day"},
                        {"id": "month", "label": "Month"},
                        {"id": "year", "label": "Year"},
                    ],
                    "required": True,
                },
                {
                    "id": "ou",
                    "label": "Organizational Unit",
                    "options": [
                        {"id": "default", "label": "Default OU"},
                        {"id": "other", "label": "Other OU"},
                    ],
                },
                {
                    "id": "service",
                    "label": "Service",
                    "options": [
                        {"id": "chrono", "label": "Chrono"},
                        {"id": "combo", "label": "Combo"},
                    ],
                    "default": "chrono",
                },
            ],
        },
        {
            'url': 'https://authentic.example.com/api/statistics/two-series/',
            'name': 'Two series stat',
            'id': 'two-series',
        },
        {
            'url': 'https://authentic.example.com/api/statistics/no-data/',
            'name': 'No data stat',
            'id': 'no-data',
        },
        {
            'url': 'https://authentic.example.com/api/statistics/not-found/',
            'name': '404 not found stat',
            'id': 'not-found',
        },
        {
            'url': 'https://authentic.example.com/api/statistics/daily/',
            'name': 'Daily discontinuous serie',
            'id': 'daily',
            "filters": [
                {
                    "default": "day",
                    "id": "time_interval",
                    "label": "Time interval",
                    "options": [
                        {"id": "day", "label": "Day"},
                    ],
                    "required": True,
                }
            ],
        },
        {
            'url': 'https://authentic.example.com/api/statistics/leap-week/',
            'name': 'Same week spanning two years',
            'id': 'leap-week',
            "filters": [
                {
                    "default": "day",
                    "id": "time_interval",
                    "label": "Time interval",
                    "options": [
                        {"id": "day", "label": "Day"},
                    ],
                    "required": True,
                }
            ],
        },
        {
            'url': 'https://authentic.example.com/api/statistics/filter-multiple/',
            'name': 'Filter on multiple values',
            'id': 'filter-multiple',
            "filters": [
                {
                    "id": "color",
                    "label": "Color",
                    "options": [
                        {"id": "red", "label": "Red"},
                        {"id": "green", "label": "Green"},
                        {"id": "blue", "label": "Blue"},
                    ],
                    "multiple": True,
                }
            ],
        },
        {
            'url': 'https://authentic.example.com/api/statistics/with-subfilter/',
            'name': 'With subfilter',
            'id': 'with-subfilter',
            'filters': [
                {
                    'id': 'form',
                    'label': 'Form',
                    'has_subfilters': True,
                    'options': [
                        {'id': 'food-request', 'label': 'Food request'},
                        {'id': 'contact', 'label': 'Contact'},
                        {'id': 'error', 'label': 'Error'},
                    ],
                },
                {
                    'id': 'other',
                    'label': 'Other',
                    'options': [
                        {'id': 'one', 'label': 'One'},
                        {'id': 'two', 'label': 'two'},
                    ],
                },
            ],
        },
        {
            'url': 'https://authentic.example.com/api/statistics/future-data/',
            'name': 'With future data',
            'id': 'with-future-data',
            'future_data': True,
        },
        {
            'url': 'https://authentic.example.com/api/statistics/option-groups/',
            'name': 'Option groups',
            'id': 'option-groups',
            "filters": [
                {
                    "id": "form",
                    "label": "Form",
                    "options": [
                        [None, [{'id': 'all', 'label': 'All'}]],
                        ['Category A', [{'id': 'test', 'label': 'Test'}]],
                        ['Category B', [{'id': 'test-2', 'label': 'test 2'}]],
                    ],
                }
            ],
        },
        {
            'url': 'https://authentic.example.com/api/statistics/deprecated-filter/',
            'name': 'Deprecated filter',
            'id': 'deprecated-filter',
            "filters": [
                {
                    "id": "form",
                    "label": "Form",
                    'deprecated': True,
                    'deprecation_hint': 'This field should not be used',
                    'options': [
                        {'id': 'one', 'label': 'One'},
                        {'id': 'two', 'label': 'two'},
                    ],
                },
                {
                    "id": "card",
                    "label": "Card",
                    'deprecated': True,
                    'options': [
                        {'id': 'one', 'label': 'One'},
                        {'id': 'two', 'label': 'two'},
                    ],
                    'default': 'one',
                },
            ],
        },
    ]
}


@remember_called
def new_api_mock(url, request):
    if url.path == '/api/statistics/':
        return {'content': json.dumps(STATISTICS_LIST), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/one-serie/':
        response = {
            'data': {
                'series': [{'data': [None, 16, 2], 'label': 'Serie 1'}],
                'x_labels': ['2020-10', '2020-11', '2020-12'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/two-series/':
        response = {
            'data': {
                'series': [
                    {'data': [None, 16, 2], 'label': 'Serie 1'},
                    {'data': [2, 1, None], 'label': 'Serie 2'},
                ],
                'x_labels': ['2020-10', '2020-11', '2020-12'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/no-data/':
        response = {
            'data': {'series': [], 'x_labels': []},
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/not-found/':
        return {'content': b'', 'request': request, 'status_code': 404}
    if url.path == '/api/statistics/daily/':
        response = {
            'data': {
                'series': [
                    {'data': [None, 1, 16, 2], 'label': 'Serie 1'},
                    {'data': [2, 2, 1, None], 'label': 'Serie 2'},
                ],
                'x_labels': ['2020-10-06', '2020-10-13', '2020-11-30', '2022-02-01'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/leap-week/':
        response = {
            'data': {
                'series': [
                    {'data': [None, 1, 16, 2], 'label': 'Serie 1'},
                ],
                'x_labels': ['2020-12-30', '2020-12-31', '2021-01-01', '2021-01-02'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/filter-multiple/':
        response = {
            'data': {
                'series': [
                    {'data': [None, 1], 'label': 'Red / Green'},
                    {'data': [1, 4], 'label': 'Red / Blue'},
                ],
                'x_labels': ['2020-12-30', '2020-12-31'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/with-subfilter/':
        response = {
            'data': {
                'series': [{'data': [None, 16, 2], 'label': 'Serie 1'}],
                'x_labels': ['2020-10', '2020-11', '2020-12'],
                'subfilters': [],
            },
        }
        if 'form=food-request' in url.query:
            response['data']['subfilters'] = [
                {
                    "id": "menu",
                    "label": "Menu",
                    "options": [
                        {"id": "meat", "label": "Meat"},
                        {"id": "vegan", "label": "Vegan"},
                    ],
                }
            ]
        if 'form=error' in url.query:
            return {'content': b'', 'request': request, 'status_code': 404}
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}
    if url.path == '/api/statistics/future-data/':
        response = {
            'data': {
                'series': [{'data': [None, 16, 2], 'label': 'Serie 1'}],
                'x_labels': ['2020-10', '2020-11', '2020-12'],
            },
        }
        return {'content': json.dumps(response), 'request': request, 'status_code': 200}


@pytest.fixture
@with_httmock(bijoe_mock)
def statistics(settings):
    settings.KNOWN_SERVICES = {
        "bijoe": {
            "plop": {"title": "test", "url": "https://bijoe.example.com", "secret": "combo", "orig": "combo"}
        }
    }
    settings.STATISTICS_PROVIDERS = ['bijoe']
    appconfig = apps.get_app_config('dataviz')
    appconfig.hourly()
    assert Statistic.objects.count() == len(VISUALIZATION_JSON)


@pytest.fixture
@with_httmock(new_api_mock)
def new_api_statistics(settings):
    settings.KNOWN_SERVICES = {
        'authentic': {
            'connection': {
                'title': 'Connection',
                'url': 'https://authentic.example.com',
                'secret': 'combo',
                'orig': 'combo',
            }
        }
    }
    settings.STATISTICS_PROVIDERS = ['authentic']
    appconfig = apps.get_app_config('dataviz')
    appconfig.hourly()
    assert Statistic.objects.count() == len(STATISTICS_LIST['data'])


@with_httmock(bijoe_mock)
def test_chartng_cell(app, statistics):
    page = Page(title='One', slug='index')
    page.save()

    cell = ChartNgCell(page=page, order=1)
    cell.statistic = Statistic.objects.get(slug='example')
    cell.save()

    # bar
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Bar'
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [([222, 134, 0, 53], {'title': ''})]

    # horizontal bar
    cell.chart_type = 'horizontal-bar'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'HorizontalBar'
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [([222, 134, 0, 53], {'title': ''})]

    # pie
    cell.chart_type = 'pie'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Pie'
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222], {'title': 'web'}),
        ([134], {'title': 'mail'}),
        ([53], {'title': 'email'}),
    ]

    # data in Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='second')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [([222, 134, 0, 53], {'title': ''})]

    # data in X/Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='third')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # stacked bar with percent
    cell.chart_type = 'stacked-bar-percent'
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([64.5, 54, 0, 61.6], {'title': 'foo'}),
        ([35.5, 46, 100, 38.4], {'title': 'bar'}),
    ]
    assert all(x + y == 100 for x, y in zip(chart.raw_series[0][0], chart.raw_series[1][0]))

    # single data point
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='fourth')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['']
    assert chart.raw_series == [([222], {'title': ''})]

    # loop/X
    cell.statistic = Statistic.objects.get(slug='fifth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # loop/Y
    cell.statistic = Statistic.objects.get(slug='sixth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # loop/X/Y
    cell.statistic = Statistic.objects.get(slug='seventh')
    cell.save()
    with pytest.raises(UnsupportedDataSet):
        chart = cell.get_chart()

    # duration
    cell.statistic = Statistic.objects.get(slug='eighth')
    cell.save()
    chart = cell.get_chart()

    # loop/X/Y
    cell.statistic = Statistic.objects.get(slug='nineth')
    cell.save()
    with pytest.raises(UnsupportedDataSet):
        chart = cell.get_chart()

    # deleted visualization
    cell.statistic = Statistic.objects.get(slug='eleventh')
    cell.save()
    with pytest.raises(HTTPError):
        chart = cell.get_chart()

    # loop and empty x_labels
    cell.statistic = Statistic.objects.get(slug='thirteenth')
    cell.save()
    with pytest.raises(UnsupportedDataSet):
        chart = cell.get_chart()

    # only loop labels
    cell.statistic = Statistic.objects.get(slug='fifteenth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['a', 'b', 'c']
    assert chart.raw_series == [
        ([1, 2, 3], {'title': ''}),
    ]


@with_httmock(new_api_mock)
def test_chartng_cell_new_api(app, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1)
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Bar'
    assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
    assert chart.raw_series == [
        (
            [None, 16, 2],
            {'title': 'Serie 1'},
        )
    ]

    cell.chart_type = 'pie'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Pie'
    assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
    assert chart.raw_series == [
        ([16], {'title': '2020-11'}),
        ([2], {'title': '2020-12'}),
    ]

    cell.statistic = Statistic.objects.get(slug='two-series')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
    assert chart.raw_series == [([None, 16, 2], {'title': 'Serie 1'}), ([2, 1, None], {'title': 'Serie 2'})]

    # stacked bar with percent
    cell.chart_type = 'stacked-bar-percent'
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
    assert chart.raw_series == [
        ([None, 94.1, 100], {'title': 'Serie 1'}),
        ([100, 5.9, None], {'title': 'Serie 2'}),
    ]
    assert all(x + y == 100 for x, y in zip(chart.raw_series[0][0], chart.raw_series[1][0]) if x and y)

    cell.statistic = Statistic.objects.get(slug='no-data')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == []
    assert chart.raw_series == []

    cell.statistic = Statistic.objects.get(slug='not-found')
    cell.save()
    with pytest.raises(HTTPError):
        chart = cell.get_chart()


@with_httmock(bijoe_mock)
def test_chartng_cell_hide_null_values(app, statistics):
    page = Page(title='One', slug='index')
    page.save()

    cell = ChartNgCell(page=page, order=1)
    cell.statistic = Statistic.objects.get(slug='example')
    cell.hide_null_values = True
    cell.save()

    # bar
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Bar'
    assert chart.x_labels == ['web', 'mail', 'email']
    assert chart.raw_series == [([222, 134, 53], {'title': ''})]

    # horizontal bar
    cell.chart_type = 'horizontal-bar'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'HorizontalBar'
    assert chart.x_labels == ['web', 'mail', 'email']
    assert chart.raw_series == [([222, 134, 53], {'title': ''})]

    # pie
    cell.chart_type = 'pie'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Pie'
    assert chart.x_labels == ['web', 'mail', 'email']
    assert chart.raw_series == [
        ([222], {'title': 'web'}),
        ([134], {'title': 'mail'}),
        ([53], {'title': 'email'}),
    ]

    # data in Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='second')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'email']
    assert chart.raw_series == [([222, 134, 53], {'title': ''})]

    # data in X/Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='third')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # single data point
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='fourth')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['']
    assert chart.raw_series == [([222], {'title': ''})]

    # loop/X
    cell.statistic = Statistic.objects.get(slug='fifth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # loop/Y
    cell.statistic = Statistic.objects.get(slug='sixth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # all null
    cell.statistic = Statistic.objects.get(slug='twelth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == []
    assert chart.raw_series == [([], {'title': ''})]


@with_httmock(new_api_mock)
def test_chartng_cell_hide_null_values_new_api(app, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, hide_null_values=True)
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.hide_null_values = True
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['2020-11', '2020-12']
    assert chart.raw_series == [
        (
            [16, 2],
            {'title': 'Serie 1'},
        )
    ]


@with_httmock(bijoe_mock)
def test_chartng_cell_sort_order_alpha(app, statistics):
    page = Page(title='One', slug='index')
    page.save()

    cell = ChartNgCell(page=page, order=1)
    cell.statistic = Statistic.objects.get(slug='example')
    cell.sort_order = 'alpha'
    cell.save()

    # bar
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Bar'
    assert chart.x_labels == ['email', 'mail', 'phone', 'web']
    assert chart.raw_series == [([53, 134, 0, 222], {'title': ''})]

    # horizontal bar
    cell.chart_type = 'horizontal-bar'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'HorizontalBar'
    assert chart.x_labels == ['email', 'mail', 'phone', 'web']
    assert chart.raw_series == [([53, 134, 0, 222], {'title': ''})]

    # pie
    cell.chart_type = 'pie'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Pie'
    assert chart.x_labels == ['email', 'mail', 'phone', 'web']
    assert chart.raw_series == [
        ([53], {'title': 'email'}),
        ([134], {'title': 'mail'}),
        ([222], {'title': 'web'}),
    ]

    # data in Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='second')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['email', 'mail', 'phone', 'web']
    assert chart.raw_series == [([53, 134, 0, 222], {'title': ''})]

    # data in X/Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='third')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # single data point
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='fourth')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['']
    assert chart.raw_series == [([222], {'title': ''})]

    # loop/X
    cell.statistic = Statistic.objects.get(slug='fifth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # loop/Y
    cell.statistic = Statistic.objects.get(slug='sixth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # empty data
    cell.statistic = Statistic.objects.get(slug='fourteenth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['a', 'b', 'c']
    assert chart.raw_series == [([], {'title': ''})]

    cell.statistic = Statistic.objects.get(slug='sixteenth')
    cell.sort_order = 'alpha'
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['4', '10', 'foo', 'foo2', 'item1', 'item2', 'Item3', 'item10', 'item20']
    assert chart.raw_series == [([8, 7, 2, 5, 9, 1, 6, 4, 3], {'title': ''})]


@with_httmock(bijoe_mock)
def test_chartng_cell_sort_order_desc(app, statistics):
    page = Page(title='One', slug='index')
    page.save()

    cell = ChartNgCell(page=page, order=1)
    cell.statistic = Statistic.objects.get(slug='example')
    cell.sort_order = 'desc'
    cell.save()

    # bar
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Bar'
    assert chart.x_labels == ['web', 'mail', 'email', 'phone']
    assert chart.raw_series == [([222, 134, 53, 0], {'title': ''})]

    # horizontal bar
    cell.chart_type = 'horizontal-bar'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'HorizontalBar'
    assert chart.x_labels == ['web', 'mail', 'email', 'phone']
    assert chart.raw_series == [([222, 134, 53, 0], {'title': ''})]

    # pie
    cell.chart_type = 'pie'
    chart = cell.get_chart()
    assert chart.__class__.__name__ == 'Pie'
    assert chart.x_labels == ['web', 'mail', 'email', 'phone']
    assert chart.raw_series == [
        ([222], {'title': 'web'}),
        ([134], {'title': 'mail'}),
        ([53], {'title': 'email'}),
    ]

    # data in Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='second')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'email', 'phone']
    assert chart.raw_series == [([222, 134, 53, 0], {'title': ''})]

    # data in X/Y
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='third')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # single data point
    cell.chart_type = 'bar'
    cell.statistic = Statistic.objects.get(slug='fourth')
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['']
    assert chart.raw_series == [([222], {'title': ''})]

    # loop/X
    cell.statistic = Statistic.objects.get(slug='fifth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # loop/Y
    cell.statistic = Statistic.objects.get(slug='sixth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['web', 'mail', 'phone', 'email']
    assert chart.raw_series == [
        ([222, 134, 0, 53], {'title': 'foo'}),
        ([122, 114, 2, 33], {'title': 'bar'}),
    ]

    # empty data
    cell.statistic = Statistic.objects.get(slug='fourteenth')
    cell.save()
    chart = cell.get_chart()
    assert chart.x_labels == ['a', 'b', 'c']
    assert chart.raw_series == [([], {'title': ''})]


@with_httmock(new_api_mock)
def test_chartng_cell_sort_order_new_api(app, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1)
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.sort_order = 'desc'
    cell.save()

    chart = cell.get_chart()
    assert chart.x_labels == ['2020-11', '2020-12', '2020-10']
    assert chart.raw_series == [
        (
            [16, 2, None],
            {'title': 'Serie 1'},
        )
    ]


@with_httmock(bijoe_mock)
def test_chartng_cell_view(app, normal_user, statistics):
    page = Page(title='One', slug='index')
    page.save()

    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='example')
    cell.save()
    location = '/api/dataviz/graph/%s/' % cell.id
    resp = app.get(location)  # get data in cache
    resp = app.get('/')
    assert 'min-height: 250px' in resp.text
    assert location in resp.text

    resp = app.get(location + '?width=400')
    assert resp.content_type == 'image/svg+xml'

    resp = app.get(location + '?width=')  # no crash
    assert resp.content_type == 'image/svg+xml'

    page.public = False
    page.save()
    resp = app.get(location + '?width=400', status=403)

    page.public = True
    page.save()
    group = Group(name='plop')
    group.save()
    cell.public = False
    cell.groups.set([group])
    cell.save()
    resp = app.get(location + '?width=400', status=403)

    app = login(app, username='normal-user', password='normal-user')
    resp = app.get(location + '?width=400', status=403)

    normal_user.groups.set([group])
    normal_user.save()
    resp = app.get(location + '?width=400', status=200)

    # table visualization
    cell.chart_type = 'table'
    cell.save()
    resp = app.get(location, status=200)
    assert '<td>222</td>' in resp.text

    # unsupported dataset
    cell.statistic = Statistic.objects.get(slug='seventh')
    cell.save()
    resp = app.get(location, status=200)
    assert '<p>Unsupported dataset.</p>' in resp.text

    cell.chart_type = 'bar'
    cell.save()
    resp = app.get(location + '?width=400', status=200)
    assert 'Unsupported dataset' in resp.text

    # durations
    cell.statistic = Statistic.objects.get(slug='eighth')
    cell.chart_type = 'table'
    cell.save()
    resp = app.get(location, status=200)
    assert '<td>Less than an hour</td>' in resp.text
    assert '<td>1 day and 10 hours</td>' in resp.text
    assert '<td>2 hours</td>' in resp.text
    assert '<td>1 day</td>' in resp.text

    cell.chart_type = 'bar'
    cell.save()
    resp = app.get(location + '?width=400', status=200)
    assert '>Less than an hour<' in resp.text
    assert '>1 day and 10 hours<' in resp.text
    assert '>2 hours<' in resp.text
    assert '>1 day<' in resp.text

    # percents
    cell.statistic = Statistic.objects.get(slug='tenth')
    cell.chart_type = 'table'
    cell.save()
    resp = app.get(location, status=200)
    assert '<td>10.0%</td>' in resp.text

    cell.chart_type = 'bar'
    cell.save()
    resp = app.get(location + '?width=400', status=200)
    assert '>10.0%<' in resp.text

    # deleted visualization
    cell.statistic = Statistic.objects.get(slug='eleventh')
    cell.save()
    resp = app.get(location)
    assert 'not found' in resp.text


@with_httmock(new_api_mock)
def test_chartng_cell_view_new_api(app, normal_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    location = '/api/dataviz/graph/%s/' % cell.id
    resp = app.get('/')
    assert 'min-height: 250px' in resp.text
    assert location in resp.text

    # table visualization
    cell.chart_type = 'table'
    cell.save()
    resp = app.get(location, status=200)
    assert '<td>18</td>' in resp.text

    # deleted visualization
    cell.statistic = Statistic.objects.get(slug='not-found')
    cell.save()
    resp = app.get(location)
    assert 'not found' in resp.text

    cell.statistic.url = ''
    cell.statistic.save()
    resp = app.get(location, status=404)


@with_httmock(bijoe_mock)
def test_chartng_cell_manager(app, admin_user, statistics):
    page = Page(title='One', slug='index')
    page.save()
    Statistic.objects.create(
        slug='unavailable-stat', label='Unavailable Stat', site_slug='plop', available=False
    )

    app = login(app)

    cell = ChartNgCell.objects.create(page=page, order=1, placeholder='content')
    resp = app.get('/manage/pages/%s/' % page.id)
    assert 'time_range' not in resp.form.fields
    assert 'time_range_start' not in resp.form.fields
    assert 'time_range_end' not in resp.form.fields
    assert 'time_range_start_template' not in resp.form.fields
    assert 'time_range_end_template_end' not in resp.form.fields

    cell.statistic = Statistic.objects.get(slug='example')
    cell.save()
    resp = app.get('/manage/pages/%s/' % page.id)
    statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id]
    # available visualizations and a blank choice
    assert len(statistics_field.options) == len(VISUALIZATION_JSON) + 1
    assert statistics_field.value == str(cell.statistic.pk)
    assert statistics_field.options[1][2] == 'test: eighth visualization (duration)'
    assert not 'Unavailable Stat' in resp.text
    assert 'time_range' not in resp.form.fields
    assert 'time_range_start' not in resp.form.fields
    assert 'time_range_end' not in resp.form.fields
    assert 'time_range_start_template' not in resp.form.fields
    assert 'time_range_end_template_end' not in resp.form.fields

    cell.statistic = Statistic.objects.get(slug='unavailable-stat')
    cell.save()
    resp = app.get('/manage/pages/%s/' % page.id)
    statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id]
    # available visualizations, a blank choice and the current unavailable visualization
    assert len(statistics_field.options) == len(VISUALIZATION_JSON) + 2
    assert 'Unavailable Stat' in resp.text


@with_httmock(new_api_mock)
def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id
    statistics_field = resp.form[field_prefix + 'statistic']
    assert len(statistics_field.options) == len(STATISTICS_LIST['data']) + 1
    assert statistics_field.value == str(cell.statistic.pk)
    selected_options = [option for option in statistics_field.options if option[1] is True]
    assert len(selected_options) == 1
    assert selected_options[0][2] == 'Connection: One serie stat'

    time_interval_field = resp.form[field_prefix + 'time_interval']
    assert time_interval_field.pos == statistics_field.pos + 1
    assert time_interval_field.value == 'month'
    assert time_interval_field.options == [
        ('day', False, 'Day'),
        ('month', True, 'Month'),
        ('year', False, 'Year'),
        ('week', False, 'Week'),
        ('weekday', False, 'Week day'),
    ]

    ou_field = resp.form[field_prefix + 'ou']
    assert ou_field.pos == statistics_field.pos + 2
    assert ou_field.value == ''
    assert ou_field.options == [
        ('', True, '---------'),
        ('default', False, 'Default OU'),
        ('other', False, 'Other OU'),
    ]

    service_field = resp.form[field_prefix + 'service']
    assert service_field.pos == statistics_field.pos + 3
    assert service_field.value == 'chrono'
    assert service_field.options == [
        ('', False, '---------'),
        ('chrono', True, 'Chrono'),
        ('combo', False, 'Combo'),
    ]
    resp.form[field_prefix + 'service'] = ''
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'service'].value == ''

    resp.form[field_prefix + 'ou'] = 'default'
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'ou'].value == 'default'
    cell.refresh_from_db()
    assert cell.get_filter_params() == {'ou': 'default', 'time_interval': 'month'}

    ou_filter = next(x for x in cell.statistic.filters if x['id'] == 'ou')
    ou_filter['options'] = [{'id': 'new', 'label': 'New'}]
    cell.statistic.save()
    resp = app.get('/manage/pages/%s/' % page.id)

    assert resp.form[field_prefix + 'ou'].value == 'default'
    assert resp.form[field_prefix + 'ou'].options == [
        ('', False, '---------'),
        ('new', False, 'New'),
        ('default', True, 'default (unavailable)'),
    ]

    resp.form[field_prefix + 'ou'] = ''
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'ou'].value == ''
    cell.refresh_from_db()
    assert cell.get_filter_params() == {'time_interval': 'month'}

    resp.form[field_prefix + 'time_range'] = 'previous-year'
    manager_submit_cell(resp.form)
    cell.refresh_from_db()
    assert cell.time_range == 'previous-year'

    resp.form[field_prefix + 'time_range'] = 'range'
    resp.form[field_prefix + 'time_range_start'] = '2020-10-01'
    resp.form[field_prefix + 'time_range_end'] = '2020-11-03'
    manager_submit_cell(resp.form)
    cell.refresh_from_db()
    assert cell.time_range == 'range'
    assert cell.time_range_start == datetime.date(year=2020, month=10, day=1)
    assert cell.time_range_end == datetime.date(year=2020, month=11, day=3)

    resp.form[field_prefix + 'time_range_start'] = ''
    resp.form[field_prefix + 'time_range_end'] = ''
    manager_submit_cell(resp.form)
    cell.refresh_from_db()
    assert cell.time_range_start is None
    assert cell.time_range_end is None

    no_filters_stat = Statistic.objects.get(slug='two-series')
    resp.form[field_prefix + 'statistic'] = no_filters_stat.pk
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'statistic'].value == str(no_filters_stat.pk)
    assert field_prefix + 'time_interval' not in resp.form.fields
    assert field_prefix + 'ou' not in resp.form.fields
    cell.refresh_from_db()
    assert cell.filter_params == {}
    assert cell.time_range == ''

    filter_multiple_stat = Statistic.objects.get(slug='filter-multiple')
    resp.form[field_prefix + 'statistic'] = filter_multiple_stat.pk
    manager_submit_cell(resp.form)
    resp.form[field_prefix + 'color'].select_multiple(texts=['Blue', 'Green'])
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'color'].value == ['green', 'blue']
    cell.refresh_from_db()
    assert cell.filter_params == {'color': ['green', 'blue']}

    color_filter = next(x for x in cell.statistic.filters if x['id'] == 'color')
    color_filter['options'] = [{'id': 'black', 'label': 'Black'}, {'id': 'green', 'label': 'Green'}]
    cell.statistic.save()
    resp = app.get('/manage/pages/%s/' % page.id)

    assert resp.form[field_prefix + 'color'].value == ['green', 'blue']
    assert resp.form[field_prefix + 'color'].options == [
        ('black', False, 'Black'),
        ('green', True, 'Green'),
        ('blue', True, 'blue (unavailable)'),
    ]

    resp.form[field_prefix + 'color'].select_multiple(texts=[])
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'color'].value is None
    cell.refresh_from_db()
    assert cell.get_filter_params() == {}

    option_groups_stat = Statistic.objects.get(slug='option-groups')
    resp.form[field_prefix + 'statistic'] = option_groups_stat.pk
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'form'].options == [
        ('', True, '---------'),
        ('all', False, 'All'),
        ('test', False, 'Test'),
        ('test-2', False, 'test 2'),
    ]
    assert resp.pyquery('optgroup[label="Category A"] option').val() == 'test'
    assert resp.pyquery('optgroup[label="Category B"] option').val() == 'test-2'

    deprecated_stat = Statistic.objects.get(slug='deprecated-filter')
    resp.form[field_prefix + 'statistic'] = deprecated_stat.pk
    manager_submit_cell(resp.form)
    assert field_prefix + 'form' not in resp.form.fields
    assert field_prefix + 'card' not in resp.form.fields

    cell.refresh_from_db()
    cell.filter_params = {'form': 'one', 'card': 'one'}
    cell.save()
    resp = app.get('/manage/pages/%s/' % page.id)
    assert field_prefix + 'form' in resp.form.fields
    assert field_prefix + 'card' not in resp.form.fields
    assert 'Form (deprecated)' in resp.text
    assert 'This field should not be used' in resp.text


@with_httmock(new_api_mock)
def test_chartng_cell_manager_future_data(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id
    time_range_field = resp.form[field_prefix + 'time_range']
    assert time_range_field.value == ''
    assert time_range_field.options == [
        ('', True, '---------'),
        ('previous-year', False, 'Previous year'),
        ('current-year', False, 'Current year'),
        ('previous-month', False, 'Previous month'),
        ('current-month', False, 'Current month'),
        ('previous-week', False, 'Previous week'),
        ('current-week', False, 'Current week'),
        ('range', False, 'Free range (date)'),
        ('range-template', False, 'Free range (template)'),
    ]

    stat_with_future_data = Statistic.objects.get(slug='with-future-data')
    resp.form[field_prefix + 'statistic'] = stat_with_future_data.pk
    manager_submit_cell(resp.form)
    time_range_field = resp.form[field_prefix + 'time_range']
    assert time_range_field.value == ''
    assert time_range_field.options == [
        ('', True, '---------'),
        ('previous-year', False, 'Previous year'),
        ('current-year', False, 'Current year'),
        ('next-year', False, 'Next year'),
        ('previous-month', False, 'Previous month'),
        ('current-month', False, 'Current month'),
        ('next-month', False, 'Next month'),
        ('previous-week', False, 'Previous week'),
        ('current-week', False, 'Current week'),
        ('next-week', False, 'Next week'),
        ('range', False, 'Free range (date)'),
        ('range-template', False, 'Free range (template)'),
    ]


@with_httmock(new_api_mock)
def test_chartng_cell_manager_subfilters(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='with-subfilter')
    cell.save()

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id

    # choice with no subfilter
    resp.form[field_prefix + 'form'] = 'contact'
    manager_submit_cell(resp.form)

    assert len(new_api_mock.call['requests']) == 1
    assert 'menu' not in resp.form.fields

    resp.form[field_prefix + 'form'] = 'error'
    manager_submit_cell(resp.form)

    assert len(new_api_mock.call['requests']) == 2
    assert 'menu' not in resp.form.fields

    # choice with subfilter
    resp.form[field_prefix + 'form'] = 'food-request'
    manager_submit_cell(resp.form)

    assert len(new_api_mock.call['requests']) == 3
    menu_field = resp.form[field_prefix + 'menu']
    assert menu_field.value == ''
    assert menu_field.options == [
        ('', True, '---------'),
        ('meat', False, 'Meat'),
        ('vegan', False, 'Vegan'),
    ]

    resp.form[field_prefix + 'menu'] = 'meat'
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'menu'].value == 'meat'
    cell.refresh_from_db()
    assert cell.get_filter_params() == {'form': 'food-request', 'menu': 'meat'}

    # choice with no subfilter
    resp.form[field_prefix + 'form'] = 'contact'
    manager_submit_cell(resp.form)

    assert len(new_api_mock.call['requests']) == 4
    assert 'menu' not in resp.form.fields
    cell.refresh_from_db()
    assert cell.get_filter_params() == {'form': 'contact'}

    # changing another filter doesn't trigger request
    resp.form[field_prefix + 'other'] = 'one'
    resp = resp.form.submit()
    assert len(new_api_mock.call['requests']) == 4


@with_httmock(new_api_mock)
@pytest.mark.freeze_time('2021-10-06')
def test_chartng_cell_manager_new_api_time_range_templates(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id

    resp.form[field_prefix + 'time_range'] = 'range-template'
    resp.form[field_prefix + 'time_range_start_template'] = 'today|add_days:"7"|adjust_to_week_monday'
    resp.form[field_prefix + 'time_range_end_template'] = 'now|add_days:"14"|adjust_to_week_monday'
    manager_submit_cell(resp.form)
    cell.refresh_from_db()
    assert cell.time_range == 'range-template'
    assert cell.time_range_start_template == 'today|add_days:"7"|adjust_to_week_monday'
    assert cell.time_range_end_template == 'now|add_days:"14"|adjust_to_week_monday'

    resp.form[field_prefix + 'time_range_start_template'] = ''
    resp.form[field_prefix + 'time_range_end_template'] = ''
    manager_submit_cell(resp.form)
    cell.refresh_from_db()
    assert cell.time_range_start_template == ''
    assert cell.time_range_end_template == ''

    resp.form[field_prefix + 'time_range_start_template'] = 'xxx'
    manager_submit_cell(resp.form, expect_errors=True)
    cell.refresh_from_db()
    assert cell.time_range_start_template == 'xxx'

    resp = app.get('/manage/pages/%s/' % page.id)
    resp.form[field_prefix + 'time_range_start_template'] = 'today|xxx'
    manager_submit_cell(resp.form, expect_errors=True)
    assert 'Invalid filter' in resp.text

    resp = app.get('/manage/pages/%s/' % page.id)
    resp.form[field_prefix + 'time_range_start_template'] = 'today|date:xxx'
    manager_submit_cell(resp.form, expect_errors=True)
    assert 'Failed lookup for key [xxx]' in resp.text


@with_httmock(new_api_mock)
def test_chartng_cell_manager_new_api_dynamic_fields(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell.objects.create(page=page, order=1, placeholder='content')
    statistic = Statistic.objects.get(slug='one-serie')

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id
    resp.form[field_prefix + 'statistic'] = statistic.pk
    resp = app.post(resp.form.action, params=resp.form.submit_fields(), xhr=True)
    assert 'time_interval' in resp.json['tabs']['general']['form']
    assert 'This field is required.' not in resp.json['tabs']['general']['form']


@with_httmock(new_api_mock)
def test_chartng_cell_manager_new_api_page_variables(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    assert '<optgroup label="Page variables">' not in resp.text

    page.extra_variables = {'foo': 'bar', 'bar_id': '{{ 40|add:2 }}'}
    page.save()
    resp = app.get('/manage/pages/%s/' % page.id)
    assert '<optgroup label="Page variables">' in resp.text

    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id
    assert resp.form[field_prefix + 'ou'].options == [
        ('', True, '---------'),
        ('default', False, 'Default OU'),
        ('other', False, 'Other OU'),
        ('variable:bar_id', False, 'bar_id'),
        ('variable:foo', False, 'foo'),
    ]
    assert resp.form[field_prefix + 'service'].options == [
        ('', False, '---------'),
        ('chrono', True, 'Chrono'),
        ('combo', False, 'Combo'),
        ('variable:bar_id', False, 'bar_id'),
        ('variable:foo', False, 'foo'),
    ]

    resp.form[field_prefix + 'ou'] = 'variable:foo'
    manager_submit_cell(resp.form)
    assert resp.form[field_prefix + 'ou'].value == 'variable:foo'
    cell.refresh_from_db()
    assert cell.filter_params['ou'] == 'variable:foo'

    del page.extra_variables['foo']
    page.save()
    resp = app.get('/manage/pages/%s/' % page.id)
    assert resp.form[field_prefix + 'ou'].options == [
        ('', False, '---------'),
        ('default', False, 'Default OU'),
        ('other', False, 'Other OU'),
        ('variable:bar_id', False, 'bar_id'),
        ('variable:foo', True, 'foo (unavailable)'),
    ]

    # no variables allowed for time_interval
    time_interval_field = resp.form[field_prefix + 'time_interval']
    assert [x[0] for x in time_interval_field.options] == ['day', 'month', 'year', 'week', 'weekday']

    # no variables allowed for multiple choice field
    cell.statistic = Statistic.objects.get(slug='filter-multiple')
    cell.save()
    resp = app.get('/manage/pages/%s/' % page.id)

    color_field = resp.form[field_prefix + 'color']
    assert [x[0] for x in color_field.options] == ['red', 'green', 'blue']


def test_chartng_cell_manager_new_api_tabs(app, admin_user):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.save()

    login(app)
    resp = app.get('/manage/pages/%s/' % page.id)

    assert not resp.pyquery('[data-tab-slug="general"] input[name$="title"]')
    assert resp.pyquery('[data-tab-slug="appearance"] input[name$="title"]')


@with_httmock(bijoe_mock)
def test_table_cell(app, admin_user, statistics):
    page = Page(title='One', slug='index')
    page.save()

    app = login(app)

    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='example')
    cell.chart_type = 'table'
    cell.save()
    location = '/api/dataviz/graph/%s/' % cell.id
    resp = app.get(location)
    assert resp.text.count('Total') == 1

    cell.statistic = Statistic.objects.get(slug='second')
    cell.save()
    resp = app.get(location)
    assert resp.text.count('Total') == 1

    cell.statistic = Statistic.objects.get(slug='third')
    cell.save()
    resp = app.get(location)
    assert '114' in resp.text
    assert resp.text.count('Total') == 2

    cell.statistic = Statistic.objects.get(slug='fourth')
    cell.save()
    resp = app.get(location)
    assert resp.text.count('Total') == 0

    # total of durations is not computed
    cell.statistic = Statistic.objects.get(slug='eighth')
    cell.save()
    resp = app.get(location)
    assert resp.text.count('Total') == 0


@with_httmock(new_api_mock)
def test_table_cell_new_api(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.chart_type = 'table'
    cell.save()

    app = login(app)
    location = '/api/dataviz/graph/%s/' % cell.id
    resp = app.get(location)
    assert resp.text.count('Total') == 1

    cell.statistic = Statistic.objects.get(slug='two-series')
    cell.save()
    resp = app.get(location)
    assert '21' in resp.text
    assert resp.text.count('Total') == 2

    cell.statistic = Statistic.objects.get(slug='no-data')
    cell.save()
    resp = app.get(location)
    assert resp.text.count('Total') == 0


def test_dataviz_hourly_unavailable_statistic(statistics, nocache):
    all_stats_count = Statistic.objects.count()
    assert Statistic.objects.filter(available=True).count() == all_stats_count

    def bijoe_mock_unavailable(url, request):
        visualization_json = VISUALIZATION_JSON[2:]
        return {'content': json.dumps(visualization_json), 'request': request, 'status_code': 200}

    appconfig = apps.get_app_config('dataviz')
    with HTTMock(bijoe_mock_unavailable):
        appconfig.hourly()
    assert Statistic.objects.filter(available=True).count() == all_stats_count - 2


def test_dataviz_import_cell():
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell.objects.create(page=page, order=1, slug='test', placeholder='content')
    statistic = Statistic.objects.create(
        slug='example', site_slug='plop', service_slug='bijoe', url='https://example.org'
    )
    cell.statistic = statistic
    cell.save()

    site_export = [page.get_serialized_page()]
    cell.delete()

    Page.load_serialized_pages(site_export)
    cell = ChartNgCell.objects.get(slug='test')
    assert cell.statistic.pk == statistic.pk

    cell.delete()
    statistic.delete()

    Page.load_serialized_pages(site_export)
    cell = ChartNgCell.objects.get(slug='test')
    assert cell.statistic.slug == statistic.slug
    assert cell.statistic.site_slug == statistic.site_slug
    assert cell.statistic.service_slug == statistic.service_slug


@with_httmock(new_api_mock)
def test_dataviz_api_list_statistics(new_api_statistics, settings):
    statistic = Statistic.objects.get(slug='one-serie')
    assert statistic.label == 'One serie stat'
    assert statistic.site_slug == 'connection'
    assert statistic.service_slug == 'authentic'
    assert statistic.site_title == 'Connection'
    assert statistic.url == 'https://authentic.example.com/api/statistics/one-serie/'
    assert statistic.available

    # try with external url
    statistics_count = Statistic.objects.count()
    settings.STATISTICS_PROVIDERS.append(
        {'url': 'https://stat.com/stats/', 'id': 'example', 'name': 'Example Provider'}
    )
    catalog = {'data': [{'url': 'https://stat.com/stats/1/', 'name': 'Test', 'id': 'test'}]}

    @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/')
    def server_error(url, request):
        return {'content': 'error', 'status_code': 500}

    appconfig = apps.get_app_config('dataviz')
    with HTTMock(server_error):
        appconfig.hourly()
    assert Statistic.objects.count() == statistics_count

    @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/')
    def success(url, request):
        return {'content': json.dumps(catalog), 'status_code': 200}

    with HTTMock(success):
        appconfig.hourly()

    assert Statistic.objects.count() == statistics_count + 1
    statistic = Statistic.objects.get(slug='test')
    assert statistic.label == 'Test'
    assert statistic.site_slug == 'example'
    assert statistic.service_slug == 'example'
    assert statistic.site_title == 'Example Provider'
    assert statistic.url == 'https://stat.com/stats/1/'
    assert statistic.available

    settings.STATISTICS_PROVIDERS.append('unknown')
    appconfig = apps.get_app_config('dataviz')
    with HTTMock(success):
        appconfig.hourly()  # unknown provider is ignored


@with_httmock(new_api_mock)
@pytest.mark.parametrize('date', ['2020-03-02 12:01', '2020-03-05 12:01'])  # Monday and Thursday
def test_chartng_cell_new_api_filter_params(app, new_api_statistics, nocache, freezer, date, settings):
    settings.TEMPLATE_VARS['test_var'] = datetime.date(year=2020, month=10, day=1)
    page = Page.objects.create(
        title='One',
        slug='index',
        extra_variables={
            'custom_date': '{{ "2021-02-03"|parse_date }}',
            'from-request': '{{ request.GET.test }}',
            'not-a-date': 'not-a-date',
            'syntax-error': '{% for %}',
            'from-template-vars': '{{ test_var }}',
        },
    )
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    cell.get_chart()
    request = new_api_mock.call['requests'][0]
    assert 'time_interval=' not in request.url
    assert 'ou=' not in request.url

    cell.filter_params = {'time_interval': 'month', 'ou': 'default', 'color': ['green', 'blue']}
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][1]
    assert 'time_interval=month' in request.url
    assert 'ou=default' in request.url
    assert 'color=green&color=blue' in request.url

    freezer.move_to(date)
    cell.time_range = 'previous-year'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][2]
    assert 'time_interval=month' in request.url
    assert 'ou=default' in request.url
    assert 'start=2019-01-01' in request.url and 'end=2020-01-01' in request.url

    cell.time_range = 'current-week'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-03-02' in request.url and 'end=2020-03-09' in request.url

    cell.time_range = 'previous-week'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-02-24' in request.url and 'end=2020-03-02' in request.url

    cell.time_range = 'next-week'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-03-09' in request.url and 'end=2020-03-16' in request.url

    cell.time_range = 'range'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][-1]
    assert 'start' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)
    assert 'end' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)

    cell.time_range_start = '2020-10-01'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-10-01' in request.url

    cell.time_range_end = '2020-11-03'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-10-01' in request.url and 'end=2020-11-03' in request.url

    location = '/api/dataviz/graph/%s/' % cell.pk
    cell.time_range = 'range-template'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)
    assert 'end' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)

    cell.time_range_start_template = 'today|add_days:"7"|adjust_to_week_monday'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-03-09' in request.url
    assert 'end' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)

    cell.time_range_end_template = 'today|add_days:"14"|adjust_to_week_monday'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-03-09' in request.url and 'end=2020-03-16' in request.url

    cell.time_range_start_template = 'xxx'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start' not in request.url

    # use page variables
    cell.time_range_start_template = 'custom_date'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start=2021-02-03' in request.url

    cell.time_range_start_template = 'not-a-date'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start' not in request.url

    cell.time_range_start_template = 'syntax-error'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start' not in request.url

    cell.time_range_start_template = 'from-request'
    cell.save()
    app.get(location + '?test=2022-05-17')
    request = new_api_mock.call['requests'][-1]
    assert 'start=2022-05-17' in request.url

    cell.time_range_start_template = 'from-template-vars'
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][-1]
    assert 'start=2020-10-01' in request.url


@with_httmock(new_api_mock)
def test_chartng_cell_new_api_filter_params_month(new_api_statistics, nocache, freezer):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    cell.filter_params = {'time_interval': 'month', 'ou': 'default'}
    cell.time_range = 'current-month'
    cell.save()

    freezer.move_to('2021-01-02')
    cell.get_chart()
    request = new_api_mock.call['requests'][0]
    assert 'start=2021-01-01' in request.url and 'end=2021-02-01' in request.url

    cell.time_range = 'previous-month'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][1]
    assert 'start=2020-12-01' in request.url and 'end=2021-01-01' in request.url

    freezer.move_to('2021-11-02')
    cell.time_range = 'next-month'
    cell.save()
    cell.get_chart()
    request = new_api_mock.call['requests'][2]
    assert 'start=2021-12-01' in request.url and 'end=2022-01-01' in request.url


@with_httmock(new_api_mock)
def test_chartng_cell_new_api_filter_params_page_variables(app, admin_user, new_api_statistics, nocache):
    Page.objects.create(title='One', slug='index')
    page = Page.objects.create(
        title='One',
        slug='cards',
        sub_slug='card_id',
        extra_variables={
            'foo': 'bar',
            'bar_id': '{{ 40|add:2 }}',
            'syntax_error': '{% for %}',
            'subslug_dependant': '{{ 40|add:card_id }}',
        },
    )
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:foo'}
    cell.save()

    location = '/api/dataviz/graph/%s/' % cell.pk
    app.get(location)
    request = new_api_mock.call['requests'][0]
    assert 'service=chrono' in request.url
    assert 'ou=bar' in request.url

    cell.filter_params = {'service': 'chrono', 'ou': 'variable:bar_id'}
    cell.save()

    app.get(location)
    request = new_api_mock.call['requests'][1]
    assert 'service=chrono' in request.url
    assert 'ou=42' in request.url

    # unknown variable
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:unknown'}
    cell.save()

    resp = app.get(location)
    assert len(new_api_mock.call['requests']) == 2
    assert 'Page variable not found.' in resp.text

    # variable with invalid syntax
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:syntax_error'}
    cell.save()

    resp = app.get(location)
    assert len(new_api_mock.call['requests']) == 2
    assert 'Syntax error in page variable.' in resp.text

    # variable with missing context
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:subslug_dependant'}
    cell.save()

    resp = app.get(location)
    assert len(new_api_mock.call['requests']) == 2
    assert 'Cannot evaluate page variable.' in resp.text

    # simulate call from page view
    app = login(app)
    resp = app.get('/cards/2/')
    ctx = resp.pyquery('.chartngcell').attr('data-extra-context')

    app.get(location + '?ctx=%s' % ctx)
    request = new_api_mock.call['requests'][2]
    assert 'service=chrono' in request.url
    assert 'ou=42' in request.url

    # reste à tester missing variable
    # et avec display table


@with_httmock(new_api_mock)
def test_chartng_cell_new_api_filter_params_page_variables_table(new_api_statistics, app, nocache):
    Page.objects.create(title='One', slug='index')
    page = Page.objects.create(
        title='One',
        slug='cards',
        sub_slug='card_id',
        extra_variables={
            'foo': 'bar',
            'syntax_error': '{% for %}',
            'subslug_dependant': '{{ 40|add:card_id }}',
        },
    )
    cell = ChartNgCell(page=page, order=1, placeholder='content', chart_type='table')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:foo'}
    cell.save()

    location = '/api/dataviz/graph/%s/' % cell.pk
    app.get(location)
    request = new_api_mock.call['requests'][0]
    assert 'service=chrono' in request.url
    assert 'ou=bar' in request.url

    # unknown variable
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:unknown'}
    cell.save()

    resp = app.get(location)
    assert len(new_api_mock.call['requests']) == 1
    assert 'Page variable not found.' in resp.text

    # variable with invalid syntax
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:syntax_error'}
    cell.save()

    resp = app.get(location)
    assert len(new_api_mock.call['requests']) == 1
    assert 'Syntax error in page variable.' in resp.text

    # variable with missing context
    cell.filter_params = {'service': 'chrono', 'ou': 'variable:subslug_dependant'}
    cell.save()

    resp = app.get(location)
    assert len(new_api_mock.call['requests']) == 1
    assert 'Cannot evaluate page variable.' in resp.text


def test_dataviz_check_validity(nocache):
    page = Page.objects.create(title='One', slug='index', extra_variables={'foo': 'bar'})
    stat = Statistic.objects.create(url='https://stat.com/stats/1/')
    cell = ChartNgCell.objects.create(page=page, order=1, placeholder='content', statistic=stat)

    @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/1/')
    def url_mock(url, request):
        return {'content': json.dumps({'data': [], 'err': 0}), 'status_code': 200}

    with HTTMock(url_mock):
        cell.check_validity()
    assert ValidityInfo.objects.exists() is False

    # cell using page variable is valid even if it cannot be evaluated
    cell.filter_params = {'test': 'variable:foo'}
    cell.save()

    with HTTMock(url_mock):
        cell.check_validity()
    assert ValidityInfo.objects.exists() is False

    cell.filter_params.clear()
    cell.save()

    @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/1/')
    def url_mock2(url, request):
        return {'content': json.dumps({'data': [], 'err': 1}), 'status_code': 404}

    with HTTMock(url_mock2):
        cell.check_validity()
    validity_info = ValidityInfo.objects.latest('pk')
    assert validity_info.invalid_reason_code == 'statistic_data_not_found'

    stat.url = ''
    stat.save()
    cell.check_validity()
    validity_info = ValidityInfo.objects.latest('pk')
    assert validity_info.invalid_reason_code == 'missing_statistic_url'


@with_httmock(new_api_mock)
def test_chartng_cell_new_api_aggregation(new_api_statistics, app, admin_user, nocache):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='daily')
    cell.save()

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id]
    assert time_interval_field.value == 'day'
    assert time_interval_field.options == [
        ('day', True, 'Day'),
        ('week', False, 'Week'),
        ('month', False, 'Month'),
        ('year', False, 'Year'),
        ('weekday', False, 'Week day'),
    ]
    manager_submit_cell(resp.form)
    cell.refresh_from_db()

    chart = cell.get_chart()
    assert len(chart.x_labels) == 484
    assert chart.x_labels[:3] == ['06 Oct 2020', '07 Oct 2020', '08 Oct 2020']
    assert chart.x_labels[-3:] == ['30 Jan 2022', '31 Jan 2022', '01 Feb 2022']
    assert chart.raw_series[0][0][:8] == [0, 0, 0, 0, 0, 0, 0, 1]
    assert chart.raw_series[1][0][:8] == [2, 0, 0, 0, 0, 0, 0, 2]

    time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id]
    time_interval_field.value = 'month'
    manager_submit_cell(resp.form)
    time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id]
    assert time_interval_field.options == [
        ('day', False, 'Day'),
        ('week', False, 'Week'),
        ('month', True, 'Month'),  # month choice is selected
        ('year', False, 'Year'),
        ('weekday', False, 'Week day'),
    ]
    cell.refresh_from_db()

    chart = cell.get_chart()
    assert 'time_interval=day' in new_api_mock.call['requests'][1].url
    assert len(chart.x_labels) == 17
    assert chart.x_labels[:3] == ['Oct 2020', 'Nov 2020', 'Dec 2020']
    assert chart.x_labels[-3:] == ['Dec 2021', 'Jan 2022', 'Feb 2022']
    assert chart.raw_series == [
        ([1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], {'title': 'Serie 1'}),
        ([4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], {'title': 'Serie 2'}),
    ]

    time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id]
    time_interval_field.value = 'year'
    resp.form.submit()
    cell.refresh_from_db()

    chart = cell.get_chart()
    assert 'time_interval=day' in new_api_mock.call['requests'][2].url
    assert chart.x_labels == ['2020', '2021', '2022']
    assert chart.raw_series == [
        ([17, 0, 2], {'title': 'Serie 1'}),
        ([5, 0, 0], {'title': 'Serie 2'}),
    ]

    time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id]
    time_interval_field.value = 'weekday'
    manager_submit_cell(resp.form)
    cell.refresh_from_db()

    chart = cell.get_chart()
    assert 'time_interval=day' in new_api_mock.call['requests'][3].url
    assert chart.x_labels == ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
    assert chart.raw_series == [
        ([16, 3, 0, 0, 0, 0, 0], {'title': 'Serie 1'}),
        ([1, 4, 0, 0, 0, 0, 0], {'title': 'Serie 2'}),
    ]

    time_interval_field = resp.form['cdataviz_chartngcell-%s-time_interval' % cell.id]
    time_interval_field.value = 'week'
    manager_submit_cell(resp.form)
    cell.refresh_from_db()

    chart = cell.get_chart()
    assert 'time_interval=day' in new_api_mock.call['requests'][1].url
    assert len(chart.x_labels) == 70
    assert chart.x_labels[:3] == ['W41-2020', 'W42-2020', 'W43-2020']
    assert chart.x_labels[-6:] == ['W52-2021', 'W1-2022', 'W2-2022', 'W3-2022', 'W4-2022', 'W5-2022']
    assert chart.raw_series == [
        ([0, 1, 0, 0, 0, 0, 0, 0, 16] + [0] * 60 + [2], {'title': 'Serie 1'}),
        ([2, 2, 0, 0, 0, 0, 0, 0, 1] + [0] * 61, {'title': 'Serie 2'}),
    ]

    cell.statistic = Statistic.objects.get(slug='leap-week')
    cell.save()
    manager_submit_cell(resp.form)
    chart = cell.get_chart()
    assert 'time_interval=day' in new_api_mock.call['requests'][1].url
    assert len(chart.x_labels) == 1
    assert chart.x_labels == ['W53-2020']
    assert chart.raw_series == [([19], {'title': 'Serie 1'})]


@with_httmock(new_api_mock)
def test_chartng_cell_new_api_month_translation(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    # populate filter params
    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)
    resp.form.submit()

    cell.refresh_from_db()
    chart = cell.get_chart()
    assert chart.x_labels == ['Oct 2020', 'Nov 2020', 'Dec 2020']


@with_httmock(new_api_mock)
def test_chart_filters_cell(new_api_statistics, app, admin_user, nocache):
    page = Page.objects.create(title='One', slug='index')
    ChartFiltersCell.objects.create(page=page, order=1, placeholder='content')
    app = login(app)
    resp = app.get('/')
    assert 'No filters are available' in resp.text

    # add unconfigured chart
    first_cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content')
    resp = app.get('/')
    assert 'No filters are available' in resp.text

    # add statistics to chart
    first_cell.statistic = Statistic.objects.get(slug='one-serie')
    first_cell.save()
    resp = app.get('/')
    assert len(resp.form.fields) == 7
    assert 'filter-time_range_start' in resp.form.fields
    assert 'filter-time_range_end' in resp.form.fields

    time_range_field = resp.form['filter-time_range']
    assert time_range_field.value == ''
    assert time_range_field.options == [
        ('', True, '---------'),
        ('previous-year', False, 'Previous year'),
        ('current-year', False, 'Current year'),
        ('previous-month', False, 'Previous month'),
        ('current-month', False, 'Current month'),
        ('previous-week', False, 'Previous week'),
        ('current-week', False, 'Current week'),
        ('range', False, 'Free range (date)'),
    ]

    time_interval_field = resp.form['filter-time_interval']
    assert time_interval_field.value == 'month'
    assert time_interval_field.options == [
        ('day', False, 'Day'),
        ('month', True, 'Month'),
        ('year', False, 'Year'),
        ('week', False, 'Week'),
        ('weekday', False, 'Week day'),
    ]

    service_field = resp.form['filter-service']
    assert service_field.value == 'chrono'
    assert service_field.options == [
        ('', False, '---------'),
        ('chrono', True, 'Chrono'),
        ('combo', False, 'Combo'),
    ]

    ou_field = resp.form['filter-ou']
    assert ou_field.value == ''
    assert ou_field.options == [
        ('', True, '---------'),
        ('default', False, 'Default OU'),
        ('other', False, 'Other OU'),
    ]

    # adding new cell with same statistics changes nothing
    cell = ChartNgCell(page=page, order=3, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()
    old_resp = resp
    resp = app.get('/')
    for field in ('time_range', 'time_interval', 'service', 'ou'):
        assert resp.form['filter-%s' % field].options == old_resp.form['filter-%s' % field].options

    # changing one filter value makes it disappear
    cell.filter_params = {'ou': 'default'}
    cell.save()
    resp = app.get('/')
    assert 'ou' not in resp.form.fields
    for field in ('time_range', 'time_interval', 'service'):
        assert resp.form['filter-%s' % field].options == old_resp.form['filter-%s' % field].options

    # setting the same value for the other cell makes it appear again
    first_cell.filter_params = {'ou': 'default'}
    first_cell.save()
    resp = app.get('/')
    assert resp.form['filter-ou'].value == 'default'

    # changing statistics type of cell remove some fields
    cell.statistic = Statistic.objects.get(slug='daily')
    cell.save()
    resp = app.get('/')
    assert 'ou' not in resp.form.fields
    assert 'service' not in resp.form.fields
    for field in ('time_range', 'time_interval'):
        assert resp.form['filter-%s' % field].options == old_resp.form['filter-%s' % field].options

    # changing time_interval value makes interval fields disappear
    cell.time_range = 'previous-year'
    cell.save()
    old_resp = resp
    resp = app.get('/')
    assert 'filter-time_range' not in resp.form.fields
    assert 'filter-time_range_start' not in resp.form.fields
    assert 'filter-time_range_end' not in resp.form.fields
    assert resp.form['filter-time_interval'].options == old_resp.form['filter-time_interval'].options

    # setting the same value for the other cell makes it appear again
    first_cell.time_range = 'previous-year'
    first_cell.save()
    resp = app.get('/')
    assert resp.form['filter-time_range'].value == 'previous-year'
    assert resp.form['filter-time_interval'].options == old_resp.form['filter-time_interval'].options

    # only common choices are shown
    first_cell.statistic.filters[0]['options'].remove({'id': 'day', 'label': 'Day'})
    first_cell.statistic.save()
    resp = app.get('/')
    assert resp.form['filter-time_interval'].options == [
        ('month', True, 'Month'),
        ('year', False, 'Year'),
    ]

    # if no common choices exist, field is removed
    first_cell.statistic.filters[0]['options'] = [{'id': 'random', 'label': 'Random'}]
    first_cell.statistic.save()
    resp = app.get('/')
    assert 'filter-time_interval' not in resp.form.fields
    assert resp.form['filter-time_range'].value == 'previous-year'

    # form is not shown if no common filters exist
    first_cell.time_range = 'current-year'
    first_cell.save()
    resp = app.get('/')
    assert 'No filters are available' in resp.text


@with_httmock(new_api_mock)
def test_chart_filters_cell_future_data(app, admin_user, new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='with-future-data')
    cell.save()
    ChartFiltersCell.objects.create(page=page, order=2, placeholder='content')

    app = login(app)
    resp = app.get('/')
    time_range_field = resp.form['filter-time_range']
    assert time_range_field.value == ''
    assert time_range_field.options == [
        ('', True, '---------'),
        ('previous-year', False, 'Previous year'),
        ('current-year', False, 'Current year'),
        ('next-year', False, 'Next year'),
        ('previous-month', False, 'Previous month'),
        ('current-month', False, 'Current month'),
        ('next-month', False, 'Next month'),
        ('previous-week', False, 'Previous week'),
        ('current-week', False, 'Current week'),
        ('next-week', False, 'Next week'),
        ('range', False, 'Free range (date)'),
    ]

    # adding cell without future data makes choice disappear
    cell = ChartNgCell(page=page, order=3, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    resp = app.get('/')
    time_range_field = resp.form['filter-time_range']
    assert time_range_field.value == ''
    assert time_range_field.options == [
        ('', True, '---------'),
        ('previous-year', False, 'Previous year'),
        ('current-year', False, 'Current year'),
        ('previous-month', False, 'Previous month'),
        ('current-month', False, 'Current month'),
        ('previous-week', False, 'Previous week'),
        ('current-week', False, 'Current week'),
        ('range', False, 'Free range (date)'),
    ]


@with_httmock(new_api_mock)
def test_chart_filters_cell_range_template(new_api_statistics, app, admin_user, nocache):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()
    ChartFiltersCell.objects.create(page=page, order=2, placeholder='content')

    app = login(app)
    resp = app.get('/')
    assert 'filter-time_range' in resp.form.fields
    assert 'filter-time_range_start' in resp.form.fields
    assert 'filter-time_range_end' in resp.form.fields

    # if time range is set using templates, time range fields are hidden from filters
    cell.time_range = 'range-template'
    cell.save()
    resp = app.get('/')
    assert 'filter-time_range' not in resp.form.fields
    assert 'filter-time_range_start' not in resp.form.fields
    assert 'filter-time_range_end' not in resp.form.fields


@with_httmock(new_api_mock)
def test_chart_filters_cell_with_subfilters(new_api_statistics, app, admin_user, nocache):
    page = Page.objects.create(title='One', slug='index')
    ChartFiltersCell.objects.create(page=page, order=1, placeholder='content')
    cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='with-subfilter')
    cell.save()

    app = login(app)
    resp = app.get('/')
    assert 'filter-form' in resp.form.fields
    assert 'filter-menu' not in resp.form.fields

    # select a choice with subfilters in manager
    resp = app.get('/manage/pages/%s/' % page.id)
    resp.forms[1]['cdataviz_chartngcell-%s-form' % cell.id] = 'food-request'
    manager_submit_cell(resp.forms[1])

    resp = app.get('/')
    assert 'filter-form' in resp.form.fields
    assert 'filter-menu' in resp.form.fields


@with_httmock(new_api_mock)
@pytest.mark.freeze_time('2021-10-06')
def test_chartng_cell_api_view_get_parameters(app, normal_user, new_api_statistics, nocache):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    location = '/api/dataviz/graph/%s/' % cell.id
    app.get(location)
    request = new_api_mock.call['requests'][0]
    assert 'time_interval=' not in request.url
    assert 'ou=' not in request.url

    cell.filter_params = {'time_interval': 'month', 'ou': 'default'}
    cell.save()
    app.get(location)
    request = new_api_mock.call['requests'][1]
    assert 'time_interval=month' in request.url
    assert 'ou=default' in request.url

    app.get(location + '?filter-time_interval=year')
    request = new_api_mock.call['requests'][2]
    assert 'time_interval=year' in request.url
    assert 'ou=default' in request.url

    cell.time_range = 'range'
    cell.time_range_start = '2022-01-01'
    cell.save()

    app.get(location)
    request = new_api_mock.call['requests'][3]
    assert 'start=2022-01-01' in request.url
    assert 'ou=default' in request.url

    app.get(location + '?filter-time_range=current-month')
    request = new_api_mock.call['requests'][4]
    assert 'start=2021-10-01' in request.url
    assert 'end=2021-11-01' in request.url
    assert 'ou=default' in request.url

    cell.filter_params.clear()
    cell.statistic = Statistic.objects.get(slug='filter-multiple')
    cell.save()
    app.get(location + '?filter-color=green&filter-color=blue')
    request = new_api_mock.call['requests'][5]
    assert 'color=green&color=blue' in request.url

    cell.filter_params.clear()
    cell.statistic = Statistic.objects.get(slug='with-subfilter')
    cell.filter_params = {'form': 'food-request'}
    cell.save()
    cell.update_subfilters()
    app.get(location + '?filter-menu=vegan')
    request = new_api_mock.call['requests'][7]
    assert 'menu=vegan' in request.url

    # unknown params
    app.get(location + '?filter-time_interval=month&filter-ou=default')
    request = new_api_mock.call['requests'][8]
    assert 'time_interval=' not in request.url
    assert 'ou=' not in request.url

    # wrong params
    resp = app.get(location + '?filter-time_range_start=xxx')
    assert 'Wrong parameters' in resp.text
    assert len(new_api_mock.call['requests']) == 9


@with_httmock(new_api_mock)
def test_spooler_refresh_statistics_data(new_api_statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='one-serie')
    cell.save()

    refresh_statistics_data(cell.pk)
    assert len(new_api_mock.call['requests']) == 1

    refresh_statistics_data(cell.pk)
    assert len(new_api_mock.call['requests']) == 2

    # variables cannot be evaluated in spooler
    page.extra_variables = {'test': 'test'}
    page.save()
    cell.filter_params = {'ou': 'variable:test'}
    cell.save()
    refresh_statistics_data(cell.pk)
    assert len(new_api_mock.call['requests']) == 2

    ChartNgCell.objects.all().delete()
    refresh_statistics_data(cell.pk)
    assert len(new_api_mock.call['requests']) == 2


@with_httmock(bijoe_mock)
def test_spooler_refresh_statistics_data_bijoe(statistics):
    page = Page.objects.create(title='One', slug='index')
    cell = ChartNgCell(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='example')
    cell.save()

    refresh_statistics_data(cell.pk)
    assert len(bijoe_mock.call['requests']) == 1


@with_httmock(new_api_mock)
def test_chartng_cell_subfilter_page_variable(new_api_statistics, app, admin_user, nocache):
    page = Page.objects.create(title='One', slug='index', extra_variables={'foo': 'bar'})
    cell = ChartNgCell.objects.create(page=page, order=1, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='with-subfilter')
    cell.save()

    app = login(app)
    resp = app.get('/manage/pages/%s/' % page.id)

    # set filter value to page variable
    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id
    resp.form[field_prefix + 'other'] = 'variable:foo'
    manager_submit_cell(resp.form)

    # change choice with subfilters
    resp.form[field_prefix + 'form'] = 'food-request'
    manager_submit_cell(resp.form)
    assert field_prefix + 'menu' in resp.form.fields

    # page variable has syntax error
    page.extra_variables = {'foo': '{% for %}'}
    page.save()
    resp.form[field_prefix + 'form'] = 'contact'
    manager_submit_cell(resp.form)
    assert field_prefix + 'menu' in resp.form.fields


@with_httmock(new_api_mock)
def test_chart_filters_cell_dynamic_subfilters(new_api_statistics, app, admin_user):
    page = Page.objects.create(title='One', slug='index')
    ChartFiltersCell.objects.create(page=page, order=1, placeholder='content')
    cell = ChartNgCell.objects.create(page=page, order=2, placeholder='content')
    cell.statistic = Statistic.objects.get(slug='with-subfilter')
    cell.save()

    app = login(app)
    resp = app.get('/')
    assert 'filter-form' in resp.form.fields
    assert 'filter-menu' not in resp.form.fields

    # ensure choice exist
    resp.form['filter-form'] = 'food-request'

    # simulate chart cell ajax refresh on form submission
    app.get('/api/dataviz/graph/%s/' % cell.pk + '?filter-form=food-request&filters_cell_id=xxx')

    # simulate filters cell ajax refresh after cell refresh
    location = resp.pyquery('.chartfilterscell').attr('data-ajax-cell-url')
    resp = app.get(location + '?filter-form=food-request&filters_cell_id=xxx')
    assert resp.form['filter-form'].value == 'food-request'
    assert 'filter-menu' in resp.form.fields

    # check isolation between pages by modifying filters_cell_id
    app.get('/api/dataviz/graph/%s/' % cell.pk + '?filter-form=contact&filters_cell_id=yyy')
    resp = app.get(location + '?filter-form=food-request&filters_cell_id=xxx')
    assert 'filter-form' in resp.form.fields
    assert 'filter-menu' in resp.form.fields