general: keep a per-language cache for skeleton pages (#70670)
gitea/hobo/pipeline/head This commit looks good Details

This commit is contained in:
Frédéric Péters 2022-10-23 09:59:06 +02:00
parent 1a4cb01666
commit 1ebd61d47a
3 changed files with 91 additions and 17 deletions

View File

@ -12,6 +12,7 @@ from django.core.cache import cache
from django.template import Template
from django.utils.encoding import smart_bytes
from django.utils.http import urlencode
from django.utils.translation import get_language, get_supported_language_variant
from hobo.scrutiny.wsgi.middleware import VersionMiddleware
@ -44,9 +45,16 @@ class RemoteTemplate:
def __init__(self, source):
self.source = source
self.set_language()
def set_language(self, language=None):
try:
self.language_code = get_supported_language_variant(language or get_language())
except LookupError:
self.language_code = settings.LANGUAGES[0][0]
def get_cached_item(self):
item = cache.get(self.PAGE_CACHE_KEY)
item = cache.get(self.get_page_cache_key())
if self.source != '404' and item:
# page_cache is a dict redirect_url -> page content, get the best
# match.
@ -65,10 +73,14 @@ class RemoteTemplate:
item = cache.get(self.cache_key)
return item
def get_page_cache_key(self):
return self.PAGE_CACHE_KEY + '-' + self.language_code
@property
def cache_key(self):
return hashlib.md5(
urllib.parse.urlunparse(urllib.parse.urlparse(self.source)[:3] + ('', '', '')).encode('ascii')
+ self.language_code.encode('ascii')
).hexdigest()
def get_template(self):
@ -101,7 +113,11 @@ class RemoteTemplate:
return Template(template_body)
def update_content(self, in_thread=True):
r = requests.get(self.theme_skeleton_url, params={'source': self.source})
r = requests.get(
self.theme_skeleton_url,
params={'source': self.source},
headers={'Accept-Language': self.language_code},
)
if r.status_code != 200:
logger.error('failed to retrieve theme (status code: %s)', r.status_code)
return None
@ -117,23 +133,29 @@ class RemoteTemplate:
return r.text
def update_all_pages_cache(self):
# always cache root
root_url = urllib.parse.urlunparse(urllib.parse.urlparse(self.source)[:2] + ('/', '', '', ''))
if not root_url in self.combo_skeleton_pages.values():
self.combo_skeleton_pages['__root'] = root_url
for lang_code, _ in settings.LANGUAGES:
self.set_language(lang_code)
# always cache root
root_url = urllib.parse.urlunparse(urllib.parse.urlparse(self.source)[:2] + ('/', '', '', ''))
if not root_url in self.combo_skeleton_pages.values():
self.combo_skeleton_pages['__root'] = root_url
page_cache = {}
for page_id, page_redirect_url in self.combo_skeleton_pages.items():
r = requests.get(self.theme_skeleton_url, params={'source': page_redirect_url})
if r.status_code != 200:
# abort
return
page_cache[page_redirect_url] = r.text
page_cache = {}
for page_id, page_redirect_url in self.combo_skeleton_pages.items():
r = requests.get(
self.theme_skeleton_url,
params={'source': page_redirect_url},
headers={'Accept-Language': lang_code},
)
if r.status_code != 200:
# abort
return
page_cache[page_redirect_url] = r.text
expiry_time = datetime.datetime.now() + datetime.timedelta(seconds=CACHE_REFRESH_TIMEOUT)
cache.set(
self.PAGE_CACHE_KEY, (page_cache, expiry_time), 2592000
) # bypass cache level expiration time
expiry_time = datetime.datetime.now() + datetime.timedelta(seconds=CACHE_REFRESH_TIMEOUT)
cache.set(
self.get_page_cache_key(), (page_cache, expiry_time), 2592000
) # bypass cache level expiration time
def cache(self, template_body):
expiry_time = datetime.datetime.now() + datetime.timedelta(seconds=CACHE_REFRESH_TIMEOUT)

View File

@ -4,6 +4,7 @@ import hobo.test_utils
LANGUAGE_CODE = 'en-us'
BROKER_URL = 'memory://'
LANGUAGES = [('en', 'English')]
INSTALLED_APPS += ('hobo.agent.common', 'hobo.user_name.apps.UserNameConfig')

View File

@ -4,6 +4,7 @@ from unittest import mock
from django.core.cache import cache
from django.test import override_settings
from django.utils import translation
from httmock import HTTMock, urlmatch
from hobo.context_processors import theme_base, user_urls
@ -68,6 +69,56 @@ def test_theme_base(settings, rf):
assert len(seen_urls) == 0
def test_theme_base_language(settings, rf):
settings.THEME_SKELETON_URL = 'http://combo.example.com/_skeleton_/'
seen_urls = []
@urlmatch(netloc=r'combo.example.com$')
def combo_mock(url, request):
language = request.headers['Accept-Language']
seen_urls.append((language, url.geturl()))
status_code = 200
assert language in ('en', 'fr'), 'invalid language'
if language == 'en':
content = 'Skeleton for English'
elif language == 'fr':
content = 'Skeleton for French'
return {
'status_code': status_code,
'content': content,
'headers': {'X-Combo-Skeleton-Pages': json.dumps({'1': 'http://testserver/foo'})},
}
cache.clear()
with HTTMock(combo_mock), override_settings(
INSTALLED_APPS=[], LANGUAGES=[('en', 'English'), ('fr', 'French')]
):
context = theme_base(rf.get('/'))
assert context['theme_base']().source == 'Skeleton for English'
for i in range(10):
# wait for the other requests, made from a thread, to happen
time.sleep(0.1)
if len(seen_urls) == 5:
break
assert len(seen_urls) == 5
assert ('en', 'http://combo.example.com/_skeleton_/?source=http%3A%2F%2Ftestserver%2F') in seen_urls
assert ('fr', 'http://combo.example.com/_skeleton_/?source=http%3A%2F%2Ftestserver%2F') in seen_urls
assert (
'en',
'http://combo.example.com/_skeleton_/?source=http%3A%2F%2Ftestserver%2Ffoo',
) in seen_urls
assert (
'fr',
'http://combo.example.com/_skeleton_/?source=http%3A%2F%2Ftestserver%2Ffoo',
) in seen_urls
assert theme_base(rf.get('/'))['theme_base']().source == 'Skeleton for English'
with translation.override('fr'):
assert theme_base(rf.get('/'))['theme_base']().source == 'Skeleton for French'
def test_user_urls(settings, rf):
settings.TEMPLATE_VARS = {
'idp_registration_url': 'https://idp/register/',