From a9dc97a46d9aa07a0b11d586f3477bc31e360b26 Mon Sep 17 00:00:00 2001 From: cuom1999 Date: Sat, 18 Feb 2023 15:12:33 -0600 Subject: [PATCH] Use infinite pagination --- .../migrations/0011_alter_message_hidden.py | 20 +++ chat_box/models.py | 2 +- chat_box/views.py | 34 ++-- judge/utils/infinite_paginator.py | 152 ++++++++++++++++++ judge/views/organization.py | 2 +- judge/views/ranked_submission.py | 6 +- judge/views/submission.py | 33 ++-- judge/views/user.py | 4 +- templates/chat/chat.html | 32 ++-- templates/chat/message.html | 2 +- templates/chat/message_list.html | 1 + 11 files changed, 230 insertions(+), 58 deletions(-) create mode 100644 chat_box/migrations/0011_alter_message_hidden.py create mode 100644 judge/utils/infinite_paginator.py diff --git a/chat_box/migrations/0011_alter_message_hidden.py b/chat_box/migrations/0011_alter_message_hidden.py new file mode 100644 index 0000000..1393e82 --- /dev/null +++ b/chat_box/migrations/0011_alter_message_hidden.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.18 on 2023-02-18 21:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("chat_box", "0010_auto_20221028_0300"), + ] + + operations = [ + migrations.AlterField( + model_name="message", + name="hidden", + field=models.BooleanField( + db_index=True, default=False, verbose_name="is hidden" + ), + ), + ] diff --git a/chat_box/models.py b/chat_box/models.py index 328e4c6..2dc2cb1 100644 --- a/chat_box/models.py +++ b/chat_box/models.py @@ -31,7 +31,7 @@ class Message(models.Model): author = models.ForeignKey(Profile, verbose_name=_("user"), on_delete=CASCADE) time = models.DateTimeField(verbose_name=_("posted time"), auto_now_add=True) body = models.TextField(verbose_name=_("body of comment"), max_length=8192) - hidden = models.BooleanField(verbose_name="is hidden", default=False) + hidden = models.BooleanField(verbose_name="is hidden", default=False, db_index=True) room = models.ForeignKey( Room, verbose_name="room id", on_delete=CASCADE, default=None, null=True ) diff --git a/chat_box/views.py b/chat_box/views.py index 34b82bb..e9f406f 100644 --- a/chat_box/views.py +++ b/chat_box/views.py @@ -48,16 +48,26 @@ class ChatView(ListView): super().__init__() self.room_id = None self.room = None - self.paginate_by = 50 self.messages = None - self.paginator = None + self.page_size = 20 def get_queryset(self): return self.messages + def has_next(self): + try: + msg = Message.objects.filter(room=self.room_id).earliest("id") + except Exception as e: + return False + return msg not in self.messages + def get(self, request, *args, **kwargs): request_room = kwargs["room_id"] - page = request.GET.get("page") + try: + last_id = int(request.GET.get("last_id")) + except Exception: + last_id = 1e15 + only_messages = request.GET.get("only_messages") if request_room: try: @@ -69,23 +79,20 @@ class ChatView(ListView): else: request_room = None - if request_room != self.room_id or not self.messages: - self.room_id = request_room - self.messages = Message.objects.filter(hidden=False, room=self.room_id) - self.paginator = Paginator(self.messages, self.paginate_by) - - if page == None: + self.room_id = request_room + self.messages = Message.objects.filter( + hidden=False, room=self.room_id, id__lt=last_id + )[: self.page_size] + if not only_messages: update_last_seen(request, **kwargs) return super().get(request, *args, **kwargs) - cur_page = self.paginator.get_page(page) - return render( request, "chat/message_list.html", { - "object_list": cur_page.object_list, - "num_pages": self.paginator.num_pages, + "object_list": self.messages, + "has_next": self.has_next(), }, ) @@ -96,6 +103,7 @@ class ChatView(ListView): context["last_msg"] = event.last() context["status_sections"] = get_status_context(self.request) context["room"] = self.room_id + context["has_next"] = self.has_next() context["unread_count_lobby"] = get_unread_count(None, self.request.profile) if self.room: users_room = [self.room.user_one, self.room.user_two] diff --git a/judge/utils/infinite_paginator.py b/judge/utils/infinite_paginator.py new file mode 100644 index 0000000..5693481 --- /dev/null +++ b/judge/utils/infinite_paginator.py @@ -0,0 +1,152 @@ +import collections +import inspect +from math import ceil + +from django.core.paginator import EmptyPage, InvalidPage +from django.http import Http404 +from django.utils.functional import cached_property +from django.utils.inspect import method_has_no_args + + +class InfinitePage(collections.abc.Sequence): + def __init__( + self, object_list, number, unfiltered_queryset, page_size, pad_pages, paginator + ): + self.object_list = list(object_list) + self.number = number + self.unfiltered_queryset = unfiltered_queryset + self.page_size = page_size + self.pad_pages = pad_pages + self.num_pages = 1e3000 + self.paginator = paginator + + def __repr__(self): + return "" % self.number + + def __len__(self): + return len(self.object_list) + + def __getitem__(self, index): + return self.object_list[index] + + @cached_property + def _after_up_to_pad(self): + first_after = self.number * self.page_size + padding_length = self.pad_pages * self.page_size + queryset = self.unfiltered_queryset[ + first_after : first_after + padding_length + 1 + ] + c = getattr(queryset, "count", None) + if callable(c) and not inspect.isbuiltin(c) and method_has_no_args(c): + return c() + return len(queryset) + + def has_next(self): + return self._after_up_to_pad > 0 + + def has_previous(self): + return self.number > 1 + + def has_other_pages(self): + return self.has_previous() or self.has_next() + + def next_page_number(self): + if not self.has_next(): + raise EmptyPage() + return self.number + 1 + + def previous_page_number(self): + if self.number <= 1: + raise EmptyPage() + return self.number - 1 + + def start_index(self): + return (self.page_size * (self.number - 1)) + 1 + + def end_index(self): + return self.start_index() + len(self.object_list) + + @cached_property + def main_range(self): + start = max(1, self.number - self.pad_pages) + end = self.number + min( + int(ceil(self._after_up_to_pad / self.page_size)), self.pad_pages + ) + return range(start, end + 1) + + @cached_property + def leading_range(self): + return range(1, min(3, self.main_range[0])) + + @cached_property + def has_trailing(self): + return self._after_up_to_pad > self.pad_pages * self.page_size + + @cached_property + def page_range(self): + result = list(self.leading_range) + main_range = self.main_range + + # Add ... element if there is space in between. + if result and result[-1] + 1 < self.main_range[0]: + result.append(False) + + result += list(main_range) + + # Add ... element if there are elements after main_range. + if self.has_trailing: + result.append(False) + return result + + +class DummyPaginator: + is_infinite = True + + def __init__(self, per_page): + self.per_page = per_page + + +def infinite_paginate(queryset, page, page_size, pad_pages, paginator=None): + if page < 1: + raise EmptyPage() + sliced = queryset[(page - 1) * page_size : page * page_size] + if page > 1 and not sliced: + raise EmptyPage() + return InfinitePage(sliced, page, queryset, page_size, pad_pages, paginator) + + +class InfinitePaginationMixin: + pad_pages = 4 + + @property + def use_infinite_pagination(self): + return True + + def paginate_queryset(self, queryset, page_size): + if not self.use_infinite_pagination: + paginator, page, object_list, has_other = super().paginate_queryset( + queryset, page_size + ) + paginator.is_infinite = False + return paginator, page, object_list, has_other + + page_kwarg = self.page_kwarg + page = self.kwargs.get(page_kwarg) or self.request.GET.get(page_kwarg) or 1 + try: + page_number = int(page) + except ValueError: + raise Http404("Page cannot be converted to an int.") + try: + paginator = DummyPaginator(page_size) + page = infinite_paginate( + queryset, page_number, page_size, self.pad_pages, paginator + ) + return paginator, page, page.object_list, page.has_other_pages() + except InvalidPage as e: + raise Http404( + "Invalid page (%(page_number)s): %(message)s" + % { + "page_number": page_number, + "message": str(e), + } + ) diff --git a/judge/views/organization.py b/judge/views/organization.py index 411074d..b633a10 100644 --- a/judge/views/organization.py +++ b/judge/views/organization.py @@ -480,7 +480,7 @@ class OrganizationSubmissions( def contest(self): return None - def _get_queryset(self): + def get_queryset(self): return ( super() ._get_entire_queryset() diff --git a/judge/views/ranked_submission.py b/judge/views/ranked_submission.py index a08bb98..e00a460 100644 --- a/judge/views/ranked_submission.py +++ b/judge/views/ranked_submission.py @@ -27,7 +27,7 @@ class RankedSubmissions(ProblemSubmissions): constraint = "" queryset = ( super(RankedSubmissions, self) - ._get_queryset() + .get_queryset() .filter(user__is_unlisted=False) ) join_sql_subquery( @@ -76,6 +76,4 @@ class RankedSubmissions(ProblemSubmissions): ) def _get_result_data(self): - return get_result_data( - super(RankedSubmissions, self)._get_queryset().order_by() - ) + return get_result_data(super(RankedSubmissions, self).get_queryset().order_by()) diff --git a/judge/views/submission.py b/judge/views/submission.py index 71287b2..5594c32 100644 --- a/judge/views/submission.py +++ b/judge/views/submission.py @@ -47,13 +47,11 @@ from judge.utils.problems import user_editable_ids from judge.utils.problem_data import get_problem_case from judge.utils.raw_sql import join_sql_subquery, use_straight_join from judge.utils.views import DiggPaginatorMixin +from judge.utils.infinite_paginator import InfinitePaginationMixin from judge.utils.views import TitleMixin from judge.utils.timedelta import nice_repr -MAX_NUMBER_OF_QUERY_SUBMISSIONS = 50000 - - def submission_related(queryset): return queryset.select_related("user__user", "problem", "language").only( "id", @@ -333,7 +331,7 @@ class SubmissionsListBase(DiggPaginatorMixin, TitleMixin, ListView): return result def _get_result_data(self): - return get_result_data(self._get_queryset().order_by()) + return get_result_data(self.get_queryset().order_by()) def access_check(self, request): pass @@ -412,7 +410,7 @@ class SubmissionsListBase(DiggPaginatorMixin, TitleMixin, ListView): return queryset - def _get_queryset(self): + def get_queryset(self): queryset = self._get_entire_queryset() if not self.in_contest: if self.request.organization: @@ -434,9 +432,6 @@ class SubmissionsListBase(DiggPaginatorMixin, TitleMixin, ListView): ) return queryset - def get_queryset(self): - return self._get_queryset()[:MAX_NUMBER_OF_QUERY_SUBMISSIONS] - def get_my_submissions_page(self): return None @@ -578,10 +573,10 @@ class GeneralSubmissions(SubmissionsListBase): class AllUserSubmissions(ConditionalUserTabMixin, UserMixin, GeneralSubmissions): - def _get_queryset(self): + def get_queryset(self): return ( super(AllUserSubmissions, self) - ._get_queryset() + .get_queryset() .filter(user_id=self.profile.id) ) @@ -608,12 +603,10 @@ class AllUserSubmissions(ConditionalUserTabMixin, UserMixin, GeneralSubmissions) class AllFriendSubmissions(LoginRequiredMixin, GeneralSubmissions): - def _get_queryset(self): + def get_queryset(self): friends = self.request.profile.get_friends() return ( - super(AllFriendSubmissions, self) - ._get_queryset() - .filter(user_id__in=friends) + super(AllFriendSubmissions, self).get_queryset().filter(user_id__in=friends) ) def get_title(self): @@ -631,7 +624,7 @@ class ProblemSubmissionsBase(SubmissionsListBase): dynamic_update = True check_contest_in_access_check = False - def _get_queryset(self): + def get_queryset(self): if ( self.in_contest and not self.contest.contest_problems.filter( @@ -723,10 +716,10 @@ class UserProblemSubmissions(ConditionalUserTabMixin, UserMixin, ProblemSubmissi if not self.is_own: self.access_check_contest(request) - def _get_queryset(self): + def get_queryset(self): return ( super(UserProblemSubmissions, self) - ._get_queryset() + .get_queryset() .filter(user_id=self.profile.id) ) @@ -804,9 +797,13 @@ def single_submission_query(request): return single_submission(request, int(request.GET["id"]), bool(show_problem)) -class AllSubmissions(GeneralSubmissions): +class AllSubmissions(InfinitePaginationMixin, GeneralSubmissions): stats_update_interval = 3600 + @property + def use_infinite_pagination(self): + return not self.in_contest + def get_context_data(self, **kwargs): context = super(AllSubmissions, self).get_context_data(**kwargs) context["dynamic_update"] = ( diff --git a/judge/views/user.py b/judge/views/user.py index d98f548..9796a90 100644 --- a/judge/views/user.py +++ b/judge/views/user.py @@ -44,12 +44,12 @@ from judge.utils.problems import contest_completed_ids, user_completed_ids from judge.utils.ranker import ranker from judge.utils.unicode import utf8text from judge.utils.views import ( - DiggPaginatorMixin, QueryStringSortMixin, TitleMixin, generic_message, SingleObjectFormView, ) +from judge.utils.infinite_paginator import InfinitePaginationMixin from .contests import ContestRanking __all__ = [ @@ -437,7 +437,7 @@ def edit_profile(request): ) -class UserList(QueryStringSortMixin, DiggPaginatorMixin, TitleMixin, ListView): +class UserList(QueryStringSortMixin, InfinitePaginationMixin, TitleMixin, ListView): model = Profile title = gettext_lazy("Leaderboard") context_object_name = "users" diff --git a/templates/chat/chat.html b/templates/chat/chat.html index 7e03526..36c250d 100644 --- a/templates/chat/chat.html +++ b/templates/chat/chat.html @@ -23,25 +23,23 @@ ];