Use infinite pagination

chat_box/migrations/0011_alter_message_hidden.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.18 on 2023-02-18 21:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("chat_box", "0010_auto_20221028_0300"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="message",
+            name="hidden",
+            field=models.BooleanField(
+                db_index=True, default=False, verbose_name="is hidden"
+            ),
+        ),
+    ]

chat_box/models.py

@@ -31,7 +31,7 @@ class Message(models.Model):
     author = models.ForeignKey(Profile, verbose_name=_("user"), on_delete=CASCADE)
     time = models.DateTimeField(verbose_name=_("posted time"), auto_now_add=True)
     body = models.TextField(verbose_name=_("body of comment"), max_length=8192)
-    hidden = models.BooleanField(verbose_name="is hidden", default=False)
+    hidden = models.BooleanField(verbose_name="is hidden", default=False, db_index=True)
     room = models.ForeignKey(
         Room, verbose_name="room id", on_delete=CASCADE, default=None, null=True
     )

chat_box/views.py

@@ -48,16 +48,26 @@ class ChatView(ListView):
         super().__init__()
         self.room_id = None
         self.room = None
-        self.paginate_by = 50
         self.messages = None
-        self.paginator = None
+        self.page_size = 20
 
     def get_queryset(self):
         return self.messages
 
+    def has_next(self):
+        try:
+            msg = Message.objects.filter(room=self.room_id).earliest("id")
+        except Exception as e:
+            return False
+        return msg not in self.messages
+
     def get(self, request, *args, **kwargs):
         request_room = kwargs["room_id"]
-        page = request.GET.get("page")
+        try:
+            last_id = int(request.GET.get("last_id"))
+        except Exception:
+            last_id = 1e15
+        only_messages = request.GET.get("only_messages")
 
         if request_room:
             try:
@@ -69,23 +79,20 @@ class ChatView(ListView):
         else:
             request_room = None
 
-        if request_room != self.room_id or not self.messages:
+        self.room_id = request_room
-            self.room_id = request_room
+        self.messages = Message.objects.filter(
-            self.messages = Message.objects.filter(hidden=False, room=self.room_id)
+            hidden=False, room=self.room_id, id__lt=last_id
-            self.paginator = Paginator(self.messages, self.paginate_by)
+        )[: self.page_size]
-
+        if not only_messages:
-        if page == None:
             update_last_seen(request, **kwargs)
             return super().get(request, *args, **kwargs)
 
-        cur_page = self.paginator.get_page(page)
-
         return render(
             request,
             "chat/message_list.html",
             {
-                "object_list": cur_page.object_list,
+                "object_list": self.messages,
-                "num_pages": self.paginator.num_pages,
+                "has_next": self.has_next(),
             },
         )
 
@@ -96,6 +103,7 @@ class ChatView(ListView):
         context["last_msg"] = event.last()
         context["status_sections"] = get_status_context(self.request)
         context["room"] = self.room_id
+        context["has_next"] = self.has_next()
         context["unread_count_lobby"] = get_unread_count(None, self.request.profile)
         if self.room:
             users_room = [self.room.user_one, self.room.user_two]

judge/utils/infinite_paginator.py

@@ -0,0 +1,152 @@
+import collections
+import inspect
+from math import ceil
+
+from django.core.paginator import EmptyPage, InvalidPage
+from django.http import Http404
+from django.utils.functional import cached_property
+from django.utils.inspect import method_has_no_args
+
+
+class InfinitePage(collections.abc.Sequence):
+    def __init__(
+        self, object_list, number, unfiltered_queryset, page_size, pad_pages, paginator
+    ):
+        self.object_list = list(object_list)
+        self.number = number
+        self.unfiltered_queryset = unfiltered_queryset
+        self.page_size = page_size
+        self.pad_pages = pad_pages
+        self.num_pages = 1e3000
+        self.paginator = paginator
+
+    def __repr__(self):
+        return "<Page %s of many>" % self.number
+
+    def __len__(self):
+        return len(self.object_list)
+
+    def __getitem__(self, index):
+        return self.object_list[index]
+
+    @cached_property
+    def _after_up_to_pad(self):
+        first_after = self.number * self.page_size
+        padding_length = self.pad_pages * self.page_size
+        queryset = self.unfiltered_queryset[
+            first_after : first_after + padding_length + 1
+        ]
+        c = getattr(queryset, "count", None)
+        if callable(c) and not inspect.isbuiltin(c) and method_has_no_args(c):
+            return c()
+        return len(queryset)
+
+    def has_next(self):
+        return self._after_up_to_pad > 0
+
+    def has_previous(self):
+        return self.number > 1
+
+    def has_other_pages(self):
+        return self.has_previous() or self.has_next()
+
+    def next_page_number(self):
+        if not self.has_next():
+            raise EmptyPage()
+        return self.number + 1
+
+    def previous_page_number(self):
+        if self.number <= 1:
+            raise EmptyPage()
+        return self.number - 1
+
+    def start_index(self):
+        return (self.page_size * (self.number - 1)) + 1
+
+    def end_index(self):
+        return self.start_index() + len(self.object_list)
+
+    @cached_property
+    def main_range(self):
+        start = max(1, self.number - self.pad_pages)
+        end = self.number + min(
+            int(ceil(self._after_up_to_pad / self.page_size)), self.pad_pages
+        )
+        return range(start, end + 1)
+
+    @cached_property
+    def leading_range(self):
+        return range(1, min(3, self.main_range[0]))
+
+    @cached_property
+    def has_trailing(self):
+        return self._after_up_to_pad > self.pad_pages * self.page_size
+
+    @cached_property
+    def page_range(self):
+        result = list(self.leading_range)
+        main_range = self.main_range
+
+        # Add ... element if there is space in between.
+        if result and result[-1] + 1 < self.main_range[0]:
+            result.append(False)
+
+        result += list(main_range)
+
+        # Add ... element if there are elements after main_range.
+        if self.has_trailing:
+            result.append(False)
+        return result
+
+
+class DummyPaginator:
+    is_infinite = True
+
+    def __init__(self, per_page):
+        self.per_page = per_page
+
+
+def infinite_paginate(queryset, page, page_size, pad_pages, paginator=None):
+    if page < 1:
+        raise EmptyPage()
+    sliced = queryset[(page - 1) * page_size : page * page_size]
+    if page > 1 and not sliced:
+        raise EmptyPage()
+    return InfinitePage(sliced, page, queryset, page_size, pad_pages, paginator)
+
+
+class InfinitePaginationMixin:
+    pad_pages = 4
+
+    @property
+    def use_infinite_pagination(self):
+        return True
+
+    def paginate_queryset(self, queryset, page_size):
+        if not self.use_infinite_pagination:
+            paginator, page, object_list, has_other = super().paginate_queryset(
+                queryset, page_size
+            )
+            paginator.is_infinite = False
+            return paginator, page, object_list, has_other
+
+        page_kwarg = self.page_kwarg
+        page = self.kwargs.get(page_kwarg) or self.request.GET.get(page_kwarg) or 1
+        try:
+            page_number = int(page)
+        except ValueError:
+            raise Http404("Page cannot be converted to an int.")
+        try:
+            paginator = DummyPaginator(page_size)
+            page = infinite_paginate(
+                queryset, page_number, page_size, self.pad_pages, paginator
+            )
+            return paginator, page, page.object_list, page.has_other_pages()
+        except InvalidPage as e:
+            raise Http404(
+                "Invalid page (%(page_number)s): %(message)s"
+                % {
+                    "page_number": page_number,
+                    "message": str(e),
+                }
+            )

judge/views/organization.py

@@ -480,7 +480,7 @@ class OrganizationSubmissions(
     def contest(self):
         return None
 
-    def _get_queryset(self):
+    def get_queryset(self):
         return (
             super()
             ._get_entire_queryset()

judge/views/ranked_submission.py

@@ -27,7 +27,7 @@ class RankedSubmissions(ProblemSubmissions):
             constraint = ""
         queryset = (
             super(RankedSubmissions, self)
-            ._get_queryset()
+            .get_queryset()
             .filter(user__is_unlisted=False)
         )
         join_sql_subquery(
@@ -76,6 +76,4 @@ class RankedSubmissions(ProblemSubmissions):
         )
 
     def _get_result_data(self):
-        return get_result_data(
+        return get_result_data(super(RankedSubmissions, self).get_queryset().order_by())
-            super(RankedSubmissions, self)._get_queryset().order_by()
-        )

judge/views/submission.py

@@ -47,13 +47,11 @@ from judge.utils.problems import user_editable_ids
 from judge.utils.problem_data import get_problem_case
 from judge.utils.raw_sql import join_sql_subquery, use_straight_join
 from judge.utils.views import DiggPaginatorMixin
+from judge.utils.infinite_paginator import InfinitePaginationMixin
 from judge.utils.views import TitleMixin
 from judge.utils.timedelta import nice_repr
 
 
-MAX_NUMBER_OF_QUERY_SUBMISSIONS = 50000
-
-
 def submission_related(queryset):
     return queryset.select_related("user__user", "problem", "language").only(
         "id",
@@ -333,7 +331,7 @@ class SubmissionsListBase(DiggPaginatorMixin, TitleMixin, ListView):
         return result
 
     def _get_result_data(self):
-        return get_result_data(self._get_queryset().order_by())
+        return get_result_data(self.get_queryset().order_by())
 
     def access_check(self, request):
         pass
@@ -412,7 +410,7 @@ class SubmissionsListBase(DiggPaginatorMixin, TitleMixin, ListView):
 
         return queryset
 
-    def _get_queryset(self):
+    def get_queryset(self):
         queryset = self._get_entire_queryset()
         if not self.in_contest:
             if self.request.organization:
@@ -434,9 +432,6 @@ class SubmissionsListBase(DiggPaginatorMixin, TitleMixin, ListView):
             )
         return queryset
 
-    def get_queryset(self):
-        return self._get_queryset()[:MAX_NUMBER_OF_QUERY_SUBMISSIONS]
-
     def get_my_submissions_page(self):
         return None
 
@@ -578,10 +573,10 @@ class GeneralSubmissions(SubmissionsListBase):
 
 
 class AllUserSubmissions(ConditionalUserTabMixin, UserMixin, GeneralSubmissions):
-    def _get_queryset(self):
+    def get_queryset(self):
         return (
             super(AllUserSubmissions, self)
-            ._get_queryset()
+            .get_queryset()
             .filter(user_id=self.profile.id)
         )
 
@@ -608,12 +603,10 @@ class AllUserSubmissions(ConditionalUserTabMixin, UserMixin, GeneralSubmissions)
 
 
 class AllFriendSubmissions(LoginRequiredMixin, GeneralSubmissions):
-    def _get_queryset(self):
+    def get_queryset(self):
         friends = self.request.profile.get_friends()
         return (
-            super(AllFriendSubmissions, self)
+            super(AllFriendSubmissions, self).get_queryset().filter(user_id__in=friends)
-            ._get_queryset()
-            .filter(user_id__in=friends)
         )
 
     def get_title(self):
@@ -631,7 +624,7 @@ class ProblemSubmissionsBase(SubmissionsListBase):
     dynamic_update = True
     check_contest_in_access_check = False
 
-    def _get_queryset(self):
+    def get_queryset(self):
         if (
             self.in_contest
             and not self.contest.contest_problems.filter(
@@ -723,10 +716,10 @@ class UserProblemSubmissions(ConditionalUserTabMixin, UserMixin, ProblemSubmissi
         if not self.is_own:
             self.access_check_contest(request)
 
-    def _get_queryset(self):
+    def get_queryset(self):
         return (
             super(UserProblemSubmissions, self)
-            ._get_queryset()
+            .get_queryset()
             .filter(user_id=self.profile.id)
         )
 
@@ -804,9 +797,13 @@ def single_submission_query(request):
     return single_submission(request, int(request.GET["id"]), bool(show_problem))
 
 
-class AllSubmissions(GeneralSubmissions):
+class AllSubmissions(InfinitePaginationMixin, GeneralSubmissions):
     stats_update_interval = 3600
 
+    @property
+    def use_infinite_pagination(self):
+        return not self.in_contest
+
     def get_context_data(self, **kwargs):
         context = super(AllSubmissions, self).get_context_data(**kwargs)
         context["dynamic_update"] = (

judge/views/user.py

@@ -44,12 +44,12 @@ from judge.utils.problems import contest_completed_ids, user_completed_ids
 from judge.utils.ranker import ranker
 from judge.utils.unicode import utf8text
 from judge.utils.views import (
-    DiggPaginatorMixin,
     QueryStringSortMixin,
     TitleMixin,
     generic_message,
     SingleObjectFormView,
 )
+from judge.utils.infinite_paginator import InfinitePaginationMixin
 from .contests import ContestRanking
 
 __all__ = [
@@ -437,7 +437,7 @@ def edit_profile(request):
     )
 
 
-class UserList(QueryStringSortMixin, DiggPaginatorMixin, TitleMixin, ListView):
+class UserList(QueryStringSortMixin, InfinitePaginationMixin, TitleMixin, ListView):
     model = Profile
     title = gettext_lazy("Leaderboard")
     context_object_name = "users"

templates/chat/chat.html

@@ -23,25 +23,23 @@
     ];
   </script>
   <script type="text/javascript">
-    window.currentPage = 1;
     window.limit_time = 24;
-    window.messages_per_page = 50;
     window.room_id = "{{room if room else ''}}";
     window.unread_message = 0;
     window.other_user_id = "{{other_user.id if other_user else ''}}";
-    window.num_pages = {{paginator.num_pages}};
     window.lock = false;
     window.lock_click_space = false;
     window.pushed_messages = new Set();
     let isMobile = window.matchMedia("only screen and (max-width: 799px)").matches;
 
-    function load_page(page, refresh_html=false) {
+    function load_next_page(last_id, refresh_html=false) {
       var param = {
-        'page': page,
+        'last_id': last_id,
+        'only_messages': true,
       }
       $.get("{{ url('chat', '') }}" + window.room_id, param)
         .fail(function() {
-        console.log("Fail to load page " + page);
+        console.log("Fail to load page, last_id = " + last_id);
       })
         .done(function(data) {
         if (refresh_html) {
@@ -49,11 +47,10 @@
           $('#chat-box').scrollTop($('#chat-box')[0].scrollHeight);
           window.lock = true;
         }
-
+        var time = refresh_html ? 0 : 200;
-        window.num_pages = parseInt($('<div>' + data + '</div>').find('#num_pages').html());
-        var time = refresh_html ? 0 : 500;
 
         setTimeout(function() {
+          $(".has_next").remove();
           let $chat_box = $('#chat-box');
           let lastMsgPos = scrollTopOfBottom($chat_box)
 
@@ -66,9 +63,6 @@
             $('#chat-log').prepend(data);
           }
 
-          $('.body-block').slice(0, window.messages_per_page).each(function() {
-          });
-
           register_time($('.time-with-rel'));
           merge_authors();
 
@@ -79,6 +73,7 @@
             $('#chat-box').scrollTop($('#chat-box')[0].scrollHeight);
           }
           window.lock = false;
+          window.has_next = parseInt($(".has_next").attr("value"));
         }, time);
       })
     }
@@ -90,10 +85,12 @@
     function scrollContainer(container, loader) {
       container.scroll(function() {
         if (container.scrollTop() == 0) {
-          if (currentPage < window.num_pages && !window.lock) {
+          if (!window.lock && window.has_next) {
-            currentPage++;
             loader.show();
-            load_page(currentPage);
+            var message_ids = $('.message').map(function() {
+              return parseInt($(this).attr('message-id'));
+            }).get();
+            load_next_page(Math.min(...message_ids));
           }
         }
       })}
@@ -323,7 +320,7 @@
 
       function callback() {
         history.replaceState(null, '', "{{url('chat', '')}}" + window.room_id);
-        load_page(window.currentPage, true, refresh_status);
+        load_next_page(null, true);
         update_last_seen();
         refresh_status();
         $('#chat-input').focus();
@@ -332,7 +329,6 @@
       if (encrypted_user) {
         $.get("{{url('get_or_create_room')}}" + `?other=${encrypted_user}`)
           .done(function(data) {
-          window.currentPage = 1;
           window.room_id = data.room;
           window.other_user_id = data.other_user_id;
           callback();
@@ -342,7 +338,6 @@
         })
       }
       else {
-        window.currentPage = 1;
         window.room_id = '';
         window.other_user_id = '';
         callback();
@@ -408,6 +403,7 @@
     $(function() {
       $('#loader').hide();
       merge_authors();
+      window.has_next = parseInt($(".has_next").attr("value"));
 
       scrollContainer($('#chat-box'), $('#loader'))
 

templates/chat/message.html

@@ -1,4 +1,4 @@
-<li class="message" id="message-{{ message.id }}">
+<li class="message" id="message-{{ message.id }}" message-id="{{ message.id }}">
   <a href="{{ url('user_page', message.author.user.username) }}">
     <img src="{{ gravatar(message.author, 135) }}" class="profile-pic">
   </a>

templates/chat/message_list.html

@@ -1,3 +1,4 @@
+<div class="has_next" style="display: none;" value="{{1 if has_next else 0}}"></div>
 {% if object_list %}
   <div style="display: none" id="num_pages">{{num_pages}}</div>
   {% for message in object_list | reverse%}