From 2854ac97e930ed112d5ac126409b4a17db032714 Mon Sep 17 00:00:00 2001 From: cuom1999 Date: Mon, 28 Aug 2023 14:20:35 -0500 Subject: [PATCH] Make chat faster --- .../migrations/0013_alter_message_time.py | 20 + .../migrations/0014_userroom_unread_count.py | 38 ++ chat_box/models.py | 26 +- chat_box/utils.py | 35 +- chat_box/views.py | 176 +++--- dmoj/settings.py | 3 + judge/migrations/0166_display_rank_index.py | 28 + judge/models/profile.py | 1 + templates/chat/chat.html | 562 +----------------- templates/chat/chat_css.html | 262 ++++---- templates/chat/chat_js.html | 553 +++++++++++++++++ 11 files changed, 903 insertions(+), 801 deletions(-) create mode 100644 chat_box/migrations/0013_alter_message_time.py create mode 100644 chat_box/migrations/0014_userroom_unread_count.py create mode 100644 judge/migrations/0166_display_rank_index.py create mode 100644 templates/chat/chat_js.html diff --git a/chat_box/migrations/0013_alter_message_time.py b/chat_box/migrations/0013_alter_message_time.py new file mode 100644 index 0000000..0f4ddc3 --- /dev/null +++ b/chat_box/migrations/0013_alter_message_time.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.18 on 2023-08-28 01:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("chat_box", "0012_auto_20230308_1417"), + ] + + operations = [ + migrations.AlterField( + model_name="message", + name="time", + field=models.DateTimeField( + auto_now_add=True, db_index=True, verbose_name="posted time" + ), + ), + ] diff --git a/chat_box/migrations/0014_userroom_unread_count.py b/chat_box/migrations/0014_userroom_unread_count.py new file mode 100644 index 0000000..8f407b3 --- /dev/null +++ b/chat_box/migrations/0014_userroom_unread_count.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.18 on 2023-08-28 06:02 + +from django.db import migrations, models + + +def migrate(apps, schema_editor): + UserRoom = apps.get_model("chat_box", "UserRoom") + Message = apps.get_model("chat_box", "Message") + + for ur in UserRoom.objects.all(): + if not ur.room: + continue + messages = ur.room.message_set + last_msg = messages.first() + try: + if last_msg and last_msg.author != ur.user: + ur.unread_count = messages.filter(time__gte=ur.last_seen).count() + else: + ur.unread_count = 0 + ur.save() + except: + continue + + +class Migration(migrations.Migration): + + dependencies = [ + ("chat_box", "0013_alter_message_time"), + ] + + operations = [ + migrations.AddField( + model_name="userroom", + name="unread_count", + field=models.IntegerField(db_index=True, default=0), + ), + migrations.RunPython(migrate, migrations.RunPython.noop, atomic=True), + ] diff --git a/chat_box/models.py b/chat_box/models.py index 61e39a6..b6d7220 100644 --- a/chat_box/models.py +++ b/chat_box/models.py @@ -1,9 +1,10 @@ from django.db import models -from django.db.models import CASCADE +from django.db.models import CASCADE, Q from django.utils.translation import gettext_lazy as _ from judge.models.profile import Profile +from judge.caching import cache_wrapper __all__ = ["Message", "Room", "UserRoom", "Ignore"] @@ -29,7 +30,9 @@ class Room(models.Model): class Message(models.Model): author = models.ForeignKey(Profile, verbose_name=_("user"), on_delete=CASCADE) - time = models.DateTimeField(verbose_name=_("posted time"), auto_now_add=True) + time = models.DateTimeField( + verbose_name=_("posted time"), auto_now_add=True, db_index=True + ) body = models.TextField(verbose_name=_("body of comment"), max_length=8192) hidden = models.BooleanField(verbose_name="is hidden", default=False) room = models.ForeignKey( @@ -56,6 +59,7 @@ class UserRoom(models.Model): Room, verbose_name="room id", on_delete=CASCADE, default=None, null=True ) last_seen = models.DateTimeField(verbose_name=_("last seen"), auto_now_add=True) + unread_count = models.IntegerField(default=0, db_index=True) class Meta: unique_together = ("user", "room") @@ -74,11 +78,9 @@ class Ignore(models.Model): @classmethod def is_ignored(self, current_user, new_friend): try: - return ( - current_user.ignored_chat_users.get() - .ignored_users.filter(id=new_friend.id) - .exists() - ) + return current_user.ignored_chat_users.ignored_users.filter( + id=new_friend.id + ).exists() except: return False @@ -89,6 +91,16 @@ class Ignore(models.Model): except: return Profile.objects.none() + @classmethod + def get_ignored_rooms(self, user): + try: + ignored_users = self.objects.get(user=user).ignored_users.all() + return Room.objects.filter(Q(user_one=user) | Q(user_two=user)).filter( + Q(user_one__in=ignored_users) | Q(user_two__in=ignored_users) + ) + except: + return Room.objects.none() + @classmethod def add_ignore(self, current_user, friend): ignore, created = self.objects.get_or_create(user=current_user) diff --git a/chat_box/utils.py b/chat_box/utils.py index e25e861..dd59d98 100644 --- a/chat_box/utils.py +++ b/chat_box/utils.py @@ -1,10 +1,12 @@ from cryptography.fernet import Fernet +import hmac +import hashlib from django.conf import settings -from django.db.models import OuterRef, Count, Subquery, IntegerField +from django.db.models import OuterRef, Count, Subquery, IntegerField, Q from django.db.models.functions import Coalesce -from chat_box.models import Ignore, Message, UserRoom +from chat_box.models import Ignore, Message, UserRoom, Room secret_key = settings.CHAT_SECRET_KEY fernet = Fernet(secret_key) @@ -24,25 +26,22 @@ def decrypt_url(message_encrypted): return None, None -def get_unread_boxes(profile): - ignored_users = Ignore.get_ignored_users(profile) - - mess = ( - Message.objects.filter(room=OuterRef("room"), time__gte=OuterRef("last_seen")) - .exclude(author=profile) - .exclude(author__in=ignored_users) - .order_by() - .values("room") - .annotate(unread_count=Count("pk")) - .values("unread_count") +def encrypt_channel(channel): + return ( + hmac.new( + settings.CHAT_SECRET_KEY.encode(), + channel.encode(), + hashlib.sha512, + ).hexdigest()[:16] + + "%s" % channel ) + +def get_unread_boxes(profile): + ignored_rooms = Ignore.get_ignored_rooms(profile) unread_boxes = ( - UserRoom.objects.filter(user=profile, room__isnull=False) - .annotate( - unread_count=Coalesce(Subquery(mess, output_field=IntegerField()), 0), - ) - .filter(unread_count__gte=1) + UserRoom.objects.filter(user=profile, unread_count__gt=0) + .exclude(room__in=ignored_rooms) .count() ) diff --git a/chat_box/views.py b/chat_box/views.py index cea58e1..879dfcf 100644 --- a/chat_box/views.py +++ b/chat_box/views.py @@ -21,6 +21,7 @@ from django.db.models import ( Exists, Count, IntegerField, + F, ) from django.db.models.functions import Coalesce from django.utils import timezone @@ -34,7 +35,7 @@ from judge.jinja2.gravatar import gravatar from judge.models import Friend from chat_box.models import Message, Profile, Room, UserRoom, Ignore -from chat_box.utils import encrypt_url, decrypt_url +from chat_box.utils import encrypt_url, decrypt_url, encrypt_channel import json @@ -49,7 +50,8 @@ class ChatView(ListView): self.room_id = None self.room = None self.messages = None - self.page_size = 20 + self.first_page_size = 20 # only for first request + self.follow_up_page_size = 50 def get_queryset(self): return self.messages @@ -63,10 +65,12 @@ class ChatView(ListView): def get(self, request, *args, **kwargs): request_room = kwargs["room_id"] + page_size = self.follow_up_page_size try: last_id = int(request.GET.get("last_id")) except Exception: last_id = 1e15 + page_size = self.first_page_size only_messages = request.GET.get("only_messages") if request_room: @@ -80,11 +84,12 @@ class ChatView(ListView): request_room = None self.room_id = request_room - self.messages = Message.objects.filter( - hidden=False, room=self.room_id, id__lt=last_id - )[: self.page_size] + self.messages = ( + Message.objects.filter(hidden=False, room=self.room_id, id__lt=last_id) + .select_related("author", "author__user") + .defer("author__about", "author__user_script")[:page_size] + ) if not only_messages: - update_last_seen(request, **kwargs) return super().get(request, *args, **kwargs) return render( @@ -101,10 +106,14 @@ class ChatView(ListView): context["title"] = self.title context["last_msg"] = event.last() - context["status_sections"] = get_status_context(self.request) + context["status_sections"] = get_status_context(self.request.profile) context["room"] = self.room_id context["has_next"] = self.has_next() context["unread_count_lobby"] = get_unread_count(None, self.request.profile) + context["chat_channel"] = encrypt_channel( + "chat_" + str(self.request.profile.id) + ) + context["chat_lobby_channel"] = encrypt_channel("chat_lobby") if self.room: users_room = [self.room.user_one, self.room.user_two] users_room.remove(self.request.profile) @@ -187,7 +196,7 @@ def post_message(request): if not room: event.post( - "chat_lobby", + encrypt_channel("chat_lobby"), { "type": "lobby", "author_id": request.profile.id, @@ -199,7 +208,7 @@ def post_message(request): else: for user in room.users(): event.post( - "chat_" + str(user.id), + encrypt_channel("chat_" + str(user.id)), { "type": "private", "author_id": request.profile.id, @@ -208,6 +217,10 @@ def post_message(request): "tmp_id": request.POST.get("tmp_id"), }, ) + if user != request.profile: + UserRoom.objects.filter(user=user, room=room).update( + unread_count=F("unread_count") + 1 + ) return JsonResponse(ret) @@ -254,35 +267,33 @@ def update_last_seen(request, **kwargs): room_id = request.POST.get("room") else: return HttpResponseBadRequest() - try: profile = request.profile room = None if room_id: - room = Room.objects.get(id=int(room_id)) + room = Room.objects.filter(id=int(room_id)).first() except Room.DoesNotExist: return HttpResponseBadRequest() - except Exception as e: - return HttpResponseBadRequest() if room and not room.contain(profile): return HttpResponseBadRequest() user_room, _ = UserRoom.objects.get_or_create(user=profile, room=room) user_room.last_seen = timezone.now() + user_room.unread_count = 0 user_room.save() return JsonResponse({"msg": "updated"}) def get_online_count(): - last_two_minutes = timezone.now() - timezone.timedelta(minutes=2) - return Profile.objects.filter(last_access__gte=last_two_minutes).count() + last_5_minutes = timezone.now() - timezone.timedelta(minutes=5) + return Profile.objects.filter(last_access__gte=last_5_minutes).count() def get_user_online_status(user): time_diff = timezone.now() - user.last_access - is_online = time_diff <= timezone.timedelta(minutes=2) + is_online = time_diff <= timezone.timedelta(minutes=5) return is_online @@ -319,47 +330,51 @@ def user_online_status_ajax(request): ) -def get_online_status(request_user, queryset, rooms=None): - if not queryset: +def get_online_status(profile, other_profile_ids, rooms=None): + if not other_profile_ids: return None - last_two_minutes = timezone.now() - timezone.timedelta(minutes=2) + joined_ids = ",".join([str(id) for id in other_profile_ids]) + other_profiles = Profile.objects.raw( + f"SELECT * from judge_profile where id in ({joined_ids}) order by field(id,{joined_ids})" + ) + last_5_minutes = timezone.now() - timezone.timedelta(minutes=5) ret = [] - if rooms: - unread_count = get_unread_count(rooms, request_user) + unread_count = get_unread_count(rooms, profile) count = {} for i in unread_count: count[i["other_user"]] = i["unread_count"] - - for user in queryset: + for other_profile in other_profiles: is_online = False - if user.last_access >= last_two_minutes: + if other_profile.last_access >= last_5_minutes: is_online = True - user_dict = {"user": user, "is_online": is_online} - if rooms and user.id in count: - user_dict["unread_count"] = count[user.id] - user_dict["url"] = encrypt_url(request_user.id, user.id) + user_dict = {"user": other_profile, "is_online": is_online} + if rooms and other_profile.id in count: + user_dict["unread_count"] = count[other_profile.id] + user_dict["url"] = encrypt_url(profile.id, other_profile.id) ret.append(user_dict) return ret -def get_status_context(request, include_ignored=False): +def get_status_context(profile, include_ignored=False): if include_ignored: - ignored_users = Profile.objects.none() + ignored_users = [] queryset = Profile.objects else: - ignored_users = Ignore.get_ignored_users(request.profile) + ignored_users = list( + Ignore.get_ignored_users(profile).values_list("id", flat=True) + ) queryset = Profile.objects.exclude(id__in=ignored_users) - last_two_minutes = timezone.now() - timezone.timedelta(minutes=2) + last_5_minutes = timezone.now() - timezone.timedelta(minutes=5) recent_profile = ( - Room.objects.filter(Q(user_one=request.profile) | Q(user_two=request.profile)) + Room.objects.filter(Q(user_one=profile) | Q(user_two=profile)) .annotate( last_msg_time=Subquery( Message.objects.filter(room=OuterRef("pk")).values("time")[:1] ), other_user=Case( - When(user_one=request.profile, then="user_two"), + When(user_one=profile, then="user_two"), default="user_one", ), ) @@ -369,50 +384,49 @@ def get_status_context(request, include_ignored=False): .values("other_user", "id")[:20] ) - recent_profile_id = [str(i["other_user"]) for i in recent_profile] - joined_id = ",".join(recent_profile_id) + recent_profile_ids = [str(i["other_user"]) for i in recent_profile] recent_rooms = [int(i["id"]) for i in recent_profile] - recent_list = None - if joined_id: - recent_list = Profile.objects.raw( - f"SELECT * from judge_profile where id in ({joined_id}) order by field(id,{joined_id})" - ) friend_list = ( - Friend.get_friend_profiles(request.profile) - .exclude(id__in=recent_profile_id) + Friend.get_friend_profiles(profile) + .exclude(id__in=recent_profile_ids) .exclude(id__in=ignored_users) .order_by("-last_access") + .values_list("id", flat=True) ) + admin_list = ( queryset.filter(display_rank="admin") .exclude(id__in=friend_list) - .exclude(id__in=recent_profile_id) + .exclude(id__in=recent_profile_ids) + .values_list("id", flat=True) ) + all_user_status = ( - queryset.filter(display_rank="user", last_access__gte=last_two_minutes) + queryset.filter(last_access__gte=last_5_minutes) .annotate(is_online=Case(default=True, output_field=BooleanField())) .order_by("-rating") .exclude(id__in=friend_list) .exclude(id__in=admin_list) - .exclude(id__in=recent_profile_id)[:30] + .exclude(id__in=recent_profile_ids) + .values_list("id", flat=True)[:30] ) return [ { "title": "Recent", - "user_list": get_online_status(request.profile, recent_list, recent_rooms), + "user_list": get_online_status(profile, recent_profile_ids, recent_rooms), }, { "title": "Following", - "user_list": get_online_status(request.profile, friend_list), + "user_list": get_online_status(profile, friend_list), }, { "title": "Admin", - "user_list": get_online_status(request.profile, admin_list), + "user_list": get_online_status(profile, admin_list), }, { "title": "Other", - "user_list": get_online_status(request.profile, all_user_status), + "user_list": get_online_status(profile, all_user_status), }, ] @@ -423,7 +437,7 @@ def online_status_ajax(request): request, "chat/online_status.html", { - "status_sections": get_status_context(request), + "status_sections": get_status_context(request.profile), "unread_count_lobby": get_unread_count(None, request.profile), }, ) @@ -447,7 +461,6 @@ def get_or_create_room(request): return HttpResponseBadRequest() request_id, other_id = decrypt_url(decrypted_other_id) - if not other_id or not request_id or request_id != request.profile.id: return HttpResponseBadRequest() @@ -475,48 +488,31 @@ def get_or_create_room(request): def get_unread_count(rooms, user): if rooms: - mess = ( - Message.objects.filter( - room=OuterRef("room"), time__gte=OuterRef("last_seen") - ) - .exclude(author=user) - .order_by() - .values("room") - .annotate(unread_count=Count("pk")) - .values("unread_count") - ) - - return ( - UserRoom.objects.filter(user=user, room__in=rooms) - .annotate( - unread_count=Coalesce(Subquery(mess, output_field=IntegerField()), 0), - other_user=Case( - When(room__user_one=user, then="room__user_two"), - default="room__user_one", - ), - ) - .filter(unread_count__gte=1) - .values("other_user", "unread_count") - ) - else: # lobby - mess = ( - Message.objects.filter(room__isnull=True, time__gte=OuterRef("last_seen")) - .exclude(author=user) - .order_by() - .values("room") - .annotate(unread_count=Count("pk")) - .values("unread_count") - ) - res = ( - UserRoom.objects.filter(user=user, room__isnull=True) - .annotate( - unread_count=Coalesce(Subquery(mess, output_field=IntegerField()), 0), + UserRoom.objects.filter(user=user, room__in=rooms, unread_count__gt=0) + .select_related("room__user_one", "room__user_two") + .values("unread_count", "room__user_one", "room__user_two") + ) + for ur in res: + ur["other_user"] = ( + ur["room__user_one"] + if ur["room__user_two"] == user.id + else ur["room__user_two"] ) - .values_list("unread_count", flat=True) + return res + else: # lobby + user_room = UserRoom.objects.filter(user=user, room__isnull=True).first() + if not user_room: + return 0 + last_seen = user_room.last_seen + res = ( + Message.objects.filter(room__isnull=True, time__gte=last_seen) + .exclude(author=user) + .exclude(hidden=True) + .count() ) - return res[0] if len(res) else 0 + return res @login_required diff --git a/dmoj/settings.py b/dmoj/settings.py index e205e60..d03dc27 100644 --- a/dmoj/settings.py +++ b/dmoj/settings.py @@ -484,3 +484,6 @@ except IOError: pass DEFAULT_AUTO_FIELD = "django.db.models.AutoField" + +# Chat +CHAT_SECRET_KEY = "QUdVFsxk6f5-Hd8g9BXv81xMqvIZFRqMl-KbRzztW-U=" diff --git a/judge/migrations/0166_display_rank_index.py b/judge/migrations/0166_display_rank_index.py new file mode 100644 index 0000000..ffea311 --- /dev/null +++ b/judge/migrations/0166_display_rank_index.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.18 on 2023-08-28 01:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("judge", "0165_drop_output_prefix_override"), + ] + + operations = [ + migrations.AlterField( + model_name="profile", + name="display_rank", + field=models.CharField( + choices=[ + ("user", "Normal User"), + ("setter", "Problem Setter"), + ("admin", "Admin"), + ], + db_index=True, + default="user", + max_length=10, + verbose_name="display rank", + ), + ), + ] diff --git a/judge/models/profile.py b/judge/models/profile.py index ccb0e87..3f6e430 100644 --- a/judge/models/profile.py +++ b/judge/models/profile.py @@ -183,6 +183,7 @@ class Profile(models.Model): ("setter", "Problem Setter"), ("admin", "Admin"), ), + db_index=True, ) mute = models.BooleanField( verbose_name=_("comment mute"), diff --git a/templates/chat/chat.html b/templates/chat/chat.html index 9cde791..857aaf9 100644 --- a/templates/chat/chat.html +++ b/templates/chat/chat.html @@ -7,21 +7,11 @@ - - + {% compress js %} + {% include "chat/chat_js.html" %} + {% endcompress %} + @@ -638,8 +89,7 @@ {% include 'chat/user_online_status.html' %}
- - + diff --git a/templates/chat/chat_css.html b/templates/chat/chat_css.html index f5bf8cd..d79a22b 100644 --- a/templates/chat/chat_css.html +++ b/templates/chat/chat_css.html @@ -1,132 +1,134 @@ - + + #content { + padding-top: 0; + } + + ::-webkit-scrollbar { + width: 20px; + } + + ::-webkit-scrollbar-track { + background-color: transparent; + } + + ::-webkit-scrollbar-thumb { + background-color: #d6dee1; + border-radius: 20px; + border: 6px solid transparent; + background-clip: content-box; + } + + ::-webkit-scrollbar-thumb:hover { + background-color: #a8bbbf; + } + + #page-container { + width: 100%; + } + + .body-message img{ + max-height: 12em; + } + + .tooltip:not(.shown) { + display: none; + } + + textarea { + resize: none; + } + + .tooltip { + position: absolute; + z-index: 1000; + } + + #loader { + display: block; + margin-left: auto; + margin-right: auto; + width: 80px; + } + .profile-pic { + height: 2.6em; + width: 2.6em; + border-radius: 0.3em; + margin-top: 0.1em; + float: left; + } + .body-message { + padding-left: 3em; + padding-bottom: 0.5em; + border-bottom: 1px dotted lightgray; + } + .user-time { + margin-bottom: 0.3em; + } + .time { + margin-left: 0.5em; + } + .clear { + clear: both; + } + .content-message { + word-wrap: break-word; + } + .content-message p { + margin: 0; + } + #content { + width: 100%; + } + #content-body { + padding-bottom: 0; + } + #page-container { + min-height: 0; + } + .sidebox h3 { + border-radius: 0; + } + .body-block { + border-radius: 4px; + padding: 0.05em 0.6em; + width: 100%; + } + #search-form { + float: inherit; + } + #search-container { + margin-bottom: 0.4em; + } + #setting { + position: relative; + } + + + @media (min-width: 800px) { + #page-container { + position:fixed; + overflow:hidden; + } + } + @media (max-width: 799px) { + html, body { + max-width: 100%; + overflow-x: hidden; + } + #mobile ul { + width: 100%; + } + .info-pic { + margin-left: 0.5em; + } + .active-span { + display: none; + } + } + +{% endcompress %} \ No newline at end of file diff --git a/templates/chat/chat_js.html b/templates/chat/chat_js.html new file mode 100644 index 0000000..116d949 --- /dev/null +++ b/templates/chat/chat_js.html @@ -0,0 +1,553 @@ + \ No newline at end of file