Add delete cache to testcases

judge/models/problem_data.py

@@ -1,11 +1,13 @@
 import errno
 import os
+from zipfile import BadZipFile, ZipFile
 
 from django.core.validators import FileExtensionValidator
+from django.core.cache import cache
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from judge.utils.problem_data import ProblemDataStorage
+from judge.utils.problem_data import ProblemDataStorage, get_file_cachekey
 
 __all__ = ['problem_data_storage', 'problem_directory_file', 'ProblemData', 'ProblemTestCase', 'CHECKERS']
 
@@ -66,7 +68,16 @@ class ProblemData(models.Model):
         self.__original_zipfile = self.zipfile
 
     def save(self, *args, **kwargs):
-        if self.zipfile != self.__original_zipfile:
+        if self.zipfile != self.__original_zipfile and self.__original_zipfile:
+            # Delete caches
+            try: 
+                files = ZipFile(self.__original_zipfile.path).namelist()
+                for file in files:
+                    cache_key = 'problem_archive:%s:%s' % (self.problem.code, get_file_cachekey(file))
+                    cache.delete(cache_key)
+            except BadZipFile:
+                pass
+
             self.__original_zipfile.delete(save=False)
         return super(ProblemData, self).save(*args, **kwargs)
 

judge/utils/problem_data.py

@@ -2,14 +2,15 @@ import json
 import os
 import re
 import shutil
-
 import yaml
+import zipfile
+
 from django.conf import settings
 from django.core.files.base import ContentFile
 from django.core.files.storage import FileSystemStorage
 from django.urls import reverse
 from django.utils.translation import gettext as _
-
+from django.core.cache import cache
 
 VALIDATOR_TEMPLATE_PATH = 'validator_template/template.py'
 
@@ -232,3 +233,61 @@ class ProblemDataCompiler(object):
     def generate(cls, *args, **kwargs):
         self = cls(*args, **kwargs)
         self.compile()
+
+
+def get_visible_content(data):
+    data = data or b''
+    data = data.replace(b'\r\n', b'\r').replace(b'\r', b'\n')
+
+    data = data.decode('utf-8')
+
+    if (len(data) > settings.TESTCASE_VISIBLE_LENGTH):
+        data = data[:settings.TESTCASE_VISIBLE_LENGTH]
+        data += '.' * 3
+    return data
+
+
+def get_file_cachekey(file):
+    return file.replace(' ', '===')
+
+def get_problem_case(problem, files):
+    result = {}
+    uncached_files = []
+
+    for file in files:
+        cache_key = 'problem_archive:%s:%s' % (problem.code, get_file_cachekey(file))
+        qs = cache.get(cache_key)
+        if qs is None:
+            uncached_files.append(file)
+        else:
+            result[file] = qs
+
+    if not uncached_files:
+        return result
+
+    archive_path = os.path.join(settings.DMOJ_PROBLEM_DATA_ROOT,
+                                    str(problem.data_files.zipfile))
+    if not os.path.exists(archive_path):
+        raise Exception(
+            'archive file "%s" does not exist' % archive_path)
+    try:
+        archive = zipfile.ZipFile(archive_path, 'r')
+    except zipfile.BadZipfile:
+        raise Exception('bad archive: "%s"' % archive_path)
+
+    for file in uncached_files:
+        cache_key = 'problem_archive:%s:%s' % (problem.code, get_file_cachekey(file))
+        with archive.open(file) as f:
+            s = f.read(settings.TESTCASE_VISIBLE_LENGTH + 3)
+            # add this so there are no characters left behind (ex, 'á' = 2 utf-8 chars)
+            while True:
+                try:
+                    s.decode('utf-8')
+                    break
+                except UnicodeDecodeError:
+                    s += f.read(1)
+            qs = get_visible_content(s)
+        cache.set(cache_key, qs, 86400)
+        result[file] = qs
+
+    return result

judge/utils/problems.py

@@ -146,62 +146,4 @@ def hot_problems(duration, limit):
         )).order_by('-ordering').defer('description')[:limit]
 
         cache.set(cache_key, qs, 900)
-    return qs
-
-
-def get_visible_content(data):
-    data = data or b''
-    data = data.replace(b'\r\n', b'\r').replace(b'\r', b'\n')
-
-    data = data.decode('utf-8')
-
-    if (len(data) > settings.TESTCASE_VISIBLE_LENGTH):
-        data = data[:settings.TESTCASE_VISIBLE_LENGTH]
-        data += '.' * 3
-    return data
-
-
-def get_cachekey_file(file):
-    return file.replace(' ', '===')
-
-def get_problem_case(problem, files):
-    result = {}
-    uncached_files = []
-
-    for file in files:
-        cache_key = 'problem_archive:%s:%s' % (problem.code, get_cachekey_file(file))
-        qs = cache.get(cache_key)
-        if qs is None:
-            uncached_files.append(file)
-        else:
-            result['file'] = qs
-
-    if not uncached_files:
-        return result
-
-    archive_path = os.path.join(settings.DMOJ_PROBLEM_DATA_ROOT,
-                                    str(problem.data_files.zipfile))
-    if not os.path.exists(archive_path):
-        raise Exception(
-            'archive file "%s" does not exist' % archive_path)
-    try:
-        archive = zipfile.ZipFile(archive_path, 'r')
-    except zipfile.BadZipfile:
-        raise Exception('bad archive: "%s"' % archive_path)
-
-    for file in uncached_files:
-        cache_key = 'problem_archive:%s:%s' % (problem.code, get_cachekey_file(file))
-        with archive.open(file) as f:
-            s = f.read(settings.TESTCASE_VISIBLE_LENGTH + 3)
-            # add this so there are no characters left behind (ex, 'á' = 2 utf-8 chars)
-            while True:
-                try:
-                    s.decode('utf-8')
-                    break
-                except UnicodeDecodeError:
-                    s += f.read(1)
-            qs = get_visible_content(s)
-        cache.set(cache_key, qs, 86400)
-        result[file] = qs
-
-    return result
+    return qs

judge/views/submission.py

@@ -43,7 +43,7 @@ from judge.utils.problems import get_result_data
 from judge.utils.problems import user_authored_ids
 from judge.utils.problems import user_completed_ids
 from judge.utils.problems import user_editable_ids
-from judge.utils.problems import get_problem_case
+from judge.utils.problem_data import get_problem_case
 from judge.utils.raw_sql import join_sql_subquery, use_straight_join
 from judge.utils.views import DiggPaginatorMixin
 from judge.utils.views import TitleMixin
@@ -172,7 +172,6 @@ class SubmissionStatus(SubmissionDetailBase):
         submission = self.object
         context['last_msg'] = event.last()
         context['batches'] = group_test_cases(submission.test_cases.all())
-        print(context['batches'])
         context['time_limit'] = submission.problem.time_limit
 
         contest = submission.contest_or_none

requirements.txt

@@ -34,7 +34,6 @@ channels==2.4.0
 channels-redis==2.4.2
 docker
 django-newsletter
-python-memcached
 netaddr
 redis
 lupa