From 8ac611f9f4a9e1106bdf5e50bb107a092e2e9b18 Mon Sep 17 00:00:00 2001 From: cuom1999 Date: Tue, 1 Jun 2021 15:34:42 -0500 Subject: [PATCH] Add delete cache to testcases --- judge/models/problem_data.py | 15 +++++++-- judge/utils/problem_data.py | 63 ++++++++++++++++++++++++++++++++++-- judge/utils/problems.py | 60 +--------------------------------- judge/views/submission.py | 3 +- requirements.txt | 1 - 5 files changed, 76 insertions(+), 66 deletions(-) diff --git a/judge/models/problem_data.py b/judge/models/problem_data.py index da443e5..fe79c49 100644 --- a/judge/models/problem_data.py +++ b/judge/models/problem_data.py @@ -1,11 +1,13 @@ import errno import os +from zipfile import BadZipFile, ZipFile from django.core.validators import FileExtensionValidator +from django.core.cache import cache from django.db import models from django.utils.translation import gettext_lazy as _ -from judge.utils.problem_data import ProblemDataStorage +from judge.utils.problem_data import ProblemDataStorage, get_file_cachekey __all__ = ['problem_data_storage', 'problem_directory_file', 'ProblemData', 'ProblemTestCase', 'CHECKERS'] @@ -66,7 +68,16 @@ class ProblemData(models.Model): self.__original_zipfile = self.zipfile def save(self, *args, **kwargs): - if self.zipfile != self.__original_zipfile: + if self.zipfile != self.__original_zipfile and self.__original_zipfile: + # Delete caches + try: + files = ZipFile(self.__original_zipfile.path).namelist() + for file in files: + cache_key = 'problem_archive:%s:%s' % (self.problem.code, get_file_cachekey(file)) + cache.delete(cache_key) + except BadZipFile: + pass + self.__original_zipfile.delete(save=False) return super(ProblemData, self).save(*args, **kwargs) diff --git a/judge/utils/problem_data.py b/judge/utils/problem_data.py index 5bf6341..da26e60 100644 --- a/judge/utils/problem_data.py +++ b/judge/utils/problem_data.py @@ -2,14 +2,15 @@ import json import os import re import shutil - import yaml +import zipfile + from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import FileSystemStorage from django.urls import reverse from django.utils.translation import gettext as _ - +from django.core.cache import cache VALIDATOR_TEMPLATE_PATH = 'validator_template/template.py' @@ -232,3 +233,61 @@ class ProblemDataCompiler(object): def generate(cls, *args, **kwargs): self = cls(*args, **kwargs) self.compile() + + +def get_visible_content(data): + data = data or b'' + data = data.replace(b'\r\n', b'\r').replace(b'\r', b'\n') + + data = data.decode('utf-8') + + if (len(data) > settings.TESTCASE_VISIBLE_LENGTH): + data = data[:settings.TESTCASE_VISIBLE_LENGTH] + data += '.' * 3 + return data + + +def get_file_cachekey(file): + return file.replace(' ', '===') + +def get_problem_case(problem, files): + result = {} + uncached_files = [] + + for file in files: + cache_key = 'problem_archive:%s:%s' % (problem.code, get_file_cachekey(file)) + qs = cache.get(cache_key) + if qs is None: + uncached_files.append(file) + else: + result[file] = qs + + if not uncached_files: + return result + + archive_path = os.path.join(settings.DMOJ_PROBLEM_DATA_ROOT, + str(problem.data_files.zipfile)) + if not os.path.exists(archive_path): + raise Exception( + 'archive file "%s" does not exist' % archive_path) + try: + archive = zipfile.ZipFile(archive_path, 'r') + except zipfile.BadZipfile: + raise Exception('bad archive: "%s"' % archive_path) + + for file in uncached_files: + cache_key = 'problem_archive:%s:%s' % (problem.code, get_file_cachekey(file)) + with archive.open(file) as f: + s = f.read(settings.TESTCASE_VISIBLE_LENGTH + 3) + # add this so there are no characters left behind (ex, 'á' = 2 utf-8 chars) + while True: + try: + s.decode('utf-8') + break + except UnicodeDecodeError: + s += f.read(1) + qs = get_visible_content(s) + cache.set(cache_key, qs, 86400) + result[file] = qs + + return result diff --git a/judge/utils/problems.py b/judge/utils/problems.py index f2196b2..ba846e6 100644 --- a/judge/utils/problems.py +++ b/judge/utils/problems.py @@ -146,62 +146,4 @@ def hot_problems(duration, limit): )).order_by('-ordering').defer('description')[:limit] cache.set(cache_key, qs, 900) - return qs - - -def get_visible_content(data): - data = data or b'' - data = data.replace(b'\r\n', b'\r').replace(b'\r', b'\n') - - data = data.decode('utf-8') - - if (len(data) > settings.TESTCASE_VISIBLE_LENGTH): - data = data[:settings.TESTCASE_VISIBLE_LENGTH] - data += '.' * 3 - return data - - -def get_cachekey_file(file): - return file.replace(' ', '===') - -def get_problem_case(problem, files): - result = {} - uncached_files = [] - - for file in files: - cache_key = 'problem_archive:%s:%s' % (problem.code, get_cachekey_file(file)) - qs = cache.get(cache_key) - if qs is None: - uncached_files.append(file) - else: - result['file'] = qs - - if not uncached_files: - return result - - archive_path = os.path.join(settings.DMOJ_PROBLEM_DATA_ROOT, - str(problem.data_files.zipfile)) - if not os.path.exists(archive_path): - raise Exception( - 'archive file "%s" does not exist' % archive_path) - try: - archive = zipfile.ZipFile(archive_path, 'r') - except zipfile.BadZipfile: - raise Exception('bad archive: "%s"' % archive_path) - - for file in uncached_files: - cache_key = 'problem_archive:%s:%s' % (problem.code, get_cachekey_file(file)) - with archive.open(file) as f: - s = f.read(settings.TESTCASE_VISIBLE_LENGTH + 3) - # add this so there are no characters left behind (ex, 'á' = 2 utf-8 chars) - while True: - try: - s.decode('utf-8') - break - except UnicodeDecodeError: - s += f.read(1) - qs = get_visible_content(s) - cache.set(cache_key, qs, 86400) - result[file] = qs - - return result + return qs \ No newline at end of file diff --git a/judge/views/submission.py b/judge/views/submission.py index faa4d60..06654c0 100644 --- a/judge/views/submission.py +++ b/judge/views/submission.py @@ -43,7 +43,7 @@ from judge.utils.problems import get_result_data from judge.utils.problems import user_authored_ids from judge.utils.problems import user_completed_ids from judge.utils.problems import user_editable_ids -from judge.utils.problems import get_problem_case +from judge.utils.problem_data import get_problem_case from judge.utils.raw_sql import join_sql_subquery, use_straight_join from judge.utils.views import DiggPaginatorMixin from judge.utils.views import TitleMixin @@ -172,7 +172,6 @@ class SubmissionStatus(SubmissionDetailBase): submission = self.object context['last_msg'] = event.last() context['batches'] = group_test_cases(submission.test_cases.all()) - print(context['batches']) context['time_limit'] = submission.problem.time_limit contest = submission.contest_or_none diff --git a/requirements.txt b/requirements.txt index 869deb8..e718308 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,6 @@ channels==2.4.0 channels-redis==2.4.2 docker django-newsletter -python-memcached netaddr redis lupa \ No newline at end of file