Implement chunk uploading

judge/utils/fine_uploader.py

@@ -0,0 +1,87 @@
+# https://github.com/FineUploader/server-examples/blob/master/python/django-fine-uploader
+
+from django.conf import settings
+from django import forms
+from django.forms import ClearableFileInput
+
+import os, os.path
+import tempfile
+import shutil
+
+__all__ = (
+    'handle_upload', 'save_upload', 'FineUploadForm', 'FineUploadFileInput'
+)
+
+
+def combine_chunks(total_parts, total_size, source_folder, dest):
+    if not os.path.exists(os.path.dirname(dest)):
+        os.makedirs(os.path.dirname(dest))
+
+    with open(dest, 'wb+') as destination:
+        for i in range(total_parts):
+            part = os.path.join(source_folder, str(i))
+            with open(part, 'rb') as source:
+                destination.write(source.read())
+
+
+def save_upload(f, path):
+    if not os.path.exists(os.path.dirname(path)):
+        os.makedirs(os.path.dirname(path))
+    with open(path, 'wb+') as destination:
+        if hasattr(f, 'multiple_chunks') and f.multiple_chunks():
+            for chunk in f.chunks():
+                destination.write(chunk)
+        else:
+            destination.write(f.read())
+
+
+# pass callback function to post_upload
+def handle_upload(f, fileattrs, upload_dir, post_upload=None):
+    chunks_dir = os.path.join(tempfile.gettempdir(), 'chunk_upload_tmp')
+    if not os.path.exists(os.path.dirname(chunks_dir)):
+        os.makedirs(os.path.dirname(chunks_dir))
+    chunked = False
+    dest_folder = upload_dir
+    dest = os.path.join(dest_folder, fileattrs['qqfilename'])
+
+    # Chunked
+    if fileattrs.get('qqtotalparts') and int(fileattrs['qqtotalparts']) > 1:
+        chunked = True
+        dest_folder = os.path.join(chunks_dir, fileattrs['qquuid'])
+        dest = os.path.join(dest_folder, fileattrs['qqfilename'], str(fileattrs['qqpartindex']))
+    save_upload(f, dest)
+
+    # If the last chunk has been sent, combine the parts.
+    if chunked and (fileattrs['qqtotalparts'] - 1 == fileattrs['qqpartindex']):
+        combine_chunks(fileattrs['qqtotalparts'],
+            fileattrs['qqtotalfilesize'],
+            source_folder=os.path.dirname(dest),
+            dest=os.path.join(upload_dir, fileattrs['qqfilename']))
+        shutil.rmtree(os.path.dirname(os.path.dirname(dest)))
+    
+    if post_upload and (not chunked or fileattrs['qqtotalparts'] - 1 == fileattrs['qqpartindex']):
+        post_upload()
+
+
+class FineUploadForm(forms.Form):
+    qqfile = forms.FileField()
+    qquuid = forms.CharField()
+    qqfilename = forms.CharField()
+    qqpartindex = forms.IntegerField(required=False)
+    qqchunksize = forms.IntegerField(required=False)
+    qqpartbyteoffset = forms.IntegerField(required=False)
+    qqtotalfilesize = forms.IntegerField(required=False)
+    qqtotalparts = forms.IntegerField(required=False)
+
+
+class FineUploadFileInput(ClearableFileInput):
+    template_name = 'widgets/fine_uploader.html'
+    def fine_uploader_id(self, name):
+        return name + '_' + 'fine_uploader'
+
+    def get_context(self, name, value, attrs):
+        context = super().get_context(name, value, attrs)
+        context['widget'].update({
+            'fine_uploader_id': self.fine_uploader_id(name),
+        })
+        return context

judge/utils/problem_data.py

@@ -291,4 +291,4 @@ def get_problem_case(problem, files):
         cache.set(cache_key, qs, 86400)
         result[file] = qs
 
-    return result
+    return result

judge/views/problem_data.py

@@ -2,14 +2,24 @@ import json
 import mimetypes
 import os
 from itertools import chain
+import shutil
+from tempfile import gettempdir
 from zipfile import BadZipfile, ZipFile
 
+from django import forms
+from django.conf import settings
+from django.http import HttpResponse, HttpRequest
+from django.shortcuts import render
+from django.views.decorators.csrf import csrf_exempt
+from django.views.generic import View
+
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.mixins import LoginRequiredMixin
+from django.core.files import File
 from django.core.exceptions import ValidationError
 from django.forms import BaseModelFormSet, HiddenInput, ModelForm, NumberInput, Select, formset_factory, FileInput
-from django.http import Http404, HttpResponse, HttpResponseRedirect
+from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse
 from django.shortcuts import get_object_or_404, render
 from django.urls import reverse
 from django.utils.html import escape, format_html
@@ -22,6 +32,7 @@ from judge.models import Problem, ProblemData, ProblemTestCase, Submission, prob
 from judge.utils.problem_data import ProblemDataCompiler
 from judge.utils.unicode import utf8text
 from judge.utils.views import TitleMixin
+from judge.utils.fine_uploader import combine_chunks, save_upload, handle_upload, FineUploadFileInput, FineUploadForm
 from judge.views.problem import ProblemMixin
 
 mimetypes.init()
@@ -52,6 +63,7 @@ class ProblemDataForm(ModelForm):
         model = ProblemData
         fields = ['zipfile', 'checker', 'checker_args', 'custom_checker', 'custom_validator']
         widgets = {
+            'zipfile': FineUploadFileInput,
             'checker_args': HiddenInput,
             'generator': HiddenInput,
             'output_limit': HiddenInput,
@@ -76,6 +88,7 @@ class ProblemCaseForm(ModelForm):
         }
 
 
+
 class ProblemCaseFormSet(formset_factory(ProblemCaseForm, formset=BaseModelFormSet, extra=1, max_num=1,
                                          can_delete=True)):
     model = ProblemTestCase
@@ -242,3 +255,39 @@ def problem_init_view(request, problem):
             format_html('<a href="{1}">{0}</a>', problem.name,
                         reverse('problem_detail', args=[problem.code])))),
     })
+
+
+class ProblemZipUploadView(ProblemManagerMixin, View):
+    def dispatch(self, *args, **kwargs):
+        return super(ProblemZipUploadView, self).dispatch(*args, **kwargs)
+    
+    def post(self, request, *args, **kwargs):
+        self.object = problem = self.get_object()
+        problem_data = get_object_or_404(ProblemData, problem=self.object)
+        form = FineUploadForm(request.POST, request.FILES)
+        
+        if form.is_valid():
+            fileuid = form.cleaned_data['qquuid']
+            filename = form.cleaned_data['qqfilename']
+            dest = os.path.join(gettempdir(), fileuid)
+
+            def post_upload():
+                zip_dest = os.path.join(dest, filename)
+                try:
+                    ZipFile(zip_dest).namelist() # check if this file is valid
+                    with open(zip_dest, 'rb') as f:
+                        problem_data.zipfile.delete()
+                        problem_data.zipfile.save(filename, File(f))
+                        f.close()
+                except Exception as e:
+                    raise Exception(e)
+                finally:
+                    shutil.rmtree(dest)
+                    
+            try:
+                handle_upload(request.FILES['qqfile'], form.cleaned_data, dest, post_upload=post_upload)
+            except Exception as e:
+                return JsonResponse({'success': False, 'error': str(e)})
+            return JsonResponse({'success': True})
+        else:
+            return HttpResponse(status_code=400)