From c309628af347edb83ad5e82bf2d8d0795eb5362c Mon Sep 17 00:00:00 2001 From: doubaniux Date: Wed, 1 Sep 2021 11:41:21 +0200 Subject: [PATCH] fix uploading and parsing bugs --- sync/apps.py | 4 ++++ sync/forms.py | 1 + sync/jobs.py | 28 +++++++++++++++++----------- sync/models.py | 14 +++----------- sync/views.py | 3 ++- users/templates/users/home.html | 7 ++----- 6 files changed, 29 insertions(+), 28 deletions(-) diff --git a/sync/apps.py b/sync/apps.py index 0d95b485..6e952eab 100644 --- a/sync/apps.py +++ b/sync/apps.py @@ -3,3 +3,7 @@ from django.apps import AppConfig class SyncConfig(AppConfig): name = 'sync' + + def ready(self): + from sync.jobs import sync_task_manager + sync_task_manager.start() \ No newline at end of file diff --git a/sync/forms.py b/sync/forms.py index 165d474c..b457db71 100644 --- a/sync/forms.py +++ b/sync/forms.py @@ -10,6 +10,7 @@ class SyncTaskForm(forms.ModelForm): model = SyncTask fields = [ "user", + "file", "overwrite", "sync_book", "sync_movie", diff --git a/sync/jobs.py b/sync/jobs.py index c892b21a..5d749efb 100644 --- a/sync/jobs.py +++ b/sync/jobs.py @@ -21,6 +21,8 @@ from .models import SyncTask __all__ = ['sync_task_manager'] logger = logging.getLogger(__name__) + + class SyncTaskManger: # in seconds @@ -41,7 +43,8 @@ class SyncTaskManger: def __start_new_worker(self, task): new_worker = threading.Thread( - target=sync_doufen_job, args=[task, self.is_stopped]) + target=sync_doufen_job, args=[task, self.is_stopped], daemon=True + ) self.__worker_threads.append(new_worker) new_worker.start() @@ -71,10 +74,9 @@ class SyncTaskManger: def start(self): self.__enqueue_existing_tasks() # enqueue - self.__listen_for_new_task() # enqueue listen_new_task_thread = threading.Thread( - target=self.__listen_for_new_task) + target=self.__listen_for_new_task, daemon=True) self.__worker_threads.append(listen_new_task_thread) @@ -169,12 +171,12 @@ class DoufenParser: # parse data for i in range(start_row_index, ws.max_row + 1): # url definitely exists - url = ws.cell(row=i, column=URL_INDEX).value + url = ws.cell(row=i, column=self.URL_INDEX).value - tags = ws.cell(row=i, column=TAG_INDEX).value + tags = ws.cell(row=i, column=self.TAG_INDEX).value tags = tags.split(',') if tags else None - time = ws.cell(row=i, column=TIME_INDEX).value + time = ws.cell(row=i, column=self.TIME_INDEX).value if time: time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S") tz = pytz.timezone('Asia/Shanghai') @@ -182,11 +184,11 @@ class DoufenParser: else: time = None - content = ws.cell(row=i, column=CONTENT_INDEX).value + content = ws.cell(row=i, column=self.CONTENT_INDEX).value if not content: content = "" - rating = ws.cell(row=i, column=RATING_INDEX).value + rating = ws.cell(row=i, column=self.RATING_INDEX).value rating = int(rating) * 2 if rating else None # store result @@ -293,6 +295,9 @@ def overwrite_mark(entity, entity_class, mark, mark_class, tag_class, data, shee def sync_doufen_job(task, stop_check_func): + """ + TODO: Update task status every certain amount of items to reduce IO consumption + """ parser = DoufenParser(task) items = parser.parse() @@ -351,15 +356,14 @@ def sync_doufen_job(task, stop_check_func): task.failed_urls.append(data.url) task.save(update_fields=['failed_urls']) continue - + task.success_items += 1 task.save(update_fields=["success_items"]) - # if task finish if len(items) == 0: task.is_finished = True - task.clear_progress() + task.clear_breakpoint() task.save(update_fields=['is_finished', 'break_point']) @@ -376,6 +380,8 @@ def translate_status(sheet_name): sync_task_manager = SyncTaskManger() +# sync_task_manager.start() + signal.signal(signal.SIGTERM, sync_task_manager.stop) if sys.platform.startswith('linux'): signal.signal(signal.SIGHUP, sync_task_manager.stop) diff --git a/sync/models.py b/sync/models.py index 52f3444a..e5264477 100644 --- a/sync/models.py +++ b/sync/models.py @@ -18,7 +18,7 @@ class SyncTask(models.Model): #-----------------------------------# user = models.ForeignKey( User, on_delete=models.CASCADE, related_name='user_%(class)ss') - file = models.FileField(upload_to=sync_file_path) + file = models.FileField(upload_to=sync_file_path, default='') #-----------------------------------# # options @@ -60,15 +60,6 @@ class SyncTask(models.Model): started_time = models.DateTimeField(auto_now_add=True) ended_time = models.DateTimeField(auto_now=True) - - # how many items are overwritten - # overwrite_books = models.PositiveIntegerField(default=0) - # overwrite_movies = models.PositiveIntegerField(default=0) - # overwrite_music = models.PositiveIntegerField(default=0) - # overwrite_games = models.PositiveIntegerField(default=0) - - # thread pid - # pid = models.PositiveIntegerField(blank=True, null=True) class Meta: """Meta definition for SyncTask.""" @@ -107,7 +98,8 @@ class SyncTask(models.Model): def get_breakpoint(self): if not self.break_point: return None, None - return self.break_point.split('-') + progress = self.break_point.split('-') + return progress[0], int(progress[1]) def set_breakpoint(self, sheet_name, row_index, save=False): self.break_point = f"{sheet_name}-{row_index}" diff --git a/sync/views.py b/sync/views.py index 440cc2db..69aa7de6 100644 --- a/sync/views.py +++ b/sync/views.py @@ -21,7 +21,7 @@ def sync_douban(request): if request.method == 'POST': # validate sunmitted data try: - uploaded_file = request.FILES['xlsx'] + uploaded_file = request.FILES['file'] wb = openpyxl.open(uploaded_file, read_only=True, data_only=True, keep_links=False) wb.close() @@ -29,6 +29,7 @@ def sync_douban(request): # raise e return HttpResponseBadRequest(content="invalid excel file") + # file_data = {'file': request.FILES['xlsx']} form = SyncTaskForm(request.POST, request.FILES) if form.is_valid(): # stop all preivous task diff --git a/users/templates/users/home.html b/users/templates/users/home.html index ca983a53..329739a8 100644 --- a/users/templates/users/home.html +++ b/users/templates/users/home.html @@ -547,9 +547,8 @@ - - {% comment %} + {% if user == request.user %}
@@ -593,7 +592,7 @@ ?
- +
@@ -654,8 +653,6 @@
{% endif %} - {% endcomment %} -