From 7cb4a376cea0cab6cb943f5cb5f5e50d9229a301 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 17 Sep 2022 23:39:08 -0400 Subject: [PATCH] rescrape --- books/templates/books/create_update.html | 18 +++++++++++++++- books/urls.py | 1 + books/views.py | 15 +++++++++++++ collection/views.py | 2 ++ common/scraper.py | 4 ++-- common/scrapers/spotify.py | 20 +++++++++--------- games/forms.py | 3 ++- games/templates/games/create_update.html | 19 +++++++++++++++-- games/urls.py | 1 + games/views.py | 15 +++++++++++++ movies/templates/movies/create_update.html | 21 ++++++++++++++++--- movies/urls.py | 1 + movies/views.py | 15 +++++++++++++ .../templates/music/create_update_album.html | 19 +++++++++++++++-- music/urls.py | 1 + music/views.py | 17 +++++++++++++++ 16 files changed, 151 insertions(+), 21 deletions(-) diff --git a/books/templates/books/create_update.html b/books/templates/books/create_update.html index 54e0ca99..de4b8ca7 100644 --- a/books/templates/books/create_update.html +++ b/books/templates/books/create_update.html @@ -22,8 +22,24 @@
+ {% if is_update and form.source_site.value != 'in-site' %} +
+
+
+
{% trans '源网站' %}: {{ form.source_site.value }}
+
+
+ {% csrf_token %} + +
+
+
+
+
+ {% endif %} +
- {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
{% csrf_token %} {{ form.media }} diff --git a/books/urls.py b/books/urls.py index 0185888e..518ea096 100644 --- a/books/urls.py +++ b/books/urls.py @@ -8,6 +8,7 @@ urlpatterns = [ path('/', retrieve, name='retrieve'), path('update//', update, name='update'), path('delete//', delete, name='delete'), + path('rescrape//', rescrape, name='rescrape'), path('mark/', create_update_mark, name='create_update_mark'), path('wish//', wish, name='wish'), re_path('(?P[0-9]+)/mark/list/(?:(?P\\d+))?', retrieve_mark_list, name='retrieve_mark_list'), diff --git a/books/views.py b/books/views.py index eeb91435..10d31b77 100644 --- a/books/views.py +++ b/books/views.py @@ -19,6 +19,7 @@ from .forms import * from .forms import BookMarkStatusTranslator from django.conf import settings from collection.models import CollectionItem +from common.scraper import get_scraper_by_url, get_normalized_url logger = logging.getLogger(__name__) @@ -89,6 +90,18 @@ def create(request): return HttpResponseBadRequest() +@login_required +def rescrape(request, id): + if request.method != 'POST': + return HttpResponseBadRequest() + item = get_object_or_404(Book, pk=id) + url = get_normalized_url(item.source_url) + scraper = get_scraper_by_url(url) + scraper.scrape(url) + form = scraper.save(request_user=request.user, instance=item) + return redirect(reverse("books:retrieve", args=[form.instance.id])) + + @login_required def update(request, id): if request.method == 'GET': @@ -99,6 +112,7 @@ def update(request, id): 'books/create_update.html', { 'form': form, + 'is_update': True, 'title': _('修改书籍'), 'submit_url': reverse("books:update", args=[book.id]), # provided for frontend js @@ -127,6 +141,7 @@ def update(request, id): 'books/create_update.html', { 'form': form, + 'is_update': True, 'title': _('修改书籍'), 'submit_url': reverse("books:update", args=[book.id]), # provided for frontend js diff --git a/collection/views.py b/collection/views.py index 85a26566..659cb586 100644 --- a/collection/views.py +++ b/collection/views.py @@ -107,6 +107,7 @@ def update(request, id): 'create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("collection:update", args=[collection.id]), # provided for frontend js @@ -130,6 +131,7 @@ def update(request, id): 'create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("collection:update", args=[collection.id]), # provided for frontend js diff --git a/common/scraper.py b/common/scraper.py index 94cae161..defe6f38 100644 --- a/common/scraper.py +++ b/common/scraper.py @@ -211,11 +211,11 @@ class AbstractScraper: return raw_img, ext @classmethod - def save(cls, request_user): + def save(cls, request_user, instance=None): entity_cover = { 'cover': SimpleUploadedFile('temp.' + cls.img_ext, cls.raw_img) } if cls.img_ext is not None else None - form = cls.form_class(cls.raw_data, entity_cover) + form = cls.form_class(data=cls.raw_data, files=entity_cover, instance=instance) if form.is_valid(): form.instance.last_editor = request_user form.instance._change_reason = 'scrape' diff --git a/common/scrapers/spotify.py b/common/scrapers/spotify.py index 53ebfde3..da891d69 100644 --- a/common/scrapers/spotify.py +++ b/common/scrapers/spotify.py @@ -181,16 +181,16 @@ class SpotifyAlbumScraper(AbstractScraper): else: return None - @classmethod - def save(cls, request_user): - form = super().save(request_user) - # task = Thread( - # target=cls.add_tracks, - # args=(form.instance, request_user), - # daemon=True - # ) - # task.start() - return form + # @classmethod + # def save(cls, request_user): + # form = super().save(request_user) + # task = Thread( + # target=cls.add_tracks, + # args=(form.instance, request_user), + # daemon=True + # ) + # task.start() + # return form @classmethod def get_api_url(cls, url): diff --git a/games/forms.py b/games/forms.py index cda8b881..e758f9fd 100644 --- a/games/forms.py +++ b/games/forms.py @@ -11,13 +11,14 @@ def GameMarkStatusTranslator(status): class GameForm(forms.ModelForm): - # id = forms.IntegerField(required=False, widget=forms.HiddenInput()) + id = forms.IntegerField(required=False, widget=forms.HiddenInput()) other_info = JSONField(required=False, label=_("其他信息")) class Meta: model = Game fields = [ + 'id', 'title', 'source_site', 'source_url', diff --git a/games/templates/games/create_update.html b/games/templates/games/create_update.html index b813363a..178089c4 100644 --- a/games/templates/games/create_update.html +++ b/games/templates/games/create_update.html @@ -22,9 +22,24 @@
+ {% if is_update and form.source_site.value != 'in-site' %} +
+
+
+
{% trans '源网站' %}: {{ form.source_site.value }}
+
+ + {% csrf_token %} + + +
+
+
+
+ {% endif %} +
- {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
{% csrf_token %} {{ form.media }} diff --git a/games/urls.py b/games/urls.py index 29f37500..88ce1629 100644 --- a/games/urls.py +++ b/games/urls.py @@ -8,6 +8,7 @@ urlpatterns = [ path('/', retrieve, name='retrieve'), path('update//', update, name='update'), path('delete//', delete, name='delete'), + path('rescrape//', rescrape, name='rescrape'), path('mark/', create_update_mark, name='create_update_mark'), path('wish//', wish, name='wish'), re_path('(?P[0-9]+)/mark/list/(?:(?P\\d+))?', retrieve_mark_list, name='retrieve_mark_list'), diff --git a/games/views.py b/games/views.py index ff171ad2..0f0fc166 100644 --- a/games/views.py +++ b/games/views.py @@ -18,6 +18,7 @@ from .models import * from .forms import * from django.conf import settings from collection.models import CollectionItem +from common.scraper import get_scraper_by_url, get_normalized_url logger = logging.getLogger(__name__) @@ -88,6 +89,18 @@ def create(request): return HttpResponseBadRequest() +@login_required +def rescrape(request, id): + if request.method != 'POST': + return HttpResponseBadRequest() + item = get_object_or_404(Game, pk=id) + url = get_normalized_url(item.source_url) + scraper = get_scraper_by_url(url) + scraper.scrape(url) + form = scraper.save(request_user=request.user, instance=item) + return redirect(reverse("games:retrieve", args=[form.instance.id])) + + @login_required def update(request, id): if request.method == 'GET': @@ -99,6 +112,7 @@ def update(request, id): 'games/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("games:update", args=[game.id]), # provided for frontend js @@ -128,6 +142,7 @@ def update(request, id): 'games/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("games:update", args=[game.id]), # provided for frontend js diff --git a/movies/templates/movies/create_update.html b/movies/templates/movies/create_update.html index f98a388d..497c9bce 100644 --- a/movies/templates/movies/create_update.html +++ b/movies/templates/movies/create_update.html @@ -21,10 +21,25 @@ {% include "partial/_navbar.html" %}
-
+
+ {% if is_update and form.source_site.value != 'in-site' %} +
+
+
+
{% trans '源网站' %}: {{ form.source_site.value }}
+
+ + {% csrf_token %} + + +
+
+
+
+ {% endif %} +
- {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
{% csrf_token %} {{ form.media }} diff --git a/movies/urls.py b/movies/urls.py index d059cfbe..0495a9f0 100644 --- a/movies/urls.py +++ b/movies/urls.py @@ -8,6 +8,7 @@ urlpatterns = [ path('/', retrieve, name='retrieve'), path('update//', update, name='update'), path('delete//', delete, name='delete'), + path('rescrape//', rescrape, name='rescrape'), path('mark/', create_update_mark, name='create_update_mark'), path('wish//', wish, name='wish'), re_path('(?P[0-9]+)/mark/list/(?:(?P\\d+))?', retrieve_mark_list, name='retrieve_mark_list'), diff --git a/movies/views.py b/movies/views.py index 50f8bdbc..d4fc21f5 100644 --- a/movies/views.py +++ b/movies/views.py @@ -18,6 +18,7 @@ from .models import * from .forms import * from django.conf import settings from collection.models import CollectionItem +from common.scraper import get_scraper_by_url, get_normalized_url logger = logging.getLogger(__name__) @@ -88,6 +89,18 @@ def create(request): return HttpResponseBadRequest() +@login_required +def rescrape(request, id): + if request.method != 'POST': + return HttpResponseBadRequest() + item = get_object_or_404(Movie, pk=id) + url = get_normalized_url(item.source_url) + scraper = get_scraper_by_url(url) + scraper.scrape(url) + form = scraper.save(request_user=request.user, instance=item) + return redirect(reverse("movies:retrieve", args=[form.instance.id])) + + @login_required def update(request, id): if request.method == 'GET': @@ -99,6 +112,7 @@ def update(request, id): 'movies/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("movies:update", args=[movie.id]), # provided for frontend js @@ -128,6 +142,7 @@ def update(request, id): 'movies/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("movies:update", args=[movie.id]), # provided for frontend js diff --git a/music/templates/music/create_update_album.html b/music/templates/music/create_update_album.html index 1ef3f190..111c60a1 100644 --- a/music/templates/music/create_update_album.html +++ b/music/templates/music/create_update_album.html @@ -22,9 +22,24 @@
+ {% if is_update and form.source_site.value != 'in-site' %} +
+
+
+
{% trans '源网站' %}: {{ form.source_site.value }}
+
+ + {% csrf_token %} + + +
+
+
+
+ {% endif %} +
- {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
{% csrf_token %} {{ form.media }} diff --git a/music/urls.py b/music/urls.py index d99248e2..09b180ca 100644 --- a/music/urls.py +++ b/music/urls.py @@ -25,6 +25,7 @@ urlpatterns = [ path('album//', retrieve_album, name='retrieve_album'), path('album/update//', update_album, name='update_album'), path('album/delete//', delete_album, name='delete_album'), + path('rescrape//', rescrape, name='rescrape'), path('album/mark/', create_update_album_mark, name='create_update_album_mark'), path('album/wish//', wish_album, name='wish_album'), re_path('album/(?P[0-9]+)/mark/list/(?:(?P\\d+))?', retrieve_album_mark_list, name='retrieve_album_mark_list'), diff --git a/music/views.py b/music/views.py index f4188890..af37c2f2 100644 --- a/music/views.py +++ b/music/views.py @@ -18,6 +18,7 @@ from django.shortcuts import render, get_object_or_404, redirect, reverse import logging from django.shortcuts import render from collection.models import CollectionItem +from common.scraper import get_scraper_by_url, get_normalized_url logger = logging.getLogger(__name__) @@ -99,6 +100,7 @@ def update_song(request, id): 'music/create_update_song.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("music:update_song", args=[song.id]), # provided for frontend js @@ -128,6 +130,7 @@ def update_song(request, id): 'music/create_update_song.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("music:update_song", args=[song.id]), # provided for frontend js @@ -637,6 +640,18 @@ def create_album(request): return HttpResponseBadRequest() +@login_required +def rescrape(request, id): + if request.method != 'POST': + return HttpResponseBadRequest() + item = get_object_or_404(Album, pk=id) + url = get_normalized_url(item.source_url) + scraper = get_scraper_by_url(url) + scraper.scrape(url) + form = scraper.save(request_user=request.user, instance=item) + return redirect(reverse("music:retrieve_album", args=[form.instance.id])) + + @login_required def update_album(request, id): if request.method == 'GET': @@ -648,6 +663,7 @@ def update_album(request, id): 'music/create_update_album.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("music:update_album", args=[album.id]), # provided for frontend js @@ -677,6 +693,7 @@ def update_album(request, id): 'music/create_update_album.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("music:update_album", args=[album.id]), # provided for frontend js