Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

rescrape

Browse source
This commit is contained in:
Your Name 2022-09-17 23:39:08 -04:00
parent 8fa1ba05a7
commit 7cb4a376ce
16 changed files with 151 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
books/templates/books/create_update.html
View file

@ -22,8 +22,24 @@
<section id="content" class="container">
<div class="grid">
{% if is_update and form.source_site.value != 'in-site' %}
<div style="float:right;padding-left:16px">
<div class="aside-section-wrapper">
<div class="action-panel">
<div class="action-panel__label">{% trans '源网站' %}: <a href="{{ form.source_url.value }}">{{ form.source_site.value }}</a></div>
<div class="action-panel__button-group">
<form method="post" action="{% url 'books:rescrape' form.id.value %}">
{% csrf_token %}
<input class="button" type="submit" value="{% trans '从源网站重新抓取' %}">
</form>
</div>
</div>
</div>
</div>
{% endif %}
<div class="single-section-wrapper" id="main">
<a href="{% url 'books:scrape' %}" class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a>
{% comment %} <a href="{% url 'books:scrape' %}" class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a> {% endcomment %}
<form class="entity-form" action="{{ submit_url }}" method="post" enctype="multipart/form-data">
{% csrf_token %}
{{ form.media }}

1
books/urls.py
View file

@ -8,6 +8,7 @@ urlpatterns = [
path('<int:id>/', retrieve, name='retrieve'),
path('update/<int:id>/', update, name='update'),
path('delete/<int:id>/', delete, name='delete'),
path('rescrape/<int:id>/', rescrape, name='rescrape'),
path('mark/', create_update_mark, name='create_update_mark'),
path('wish/<int:id>/', wish, name='wish'),
re_path('(?P<book_id>[0-9]+)/mark/list/(?:(?P<following_only>\\d+))?', retrieve_mark_list, name='retrieve_mark_list'),

15
books/views.py
View file

@ -19,6 +19,7 @@ from .forms import *
from .forms import BookMarkStatusTranslator
from django.conf import settings
from collection.models import CollectionItem
from common.scraper import get_scraper_by_url, get_normalized_url
logger = logging.getLogger(__name__)
@ -89,6 +90,18 @@ def create(request):
return HttpResponseBadRequest()
@login_required
def rescrape(request, id):
if request.method != 'POST':
return HttpResponseBadRequest()
item = get_object_or_404(Book, pk=id)
url = get_normalized_url(item.source_url)
scraper = get_scraper_by_url(url)
scraper.scrape(url)
form = scraper.save(request_user=request.user, instance=item)
return redirect(reverse("books:retrieve", args=[form.instance.id]))
@login_required
def update(request, id):
if request.method == 'GET':
@ -99,6 +112,7 @@ def update(request, id):
'books/create_update.html',
{
'form': form,
'is_update': True,
'title': _('修改书籍'),
'submit_url': reverse("books:update", args=[book.id]),
# provided for frontend js
@ -127,6 +141,7 @@ def update(request, id):
'books/create_update.html',
{
'form': form,
'is_update': True,
'title': _('修改书籍'),
'submit_url': reverse("books:update", args=[book.id]),
# provided for frontend js

2
collection/views.py
View file

@ -107,6 +107,7 @@ def update(request, id):
'create_update.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("collection:update", args=[collection.id]),
# provided for frontend js
@ -130,6 +131,7 @@ def update(request, id):
'create_update.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("collection:update", args=[collection.id]),
# provided for frontend js

4
common/scraper.py
View file

@ -211,11 +211,11 @@ class AbstractScraper:
return raw_img, ext
@classmethod
def save(cls, request_user):
def save(cls, request_user, instance=None):
entity_cover = {
'cover': SimpleUploadedFile('temp.' + cls.img_ext, cls.raw_img)
} if cls.img_ext is not None else None
form = cls.form_class(cls.raw_data, entity_cover)
form = cls.form_class(data=cls.raw_data, files=entity_cover, instance=instance)
if form.is_valid():
form.instance.last_editor = request_user
form.instance._change_reason = 'scrape'

20
common/scrapers/spotify.py
View file

@ -181,16 +181,16 @@ class SpotifyAlbumScraper(AbstractScraper):
else:
return None
@classmethod
def save(cls, request_user):
form = super().save(request_user)
# task = Thread(
# target=cls.add_tracks,
# args=(form.instance, request_user),
# daemon=True
# )
# task.start()
return form
# @classmethod
# def save(cls, request_user):
# form = super().save(request_user)
# task = Thread(
# target=cls.add_tracks,
# args=(form.instance, request_user),
# daemon=True
# )
# task.start()
# return form
@classmethod
def get_api_url(cls, url):

3
games/forms.py
View file

@ -11,13 +11,14 @@ def GameMarkStatusTranslator(status):
class GameForm(forms.ModelForm):
# id = forms.IntegerField(required=False, widget=forms.HiddenInput())
id = forms.IntegerField(required=False, widget=forms.HiddenInput())
other_info = JSONField(required=False, label=_("其他信息"))
class Meta:
model = Game
fields = [
'id',
'title',
'source_site',
'source_url',

19
games/templates/games/create_update.html
View file

@ -22,9 +22,24 @@
<section id="content" class="container">
<div class="grid">
{% if is_update and form.source_site.value != 'in-site' %}
<div style="float:right;padding-left:16px">
<div class="aside-section-wrapper">
<div class="action-panel">
<div class="action-panel__label">{% trans '源网站' %}: <a href="{{ form.source_url.value }}">{{ form.source_site.value }}</a></div>
<div class="action-panel__button-group">
<form method="post" action="{% url 'games:rescrape' form.id.value %}">
{% csrf_token %}
<input class="button" type="submit" value="{% trans '从源网站重新抓取' %}">
</form>
</div>
</div>
</div>
</div>
{% endif %}
<div class="single-section-wrapper" id="main">
<a href="{% url 'games:scrape' %}"
class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a>
{% comment %} <a href="{% url 'games:scrape' %}" class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a> {% endcomment %}
<form class="entity-form" action="{{ submit_url }}" method="post" enctype="multipart/form-data">
{% csrf_token %}
{{ form.media }}

1
games/urls.py
View file

@ -8,6 +8,7 @@ urlpatterns = [
path('<int:id>/', retrieve, name='retrieve'),
path('update/<int:id>/', update, name='update'),
path('delete/<int:id>/', delete, name='delete'),
path('rescrape/<int:id>/', rescrape, name='rescrape'),
path('mark/', create_update_mark, name='create_update_mark'),
path('wish/<int:id>/', wish, name='wish'),
re_path('(?P<game_id>[0-9]+)/mark/list/(?:(?P<following_only>\\d+))?', retrieve_mark_list, name='retrieve_mark_list'),

15
games/views.py
View file

@ -18,6 +18,7 @@ from .models import *
from .forms import *
from django.conf import settings
from collection.models import CollectionItem
from common.scraper import get_scraper_by_url, get_normalized_url
logger = logging.getLogger(__name__)
@ -88,6 +89,18 @@ def create(request):
return HttpResponseBadRequest()
@login_required
def rescrape(request, id):
if request.method != 'POST':
return HttpResponseBadRequest()
item = get_object_or_404(Game, pk=id)
url = get_normalized_url(item.source_url)
scraper = get_scraper_by_url(url)
scraper.scrape(url)
form = scraper.save(request_user=request.user, instance=item)
return redirect(reverse("games:retrieve", args=[form.instance.id]))
@login_required
def update(request, id):
if request.method == 'GET':
@ -99,6 +112,7 @@ def update(request, id):
'games/create_update.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("games:update", args=[game.id]),
# provided for frontend js
@ -128,6 +142,7 @@ def update(request, id):
'games/create_update.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("games:update", args=[game.id]),
# provided for frontend js

21
movies/templates/movies/create_update.html
View file

@ -21,10 +21,25 @@
{% include "partial/_navbar.html" %}
<section id="content" class="container">
<div class="grid">
<div class="grid" class="single-section-wrapper">
{% if is_update and form.source_site.value != 'in-site' %}
<div style="float:right;padding-left:16px">
<div class="aside-section-wrapper">
<div class="action-panel">
<div class="action-panel__label">{% trans '源网站' %}: <a href="{{ form.source_url.value }}">{{ form.source_site.value }}</a></div>
<div class="action-panel__button-group">
<form method="post" action="{% url 'movies:rescrape' form.id.value %}">
{% csrf_token %}
<input class="button" type="submit" value="{% trans '从源网站重新抓取' %}">
</form>
</div>
</div>
</div>
</div>
{% endif %}
<div class="single-section-wrapper" id="main">
<a href="{% url 'movies:scrape' %}"
class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a>
{% comment %} <a href="{% url 'movies:scrape' %}" class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a> {% endcomment %}
<form class="entity-form" action="{{ submit_url }}" method="post" enctype="multipart/form-data">
{% csrf_token %}
{{ form.media }}

1
movies/urls.py
View file

@ -8,6 +8,7 @@ urlpatterns = [
path('<int:id>/', retrieve, name='retrieve'),
path('update/<int:id>/', update, name='update'),
path('delete/<int:id>/', delete, name='delete'),
path('rescrape/<int:id>/', rescrape, name='rescrape'),
path('mark/', create_update_mark, name='create_update_mark'),
path('wish/<int:id>/', wish, name='wish'),
re_path('(?P<movie_id>[0-9]+)/mark/list/(?:(?P<following_only>\\d+))?', retrieve_mark_list, name='retrieve_mark_list'),

15
movies/views.py
View file

@ -18,6 +18,7 @@ from .models import *
from .forms import *
from django.conf import settings
from collection.models import CollectionItem
from common.scraper import get_scraper_by_url, get_normalized_url
logger = logging.getLogger(__name__)
@ -88,6 +89,18 @@ def create(request):
return HttpResponseBadRequest()
@login_required
def rescrape(request, id):
if request.method != 'POST':
return HttpResponseBadRequest()
item = get_object_or_404(Movie, pk=id)
url = get_normalized_url(item.source_url)
scraper = get_scraper_by_url(url)
scraper.scrape(url)
form = scraper.save(request_user=request.user, instance=item)
return redirect(reverse("movies:retrieve", args=[form.instance.id]))
@login_required
def update(request, id):
if request.method == 'GET':
@ -99,6 +112,7 @@ def update(request, id):
'movies/create_update.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("movies:update", args=[movie.id]),
# provided for frontend js
@ -128,6 +142,7 @@ def update(request, id):
'movies/create_update.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("movies:update", args=[movie.id]),
# provided for frontend js

19
music/templates/music/create_update_album.html
View file

@ -22,9 +22,24 @@
<section id="content" class="container">
<div class="grid">
{% if is_update and form.source_site.value != 'in-site' %}
<div style="float:right;padding-left:16px">
<div class="aside-section-wrapper">
<div class="action-panel">
<div class="action-panel__label">{% trans '源网站' %}: <a href="{{ form.source_url.value }}">{{ form.source_site.value }}</a></div>
<div class="action-panel__button-group">
<form method="post" action="{% url 'music:rescrape' form.id.value %}">
{% csrf_token %}
<input class="button" type="submit" value="{% trans '从源网站重新抓取' %}">
</form>
</div>
</div>
</div>
</div>
{% endif %}
<div class="single-section-wrapper" id="main">
<a href="{% url 'music:scrape_album' %}"
class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a>
{% comment %} <a href="{% url 'music:scrape_album' %}" class="single-section-wrapper__link single-section-wrapper__link--secondary">{% trans '>>> 试试一键剽取~ <<<' %}</a> {% endcomment %}
<form class="entity-form" action="{{ submit_url }}" method="post" enctype="multipart/form-data">
{% csrf_token %}
{{ form.media }}

1
music/urls.py
View file

@ -25,6 +25,7 @@ urlpatterns = [
path('album/<int:id>/', retrieve_album, name='retrieve_album'),
path('album/update/<int:id>/', update_album, name='update_album'),
path('album/delete/<int:id>/', delete_album, name='delete_album'),
path('rescrape/<int:id>/', rescrape, name='rescrape'),
path('album/mark/', create_update_album_mark, name='create_update_album_mark'),
path('album/wish/<int:id>/', wish_album, name='wish_album'),
re_path('album/(?P<album_id>[0-9]+)/mark/list/(?:(?P<following_only>\\d+))?', retrieve_album_mark_list, name='retrieve_album_mark_list'),

17
music/views.py
View file

@ -18,6 +18,7 @@ from django.shortcuts import render, get_object_or_404, redirect, reverse
import logging
from django.shortcuts import render
from collection.models import CollectionItem
from common.scraper import get_scraper_by_url, get_normalized_url
logger = logging.getLogger(__name__)
@ -99,6 +100,7 @@ def update_song(request, id):
'music/create_update_song.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("music:update_song", args=[song.id]),
# provided for frontend js
@ -128,6 +130,7 @@ def update_song(request, id):
'music/create_update_song.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("music:update_song", args=[song.id]),
# provided for frontend js
@ -637,6 +640,18 @@ def create_album(request):
return HttpResponseBadRequest()
@login_required
def rescrape(request, id):
if request.method != 'POST':
return HttpResponseBadRequest()
item = get_object_or_404(Album, pk=id)
url = get_normalized_url(item.source_url)
scraper = get_scraper_by_url(url)
scraper.scrape(url)
form = scraper.save(request_user=request.user, instance=item)
return redirect(reverse("music:retrieve_album", args=[form.instance.id]))
@login_required
def update_album(request, id):
if request.method == 'GET':
@ -648,6 +663,7 @@ def update_album(request, id):
'music/create_update_album.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("music:update_album", args=[album.id]),
# provided for frontend js
@ -677,6 +693,7 @@ def update_album(request, id):
'music/create_update_album.html',
{
'form': form,
'is_update': True,
'title': page_title,
'submit_url': reverse("music:update_album", args=[album.id]),
# provided for frontend js