implement import douban data function

This commit is contained in:
doubaniux 2021-06-14 22:18:39 +02:00
parent b744846e8f
commit 674d95ca49
19 changed files with 969 additions and 35 deletions

View file

@ -27,6 +27,7 @@ urlpatterns = [
path('movies/', include('movies.urls')),
path('music/', include('music.urls')),
path('games/', include('games.urls')),
path('sync/', include('sync.urls')),
path('announcement/', include('management.urls')),
path('', include('common.urls')),

View file

@ -10,6 +10,7 @@ from markdownx.models import MarkdownxField
from users.models import User
from mastodon.api import get_relationships, get_cross_site_id
from boofilsic.settings import CLIENT_NAME
from django.utils import timezone
RE_HTML_TAG = re.compile(r"<[^>]*>")
@ -33,7 +34,7 @@ class Entity(models.Model):
rating = models.DecimalField(
null=True, blank=True, max_digits=3, decimal_places=1)
created_time = models.DateTimeField(auto_now_add=True)
edited_time = models.DateTimeField(auto_now_add=True)
edited_time = models.DateTimeField(auto_now=True)
last_editor = models.ForeignKey(
User, on_delete=models.SET_NULL, related_name='%(class)s_last_editor', null=True, blank=False)
brief = models.TextField(_("简介"), blank=True, default="")
@ -101,6 +102,10 @@ class Entity(models.Model):
pass
def update_rating(self, old_rating, new_rating):
"""
@param old_rating: the old mark rating
@param new_rating: the new mark rating
"""
self.calculate_rating(old_rating, new_rating)
self.save()
@ -145,8 +150,8 @@ class UserOwnedEntity(models.Model):
is_private = models.BooleanField()
owner = models.ForeignKey(
User, on_delete=models.CASCADE, related_name='user_%(class)ss')
created_time = models.DateTimeField(auto_now_add=True)
edited_time = models.DateTimeField(auto_now_add=True)
created_time = models.DateTimeField(default=timezone.now)
edited_time = models.DateTimeField(default=timezone.now)
class Meta:
abstract = True
@ -245,6 +250,9 @@ class Mark(UserOwnedEntity):
models.CheckConstraint(check=models.Q(
rating__lte=10), name='mark_rating_upperbound'),
]
# TODO update entity rating when save
# TODO update tags
class Review(UserOwnedEntity):

View file

@ -1039,7 +1039,7 @@ class DoubanGameScraper(AbstractScraper):
raw_title = content.xpath(
"//div[@id='content']/h1/text()")[0].strip()
except IndexError:
raise ValueError("given url contains no movie info")
raise ValueError("given url contains no game info")
title = raw_title

View file

@ -2126,6 +2126,10 @@ select::placeholder {
justify-content: flex-start;
}
.user-relation .user-relation__related-user-list:last-of-type {
margin-bottom: 0;
}
.user-relation .user-relation__related-user {
-ms-flex-preferred-size: 25%;
flex-basis: 25%;
@ -2140,7 +2144,17 @@ select::placeholder {
}
.user-relation .user-relation__related-user-avatar {
background-image: url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7");
width: 48px;
height: 48px;
}
@media (min-width: 575.98px) and (max-width: 991.98px) {
.user-relation .user-relation__related-user-avatar {
height: unset;
width: 60%;
max-width: 96px;
}
}
.user-relation .user-relation__related-user-name {
@ -2156,6 +2170,10 @@ select::placeholder {
margin-bottom: 10px;
}
.report-panel .report-panel__body {
padding-left: 0;
}
.report-panel .report-panel__report {
margin: 2px 0;
}
@ -2168,6 +2186,125 @@ select::placeholder {
margin-left: 5px;
}
.import-panel {
overflow-x: hidden;
}
.import-panel .import-panel__label {
display: inline-block;
margin-bottom: 10px;
}
.import-panel .import-panel__body {
padding-left: 0;
border: 2px dashed #00a1cc;
padding: 6px 9px;
}
.import-panel .import-panel__body form {
margin: 0;
}
@media (max-width: 991.98px) {
.import-panel .import-panel__body {
border: unset;
padding-left: 0;
}
}
.import-panel .import-panel__help {
background-color: #e5e5e5;
border-radius: 100000px;
display: inline-block;
width: 16px;
height: 16px;
text-align: center;
font-size: 12px;
cursor: help;
}
.import-panel .import-panel__checkbox {
display: inline-block;
margin-right: 10px;
}
.import-panel .import-panel__checkbox label {
display: inline;
}
.import-panel .import-panel__checkbox input[type="checkbox"] {
margin: 0;
position: relative;
top: 2px;
}
.import-panel .import-panel__checkbox--last {
margin-right: 0;
}
.import-panel .import-panel__file-input {
margin-top: 10px;
}
.import-panel .import-panel__button {
line-height: unset;
height: unset;
padding: 4px 15px;
}
.import-panel .import-panel__progress {
padding-top: 10px;
}
.import-panel .import-panel__progress:not(:first-child) {
border-top: #bbb 1px dashed;
}
.import-panel .import-panel__progress label {
display: inline;
}
.import-panel .import-panel__progress progress {
background-color: #d5d5d5;
border-radius: 0;
height: 10px;
width: 65%;
}
.import-panel .import-panel__progress progress::-webkit-progress-bar {
background-color: #d5d5d5;
}
.import-panel .import-panel__progress progress::-webkit-progress-value {
background-color: #00a1cc;
}
.import-panel .import-panel__progress progress::-moz-progress-bar {
background-color: #d5d5d5;
}
.import-panel .import-panel__last-task:not(:first-child) {
padding-top: 4px;
border-top: #bbb 1px dashed;
}
.import-panel .import-panel__last-task .index:not(:last-of-type) {
margin-right: 8px;
}
.import-panel .import-panel__fail-urls {
margin-top: 10px;
}
.import-panel .import-panel__fail-urls li {
word-break: break-all;
}
.import-panel .import-panel__fail-urls ul {
max-height: 100px;
overflow-y: auto;
}
.relation-dropdown .relation-dropdown__button {
display: none;
}
@ -2309,6 +2446,10 @@ select::placeholder {
-ms-flex-align: center;
align-items: center;
cursor: pointer;
-webkit-transition: -webkit-transform 0.3s;
transition: -webkit-transform 0.3s;
transition: transform 0.3s;
transition: transform 0.3s, -webkit-transform 0.3s;
}
.relation-dropdown .relation-dropdown__button:focus {
background-color: red;
@ -2327,15 +2468,15 @@ select::placeholder {
transform: rotate(-180deg);
}
.relation-dropdown .relation-dropdown__button + .relation-dropdown__body--expand {
max-height: 500px;
-webkit-transition: max-height 0.6s ease-in;
transition: max-height 0.6s ease-in;
max-height: 2000px;
-webkit-transition: max-height 1s ease-in;
transition: max-height 1s ease-in;
}
.relation-dropdown .relation-dropdown__body {
background-color: #f7f7f7;
max-height: 0;
-webkit-transition: max-height 0.6s ease-out;
transition: max-height 0.6s ease-out;
-webkit-transition: max-height 1s ease-out;
transition: max-height 1s ease-out;
overflow: hidden;
}
.entity-card {
@ -2515,3 +2656,9 @@ select::placeholder {
.ms-parent > .ms-drop > ul > li > label > input {
width: unset;
}
.tippy-box {
border: #606c76 1px solid;
background-color: #f7f7f7;
padding: 3px 5px;
}

File diff suppressed because one or more lines are too long

View file

@ -111,20 +111,133 @@ $(document).ready( function() {
);
// mobile dropdown
$(".relation-dropdown__button").click(e => {
const button = $(e.currentTarget);
$(".relation-dropdown__button").data("collapse", true);
function onClickDropdownButton(e) {
const button = $(".relation-dropdown__button");
button.data("collapse", !button.data("collapse"));
button.children('.icon-arrow').toggleClass("icon-arrow--expand");
button.siblings('.relation-dropdown__body').toggleClass("relation-dropdown__body--expand");
})
}
$(".relation-dropdown__button").click(onClickDropdownButton)
// close when click outside
window.onclick = evt => {
const button = $(".relation-dropdown__button");
const target = $(evt.target);
if (!target.parents('.relation-dropdown__button').length && !target.hasClass("relation-dropdown__button")) {
button.children('.icon-arrow').removeClass("icon-arrow--expand");
button.siblings('.relation-dropdown__body').removeClass("relation-dropdown__body--expand");
if (!target.parents('.relation-dropdown').length && !$(".relation-dropdown__button").data("collapse")) {
onClickDropdownButton();
}
};
// import panel
$("#uploadBtn").click(e => {
const btn = $("#uploadBtn")
const form = $(".import-panel__body form")
// validate form
let isValidForm = form[0].checkValidity();
if (!isValidForm) {
if (form[0].reportValidity) {
form[0].reportValidity();
} else {
alert("Invalid File");
}
return
}
e.preventDefault();
let formData = new FormData(form[0]);
// disable submit button for a while
btn.prop('disabled', true);
setTimeout(() => {
btn.prop('disabled', false);
}, 2000);
// show progress bar & hide last status
$(".import-panel__progress").show();
$(".import-panel__last-task").hide();
// flush failed urls
$("#failedUrls").html("");
// reset progress bar
$("#importProgress").attr("max", 1);
$("#importProgress").attr("value", 0);
percent.text('0%');
$.ajax({
url: form.attr("action"),
type: 'POST',
data: formData,
contentType: false,
dataType: "json",
processData: false,
enctype: form.attr("enctype"),
success: function (response) {
// console.log("here");
// console.log(response);
// long polling
poll();
},
error: function (response) {
// console.log("there")
// console.log(response)
},
complete: function (response) {
// console.log("somewhere")
// console.log(response)
},
});
});
// if progress is visible start to poll
if ($(".import-panel__progress").is(":visible")) {
poll();
}
// long polling function
const pollingInterval = 1500;
function poll() {
$.ajax({
url: $("#querySyncInfoURL").data("url"),
success: function (data) {
console.log(data);
const progress = $("#importProgress");
const percent = $("#progressPercent");
if (!data.is_finished) {
// update progress bar
if (!data.total_items == 0) {
progress.attr("max", data.total_items);
progress.attr("value", data.finished_items);
percent.text(Math.floor(100 * data.finished_items / data.total_items) + '%');
}
setTimeout(() => {
poll();
}, pollingInterval);
} else {
// task finishes
// update progress bar
percent.text('100%');
progress.attr("max", 1);
progress.attr("value", 1);
// update last task summary
$("#lastTaskTotalItems").text(data.total_items);
$("#lastTaskSuccessItems").text(data.success_items);
$("#lastTaskStatus").text(data.status);
// display failed urls
data.failed_urls.forEach((v, i) => {
console.log(v)
$("#failedUrls").append($("<li>" + v + "</li>"));
});
// hide progress & show last task
$(".import-panel__progress").hide();
$(".import-panel__last-task").show();
}
},
dataType: "json",
complete: () => {
// setTimeout(() => {
// poll();
// }, pollingInterval);
},
});
}
});

View file

@ -125,6 +125,8 @@ $aside-section-padding-mobile: 24px 25px 10px 25px
& &__related-user-list
display: flex
justify-content: flex-start
&:last-of-type
margin-bottom: 0
& &__related-user
flex-basis: 25%
@ -137,7 +139,14 @@ $aside-section-padding-mobile: 24px 25px 10px 25px
color: $color-secondary
& &__related-user-avatar
background-image: url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")
width: 48px
height: 48px
@media (min-width: $small-devices) and (max-width: $large-devices)
height: unset
width: 60%
max-width: 96px
& &__related-user-name
color: inherit
@ -147,11 +156,16 @@ $aside-section-padding-mobile: 24px 25px 10px 25px
-webkit-box-orient: vertical;
-webkit-line-clamp: 2;
$panel-padding : 0
.report-panel
& &__label
display: inline-block
margin-bottom: 10px
& &__body
padding-left: $panel-padding
& &__report-list
& &__report
@ -163,6 +177,93 @@ $aside-section-padding-mobile: 24px 25px 10px 25px
& &__all-link
margin-left: 5px
.import-panel
overflow-x: hidden
& &__label
display: inline-block
margin-bottom: 10px
& &__body
padding-left: $panel-padding
border: 2px dashed #00a1cc
padding: 6px 9px
form
margin: 0
@media (max-width: $large-devices)
border: unset
padding-left: 0
& &__help
background-color: $color-quinary
border-radius: 100000px
display: inline-block
width: 16px
height: 16px
text-align: center
font-size: 12px
cursor: help
& &__checkbox
display: inline-block
margin-right: 10px
label
display: inline
input[type="checkbox"]
margin: 0
position: relative
top: 2px
&--last
margin-right: 0
& &__file-input
margin-top: 10px
& &__button
line-height: unset
height: unset
padding: 4px 15px
& &__progress
padding-top: 10px
// padding-top: 4px
&:not(:first-child)
border-top: $color-tertiary 1px dashed
label
display: inline
progress
background-color: $color-quaternary
border-radius: 0
height: 10px
width: 65%
progress::-webkit-progress-bar
background-color: $color-quaternary
progress::-webkit-progress-value
background-color: $color-primary
progress::-moz-progress-bar
background-color: $color-quaternary
& &__last-task
&:not(:first-child)
padding-top: 4px
border-top: $color-tertiary 1px dashed
.index:not(:last-of-type)
margin-right: 8px
& &__fail-urls
margin-top: 10px
li
word-break: break-all
ul
// padding: 4px
max-height: 100px
overflow-y: auto
.relation-dropdown
& &__button
display: none
@ -267,6 +368,7 @@ $aside-section-padding-mobile: 24px 25px 10px 25px
justify-content: center
align-items: center
cursor: pointer
transition: transform 0.3s
&:focus
background-color: red
@ -280,13 +382,13 @@ $aside-section-padding-mobile: 24px 25px 10px 25px
transform: rotate(-180deg)
& &__button + &__body--expand
max-height: 500px
transition: max-height 0.6s ease-in
max-height: 2000px
transition: max-height 1s ease-in
& &__body
background-color: $color-bright
max-height: 0
transition: max-height 0.6s ease-out
transition: max-height 1s ease-out
overflow: hidden
.entity-card

View file

@ -41,3 +41,10 @@
& > input
width: unset
.tippy-box
border: $color-secondary 1px solid
// border-radius: 2px
background-color: $color-bright
padding: 3px 5px
.tippy-content

0
sync/__init__.py Normal file
View file

4
sync/admin.py Normal file
View file

@ -0,0 +1,4 @@
from django.contrib import admin
from .models import *
admin.site.register(SyncTask)

5
sync/apps.py Normal file
View file

@ -0,0 +1,5 @@
from django.apps import AppConfig
class SyncConfig(AppConfig):
name = 'sync'

20
sync/forms.py Normal file
View file

@ -0,0 +1,20 @@
from django import forms
from .models import SyncTask
class SyncTaskForm(forms.ModelForm):
"""Form definition for SyncTask."""
class Meta:
"""Meta definition for SyncTaskform."""
model = SyncTask
fields = [
"user",
"overwrite",
"sync_book",
"sync_movie",
"sync_music",
"sync_game",
"default_public",
]

205
sync/jobs.py Normal file
View file

@ -0,0 +1,205 @@
import logging
import pytz
from datetime import datetime
from django.utils import timezone
from django.core.exceptions import ObjectDoesNotExist
from openpyxl import load_workbook
from books.models import BookMark, Book, BookTag
from movies.models import MovieMark, Movie, MovieTag
from music.models import AlbumMark, Album, AlbumTag
from games.models import GameMark, Game, GameTag
from common.scraper import DoubanAlbumScraper, DoubanBookScraper, DoubanGameScraper, DoubanMovieScraper
from common.models import MarkStatusEnum
logger = logging.getLogger(__name__)
def sync_douban_job(task, user, filename, temp_dir):
try:
# NOTE use python IO since bug occurs using openpyxl
fp = open(filename, 'rb')
wb = load_workbook(
fp,
read_only=True,
data_only=True,
keep_links=False
)
# count items
items_count = 0
# sheet names
categories = []
# substract headers
if task.sync_book:
categories.append({'sheets': ['想读', '在读', '读过'], 'mark': BookMark, 'entity': Book, 'tag': BookTag, 'scraper': DoubanBookScraper})
items_count += wb['想读'].max_row + wb['在读'].max_row + wb['读过'].max_row - 3
if task.sync_movie:
categories.append({'sheets': ['想看', '在看', '看过'], 'mark': MovieMark, 'entity': Movie, 'tag': MovieTag, 'scraper': DoubanMovieScraper})
items_count += wb['想看'].max_row + wb['在看'].max_row + wb['看过'].max_row - 3
if task.sync_music:
categories.append({'sheets': ['想听', '在听', '听过'], 'mark': AlbumMark, 'entity': Album, 'tag': AlbumTag, 'scraper': DoubanAlbumScraper})
items_count += wb['想听'].max_row + wb['在听'].max_row + wb['听过'].max_row - 3
if task.sync_game:
categories.append({'sheets': ['想玩', '在玩', '玩过'], 'mark': GameMark, 'entity': Game, 'tag': GameTag, 'scraper': DoubanGameScraper})
items_count += wb['想玩'].max_row + wb['在玩'].max_row + wb['玩过'].max_row - 3
if items_count == 0:
task.is_finished = True
task.ended_time = timezone.now()
task.save()
wb.close()
temp_dir.cleanup()
return
else:
task.total_items = items_count
task.save(update_fields=["total_items"])
# add marks
# indices in xlsx
URL_INDEX = 4
CONTENT_INDEX = 8
TAG_INDEX = 7
TIME_INDEX = 5
RATING_INDEX = 6
for category in categories:
for sheet in category['sheets']:
ws = wb[sheet]
if ws.max_row <= 1:
continue
for i in range(2, ws.max_row + 1):
task.finished_items += 1
task.save(update_fields=["finished_items"])
# collect info
# url definitely exists
url = ws.cell(row=i, column=URL_INDEX).value
tags = ws.cell(row=i, column=TAG_INDEX).value
if tags:
tags = tags.split(',')
else:
tags = None
time = ws.cell(row=i, column=TIME_INDEX).value
if time:
time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S")
tz = pytz.timezone('Asia/Shanghai')
time = time.replace(tzinfo=tz)
else:
time = None
content = ws.cell(row=i, column=CONTENT_INDEX).value
if not content:
content = ""
rating = ws.cell(row=i, column=RATING_INDEX).value
if rating:
rating = int(rating) * 2
else:
rating = None
# scrape the entity if not exists
try:
item = category['entity'].objects.get(source_url=url)
except ObjectDoesNotExist:
try:
scraper = category['scraper']
scraper.scrape(url)
form = scraper.save(request_user=user)
item = form.instance
except Exception as e:
logger.error(f"Scrape Failed URL: {url}")
logger.error("Expections during saving scraped data:", exc_info=e)
task.failed_urls.append(url)
task.save(update_fields=['failed_urls'])
continue
# sync mark
try:
# already exists
params = {
'owner': user,
category['entity'].__name__.lower(): item
}
mark = category['mark'].objects.get(**params)
old_rating = mark.rating
old_tags = getattr(
mark, category['mark'].__name__.lower()+'_tags').all()
if task.overwrite:
# update mark logic
mark.created_time = time
mark.edited_time = time
mark.text = content
mark.rating = rating
mark.status = translate_status(sheet)
mark.save()
item.update_rating(old_rating, rating)
if old_tags:
for tag in old_tags:
tag.delete()
if tags:
for tag in tags:
params = {
'content': tag,
category['entity'].__name__.lower(): item,
'mark': mark
}
category['tag'].objects.create(**params)
else:
continue
except ObjectDoesNotExist:
# add new mark
params = {
'owner': user,
'created_time': time,
'edited_time': time,
'rating': rating,
'text': content,
'status': translate_status(sheet),
'is_private': not task.default_public,
category['entity'].__name__.lower(): item,
}
mark = category['mark'].objects.create(**params)
item.update_rating(None, rating)
if tags:
for tag in tags:
params = {
'content': tag,
category['entity'].__name__.lower(): item,
'mark': mark
}
category['tag'].objects.create(**params)
except Exception as e:
logger.error("Unknown exception when syncing marks", exc_info=e)
task.failed_urls.append(url)
task.save(update_fields=['failed_urls'])
continue
task.success_items += 1
task.save(update_fields=["success_items"])
task.is_finished = True
task.ended_time = timezone.now()
task.save()
wb.close()
except Exception as e:
task.is_failed = True
task.is_finished = True
task.ended_time = timezone.now()
task.save()
logger.error("Sync task failed", exc_info=e)
raise e
finally:
if wb is not None:
wb.close()
fp.close()
temp_dir.cleanup()
def translate_status(sheet_name):
if '' in sheet_name:
return MarkStatusEnum.WISH
elif '' in sheet_name:
return MarkStatusEnum.DO
elif '' in sheet_name:
return MarkStatusEnum.COLLECT
raise ValueError("Not valid status")

87
sync/models.py Normal file
View file

@ -0,0 +1,87 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
import django.contrib.postgres.fields as postgres
from users.models import User
class SyncTask(models.Model):
"""A class that records information about douban data synchronization task."""
user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name='user_%(class)ss')
is_failed = models.BooleanField(default=False)
# fail_reason = models.TextField(default='')
is_finished = models.BooleanField(default=False)
# how many items to synchronize
total_items = models.PositiveIntegerField(default=0)
# how many items are handled
finished_items = models.PositiveIntegerField(default=0)
# how many imtes have been synchronized successfully
success_items = models.PositiveIntegerField(default=0)
failed_urls = postgres.ArrayField(
models.URLField(blank=True, default='', max_length=200),
null=True,
blank=True,
default=list,
)
started_time = models.DateTimeField(auto_now_add=True)
ended_time = models.DateTimeField(auto_now=True)
# how many items are overwritten
# overwrite_books = models.PositiveIntegerField(default=0)
# overwrite_movies = models.PositiveIntegerField(default=0)
# overwrite_music = models.PositiveIntegerField(default=0)
# overwrite_games = models.PositiveIntegerField(default=0)
# options
# for the same book, if is already marked before sync, overwrite the previous mark or not
overwrite = models.BooleanField(default=False)
# sync book marks or not
sync_book = models.BooleanField()
# sync movie marks or not
sync_movie = models.BooleanField()
# sync music marks or not
sync_music = models.BooleanField()
# sync game marks or not
sync_game = models.BooleanField()
# default visibility of marks
default_public = models.BooleanField()
# thread pid
pid = models.PositiveIntegerField(blank=True, null=True)
class Meta:
"""Meta definition for SyncTask."""
verbose_name = 'SyncTask'
verbose_name_plural = 'SyncTasks'
def __str__(self):
"""Unicode representation of SyncTask."""
return str(self.user.username) + '@' + str(self.started_time) + self.get_status_emoji()
def get_status_emoji(self):
return ("" if self.is_failed else "") if self.is_finished else ""
def get_duration(self):
return self.ended_time - self.started_time
def get_overwritten_items(self):
if self.overwrite:
return self.overwrite_books + self.overwrite_games + self.overwrite_movies + self.overwrite_music
else:
return 0
def get_progress(self):
"""
@return: return percentage
"""
if self.is_finished:
return 100
else:
if self.total_items > 0:
return 100 * self.finished_items / self.total_items
else:
return 0

3
sync/tests.py Normal file
View file

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

10
sync/urls.py Normal file
View file

@ -0,0 +1,10 @@
from django.urls import path
from .views import *
app_name = 'sync'
urlpatterns = [
path('douban/', sync_douban, name='douban'),
path('progress/', query_progress, name='progress'),
path('last/', query_last_task, name='last'),
]

81
sync/views.py Normal file
View file

@ -0,0 +1,81 @@
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseBadRequest, JsonResponse, HttpResponse
from .models import SyncTask
from .forms import SyncTaskForm
from .jobs import sync_douban_job
import tempfile
import os
from threading import Thread
import openpyxl
from django.utils.datastructures import MultiValueDictKeyError
from openpyxl.utils.exceptions import InvalidFileException
from zipfile import BadZipFile
@login_required
def sync_douban(request):
"""
Sync douban data from .xlsx file generated by doufen
"""
if request.method == 'POST':
# validate sunmitted data
try:
uploaded_file = request.FILES['xlsx']
wb = openpyxl.open(uploaded_file, read_only=True,
data_only=True, keep_links=False)
wb.close()
except (MultiValueDictKeyError, InvalidFileException, BadZipFile) as e :
# raise e
return HttpResponseBadRequest(content="invalid excel file")
form = SyncTaskForm(request.POST)
if form.is_valid():
# stop all preivous task
SyncTask.objects.filter(user=request.user, is_finished=False).update(is_finished=True)
form.save()
temp_dir = tempfile.TemporaryDirectory()
filename = os.path.join(temp_dir.name, uploaded_file.name)
with open(filename, 'wb+') as destination:
for chunk in uploaded_file.chunks():
destination.write(chunk)
task = Thread(
target=sync_douban_job,
args=(form.instance, request.user, filename, temp_dir),
daemon=True
)
task.start()
return HttpResponse(status=204)
else:
return HttpResponseBadRequest()
else:
return HttpResponseBadRequest()
@login_required
def query_progress(request):
task = request.user.user_synctasks.order_by('-id').first()
if task is not None:
return JsonResponse({
'progress': task.get_progress()
})
else:
return JsonResponse()
def query_last_task(request):
task = request.user.user_synctasks.order_by('-id').first()
if task is not None:
return JsonResponse({
'total_items': task.total_items,
'success_items': task.success_items,
'finished_items': task.finished_items,
'status': task.get_status_emoji(),
'is_finished': task.is_finished,
'failed_urls': task.failed_urls,
'ended_time': task.ended_time if task.is_finished else None,
})
else:
return JsonResponse()

View file

@ -499,6 +499,7 @@
</div>
</div>
<!-- contains relations, reports, and upload excel entry -->
<div class="relation-dropdown">
<div class="relation-dropdown__button">
<span class="icon-arrow">
@ -546,6 +547,113 @@
</div>
</div>
<!-- import douban data -->
{% if user == request.user %}
<div class="aside-section-wrapper aside-section-wrapper--transparent aside-section-wrapper--collapse">
<div class="import-panel">
<h5 class="import-panel__label">{% trans '导入豆瓣标记数据' %}</h5>
<span id="importHelp" class="import-panel__help">?</span>
<div class="import-panel__body">
<form action="{% url 'sync:douban' %}" method="POST" enctype="multipart/form-data" >
{% csrf_token %}
<input type="hidden" name="user" value="{{ request.user.id }}">
<span>{% trans '导入:' %}</span>
<div class="import-panel__checkbox">
<input type="checkbox" name="sync_book" id="syncBook">
<label for="syncBook">{% trans '书' %}</label>
</div>
<div class="import-panel__checkbox">
<input type="checkbox" name="sync_movie" id="syncMovie">
<label for="syncMovie">{% trans '电影' %}</label>
</div>
<div class="import-panel__checkbox">
<input type="checkbox" name="sync_music" id="syncMusic">
<label for="syncMusic">{% trans '音乐' %}</label>
</div>
<div class="import-panel__checkbox">
<input type="checkbox" name="sync_game" id="syncGame">
<label for="syncGame">{% trans '游戏' %}</label>
</div>
<div></div>
<span>{% trans '覆盖:' %}</span>
<div class="import-panel__checkbox import-panel__checkbox--last">
<input type="checkbox" name="overwrite" id="overwrite">
<label for="overwrite">{% trans '覆盖原有标记' %}</label>
</div>
<span id="overwriteHelp" class="import-panel__help">?</span>
<div></div>
<span>{% trans '可见性:' %}</span>
<div class="import-panel__checkbox import-panel__checkbox--last">
<input type="checkbox" name="default_public" id="visibility">
<label for="visibility">{% trans '公开' %}</label>
</div>
<span id="visibilityHelp" class="import-panel__help">?</span>
<div></div>
<div class="import-panel__file-input">
<input type="file" name="xlsx" id="excelFile" required accept=".xlsx">
</div>
<input type="submit" class="import-panel__button" value="{% trans '导入' %}" id="uploadBtn">
</form>
<div class="import-panel__progress"
{% if latest_task.is_finished or latest_task is None %}
style="display: none;"
{% endif %}
>
<label for="importProgress">{% trans '进度' %}</label>
<progress id="importProgress" value="{{ latest_task.finished_items }}" max="{{ latest_task.total_items }}"></progress>
<span class="float-right" id="progressPercent">{{ latest_task.get_progress | floatformat:"0" }}%</span>
<span class="clearfix"></span>
</div>
<div class="import-panel__last-task"
{% if not latest_task.is_finished %}`
style="display: none;"
{% endif %}
>
{% trans '上次导入:' %}
<span class="index">{% trans '总数' %} <span id="lastTaskTotalItems">{{ latest_task.total_items }}</span></span>
<span class="index">{% trans '同步' %} <span id="lastTaskSuccessItems">{{ latest_task.success_items }}</span></span>
<span class="index">{% trans '状态' %} <span id="lastTaskStatus">{{ latest_task.get_status_emoji }}</span></span>
<div class="import-panel__fail-urls"
{% if not latest_task.failed_urls %}
style="display: none;"
{% endif %}
>
<span>
{% trans '失败条目链接' %}
</span>
<a class="float-right" style="cursor: pointer;" id="failedUrlsBtn">
</a>
<script>
$("#failedUrlsBtn").data("collapse", true);
$("#failedUrlsBtn").click(()=>{
const btn = $("#failedUrlsBtn");
if(btn.data("collapse") == true) {
btn.data("collapse", false);
btn.text("▼");
$("#failedUrls").show();
} else {
btn.data("collapse", true);
btn.text("▶");
$("#failedUrls").hide();
}
});
</script>
<span class="clearfix"></span>
<ul id="failedUrls" style="display: none;">
{% for url in latest_task.failed_urls %}
<li>{{ url }}</li>
{% endfor %}
</ul>
</div>
</div>
</div>
</div>
</div>
{% endif %}
<div
class="aside-section-wrapper aside-section-wrapper--transparent aside-section-wrapper--collapse">
{% if request.user.is_staff and request.user == user%}
@ -553,19 +661,21 @@
<h5 class="report-panel__label">{% trans '举报信息' %}</h5>
<a class="report-panel__all-link"
href="{% url 'users:manage_report' %}">全部举报</a>
<ul class="report-panel__report-list">
{% for report in reports %}
<li class="report-panel__report">
<a href="{% url 'users:home' report.submit_user.id %}"
class="report-panel__user-link">{{ report.submit_user }}</a>{% trans '举报了' %}<a
href="{% url 'users:home' report.reported_user.id %}"
class="report-panel__user-link">{{ report.reported_user }}</a>
</li>
{% empty %}
<div>{% trans '暂无新举报' %}</div>
{% endfor %}
</ul>
<div class="report-panel__body">
<ul class="report-panel__report-list">
{% for report in reports %}
<li class="report-panel__report">
<a href="{% url 'users:home' report.submit_user.id %}"
class="report-panel__user-link">{{ report.submit_user }}</a>{% trans '举报了' %}<a
href="{% url 'users:home' report.reported_user.id %}"
class="report-panel__user-link">{{ report.reported_user }}</a>
</li>
{% empty %}
<div>{% trans '暂无新举报' %}</div>
{% endfor %}
</ul>
</div>
</div>
{% endif %}
</div>
@ -583,6 +693,8 @@
<div id="oauth2Token" hidden="true">{% oauth_token %}</div>
<div id="mastodonURI" hidden="true">{% mastodon request.user.mastodon_site %}</div>
<div id="queryProgressURL" data-url="{% url 'sync:progress' %}"></div>
<div id="querySyncInfoURL" data-url="{% url 'sync:last' %}"></div>
<!--current user mastodon id-->
{% if user == request.user %}
@ -669,6 +781,32 @@
});
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/2.9.2/umd/popper.min.js"
integrity="sha512-2rNj2KJ+D8s1ceNasTIex6z4HWyOnEYLVC3FigGOmyQCZc2eBXKgOxQmo3oKLHyfcj53uz4QMsRCWNbLd32Q1g=="
crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/tippy.js/6.3.1/tippy.umd.min.js"
integrity="sha512-Ns7w8bjVjVcBVa+k3XLt0ObfsG2LQfr573HoIYtC4wh8gUKLvCx+rlggxfvsHqup6jvMAEmBtYXmhcKHL+6R5A=="
crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
tippy('#importHelp', {
content: "{% trans '上传导入由<a href=\"https://github.com/doufen-org/tofu\" target=\"_blank\">豆伴</a>豆坟导出的Excel文件<strong>请勿手动修改该文件</strong>。部分条目由于需要登陆无法自动同步。' %}",
interactive: true,
allowHTML: true,
duration: 0,
});
tippy('#overwriteHelp', {
content: "{% trans '在导入之前如果已经在本站标记了某一个条目,是否使用来自豆瓣的标记覆盖原有的。' %}",
interactive: true,
allowHTML: true,
duration: 0,
});
tippy('#visibilityHelp', {
content: "{% trans '所同步的标记可见性是否为公开;对于已有的标记即便覆盖也不会改变可见性。' %}",
interactive: true,
allowHTML: true,
duration: 0,
});
</script>
</body>

View file

@ -195,6 +195,8 @@ def home(request, id):
song_marks = request.user.user_songmarks.all()
game_marks = request.user.user_gamemarks.all()
latest_task = user.user_synctasks.order_by("-id").first()
# visit other's home page
else:
# no these value on other's home page
@ -271,6 +273,7 @@ def home(request, id):
'layout': layout,
'reports': reports,
'unread_announcements': unread_announcements,
'latest_task': latest_task,
}
)
else: