review url mounts for deprecated modules.

catalog/book/models.py

@@ -90,6 +90,10 @@ class Edition(Item):
                 # if not work:
                 #     _logger.info(f'Unable to find link for {w["url"]}')
 
+    def get_related_books(self):
+        # TODO
+        return []
+
 
 class Work(Item):
     category = ItemCategory.Book

catalog/models.py

@@ -70,6 +70,8 @@ def all_categories():
 
 
 def init_catalog_search_models():
+    if settings.DISABLE_MODEL_SIGNAL:
+        return
     Indexer.update_model_indexable(Edition)
     Indexer.update_model_indexable(Work)
     Indexer.update_model_indexable(Movie)

catalog/search/external.py

@@ -0,0 +1,259 @@
+from urllib.parse import quote_plus
+from django.conf import settings
+from catalog.common import *
+from catalog.models import *
+from catalog.sites.spotify import get_spotify_token
+import requests
+from lxml import html
+import logging
+
+SEARCH_PAGE_SIZE = 5  # not all apis support page size
+logger = logging.getLogger(__name__)
+
+
+class SearchResultItem:
+    def __init__(
+        self, category, source_site, source_url, title, subtitle, brief, cover_url
+    ):
+        self.category = category
+        self.source_site = source_site
+        self.source_url = source_url
+        self.title = title
+        self.subtitle = subtitle
+        self.brief = brief
+        self.cover_url = cover_url
+
+    @property
+    def verbose_category_name(self):
+        return self.category.label
+
+    @property
+    def link(self):
+        return f"/search?q={quote_plus(self.source_url)}"
+
+    @property
+    def scraped(self):
+        return False
+
+
+class ProxiedRequest:
+    @classmethod
+    def get(cls, url):
+        u = f"http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={quote_plus(url)}"
+        return requests.get(u, timeout=10)
+
+
+class Goodreads:
+    @classmethod
+    def search(cls, q, page=1):
+        results = []
+        try:
+            search_url = (
+                f"https://www.goodreads.com/search?page={page}&q={quote_plus(q)}"
+            )
+            r = requests.get(search_url)
+            if r.url.startswith("https://www.goodreads.com/book/show/"):
+                # Goodreads will 302 if only one result matches ISBN
+                res = SiteManager.get_site_by_url(r.url).get_resource_ready()
+                subtitle = f"{res.metadata['pub_year']} {', '.join(res.metadata['author'])} {', '.join(res.metadata['translator'] if res.metadata['translator'] else [])}"
+                results.append(
+                    SearchResultItem(
+                        ItemCategory.Book,
+                        SiteName.Goodreads,
+                        res.url,
+                        res.metadata["title"],
+                        subtitle,
+                        res.metadata["brief"],
+                        res.metadata["cover_image_url"],
+                    )
+                )
+            else:
+                h = html.fromstring(r.content.decode("utf-8"))
+                for c in h.xpath('//tr[@itemtype="http://schema.org/Book"]'):
+                    el_cover = c.xpath('.//img[@class="bookCover"]/@src')
+                    cover = el_cover[0] if el_cover else None
+                    el_title = c.xpath('.//a[@class="bookTitle"]//text()')
+                    title = "".join(el_title).strip() if el_title else None
+                    el_url = c.xpath('.//a[@class="bookTitle"]/@href')
+                    url = "https://www.goodreads.com" + el_url[0] if el_url else None
+                    el_authors = c.xpath('.//a[@class="authorName"]//text()')
+                    subtitle = ", ".join(el_authors) if el_authors else None
+                    results.append(
+                        SearchResultItem(
+                            ItemCategory.Book,
+                            SiteName.Goodreads,
+                            url,
+                            title,
+                            subtitle,
+                            "",
+                            cover,
+                        )
+                    )
+        except Exception as e:
+            logger.error(f"Goodreads search '{q}' error: {e}")
+        return results
+
+
+class GoogleBooks:
+    @classmethod
+    def search(cls, q, page=1):
+        results = []
+        try:
+            api_url = f"https://www.googleapis.com/books/v1/volumes?country=us&q={quote_plus(q)}&startIndex={SEARCH_PAGE_SIZE*(page-1)}&maxResults={SEARCH_PAGE_SIZE}&maxAllowedMaturityRating=MATURE"
+            j = requests.get(api_url).json()
+            if "items" in j:
+                for b in j["items"]:
+                    if "title" not in b["volumeInfo"]:
+                        continue
+                    title = b["volumeInfo"]["title"]
+                    subtitle = ""
+                    if "publishedDate" in b["volumeInfo"]:
+                        subtitle += b["volumeInfo"]["publishedDate"] + " "
+                    if "authors" in b["volumeInfo"]:
+                        subtitle += ", ".join(b["volumeInfo"]["authors"])
+                    if "description" in b["volumeInfo"]:
+                        brief = b["volumeInfo"]["description"]
+                    elif "textSnippet" in b["volumeInfo"]:
+                        brief = b["volumeInfo"]["textSnippet"]["searchInfo"]
+                    else:
+                        brief = ""
+                    category = ItemCategory.Book
+                    # b['volumeInfo']['infoLink'].replace('http:', 'https:')
+                    url = "https://books.google.com/books?id=" + b["id"]
+                    cover = (
+                        b["volumeInfo"]["imageLinks"]["thumbnail"]
+                        if "imageLinks" in b["volumeInfo"]
+                        else None
+                    )
+                    results.append(
+                        SearchResultItem(
+                            category,
+                            SiteName.GoogleBooks,
+                            url,
+                            title,
+                            subtitle,
+                            brief,
+                            cover,
+                        )
+                    )
+        except Exception as e:
+            logger.error(f"GoogleBooks search '{q}' error: {e}")
+        return results
+
+
+class TheMovieDatabase:
+    @classmethod
+    def search(cls, q, page=1):
+        results = []
+        try:
+            api_url = f"https://api.themoviedb.org/3/search/multi?query={quote_plus(q)}&page={page}&api_key={settings.TMDB_API3_KEY}&language=zh-CN&include_adult=true"
+            j = requests.get(api_url).json()
+            for m in j["results"]:
+                if m["media_type"] in ["tv", "movie"]:
+                    url = f"https://www.themoviedb.org/{m['media_type']}/{m['id']}"
+                    if m["media_type"] == "tv":
+                        cat = ItemCategory.TV
+                        title = m["name"]
+                        subtitle = f"{m.get('first_air_date')} {m.get('original_name')}"
+                    else:
+                        cat = ItemCategory.Movie
+                        title = m["title"]
+                        subtitle = f"{m.get('release_date')} {m.get('original_name')}"
+                    cover = f"https://image.tmdb.org/t/p/w500/{m.get('poster_path')}"
+                    results.append(
+                        SearchResultItem(
+                            cat,
+                            SiteName.TMDB,
+                            url,
+                            title,
+                            subtitle,
+                            m.get("overview"),
+                            cover,
+                        )
+                    )
+        except Exception as e:
+            logger.error(f"TMDb search '{q}' error: {e}")
+        return results
+
+
+class Spotify:
+    @classmethod
+    def search(cls, q, page=1):
+        results = []
+        try:
+            api_url = f"https://api.spotify.com/v1/search?q={q}&type=album&limit={SEARCH_PAGE_SIZE}&offset={page*SEARCH_PAGE_SIZE}"
+            headers = {"Authorization": f"Bearer {get_spotify_token()}"}
+            j = requests.get(api_url, headers=headers).json()
+            for a in j["albums"]["items"]:
+                title = a["name"]
+                subtitle = a["release_date"]
+                for artist in a["artists"]:
+                    subtitle += " " + artist["name"]
+                url = a["external_urls"]["spotify"]
+                cover = a["images"][0]["url"]
+                results.append(
+                    SearchResultItem(
+                        ItemCategory.Music,
+                        SiteName.Spotify,
+                        url,
+                        title,
+                        subtitle,
+                        "",
+                        cover,
+                    )
+                )
+        except Exception as e:
+            logger.error(f"Spotify search '{q}' error: {e}")
+        return results
+
+
+class Bandcamp:
+    @classmethod
+    def search(cls, q, page=1):
+        results = []
+        try:
+            search_url = f"https://bandcamp.com/search?from=results&item_type=a&page={page}&q={quote_plus(q)}"
+            r = requests.get(search_url)
+            h = html.fromstring(r.content.decode("utf-8"))
+            for c in h.xpath('//li[@class="searchresult data-search"]'):
+                el_cover = c.xpath('.//div[@class="art"]/img/@src')
+                cover = el_cover[0] if el_cover else None
+                el_title = c.xpath('.//div[@class="heading"]//text()')
+                title = "".join(el_title).strip() if el_title else None
+                el_url = c.xpath('..//div[@class="itemurl"]/a/@href')
+                url = el_url[0] if el_url else None
+                el_authors = c.xpath('.//div[@class="subhead"]//text()')
+                subtitle = ", ".join(el_authors) if el_authors else None
+                results.append(
+                    SearchResultItem(
+                        ItemCategory.Music,
+                        SiteName.Bandcamp,
+                        url,
+                        title,
+                        subtitle,
+                        "",
+                        cover,
+                    )
+                )
+        except Exception as e:
+            logger.error(f"Goodreads search '{q}' error: {e}")
+        return results
+
+
+class ExternalSources:
+    @classmethod
+    def search(cls, c, q, page=1):
+        if not q:
+            return []
+        results = []
+        if c == "" or c is None:
+            c = "all"
+        if c == "all" or c == "movie":
+            results.extend(TheMovieDatabase.search(q, page))
+        if c == "all" or c == "book":
+            results.extend(GoogleBooks.search(q, page))
+            results.extend(Goodreads.search(q, page))
+        if c == "all" or c == "music":
+            results.extend(Spotify.search(q, page))
+            results.extend(Bandcamp.search(q, page))
+        return results

catalog/search/typesense.py

@@ -4,7 +4,7 @@ import typesense
 from typesense.exceptions import ObjectNotFound
 from django.conf import settings
 from django.db.models.signals import post_save, post_delete
-
+from catalog.models import Item
 
 INDEX_NAME = "catalog"
 SEARCHABLE_ATTRIBUTES = [
@@ -257,7 +257,8 @@ class Indexer:
     @classmethod
     def item_to_obj(cls, item):
         try:
-            return cls.class_map[item["class_name"]].get_by_url(item["id"])
+            return Item.get_by_url(item["id"])
         except Exception as e:
+            print(e)
             logger.error(f"unable to load search result item from db:\n{item}")
             return None

catalog/templates/album.html

@@ -97,13 +97,13 @@
 	</div>
 
 	{% if item.last_editor and item.last_editor.preference.show_last_edit %}
-	<div>{% trans '最近编辑者：' %}<a href="{% url 'users:home' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
+	<div>{% trans '最近编辑者：' %}<a href="{% url 'journal:user_profile' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
 	{% endif %}
 
 	<div>
-		<a href="{% url 'music:update_album' item.id %}">{% trans '编辑这个作品' %}</a>
+		<a href="{% url 'catalog:edit' item.url_path item.uuid %}">{% trans '编辑这张专辑' %}</a>
 		{% if user.is_staff %}
-		/<a href="{% url 'music:delete_album' item.id %}"> {% trans '删除' %}</a>
+		/<a href="{% url 'catalog:delete' item.url_path item.uuid %}"> {% trans '删除' %}</a>
 		{% endif %}
 	</div>
 </div>

catalog/templates/edition.html

@@ -63,13 +63,13 @@
 
 
 	{% if item.last_editor and item.last_editor.preference.show_last_edit %}
-	<div>{% trans '最近编辑者：' %}<a href="{% url 'users:home' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
+	<div>{% trans '最近编辑者：' %}<a href="{% url 'journal:user_profile' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
 	{% endif %}
 
 	<div>
-		<a href="{% url 'books:update' item.id %}">{% trans '编辑这本书' %}</a>
+		<a href="{% url 'catalog:edit' item.url_path item.uuid %}">{% trans '编辑这本书' %}</a>
 		{% if user.is_staff %}
-		/<a href="{% url 'books:delete' item.id %}"> {% trans '删除' %}</a>
+		/<a href="{% url 'catalog:delete' item.url_path item.uuid %}"> {% trans '删除' %}</a>
 		{% endif %}
 	</div>
 </div>
@@ -84,7 +84,7 @@
 		<div >
 			{% for b in item.get_related_books %}
 			<p>
-				<a href="{% url 'books:retrieve' b.id %}">{{ b.title }}</a>
+				<a href="{{ b.url }}">{{ b.title }}</a>
 				<small>({{ b.pub_house }} {{ b.pub_year }})</small>
 				<span class="source-label source-label__{{ b.source_site }}">{{ b.get_source_site_display }}</span>
 			</p>

catalog/templates/external_search_results.html

@@ -13,7 +13,9 @@
 <li class="entity-list__entity">
     <div class="entity-list__entity-img-wrapper">
         <a href="{{ item.link }}">
+            {% if item.cover_url %}
             <img src="{{ item.cover_url }}" alt="" class="entity-list__entity-img">
+            {% endif %}
         </a>
     </div>
     <div class="entity-list__entity-text">

catalog/templates/game.html

@@ -92,14 +92,13 @@
 
 
 	{% if item.last_editor and item.last_editor.preference.show_last_edit  %}
-	<div>{% trans '最近编辑者：' %}<a href="{% url 'users:home' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
+	<div>{% trans '最近编辑者：' %}<a href="{% url 'journal:user_profile' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
 	{% endif %}
 
 	<div>
-		<a href="{% url 'games:update' item.id %}">{% trans '编辑这个游戏' %}</a>
-
+		<a href="{% url 'catalog:edit' item.url_path item.uuid %}">{% trans '编辑这个游戏' %}</a>
 		{% if user.is_staff %}
-		/<a href="{% url 'games:delete' item.id %}"> {% trans '删除' %}</a>
+		/<a href="{% url 'catalog:delete' item.url_path item.uuid %}"> {% trans '删除' %}</a>
 		{% endif %}
 	</div>
 </div>

catalog/templates/item_base.html

@@ -79,7 +79,7 @@
                   <div class="tag-collection">
                     {% for tag in item.tags %}
                     <span class="tag-collection__tag">
-                      <a href="{% url 'common:search' %}?tag={{ tag }}">{{ tag }}</a>
+                      <a href="{% url 'catalog:search' %}?tag={{ tag }}">{{ tag }}</a>
                     </span>
                     {% endfor %}
                   </div>
@@ -127,7 +127,7 @@
                 <ul class="entity-marks__mark-list">
                 {% for others_mark in mark_list %}
                 <li class="entity-marks__mark">
-                    <a href="{% url 'users:home' others_mark.owner.mastodon_username %}" class="entity-marks__owner-link">{{ others_mark.owner.username }}</a>
+                    <a href="{% url 'journal:user_profile' others_mark.owner.mastodon_username %}" class="entity-marks__owner-link">{{ others_mark.owner.username }}</a>
 
                     <span>{{ others_mark.shelf_label }}</span>
 
@@ -165,7 +165,7 @@
                 <ul class="entity-reviews__review-list">
                   {% for others_review in review_list %}
                   <li class="entity-reviews__review">
-                    <a href="{% url 'users:home' others_review.owner.mastodon_username %}" class="entity-reviews__owner-link">{{ others_review.owner.username }}</a>
+                    <a href="{% url 'journal:user_profile' others_review.owner.mastodon_username %}" class="entity-reviews__owner-link">{{ others_review.owner.username }}</a>
                     {% if others_review.visibility > 0 %}
                     <span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path d="M17,8.48h-.73V6.27a6.27,6.27,0,1,0-12.53,0V8.48H3a.67.67,0,0,0-.67.67V19.33A.67.67,0,0,0,3,20H17a.67.67,0,0,0,.67-.67V9.15A.67.67,0,0,0,17,8.48ZM6.42,6.27h0a3.57,3.57,0,0,1,7.14,0h0V8.48H6.42Z"/></svg></span>
                     {% endif %}

catalog/templates/item_mark_list.html

@@ -32,7 +32,7 @@
                 <ul class="entity-marks__mark-list">
                   {% for others_mark in marks %}
                   <li class="entity-marks__mark">
-                    <a href="{% url 'users:home' others_mark.owner.mastodon_username %}" class="entity-marks__owner-link">{{ others_mark.owner.username }}</a>
+                    <a href="{% url 'journal:user_profile' others_mark.owner.mastodon_username %}" class="entity-marks__owner-link">{{ others_mark.owner.username }}</a>
 
                     <span>{{ others_mark.mark.get_status_display }}</span>
 

catalog/templates/item_review_list.html

@@ -32,7 +32,7 @@
 
                   {% for review in reviews %}
                   <li class="entity-reviews__review entity-reviews__review--wider">
-                    <a href="{% url 'users:home' review.owner.mastodon_username %}" class="entity-reviews__owner-link">{{ review.owner.username }}</a>
+                    <a href="{% url 'journal:user_profile' review.owner.mastodon_username %}" class="entity-reviews__owner-link">{{ review.owner.username }}</a>
                     {% if review.visibility > 0 %}
                     <span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path d="M17,8.48h-.73V6.27a6.27,6.27,0,1,0-12.53,0V8.48H3a.67.67,0,0,0-.67.67V19.33A.67.67,0,0,0,3,20H17a.67.67,0,0,0,.67-.67V9.15A.67.67,0,0,0,17,8.48ZM6.42,6.27h0a3.57,3.57,0,0,1,7.14,0h0V8.48H6.42Z"/></svg></span>
                     {% endif %}

catalog/templates/movie.html

@@ -156,13 +156,13 @@
 
 
 	{% if item.last_editor and item.last_editor.preference.show_last_edit  %}
-	<div>{% trans '最近编辑者：' %}<a href="{% url 'users:home' 'fixme' %}">{{ item.last_editor | default:"" }}</a></div>
+	<div>{% trans '最近编辑者：' %}<a href="{% url 'journal:user_profile' 'fixme' %}">{{ item.last_editor | default:"" }}</a></div>
 	{% endif %}
 
 	<div>
-		<a href="{% url 'movies:update' item.id %}">{% trans '编辑这部电影' %}</a>
+		<a href="{% url 'catalog:edit' item.url_path item.uuid %}">{% trans '编辑这部电影' %}</a>
 		{% if user.is_staff %}
-		/<a href="{% url 'movies:delete' item.id %}"> {% trans '删除' %}</a>
+		/<a href="{% url 'catalog:delete' item.url_path item.uuid %}"> {% trans '删除' %}</a>
 		{% endif %}
 	</div>
 </div>

catalog/templates/search_results.html

@@ -54,7 +54,7 @@
                                     </li>
                                     {% endfor %}
                                     {% if request.GET.q and user.is_authenticated %}
-                                    <li class="entity-list__entity" hx-get="{% url 'common:external_search' %}?q={{ request.GET.q }}&c={{ request.GET.c }}&page={% if pagination.current_page %}{{ pagination.current_page }}{% else %}1{% endif %}" hx-trigger="load" hx-swap="outerHTML">
+                                    <li class="entity-list__entity" hx-get="{% url 'catalog:external_search' %}?q={{ request.GET.q }}&c={{ request.GET.c }}&page={% if pagination.current_page %}{{ pagination.current_page }}{% else %}1{% endif %}" hx-trigger="load" hx-swap="outerHTML">
                                     {% trans '正在实时搜索站外条目' %}
                                     <div id="spinner">
                                         <div class="spinner">
@@ -111,117 +111,22 @@
                                     <div class="add-entity-entries__label">
                                         {% trans '没有想要的结果？' %}
                                     </div>
-                                    {% if request.GET.c and request.GET.c in categories %}
-                                    
-                                        {% if request.GET.c|lower == 'book' %}
-                                            
-                                        <a href="{% url 'books:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加书' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'movie' %}
-                                        
-                                        <a href="{% url 'movies:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加电影/剧集' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'music' %}
-                                        
-                                        <a href="{% url 'music:create_album' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加专辑' %}</button>
-                                        </a>
-                                        <a href="{% url 'music:create_song' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加单曲' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'game' %}
-                                        
-                                        <a href="{% url 'games:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加游戏' %}</button>
-                                        </a>
-                                        
-                                        {% endif %}
-                                        
-                                    {% else %}
-                                        <a href="{% url 'books:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加书' %}</button>
-                                        </a>
-                                        <a href="{% url 'movies:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加电影/剧集' %}</button>
-                                        </a>
-                                        <a href="{% url 'music:create_album' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加专辑' %}</button>
-                                        </a>
-                                        <a href="{% url 'music:create_song' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加单曲' %}</button>
-                                        </a>
-                                        <a href="{% url 'games:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加游戏' %}</button>
-                                        </a>
-                                    {% endif %}
+                                    <a href="#">
+                                        <button class="add-entity-entries__button">{% trans '添加书' %}</button>
+                                    </a>
+                                    <a href="#">
+                                        <button class="add-entity-entries__button">{% trans '添加电影' %}</button>
+                                    </a>
+                                    <a href="#">
+                                        <button class="add-entity-entries__button">{% trans '添加剧集' %}</button>
+                                    </a>
+                                    <a href="#">
+                                        <button class="add-entity-entries__button">{% trans '添加专辑' %}</button>
+                                    </a>
+                                    <a href="#">
+                                        <button class="add-entity-entries__button">{% trans '添加游戏' %}</button>
+                                    </a>
                                 </div>
-                                <!-- div class="add-entity-entries__entry">
-                                    {% if request.GET.c and request.GET.c in categories %}
-                                    
-                                        {% if request.GET.c|lower == 'book' %}
-
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'books:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'movie' %}
-                                        
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'movies:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'game' %}
-                                        
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'games:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'music' %}
-                                        
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'music:scrape_album' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                                                                
-                                        {% endif %}
-                                        
-                                    {% else %}
-                                        
-                                    <div class="add-entity-entries__label">
-                                        {% trans '或从表瓣剽取' %}
-                                    </div>
-                                    <a href="{% url 'books:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '书' %}</button>
-                                    </a>
-                                    <a href="{% url 'movies:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '电影/剧集' %}</button>
-                                    </a>
-                                    <a href="{% url 'music:scrape_album' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '专辑' %}</button>
-                                    </a>
-                                    <a href="{% url 'games:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '游戏' %}</button>
-                                    </a>
-
-                                    {% endif %}
-                                </div -->
-
                             </div>
 
                         </div>

catalog/templates/tvseason.html

@@ -156,13 +156,13 @@
 
 
 	{% if item.last_editor and item.last_editor.preference.show_last_edit  %}
-	<div>{% trans '最近编辑者：' %}<a href="{% url 'users:home' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
+	<div>{% trans '最近编辑者：' %}<a href="{% url 'journal:user_profile' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
 	{% endif %}
 
 	<div>
-		<a href="{% url 'movies:update' item.id %}">{% trans '编辑这部电影' %}</a>
+		<a href="{% url 'catalog:edit' item.url_path item.uuid %}">{% trans '编辑这部剧集' %}</a>
 		{% if user.is_staff %}
-		/<a href="{% url 'movies:delete' item.id %}"> {% trans '删除' %}</a>
+		/<a href="{% url 'catalog:delete' item.url_path item.uuid %}"> {% trans '删除' %}</a>
 		{% endif %}
 	</div>
 </div>

catalog/templates/tvshow.html

@@ -156,13 +156,13 @@
 
 
 	{% if item.last_editor and item.last_editor.preference.show_last_edit %}
-	<div>{% trans '最近编辑者：' %}<a href="{% url 'users:home' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
+	<div>{% trans '最近编辑者：' %}<a href="{% url 'journal:user_profile' item.last_editor.mastodon_username %}">{{ item.last_editor | default:"" }}</a></div>
 	{% endif %}
 
 	<div>
-		<a href="{% url 'movies:update' item.id %}">{% trans '编辑这部电影' %}</a>
+		<a href="{% url 'catalog:edit' item.url_path item.uuid %}">{% trans '编辑这部剧集' %}</a>
 		{% if user.is_staff %}
-		/<a href="{% url 'movies:delete' item.id %}"> {% trans '删除' %}</a>
+		/<a href="{% url 'catalog:delete' item.url_path item.uuid %}"> {% trans '删除' %}</a>
 		{% endif %}
 	</div>
 </div>

catalog/urls.py

@@ -29,6 +29,20 @@ urlpatterns = [
         retrieve,
         name="retrieve",
     ),
+    re_path(
+        r"^(?P<item_path>"
+        + _get_all_url_paths()
+        + ")/(?P<item_uuid>[A-Za-z0-9]{21,22})/edit$",
+        edit,
+        name="edit",
+    ),
+    re_path(
+        r"^(?P<item_path>"
+        + _get_all_url_paths()
+        + ")/(?P<item_uuid>[A-Za-z0-9]{21,22})/delete$",
+        delete,
+        name="delete",
+    ),
     re_path(
         r"^(?P<item_path>"
         + _get_all_url_paths()
@@ -43,7 +57,8 @@ urlpatterns = [
         mark_list,
         name="mark_list",
     ),
-    path("search2/", search, name="search"),
+    path("search/", search, name="search"),
+    path("search/external/", external_search, name="external_search"),
     path("fetch_refresh/<str:job_id>", fetch_refresh, name="fetch_refresh"),
     path("api/", api.urls),
 ]

catalog/views.py

@@ -1,6 +1,6 @@
 import uuid
 import logging
-from django.shortcuts import render, get_object_or_404, redirect, reverse
+from django.shortcuts import render, get_object_or_404, redirect
 from django.contrib.auth.decorators import login_required, permission_required
 from django.utils.translation import gettext_lazy as _
 from django.http import (
@@ -8,8 +8,10 @@ from django.http import (
     HttpResponseServerError,
     HttpResponse,
     HttpResponseRedirect,
+    HttpResponseNotFound,
 )
-from django.core.exceptions import BadRequest, ObjectDoesNotExist, PermissionDenied
+from django.contrib.auth.decorators import login_required, permission_required
+from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
 from django.db import IntegrityError, transaction
 from django.db.models import Count
 from django.utils import timezone
@@ -21,15 +23,15 @@ from mastodon.models import MastodonApplication
 from mastodon.api import share_mark, share_review
 from .models import *
 from django.conf import settings
-from common.scraper import get_scraper_by_url, get_normalized_url
 from django.utils.baseconv import base62
 from journal.models import Mark, ShelfMember, Review
 from journal.models import query_visible, query_following
 from common.utils import PageLinksGenerator
-from common.views import PAGE_LINK_NUMBER
+from common.config import PAGE_LINK_NUMBER
 from journal.models import ShelfTypeNames
 import django_rq
 from rq.job import Job
+from .search.external import ExternalSources
 
 _logger = logging.getLogger(__name__)
 
@@ -107,10 +109,11 @@ def retrieve(request, item_path, item_uuid):
         return HttpResponseBadRequest()
 
 
+@login_required
 def mark_list(request, item_path, item_uuid, following_only=False):
     item = get_object_or_404(Item, uid=base62.decode(item_uuid))
     if not item:
-        return HttpResponseNotFound("item not found")
+        return HttpResponseNotFound(b"item not found")
     queryset = ShelfMember.objects.filter(item=item).order_by("-created_time")
     if following_only:
         queryset = queryset.filter(query_following(request.user))
@@ -135,7 +138,7 @@ def mark_list(request, item_path, item_uuid, following_only=False):
 def review_list(request, item_path, item_uuid):
     item = get_object_or_404(Item, uid=base62.decode(item_uuid))
     if not item:
-        return HttpResponseNotFound("item not found")
+        return HttpResponseNotFound(b"item not found")
     queryset = Review.objects.filter(item=item).order_by("-created_time")
     queryset = queryset.filter(query_visible(request.user))
     paginator = Paginator(queryset, NUM_REVIEWS_ON_LIST_PAGE)
@@ -164,6 +167,7 @@ def fetch_task(url):
         return "-"
 
 
+@login_required
 def fetch_refresh(request, job_id):
     retry = request.GET
     job = Job.fetch(id=job_id, connection=django_rq.get_connection("fetch"))
@@ -185,9 +189,10 @@ def fetch_refresh(request, job_id):
             )
 
 
+@login_required
 def fetch(request, url, site: AbstractSite = None):
     if not site:
-        site = SiteManager.get_site_by_url(keywords)
+        site = SiteManager.get_site_by_url(url)
         if not site:
             return HttpResponseBadRequest()
     item = site.get_item()
@@ -250,13 +255,13 @@ def search(request):
             items.append(i)
         for res in i.external_resources.all():
             urls.append(res.url)
-    if request.path.endswith(".json/"):
-        return JsonResponse(
-            {
-                "num_pages": result.num_pages,
-                "items": list(map(lambda i: i.get_json(), items)),
-            }
-        )
+    # if request.path.endswith(".json/"):
+    #     return JsonResponse(
+    #         {
+    #             "num_pages": result.num_pages,
+    #             "items": list(map(lambda i: i.get_json(), items)),
+    #         }
+    #     )
     request.session["search_dedupe_urls"] = urls
     return render(
         request,
@@ -270,3 +275,33 @@ def search(request):
             "hide_category": category is not None,
         },
     )
+
+
+@login_required
+def external_search(request):
+    category = request.GET.get("c", default="all").strip().lower()
+    if category == "all":
+        category = None
+    keywords = request.GET.get("q", default="").strip()
+    page_number = int(request.GET.get("page", default=1))
+    items = ExternalSources.search(category, keywords, page_number) if keywords else []
+    dedupe_urls = request.session.get("search_dedupe_urls", [])
+    items = [i for i in items if i.source_url not in dedupe_urls]
+
+    return render(
+        request,
+        "external_search_results.html",
+        {
+            "external_items": items,
+        },
+    )
+
+
+@login_required
+def edit(request, item_uuid):
+    return HttpResponseBadRequest()
+
+
+@login_required
+def delete(request, item_uuid):
+    return HttpResponseBadRequest()

common/importers/douban.py

@@ -1,270 +0,0 @@
-import openpyxl
-import requests
-import re
-from lxml import html
-from markdownify import markdownify as md
-from datetime import datetime
-from common.scraper import get_scraper_by_url
-import logging
-import pytz
-from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist
-from user_messages import api as msg
-import django_rq
-from common.utils import GenerateDateUUIDMediaFilePath
-import os
-from books.models import BookReview, Book, BookMark, BookTag
-from movies.models import MovieReview, Movie, MovieMark, MovieTag
-from music.models import AlbumReview, Album, AlbumMark, AlbumTag
-from games.models import GameReview, Game, GameMark, GameTag
-from common.scraper import DoubanAlbumScraper, DoubanBookScraper, DoubanGameScraper, DoubanMovieScraper
-from PIL import Image
-from io import BytesIO
-import filetype
-from common.models import MarkStatusEnum
-
-
-logger = logging.getLogger(__name__)
-tz_sh = pytz.timezone('Asia/Shanghai')
-
-
-def fetch_remote_image(url):
-    try:
-        print(f'fetching remote image {url}')
-        raw_img = None
-        ext = None
-        if settings.SCRAPESTACK_KEY is not None:
-            dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}'
-        elif settings.SCRAPERAPI_KEY is not None:
-            dl_url = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}'
-        else:
-            dl_url = url
-        img_response = requests.get(dl_url, timeout=settings.SCRAPING_TIMEOUT)
-        raw_img = img_response.content
-        img = Image.open(BytesIO(raw_img))
-        img.load()  # corrupted image will trigger exception
-        content_type = img_response.headers.get('Content-Type')
-        ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension
-        f = GenerateDateUUIDMediaFilePath(None, "x." + ext, settings.MARKDOWNX_MEDIA_PATH)
-        file = settings.MEDIA_ROOT + f
-        local_url = settings.MEDIA_URL + f
-        os.makedirs(os.path.dirname(file), exist_ok=True)
-        img.save(file)
-        # print(f'remote image saved as {local_url}')
-        return local_url
-    except Exception:
-        print(f'unable to fetch remote image {url}')
-        return url
-
-
-class DoubanImporter:
-    total = 0
-    processed = 0
-    skipped = 0
-    imported = 0
-    failed = []
-    user = None
-    visibility = 0
-    file = None
-
-    def __init__(self, user, visibility):
-        self.user = user
-        self.visibility = visibility
-
-    def update_user_import_status(self, status):
-        self.user.preference.import_status['douban_pending'] = status
-        self.user.preference.import_status['douban_file'] = self.file
-        self.user.preference.import_status['douban_visibility'] = self.visibility
-        self.user.preference.import_status['douban_total'] = self.total
-        self.user.preference.import_status['douban_processed'] = self.processed
-        self.user.preference.import_status['douban_skipped'] = self.skipped
-        self.user.preference.import_status['douban_imported'] = self.imported
-        self.user.preference.import_status['douban_failed'] = self.failed
-        self.user.preference.save(update_fields=['import_status'])
-
-    def import_from_file(self, uploaded_file):
-        try:
-            wb = openpyxl.open(uploaded_file, read_only=True, data_only=True, keep_links=False)
-            wb.close()
-            file = settings.MEDIA_ROOT + GenerateDateUUIDMediaFilePath(None, "x.xlsx", settings.SYNC_FILE_PATH_ROOT)
-            os.makedirs(os.path.dirname(file), exist_ok=True)
-            with open(file, 'wb') as destination:
-                for chunk in uploaded_file.chunks():
-                    destination.write(chunk)
-            self.file = file
-            self.update_user_import_status(2)
-            jid = f'Douban_{self.user.id}_{os.path.basename(self.file)}'
-            django_rq.get_queue('doufen').enqueue(self.import_from_file_task, job_id=jid)
-        except Exception:
-            return False
-        # self.import_from_file_task(file, user, visibility)
-        return True
-
-    mark_sheet_config = {
-        '想读': [MarkStatusEnum.WISH, DoubanBookScraper, Book, BookMark, BookTag],
-        '在读': [MarkStatusEnum.DO, DoubanBookScraper, Book, BookMark, BookTag],
-        '读过': [MarkStatusEnum.COLLECT, DoubanBookScraper, Book, BookMark, BookTag],
-        '想看': [MarkStatusEnum.WISH, DoubanMovieScraper, Movie, MovieMark, MovieTag],
-        '在看': [MarkStatusEnum.DO, DoubanMovieScraper, Movie, MovieMark, MovieTag],
-        '想看': [MarkStatusEnum.COLLECT, DoubanMovieScraper, Movie, MovieMark, MovieTag],
-        '想听': [MarkStatusEnum.WISH, DoubanAlbumScraper, Album, AlbumMark, AlbumTag],
-        '在听': [MarkStatusEnum.DO, DoubanAlbumScraper, Album, AlbumMark, AlbumTag],
-        '听过': [MarkStatusEnum.COLLECT, DoubanAlbumScraper, Album, AlbumMark, AlbumTag],
-        '想玩': [MarkStatusEnum.WISH, DoubanGameScraper, Game, GameMark, GameTag],
-        '在玩': [MarkStatusEnum.DO, DoubanGameScraper, Game, GameMark, GameTag],
-        '玩过': [MarkStatusEnum.COLLECT, DoubanGameScraper, Game, GameMark, GameTag],
-    }
-    review_sheet_config = {
-        '书评': [DoubanBookScraper, Book, BookReview],
-        '影评': [DoubanMovieScraper, Movie, MovieReview],
-        '乐评': [DoubanAlbumScraper, Album, AlbumReview],
-        '游戏评论&攻略': [DoubanGameScraper, Game, GameReview],
-    }
-    mark_data = {}
-    review_data = {}
-    entity_lookup = {}
-
-    def load_sheets(self):
-        f = open(self.file, 'rb')
-        wb = openpyxl.load_workbook(f, read_only=True, data_only=True, keep_links=False)
-        for data, config in [(self.mark_data, self.mark_sheet_config), (self.review_data, self.review_sheet_config)]:
-            for name in config:
-                data[name] = []
-                if name in wb:
-                    print(f'{self.user} parsing {name}')
-                    for row in wb[name].iter_rows(min_row=2, values_only=True):
-                        cells = [cell for cell in row]
-                        if len(cells) > 6:
-                            data[name].append(cells)
-        for sheet in self.mark_data.values():
-            for cells in sheet:
-                # entity_lookup["title|rating"] = [(url, time), ...]
-                k = f'{cells[0]}|{cells[5]}'
-                v = (cells[3], cells[4])
-                if k in self.entity_lookup:
-                    self.entity_lookup[k].append(v)
-                else:
-                    self.entity_lookup[k] = [v]
-        self.total = sum(map(lambda a: len(a), self.review_data.values()))
-
-    def guess_entity_url(self, title, rating, timestamp):
-        k = f'{title}|{rating}'
-        if k not in self.entity_lookup:
-            return None
-        v = self.entity_lookup[k]
-        if len(v) > 1:
-            v.sort(key=lambda c: abs(timestamp - (datetime.strptime(c[1], "%Y-%m-%d %H:%M:%S") if type(c[1])==str else c[1]).replace(tzinfo=tz_sh)))
-        return v[0][0]
-        # for sheet in self.mark_data.values():
-        #     for cells in sheet:
-        #         if cells[0] == title and cells[5] == rating:
-        #             return cells[3]
-
-    def import_from_file_task(self):
-        print(f'{self.user} import start')
-        msg.info(self.user, f'开始导入豆瓣评论')
-        self.update_user_import_status(1)
-        self.load_sheets()
-        print(f'{self.user} sheet loaded, {self.total} lines total')
-        self.update_user_import_status(1)
-        for name, param in self.review_sheet_config.items():
-            self.import_review_sheet(self.review_data[name], param[0], param[1], param[2])
-        self.update_user_import_status(0)
-        msg.success(self.user, f'豆瓣评论导入完成，共处理{self.total}篇，已存在{self.skipped}篇，新增{self.imported}篇。')
-        if len(self.failed):
-            msg.error(self.user, f'豆瓣评论导入时未能处理以下网址：\n{" , ".join(self.failed)}')
-
-    def import_review_sheet(self, worksheet, scraper, entity_class, review_class):
-        prefix = f'{self.user} |'
-        if worksheet is None:  # or worksheet.max_row < 2:
-            print(f'{prefix} {review_class.__name__} empty sheet')
-            return
-        for cells in worksheet:
-            if len(cells) < 6:
-                continue
-            title = cells[0]
-            entity_title = re.sub('^《', '', re.sub('》$', '', cells[1]))
-            review_url = cells[2]
-            time = cells[3]
-            rating = cells[4]
-            content = cells[6]
-            self.processed += 1
-            if time:
-                if type(time) == str:
-                    time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S")
-                time = time.replace(tzinfo=tz_sh)
-            else:
-                time = None
-            if not content:
-                content = ""
-            if not title:
-                title = ""
-            r = self.import_review(entity_title, rating, title, review_url, content, time, scraper, entity_class, review_class)
-            if r == 1:
-                self.imported += 1
-            elif r == 2:
-                self.skipped += 1
-            else:
-                self.failed.append(review_url)
-            self.update_user_import_status(1)
-
-    def import_review(self, entity_title, rating, title, review_url, content, time, scraper, entity_class, review_class):
-        # return 1: done / 2: skipped / None: failed
-        prefix = f'{self.user} |'
-        url = self.guess_entity_url(entity_title, rating, time)
-        if url is None:
-            print(f'{prefix} fetching {review_url}')
-            try:
-                if settings.SCRAPESTACK_KEY is not None:
-                    _review_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={review_url}'
-                else:
-                    _review_url = review_url
-                r = requests.get(_review_url, timeout=settings.SCRAPING_TIMEOUT)
-                if r.status_code != 200:
-                    print(f'{prefix} fetching error {review_url} {r.status_code}')
-                    return
-                h = html.fromstring(r.content.decode('utf-8'))
-                for u in h.xpath("//header[@class='main-hd']/a/@href"):
-                    if '.douban.com/subject/' in u:
-                        url = u
-                if not url:
-                    print(f'{prefix} fetching error {review_url} unable to locate entity url')
-                    return
-            except Exception:
-                print(f'{prefix} fetching exception {review_url}')
-                return
-        try:
-            entity = entity_class.objects.get(source_url=url)
-            print(f'{prefix} matched {url}')
-        except ObjectDoesNotExist:
-            try:
-                print(f'{prefix} scraping {url}')
-                scraper.scrape(url)
-                form = scraper.save(request_user=self.user)
-                entity = form.instance
-            except Exception as e:
-                print(f"{prefix} scrape failed: {url} {e}")
-                logger.error(f"{prefix} scrape failed: {url}", exc_info=e)
-                return
-        params = {
-            'owner': self.user,
-            entity_class.__name__.lower(): entity
-        }
-        if review_class.objects.filter(**params).exists():
-            return 2
-        content = re.sub(r'<span style="font-weight: bold;">([^<]+)</span>', r'<b>\1</b>', content)
-        content = re.sub(r'(<img [^>]+>)', r'\1<br>', content)
-        content = re.sub(r'<div class="image-caption">([^<]+)</div>', r'<br><i>\1</i><br>', content)
-        content = md(content)
-        content = re.sub(r'(?<=!\[\]\()([^)]+)(?=\))', lambda x: fetch_remote_image(x[1]), content)
-        params = {
-            'owner': self.user,
-            'created_time': time,
-            'edited_time': time,
-            'title': title,
-            'content': content,
-            'visibility': self.visibility,
-            entity_class.__name__.lower(): entity,
-        }
-        review_class.objects.create(**params)
-        return 1

common/importers/goodreads.py

@@ -1,202 +0,0 @@
-import re
-import requests
-from lxml import html
-from datetime import datetime
-# from common.scrapers.goodreads import GoodreadsScraper
-from common.scraper import get_scraper_by_url
-from books.models import Book, BookMark
-from collection.models import Collection
-from common.models import MarkStatusEnum
-from django.conf import settings
-from user_messages import api as msg
-import django_rq
-from django.utils.timezone import make_aware
-
-
-re_list = r'^https://www.goodreads.com/list/show/\d+'
-re_shelf = r'^https://www.goodreads.com/review/list/\d+[^?]*\?shelf=[^&]+'
-re_profile = r'^https://www.goodreads.com/user/show/(\d+)'
-gr_rating = {
-    'did not like it': 2,
-    'it was ok': 4,
-    'liked it': 6,
-    'really liked it': 8,
-    'it was amazing': 10
-}
-
-
-class GoodreadsImporter:
-    @classmethod
-    def import_from_url(self, raw_url, user):
-        match_list = re.match(re_list, raw_url)
-        match_shelf = re.match(re_shelf, raw_url)
-        match_profile = re.match(re_profile, raw_url)
-        if match_profile or match_shelf or match_list:
-            django_rq.get_queue('doufen').enqueue(self.import_from_url_task, raw_url, user)
-            return True
-        else:
-            return False
-
-    @classmethod
-    def import_from_url_task(cls, url, user):
-        match_list = re.match(re_list, url)
-        match_shelf = re.match(re_shelf, url)
-        match_profile = re.match(re_profile, url)
-        total = 0
-        if match_list or match_shelf:
-            shelf = cls.parse_shelf(match_shelf[0], user) if match_shelf else cls.parse_list(match_list[0], user)
-            if shelf['title'] and shelf['books']:
-                collection = Collection.objects.create(title=shelf['title'],
-                                                       description=shelf['description'] + '\n\nImported from [Goodreads](' + url + ')',
-                                                       owner=user)
-                for book in shelf['books']:
-                    collection.append_item(book['book'], book['review'])
-                    total += 1
-                collection.save()
-            msg.success(user, f'成功从Goodreads导入包含{total}本书的收藏单{shelf["title"]}。')
-        elif match_profile:
-            uid = match_profile[1]
-            shelves = {
-                MarkStatusEnum.WISH: f'https://www.goodreads.com/review/list/{uid}?shelf=to-read',
-                MarkStatusEnum.DO: f'https://www.goodreads.com/review/list/{uid}?shelf=currently-reading',
-                MarkStatusEnum.COLLECT: f'https://www.goodreads.com/review/list/{uid}?shelf=read',
-            }
-            for status in shelves:
-                shelf_url = shelves.get(status)
-                shelf = cls.parse_shelf(shelf_url, user)
-                for book in shelf['books']:
-                    params = {
-                        'owner': user,
-                        'rating': book['rating'],
-                        'text': book['review'],
-                        'status': status,
-                        'visibility': user.preference.default_visibility,
-                        'book': book['book'],
-                    }
-                    if book['last_updated']:
-                        params['created_time'] = book['last_updated']
-                        params['edited_time'] = book['last_updated']
-                    try:
-                        mark = BookMark.objects.create(**params)
-                        mark.book.update_rating(None, mark.rating)
-                    except Exception:
-                        print(f'Skip mark for {book["book"]}')
-                        pass
-                    total += 1
-            msg.success(user, f'成功从Goodreads用户主页导入{total}个标记。')
-
-    @classmethod
-    def parse_shelf(cls, url, user):  # return {'title': 'abc', books: [{'book': obj, 'rating': 10, 'review': 'txt'}, ...]}
-        title = None
-        books = []
-        url_shelf = url + '&view=table'
-        while url_shelf:
-            print(f'Shelf loading {url_shelf}')
-            r = requests.get(url_shelf, timeout=settings.SCRAPING_TIMEOUT)
-            if r.status_code != 200:
-                print(f'Shelf loading error {url_shelf}')
-                break
-            url_shelf = None
-            content = html.fromstring(r.content.decode('utf-8'))
-            title_elem = content.xpath("//span[@class='h1Shelf']/text()")
-            if not title_elem:
-                print(f'Shelf parsing error {url_shelf}')
-                break
-            title = title_elem[0].strip()
-            print("Shelf title: " + title)
-            for cell in content.xpath("//tbody[@id='booksBody']/tr"):
-                url_book = 'https://www.goodreads.com' + \
-                    cell.xpath(
-                        ".//td[@class='field title']//a/@href")[0].strip()
-                # has_review = cell.xpath(
-                #     ".//td[@class='field actions']//a/text()")[0].strip() == 'view (with text)'
-                rating_elem = cell.xpath(
-                    ".//td[@class='field rating']//span/@title")
-                rating = gr_rating.get(
-                    rating_elem[0].strip()) if rating_elem else None
-                url_review = 'https://www.goodreads.com' + \
-                    cell.xpath(
-                        ".//td[@class='field actions']//a/@href")[0].strip()
-                review = ''
-                last_updated = None
-                try:
-                    r2 = requests.get(
-                        url_review, timeout=settings.SCRAPING_TIMEOUT)
-                    if r2.status_code == 200:
-                        c2 = html.fromstring(r2.content.decode('utf-8'))
-                        review_elem = c2.xpath(
-                            "//div[@itemprop='reviewBody']/text()")
-                        review = '\n'.join(
-                            p.strip() for p in review_elem) if review_elem else ''
-                        date_elem = c2.xpath(
-                            "//div[@class='readingTimeline__text']/text()")
-                        for d in date_elem:
-                            date_matched = re.search(r'(\w+)\s+(\d+),\s+(\d+)', d)
-                            if date_matched:
-                                last_updated = make_aware(datetime.strptime(date_matched[1] + ' ' + date_matched[2] + ' ' + date_matched[3], '%B %d %Y'))
-                    else:
-                        print(f"Error loading review{url_review}, ignored")
-                    scraper = get_scraper_by_url(url_book)
-                    url_book = scraper.get_effective_url(url_book)
-                    book = Book.objects.filter(source_url=url_book).first()
-                    if not book:
-                        print("add new book " + url_book)
-                        scraper.scrape(url_book)
-                        form = scraper.save(request_user=user)
-                        book = form.instance
-                    books.append({
-                        'url': url_book,
-                        'book': book,
-                        'rating': rating,
-                        'review': review,
-                        'last_updated': last_updated
-                    })
-                except Exception:
-                    print("Error adding " + url_book)
-                    pass  # likely just download error
-            next_elem = content.xpath("//a[@class='next_page']/@href")
-            url_shelf = ('https://www.goodreads.com' + next_elem[0].strip()) if next_elem else None
-        return {'title': title, 'description': '', 'books': books}
-
-    @classmethod
-    def parse_list(cls, url, user):  # return {'title': 'abc', books: [{'book': obj, 'rating': 10, 'review': 'txt'}, ...]}
-        title = None
-        books = []
-        url_shelf = url
-        while url_shelf:
-            print(f'List loading {url_shelf}')
-            r = requests.get(url_shelf, timeout=settings.SCRAPING_TIMEOUT)
-            if r.status_code != 200:
-                print(f'List loading error {url_shelf}')
-                break
-            url_shelf = None
-            content = html.fromstring(r.content.decode('utf-8'))
-            title_elem = content.xpath('//h1[@class="gr-h1 gr-h1--serif"]/text()')
-            if not title_elem:
-                print(f'List parsing error {url_shelf}')
-                break
-            title = title_elem[0].strip()
-            description = content.xpath('//div[@class="mediumText"]/text()')[0].strip()
-            print("List title: " + title)
-            for link in content.xpath('//a[@class="bookTitle"]/@href'):
-                url_book = 'https://www.goodreads.com' + link
-                try:
-                    scraper = get_scraper_by_url(url_book)
-                    url_book = scraper.get_effective_url(url_book)
-                    book = Book.objects.filter(source_url=url_book).first()
-                    if not book:
-                        print("add new book " + url_book)
-                        scraper.scrape(url_book)
-                        form = scraper.save(request_user=user)
-                        book = form.instance
-                    books.append({
-                        'url': url_book,
-                        'book': book,
-                        'review': '',
-                    })
-                except Exception:
-                    print("Error adding " + url_book)
-                    pass  # likely just download error
-            next_elem = content.xpath("//a[@class='next_page']/@href")
-            url_shelf = ('https://www.goodreads.com' + next_elem[0].strip()) if next_elem else None
-        return {'title': title, 'description': description, 'books': books}

common/management/commands/index_stats.py

@@ -1,40 +0,0 @@
-from django.core.management.base import BaseCommand
-from common.index import Indexer
-from django.conf import settings
-from movies.models import Movie
-from books.models import Book
-from games.models import Game
-from music.models import Album, Song
-from django.core.paginator import Paginator
-from tqdm import tqdm
-from time import sleep
-from datetime import timedelta
-from django.utils import timezone
-
-
-class Command(BaseCommand):
-    help = 'Check search index'
-
-    def handle(self, *args, **options):
-        print(f'Connecting to search server')
-        stats = Indexer.get_stats()
-        print(stats)
-        st = Indexer.instance().get_all_update_status() 
-        cnt = {"enqueued": [0, 0], "processing": [0, 0], "processed": [0, 0], "failed": [0, 0]}
-        lastEnq = {"enqueuedAt": ""}
-        lastProc = {"enqueuedAt": ""}
-        for s in st:
-            n = s["type"].get("number")
-            cnt[s["status"]][0] += 1
-            cnt[s["status"]][1] += n if n else 0
-            if s["status"] == "processing":
-                print(s)
-            elif s["status"] == "enqueued":
-                if s["enqueuedAt"] > lastEnq["enqueuedAt"]:
-                    lastEnq = s
-            elif s["status"] == "processed":
-                if s["enqueuedAt"] > lastProc["enqueuedAt"]:
-                    lastProc = s
-        print(lastEnq)
-        print(lastProc)
-        print(cnt)

common/management/commands/init_index.py

@@ -1,18 +0,0 @@
-from django.core.management.base import BaseCommand
-from common.index import Indexer
-from django.conf import settings
-
-
-class Command(BaseCommand):
-    help = 'Initialize the search index'
-
-    def handle(self, *args, **options):
-        print(f'Connecting to search server')
-        Indexer.init()
-        self.stdout.write(self.style.SUCCESS('Index created.'))
-        # try:
-        #     Indexer.init()
-        #     self.stdout.write(self.style.SUCCESS('Index created.'))
-        # except Exception:
-        #     Indexer.update_settings()
-        #     self.stdout.write(self.style.SUCCESS('Index settings updated.'))

common/management/commands/reindex.py

@@ -1,40 +0,0 @@
-from django.core.management.base import BaseCommand
-from common.index import Indexer
-from django.conf import settings
-from movies.models import Movie
-from books.models import Book
-from games.models import Game
-from music.models import Album, Song
-from django.core.paginator import Paginator
-from tqdm import tqdm
-from time import sleep
-from datetime import timedelta
-from django.utils import timezone
-
-
-BATCH_SIZE = 1000
-
-
-class Command(BaseCommand):
-    help = 'Regenerate the search index'
-
-    # def add_arguments(self, parser):
-    #     parser.add_argument('hours', type=int, help='Re-index items modified in last N hours, 0 to reindex all')
-
-    def handle(self, *args, **options):
-        # h = int(options['hours'])
-        print(f'Connecting to search server')
-        if Indexer.busy():
-            print('Please wait for previous updates')
-        # Indexer.update_settings()
-        # self.stdout.write(self.style.SUCCESS('Index settings updated.'))
-        for c in [Book, Song, Album, Game, Movie]:
-            print(f'Re-indexing {c}')
-            qs = c.objects.all()  # if h == 0 else c.objects.filter(edited_time__gt=timezone.now() - timedelta(hours=h))
-            pg = Paginator(qs.order_by('id'), BATCH_SIZE)
-            for p in tqdm(pg.page_range):
-                items = list(map(lambda o: Indexer.obj_to_dict(o), pg.get_page(p).object_list))
-                if items:
-                    Indexer.replace_batch(items)
-                    while Indexer.busy():
-                        sleep(0.5)

common/scraper.py

@@ -1,265 +0,0 @@
-import requests
-import functools
-import random
-import logging
-import re
-import dateparser
-import datetime
-import time
-import filetype
-import dns.resolver
-import urllib.parse
-from lxml import html
-from threading import Thread
-from django.utils import timezone
-from django.utils.translation import ugettext_lazy as _
-from django.core.exceptions import ObjectDoesNotExist, ValidationError
-from django.core.files.uploadedfile import SimpleUploadedFile
-from common.models import SourceSiteEnum
-from django.conf import settings
-from django.core.exceptions import ValidationError
-
-
-RE_NUMBERS = re.compile(r"\d+\d*")
-RE_WHITESPACES = re.compile(r"\s+")
-
-
-DEFAULT_REQUEST_HEADERS = {
-    'Host': '',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:70.0) Gecko/20100101 Firefox/70.0',
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
-    # well, since brotli lib is so bothering, remove `br`
-    'Accept-Encoding': 'gzip, deflate',
-    'Connection': 'keep-alive',
-    'DNT': '1',
-    'Upgrade-Insecure-Requests': '1',
-    'Cache-Control': 'no-cache',
-}
-
-
-# luminati account credentials
-PORT = 22225
-
-
-logger = logging.getLogger(__name__)
-
-
-# register all implemented scraper in form of {host: scraper_class,}
-scraper_registry = {}
-
-
-def get_normalized_url(raw_url):
-    url = re.sub(r'//m.douban.com/(\w+)/', r'//\1.douban.com/', raw_url)
-    url = re.sub(r'//www.google.com/books/edition/_/([A-Za-z0-9_\-]+)[\?]*', r'//books.google.com/books?id=\1&', url)
-    return url
-
-
-def log_url(func):
-    """
-    Catch exceptions and log then pass the exceptions.
-    First postion argument (except cls/self) of decorated function must be the url.
-    """
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            # log the url and trace stack
-            logger.error(f"Scrape Failed URL: {args[1]}\n{e}")
-            if settings.DEBUG:
-                logger.error("Expections during scraping:", exc_info=e)
-            raise e
-
-    return wrapper
-
-
-def parse_date(raw_str):
-    return dateparser.parse(
-        raw_str,
-        settings={
-            "RELATIVE_BASE": datetime.datetime(1900, 1, 1)
-        }
-    )
-
-
-class AbstractScraper:
-    """
-    Scrape entities. The entities means those defined in the models.py file,
-    like Book, Movie......
-    """
-
-    # subclasses must specify those two variables
-    # site means general sites, like amazon/douban etc
-    site_name = None
-    # host means technically hostname
-    host = None
-    # corresponding data class
-    data_class = None
-    # corresponding form class
-    form_class = None
-    # used to extract effective url
-    regex = None
-    # scraped raw image
-    raw_img = None
-    # scraped raw data
-    raw_data = {}
-
-    def __init_subclass__(cls, **kwargs):
-        # this statement initialize the subclasses
-        super().__init_subclass__(**kwargs)
-        assert cls.site_name is not None, "class variable `site_name` must be specified"
-        assert bool(cls.host), "class variable `host` must be specified"
-        assert cls.data_class is not None, "class variable `data_class` must be specified"
-        assert cls.form_class is not None, "class variable `form_class` must be specified"
-        assert cls.regex is not None, "class variable `regex` must be specified"
-        assert isinstance(cls.host, str) or (isinstance(cls.host, list) and isinstance(
-            cls.host[0], str)), "`host` must be type str or list"
-        assert cls.site_name in SourceSiteEnum, "`site_name` must be one of `SourceSiteEnum` value"
-        assert hasattr(cls, 'scrape') and callable(
-            cls.scrape), "scaper must have method `.scrape()`"
-
-        # decorate the scrape method
-        cls.scrape = classmethod(log_url(cls.scrape))
-
-        # register scraper
-        if isinstance(cls.host, list):
-            for host in cls.host:
-                scraper_registry[host] = cls
-        else:
-            scraper_registry[cls.host] = cls
-
-    def scrape(self, url):
-        """
-        Scrape/request model schema specified data from given url and return it.
-        Implementations of subclasses to this method would be decorated as class method.
-        return (data_dict, image)
-        Should set the `raw_data` and the `raw_img`
-        """
-        raise NotImplementedError("Subclass should implement this method")
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        """
-        The return value should be identical with that saved in DB as `source_url`
-        """
-        url = cls.regex.findall(raw_url.replace('http:', 'https:'))  # force all http to be https
-        if not url:
-            raise ValueError(f"not valid url: {raw_url}")
-        return url[0]
-
-    @classmethod
-    def download_page(cls, url, headers):
-        url = cls.get_effective_url(url)
-
-        if settings.LUMINATI_USERNAME is None:
-            proxies = None
-            if settings.PROXYCRAWL_KEY is not None:
-                url = f'https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={url}'
-            # if settings.SCRAPESTACK_KEY is not None:
-            #     url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}'
-        else:
-            session_id = random.random()
-            proxy_url = ('http://%s-country-cn-session-%s:%s@zproxy.lum-superproxy.io:%d' %
-                         (settings.LUMINATI_USERNAME, session_id, settings.LUMINATI_PASSWORD, PORT))
-            proxies = {
-                'http': proxy_url,
-                'https': proxy_url,
-            }
-
-        r = requests.get(url, proxies=proxies,
-                         headers=headers, timeout=settings.SCRAPING_TIMEOUT)
-
-        if r.status_code != 200:
-            raise RuntimeError(f"download page failed, status code {r.status_code}")
-        # with open('temp.html', 'w', encoding='utf-8') as fp:
-        #     fp.write(r.content.decode('utf-8'))
-        return html.fromstring(r.content.decode('utf-8'))
-
-    @classmethod
-    def download_image(cls, url, item_url=None):
-        if url is None:
-            return None, None
-        raw_img = None
-        session_id = random.random()
-        proxy_url = ('http://%s-country-cn-session-%s:%s@zproxy.lum-superproxy.io:%d' %
-                     (settings.LUMINATI_USERNAME, session_id, settings.LUMINATI_PASSWORD, PORT))
-        proxies = {
-            'http': proxy_url,
-            'https': proxy_url,
-        }
-        if settings.LUMINATI_USERNAME is None:
-            proxies = None
-        if url:
-            img_response = requests.get(
-                url,
-                headers={
-                    'accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
-                    'accept-encoding': 'gzip, deflate',
-                    'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,fr-FR;q=0.6,fr;q=0.5,zh-TW;q=0.4',
-                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 Edg/81.0.416.72',
-                    'cache-control': 'no-cache',
-                    'dnt': '1',
-                },
-                proxies=proxies,
-                timeout=settings.SCRAPING_TIMEOUT,
-            )
-            if img_response.status_code == 200:
-                raw_img = img_response.content
-                content_type = img_response.headers.get('Content-Type')
-                ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension
-            else:
-                ext = None
-        return raw_img, ext
-
-    @classmethod
-    def save(cls, request_user, instance=None):
-        entity_cover = {
-            'cover': SimpleUploadedFile('temp.' + cls.img_ext, cls.raw_img)
-        } if cls.img_ext is not None else None
-        form = cls.form_class(data=cls.raw_data, files=entity_cover, instance=instance)
-        if form.is_valid():
-            form.instance.last_editor = request_user
-            form.instance._change_reason = 'scrape'
-            form.save()
-            cls.instance = form.instance
-        else:
-            logger.error(str(form.errors))
-            raise ValidationError("Form invalid.")
-        return form
-
-
-from common.scrapers.bandcamp import BandcampAlbumScraper
-from common.scrapers.goodreads import GoodreadsScraper
-from common.scrapers.google import GoogleBooksScraper
-from common.scrapers.tmdb import TmdbMovieScraper
-from common.scrapers.steam import SteamGameScraper
-from common.scrapers.imdb import ImdbMovieScraper
-from common.scrapers.igdb import IgdbGameScraper
-from common.scrapers.spotify import SpotifyAlbumScraper, SpotifyTrackScraper
-from common.scrapers.douban import DoubanAlbumScraper, DoubanBookScraper, DoubanGameScraper, DoubanMovieScraper
-from common.scrapers.bangumi import BangumiScraper
-
-
-def get_scraper_by_url(url):
-    parsed_url = urllib.parse.urlparse(url)
-    hostname = parsed_url.netloc
-    for host in scraper_registry:
-        if host in url:
-            return scraper_registry[host]
-    # TODO move this logic to scraper class
-    try:
-        answers = dns.resolver.query(hostname, 'CNAME')
-        for rdata in answers:
-            if str(rdata.target) == 'dom.bandcamp.com.':
-                return BandcampAlbumScraper
-    except Exception as e:
-        pass
-    try:
-        answers = dns.resolver.query(hostname, 'A')
-        for rdata in answers:
-            if str(rdata.address) == '35.241.62.186':
-                return BandcampAlbumScraper
-    except Exception as e:
-        pass
-    return None

common/scrapers/bandcamp.py

@@ -1,71 +0,0 @@
-import re
-import dateparser
-import json
-from lxml import html
-from common.models import SourceSiteEnum
-from common.scraper import AbstractScraper
-from music.models import Album
-from music.forms import AlbumForm
-
-
-class BandcampAlbumScraper(AbstractScraper):
-    site_name = SourceSiteEnum.BANDCAMP.value
-    # API URL
-    host = '.bandcamp.com/'
-    data_class = Album
-    form_class = AlbumForm
-
-    regex = re.compile(r"https://[a-zA-Z0-9\-\.]+/album/[^?#]+")
-
-    def scrape(self, url, response=None):
-        effective_url = self.get_effective_url(url)
-        if effective_url is None:
-            raise ValueError("not valid url")
-        if response is not None:
-            content = html.fromstring(response.content.decode('utf-8'))
-        else:
-            content = self.download_page(url, {})
-        try:
-            title = content.xpath("//h2[@class='trackTitle']/text()")[0].strip()
-            artist = [content.xpath("//div[@id='name-section']/h3/span/a/text()")[0].strip()]
-        except IndexError:
-            raise ValueError("given url contains no valid info")
-
-        genre = []  # TODO: parse tags
-        track_list = []
-        release_nodes = content.xpath("//div[@class='tralbumData tralbum-credits']/text()")
-        release_date = dateparser.parse(re.sub(r'releas\w+ ', '', release_nodes[0].strip())) if release_nodes else None
-        duration = None
-        company = None
-        brief_nodes = content.xpath("//div[@class='tralbumData tralbum-about']/text()")
-        brief = "".join(brief_nodes) if brief_nodes else None
-        cover_url = content.xpath("//div[@id='tralbumArt']/a/@href")[0].strip()
-        bandcamp_page_data = json.loads(content.xpath(
-            "//meta[@name='bc-page-properties']/@content")[0].strip())
-        other_info = {}
-        other_info['bandcamp_album_id'] = bandcamp_page_data['item_id']
-
-        raw_img, ext = self.download_image(cover_url, url)
-
-        data = {
-            'title': title,
-            'artist': artist,
-            'genre': genre,
-            'track_list': track_list,
-            'release_date': release_date,
-            'duration': duration,
-            'company': company,
-            'brief': brief,
-            'other_info': other_info,
-            'source_site': self.site_name,
-            'source_url': effective_url,
-            'cover_url': cover_url,
-        }
-
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        url = cls.regex.findall(raw_url)
-        return url[0] if len(url) > 0 else None

common/scrapers/bangumi.py

@@ -1,199 +0,0 @@
-import re
-from common.models import SourceSiteEnum
-from movies.models import Movie, MovieGenreEnum
-from movies.forms import MovieForm
-from books.models import Book
-from books.forms import BookForm
-from music.models import Album, Song
-from music.forms import AlbumForm, SongForm
-from games.models import Game
-from games.forms import GameForm
-from common.scraper import *
-from django.core.exceptions import ObjectDoesNotExist
-
-
-def find_entity(source_url):
-    """
-    for bangumi
-    """
-    # to be added when new scrape method is implemented
-    result = Game.objects.filter(source_url=source_url)
-    if result:
-        return result[0]
-    else:
-        raise ObjectDoesNotExist
-
-
-class BangumiScraper(AbstractScraper):
-    site_name = SourceSiteEnum.BANGUMI.value
-    host = 'bgm.tv'
-
-    # for interface coherence
-    data_class = type("FakeDataClass", (object,), {})()
-    data_class.objects = type("FakeObjectsClass", (object,), {})()
-    data_class.objects.get = find_entity
-    # should be set at scrape_* method
-    form_class = ''
-
-    regex = re.compile(r"https{0,1}://bgm\.tv/subject/\d+")
-
-    def scrape(self, url):
-        """
-        This is the scraping portal
-        """
-        headers = DEFAULT_REQUEST_HEADERS.copy()
-        headers['Host'] = self.host
-        content = self.download_page(url, headers)
-
-        # download image
-        img_url = 'http:' + content.xpath("//div[@class='infobox']//img[1]/@src")[0]
-        raw_img, ext = self.download_image(img_url, url)
-
-        # Test category
-        category_code = content.xpath("//div[@id='headerSearch']//option[@selected]/@value")[0]
-        handler_map = {
-            '1': self.scrape_book,
-            '2': self.scrape_movie,
-            '3': self.scrape_album,
-            '4': self.scrape_game
-        }
-        data = handler_map[category_code](self, content)
-        data['source_url'] = self.get_effective_url(url)
-
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-    def scrape_game(self, content):
-        self.data_class = Game
-        self.form_class = GameForm
-
-        title_elem = content.xpath("//a[@property='v:itemreviewed']/text()")
-        if not title_elem:
-            raise ValueError("no game info found on this page")
-            title = None
-        else:
-            title = title_elem[0].strip()
-
-        other_title_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'别名')]]/text()")
-        if not other_title_elem:
-            other_title_elem = content.xpath(
-                "//ul[@id='infobox']/li[child::span[contains(text(),'别名')]]/a/text()")
-        other_title = other_title_elem if other_title_elem else []
-
-        chinese_name_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'中文')]]/text()")
-        if not chinese_name_elem:
-            chinese_name_elem = content.xpath(
-                "//ul[@id='infobox']/li[child::span[contains(text(),'中文')]]/a/text()")
-        if chinese_name_elem:
-            chinese_name = chinese_name_elem[0]
-            # switch chinese name with original name
-            title, chinese_name = chinese_name, title
-            # actually the name appended is original
-            other_title.append(chinese_name)
-
-        developer_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'开发')]]/text()")
-        if not developer_elem:
-            developer_elem = content.xpath(
-                "//ul[@id='infobox']/li[child::span[contains(text(),'开发')]]/a/text()")
-        developer = developer_elem if developer_elem else None
-
-        publisher_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'发行:')]]/text()")
-        if not publisher_elem:
-            publisher_elem = content.xpath(
-                "//ul[@id='infobox']/li[child::span[contains(text(),'发行:')]]/a/text()")
-        publisher = publisher_elem if publisher_elem else None
-
-        platform_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'平台')]]/text()")
-        if not platform_elem:
-            platform_elem = content.xpath(
-                "//ul[@id='infobox']/li[child::span[contains(text(),'平台')]]/a/text()")
-        platform = platform_elem if platform_elem else None
-
-        genre_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'类型')]]/text()")
-        if not genre_elem:
-            genre_elem = content.xpath(
-                "//ul[@id='infobox']/li[child::span[contains(text(),'类型')]]/a/text()")
-        genre = genre_elem if genre_elem else None
-
-        date_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'发行日期')]]/text()")
-        if not date_elem:
-            date_elem = content.xpath(
-                "//ul[@id='infobox']/li[child::span[contains(text(),'发行日期')]]/a/text()")
-        release_date = parse_date(date_elem[0]) if date_elem else None
-
-        brief = ''.join(content.xpath("//div[@property='v:summary']/text()"))
-
-        other_info = {}
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'人数')]]/text()")
-        if other_elem:
-            other_info['游玩人数'] = other_elem[0]
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'引擎')]]/text()")
-        if other_elem:
-            other_info['引擎'] = ' '.join(other_elem)
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'售价')]]/text()")
-        if other_elem:
-            other_info['售价'] = ' '.join(other_elem)
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'官方网站')]]/text()")
-        if other_elem:
-            other_info['网站'] = other_elem[0]
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'剧本')]]/a/text()") or content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'剧本')]]/text()")
-        if other_elem:
-            other_info['剧本'] = ' '.join(other_elem)
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'编剧')]]/a/text()") or content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'编剧')]]/text()")
-        if other_elem:
-            other_info['编剧'] = ' '.join(other_elem)
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'音乐')]]/a/text()") or content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'音乐')]]/text()")
-        if other_elem:
-            other_info['音乐'] = ' '.join(other_elem)
-        other_elem = content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'美术')]]/a/text()") or content.xpath(
-            "//ul[@id='infobox']/li[child::span[contains(text(),'美术')]]/text()")
-        if other_elem:
-            other_info['美术'] = ' '.join(other_elem)
-
-        data = {
-            'title': title,
-            'other_title': None,
-            'developer': developer,
-            'publisher': publisher,
-            'release_date': release_date,
-            'genre': genre,
-            'platform': platform,
-            'brief': brief,
-            'other_info': other_info,
-            'source_site': self.site_name,
-        }
-
-        return data
-
-    def scrape_movie(self, content):
-        self.data_class = Movie
-        self.form_class = MovieForm
-        raise NotImplementedError
-
-    def scrape_book(self, content):
-        self.data_class = Book
-        self.form_class = BookForm
-        raise NotImplementedError
-
-    def scrape_album(self, content):
-        self.data_class = Album
-        self.form_class = AlbumForm
-        raise NotImplementedError

common/scrapers/douban.py

@@ -1,714 +0,0 @@
-import requests
-import re
-import filetype
-from lxml import html
-from common.models import SourceSiteEnum
-from movies.models import Movie, MovieGenreEnum
-from movies.forms import MovieForm
-from books.models import Book
-from books.forms import BookForm
-from music.models import Album
-from music.forms import AlbumForm
-from games.models import Game
-from games.forms import GameForm
-from django.core.validators import URLValidator
-from django.conf import settings
-from PIL import Image
-from io import BytesIO
-from common.scraper import *
-
-
-class DoubanScrapperMixin:
-    @classmethod
-    def download_page(cls, url, headers):
-        url = cls.get_effective_url(url)
-        r = None
-        error = 'DoubanScrapper: error occured when downloading ' + url
-        content = None
-        last_error = None
-
-        def get(url):
-            nonlocal r
-            # print('Douban GET ' + url)
-            try:
-                r = requests.get(url, timeout=settings.SCRAPING_TIMEOUT)
-            except Exception as e:
-                r = requests.Response()
-                r.status_code = f"Exception when GET {url} {e}" + url
-            # print('Douban CODE ' + str(r.status_code))
-            return r
-
-        def check_content():
-            nonlocal r, error, content, last_error
-            content = None
-            last_error = None
-            if r.status_code == 200:
-                content = r.content.decode('utf-8')
-                if content.find('关于豆瓣') == -1:
-                    if content.find('你的 IP 发出') == -1:
-                        error = error + 'Content not authentic'  # response is garbage
-                    else:
-                        error = error + 'IP banned'
-                    content = None
-                    last_error = 'network'
-                elif content.find('<title>页面不存在</title>') != -1 or content.find('呃... 你想访问的条目豆瓣不收录。') != -1:  # re.search('不存在[^<]+</title>', content, re.MULTILINE):
-                    content = None
-                    last_error = 'censorship'
-                    error = error + 'Not found or hidden by Douban'
-            elif r.status_code == 204:
-                content = None
-                last_error = 'censorship'
-                error = error + 'Not found or hidden by Douban'
-            else:
-                content = None
-                last_error = 'network'
-                error = error + str(r.status_code)
-
-        def fix_wayback_links():
-            nonlocal content
-            # fix links
-            content = re.sub(r'href="http[^"]+http', r'href="http', content)
-            # https://img9.doubanio.com/view/subject/{l|m|s}/public/s1234.jpg
-            content = re.sub(r'src="[^"]+/(s\d+\.\w+)"',
-                             r'src="https://img9.doubanio.com/view/subject/m/public/\1"', content)
-            # https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2681329386.jpg
-            # https://img9.doubanio.com/view/photo/{l|m|s}/public/p1234.webp
-            content = re.sub(r'src="[^"]+/(p\d+\.\w+)"',
-                             r'src="https://img9.doubanio.com/view/photo/m/public/\1"', content)
-
-        # Wayback Machine: get latest available
-        def wayback():
-            nonlocal r, error, content
-            error = error + '\nWayback: '
-            get('http://archive.org/wayback/available?url=' + url)
-            if r.status_code == 200:
-                w = r.json()
-                if w['archived_snapshots'] and w['archived_snapshots']['closest']:
-                    get(w['archived_snapshots']['closest']['url'])
-                    check_content()
-                    if content is not None:
-                        fix_wayback_links()
-                else:
-                    error = error + 'No snapshot available'
-            else:
-                error = error + str(r.status_code)
-
-        # Wayback Machine: guess via CDX API
-        def wayback_cdx():
-            nonlocal r, error, content
-            error = error + '\nWayback: '
-            get('http://web.archive.org/cdx/search/cdx?url=' + url)
-            if r.status_code == 200:
-                dates = re.findall(r'[^\s]+\s+(\d+)\s+[^\s]+\s+[^\s]+\s+\d+\s+[^\s]+\s+\d{5,}',
-                                   r.content.decode('utf-8'))
-                # assume snapshots whose size >9999 contain real content, use the latest one of them
-                if len(dates) > 0:
-                    get('http://web.archive.org/web/' + dates[-1] + '/' + url)
-                    check_content()
-                    if content is not None:
-                        fix_wayback_links()
-                else:
-                    error = error + 'No snapshot available'
-            else:
-                error = error + str(r.status_code)
-
-        def latest():
-            nonlocal r, error, content
-            if settings.SCRAPESTACK_KEY is not None:
-                error = error + '\nScrapeStack: '
-                get(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}')
-            elif settings.SCRAPERAPI_KEY is not None:
-                error = error + '\nScraperAPI: '
-                get(f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}')
-            else:
-                error = error + '\nDirect: '
-                get(url)
-            check_content()
-            if last_error == 'network' and settings.PROXYCRAWL_KEY is not None:
-                error = error + '\nProxyCrawl: '
-                get(f'https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={url}')
-                check_content()
-            if last_error == 'censorship' and settings.LOCAL_PROXY is not None:
-                error = error + '\nLocal: '
-                get(f'{settings.LOCAL_PROXY}?url={url}')
-                check_content()
-
-        latest()
-        if content is None:
-            wayback_cdx()
-
-        if content is None:
-            raise RuntimeError(error)
-        # with open('/tmp/temp.html', 'w', encoding='utf-8') as fp:
-        #     fp.write(content)
-        return html.fromstring(content)
-
-    @classmethod
-    def download_image(cls, url, item_url=None):
-        raw_img = None
-        ext = None
-
-        if settings.SCRAPESTACK_KEY is not None:
-            dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}'
-        elif settings.SCRAPERAPI_KEY is not None:
-            dl_url = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}'
-        else:
-            dl_url = url
-
-        try:
-            img_response = requests.get(dl_url, timeout=settings.SCRAPING_TIMEOUT)
-            if img_response.status_code == 200:
-                raw_img = img_response.content
-                img = Image.open(BytesIO(raw_img))
-                img.load()  # corrupted image will trigger exception
-                content_type = img_response.headers.get('Content-Type')
-                ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension
-            else:
-                logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}")
-                # raise RuntimeError(f"Douban: download image failed {img_response.status_code} {dl_url}")
-        except Exception as e:
-            raw_img = None
-            ext = None
-            logger.error(f"Douban: download image failed {e} {dl_url} {item_url}")
-
-        if raw_img is None and settings.PROXYCRAWL_KEY is not None:
-            try:
-                dl_url = f'https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={url}'
-                img_response = requests.get(dl_url, timeout=settings.SCRAPING_TIMEOUT)
-                if img_response.status_code == 200:
-                    raw_img = img_response.content
-                    img = Image.open(BytesIO(raw_img))
-                    img.load()  # corrupted image will trigger exception
-                    content_type = img_response.headers.get('Content-Type')
-                    ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension
-                else:
-                    logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}")
-            except Exception as e:
-                raw_img = None
-                ext = None
-                logger.error(f"Douban: download image failed {e} {dl_url} {item_url}")
-        return raw_img, ext
-
-
-class DoubanBookScraper(DoubanScrapperMixin, AbstractScraper):
-    site_name = SourceSiteEnum.DOUBAN.value
-    host = "book.douban.com"
-    data_class = Book
-    form_class = BookForm
-
-    regex = re.compile(r"https://book\.douban\.com/subject/\d+/{0,1}")
-
-    def scrape(self, url):
-        headers = DEFAULT_REQUEST_HEADERS.copy()
-        headers['Host'] = self.host
-        content = self.download_page(url, headers)
-
-        isbn_elem = content.xpath("//div[@id='info']//span[text()='ISBN:']/following::text()")
-        isbn = isbn_elem[0].strip() if isbn_elem else None
-        title_elem = content.xpath("/html/body//h1/span/text()")
-        title = title_elem[0].strip() if title_elem else None
-        if not title:
-            if isbn:
-                title = 'isbn: ' + isbn
-            else:
-                raise ValueError("given url contains no book title or isbn")
-
-        subtitle_elem = content.xpath(
-            "//div[@id='info']//span[text()='副标题:']/following::text()")
-        subtitle = subtitle_elem[0].strip()[:500] if subtitle_elem else None
-
-        orig_title_elem = content.xpath(
-            "//div[@id='info']//span[text()='原作名:']/following::text()")
-        orig_title = orig_title_elem[0].strip()[:500] if orig_title_elem else None
-
-        language_elem = content.xpath(
-            "//div[@id='info']//span[text()='语言:']/following::text()")
-        language = language_elem[0].strip() if language_elem else None
-
-        pub_house_elem = content.xpath(
-            "//div[@id='info']//span[text()='出版社:']/following::text()")
-        pub_house = pub_house_elem[0].strip() if pub_house_elem else None
-
-        pub_date_elem = content.xpath(
-            "//div[@id='info']//span[text()='出版年:']/following::text()")
-        pub_date = pub_date_elem[0].strip() if pub_date_elem else ''
-        year_month_day = RE_NUMBERS.findall(pub_date)
-        if len(year_month_day) in (2, 3):
-            pub_year = int(year_month_day[0])
-            pub_month = int(year_month_day[1])
-        elif len(year_month_day) == 1:
-            pub_year = int(year_month_day[0])
-            pub_month = None
-        else:
-            pub_year = None
-            pub_month = None
-        if pub_year and pub_month and pub_year < pub_month:
-            pub_year, pub_month = pub_month, pub_year
-        pub_year = None if pub_year is not None and pub_year not in range(
-            0, 3000) else pub_year
-        pub_month = None if pub_month is not None and pub_month not in range(
-            1, 12) else pub_month
-
-        binding_elem = content.xpath(
-            "//div[@id='info']//span[text()='装帧:']/following::text()")
-        binding = binding_elem[0].strip() if binding_elem else None
-
-        price_elem = content.xpath(
-            "//div[@id='info']//span[text()='定价:']/following::text()")
-        price = price_elem[0].strip() if price_elem else None
-
-        pages_elem = content.xpath(
-            "//div[@id='info']//span[text()='页数:']/following::text()")
-        pages = pages_elem[0].strip() if pages_elem else None
-        if pages is not None:
-            pages = int(RE_NUMBERS.findall(pages)[
-                        0]) if RE_NUMBERS.findall(pages) else None
-            if pages and (pages > 999999 or pages < 1):
-                pages = None
-
-        brief_elem = content.xpath(
-            "//h2/span[text()='内容简介']/../following-sibling::div[1]//div[@class='intro'][not(ancestor::span[@class='short'])]/p/text()")
-        brief = '\n'.join(p.strip()
-                          for p in brief_elem) if brief_elem else None
-
-        contents = None
-        try:
-            contents_elem = content.xpath(
-                "//h2/span[text()='目录']/../following-sibling::div[1]")[0]
-            # if next the id of next sibling contains `dir`, that would be the full contents
-            if "dir" in contents_elem.getnext().xpath("@id")[0]:
-                contents_elem = contents_elem.getnext()
-                contents = '\n'.join(p.strip() for p in contents_elem.xpath(
-                    "text()")[:-2]) if contents_elem else None
-            else:
-                contents = '\n'.join(p.strip() for p in contents_elem.xpath(
-                    "text()")) if contents_elem else None
-        except Exception:
-            pass
-
-        img_url_elem = content.xpath("//*[@id='mainpic']/a/img/@src")
-        img_url = img_url_elem[0].strip() if img_url_elem else None
-        raw_img, ext = self.download_image(img_url, url)
-
-        # there are two html formats for authors and translators
-        authors_elem = content.xpath("""//div[@id='info']//span[text()='作者:']/following-sibling::br[1]/
-            preceding-sibling::a[preceding-sibling::span[text()='作者:']]/text()""")
-        if not authors_elem:
-            authors_elem = content.xpath(
-                """//div[@id='info']//span[text()=' 作者']/following-sibling::a/text()""")
-        if authors_elem:
-            authors = []
-            for author in authors_elem:
-                authors.append(RE_WHITESPACES.sub(' ', author.strip())[:200])
-        else:
-            authors = None
-
-        translators_elem = content.xpath("""//div[@id='info']//span[text()='译者:']/following-sibling::br[1]/
-            preceding-sibling::a[preceding-sibling::span[text()='译者:']]/text()""")
-        if not translators_elem:
-            translators_elem = content.xpath(
-                """//div[@id='info']//span[text()=' 译者']/following-sibling::a/text()""")
-        if translators_elem:
-            translators = []
-            for translator in translators_elem:
-                translators.append(RE_WHITESPACES.sub(' ', translator.strip()))
-        else:
-            translators = None
-
-        other = {}
-        cncode_elem = content.xpath(
-            "//div[@id='info']//span[text()='统一书号:']/following::text()")
-        if cncode_elem:
-            other['统一书号'] = cncode_elem[0].strip()
-        series_elem = content.xpath(
-            "//div[@id='info']//span[text()='丛书:']/following-sibling::a[1]/text()")
-        if series_elem:
-            other['丛书'] = series_elem[0].strip()
-        imprint_elem = content.xpath(
-            "//div[@id='info']//span[text()='出品方:']/following-sibling::a[1]/text()")
-        if imprint_elem:
-            other['出品方'] = imprint_elem[0].strip()
-
-        data = {
-            'title': title,
-            'subtitle': subtitle,
-            'orig_title': orig_title,
-            'author': authors,
-            'translator': translators,
-            'language': language,
-            'pub_house': pub_house,
-            'pub_year': pub_year,
-            'pub_month': pub_month,
-            'binding': binding,
-            'price': price,
-            'pages': pages,
-            'isbn': isbn,
-            'brief': brief,
-            'contents': contents,
-            'other_info': other,
-            'source_site': self.site_name,
-            'source_url': self.get_effective_url(url),
-        }
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-
-class DoubanMovieScraper(DoubanScrapperMixin, AbstractScraper):
-    site_name = SourceSiteEnum.DOUBAN.value
-    host = 'movie.douban.com'
-    data_class = Movie
-    form_class = MovieForm
-
-    regex = re.compile(r"https://movie\.douban\.com/subject/\d+/{0,1}")
-
-    def scrape(self, url):
-        headers = DEFAULT_REQUEST_HEADERS.copy()
-        headers['Host'] = self.host
-        content = self.download_page(url, headers)
-
-        # parsing starts here
-        try:
-            raw_title = content.xpath(
-                "//span[@property='v:itemreviewed']/text()")[0].strip()
-        except IndexError:
-            raise ValueError("given url contains no movie info")
-
-        orig_title = content.xpath(
-            "//img[@rel='v:image']/@alt")[0].strip()
-        title = raw_title.split(orig_title)[0].strip()
-        # if has no chinese title
-        if title == '':
-            title = orig_title
-
-        if title == orig_title:
-            orig_title = None
-
-        # there are two html formats for authors and translators
-        other_title_elem = content.xpath(
-            "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]")
-        other_title = other_title_elem[0].strip().split(
-            ' / ') if other_title_elem else None
-
-        imdb_elem = content.xpath(
-            "//div[@id='info']//span[text()='IMDb链接:']/following-sibling::a[1]/text()")
-        if not imdb_elem:
-            imdb_elem = content.xpath(
-                "//div[@id='info']//span[text()='IMDb:']/following-sibling::text()[1]")
-        imdb_code = imdb_elem[0].strip() if imdb_elem else None
-
-        director_elem = content.xpath(
-            "//div[@id='info']//span[text()='导演']/following-sibling::span[1]/a/text()")
-        director = director_elem if director_elem else None
-
-        playwright_elem = content.xpath(
-            "//div[@id='info']//span[text()='编剧']/following-sibling::span[1]/a/text()")
-        playwright = list(map(lambda a: a[:200], playwright_elem)) if playwright_elem else None
-
-        actor_elem = content.xpath(
-            "//div[@id='info']//span[text()='主演']/following-sibling::span[1]/a/text()")
-        actor = list(map(lambda a: a[:200], actor_elem)) if actor_elem else None
-
-        # construct genre translator
-        genre_translator = {}
-        attrs = [attr for attr in dir(MovieGenreEnum) if '__' not in attr]
-        for attr in attrs:
-            genre_translator[getattr(MovieGenreEnum, attr).label] = getattr(
-                MovieGenreEnum, attr).value
-
-        genre_elem = content.xpath("//span[@property='v:genre']/text()")
-        if genre_elem:
-            genre = []
-            for g in genre_elem:
-                g = g.split(' ')[0]
-                if g == '紀錄片':  # likely some original data on douban was corrupted
-                    g = '纪录片'
-                elif g == '鬼怪':
-                    g = '惊悚'
-                if g in genre_translator:
-                    genre.append(genre_translator[g])
-                elif g in genre_translator.values():
-                    genre.append(g)
-                else:
-                    logger.error(f'unable to map genre {g}')
-        else:
-            genre = None
-
-        showtime_elem = content.xpath(
-            "//span[@property='v:initialReleaseDate']/text()")
-        if showtime_elem:
-            showtime = []
-            for st in showtime_elem:
-                parts = st.split('(')
-                if len(parts) == 1:
-                    time = st.split('(')[0]
-                    region = ''
-                else:
-                    time = st.split('(')[0]
-                    region = st.split('(')[1][0:-1]
-                showtime.append({time: region})
-        else:
-            showtime = None
-
-        site_elem = content.xpath(
-            "//div[@id='info']//span[text()='官方网站:']/following-sibling::a[1]/@href")
-        site = site_elem[0].strip()[:200] if site_elem else None
-        try:
-            validator = URLValidator()
-            validator(site)
-        except ValidationError:
-            site = None
-
-        area_elem = content.xpath(
-            "//div[@id='info']//span[text()='制片国家/地区:']/following-sibling::text()[1]")
-        if area_elem:
-            area = [a.strip()[:100] for a in area_elem[0].split('/')]
-        else:
-            area = None
-
-        language_elem = content.xpath(
-            "//div[@id='info']//span[text()='语言:']/following-sibling::text()[1]")
-        if language_elem:
-            language = [a.strip() for a in language_elem[0].split(' / ')]
-        else:
-            language = None
-
-        year_elem = content.xpath("//span[@class='year']/text()")
-        year = int(re.search(r'\d+', year_elem[0])[0]) if year_elem and re.search(r'\d+', year_elem[0]) else None
-
-        duration_elem = content.xpath("//span[@property='v:runtime']/text()")
-        other_duration_elem = content.xpath(
-            "//span[@property='v:runtime']/following-sibling::text()[1]")
-        if duration_elem:
-            duration = duration_elem[0].strip()
-            if other_duration_elem:
-                duration += other_duration_elem[0].rstrip()
-            duration = duration.split('/')[0].strip()
-        else:
-            duration = None
-
-        season_elem = content.xpath(
-            "//*[@id='season']/option[@selected='selected']/text()")
-        if not season_elem:
-            season_elem = content.xpath(
-                "//div[@id='info']//span[text()='季数:']/following-sibling::text()[1]")
-            season = int(season_elem[0].strip()) if season_elem else None
-        else:
-            season = int(season_elem[0].strip())
-
-        episodes_elem = content.xpath(
-            "//div[@id='info']//span[text()='集数:']/following-sibling::text()[1]")
-        episodes = int(episodes_elem[0].strip()) if episodes_elem and episodes_elem[0].strip().isdigit() else None
-
-        single_episode_length_elem = content.xpath(
-            "//div[@id='info']//span[text()='单集片长:']/following-sibling::text()[1]")
-        single_episode_length = single_episode_length_elem[0].strip(
-        )[:100] if single_episode_length_elem else None
-
-        # if has field `episodes` not none then must be series
-        is_series = True if episodes else False
-
-        brief_elem = content.xpath("//span[@class='all hidden']")
-        if not brief_elem:
-            brief_elem = content.xpath("//span[@property='v:summary']")
-        brief = '\n'.join([e.strip() for e in brief_elem[0].xpath(
-            './text()')]) if brief_elem else None
-
-        img_url_elem = content.xpath("//img[@rel='v:image']/@src")
-        img_url = img_url_elem[0].strip() if img_url_elem else None
-        raw_img, ext = self.download_image(img_url, url)
-
-        data = {
-            'title': title,
-            'orig_title': orig_title,
-            'other_title': other_title,
-            'imdb_code': imdb_code,
-            'director': director,
-            'playwright': playwright,
-            'actor': actor,
-            'genre': genre,
-            'showtime': showtime,
-            'site': site,
-            'area': area,
-            'language': language,
-            'year': year,
-            'duration': duration,
-            'season': season,
-            'episodes': episodes,
-            'single_episode_length': single_episode_length,
-            'brief': brief,
-            'is_series': is_series,
-            'source_site': self.site_name,
-            'source_url': self.get_effective_url(url),
-        }
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-
-class DoubanAlbumScraper(DoubanScrapperMixin, AbstractScraper):
-    site_name = SourceSiteEnum.DOUBAN.value
-    host = 'music.douban.com'
-    data_class = Album
-    form_class = AlbumForm
-
-    regex = re.compile(r"https://music\.douban\.com/subject/\d+/{0,1}")
-
-    def scrape(self, url):
-        headers = DEFAULT_REQUEST_HEADERS.copy()
-        headers['Host'] = self.host
-        content = self.download_page(url, headers)
-
-        # parsing starts here
-        try:
-            title = content.xpath("//h1/span/text()")[0].strip()
-        except IndexError:
-            raise ValueError("given url contains no album info")
-        if not title:
-            raise ValueError("given url contains no album info")
-
-        artists_elem = content.xpath("//div[@id='info']/span/span[@class='pl']/a/text()")
-        artist = None if not artists_elem else list(map(lambda a: a[:200], artists_elem))
-
-        genre_elem = content.xpath(
-            "//div[@id='info']//span[text()='流派:']/following::text()[1]")
-        genre = genre_elem[0].strip() if genre_elem else None
-
-        date_elem = content.xpath(
-            "//div[@id='info']//span[text()='发行时间:']/following::text()[1]")
-        release_date = parse_date(date_elem[0].strip()) if date_elem else None
-
-        company_elem = content.xpath(
-            "//div[@id='info']//span[text()='出版者:']/following::text()[1]")
-        company = company_elem[0].strip() if company_elem else None
-
-        track_list_elem = content.xpath(
-            "//div[@class='track-list']/div[@class='indent']/div/text()"
-        )
-        if track_list_elem:
-            track_list = '\n'.join([track.strip() for track in track_list_elem])
-        else:
-            track_list = None
-
-        brief_elem = content.xpath("//span[@class='all hidden']")
-        if not brief_elem:
-            brief_elem = content.xpath("//span[@property='v:summary']")
-        brief = '\n'.join([e.strip() for e in brief_elem[0].xpath(
-            './text()')]) if brief_elem else None
-
-        other_info = {}
-        other_elem = content.xpath(
-            "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]")
-        if other_elem:
-            other_info['又名'] = other_elem[0].strip()
-        other_elem = content.xpath(
-            "//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]")
-        if other_elem:
-            other_info['专辑类型'] = other_elem[0].strip()
-        other_elem = content.xpath(
-            "//div[@id='info']//span[text()='介质:']/following-sibling::text()[1]")
-        if other_elem:
-            other_info['介质'] = other_elem[0].strip()
-        other_elem = content.xpath(
-            "//div[@id='info']//span[text()='ISRC:']/following-sibling::text()[1]")
-        if other_elem:
-            other_info['ISRC'] = other_elem[0].strip()
-        other_elem = content.xpath(
-            "//div[@id='info']//span[text()='条形码:']/following-sibling::text()[1]")
-        if other_elem:
-            other_info['条形码'] = other_elem[0].strip()
-        other_elem = content.xpath(
-            "//div[@id='info']//span[text()='碟片数:']/following-sibling::text()[1]")
-        if other_elem:
-            other_info['碟片数'] = other_elem[0].strip()
-
-        img_url_elem = content.xpath("//div[@id='mainpic']//img/@src")
-        img_url = img_url_elem[0].strip() if img_url_elem else None
-        raw_img, ext = self.download_image(img_url, url)
-
-        data = {
-            'title': title,
-            'artist': artist,
-            'genre': genre,
-            'release_date': release_date,
-            'duration': None,
-            'company': company,
-            'track_list': track_list,
-            'brief': brief,
-            'other_info': other_info,
-            'source_site': self.site_name,
-            'source_url': self.get_effective_url(url),
-        }
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-
-class DoubanGameScraper(DoubanScrapperMixin, AbstractScraper):
-    site_name = SourceSiteEnum.DOUBAN.value
-    host = 'www.douban.com/game/'
-    data_class = Game
-    form_class = GameForm
-
-    regex = re.compile(r"https://www\.douban\.com/game/\d+/{0,1}")
-
-    def scrape(self, url):
-        headers = DEFAULT_REQUEST_HEADERS.copy()
-        headers['Host'] = 'www.douban.com'
-        content = self.download_page(url, headers)
-
-        try:
-            raw_title = content.xpath(
-                "//div[@id='content']/h1/text()")[0].strip()
-        except IndexError:
-            raise ValueError("given url contains no game info")
-
-        title = raw_title
-
-        other_title_elem = content.xpath(
-            "//dl[@class='game-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()")
-        other_title = other_title_elem[0].strip().split(' / ') if other_title_elem else None
-
-        developer_elem = content.xpath(
-            "//dl[@class='game-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()")
-        developer = developer_elem[0].strip().split(' / ') if developer_elem else None
-
-        publisher_elem = content.xpath(
-            "//dl[@class='game-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()")
-        publisher = publisher_elem[0].strip().split(' / ') if publisher_elem else None
-
-        platform_elem = content.xpath(
-            "//dl[@class='game-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()")
-        platform = platform_elem if platform_elem else None
-
-        genre_elem = content.xpath(
-            "//dl[@class='game-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()")
-        genre = None
-        if genre_elem:
-            genre = [g for g in genre_elem if g != '游戏']
-
-        date_elem = content.xpath(
-            "//dl[@class='game-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()")
-        release_date = parse_date(date_elem[0].strip()) if date_elem else None
-
-        brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()")
-        brief = '\n'.join(brief_elem) if brief_elem else None
-
-        img_url_elem = content.xpath(
-            "//div[@class='item-subject-info']/div[@class='pic']//img/@src")
-        img_url = img_url_elem[0].strip() if img_url_elem else None
-        raw_img, ext = self.download_image(img_url, url)
-
-        data = {
-            'title': title,
-            'other_title': other_title,
-            'developer': developer,
-            'publisher': publisher,
-            'release_date': release_date,
-            'genre': genre,
-            'platform': platform,
-            'brief': brief,
-            'other_info': None,
-            'source_site': self.site_name,
-            'source_url': self.get_effective_url(url),
-        }
-
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img

common/scrapers/goodreads.py

@@ -1,157 +0,0 @@
-import requests
-import re
-import filetype
-from lxml import html
-from common.models import SourceSiteEnum
-from movies.models import Movie, MovieGenreEnum
-from movies.forms import MovieForm
-from books.models import Book
-from books.forms import BookForm
-from music.models import Album, Song
-from music.forms import AlbumForm, SongForm
-from games.models import Game
-from games.forms import GameForm
-from django.conf import settings
-from PIL import Image
-from io import BytesIO
-from common.scraper import *
-
-
-class GoodreadsScraper(AbstractScraper):
-    site_name = SourceSiteEnum.GOODREADS.value
-    host = "www.goodreads.com"
-    data_class = Book
-    form_class = BookForm
-    regex = re.compile(r"https://www\.goodreads\.com/book/show/\d+")
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        u = re.match(r".+/book/show/(\d+)", raw_url)
-        if not u:
-            u = re.match(r".+book/(\d+)", raw_url)
-        return "https://www.goodreads.com/book/show/" + u[1] if u else None
-
-    def scrape(self, url, response=None):
-        """
-        This is the scraping portal
-        """
-        if response is not None:
-            content = html.fromstring(response.content.decode('utf-8'))
-        else:
-            headers = None  # DEFAULT_REQUEST_HEADERS.copy()
-            content = self.download_page(url, headers)
-
-        try:
-            title = content.xpath("//h1/text()")[0].strip()
-        except IndexError:
-            raise ValueError("given url contains no book info")
-
-        subtitle = None
-
-        orig_title_elem = content.xpath("//div[@id='bookDataBox']//div[text()='Original Title']/following-sibling::div/text()")
-        orig_title = orig_title_elem[0].strip() if orig_title_elem else None
-
-        language_elem = content.xpath('//div[@itemprop="inLanguage"]/text()')
-        language = language_elem[0].strip() if language_elem else None
-
-        pub_house_elem = content.xpath("//div[contains(text(), 'Published') and @class='row']/text()")
-        try:
-            months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
-            r = re.compile('.*Published.*(' + '|'.join(months) + ').*(\\d\\d\\d\\d).+by\\s*(.+)\\s*', re.DOTALL)
-            pub = r.match(pub_house_elem[0])
-            pub_year = pub[2]
-            pub_month = months.index(pub[1]) + 1
-            pub_house = pub[3].strip()
-        except Exception:
-            pub_year = None
-            pub_month = None
-            pub_house = None
-
-        pub_house_elem = content.xpath("//nobr[contains(text(), 'first published')]/text()")
-        try:
-            pub = re.match(r'.*first published\s+(.+\d\d\d\d).*', pub_house_elem[0], re.DOTALL)
-            first_pub = pub[1]
-        except Exception:
-            first_pub = None
-
-        binding_elem = content.xpath('//span[@itemprop="bookFormat"]/text()')
-        binding = binding_elem[0].strip() if binding_elem else None
-
-        pages_elem = content.xpath('//span[@itemprop="numberOfPages"]/text()')
-        pages = pages_elem[0].strip() if pages_elem else None
-        if pages is not None:
-            pages = int(RE_NUMBERS.findall(pages)[
-                        0]) if RE_NUMBERS.findall(pages) else None
-
-        isbn_elem = content.xpath('//span[@itemprop="isbn"]/text()')
-        if not isbn_elem:
-            isbn_elem = content.xpath('//div[@itemprop="isbn"]/text()')  # this is likely ASIN
-        isbn = isbn_elem[0].strip() if isbn_elem else None
-
-        brief_elem = content.xpath('//div[@id="description"]/span[@style="display:none"]/text()')
-        if brief_elem:
-            brief = '\n'.join(p.strip() for p in brief_elem)
-        else:
-            brief_elem = content.xpath('//div[@id="description"]/span/text()')
-            brief = '\n'.join(p.strip() for p in brief_elem) if brief_elem else None
-
-        genre = content.xpath('//div[@class="bigBoxBody"]/div/div/div/a/text()')
-        genre = genre[0] if genre else None
-        book_title = re.sub('\n', '', content.xpath('//h1[@id="bookTitle"]/text()')[0]).strip()
-        author = content.xpath('//a[@class="authorName"]/span/text()')[0]
-        contents = None
-
-        img_url_elem = content.xpath("//img[@id='coverImage']/@src")
-        img_url = img_url_elem[0].strip() if img_url_elem else None
-        raw_img, ext = self.download_image(img_url, url)
-
-        authors_elem = content.xpath("//a[@class='authorName'][not(../span[@class='authorName greyText smallText role'])]/span/text()")
-        if authors_elem:
-            authors = []
-            for author in authors_elem:
-                authors.append(RE_WHITESPACES.sub(' ', author.strip()))
-        else:
-            authors = None
-
-        translators = None
-        authors_elem = content.xpath("//a[@class='authorName'][../span/text()='(Translator)']/span/text()")
-        if authors_elem:
-            translators = []
-            for translator in authors_elem:
-                translators.append(RE_WHITESPACES.sub(' ', translator.strip()))
-        else:
-            translators = None
-
-        other = {}
-        if first_pub:
-            other['首版时间'] = first_pub
-        if genre:
-            other['分类'] = genre
-        series_elem = content.xpath("//h2[@id='bookSeries']/a/text()")
-        if series_elem:
-            other['丛书'] = re.sub(r'\(\s*(.+[^\s])\s*#.*\)', '\\1', series_elem[0].strip())
-
-        data = {
-            'title': title,
-            'subtitle': subtitle,
-            'orig_title': orig_title,
-            'author': authors,
-            'translator': translators,
-            'language': language,
-            'pub_house': pub_house,
-            'pub_year': pub_year,
-            'pub_month': pub_month,
-            'binding': binding,
-            'pages': pages,
-            'isbn': isbn,
-            'brief': brief,
-            'contents': contents,
-            'other_info': other,
-            'cover_url': img_url,
-            'source_site': self.site_name,
-            'source_url': self.get_effective_url(url),
-        }
-        data['source_url'] = self.get_effective_url(url)
-
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img

common/scrapers/google.py

@@ -1,102 +0,0 @@
-import requests
-import re
-import filetype
-from lxml import html
-from common.models import SourceSiteEnum
-from movies.models import Movie, MovieGenreEnum
-from movies.forms import MovieForm
-from books.models import Book
-from books.forms import BookForm
-from music.models import Album, Song
-from music.forms import AlbumForm, SongForm
-from games.models import Game
-from games.forms import GameForm
-from django.conf import settings
-from PIL import Image
-from io import BytesIO
-from common.scraper import *
-
-
-# https://developers.google.com/youtube/v3/docs/?apix=true
-# https://developers.google.com/books/docs/v1/using
-class GoogleBooksScraper(AbstractScraper):
-    site_name = SourceSiteEnum.GOOGLEBOOKS.value
-    host = ["books.google.com", "www.google.com/books"]
-    data_class = Book
-    form_class = BookForm
-    regex = re.compile(r"https://books\.google\.com/books\?id=([^&#]+)")
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        # https://books.google.com/books?id=wUHxzgEACAAJ
-        # https://books.google.com/books/about/%E7%8F%BE%E5%A0%B4%E6%AD%B7%E5%8F%B2.html?id=nvNoAAAAIAAJ
-        # https://www.google.com/books/edition/_/nvNoAAAAIAAJ?hl=en&gbpv=1
-        u = re.match(r"https://books\.google\.com/books.*id=([^&#]+)", raw_url)
-        if not u:
-            u = re.match(r"https://www\.google\.com/books/edition/[^/]+/([^&#?]+)", raw_url)
-        return 'https://books.google.com/books?id=' + u[1] if u else None
-
-    def scrape(self, url, response=None):
-        url = self.get_effective_url(url)
-        m = self.regex.match(url)
-        if m:
-            api_url = f'https://www.googleapis.com/books/v1/volumes/{m[1]}'
-        else:
-            raise ValueError("not valid url")
-        b = requests.get(api_url).json()
-        other = {}
-        title = b['volumeInfo']['title']
-        subtitle = b['volumeInfo']['subtitle'] if 'subtitle' in b['volumeInfo'] else None
-        pub_year = None
-        pub_month = None
-        if 'publishedDate' in b['volumeInfo']:
-            pub_date = b['volumeInfo']['publishedDate'].split('-')
-            pub_year = pub_date[0]
-            pub_month = pub_date[1] if len(pub_date) > 1 else None
-        pub_house = b['volumeInfo']['publisher'] if 'publisher' in b['volumeInfo'] else None
-        language = b['volumeInfo']['language'] if 'language' in b['volumeInfo'] else None
-        pages = b['volumeInfo']['pageCount'] if 'pageCount' in b['volumeInfo'] else None
-        if 'mainCategory' in b['volumeInfo']:
-            other['分类'] = b['volumeInfo']['mainCategory']
-        authors = b['volumeInfo']['authors'] if 'authors' in b['volumeInfo'] else None
-        if 'description' in b['volumeInfo']:
-            brief = b['volumeInfo']['description']
-        elif 'textSnippet' in b['volumeInfo']:
-            brief = b["volumeInfo"]["textSnippet"]["searchInfo"]
-        else:
-            brief = ''
-        brief = re.sub(r'<.*?>', '', brief.replace('<br', '\n<br'))
-        img_url = b['volumeInfo']['imageLinks']['thumbnail'] if 'imageLinks' in b['volumeInfo'] else None
-        isbn10 = None
-        isbn13 = None
-        for iid in b['volumeInfo']['industryIdentifiers'] if 'industryIdentifiers' in b['volumeInfo'] else []:
-            if iid['type'] == 'ISBN_10':
-                isbn10 = iid['identifier']
-            if iid['type'] == 'ISBN_13':
-                isbn13 = iid['identifier']
-        isbn = isbn13 if isbn13 is not None else isbn10
-
-        data = {
-            'title': title,
-            'subtitle': subtitle,
-            'orig_title': None,
-            'author': authors,
-            'translator': None,
-            'language': language,
-            'pub_house': pub_house,
-            'pub_year': pub_year,
-            'pub_month': pub_month,
-            'binding': None,
-            'pages': pages,
-            'isbn': isbn,
-            'brief': brief,
-            'contents': None,
-            'other_info': other,
-            'cover_url': img_url,
-            'source_site': self.site_name,
-            'source_url': self.get_effective_url(url),
-        }
-        raw_img, ext = self.download_image(img_url, url)
-
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img

common/scrapers/igdb.py

@@ -1,101 +0,0 @@
-import requests
-import re
-from common.models import SourceSiteEnum
-from games.models import Game
-from games.forms import GameForm
-from django.conf import settings
-from common.scraper import *
-from igdb.wrapper import IGDBWrapper
-import json
-import datetime
-import logging
-
-
-_logger = logging.getLogger(__name__)
-
-
-def _igdb_access_token():
-    try:
-        token = requests.post(f'https://id.twitch.tv/oauth2/token?client_id={settings.IGDB_CLIENT_ID}&client_secret={settings.IGDB_CLIENT_SECRET}&grant_type=client_credentials').json()['access_token']
-    except Exception:
-        _logger.error('unable to obtain IGDB token')
-        token = '<invalid>'
-    return token
-
-
-wrapper = IGDBWrapper(settings.IGDB_CLIENT_ID, _igdb_access_token())
-
-
-class IgdbGameScraper(AbstractScraper):
-    site_name = SourceSiteEnum.IGDB.value
-    host = 'https://www.igdb.com/'
-    data_class = Game
-    form_class = GameForm
-    regex = re.compile(r"https://www\.igdb\.com/games/([a-zA-Z0-9\-_]+)")
-
-    def scrape_steam(self, steam_url):
-        r = json.loads(wrapper.api_request('websites', f'fields *, game.*; where url = "{steam_url}";'))
-        if not r:
-            raise ValueError("Cannot find steam url in IGDB")
-        r = sorted(r, key=lambda w: w['game']['id'])
-        return self.scrape(r[0]['game']['url'])
-
-    def scrape(self, url):
-        m = self.regex.match(url)
-        if m:
-            effective_url = m[0]
-        else:
-            raise ValueError("not valid url")
-        effective_url = m[0]
-        slug = m[1]
-        fields = '*, cover.url, genres.name, platforms.name, involved_companies.*, involved_companies.company.name'
-        r = json.loads(wrapper.api_request('games', f'fields {fields}; where url = "{effective_url}";'))[0]
-        brief = r['summary'] if 'summary' in r else ''
-        brief += "\n\n" + r['storyline'] if 'storyline' in r else ''
-        developer = None
-        publisher = None
-        release_date = None
-        genre = None
-        platform = None
-        if 'involved_companies' in r:
-            developer = next(iter([c['company']['name'] for c in r['involved_companies'] if c['developer'] == True]), None)
-            publisher = next(iter([c['company']['name'] for c in r['involved_companies'] if c['publisher'] == True]), None)
-        if 'platforms' in r:
-            ps = sorted(r['platforms'], key=lambda p: p['id'])
-            platform = [(p['name'] if p['id'] != 6 else 'Windows') for p in ps]
-        if 'first_release_date' in r:
-            release_date = datetime.datetime.fromtimestamp(r['first_release_date'], datetime.timezone.utc)
-        if 'genres' in r:
-            genre = [g['name'] for g in r['genres']]
-        other_info = {'igdb_id': r['id']}
-        websites = json.loads(wrapper.api_request('websites', f'fields *; where game.url = "{effective_url}";'))
-        for website in websites:
-            if website['category'] == 1:
-                other_info['official_site'] = website['url']
-            elif website['category'] == 13:
-                other_info['steam_url'] = website['url']
-        data = {
-            'title': r['name'],
-            'other_title': None,
-            'developer': developer,
-            'publisher': publisher,
-            'release_date': release_date,
-            'genre': genre,
-            'platform': platform,
-            'brief': brief,
-            'other_info': other_info,
-            'source_site': self.site_name,
-            'source_url': self.get_effective_url(url),
-        }
-        raw_img, ext = self.download_image('https:' + r['cover']['url'].replace('t_thumb', 't_cover_big'), url)
-
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        m = cls.regex.match(raw_url)
-        if m:
-            return m[0]
-        else:
-            return None

common/scrapers/imdb.py

@@ -1,116 +0,0 @@
-import requests
-import re
-from common.models import SourceSiteEnum
-from movies.forms import MovieForm
-from movies.models import Movie
-from django.conf import settings
-from common.scraper import *
-
-
-class ImdbMovieScraper(AbstractScraper):
-    site_name = SourceSiteEnum.IMDB.value
-    host = 'https://www.imdb.com/title/'
-    data_class = Movie
-    form_class = MovieForm
-
-    regex = re.compile(r"(?<=https://www\.imdb\.com/title/)[a-zA-Z0-9]+")
-
-    def scrape(self, url):
-        effective_url = self.get_effective_url(url)
-        if effective_url is None:
-            raise ValueError("not valid url")
-        code = self.regex.findall(effective_url)[0]
-        s = TmdbMovieScraper()
-        s.scrape_imdb(code)
-        self.raw_data = s.raw_data
-        self.raw_img = s.raw_img
-        self.img_ext = s.img_ext
-        self.raw_data['source_site'] = self.site_name
-        self.raw_data['source_url'] = effective_url
-        return self.raw_data, self.raw_img
-
-        api_url = self.get_api_url(effective_url)
-        r = requests.get(api_url)
-        res_data = r.json()
-
-        if not res_data['type'] in ['Movie', 'TVSeries']:
-            raise ValueError("not movie/series item")
-
-        if res_data['type'] == 'Movie':
-            is_series = False
-        elif res_data['type'] == 'TVSeries':
-            is_series = True
-
-        title = res_data['title']
-        orig_title = res_data['originalTitle']
-        imdb_code = self.regex.findall(effective_url)[0]
-        director = []
-        for direct_dict in res_data['directorList']:
-            director.append(direct_dict['name'])
-        playwright = []
-        for writer_dict in res_data['writerList']:
-            playwright.append(writer_dict['name'])
-        actor = []
-        for actor_dict in res_data['actorList']:
-            actor.append(actor_dict['name'])
-        genre = res_data['genres'].split(', ')
-        area = res_data['countries'].split(', ')
-        language = res_data['languages'].split(', ')
-        year = int(res_data['year'])
-        duration = res_data['runtimeStr']
-        brief = res_data['plotLocal'] if res_data['plotLocal'] else res_data['plot']
-        if res_data['releaseDate']:
-            showtime = [{res_data['releaseDate']: "发布日期"}]
-        else:
-            showtime = None
-
-        other_info = {}
-        if res_data['contentRating']:
-            other_info['分级'] = res_data['contentRating']
-        if res_data['imDbRating']:
-            other_info['IMDb评分'] = res_data['imDbRating']
-        if res_data['metacriticRating']:
-            other_info['Metacritic评分'] = res_data['metacriticRating']
-        if res_data['awards']:
-            other_info['奖项'] = res_data['awards']
-
-        raw_img, ext = self.download_image(res_data['image'], url)
-
-        data = {
-            'title': title,
-            'orig_title': orig_title,
-            'other_title': None,
-            'imdb_code': imdb_code,
-            'director': director,
-            'playwright': playwright,
-            'actor': actor,
-            'genre': genre,
-            'showtime': showtime,
-            'site': None,
-            'area': area,
-            'language': language,
-            'year': year,
-            'duration': duration,
-            'season': None,
-            'episodes': None,
-            'single_episode_length': None,
-            'brief': brief,
-            'is_series': is_series,
-            'other_info': other_info,
-            'source_site': self.site_name,
-            'source_url': effective_url,
-        }
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        code = cls.regex.findall(raw_url)
-        if code:
-            return f"https://www.imdb.com/title/{code[0]}/"
-        else:
-            return None
-
-    @classmethod
-    def get_api_url(cls, url):
-        return f"https://imdb-api.com/zh/API/Title/{settings.IMDB_API_KEY}/{cls.regex.findall(url)[0]}/FullActor,"

common/scrapers/spotify.py

@@ -1,287 +0,0 @@
-import requests
-import re
-import time
-from common.models import SourceSiteEnum
-from music.models import Album, Song
-from music.forms import AlbumForm, SongForm
-from django.conf import settings
-from common.scraper import *
-from threading import Thread
-from django.core.exceptions import ObjectDoesNotExist
-from django.utils import timezone
-
-
-spotify_token = None
-spotify_token_expire_time = time.time()
-
-
-class SpotifyTrackScraper(AbstractScraper):
-    site_name = SourceSiteEnum.SPOTIFY.value
-    host = 'https://open.spotify.com/track/'
-    data_class = Song
-    form_class = SongForm
-
-    regex = re.compile(r"(?<=https://open\.spotify\.com/track/)[a-zA-Z0-9]+")
-
-    def scrape(self, url):
-        """
-        Request from API, not really scraping
-        """
-        global spotify_token, spotify_token_expire_time
-
-        if spotify_token is None or is_spotify_token_expired():
-            invoke_spotify_token()
-        effective_url = self.get_effective_url(url)
-        if effective_url is None:
-            raise ValueError("not valid url")
-
-        api_url = self.get_api_url(effective_url)
-        headers = {
-            'Authorization': f"Bearer {spotify_token}"
-        }
-        r = requests.get(api_url, headers=headers)
-        res_data = r.json()
-
-        artist = []
-        for artist_dict in res_data['artists']:
-            artist.append(artist_dict['name'])
-        if not artist:
-            artist = None
-
-        title = res_data['name']
-
-        release_date = parse_date(res_data['album']['release_date'])
-
-        duration = res_data['duration_ms']
-
-        if res_data['external_ids'].get('isrc'):
-            isrc = res_data['external_ids']['isrc']
-        else:
-            isrc = None
-
-        raw_img, ext = self.download_image(res_data['album']['images'][0]['url'], url)
-
-        data = {
-            'title': title,
-            'artist': artist,
-            'genre': None,
-            'release_date': release_date,
-            'duration': duration,
-            'isrc': isrc,
-            'album': None,
-            'brief': None,
-            'other_info': None,
-            'source_site': self.site_name,
-            'source_url': effective_url,
-        }
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        code = cls.regex.findall(raw_url)
-        if code:
-            return f"https://open.spotify.com/track/{code[0]}"
-        else:
-            return None
-
-    @classmethod
-    def get_api_url(cls, url):
-        return "https://api.spotify.com/v1/tracks/" + cls.regex.findall(url)[0]
-
-
-class SpotifyAlbumScraper(AbstractScraper):
-    site_name = SourceSiteEnum.SPOTIFY.value
-    # API URL
-    host = 'https://open.spotify.com/album/'
-    data_class = Album
-    form_class = AlbumForm
-
-    regex = re.compile(r"(?<=https://open\.spotify\.com/album/)[a-zA-Z0-9]+")
-
-    def scrape(self, url):
-        """
-        Request from API, not really scraping
-        """
-        global spotify_token, spotify_token_expire_time
-
-        if spotify_token is None or is_spotify_token_expired():
-            invoke_spotify_token()
-        effective_url = self.get_effective_url(url)
-        if effective_url is None:
-            raise ValueError("not valid url")
-
-        api_url = self.get_api_url(effective_url)
-        headers = {
-            'Authorization': f"Bearer {spotify_token}"
-        }
-        r = requests.get(api_url, headers=headers)
-        res_data = r.json()
-
-        artist = []
-        for artist_dict in res_data['artists']:
-            artist.append(artist_dict['name'])
-
-        title = res_data['name']
-
-        genre = ', '.join(res_data['genres'])
-
-        company = []
-        for com in res_data['copyrights']:
-            company.append(com['text'])
-
-        duration = 0
-        track_list = []
-        track_urls = []
-        for track in res_data['tracks']['items']:
-            track_urls.append(track['external_urls']['spotify'])
-            duration += track['duration_ms']
-            if res_data['tracks']['items'][-1]['disc_number'] > 1:
-                # more than one disc
-                track_list.append(str(
-                    track['disc_number']) + '-' + str(track['track_number']) + '. ' + track['name'])
-            else:
-                track_list.append(str(track['track_number']) + '. ' + track['name'])
-        track_list = '\n'.join(track_list)
-
-        release_date = parse_date(res_data['release_date'])
-
-        other_info = {}
-        if res_data['external_ids'].get('upc'):
-            # bar code
-            other_info['UPC'] = res_data['external_ids']['upc']
-
-        raw_img, ext = self.download_image(res_data['images'][0]['url'], url)
-
-        data = {
-            'title': title,
-            'artist': artist,
-            'genre': genre,
-            'track_list': track_list,
-            'release_date': release_date,
-            'duration': duration,
-            'company': company,
-            'brief': None,
-            'other_info': other_info,
-            'source_site': self.site_name,
-            'source_url': effective_url,
-        }
-
-        # set tracks_data, used for adding tracks
-        self.track_urls = track_urls
-
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        code = cls.regex.findall(raw_url)
-        if code:
-            return f"https://open.spotify.com/album/{code[0]}"
-        else:
-            return None
-
-    # @classmethod
-    # def save(cls, request_user):
-    #     form = super().save(request_user)
-    #     task = Thread(
-    #         target=cls.add_tracks,
-    #         args=(form.instance, request_user),
-    #         daemon=True
-    #     )
-    #     task.start()
-    #     return form
-
-    @classmethod
-    def get_api_url(cls, url):
-        return "https://api.spotify.com/v1/albums/" + cls.regex.findall(url)[0]
-
-    @classmethod
-    def add_tracks(cls, album: Album, request_user):
-        to_be_updated_tracks = []
-        for track_url in cls.track_urls:
-            track = cls.get_track_or_none(track_url)
-            # seems lik if fire too many requests at the same time
-            # spotify would limit access
-            if track is None:
-                task = Thread(
-                    target=cls.scrape_and_save_track,
-                    args=(track_url, album, request_user),
-                    daemon=True
-                )
-                task.start()
-                task.join()
-            else:
-                to_be_updated_tracks.append(track)
-        cls.bulk_update_track_album(to_be_updated_tracks, album, request_user)
-
-    @classmethod
-    def get_track_or_none(cls, track_url: str):
-        try:
-            instance = Song.objects.get(source_url=track_url)
-            return instance
-        except ObjectDoesNotExist:
-            return None
-
-    @classmethod
-    def scrape_and_save_track(cls, url: str, album: Album, request_user):
-        data, img = SpotifyTrackScraper.scrape(url)
-        SpotifyTrackScraper.raw_data['album'] = album
-        SpotifyTrackScraper.save(request_user)
-
-    @classmethod
-    def bulk_update_track_album(cls, tracks, album, request_user):
-        for track in tracks:
-            track.last_editor = request_user
-            track.edited_time = timezone.now()
-            track.album = album
-        Song.objects.bulk_update(tracks, [
-            'last_editor',
-            'edited_time',
-            'album'
-        ])
-
-
-def get_spotify_token():
-    global spotify_token, spotify_token_expire_time
-    if spotify_token is None or is_spotify_token_expired():
-        invoke_spotify_token()
-    return spotify_token
-
-    
-def is_spotify_token_expired():
-    global spotify_token_expire_time
-    return True if spotify_token_expire_time <= time.time() else False
-
-
-def invoke_spotify_token():
-    global spotify_token, spotify_token_expire_time
-    r = requests.post(
-        "https://accounts.spotify.com/api/token",
-        data={
-            "grant_type": "client_credentials"
-        },
-        headers={
-            "Authorization": f"Basic {settings.SPOTIFY_CREDENTIAL}"
-        }
-    )
-    data = r.json()
-    if r.status_code == 401:
-        # token expired, try one more time
-        # this maybe caused by external operations,
-        # for example debugging using a http client
-        r = requests.post(
-            "https://accounts.spotify.com/api/token",
-            data={
-                "grant_type": "client_credentials"
-            },
-            headers={
-                "Authorization": f"Basic {settings.SPOTIFY_CREDENTIAL}"
-            }
-        )
-        data = r.json()
-    elif r.status_code != 200:
-        raise Exception(f"Request to spotify API fails. Reason: {r.reason}")
-    # minus 2 for execution time error
-    spotify_token_expire_time = int(data['expires_in']) + time.time() - 2
-    spotify_token = data['access_token']

common/scrapers/steam.py

@@ -1,92 +0,0 @@
-import re
-from common.models import SourceSiteEnum
-from games.models import Game
-from games.forms import GameForm
-from common.scraper import *
-from common.scrapers.igdb import IgdbGameScraper
-
-
-class SteamGameScraper(AbstractScraper):
-    site_name = SourceSiteEnum.STEAM.value
-    host = 'store.steampowered.com'
-    data_class = Game
-    form_class = GameForm
-
-    regex = re.compile(r"https://store\.steampowered\.com/app/\d+")
-
-    def scrape(self, url):
-        m = self.regex.match(url)
-        if m:
-            effective_url = m[0]
-        else:
-            raise ValueError("not valid url")
-        try:
-            s = IgdbGameScraper()
-            s.scrape_steam(effective_url)
-            self.raw_data = s.raw_data
-            self.raw_img = s.raw_img
-            self.img_ext = s.img_ext
-            self.raw_data['source_site'] = self.site_name
-            self.raw_data['source_url'] = effective_url
-            # return self.raw_data, self.raw_img
-        except:
-            self.raw_img = None
-            self.raw_data = {}
-        headers = DEFAULT_REQUEST_HEADERS.copy()
-        headers['Host'] = self.host
-        headers['Cookie'] = "wants_mature_content=1; birthtime=754700401;"
-        content = self.download_page(url, headers)
-
-        title = content.xpath("//div[@class='apphub_AppName']/text()")[0]
-        developer = content.xpath("//div[@id='developers_list']/a/text()")
-        publisher = content.xpath("//div[@class='glance_ctn']//div[@class='dev_row'][2]//a/text()")
-        release_date = parse_date(
-            content.xpath(
-                "//div[@class='release_date']/div[@class='date']/text()")[0]
-        )
-
-        genre = content.xpath(
-            "//div[@class='details_block']/b[2]/following-sibling::a/text()")
-
-        platform = ['PC']
-
-        brief = content.xpath(
-            "//div[@class='game_description_snippet']/text()")[0].strip()
-
-        img_url = content.xpath(
-            "//img[@class='game_header_image_full']/@src"
-        )[0].replace("header.jpg", "library_600x900.jpg")
-        raw_img, img_ext = self.download_image(img_url, url)
-
-        # no 600x900 picture
-        if raw_img is None:
-            img_url = content.xpath("//img[@class='game_header_image_full']/@src")[0]
-            raw_img, img_ext = self.download_image(img_url, url)
-
-        if raw_img is not None:
-            self.raw_img = raw_img
-            self.img_ext = img_ext
-
-        data = {
-            'title': title if title else self.raw_data['title'],
-            'other_title': None,
-            'developer': developer if 'developer' not in self.raw_data else self.raw_data['developer'],
-            'publisher': publisher if 'publisher' not in self.raw_data else self.raw_data['publisher'],
-            'release_date': release_date if 'release_date' not in self.raw_data else self.raw_data['release_date'],
-            'genre': genre if 'genre' not in self.raw_data else self.raw_data['genre'],
-            'platform': platform if 'platform' not in self.raw_data else self.raw_data['platform'],
-            'brief': brief if brief else self.raw_data['brief'],
-            'other_info': None if 'other_info' not in self.raw_data else self.raw_data['other_info'],
-            'source_site': self.site_name,
-            'source_url': effective_url
-        }
-        self.raw_data = data
-        return self.raw_data, self.raw_img
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        m = cls.regex.match(raw_url)
-        if m:
-            return m[0]
-        else:
-            return None

common/scrapers/tmdb.py

@@ -1,150 +0,0 @@
-import requests
-import re
-from common.models import SourceSiteEnum
-from movies.models import Movie
-from movies.forms import MovieForm
-from django.conf import settings
-from common.scraper import *
-
-
-class TmdbMovieScraper(AbstractScraper):
-    site_name = SourceSiteEnum.TMDB.value
-    host = 'https://www.themoviedb.org/'
-    data_class = Movie
-    form_class = MovieForm
-    regex = re.compile(r"https://www\.themoviedb\.org/(movie|tv)/([a-zA-Z0-9]+)")
-    # http://api.themoviedb.org/3/genre/movie/list?api_key=&language=zh
-    # http://api.themoviedb.org/3/genre/tv/list?api_key=&language=zh
-    genre_map = {
-        'Sci-Fi & Fantasy': 'Sci-Fi',
-        'War & Politics':   'War',
-        '儿童':             'Kids',
-        '冒险':             'Adventure',
-        '剧情':             'Drama',
-        '动作':             'Action',
-        '动作冒险':         'Action',
-        '动画':             'Animation',
-        '历史':             'History',
-        '喜剧':             'Comedy',
-        '奇幻':             'Fantasy',
-        '家庭':             'Family',
-        '恐怖':             'Horror',
-        '悬疑':             'Mystery',
-        '惊悚':             'Thriller',
-        '战争':             'War',
-        '新闻':             'News',
-        '爱情':             'Romance',
-        '犯罪':             'Crime',
-        '电视电影':         'TV Movie',
-        '真人秀':           'Reality-TV',
-        '科幻':             'Sci-Fi',
-        '纪录':             'Documentary',
-        '肥皂剧':           'Soap',
-        '脱口秀':           'Talk-Show',
-        '西部':             'Western',
-        '音乐':             'Music',
-    }
-
-    def scrape_imdb(self, imdb_code):
-        api_url = f"https://api.themoviedb.org/3/find/{imdb_code}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&external_source=imdb_id"
-        r = requests.get(api_url)
-        res_data = r.json()
-        if 'movie_results' in res_data and len(res_data['movie_results']) > 0:
-            url = f"https://www.themoviedb.org/movie/{res_data['movie_results'][0]['id']}"
-        elif 'tv_results' in res_data and len(res_data['tv_results']) > 0:
-            url = f"https://www.themoviedb.org/tv/{res_data['tv_results'][0]['id']}"
-        else:
-            raise ValueError("Cannot find IMDb ID in TMDB")
-        return self.scrape(url)
-
-    def scrape(self, url):
-        m = self.regex.match(url)
-        if m:
-            effective_url = m[0]
-        else:
-            raise ValueError("not valid url")
-        effective_url = m[0]
-        is_series = m[1] == 'tv'
-        id = m[2]
-        if is_series:
-            api_url = f"https://api.themoviedb.org/3/tv/{id}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits"
-        else:
-            api_url = f"https://api.themoviedb.org/3/movie/{id}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits"
-        r = requests.get(api_url)
-        res_data = r.json()
-
-        if is_series:
-            title = res_data['name']
-            orig_title = res_data['original_name']
-            year = int(res_data['first_air_date'].split('-')[0]) if res_data['first_air_date'] else None
-            imdb_code = res_data['external_ids']['imdb_id']
-            showtime = [{res_data['first_air_date']: "首播日期"}] if res_data['first_air_date'] else None
-            duration = None
-        else:
-            title = res_data['title']
-            orig_title = res_data['original_title']
-            year = int(res_data['release_date'].split('-')[0]) if res_data['release_date'] else None
-            showtime = [{res_data['release_date']: "发布日期"}] if res_data['release_date'] else None
-            imdb_code = res_data['imdb_id']
-            duration = res_data['runtime'] if res_data['runtime'] else None # in minutes
-
-        genre = list(map(lambda x: self.genre_map[x['name']] if x['name'] in self.genre_map else 'Other', res_data['genres']))
-        language = list(map(lambda x: x['name'], res_data['spoken_languages']))
-        brief = res_data['overview']
-
-        if is_series:
-            director = list(map(lambda x: x['name'], res_data['created_by']))
-        else:
-            director = list(map(lambda x: x['name'], filter(lambda c: c['job'] == 'Director', res_data['credits']['crew'])))
-        playwright = list(map(lambda x: x['name'], filter(lambda c: c['job'] == 'Screenplay', res_data['credits']['crew'])))
-        actor = list(map(lambda x: x['name'], res_data['credits']['cast']))
-        area = []
-
-        other_info = {}
-        other_info['TMDB评分'] = res_data['vote_average']
-        # other_info['分级'] = res_data['contentRating']
-        # other_info['Metacritic评分'] = res_data['metacriticRating']
-        # other_info['奖项'] = res_data['awards']
-        other_info['TMDB_ID'] = id
-        if is_series:
-            other_info['Seasons'] = res_data['number_of_seasons']
-            other_info['Episodes'] = res_data['number_of_episodes']
-
-        img_url = ('https://image.tmdb.org/t/p/original/' + res_data['poster_path']) if res_data['poster_path'] is not None else None
-        # TODO: use GET /configuration to get base url
-        raw_img, ext = self.download_image(img_url, url)
-
-        data = {
-            'title': title,
-            'orig_title': orig_title,
-            'other_title': None,
-            'imdb_code': imdb_code,
-            'director': director,
-            'playwright': playwright,
-            'actor': actor,
-            'genre': genre,
-            'showtime': showtime,
-            'site': None,
-            'area': area,
-            'language': language,
-            'year': year,
-            'duration': duration,
-            'season': None,
-            'episodes': None,
-            'single_episode_length': None,
-            'brief': brief,
-            'is_series': is_series,
-            'other_info': other_info,
-            'source_site': self.site_name,
-            'source_url': effective_url,
-        }
-        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
-        return data, raw_img
-
-    @classmethod
-    def get_effective_url(cls, raw_url):
-        m = cls.regex.match(raw_url)
-        if raw_url:
-            return m[0]
-        else:
-            return None

common/searcher.py

@@ -1,209 +0,0 @@
-from urllib.parse import quote_plus
-from enum import Enum
-from common.models import SourceSiteEnum
-from django.conf import settings
-from common.scrapers.goodreads import GoodreadsScraper
-from common.scrapers.spotify import get_spotify_token
-import requests
-from lxml import html
-import logging
-
-SEARCH_PAGE_SIZE = 5  # not all apis support page size
-logger = logging.getLogger(__name__)
-
-
-class Category(Enum):
-    Book = '书籍'
-    Movie = '电影'
-    Music = '音乐'
-    Game = '游戏'
-    TV = '剧集'
-
-
-class SearchResultItem:
-    def __init__(self, category, source_site, source_url, title, subtitle, brief, cover_url):
-        self.category = category
-        self.source_site = source_site
-        self.source_url = source_url
-        self.title = title
-        self.subtitle = subtitle
-        self.brief = brief
-        self.cover_url = cover_url
-
-    @property
-    def verbose_category_name(self):
-        return self.category.value
-
-    @property
-    def link(self):
-        return f"/search?q={quote_plus(self.source_url)}"
-
-    @property
-    def scraped(self):
-        return False
-
-
-class ProxiedRequest:
-    @classmethod
-    def get(cls, url):
-        u = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={quote_plus(url)}'
-        return requests.get(u, timeout=10)
-
-
-class Goodreads:
-    @classmethod
-    def search(self, q, page=1):
-        results = []
-        try:
-            search_url = f'https://www.goodreads.com/search?page={page}&q={quote_plus(q)}'
-            r = requests.get(search_url)
-            if r.url.startswith('https://www.goodreads.com/book/show/'):
-                # Goodreads will 302 if only one result matches ISBN
-                data, img = GoodreadsScraper.scrape(r.url, r)
-                subtitle = f"{data['pub_year']} {', '.join(data['author'])} {', '.join(data['translator'] if data['translator'] else [])}"
-                results.append(SearchResultItem(Category.Book, SourceSiteEnum.GOODREADS,
-                                                data['source_url'], data['title'], subtitle,
-                                                data['brief'], data['cover_url']))
-            else:
-                h = html.fromstring(r.content.decode('utf-8'))
-                for c in h.xpath('//tr[@itemtype="http://schema.org/Book"]'):
-                    el_cover = c.xpath('.//img[@class="bookCover"]/@src')
-                    cover = el_cover[0] if el_cover else None
-                    el_title = c.xpath('.//a[@class="bookTitle"]//text()')
-                    title = ''.join(el_title).strip() if el_title else None
-                    el_url = c.xpath('.//a[@class="bookTitle"]/@href')
-                    url = 'https://www.goodreads.com' + \
-                        el_url[0] if el_url else None
-                    el_authors = c.xpath('.//a[@class="authorName"]//text()')
-                    subtitle = ', '.join(el_authors) if el_authors else None
-                    results.append(SearchResultItem(
-                        Category.Book, SourceSiteEnum.GOODREADS, url, title, subtitle, '', cover))
-        except Exception as e:
-            logger.error(f"Goodreads search '{q}' error: {e}")
-        return results
-
-
-class GoogleBooks:
-    @classmethod
-    def search(self, q, page=1):
-        results = []
-        try:
-            api_url = f'https://www.googleapis.com/books/v1/volumes?country=us&q={quote_plus(q)}&startIndex={SEARCH_PAGE_SIZE*(page-1)}&maxResults={SEARCH_PAGE_SIZE}&maxAllowedMaturityRating=MATURE'
-            j = requests.get(api_url).json()
-            if 'items' in j:
-                for b in j['items']:
-                    if 'title' not in b['volumeInfo']:
-                        continue
-                    title = b['volumeInfo']['title']
-                    subtitle = ''
-                    if 'publishedDate' in b['volumeInfo']:
-                        subtitle += b['volumeInfo']['publishedDate'] + ' '
-                    if 'authors' in b['volumeInfo']:
-                        subtitle += ', '.join(b['volumeInfo']['authors'])
-                    if 'description' in b['volumeInfo']:
-                        brief = b['volumeInfo']['description']
-                    elif 'textSnippet' in b['volumeInfo']:
-                        brief = b["volumeInfo"]["textSnippet"]["searchInfo"]
-                    else:
-                        brief = ''
-                    category = Category.Book
-                    # b['volumeInfo']['infoLink'].replace('http:', 'https:')
-                    url = 'https://books.google.com/books?id=' + b['id']
-                    cover = b['volumeInfo']['imageLinks']['thumbnail'] if 'imageLinks' in b['volumeInfo'] else None
-                    results.append(SearchResultItem(
-                        category, SourceSiteEnum.GOOGLEBOOKS, url, title, subtitle, brief, cover))
-        except Exception as e:
-            logger.error(f"GoogleBooks search '{q}' error: {e}")
-        return results
-
-
-class TheMovieDatabase:
-    @classmethod
-    def search(self, q, page=1):
-        results = []
-        try:
-            api_url = f'https://api.themoviedb.org/3/search/multi?query={quote_plus(q)}&page={page}&api_key={settings.TMDB_API3_KEY}&language=zh-CN&include_adult=true'
-            j = requests.get(api_url).json()
-            for m in j['results']:
-                if m['media_type'] in ['tv', 'movie']:
-                    url = f"https://www.themoviedb.org/{m['media_type']}/{m['id']}"
-                    if m['media_type'] == 'tv':
-                        cat = Category.TV
-                        title = m['name']
-                        subtitle = f"{m.get('first_air_date')} {m.get('original_name')}"
-                    else:
-                        cat = Category.Movie
-                        title = m['title']
-                        subtitle = f"{m.get('release_date')} {m.get('original_name')}"
-                    cover = f"https://image.tmdb.org/t/p/w500/{m.get('poster_path')}"
-                    results.append(SearchResultItem(
-                        cat, SourceSiteEnum.TMDB, url, title, subtitle, m.get('overview'), cover))
-        except Exception as e:
-            logger.error(f"TMDb search '{q}' error: {e}")
-        return results
-
-
-class Spotify:
-    @classmethod
-    def search(self, q, page=1):
-        results = []
-        try:
-            api_url = f"https://api.spotify.com/v1/search?q={q}&type=album&limit={SEARCH_PAGE_SIZE}&offset={page*SEARCH_PAGE_SIZE}"
-            headers = {
-                'Authorization': f"Bearer {get_spotify_token()}"
-            }
-            j = requests.get(api_url, headers=headers).json()
-            for a in j['albums']['items']:
-                title = a['name']
-                subtitle = a['release_date']
-                for artist in a['artists']:
-                    subtitle += ' ' + artist['name']
-                url = a['external_urls']['spotify']
-                cover = a['images'][0]['url']
-                results.append(SearchResultItem(
-                    Category.Music, SourceSiteEnum.SPOTIFY, url, title, subtitle, '', cover))
-        except Exception as e:
-            logger.error(f"Spotify search '{q}' error: {e}")
-        return results
-
-
-class Bandcamp:
-    @classmethod
-    def search(self, q, page=1):
-        results = []
-        try:
-            search_url = f'https://bandcamp.com/search?from=results&item_type=a&page={page}&q={quote_plus(q)}'
-            r = requests.get(search_url)
-            h = html.fromstring(r.content.decode('utf-8'))
-            for c in h.xpath('//li[@class="searchresult data-search"]'):
-                el_cover = c.xpath('.//div[@class="art"]/img/@src')
-                cover = el_cover[0] if el_cover else None
-                el_title = c.xpath('.//div[@class="heading"]//text()')
-                title = ''.join(el_title).strip() if el_title else None
-                el_url = c.xpath('..//div[@class="itemurl"]/a/@href')
-                url = el_url[0] if el_url else None
-                el_authors = c.xpath('.//div[@class="subhead"]//text()')
-                subtitle = ', '.join(el_authors) if el_authors else None
-                results.append(SearchResultItem(Category.Music, SourceSiteEnum.BANDCAMP, url, title, subtitle, '', cover))
-        except Exception as e:
-            logger.error(f"Goodreads search '{q}' error: {e}")
-        return results
-
-
-class ExternalSources:
-    @classmethod
-    def search(self, c, q, page=1):
-        if not q:
-            return []
-        results = []
-        if c == '' or c is None:
-            c = 'all'
-        if c == 'all' or c == 'movie':
-            results.extend(TheMovieDatabase.search(q, page))
-        if c == 'all' or c == 'book':
-            results.extend(GoogleBooks.search(q, page))
-            results.extend(Goodreads.search(q, page))
-        if c == 'all' or c == 'music':
-            results.extend(Spotify.search(q, page))
-            results.extend(Bandcamp.search(q, page))
-        return results

common/templates/common/search_result.html

@@ -1,243 +0,0 @@
-{% load static %}
-{% load i18n %}
-{% load l10n %}
-{% load humanize %}
-{% load admin_url %}
-{% load mastodon %}
-{% load oauth_token %}
-{% load truncate %}
-{% load highlight %}
-{% load thumb %}
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{{ site_name }} - {% trans '搜索结果' %}</title>
-    <script src="https://cdn.staticfile.org/jquery/3.6.1/jquery.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/htmx/1.8.4/htmx.min.js"></script>
-    <script src="{% static 'lib/js/rating-star.js' %}"></script>
-    <script src="{% static 'js/rating-star-readonly.js' %}"></script>
-    <link rel="stylesheet" href="{% static 'css/boofilsic.min.css' %}">
-    <link rel="stylesheet" href="{% static 'lib/css/rating-star.css' %}">
-    <link rel="stylesheet" href="{% static 'lib/css/collection.css' %}">
-</head>
-
-<body>
-    <div id="page-wrapper">
-        <div id="content-wrapper">
-            {% include 'partial/_navbar.html' %}
-        
-            <section id="content">
-                <div class="grid">
-                    <div class="grid__main">
-                        <div class="main-section-wrapper">
-
-                            <div class="entity-list">
-                                {% if request.GET.q %}
-                                <h5 class="entity-list__title">“{{ request.GET.q }}” {% trans '的搜索结果' %}</h5>
-                                {% endif %}
-                                
-                                {% if request.GET.tag %}
-                                <h5 class="entity-list__title">{% trans '含有标签' %} “{{ request.GET.tag }}” {% trans '的结果' %}</h5>
-                                {% endif %}
-                                    
-                                <ul class="entity-list__entities">
-                                    {% for item in items %}
-                                    {% include "partial/list_item.html" %}
-                                    {% empty %}
-                                    <li class="entity-list__entity">
-                                    {% trans '无站内条目匹配' %}
-                                    </li>
-                                    {% endfor %}
-                                    {% if request.GET.q and user.is_authenticated %}
-                                    <li class="entity-list__entity" hx-get="{% url 'common:external_search' %}?q={{ request.GET.q }}&c={{ request.GET.c }}&page={% if pagination.current_page %}{{ pagination.current_page }}{% else %}1{% endif %}" hx-trigger="load" hx-swap="outerHTML">
-                                    {% trans '正在实时搜索站外条目' %}
-                                    <div id="spinner">
-                                        <div class="spinner">
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                            <div></div>
-                                        </div>
-                                    </div>
-                                    </li>
-                                    {% endif %}
-                                </ul>
-                            </div>
-                            <div class="pagination" >
-                                
-                                {% if pagination.has_prev %}
-                                    <a href="?page=1&{% if request.GET.q %}q={{ request.GET.q }}{% elif request.GET.tag %}tag={{ request.GET.tag }}{% endif %}{% if request.GET.c %}&c={{ request.GET.c }}{% endif %}" class="pagination__nav-link pagination__nav-link">&laquo;</a>
-                                    <a href="?page={{ pagination.previous_page }}&{% if request.GET.q %}q={{ request.GET.q }}{% elif request.GET.tag %}tag={{ request.GET.tag }}{% endif %}{% if request.GET.c %}&c={{ request.GET.c }}{% endif %}" class="pagination__nav-link pagination__nav-link--right-margin pagination__nav-link">&lsaquo;</a>
-                                {% endif %}
-                                    
-                                {% for page in pagination.page_range %}
-                                    
-                                    {% if page == pagination.current_page %}
-                                    <a href="?page={{ page }}&{% if request.GET.q %}q={{ request.GET.q }}{% elif request.GET.tag %}tag={{ request.GET.tag }}{% endif %}{% if request.GET.c %}&c={{ request.GET.c }}{% endif %}" class="pagination__page-link pagination__page-link--current">{{ page }}</a>
-                                    {% else %}
-                                    <a href="?page={{ page }}&{% if request.GET.q %}q={{ request.GET.q }}{% elif request.GET.tag %}tag={{ request.GET.tag }}{% endif %}{% if request.GET.c %}&c={{ request.GET.c }}{% endif %}" class="pagination__page-link">{{ page }}</a>
-                                    {% endif %}
-                                        
-                                {% endfor %}
-                                    
-                                {% if pagination.has_next %}
-                                    <a href="?page={{ pagination.next_page }}&{% if request.GET.q %}q={{ request.GET.q }}{% elif request.GET.tag %}tag={{ request.GET.tag }}{% endif %}{% if request.GET.c %}&c={{ request.GET.c }}{% endif %}" class="pagination__nav-link pagination__nav-link--left-margin">&rsaquo;</a>
-                                    <a href="?page={{ pagination.last_page }}&{% if request.GET.q %}q={{ request.GET.q }}{% elif request.GET.tag %}tag={{ request.GET.tag }}{% endif %}{% if request.GET.c %}&c={{ request.GET.c }}{% endif %}" class="pagination__nav-link">&raquo;</a>
-                                {% endif %}           
-                              
-                            </div>            
-                        </div>
-           
-                    </div>
-        
-                    <div class="grid__aside">
-                        <div class="aside-section-wrapper">
-                            
-                            <div class="add-entity-entries">
-                                <div class="add-entity-entries__entry">
-                                    <div class="add-entity-entries__label">
-                                        {% trans '没有想要的结果？' %}
-                                    </div>
-                                    {% if request.GET.c and request.GET.c in categories %}
-                                    
-                                        {% if request.GET.c|lower == 'book' %}
-                                            
-                                        <a href="{% url 'books:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加书' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'movie' %}
-                                        
-                                        <a href="{% url 'movies:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加电影/剧集' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'music' %}
-                                        
-                                        <a href="{% url 'music:create_album' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加专辑' %}</button>
-                                        </a>
-                                        <a href="{% url 'music:create_song' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加单曲' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'game' %}
-                                        
-                                        <a href="{% url 'games:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加游戏' %}</button>
-                                        </a>
-                                        
-                                        {% endif %}
-                                        
-                                    {% else %}
-                                        <a href="{% url 'books:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加书' %}</button>
-                                        </a>
-                                        <a href="{% url 'movies:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加电影/剧集' %}</button>
-                                        </a>
-                                        <a href="{% url 'music:create_album' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加专辑' %}</button>
-                                        </a>
-                                        <a href="{% url 'music:create_song' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加单曲' %}</button>
-                                        </a>
-                                        <a href="{% url 'games:create' %}">
-                                            <button class="add-entity-entries__button">{% trans '添加游戏' %}</button>
-                                        </a>
-                                    {% endif %}
-                                </div>
-                                <!-- div class="add-entity-entries__entry">
-                                    {% if request.GET.c and request.GET.c in categories %}
-                                    
-                                        {% if request.GET.c|lower == 'book' %}
-
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'books:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'movie' %}
-                                        
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'movies:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'game' %}
-                                        
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'games:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                        
-                                        {% elif request.GET.c|lower == 'music' %}
-                                        
-                                        <div class="add-entity-entries__label">
-                                            {% trans '或者(≖ ◡ ≖)✧' %}
-                                        </div>
-                                        <a href="{% url 'music:scrape_album' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                            <button class="add-entity-entries__button">{% trans '从表瓣剽取数据' %}</button>
-                                        </a>
-                                                                                
-                                        {% endif %}
-                                        
-                                    {% else %}
-                                        
-                                    <div class="add-entity-entries__label">
-                                        {% trans '或从表瓣剽取' %}
-                                    </div>
-                                    <a href="{% url 'books:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '书' %}</button>
-                                    </a>
-                                    <a href="{% url 'movies:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '电影/剧集' %}</button>
-                                    </a>
-                                    <a href="{% url 'music:scrape_album' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '专辑' %}</button>
-                                    </a>
-                                    <a href="{% url 'games:scrape' %}{% if request.GET.q %}?q={{ request.GET.q }}{% endif %}">
-                                        <button class="add-entity-entries__button">{% trans '游戏' %}</button>
-                                    </a>
-
-                                    {% endif %}
-                                </div -->
-
-                            </div>
-
-                        </div>
-                    </div>
-                </div>
-            </section>
-        </div>
-        {% include 'partial/_footer.html' %}
-    </div>
-
-    
-        
-    <script>
-      document.body.addEventListener('htmx:configRequest', (event) => {
-        event.detail.headers['X-CSRFToken'] = '{{ csrf_token }}';
-      })
-    </script>
-</body>
-
-
-</html>

common/templates/partial/_navbar.html

@@ -1,7 +1,7 @@
 {% load static %}
 {% load i18n %}
 {% load admin_url %}
-<form method="get" action="{% url 'common:search' %}">
+<form method="get" action="{% url 'catalog:search' %}">
 <section id="navbar">
     <nav class="navbar">
         <div class="grid">
@@ -26,8 +26,8 @@
 
                     {% if request.user.is_authenticated %}
 
-                        <a class="navbar__link {% if current == 'home' %}current{% endif %}" href="{% url 'users:home' request.user.mastodon_username %}">{% trans '主页' %}</a>
-                        <a class="navbar__link {% if current == 'timeline' %}current{% endif %}" href="{% url 'timeline:timeline' %}">{% trans '动态' %}</a>
+                        <a class="navbar__link {% if current == 'home' %}current{% endif %}" href="{% url 'journal:user_profile' request.user.mastodon_username %}">{% trans '主页' %}</a>
+                        <a class="navbar__link {% if current == 'timeline' %}current{% endif %}" href="{% url 'social:feed' %}">{% trans '动态' %}</a>
                         <a class="navbar__link {% if current == 'data' %}current{% endif %}" href="{% url 'users:data' %}">{% trans '数据' %}</a>
                         <a class="navbar__link {% if current == 'preferences' %}current{% endif %}" href="{% url 'users:preferences' %}">{% trans '设置' %}</a>
                         <a class="navbar__link" id="logoutLink" href="{% url 'users:logout' %}">{% trans '登出' %}</a>

common/templates/partial/_sidebar.html

@@ -5,14 +5,14 @@
 {% load oauth_token %}
 {% load truncate %}
 {% load thumb %}
-{% load user_item %}
+
 <div class="grid__aside grid__aside--reverse-order grid__aside--tablet-column">
     <div class="aside-section-wrapper aside-section-wrapper--no-margin">
         <div class="user-profile" id="userInfoCard">
             <div class="user-profile__header">
                 <!-- <img src="" class="user-profile__avatar mast-avatar" alt="{{ user.username }}"> -->
                 <img src="{{ user.mastodon_account.avatar }}" class="user-profile__avatar mast-avatar">
-                <a href="{% url 'users:home' user.mastodon_username %}">
+                <a href="{% url 'journal:user_profile' user.mastodon_username %}">
                     <h5 class="user-profile__username mast-displayname">{{ user.mastodon_account.display_name }}</h5>
                 </a>
             </div>
@@ -85,7 +85,7 @@
                     <h5 class="user-relation__label">
                         {% trans '常用标签' %}
                     </h5>
-                    <a href="{% url 'users:tag_list' user.mastodon_username %}">{% trans '更多' %}</a>
+                    <a href="{% url 'journal:user_tag_list' user.mastodon_username %}">{% trans '更多' %}</a>
                     <div class="tag-collection" style="margin-left: 0;">
                         {% if top_tags %}
                         {% for t in top_tags %}
@@ -150,9 +150,9 @@
                         <ul class="report-panel__report-list">
                             {% for report in reports %}
                             <li class="report-panel__report">
-                                <a href="{% url 'users:home' report.submit_user.mastodon_username %}"
+                                <a href="{% url 'journal:user_profile' report.submit_user.mastodon_username %}"
                                     class="report-panel__user-link">{{ report.submit_user }}</a>{% trans '已投诉' %}<a
-                                    href="{% url 'users:home' report.reported_user.mastodon_username %}"
+                                    href="{% url 'journal:user_profile' report.reported_user.mastodon_username %}"
                                     class="report-panel__user-link">{{ report.reported_user }}</a>
                             </li>
                             {% empty %}
@@ -173,7 +173,7 @@
 <div id="oauth2Token" hidden="true">{{ request.user.mastodon_token }}</div>
 <div id="mastodonURI" hidden="true">{{ request.user.mastodon_site }}</div>
 <div id="userMastodonID" hidden="true">{{ user.mastodon_id }}</div>
-<div id="userPageURL" hidden="true">{% url 'users:home' 0 %}</div>
+<div id="userPageURL" hidden="true">{% url 'journal:user_profile' 0 %}</div>
 
 <div id="spinner" hidden>
     <div class="spinner">

common/templates/partial/list_item.html

@@ -1,9 +0,0 @@
-{% if item.category_name|lower == 'book' %}
-{% include "partial/list_item_book.html" with book=item %}
-{% elif item.category_name|lower == 'movie' %}    
-{% include "partial/list_item_movie.html" with movie=item %}
-{% elif item.category_name|lower == 'game' %}    
-{% include "partial/list_item_game.html" with game=item %}
-{% elif item.category_name|lower == 'album' or item.category_name|lower == 'song' %}    
-{% include "partial/list_item_music.html" with music=item %}
-{% endif %}

common/templates/partial/list_item_book.html

@@ -1,159 +0,0 @@
-{% load thumb %}
-{% load highlight %}
-{% load i18n %}
-{% load l10n %}
-{% load user_item %}
-{% current_user_marked_item book as marked %}
-<li class="entity-list__entity">
-    <div class="entity-list__entity-img-wrapper">
-        <a href="{% url 'books:retrieve' book.id %}">
-            <img src="{{ book.cover|thumb:'normal' }}" alt="" class="entity-list__entity-img">
-        </a>
-        {% if not marked %}
-        <a class="entity-list__entity-action-icon" hx-post="{% url 'books:wish' book.id %}" title="加入想读">➕</a>
-        {% endif %}
-    </div>
-
-    <div class="entity-list__entity-text">
-        {% if editable %}
-        <div class="collection-item-position-edit">
-            {% if not forloop.first %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_up_item' form.instance.id collectionitem.id %}">▲</a>
-            {% endif %}
-            {% if not forloop.last %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_down_item' form.instance.id collectionitem.id %}">▼</a>
-            {% endif %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:delete_item' form.instance.id collectionitem.id %}">✖</a>
-        </div>
-        {% endif %}
-
-        <div class="entity-list__entity-title">
-
-            <a href="{% url 'books:retrieve' book.id %}" class="entity-list__entity-link">
-                {% if request.GET.q %}
-                {{ book.title | highlight:request.GET.q }}
-                {% else %}
-                {{ book.title }}
-                {% endif %}
-
-            </a>
-            {% if not request.GET.c and not hide_category %}
-            <span class="entity-list__entity-category">[{{book.verbose_category_name}}]</span>
-            {% endif %}
-            <a href="{{ book.source_url }}">
-                <span class="source-label source-label__{{ book.source_site }}">{{ book.get_source_site_display }}</span>
-            </a>
-        </div>
-
-        {% if book.rating %}
-        <div class="rating-star entity-list__rating-star" data-rating-score="{{ book.rating | floatformat:"0" }}"></div>
-        <span class="entity-list__rating-score rating-score">{{ book.rating }}</span>
-        {% else %}
-        <div class="entity-list__rating entity-list__rating--empty"> {% trans '暂无评分' %}</div>
-        {% endif %}
-
-        <span class="entity-list__entity-info">
-            {% if book.pub_year %} /
-            {{ book.pub_year }}{% trans '年' %}{% if book.pub_month %}{{book.pub_month }}{% trans '月' %}{% endif %}
-            {% endif %}
-
-            {% if book.author %} /
-            {% for author in book.author %}
-            {% if request.GET.q %}
-            {{ author | highlight:request.GET.q }}
-            {% else %}
-            {{ author }}
-            {% endif %}
-            {% if not forloop.last %},{% endif %}
-            {% endfor %}
-            {% endif %}
-
-            {% if book.translator %} /
-            {% trans '翻译' %}:
-            {% for translator in book.translator %}
-            {% if request.GET.q %}
-            {{ translator | highlight:request.GET.q }}
-            {% else %}
-            {{ translator }}
-            {% endif %}
-            {% if not forloop.last %},{% endif %}
-            {% endfor %}
-            {% endif %}
-
-            {% if book.subtitle %} /
-            {% trans '副标题' %}:
-            {% if request.GET.q %}
-            {{ book.subtitle | highlight:request.GET.q }}
-            {% else %}
-            {{ book.subtitle }}
-            {% endif %}
-            {% endif %}
-
-            {% if book.orig_title %} /
-            {% trans '原名' %}:
-            {% if request.GET.q %}
-            {{ book.orig_title | highlight:request.GET.q }}
-            {% else %}
-            {{ book.orig_title }}
-            {% endif %}
-            {% endif %}
-        </span>
-        <p class="entity-list__entity-brief">
-            {{ book.brief }}
-        </p>
-
-        <div class="tag-collection">
-            {% for tag_dict in book.top_tags %}
-            <span class="tag-collection__tag">
-                <a href="{% url 'common:search' %}?tag={{ tag_dict.content }}">{{ tag_dict.content }}</a>
-            </span>
-            {% endfor %}
-        </div>
-
-        {% if mark %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    {% if mark.rating %}
-                    <span class="entity-marks__rating-star rating-star"
-                        data-rating-score="{{ mark.rating | floatformat:"0" }}" style="left: -4px;"></span>
-                    {% endif %}
-                    {% if mark.visibility > 0 %}
-                    <span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20">
-                                <path
-                                    d="M17,8.48h-.73V6.27a6.27,6.27,0,1,0-12.53,0V8.48H3a.67.67,0,0,0-.67.67V19.33A.67.67,0,0,0,3,20H17a.67.67,0,0,0,.67-.67V9.15A.67.67,0,0,0,17,8.48ZM6.42,6.27h0a3.57,3.57,0,0,1,7.14,0h0V8.48H6.42Z" />
-                        </svg></span>
-                    {% endif %}
-                    <span class="entity-marks__mark-time">
-                        {% trans '于' %} {{ mark.created_time }}
-                        {% if status == 'reviewed' %}
-                        {% trans '评论' %}: <a href="{% url 'books:retrieve_review' mark.id %}">{{ mark.title }}</a>
-                        {% else %}
-                        {% trans '标记' %}
-                        {% endif %}
-                    </span>
-                    {% if mark.text %}
-                    <p class="entity-marks__mark-content">{{ mark.text }}</p>
-                    {% endif %}
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-
-        {% if collectionitem %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    <p class="entity-marks__mark-content" hx-target="this" hx-swap="innerHTML">
-                        {% include "show_item_comment.html" %}
-                    </p>
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-    </div>
-</li>

common/templates/partial/list_item_game.html

@@ -1,139 +0,0 @@
-{% load thumb %}
-{% load highlight %}
-{% load i18n %}
-{% load l10n %}
-{% load user_item %}
-{% current_user_marked_item game as marked %}
-<li class="entity-list__entity">
-    <div class="entity-list__entity-img-wrapper">
-        <a href="{% url 'games:retrieve' game.id %}">
-            <img src="{{ game.cover|thumb:'normal' }}" alt="" class="entity-list__entity-img">
-        </a>
-        {% if not marked %}
-        <a class="entity-list__entity-action-icon" hx-post="{% url 'games:wish' game.id %}" title="加入想玩">➕</a>
-        {% endif %}
-    </div>
-    <div class="entity-list__entity-text">
-        {% if editable %}
-        <div class="collection-item-position-edit">
-            {% if not forloop.first %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_up_item' form.instance.id collectionitem.id %}">▲</a>
-            {% endif %}
-            {% if not forloop.last %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_down_item' form.instance.id collectionitem.id %}">▼</a>
-            {% endif %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:delete_item' form.instance.id collectionitem.id %}">✖</a>
-        </div>
-        {% endif %}
-
-        <div class="entity-list__entity-title">
-            <a href="{% url 'games:retrieve' game.id %}" class="entity-list__entity-link">
-                {% if request.GET.q %}
-                    {{ game.title | highlight:request.GET.q }}
-                {% else %}
-                    {{ game.title }}
-                {% endif %}
-            </a>
-            
-            {% if not request.GET.c and not hide_category %}
-            <span class="entity-list__entity-category">[{{item.verbose_category_name}}]</span>
-            {% endif %}
-            <a href="{{ game.source_url }}">
-                <span class="source-label source-label__{{ game.source_site }}">{{ game.get_source_site_display }}</span>
-            </a>
-        </div>
-
-        {% if game.rating %}
-        <div class="rating-star entity-list__rating-star" data-rating-score="{{ game.rating | floatformat:"0" }}"></div>
-        <span class="entity-list__rating-score rating-score">{{ game.rating }}</span>
-        {% else %}
-        <div class="entity-list__rating entity-list__rating--empty"> {% trans '暂无评分' %}</div>
-        {% endif %}
-
-        <span class="entity-list__entity-info entity-list__entity-info--full-length">
-
-            {% if game.other_title %}{% trans '别名' %}:
-            {% for other_title in game.other_title %}
-            {{ other_title }}{% if not forloop.last %} {% endif %}
-            {% endfor %}/
-            {% endif %}
-
-            {% if game.developer %}{% trans '开发商' %}:
-            {% for developer in game.developer %}
-            {{ developer }}{% if not forloop.last %} {% endif %}
-            {% endfor %}/
-            {% endif %}
-
-            {% if game.genre %}{% trans '类型' %}:
-            {% for genre in game.genre %}
-            {{ genre }}{% if not forloop.last %} {% endif %}
-            {% endfor %}/
-            {% endif %}
-
-            {% if game.platform %}{% trans '平台' %}:
-            {% for platform in game.platform %}
-            {{ platform }}{% if not forloop.last %} {% endif %}
-            {% endfor %}
-            {% endif %}
-
-        </span>
-        <p class="entity-list__entity-brief">
-            {{ game.brief }}
-        </p>
-
-        <div class="tag-collection">
-            {% for tag_dict in game.top_tags %}
-            <span class="tag-collection__tag">
-                <a href="{% url 'common:search' %}?tag={{ tag_dict.content }}">{{ tag_dict.content }}</a>
-            </span>
-            {% endfor %}
-        </div>
-
-        {% if mark %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    {% if mark.rating %}
-                    <span class="entity-marks__rating-star rating-star"
-                        data-rating-score="{{ mark.rating | floatformat:"0" }}" style="left: -4px;"></span>
-                    {% endif %}
-                    {% if mark.visibility > 0 %}
-                    <span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20">
-                                <path
-                                    d="M17,8.48h-.73V6.27a6.27,6.27,0,1,0-12.53,0V8.48H3a.67.67,0,0,0-.67.67V19.33A.67.67,0,0,0,3,20H17a.67.67,0,0,0,.67-.67V9.15A.67.67,0,0,0,17,8.48ZM6.42,6.27h0a3.57,3.57,0,0,1,7.14,0h0V8.48H6.42Z" />
-                        </svg></span>
-                    {% endif %}
-                    <span class="entity-marks__mark-time">
-                        {% trans '于' %} {{ mark.created_time }}
-                        {% if status == 'reviewed' %}
-                        {% trans '评论' %}: <a href="{% url 'games:retrieve_review' mark.id %}">{{ mark.title }}</a>
-                        {% else %}
-                        {% trans '标记' %}
-                        {% endif %}
-                    </span>
-                    {% if mark.text %}
-                    <p class="entity-marks__mark-content">{{ mark.text }}</p>
-                    {% endif %}
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-
-        {% if collectionitem %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    <p class="entity-marks__mark-content" hx-target="this" hx-swap="innerHTML">
-                        {% include "show_item_comment.html" %}
-                    </p>
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-    </div>
-
-</li>

common/templates/partial/list_item_movie.html

@@ -1,164 +0,0 @@
-{% load thumb %}
-{% load highlight %}
-{% load i18n %}
-{% load l10n %}
-{% load humanize %}
-{% load user_item %}
-{% current_user_marked_item movie as marked %}
-<li class="entity-list__entity">
-    <div class="entity-list__entity-img-wrapper">
-        <a href="{% url 'movies:retrieve' movie.id %}">
-            <img src="{{ movie.cover|thumb:'normal' }}" alt="" class="entity-list__entity-img">
-        </a>
-        {% if not marked %}
-        <a class="entity-list__entity-action-icon" hx-post="{% url 'movies:wish' movie.id %}" title="加入想看">➕</a>
-        {% endif %}
-    </div>
-    <div class="entity-list__entity-text">
-        {% if editable %}
-        <div class="collection-item-position-edit">
-            {% if not forloop.first %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_up_item' form.instance.id collectionitem.id %}">▲</a>
-            {% endif %}
-            {% if not forloop.last %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_down_item' form.instance.id collectionitem.id %}">▼</a>
-            {% endif %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:delete_item' form.instance.id collectionitem.id %}">✖</a>
-        </div>
-        {% endif %}
-
-        <div class="entity-list__entity-title">
-            <a href="{% url 'movies:retrieve' movie.id %}" class="entity-list__entity-link">
-                {% if movie.season %}
-                
-                    {% if request.GET.q %}
-                        {{ movie.title | highlight:request.GET.q }} {% trans '第' %}{{ movie.season|apnumber }}{% trans '季' %}
-                        {{ movie.orig_title | highlight:request.GET.q }} Season {{ movie.season }}
-                        {% if movie.year %}({{ movie.year }}){% endif %}
-                    {% else %}
-                        {{ movie.title }} {% trans '第' %}{{ movie.season|apnumber }}{% trans '季' %}
-                        {{ movie.orig_title }} Season {{ movie.season }}
-                        {% if movie.year %}({{ movie.year }}){% endif %}
-                    {% endif %}
-
-                {% else %}
-                    {% if request.GET.q %}
-                        {{ movie.title | highlight:request.GET.q }} {{ movie.orig_title | highlight:request.GET.q }}
-                        {% if movie.year %}({{ movie.year }}){% endif %}
-                    {% else %}
-                        {{ movie.title }} {{ movie.orig_title }}
-                        {% if movie.year %}({{ movie.year }}){% endif %}
-                    {% endif %}
-                {% endif %}
-            </a>
-            
-            {% if not request.GET.c and not hide_category %}
-            <span class="entity-list__entity-category">[{{movie.verbose_category_name}}]</span>
-            {% endif %}
-            <a href="{{ movie.source_url }}">
-                <span class="source-label source-label__{{ movie.source_site }}">{{ movie.get_source_site_display }}</span>
-            </a>
-        </div>
-
-        {% if movie.rating %}
-        <div class="rating-star entity-list__rating-star" data-rating-score="{{ movie.rating | floatformat:"0" }}"></div>
-        <span class="entity-list__rating-score rating-score">{{ movie.rating }}</span>
-        {% else %}
-        <div class="entity-list__rating entity-list__rating--empty"> {% trans '暂无评分' %}</div>
-        {% endif %}
-
-        <span class="entity-list__entity-info ">
-
-            {% if movie.director %}{% trans '导演' %}:
-            {% for director in movie.director %}
-            {% if request.GET.q %}
-            {{ director | highlight:request.GET.q }}
-            {% else %}
-            {{ director }}
-            {% endif %}
-            {% if not forloop.last %},{% endif %}
-            {% endfor %}/
-            {% endif %}
-
-            {% if movie.genre %}{% trans '类型' %}:
-            {% for genre in movie.get_genre_display %}
-            {{ genre }}{% if not forloop.last %} {% endif %}
-            {% endfor %}/
-            {% endif %}
-
-        </span>
-        <span class="entity-list__entity-info entity-list__entity-info--full-length">
-            {% if movie.actor %}{% trans '主演' %}:
-            {% for actor in movie.actor %}
-            <span {% if forloop.counter > 5 %}style="display: none;" {% endif %}>
-            {% if request.GET.q %}
-            {{ actor | highlight:request.GET.q }}
-            {% else %}
-            {{ actor }}
-            {% endif %}
-            </span>
-            {% if forloop.counter <= 5 %}
-                {% if not forloop.counter == 5 and not forloop.last %} {% endif %}
-            {% endif %}
-            {% endfor %}
-            {% endif %}
-        </span>
-        <p class="entity-list__entity-brief">
-            {{ movie.brief }}
-        </p>
-        <div class="tag-collection">
-            {% for tag_dict in movie.top_tags %}
-            <span class="tag-collection__tag">
-                <a href="{% url 'common:search' %}?tag={{ tag_dict.content }}">{{ tag_dict.content }}</a>
-            </span>
-            {% endfor %}
-        </div>
-
-        {% if mark %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    {% if mark.rating %}
-                    <span class="entity-marks__rating-star rating-star"
-                        data-rating-score="{{ mark.rating | floatformat:"0" }}" style="left: -4px;"></span>
-                    {% endif %}
-                    {% if mark.visibility > 0 %}
-                    <span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20">
-                                <path
-                                    d="M17,8.48h-.73V6.27a6.27,6.27,0,1,0-12.53,0V8.48H3a.67.67,0,0,0-.67.67V19.33A.67.67,0,0,0,3,20H17a.67.67,0,0,0,.67-.67V9.15A.67.67,0,0,0,17,8.48ZM6.42,6.27h0a3.57,3.57,0,0,1,7.14,0h0V8.48H6.42Z" />
-                        </svg></span>
-                    {% endif %}
-                    <span class="entity-marks__mark-time">
-                        {% trans '于' %} {{ mark.created_time }}
-                        {% if status == 'reviewed' %}
-                        {% trans '评论' %}: <a href="{% url 'movies:retrieve_review' mark.id %}">{{ mark.title }}</a>
-                        {% else %}
-                        {% trans '标记' %}
-                        {% endif %}
-                    </span>
-                    {% if mark.text %}
-                    <p class="entity-marks__mark-content">{{ mark.text }}</p>
-                    {% endif %}
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-
-        {% if collectionitem %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    <p class="entity-marks__mark-content" hx-target="this" hx-swap="innerHTML">
-                        {% include "show_item_comment.html" %}
-                    </p>
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-    </div>
-
-</li>

common/templates/partial/list_item_music.html

@@ -1,171 +0,0 @@
-{% load thumb %}
-{% load highlight %}
-{% load i18n %}
-{% load l10n %}
-{% load user_item %}
-{% current_user_marked_item music as marked %}
-<li class="entity-list__entity">
-    <div class="entity-list__entity-img-wrapper">
-        
-        {% if music.category_name|lower == 'album' %}
-            <a href="{% url 'music:retrieve_album' music.id %}">
-                <img src="{{ music.cover|thumb:'normal' }}" alt="" class="entity-list__entity-img">
-            </a>
-            {% if not marked %}
-            <a class="entity-list__entity-action-icon" hx-post="{% url 'music:wish_album' music.id %}" title="加入想听">➕</a>
-            {% endif %}
-        {% elif music.category_name|lower == 'song' %}
-            <a href="{% url 'music:retrieve_song' music.id %}">
-                <img src="{{ music.cover|thumb:'normal' }}" alt="" class="entity-list__entity-img">
-            </a>
-            {% if not marked %}
-            <a class="entity-list__entity-action-icon" hx-post="{% url 'music:wish_song' music.id %}" title="加入想听">➕</a>
-            {% endif %}
-        {% endif %}
-    </div>
-    <div class="entity-list__entity-text">
-        {% if editable %}
-        <div class="collection-item-position-edit">
-            {% if not forloop.first %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_up_item' form.instance.id collectionitem.id %}">▲</a>
-            {% endif %}
-            {% if not forloop.last %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:move_down_item' form.instance.id collectionitem.id %}">▼</a>
-            {% endif %}
-            <a hx-target=".entity-list" hx-post="{% url 'collection:delete_item' form.instance.id collectionitem.id %}">✖</a>
-        </div>
-        {% endif %}
-
-        <div class="entity-list__entity-title">
-            
-            {% if music.category_name|lower == 'album' %}
-                <a href="{% url 'music:retrieve_album' music.id %}" class="entity-list__entity-link">                                                        
-                    {% if request.GET.q %}
-                    {{ music.title | highlight:request.GET.q }}
-                    {% else %}
-                    {{ music.title }}
-                    {% endif %}
-                </a>
-            {% elif music.category_name|lower == 'song' %}
-                <a href="{% url 'music:retrieve_song' music.id %}" class="entity-list__entity-link">                                                        
-                    {% if request.GET.q %}
-                    {{ music.title | highlight:request.GET.q }}
-                    {% else %}
-                    {{ music.title }}
-                    {% endif %}
-                </a>
-            {% endif %}
-                
-            
-            {% if not request.GET.c and not hide_category %}
-            <span class="entity-list__entity-category">[{{music.verbose_category_name}}]</span>
-            {% endif %}
-            <a href="{{ music.source_url }}">
-                <span class="source-label source-label__{{ music.source_site }}">{{ music.get_source_site_display }}</span>
-            </a>
-        </div>
-
-        {% if music.rating %}
-        <div class="rating-star entity-list__rating-star" data-rating-score="{{ music.rating | floatformat:"0" }}"></div>
-        <span class="entity-list__rating-score rating-score">{{ music.rating }}</span>
-        {% else %}
-        <div class="entity-list__rating entity-list__rating--empty"> {% trans '暂无评分' %}</div>
-        {% endif %}
-
-        <span class="entity-list__entity-info ">
-            {% if music.artist %}{% trans '艺术家' %}:
-            {% for artist in music.artist %}
-            <span>{{ artist }}</span>
-            {% if not forloop.last %} {% endif %}
-            {% endfor %}
-            {% endif %}
-            
-            {% if music.genre %}/ {% trans '流派' %}:
-            {{ music.genre }}
-            {% endif %}
-
-            {% if music.release_date %}/ {% trans '发行日期' %}:
-                {{ music.release_date }}
-            {% endif %}
-        </span>
-        <span class="entity-list__entity-info entity-list__entity-info--full-length">
-
-        </span>
-        
-        {% if music.brief %}
-        <p class="entity-list__entity-brief">
-            {{ music.brief }}
-        </p>
-        {% elif music.category_name|lower == 'album' %}
-        <p class="entity-list__entity-brief">
-            {% trans '曲目：' %}{{ music.track_list }}
-        </p>
-        {% else %}
-        <!-- song -->
-        <p class="entity-list__entity-brief">
-            {% trans '所属专辑：' %}{{ music.album }}
-        </p>
-        {% endif %}
-            
-        <div class="tag-collection">
-            {% for tag_dict in music.top_tags %}
-            <span class="tag-collection__tag">
-                <a href="{% url 'common:search' %}?tag={{ tag_dict.content }}">{{ tag_dict.content }}</a>
-            </span>
-            {% endfor %}
-        </div>
-
-        {% if mark %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    {% if mark.rating %}
-                    <span class="entity-marks__rating-star rating-star"
-                        data-rating-score="{{ mark.rating | floatformat:"0" }}" style="left: -4px;"></span>
-                    {% endif %}
-                    {% if mark.visibility > 0 %}
-                    <span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20">
-                                <path
-                                    d="M17,8.48h-.73V6.27a6.27,6.27,0,1,0-12.53,0V8.48H3a.67.67,0,0,0-.67.67V19.33A.67.67,0,0,0,3,20H17a.67.67,0,0,0,.67-.67V9.15A.67.67,0,0,0,17,8.48ZM6.42,6.27h0a3.57,3.57,0,0,1,7.14,0h0V8.48H6.42Z" />
-                        </svg></span>
-                    {% endif %}
-                    <span class="entity-marks__mark-time">
-                        {% trans '于' %} {{ mark.created_time }}
-                        {% if status == 'reviewed' %}
-                        {% trans '评论' %}: 
-                        {% if music.category_name|lower == 'album' %}
-                        <a href="{% url 'music:retrieve_album_review' mark.id %}">{{ mark.title }}</a>
-                        {% else %}
-                        <a href="{% url 'music:retrieve_song_review' mark.id %}">{{ mark.title }}</a>
-                        {% endif %}
-                        {% else %}
-                        {% trans '标记' %}
-                        {% endif %}
-                    </span>
-                    {% if mark.text %}
-                    <p class="entity-marks__mark-content">{{ mark.text }}</p>
-                    {% endif %}
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-
-        {% if collectionitem %}
-        <div class="clearfix"></div>
-        <div class="dividing-line dividing-line--dashed"></div>
-        <div class="entity-marks" style="margin-bottom: 0;">
-            <ul class="entity-marks__mark-list">
-                <li class="entity-marks__mark">
-                    <p class="entity-marks__mark-content" hx-target="this" hx-swap="innerHTML">
-                        {% include "show_item_comment.html" %}
-                    </p>
-                </li>
-            </ul>
-        </div>
-        {% endif %}
-
-    </div>
-
-</li>

common/templates/partial/mark_list.html

@@ -1,37 +0,0 @@
-{% load i18n %}
-
-<ul class="entity-marks__mark-list">
-{% for others_mark in mark_list %}
-<li class="entity-marks__mark">
-    <a href="{% url 'users:home' others_mark.owner.mastodon_username %}" class="entity-marks__owner-link">{{ others_mark.owner.username }}</a>
-
-    <span>{{ others_mark.get_status_display }}</span>
-
-    {% if others_mark.rating %}
-    <span class="entity-marks__rating-star rating-star" data-rating-score="{{ others_mark.rating | floatformat:"0" }}"></span>
-    {% endif %}
-
-    {% if others_mark.visibility > 0 %}
-    <span class="icon-lock"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path d="M17,8.48h-.73V6.27a6.27,6.27,0,1,0-12.53,0V8.48H3a.67.67,0,0,0-.67.67V19.33A.67.67,0,0,0,3,20H17a.67.67,0,0,0,.67-.67V9.15A.67.67,0,0,0,17,8.48ZM6.42,6.27h0a3.57,3.57,0,0,1,7.14,0h0V8.48H6.42Z"/></svg></span>
-    {% endif %}
-
-    {% if others_mark.shared_link %}
-    <a href="{{ others_mark.shared_link }}" target="_blank"><span class="entity-marks__mark-time">{{ others_mark.created_time }}</span></a>
-    {% else %}
-    <span class="entity-marks__mark-time">{{ others_mark.created_time }}</span>
-    {% endif %}
-
-    {% if current_item and others_mark.item != current_item %}
-    <span class="entity-marks__mark-time source-label"><a class="entity-marks__mark-time" href="{% url 'books:retrieve' others_mark.item.id %}">{{ others_mark.item.get_source_site_display }}</a></span>
-    {% endif %}
-
-    {% if others_mark.text %}
-    <p class="entity-marks__mark-content">{{ others_mark.text }}</p>
-    {% endif %}
-</li>
-{% empty %}
-
-<div> {% trans '暂无标记' %} </div>
-
-{% endfor %}
-</ul>

common/templatetags/user_item.py

@@ -1,19 +0,0 @@
-from django import template
-from collection.models import Collection
-
-
-register = template.Library()
-
-
-@register.simple_tag(takes_context=True)
-def current_user_marked_item(context, item):
-    # NOTE weird to put business logic in tags
-    user = context['request'].user
-    if user and user.is_authenticated:
-        if isinstance(item, Collection) and item.owner == user:
-            return item
-        else:
-            return context['request'].user.get_mark_for_item(item)
-    return None
-
-

common/urls.py

@@ -1,11 +1,5 @@
 from django.urls import path
 from .views import *
 
-app_name = 'common'
-urlpatterns = [
-    path('', home),
-    path('home/', home, name='home'),
-    path('search/', search, name='search'),
-    path('search.json/', search, name='search.json'),
-    path('external_search/', external_search, name='external_search'),
-]
+app_name = "common"
+urlpatterns = [path("", home), path("home/", home, name="home")]

common/views.py

@@ -1,553 +1,15 @@
-import operator
 import logging
-from difflib import SequenceMatcher
-from urllib.parse import urlparse
-from django.shortcuts import render, redirect, reverse
+from django.shortcuts import render, redirect
+from django.urls import reverse
 from django.contrib.auth.decorators import login_required
 from django.utils.translation import gettext_lazy as _
-from django.core.paginator import Paginator
-from django.core.validators import URLValidator
-from django.core.exceptions import ValidationError, ObjectDoesNotExist
-from django.db.models import Q, Count
-from django.http import HttpResponseBadRequest
-from books.models import Book
-from movies.models import Movie
-from games.models import Game
-from music.models import Album, Song, AlbumMark, SongMark
-from users.models import Report, User, Preference
-from mastodon.decorators import mastodon_request_included
-from users.views import home as user_home
-from timeline.views import timeline as user_timeline
-from common.models import MarkStatusEnum
-from common.utils import PageLinksGenerator
-from common.scraper import get_scraper_by_url, get_normalized_url
-from common.config import *
-from common.searcher import ExternalSources
-from management.models import Announcement
-from django.conf import settings
-from common.index import Indexer
-from django.http import JsonResponse
-from django.db.utils import IntegrityError
 
-
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 @login_required
 def home(request):
     if request.user.get_preference().classic_homepage:
-        return redirect(reverse("users:home", args=[request.user.mastodon_username]))
+        return redirect(reverse("journal:home", args=[request.user.mastodon_username]))
     else:
-        return redirect(reverse("timeline:timeline"))
-
-
-@login_required
-def external_search(request):
-    category = request.GET.get("c", default="all").strip().lower()
-    if category == "all":
-        category = None
-    keywords = request.GET.get("q", default="").strip()
-    page_number = int(request.GET.get("page", default=1))
-    items = ExternalSources.search(category, keywords, page_number) if keywords else []
-    dedupe_urls = request.session.get("search_dedupe_urls", [])
-    items = [i for i in items if i.source_url not in dedupe_urls]
-
-    return render(
-        request,
-        "common/external_search_result.html",
-        {
-            "external_items": items,
-        },
-    )
-
-
-def search(request):
-    if settings.ENABLE_NEW_MODEL:
-        from catalog.views import search as new_search
-
-        return new_search(request)
-
-    if settings.SEARCH_BACKEND is None:
-        return search2(request)
-    category = request.GET.get("c", default="all").strip().lower()
-    if category == "all":
-        category = None
-    keywords = request.GET.get("q", default="").strip()
-    tag = request.GET.get("tag", default="").strip()
-    p = request.GET.get("page", default="1")
-    page_number = int(p) if p.isdigit() else 1
-    if not (keywords or tag):
-        return render(
-            request,
-            "common/search_result.html",
-            {
-                "items": None,
-            },
-        )
-    if request.user.is_authenticated:
-        url_validator = URLValidator()
-        try:
-            url_validator(keywords)
-            # validation success
-            return jump_or_scrape(request, keywords)
-        except ValidationError as e:
-            pass
-
-    result = Indexer.search(keywords, page=page_number, category=category, tag=tag)
-    keys = []
-    items = []
-    urls = []
-    for i in result.items:
-        key = (
-            i.isbn
-            if hasattr(i, "isbn")
-            else (i.imdb_code if hasattr(i, "imdb_code") else None)
-        )
-        if key is None:
-            items.append(i)
-        elif key not in keys:
-            keys.append(key)
-            items.append(i)
-        urls.append(i.source_url)
-        i.tag_list = i.all_tag_list[:TAG_NUMBER_ON_LIST]
-
-    if request.path.endswith(".json/"):
-        return JsonResponse(
-            {
-                "num_pages": result.num_pages,
-                "items": list(map(lambda i: i.get_json(), items)),
-            }
-        )
-
-    request.session["search_dedupe_urls"] = urls
-    return render(
-        request,
-        "common/search_result.html",
-        {
-            "items": items,
-            "pagination": PageLinksGenerator(
-                PAGE_LINK_NUMBER, page_number, result.num_pages
-            ),
-            "categories": ["book", "movie", "music", "game"],
-        },
-    )
-
-
-def search2(request):
-    if request.method == "GET":
-
-        # test if input serach string is empty or not excluding param ?c=
-        empty_querystring_criteria = {k: v for k, v in request.GET.items() if k != "c"}
-        if not len(empty_querystring_criteria):
-            return HttpResponseBadRequest()
-
-        # test if user input an URL, if so jump to URL handling function
-        url_validator = URLValidator()
-        input_string = request.GET.get("q", default="").strip()
-        try:
-            url_validator(input_string)
-            # validation success
-            return jump_or_scrape(request, input_string)
-        except ValidationError as e:
-            pass
-
-        # category, book/movie/music etc
-        category = request.GET.get("c", default="").strip().lower()
-        # keywords, seperated by blank space
-        # it is better not to split the keywords
-        keywords = request.GET.get("q", default="").strip()
-        keywords = [keywords] if keywords else ""
-        # tag, when tag is provided there should be no keywords , for now
-        tag = request.GET.get("tag", default="")
-
-        # white space string, empty query
-        if not (keywords or tag):
-            return render(
-                request,
-                "common/search_result.html",
-                {
-                    "items": None,
-                },
-            )
-
-        def book_param_handler(**kwargs):
-            # keywords
-            keywords = kwargs.get("keywords")
-            # tag
-            tag = kwargs.get("tag")
-
-            query_args = []
-            q = Q()
-
-            for keyword in keywords:
-                q = q | Q(title__icontains=keyword)
-                q = q | Q(subtitle__icontains=keyword)
-                q = q | Q(orig_title__icontains=keyword)
-            if tag:
-                q = q & Q(book_tags__content__iexact=tag)
-
-            query_args.append(q)
-            queryset = Book.objects.filter(*query_args).distinct()
-
-            def calculate_similarity(book):
-                if keywords:
-                    # search by keywords
-                    similarity, n = 0, 0
-                    for keyword in keywords:
-                        similarity += (
-                            1
-                            / 2
-                            * SequenceMatcher(None, keyword, book.title).quick_ratio()
-                        )
-                        +1 / 3 * SequenceMatcher(
-                            None, keyword, book.orig_title
-                        ).quick_ratio()
-                        +1 / 6 * SequenceMatcher(
-                            None, keyword, book.subtitle
-                        ).quick_ratio()
-                        n += 1
-                    book.similarity = similarity / n
-
-                elif tag:
-                    # search by single tag
-                    book.similarity = (
-                        0 if book.rating_number is None else book.rating_number
-                    )
-                else:
-                    book.similarity = 0
-                return book.similarity
-
-            if len(queryset) > 0:
-                ordered_queryset = sorted(
-                    queryset, key=calculate_similarity, reverse=True
-                )
-            else:
-                ordered_queryset = list(queryset)
-            return ordered_queryset
-
-        def movie_param_handler(**kwargs):
-            # keywords
-            keywords = kwargs.get("keywords")
-            # tag
-            tag = kwargs.get("tag")
-
-            query_args = []
-            q = Q()
-
-            for keyword in keywords:
-                q = q | Q(title__icontains=keyword)
-                q = q | Q(other_title__icontains=keyword)
-                q = q | Q(orig_title__icontains=keyword)
-            if tag:
-                q = q & Q(movie_tags__content__iexact=tag)
-
-            query_args.append(q)
-            queryset = Movie.objects.filter(*query_args).distinct()
-
-            def calculate_similarity(movie):
-                if keywords:
-                    # search by name
-                    similarity, n = 0, 0
-                    for keyword in keywords:
-                        similarity += (
-                            1
-                            / 2
-                            * SequenceMatcher(None, keyword, movie.title).quick_ratio()
-                        )
-                        +1 / 4 * SequenceMatcher(
-                            None, keyword, movie.orig_title
-                        ).quick_ratio()
-                        +1 / 4 * SequenceMatcher(
-                            None, keyword, movie.other_title
-                        ).quick_ratio()
-                        n += 1
-                    movie.similarity = similarity / n
-                elif tag:
-                    # search by single tag
-                    movie.similarity = (
-                        0 if movie.rating_number is None else movie.rating_number
-                    )
-                else:
-                    movie.similarity = 0
-                return movie.similarity
-
-            if len(queryset) > 0:
-                ordered_queryset = sorted(
-                    queryset, key=calculate_similarity, reverse=True
-                )
-            else:
-                ordered_queryset = list(queryset)
-            return ordered_queryset
-
-        def game_param_handler(**kwargs):
-            # keywords
-            keywords = kwargs.get("keywords")
-            # tag
-            tag = kwargs.get("tag")
-
-            query_args = []
-            q = Q()
-
-            for keyword in keywords:
-                q = q | Q(title__icontains=keyword)
-                q = q | Q(other_title__icontains=keyword)
-                q = q | Q(developer__icontains=keyword)
-                q = q | Q(publisher__icontains=keyword)
-            if tag:
-                q = q & Q(game_tags__content__iexact=tag)
-
-            query_args.append(q)
-            queryset = Game.objects.filter(*query_args).distinct()
-
-            def calculate_similarity(game):
-                if keywords:
-                    # search by name
-                    developer_dump = " ".join(game.developer)
-                    publisher_dump = " ".join(game.publisher)
-                    similarity, n = 0, 0
-                    for keyword in keywords:
-                        similarity += (
-                            1
-                            / 2
-                            * SequenceMatcher(None, keyword, game.title).quick_ratio()
-                        )
-                        +1 / 4 * SequenceMatcher(
-                            None, keyword, game.other_title
-                        ).quick_ratio()
-                        +1 / 16 * SequenceMatcher(
-                            None, keyword, developer_dump
-                        ).quick_ratio()
-                        +1 / 16 * SequenceMatcher(
-                            None, keyword, publisher_dump
-                        ).quick_ratio()
-                        n += 1
-                    game.similarity = similarity / n
-                elif tag:
-                    # search by single tag
-                    game.similarity = (
-                        0 if game.rating_number is None else game.rating_number
-                    )
-                else:
-                    game.similarity = 0
-                return game.similarity
-
-            if len(queryset) > 0:
-                ordered_queryset = sorted(
-                    queryset, key=calculate_similarity, reverse=True
-                )
-            else:
-                ordered_queryset = list(queryset)
-            return ordered_queryset
-
-        def music_param_handler(**kwargs):
-            # keywords
-            keywords = kwargs.get("keywords")
-            # tag
-            tag = kwargs.get("tag")
-
-            query_args = []
-            q = Q()
-
-            # search albums
-            for keyword in keywords:
-                q = q | Q(title__icontains=keyword)
-                q = q | Q(artist__icontains=keyword)
-            if tag:
-                q = q & Q(album_tags__content__iexact=tag)
-
-            query_args.append(q)
-            album_queryset = Album.objects.filter(*query_args).distinct()
-
-            # extra query args for songs
-            q = Q()
-            for keyword in keywords:
-                q = q | Q(album__title__icontains=keyword)
-                q = q | Q(title__icontains=keyword)
-                q = q | Q(artist__icontains=keyword)
-            if tag:
-                q = q & Q(song_tags__content__iexact=tag)
-            query_args.clear()
-            query_args.append(q)
-            song_queryset = Song.objects.filter(*query_args).distinct()
-            queryset = list(album_queryset) + list(song_queryset)
-
-            def calculate_similarity(music):
-                if keywords:
-                    # search by name
-                    similarity, n = 0, 0
-                    artist_dump = " ".join(music.artist)
-                    for keyword in keywords:
-                        if music.__class__ == Album:
-                            similarity += (
-                                1
-                                / 2
-                                * SequenceMatcher(
-                                    None, keyword, music.title
-                                ).quick_ratio()
-                                + 1
-                                / 2
-                                * SequenceMatcher(
-                                    None, keyword, artist_dump
-                                ).quick_ratio()
-                            )
-                        elif music.__class__ == Song:
-                            similarity += (
-                                1
-                                / 2
-                                * SequenceMatcher(
-                                    None, keyword, music.title
-                                ).quick_ratio()
-                                + 1
-                                / 6
-                                * SequenceMatcher(
-                                    None, keyword, artist_dump
-                                ).quick_ratio()
-                                + 1
-                                / 6
-                                * (
-                                    SequenceMatcher(
-                                        None, keyword, music.album.title
-                                    ).quick_ratio()
-                                    if music.album is not None
-                                    else 0
-                                )
-                            )
-                        n += 1
-                    music.similarity = similarity / n
-                elif tag:
-                    # search by single tag
-                    music.similarity = (
-                        0 if music.rating_number is None else music.rating_number
-                    )
-                else:
-                    music.similarity = 0
-                return music.similarity
-
-            if len(queryset) > 0:
-                ordered_queryset = sorted(
-                    queryset, key=calculate_similarity, reverse=True
-                )
-            else:
-                ordered_queryset = list(queryset)
-            return ordered_queryset
-
-        def all_param_handler(**kwargs):
-            book_queryset = book_param_handler(**kwargs)
-            movie_queryset = movie_param_handler(**kwargs)
-            music_queryset = music_param_handler(**kwargs)
-            game_queryset = game_param_handler(**kwargs)
-            ordered_queryset = sorted(
-                book_queryset + movie_queryset + music_queryset + game_queryset,
-                key=operator.attrgetter("similarity"),
-                reverse=True,
-            )
-            return ordered_queryset
-
-        param_handler = {
-            "book": book_param_handler,
-            "movie": movie_param_handler,
-            "music": music_param_handler,
-            "game": game_param_handler,
-            "all": all_param_handler,
-            "": all_param_handler,
-        }
-
-        categories = [k for k in param_handler.keys() if not k in ["all", ""]]
-
-        try:
-            queryset = param_handler[category](keywords=keywords, tag=tag)
-        except KeyError as e:
-            queryset = param_handler["all"](keywords=keywords, tag=tag)
-        paginator = Paginator(queryset, ITEMS_PER_PAGE)
-        page_number = request.GET.get("page", default=1)
-        items = paginator.get_page(page_number)
-        items.pagination = PageLinksGenerator(
-            PAGE_LINK_NUMBER, page_number, paginator.num_pages
-        )
-        for item in items:
-            item.tag_list = (
-                item.get_tags_manager()
-                .values("content")
-                .annotate(tag_frequency=Count("content"))
-                .order_by("-tag_frequency")[:TAG_NUMBER_ON_LIST]
-            )
-
-        return render(
-            request,
-            "common/search_result.html",
-            {
-                "items": items,
-                "categories": categories,
-            },
-        )
-
-    else:
-        return HttpResponseBadRequest()
-
-
-@login_required
-@mastodon_request_included
-def jump_or_scrape(request, url):
-    """
-    1. match url to registered scrapers
-    2. try to find the url in the db, if exits then jump, else scrape and jump
-    """
-
-    # redirect to this site
-    this_site = request.get_host()
-    if this_site in url:
-        return redirect(url)
-
-    url = get_normalized_url(url)
-    scraper = get_scraper_by_url(url)
-    if scraper is None:
-        # invalid url
-        return render(request, "common/error.html", {"msg": _("链接无效，查询失败")})
-    else:
-        try:
-            effective_url = scraper.get_effective_url(url)
-        except ValueError:
-            return render(request, "common/error.html", {"msg": _("链接无效，查询失败")})
-        try:
-            # raise ObjectDoesNotExist
-            entity = scraper.data_class.objects.get(source_url=effective_url)
-            # if exists then jump to detail page
-            if request.path.endswith(".json/"):
-                return JsonResponse({"num_pages": 1, "items": [entity.get_json()]})
-            return redirect(entity)
-        except ObjectDoesNotExist:
-            # scrape if not exists
-            try:
-                scraper.scrape(url)
-                form = scraper.save(request_user=request.user)
-            except IntegrityError as ie:  # duplicate key on source_url may be caused by user's double submission
-                try:
-                    entity = scraper.data_class.objects.get(source_url=effective_url)
-                    return redirect(entity)
-                except Exception as e:
-                    logger.error(f"Scrape Failed URL: {url}\n{e}")
-                    if settings.DEBUG:
-                        logger.error(
-                            "Expections during saving scraped data:", exc_info=e
-                        )
-                    return render(request, "common/error.html", {"msg": _("爬取数据失败😫")})
-            except Exception as e:
-                logger.error(f"Scrape Failed URL: {url}\n{e}")
-                if settings.DEBUG:
-                    logger.error("Expections during saving scraped data:", exc_info=e)
-                return render(request, "common/error.html", {"msg": _("爬取数据失败😫")})
-            return redirect(form.instance)
-
-
-def go_relogin(request):
-    return render(
-        request,
-        "common/error.html",
-        {
-            "url": reverse("users:connect") + "?domain=" + request.user.mastodon_site,
-            "msg": _("信息已保存，但是未能分享到联邦网络"),
-            "secondary_msg": _(
-                "可能是你在联邦网络(Mastodon/Pleroma/...)的登录状态过期了，正在跳转到联邦网络重新登录😼"
-            ),
-        },
-    )
+        return redirect(reverse("social:feed"))

journal/feeds.py

@@ -1,11 +1,7 @@
 from django.contrib.syndication.views import Feed
-from django.urls import reverse
-from books.models import BookReview
-from .models import User
 from markdown import markdown
-import operator
 import mimetypes
-
+from .models import *
 
 MAX_ITEM_PER_TYPE = 10
 
@@ -24,33 +20,27 @@ class ReviewFeed(Feed):
         return "%s的评论合集 - NeoDB" % user.display_name
 
     def items(self, user):
-        if user is None:
+        if user is None or user.preference.no_anonymous_view:
             return None
-        book_reviews = list(user.user_bookreviews.filter(visibility=0)[:MAX_ITEM_PER_TYPE])
-        movie_reviews = list(user.user_moviereviews.filter(visibility=0)[:MAX_ITEM_PER_TYPE])
-        album_reviews = list(user.user_albumreviews.filter(visibility=0)[:MAX_ITEM_PER_TYPE])
-        game_reviews = list(user.user_gamereviews.filter(visibility=0)[:MAX_ITEM_PER_TYPE])
-        all_reviews = sorted(
-                book_reviews + movie_reviews + album_reviews + game_reviews, 
-                key=operator.attrgetter('created_time'), 
-                reverse=True
-            )
-        return all_reviews
+        reviews = Review.objects.filter(owner=user, visibility=0)[:MAX_ITEM_PER_TYPE]
+        return reviews
 
-    def item_title(self, item):
+    def item_title(self, item: Review):
         return f"{item.title} - 评论《{item.item.title}》"
 
-    def item_description(self, item):
-        target_html = f'<p><a href="{item.item.absolute_url}">{item.item.title}</a></p>\n' 
-        html = markdown(item.content)
+    def item_description(self, item: Review):
+        target_html = (
+            f'<p><a href="{item.item.absolute_url}">{item.item.title}</a></p>\n'
+        )
+        html = markdown(item.body)
         return target_html + html
 
     # item_link is only needed if NewsItem has no get_absolute_url method.
-    def item_link(self, item):
-        return item.url
+    def item_link(self, item: Review):
+        return item.absolute_url
 
     def item_categories(self, item):
-        return [item.item.verbose_category_name]
+        return [item.item.category.label]
 
     def item_pubdate(self, item):
         return item.created_time
@@ -66,7 +56,11 @@ class ReviewFeed(Feed):
         return t
 
     def item_enclosure_length(self, item):
-        return item.item.cover.file.size
+        try:
+            size = item.item.cover.file.size
+        except Exception:
+            size = None
+        return size
 
     def item_comments(self, item):
-        return item.shared_link
+        return item.absolute_url

journal/models.py

@@ -39,7 +39,7 @@ def q_visible_to(viewer, owner):
         return Q()
     # elif viewer.is_blocked_by(owner):
     #     return Q(pk__in=[])
-    elif viewer.is_following(owner):
+    elif viewer.is_authenticated and viewer.is_following(owner):
         return Q(visibility__ne=2)
     else:
         return Q(visibility=0)
@@ -48,7 +48,7 @@ def q_visible_to(viewer, owner):
 def query_visible(user):
     return (
         Q(visibility=0)
-        | Q(owner_id__in=user.following, visibility=1)
+        | Q(owner_id__in=user.following if user.is_authenticated else [], visibility=1)
         | Q(owner_id=user.id)
     )
 
@@ -927,3 +927,10 @@ def reset_visibility_for_user(user: User, visibility: int):
     Comment.objects.filter(owner=user).update(visibility=visibility)
     Rating.objects.filter(owner=user).update(visibility=visibility)
     Review.objects.filter(owner=user).update(visibility=visibility)
+
+
+def remove_data_by_user(user: User):
+    ShelfMember.objects.filter(owner=user).delete()
+    Comment.objects.filter(owner=user).delete()
+    Rating.objects.filter(owner=user).delete()
+    Review.objects.filter(owner=user).delete()

journal/templates/collection.html

@@ -47,7 +47,7 @@
                 <div class="review-head__body">
                   <div class="review-head__info">
 
-                    <a href="{% url 'users:home' collection.owner.mastodon_username %}" class="review-head__owner-link">{{ collection.owner.mastodon_username }}</a>
+                    <a href="{% url 'journal:user_profile' collection.owner.mastodon_username %}" class="review-head__owner-link">{{ collection.owner.mastodon_username }}</a>
 
 
                     <span class="review-head__time">{{ collection.edited_time }}</span>

journal/templates/collection_edit.html

@@ -30,11 +30,13 @@
               <input class="button" type="submit" value="{% trans '提交' %}">
             </form>
             {{ form.media }}
-            <div class="dividing-line"></div>
           </div>
+          {% if collection %}
+          <div class="dividing-line"></div>
           <div class="single-section-wrapper">
             <div id="collection_items" class="entity-list" hx-get="{% url 'journal:collection_retrieve_items' collection.uuid %}?edit=1" hx-trigger="load"></div>
           </div>
+          {% endif %}
         </div>
       </section>
     </div>

journal/templates/list_item_base.html

@@ -70,7 +70,7 @@
     <div class="tag-collection">
       {% for tag_dict in item.tags %}
       <span class="tag-collection__tag">
-        <a href="{% url 'common:search' %}?tag={{ tag_dict }}">{{ tag_dict }}</a>
+        <a href="{% url 'catalog:search' %}?tag={{ tag_dict }}">{{ tag_dict }}</a>
       </span>
       {% endfor %}
     </div>

journal/templates/piece_delete.html

@@ -37,7 +37,7 @@
             {% endif %}
             <div class="review-head__body">
               <div class="review-head__info">
-                <a href="{% url 'users:home' piece.owner.mastodon_username %}" class="review-head__owner-link">{{ piece.owner.username }}</a>
+                <a href="{% url 'journal:user_profile' piece.owner.mastodon_username %}" class="review-head__owner-link">{{ piece.owner.username }}</a>
                 <span class="review-head__time">{{ piece.edited_time }}</span>
               </div>
             </div>

journal/templates/review.html

@@ -42,7 +42,7 @@
                 <div class="review-head__body">
                   <div class="review-head__info">
 
-                    <a href="{% url 'users:home' review.owner.mastodon_username %}" class="review-head__owner-link">{{ review.owner.username }}</a>
+                    <a href="{% url 'journal:user_profile' review.owner.mastodon_username %}" class="review-head__owner-link">{{ review.owner.username }}</a>
 
                     {% if mark %}    
 

journal/urls.py

@@ -1,5 +1,6 @@
 from django.urls import path, re_path
 from .views import *
+from .feeds import ReviewFeed
 from catalog.models import *
 
 
@@ -65,7 +66,7 @@ urlpatterns = [
         name="collection_update_item_note",
     ),
     re_path(
-        r"^user/(?P<user_name>[A-Za-z0-9_\-.@]+)/(?P<shelf_type>"
+        r"^users/(?P<user_name>[A-Za-z0-9_\-.@]+)/(?P<shelf_type>"
         + _get_all_shelf_types()
         + ")/(?P<item_category>"
         + _get_all_categories()
@@ -74,31 +75,32 @@ urlpatterns = [
         name="user_mark_list",
     ),
     re_path(
-        r"^user/(?P<user_name>[A-Za-z0-9_\-.@]+)/reviews/(?P<item_category>"
+        r"^users/(?P<user_name>[A-Za-z0-9_\-.@]+)/reviews/(?P<item_category>"
         + _get_all_categories()
         + ")/$",
         user_review_list,
         name="user_review_list",
     ),
     re_path(
-        r"^user/(?P<user_name>[A-Za-z0-9_\-.@]+)/tags/(?P<tag_title>[^/]+)/$",
+        r"^users/(?P<user_name>[A-Za-z0-9_\-.@]+)/tags/(?P<tag_title>[^/]+)/$",
         user_tag_member_list,
         name="user_tag_member_list",
     ),
     re_path(
-        r"^user/(?P<user_name>[A-Za-z0-9_\-.@]+)/collections/$",
+        r"^users/(?P<user_name>[A-Za-z0-9_\-.@]+)/collections/$",
         user_collection_list,
         name="user_collection_list",
     ),
     re_path(
-        r"^user/(?P<user_name>[A-Za-z0-9_\-.@]+)/like/collections/$",
+        r"^users/(?P<user_name>[A-Za-z0-9_\-.@]+)/like/collections/$",
         user_liked_collection_list,
         name="user_liked_collection_list",
     ),
     re_path(
-        r"^user/(?P<user_name>[A-Za-z0-9_\-.@]+)/tags/$",
+        r"^users/(?P<user_name>[A-Za-z0-9_\-.@]+)/tags/$",
         user_tag_list,
         name="user_tag_list",
     ),
-    re_path(r"^user/(?P<user_name>[A-Za-z0-9_\-.@]+)/$", home, name="user_profile"),
+    re_path(r"^users/(?P<user_name>[A-Za-z0-9_\-.@]+)/$", profile, name="user_profile"),
+    path("users/<str:id>/feed/reviews/", ReviewFeed(), name="review_feed"),
 ]

journal/views.py

@@ -1,11 +1,11 @@
 import logging
-from django.shortcuts import render, get_object_or_404, redirect, reverse
+from django.shortcuts import render, get_object_or_404, redirect
+from django.urls import reverse
 from django.contrib.auth.decorators import login_required, permission_required
 from django.utils.translation import gettext_lazy as _
 from django.http import (
     HttpResponse,
     HttpResponseBadRequest,
-    HttpResponseServerError,
     HttpResponseNotFound,
 )
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
@@ -14,10 +14,8 @@ from django.utils import timezone
 from django.core.paginator import Paginator
 from .models import *
 from django.conf import settings
-import re
 from django.http import HttpResponseRedirect
 from django.db.models import Q
-import time
 from management.models import Announcement
 from django.utils.baseconv import base62
 from .forms import *
@@ -157,6 +155,7 @@ def mark(request, item_uuid):
             except Exception:
                 return render_relogin(request)
             return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+    return HttpResponseBadRequest()
 
 
 def collection_retrieve(request, collection_uuid):
@@ -436,7 +435,6 @@ def user_tag_list(request, user_name):
     ):
         return render_user_blocked(request)
     tags = Tag.objects.filter(owner=user)
-    tags = user.tag_set.all()
     if user != request.user:
         tags = tags.filter(visibility=0)
     tags = tags.values("title").annotate(total=Count("members")).order_by("-total")
@@ -497,7 +495,7 @@ def user_liked_collection_list(request, user_name):
     )
 
 
-def home_anonymous(request, id):
+def profile_anonymous(request, id):
     login_url = settings.LOGIN_URL + "?next=" + request.get_full_path()
     try:
         username = id.split("@")[0]
@@ -515,15 +513,14 @@ def home_anonymous(request, id):
         return redirect(login_url)
 
 
-def home(request, user_name):
-    if not request.user.is_authenticated:
-        return home_anonymous(request, user_name)
+def profile(request, user_name):
     if request.method != "GET":
         return HttpResponseBadRequest()
     user = User.get(user_name)
     if user is None:
         return render_user_not_found(request)
-
+    if not request.user.is_authenticated and user.get_preference().no_anonymous_view:
+        return profile_anonymous(request, user_name)
     # access one's own home page
     if user == request.user:
         reports = Report.objects.order_by("-submitted_time").filter(is_read=False)
@@ -538,7 +535,7 @@ def home(request, user_name):
             pass
     # visit other's home page
     else:
-        if request.user.is_blocked_by(user) or request.user.is_blocking(user):
+        if user.is_blocked_by(request.user) or user.is_blocking(request.user):
             return render_user_blocked(request)
         # no these value on other's home page
         reports = None
@@ -583,12 +580,13 @@ def home(request, user_name):
     if user != request.user:
         liked_collections = liked_collections.filter(query_visible(request.user))
 
-    layout = user.get_preference().get_serialized_home_layout()
+    layout = user.get_preference().get_serialized_profile_layout()
     return render(
         request,
         "profile.html",
         {
             "user": user,
+            "top_tags": user.tag_manager.all_tags[:10],
             "shelf_list": shelf_list,
             "collections": collections[:5],
             "collections_count": collections.count(),

social/models.py

@@ -91,7 +91,7 @@ class DataSignalManager:
 
     @staticmethod
     def add_handler_for_model(model):
-        if not settings.DISABLE_SOCIAL:
+        if not settings.DISABLE_MODEL_SIGNAL:
             post_save.connect(DataSignalManager.save_handler, sender=model)
             pre_delete.connect(DataSignalManager.delete_handler, sender=model)
 

social/templates/activity/create_collection.html

@@ -31,7 +31,7 @@
       </span>
     </div>
     <span class="entity-list__entity-info" style="top:0px;">
-      <a href="{% url 'users:home' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> {% trans '创建了收藏单' %}
+      <a href="{% url 'journal:user_profile' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> {% trans '创建了收藏单' %}
     </span>
     <div class="entity-list__entity-title">
       <a href="{{ activity.action_object.url }}" class="entity-list__entity-link" style="font-weight:bold;">{{ activity.action_object.title }}

social/templates/activity/like_collection.html

@@ -31,8 +31,8 @@
       </span>
     </div>
     <span class="entity-list__entity-info" style="top:0px;">
-      <a href="{% url 'users:home' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> 关注了
-      <a href="{% url 'users:home' activity.action_object.target.owner.mastodon_username %}">{{ activity.action_object.target.owner.display_name }}</a>
+      <a href="{% url 'journal:user_profile' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> 关注了
+      <a href="{% url 'journal:user_profile' activity.action_object.target.owner.mastodon_username %}">{{ activity.action_object.target.owner.display_name }}</a>
       的收藏单
     </span>
     <div class="entity-list__entity-title">

social/templates/activity/mark_item.html

@@ -32,7 +32,7 @@
       </span>
     </div>
     <span class="entity-list__entity-info" style="top:0px;">
-      <a href="{% url 'users:home' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> {{ activity.action_object.parent.shelf_label }}
+      <a href="{% url 'journal:user_profile' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> {{ activity.action_object.parent.shelf_label }}
     </span>
     <div class="entity-list__entity-title">
       <a href="{{ activity.action_object.item.url }}" class="entity-list__entity-link" style="font-weight:bold;">{{ activity.action_object.item.title }}

social/templates/activity/review_item.html

@@ -32,7 +32,7 @@
       </span>
     </div>
     <span class="entity-list__entity-info" style="top:0px;">
-      <a href="{% url 'users:home' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> {% trans '评论了' %}
+      <a href="{% url 'journal:user_profile' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> {% trans '评论了' %}
       <a href="{{ activity.action_object.item.url }}">{{ activity.action_object.item.title }}
         {% if activity.action_object.item.year %}<small style="font-weight: lighter">({{ activity.action_object.item.year }})</small>{% endif %}
       </a>

social/templates/feed_data.html

@@ -76,7 +76,11 @@ hx-swap="outerHTML">
 </div>
 {% endif %}
 {% empty %}
+{% if request.GET.last %}
 <div>{% trans '目前没有更多内容了' %}</div>
+{% else %}
+<div>{% trans '在NeoDB导入或标记一些书影音，去联邦宇宙（长毛象）关注一些正在使用NeoDB的用户，这里就会显示你和她们的近期动态。' %}</div>
+{% endif %}
 {% endfor %}
 <script>
   // readonly star rating at aside section

sync/admin.py

@@ -1,4 +0,0 @@
-from django.contrib import admin
-from .models import *
-
-admin.site.register(SyncTask)

sync/apps.py

@@ -1,6 +0,0 @@
-from django.apps import AppConfig
-from django.conf import settings
-
-
-class SyncConfig(AppConfig):
-    name = 'sync'

sync/forms.py

@@ -1,21 +0,0 @@
-from django import forms
-from .models import SyncTask
-
-class SyncTaskForm(forms.ModelForm):
-    """Form definition for SyncTask."""
-
-    class Meta:
-        """Meta definition for SyncTaskform."""
-
-        model = SyncTask
-        fields = [
-            "user",
-            "file",
-            "overwrite",
-            "sync_book",
-            "sync_movie",
-            "sync_music",
-            "sync_game",
-            "default_public",
-        ]
-

sync/jobs.py

@@ -1,344 +0,0 @@
-import logging
-import pytz
-from dataclasses import dataclass
-from datetime import datetime
-from django.conf import settings
-from django.utils import timezone
-from django.core.exceptions import ObjectDoesNotExist
-from openpyxl import load_workbook
-from books.models import BookMark, Book, BookTag
-from movies.models import MovieMark, Movie, MovieTag
-from music.models import AlbumMark, Album, AlbumTag
-from games.models import GameMark, Game, GameTag
-from common.scraper import DoubanAlbumScraper, DoubanBookScraper, DoubanGameScraper, DoubanMovieScraper
-from common.models import MarkStatusEnum
-from .models import SyncTask
-
-
-logger = logging.getLogger(__name__)
-
-
-def __import_should_stop():
-    # TODO: using queue.connection.set(job.key + b':should_stop', 1, ex=30) on the caller side and connection.get(job.key + b':should_stop') on the worker side.
-    pass
-
-
-def import_doufen_task(synctask):
-    sync_doufen_job(synctask, __import_should_stop)
-
-
-class DoufenParser:
-
-    # indices in xlsx
-    URL_INDEX = 4
-    CONTENT_INDEX = 8
-    TAG_INDEX = 7
-    TIME_INDEX = 5
-    RATING_INDEX = 6
-
-    def __init__(self, task):
-        self.__file_path = task.file.path
-        self.__progress_sheet, self.__progress_row = task.get_breakpoint()
-        self.__is_new_task = True
-        if self.__progress_sheet is not None:
-            self.__is_new_task = False
-        if self.__progress_row is None:
-            self.__progress_row = 2
-        # data in the excel parse in python types
-        self.task = task
-        self.items = []
-
-    def __open_file(self):
-        self.__fp = open(self.__file_path, 'rb')
-        self.__wb = load_workbook(
-            self.__fp,
-            read_only=True,
-            data_only=True,
-            keep_links=False
-        )
-
-    def __close_file(self):
-        if self.__wb is not None:
-            self.__wb.close()
-        self.__fp.close()
-
-    def __get_item_classes_mapping(self):
-        '''
-        We assume that the sheets names won't change
-        '''
-        mappings = []
-        if self.task.sync_movie:
-            for sheet_name in ['想看', '在看', '看过']:
-                mappings.append({'sheet': sheet_name, 'mark_class': MovieMark,
-                                 'entity_class': Movie, 'tag_class': MovieTag, 'scraper': DoubanMovieScraper})
-        if self.task.sync_music:
-            for sheet_name in ['想听', '在听', '听过']:
-                mappings.append({'sheet': sheet_name, 'mark_class': AlbumMark,
-                                 'entity_class': Album, 'tag_class': AlbumTag, 'scraper': DoubanAlbumScraper})
-        if self.task.sync_book:
-            for sheet_name in ['想读', '在读', '读过']:
-                mappings.append({'sheet': sheet_name, 'mark_class': BookMark,
-                                 'entity_class': Book, 'tag_class': BookTag, 'scraper': DoubanBookScraper})
-        if self.task.sync_game:
-            for sheet_name in ['想玩', '在玩', '玩过']:
-                mappings.append({'sheet': sheet_name, 'mark_class': GameMark,
-                                 'entity_class': Game, 'tag_class': GameTag, 'scraper': DoubanGameScraper})
-
-        mappings.sort(key=lambda mapping: mapping['sheet'])
-
-        if not self.__is_new_task:
-            start_index = [mapping['sheet']
-                           for mapping in mappings].index(self.__progress_sheet)
-            mappings = mappings[start_index:]
-
-        self.__mappings = mappings
-        return mappings
-
-    def __parse_items(self):
-        assert self.__wb is not None, 'workbook not found'
-
-        item_classes_mappings = self.__get_item_classes_mapping()
-
-        is_first_sheet = True
-        for mapping in item_classes_mappings:
-            if mapping['sheet'] not in self.__wb:
-                print(f"Sheet not found: {mapping['sheet']}")
-                continue
-            ws = self.__wb[mapping['sheet']]
-
-            max_row = ws.max_row
-            # empty sheet
-            if max_row <= 1:
-                continue
-
-            # decide starting position
-            start_row_index = 2
-            if not self.__is_new_task and is_first_sheet:
-                start_row_index = self.__progress_row
-
-            # parse data
-            tz = pytz.timezone('Asia/Shanghai')
-            i = start_row_index
-            for row in ws.iter_rows(min_row=start_row_index, max_row=max_row, values_only=True):
-                cells = [cell for cell in row]
-                url = cells[self.URL_INDEX - 1]
-                tags = cells[self.TAG_INDEX - 1]
-                tags = list(set(tags.lower().split(','))) if tags else None
-                time = cells[self.TIME_INDEX - 1]
-                if time and type(time) == str:
-                    time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S")
-                    time = time.replace(tzinfo=tz)
-                elif time and type(time) == datetime:
-                    time = time.replace(tzinfo=tz)
-                else:
-                    time = None
-                content = cells[self.CONTENT_INDEX - 1]
-                if not content:
-                    content = ""
-                rating = cells[self.RATING_INDEX - 1]
-                rating = int(rating) * 2 if rating else None
-                self.items.append({
-                    'data': DoufenRowData(url, tags, time, content, rating),
-                    'entity_class': mapping['entity_class'],
-                    'mark_class': mapping['mark_class'],
-                    'tag_class': mapping['tag_class'],
-                    'scraper': mapping['scraper'],
-                    'sheet': mapping['sheet'],
-                    'row_index': i,
-                })
-                i = i + 1
-
-            # set first sheet flag
-            is_first_sheet = False
-
-    def __get_item_number(self):
-        assert self.__wb is not None, 'workbook not found'
-        assert self.__mappings is not None, 'mappings not found'
-
-        sheets = [mapping['sheet'] for mapping in self.__mappings]
-        item_number = 0
-        for sheet in sheets:
-            if sheet in self.__wb:
-                item_number += self.__wb[sheet].max_row - 1
-
-        return item_number
-
-    def __update_total_items(self):
-        total = self.__get_item_number()
-        self.task.total_items = total
-        self.task.save(update_fields=["total_items"])
-
-    def parse(self):
-        try:
-            self.__open_file()
-            self.__parse_items()
-            if self.__is_new_task:
-                self.__update_total_items()
-            self.__close_file()
-            return self.items
-        except Exception as e:
-            logger.error(f'Error parsing {self.__file_path} {e}')
-            self.task.is_failed = True
-        finally:
-            self.__close_file()
-        return []
-
-
-@dataclass
-class DoufenRowData:
-    url: str
-    tags: list
-    time: datetime
-    content: str
-    rating: int
-
-
-def add_new_mark(data, user, entity, entity_class, mark_class, tag_class, sheet, default_public):
-    params = {
-        'owner': user,
-        'created_time': data.time,
-        'edited_time': data.time,
-        'rating': data.rating,
-        'text': data.content,
-        'status': translate_status(sheet),
-        'visibility': 0 if default_public else 1,
-        entity_class.__name__.lower(): entity,
-    }
-    mark = mark_class.objects.create(**params)
-    entity.update_rating(None, data.rating)
-    if data.tags:
-        for tag in data.tags:
-            params = {
-                'content': tag,
-                entity_class.__name__.lower(): entity,
-                'mark': mark
-            }
-            try:
-                tag_class.objects.create(**params)
-            except Exception as e:
-                logger.error(f'Error creating tag {tag} {mark}: {e}')
-
-
-def overwrite_mark(entity, entity_class, mark, mark_class, tag_class, data, sheet):
-    old_rating = mark.rating
-    old_tags = getattr(mark, mark_class.__name__.lower() + '_tags').all()
-    # update mark logic
-    mark.created_time = data.time
-    mark.edited_time = data.time
-    mark.text = data.content
-    mark.rating = data.rating
-    mark.status = translate_status(sheet)
-    mark.save()
-    entity.update_rating(old_rating, data.rating)
-    if old_tags:
-        for tag in old_tags:
-            tag.delete()
-    if data.tags:
-        for tag in data.tags:
-            params = {
-                'content': tag,
-                entity_class.__name__.lower(): entity,
-                'mark': mark
-            }
-            try:
-                tag_class.objects.create(**params)
-            except Exception as e:
-                logger.error(f'Error creating tag {tag} {mark}: {e}')
-
-
-def sync_doufen_job(task, stop_check_func):
-    """
-    TODO: Update task status every certain amount of items to reduce IO consumption
-    """
-    task = SyncTask.objects.get(pk=task.pk)
-    if task.is_finished:
-        return
-
-    print(f'Task {task.pk}: loading')
-    parser = DoufenParser(task)
-    items = parser.parse()
-
-    # use pop to reduce memo consumption
-    while len(items) > 0 and not stop_check_func():
-        item = items.pop(0)
-        data = item['data']
-        entity_class = item['entity_class']
-        mark_class = item['mark_class']
-        tag_class = item['tag_class']
-        scraper = item['scraper']
-        sheet = item['sheet']
-        row_index = item['row_index']
-
-        # update progress
-        task.set_breakpoint(sheet, row_index, save=True)
-
-        # scrape the entity if not exists
-        try:
-            entity = entity_class.objects.get(source_url=data.url)
-            print(f'Task {task.pk}: {len(items)+1} remaining; matched {data.url}')
-        except ObjectDoesNotExist:
-            try:
-                print(f'Task {task.pk}: {len(items)+1} remaining; scraping {data.url}')
-                scraper.scrape(data.url)
-                form = scraper.save(request_user=task.user)
-                entity = form.instance
-            except Exception as e:
-                logger.error(f"Task {task.pk}: scrape failed: {data.url} {e}")
-                if settings.DEBUG:
-                    logger.error("Expections during scraping data:", exc_info=e)
-                task.failed_urls.append(data.url)
-                task.finished_items += 1
-                task.save(update_fields=['failed_urls', 'finished_items'])
-                continue
-
-        # sync mark
-        try:
-            # already exists
-            params = {
-                'owner': task.user,
-                entity_class.__name__.lower(): entity
-            }
-            mark = mark_class.objects.get(**params)
-
-            if task.overwrite:
-                overwrite_mark(entity, entity_class, mark,
-                               mark_class, tag_class, data, sheet)
-            else:
-                task.success_items += 1
-                task.finished_items += 1
-                task.save(update_fields=['success_items', 'finished_items'])
-                continue
-
-        except ObjectDoesNotExist:
-            add_new_mark(data, task.user, entity, entity_class,
-                         mark_class, tag_class, sheet, task.default_public)
-
-        except Exception as e:
-            logger.error(
-                f"Task {task.pk}: error when syncing marks", exc_info=e)
-            task.failed_urls.append(data.url)
-            task.finished_items += 1
-            task.save(update_fields=['failed_urls', 'finished_items'])
-            continue
-
-        task.success_items += 1
-        task.finished_items += 1
-        task.save(update_fields=['success_items', 'finished_items'])
-
-    # if task finish
-    print(f'Task {task.pk}: stopping')
-    if len(items) == 0:
-        task.is_finished = True
-        task.clear_breakpoint()
-        task.save(update_fields=['is_finished', 'break_point'])
-
-
-def translate_status(sheet_name):
-    if '想' in sheet_name:
-        return MarkStatusEnum.WISH
-    elif '在' in sheet_name:
-        return MarkStatusEnum.DO
-    elif '过' in sheet_name:
-        return MarkStatusEnum.COLLECT
-
-    raise ValueError("Not valid status")

sync/management/commands/resync.py

@@ -1,91 +0,0 @@
-from django.core.management.base import BaseCommand
-from common.scraper import get_scraper_by_url, get_normalized_url
-import pprint
-from sync.models import SyncTask
-from users.models import User
-from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
-from tqdm import tqdm
-from django.conf import settings
-import requests
-import os
-
-
-class Command(BaseCommand):
-    help = 'Re-scrape failed urls (via local proxy)'
-
-    def add_arguments(self, parser):
-        parser.add_argument('action', type=str, help='list/download')
-
-    def handle(self, *args, **options):
-        if options['action'] == 'list':
-            self.do_list()
-        else:
-            self.do_download()
-
-    def do_list(self):
-        tasks = SyncTask.objects.filter(failed_urls__isnull=False)
-        urls = []
-        for task in tqdm(tasks):
-            for url in task.failed_urls:
-                if url not in urls and url not in urls:
-                    url = get_normalized_url(str(url))
-                    scraper = get_scraper_by_url(url)
-                    if scraper is not None:
-                        try:
-                            url = scraper.get_effective_url(url)
-                            entity = scraper.data_class.objects.get(source_url=url)
-                        except ObjectDoesNotExist:
-                            urls.append(url)
-        f = open("/tmp/resync_todo.txt", "w")
-        f.write("\n".join(urls))
-        f.close()
-
-    def do_download(self):
-        self.stdout.write(f'Checking local proxy...{settings.LOCAL_PROXY}')
-        url = f'{settings.LOCAL_PROXY}?url=https://www.douban.com/doumail/'
-        try:
-            r = requests.get(url, timeout=settings.SCRAPING_TIMEOUT)
-        except Exception as e:
-            self.stdout.write(self.style.ERROR(e))
-            return
-        content = r.content.decode('utf-8')
-        if content.find('我的豆邮') == -1:
-            self.stdout.write(self.style.ERROR(f'Proxy check failed.'))
-            return
-
-        self.stdout.write(f'Loading urls...')
-        with open("/tmp/resync_todo.txt") as file:
-            todos = file.readlines()
-            todos = [line.strip() for line in todos]
-        with open("/tmp/resync_success.txt") as file:
-            skips = file.readlines()
-            skips = [line.strip() for line in skips]
-        f_f = open("/tmp/resync_failed.txt", "a")
-        f_i = open("/tmp/resync_ignore.txt", "a")
-        f_s = open("/tmp/resync_success.txt", "a")
-        user = User.objects.get(id=1)
-
-        for url in tqdm(todos):
-            scraper = get_scraper_by_url(url)
-            url = scraper.get_effective_url(url)
-            if url in skips:
-                self.stdout.write(f'Skip {url}')
-            elif scraper is None:
-                self.stdout.write(self.style.ERROR(f'Unable to find scraper for {url}'))
-                f_i.write(url + '\n')
-            else:
-                try:
-                    entity = scraper.data_class.objects.get(source_url=url)
-                    f_i.write(url + '\n')
-                except ObjectDoesNotExist:
-                    try:
-                        # self.stdout.write(f'Fetching {url} via {scraper.__name__}')
-                        scraper.scrape(url)
-                        form = scraper.save(request_user=user)
-                        f_s.write(url + '\n')
-                        f_s.flush()
-                        os.fsync(f_s.fileno())
-                        self.stdout.write(self.style.SUCCESS(f'Saved {url}'))
-                    except Exception as e:
-                        f_f.write(url + '\n')
-                        self.stdout.write(self.style.ERROR(f'Error {url}'))

sync/models.py

@@ -1,112 +0,0 @@
-from django.db import models
-from django.utils.translation import gettext_lazy as _
-import django.contrib.postgres.fields as postgres
-from users.models import User
-from common.utils import GenerateDateUUIDMediaFilePath
-from django.conf import settings
-
-
-def sync_file_path(instance, filename):
-    return GenerateDateUUIDMediaFilePath(instance, filename, settings.SYNC_FILE_PATH_ROOT)
-
-
-class SyncTask(models.Model):
-    """A class that records information about douban data synchronization task."""
-
-    #-----------------------------------#
-    # basic infos
-    #-----------------------------------#
-    user = models.ForeignKey(
-        User, on_delete=models.CASCADE, related_name='user_%(class)ss')
-    file = models.FileField(upload_to=sync_file_path, default='')
-
-    #-----------------------------------#
-    # options
-    #-----------------------------------#
-    # for the same book, if is already marked before sync, overwrite the previous mark or not
-    overwrite = models.BooleanField(default=False)
-    # sync book marks or not
-    sync_book = models.BooleanField()
-    # sync movie marks or not
-    sync_movie = models.BooleanField()
-    # sync music marks or not
-    sync_music = models.BooleanField()    
-    # sync game marks or not
-    sync_game = models.BooleanField()
-    # default visibility of marks
-    default_public = models.BooleanField()
-
-    #-----------------------------------#
-    # execution status
-    #-----------------------------------#
-    is_failed = models.BooleanField(default=False)
-    # fail_reason = models.TextField(default='')
-    is_finished = models.BooleanField(default=False)
-    # how many items to synchronize
-    total_items = models.PositiveIntegerField(default=0)
-    # how many items are handled
-    finished_items = models.PositiveIntegerField(default=0)
-    # how many imtes have been synchronized successfully
-    success_items = models.PositiveIntegerField(default=0)
-
-    failed_urls = postgres.ArrayField(
-        models.URLField(blank=True, default='', max_length=200),
-        null=True,
-        blank=True,
-        default=list,
-    )
-
-    break_point = models.CharField(max_length=100, default="", blank=True)
-
-    started_time = models.DateTimeField(auto_now_add=True)
-    ended_time = models.DateTimeField(auto_now=True)
-
-    class Meta:
-        """Meta definition for SyncTask."""
-
-        verbose_name = 'SyncTask'
-        verbose_name_plural = 'SyncTasks'
-
-    def __str__(self):
-        """Unicode representation of SyncTask."""
-        return f'{self.id} {self.user} {self.file} {self.get_status_emoji()} {self.success_items}/{self.finished_items}/{self.total_items}'
-
-    def get_status_emoji(self):
-        return ("❌" if self.is_failed else "✔") if self.is_finished else "⚡"
-
-    def get_duration(self):
-        return self.ended_time - self.started_time
-    
-    def get_overwritten_items(self):
-        if self.overwrite:
-            return self.overwrite_books + self.overwrite_games + self.overwrite_movies + self.overwrite_music
-        else:
-            return 0
-
-    def get_progress(self):
-        """
-        @return: return percentage
-        """
-        if self.is_finished:
-            return 100
-        else:
-            if self.total_items > 0:
-                return 100 * self.finished_items / self.total_items
-            else:
-                return 0
-
-    def get_breakpoint(self):
-        if not self.break_point:
-            return None, None
-        progress = self.break_point.split('-')
-        return progress[0], int(progress[1])
-
-    def set_breakpoint(self, sheet_name, row_index, save=False):
-        self.break_point = f"{sheet_name}-{row_index}"
-        if save:
-            self.save(update_fields=['break_point'])
-
-    def clear_breakpoint(self, save=False):
-        self.break_point = ""
-        if save:
-            self.save(update_fields=['break_point'])

sync/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

sync/urls.py

@@ -1,10 +0,0 @@
-from django.urls import path
-from .views import *
-
-
-app_name = 'sync'
-urlpatterns = [
-    path('douban/', sync_douban, name='douban'),
-    path('progress/', query_progress, name='progress'),
-    path('last/', query_last_task, name='last'),
-]

sync/views.py

@@ -1,72 +0,0 @@
-from django.shortcuts import render
-from django.contrib.auth.decorators import login_required
-from django.http import HttpResponseBadRequest, JsonResponse, HttpResponse
-from .models import SyncTask
-from .forms import SyncTaskForm
-from .jobs import import_doufen_task
-import tempfile
-import os
-from threading import Thread
-import openpyxl
-from django.utils.datastructures import MultiValueDictKeyError
-from openpyxl.utils.exceptions import InvalidFileException
-from zipfile import BadZipFile
-import django_rq
-
-
-@login_required
-def sync_douban(request):
-    """
-    Sync douban data from .xlsx file generated by doufen
-    """
-    if request.method == 'POST':
-        # validate sunmitted data
-        try:
-            uploaded_file = request.FILES['file']
-            wb = openpyxl.open(uploaded_file, read_only=True,
-                               data_only=True, keep_links=False)
-            wb.close()
-        except (MultiValueDictKeyError, InvalidFileException, BadZipFile) as e:
-            # raise e
-            return HttpResponseBadRequest(content="invalid excel file")
-
-        # file_data = {'file': request.FILES['xlsx']}
-        form = SyncTaskForm(request.POST, request.FILES)
-        if form.is_valid():
-            # stop all preivous task
-            SyncTask.objects.filter(user=request.user, is_finished=False).update(is_finished=True)
-            form.save()
-            django_rq.get_queue('doufen').enqueue(import_doufen_task, form.instance, job_id=f'SyncTask_{form.instance.id}')
-            return HttpResponse(status=204)
-        else:
-            return HttpResponseBadRequest()
-    else:
-        return HttpResponseBadRequest()
-
-
-@login_required
-def query_progress(request):
-    task = request.user.user_synctasks.order_by('-id').first()
-    if task is not None:
-        return JsonResponse({
-            'progress': task.get_progress()
-        })
-    else:
-        return JsonResponse()
-
-
-@login_required
-def query_last_task(request):
-    task = request.user.user_synctasks.order_by('-id').first()
-    if task is not None:
-        return JsonResponse({
-            'total_items': task.total_items,
-            'success_items': task.success_items,
-            'finished_items': task.finished_items,
-            'status': task.get_status_emoji(),
-            'is_finished': task.is_finished,
-            'failed_urls': task.failed_urls,
-            'ended_time': task.ended_time if task.is_finished else None,
-        })
-    else:
-        return JsonResponse()

timeline/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

timeline/apps.py

@@ -1,15 +0,0 @@
-from django.apps import AppConfig
-
-
-class TimelineConfig(AppConfig):
-    name = 'timeline'
-
-    def ready(self):
-        from .models import init_post_save_handler
-        from books.models import BookMark, BookReview
-        from movies.models import MovieMark, MovieReview
-        from games.models import GameMark, GameReview
-        from music.models import AlbumMark, AlbumReview, SongMark, SongReview
-        from collection.models import Collection, CollectionMark
-        for m in [BookMark, BookReview, MovieMark, MovieReview, GameMark, GameReview, AlbumMark, AlbumReview, SongMark, SongReview, Collection, CollectionMark]:
-            init_post_save_handler(m)

timeline/management/commands/regen_activity.py

@@ -1,20 +0,0 @@
-from django.core.management.base import BaseCommand
-from users.models import User
-from datetime import timedelta
-from django.utils import timezone
-from timeline.models import Activity
-from books.models import BookMark, BookReview
-from movies.models import MovieMark, MovieReview
-from games.models import GameMark, GameReview
-from music.models import AlbumMark, AlbumReview, SongMark, SongReview
-from collection.models import Collection, CollectionMark
-from tqdm import tqdm
-
-
-class Command(BaseCommand):
-    help = 'Re-populating activity for timeline'
-
-    def handle(self, *args, **options):
-        for cl in [BookMark, BookReview, MovieMark, MovieReview, GameMark, GameReview, AlbumMark, AlbumReview, SongMark, SongReview, Collection, CollectionMark]:
-            for a in tqdm(cl.objects.filter(created_time__gt='2022-1-1 00:00+0800'), desc=f'Populating {cl.__name__}'):
-                Activity.upsert_item(a)

timeline/models.py

@@ -1,63 +0,0 @@
-from django.db import models
-from common.models import UserOwnedEntity
-from books.models import BookMark, BookReview
-from movies.models import MovieMark, MovieReview
-from games.models import GameMark, GameReview
-from music.models import AlbumMark, AlbumReview, SongMark, SongReview
-from collection.models import Collection, CollectionMark
-from django.db.models.signals import post_save, post_delete
-
-
-class Activity(UserOwnedEntity):
-    bookmark = models.ForeignKey(BookMark, models.CASCADE, null=True)
-    bookreview = models.ForeignKey(BookReview, models.CASCADE, null=True)
-    moviemark = models.ForeignKey(MovieMark, models.CASCADE, null=True)
-    moviereview = models.ForeignKey(MovieReview, models.CASCADE, null=True)
-    gamemark = models.ForeignKey(GameMark, models.CASCADE, null=True)
-    gamereview = models.ForeignKey(GameReview, models.CASCADE, null=True)
-    albummark = models.ForeignKey(AlbumMark, models.CASCADE, null=True)
-    albumreview = models.ForeignKey(AlbumReview, models.CASCADE, null=True)
-    songmark = models.ForeignKey(SongMark, models.CASCADE, null=True)
-    songreview = models.ForeignKey(SongReview, models.CASCADE, null=True)
-    collection = models.ForeignKey(Collection, models.CASCADE, null=True)
-    collectionmark = models.ForeignKey(CollectionMark, models.CASCADE, null=True)
-
-    @property
-    def target(self):
-        items = [self.bookmark, self.bookreview, self.moviemark, self.moviereview, self.gamemark, self.gamereview,
-                 self.songmark, self.songreview, self.albummark, self.albumreview, self.collection, self.collectionmark]
-        return next((x for x in items if x is not None), None)
-
-    @property
-    def mark(self):
-        items = [self.bookmark, self.moviemark, self.gamemark, self.songmark, self.albummark]
-        return next((x for x in items if x is not None), None)
-
-    @property
-    def review(self):
-        items = [self.bookreview, self.moviereview, self.gamereview, self.songreview, self.albumreview]
-        return next((x for x in items if x is not None), None)
-
-    @classmethod
-    def upsert_item(self, item):
-        attr = item.__class__.__name__.lower()
-        f = {'owner': item.owner, attr: item}
-        activity = Activity.objects.filter(**f).first()
-        if not activity:
-            activity = Activity.objects.create(**f)
-        activity.created_time = item.created_time
-        activity.visibility = item.visibility
-        activity.save()
-
-
-def _post_save_handler(sender, instance, created, **kwargs):
-    Activity.upsert_item(instance)
-
-
-# def activity_post_delete_handler(sender, instance, **kwargs):
-#     pass
-
-
-def init_post_save_handler(model):
-    post_save.connect(_post_save_handler, sender=model)
-    # post_delete.connect(activity_post_delete_handler, sender=model)  # delete handled by database

timeline/templates/timeline.html

@@ -1,83 +0,0 @@
-{% load static %}
-{% load i18n %}
-{% load l10n %}
-{% load admin_url %}
-{% load mastodon %}
-{% load oauth_token %}
-{% load truncate %}
-{% load thumb %}
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{{ site_name }}</title>
-
-    {% include "partial/_common_libs.html" with jquery=1 %}
-
-    <script src="{% static 'lib/js/rating-star.js' %}"></script>
-    <script>
-        $(document).ready( function() {
-        let render = function() {
-            let ratingLabels = $(".rating-star");
-            $(ratingLabels).each( function(index, value) {
-                let ratingScore = $(this).data("rating-score") / 2;
-                $(this).starRating({
-                    initialRating: ratingScore,
-                    readOnly: true,
-                    starSize: 16,
-                });
-            });
-        };
-        document.body.addEventListener('htmx:load', function(evt) {
-            render();
-        });
-        render();
-        });
-    </script>
-    <script src="{% static 'js/mastodon.js' %}"></script>
-    <script src="{% static 'js/home.js' %}"></script>
-</head>
-
-<body>
-    <div id="page-wrapper">
-        <div id="content-wrapper">
-            {% include "partial/_navbar.html" with current="timeline" %}
-
-            <section id="content" class="container">
-                <div class="grid grid--reverse-order">
-                    <div class="grid__main grid__main--reverse-order">
-                        <div class="main-section-wrapper">
-                            <div class="entity-list">
-
-                                <!-- <div class="set">
-                                    <h5 class="entity-list__title">
-                                        我的时间轴
-                                    </h5>
-                                </div> -->
-                                <ul class="entity-list__entities">
-                                    <div hx-get="{% url 'timeline:data' %}" hx-trigger="revealed" hx-swap="outerHTML"></div>
-                                </ul>
-                            </div>
-                        </div>
-                    </div>
-
-                    {% include "partial/_sidebar.html" %}
-                </div>
-            </section>
-        </div>
-        {% include "partial/_footer.html" %}
-    </div>
-
-    <script>
-      document.body.addEventListener('htmx:configRequest', (event) => {
-        event.detail.headers['X-CSRFToken'] = '{{ csrf_token }}';
-      })
-    </script>
-
-{% if unread_announcements %}
-{% include "partial/_announcement.html" %}
-{% endif %}
-</body>
-</html>

timeline/templates/timeline_data.html

@@ -1,125 +0,0 @@
-{% load static %}
-{% load i18n %}
-{% load l10n %}
-{% load admin_url %}
-{% load mastodon %}
-{% load oauth_token %}
-{% load truncate %}
-{% load thumb %}
-{% load prettydate %}
-{% load user_item %}
-
-{% for activity in activities %}
-{% current_user_marked_item activity.target.item as marked %}
-<li class="entity-list__entity">
-    <div class="entity-list__entity-img-wrapper">
-        <a href="{{ activity.target.item.url }}">
-            <img src="{{ activity.target.item.cover|thumb:'normal' }}" alt="" class="entity-list__entity-img" style="min-width:80px;max-width:80px">
-        </a>
-        {% if not marked %}
-        <a class="entity-list__entity-action-icon" hx-post="{{ activity.target.item.wish_url }}">➕</a>
-        {% endif %}
-    </div>
-    <div class="entity-list__entity-text">
-        <div class="collection-item-position-edit">
-            <span class="entity-marks__mark-time">
-                {% if activity.target.shared_link %}
-                <a href="{{ activity.target.shared_link }}" target="_blank">
-                    <img src="{% static 'img/fediverse.svg' %}" style="filter: invert(93%) sepia(1%) saturate(53%) hue-rotate(314deg) brightness(95%) contrast(80%); vertical-align:text-top; max-width:14px; margin-right:6px;" />
-                    <span class="entity-marks__mark-time">{{ activity.target.created_time|prettydate }}</span></a>
-                {% else %}
-                <a><span class="entity-marks__mark-time">{{ activity.target.created_time|prettydate }}</span></a>
-                {% endif %}
-            </span>
-        </div>
-        <span class="entity-list__entity-info" style="top:0px;">
-            <a href="{% url 'users:home' activity.owner.mastodon_username %}">{{ activity.owner.display_name }}</a> {{ activity.target.translated_status }}
-        </span>
-        <div class="entity-list__entity-title">
-            <a href="{{ activity.target.item.url }}" class="entity-list__entity-link" style="font-weight:bold;">{{ activity.target.item.title }}
-            {% if activity.target.item.year %}<small style="font-weight: lighter">({{ activity.target.item.year }})</small>{% endif %}
-            </a>
-            {% if activity.target.item.source_url %}
-            <a href="{{ activity.target.item.source_url }}">
-                <span class="source-label source-label__{{ activity.target.item.source_site }}" style="font-size:xx-small;">{{ activity.target.item.get_source_site_display }}</span>
-            </a>
-            {% endif %}
-        </div>
-        <p class="entity-list__entity-brief">
-            {% if activity.review %}
-                <a href="{{ activity.review.url }}">{{ activity.review.title }}</a>
-            {% endif %}
-            {% if activity.mark %}
-                {% if activity.mark.rating %}
-                <span class="entity-marks__rating-star rating-star" data-rating-score="{{ activity.mark.rating | floatformat:"0" }}" style=""></span>
-                {% endif %}
-
-                {% if activity.mark.text %}
-                <p class="entity-marks__mark-content">{{ activity.mark.text }}</p>
-                {% endif %}
-            {% endif %}
-        </p>
-    </div>
-</li>
-{% if forloop.last %}
-<div class="htmx-indicator" style="margin-left: 60px;" 
-    hx-get="{% url 'timeline:data' %}?last={{ activity.created_time|date:'Y-m-d H:i:s.uO'|urlencode }}"
-    hx-trigger="revealed"
-    hx-swap="outerHTML">
-<svg width="16" height="16" viewBox="0 0 135 140" xmlns="http://www.w3.org/2000/svg" fill="#ccc">
-    <rect y="10" width="15" height="120" rx="6">
-        <animate attributeName="height"
-             begin="0.5s" dur="1s"
-             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
-             repeatCount="indefinite" />
-        <animate attributeName="y"
-             begin="0.5s" dur="1s"
-             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
-             repeatCount="indefinite" />
-    </rect>
-    <rect x="30" y="10" width="15" height="120" rx="6">
-        <animate attributeName="height"
-             begin="0.25s" dur="1s"
-             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
-             repeatCount="indefinite" />
-        <animate attributeName="y"
-             begin="0.25s" dur="1s"
-             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
-             repeatCount="indefinite" />
-    </rect>
-    <rect x="60" width="15" height="140" rx="6">
-        <animate attributeName="height"
-             begin="0s" dur="1s"
-             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
-             repeatCount="indefinite" />
-        <animate attributeName="y"
-             begin="0s" dur="1s"
-             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
-             repeatCount="indefinite" />
-    </rect>
-    <rect x="90" y="10" width="15" height="120" rx="6">
-        <animate attributeName="height"
-             begin="0.25s" dur="1s"
-             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
-             repeatCount="indefinite" />
-        <animate attributeName="y"
-             begin="0.25s" dur="1s"
-             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
-             repeatCount="indefinite" />
-    </rect>
-    <rect x="120" y="10" width="15" height="120" rx="6">
-        <animate attributeName="height"
-             begin="0.5s" dur="1s"
-             values="120;110;100;90;80;70;60;50;40;140;120" calcMode="linear"
-             repeatCount="indefinite" />
-        <animate attributeName="y"
-             begin="0.5s" dur="1s"
-             values="10;15;20;25;30;35;40;45;50;0;10" calcMode="linear"
-             repeatCount="indefinite" />
-    </rect>
-</svg>
-</div>
-{% endif %}
-{% empty %}
-<div>{% trans '目前没有更多内容了' %}</div>
-{% endfor %}

timeline/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

timeline/urls.py

@@ -1,9 +0,0 @@
-from django.urls import path, re_path
-from .views import *
-
-
-app_name = 'timeline'
-urlpatterns = [
-    path('', timeline, name='timeline'),
-    path('data', data, name='data'),
-]

timeline/views.py

@@ -1,70 +0,0 @@
-import logging
-from django.shortcuts import render, get_object_or_404, redirect, reverse
-from django.contrib.auth.decorators import login_required, permission_required
-from django.utils.translation import gettext_lazy as _
-from django.http import HttpResponseBadRequest, HttpResponseServerError
-from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
-from django.db import IntegrityError, transaction
-from django.db.models import Count
-from django.utils import timezone
-from django.core.paginator import Paginator
-from mastodon import mastodon_request_included
-from mastodon.models import MastodonApplication
-from common.utils import PageLinksGenerator
-from .models import *
-from books.models import BookTag
-from movies.models import MovieTag
-from games.models import GameTag
-from music.models import AlbumTag
-from django.conf import settings
-import re
-from users.models import User
-from django.http import HttpResponseRedirect
-from django.db.models import Q
-import time
-from management.models import Announcement
-
-
-logger = logging.getLogger(__name__)
-mastodon_logger = logging.getLogger("django.mastodon")
-PAGE_SIZE = 20
-
-
-@login_required
-def timeline(request):
-    if request.method != 'GET':
-        return
-    user = request.user
-    unread = Announcement.objects.filter(pk__gt=user.read_announcement_index).order_by('-pk')
-    if unread:
-        user.read_announcement_index = Announcement.objects.latest('pk').pk
-        user.save(update_fields=['read_announcement_index'])
-    return render(
-        request,
-        'timeline.html',
-        {
-            'book_tags': BookTag.all_by_user(user)[:10],
-            'movie_tags': MovieTag.all_by_user(user)[:10],
-            'music_tags': AlbumTag.all_by_user(user)[:10],
-            'game_tags': GameTag.all_by_user(user)[:10],
-            'unread_announcements': unread,
-        }
-    )
-
-
-@login_required
-def data(request):
-    if request.method != 'GET':
-        return
-    q = Q(owner_id__in=request.user.following, visibility__lt=2) | Q(owner_id=request.user.id)
-    last = request.GET.get('last')
-    if last:
-        q = q & Q(created_time__lt=last)
-    activities = Activity.objects.filter(q).order_by('-created_time')[:PAGE_SIZE]
-    return render(
-        request,
-        'timeline_data.html',
-        {
-            'activities': activities,
-        }
-    )

users/account.py

@@ -12,17 +12,8 @@ from .forms import ReportForm
 from mastodon.api import *
 from mastodon import mastodon_request_included
 from common.config import *
-from common.models import MarkStatusEnum
 from common.utils import PageLinksGenerator
 from management.models import Announcement
-from books.models import *
-from movies.models import *
-from music.models import *
-from games.models import *
-from books.forms import BookMarkStatusTranslator
-from movies.forms import MovieMarkStatusTranslator
-from music.forms import MusicMarkStatusTranslator
-from games.forms import GameMarkStatusTranslator
 from mastodon.models import MastodonApplication
 from mastodon.api import verify_account
 from django.conf import settings
@@ -34,35 +25,28 @@ from datetime import timedelta
 from django.utils import timezone
 import json
 from django.contrib import messages
-from books.models import BookMark, BookReview
-from movies.models import MovieMark, MovieReview
-from games.models import GameMark, GameReview
-from music.models import AlbumMark, SongMark, AlbumReview, SongReview
-from collection.models import Collection, CollectionMark
-from common.importers.goodreads import GoodreadsImporter
-from common.importers.douban import DoubanImporter
-
+from journal.models import remove_data_by_user
 
 # the 'login' page that user can see
 def login(request):
-    if request.method == 'GET':
-        selected_site = request.GET.get('site', default='')
+    if request.method == "GET":
+        selected_site = request.GET.get("site", default="")
 
         sites = MastodonApplication.objects.all().order_by("domain_name")
 
         # store redirect url in the cookie
-        if request.GET.get('next'):
-            request.session['next_url'] = request.GET.get('next')
+        if request.GET.get("next"):
+            request.session["next_url"] = request.GET.get("next")
 
         return render(
             request,
-            'users/login.html',
+            "users/login.html",
             {
-                'sites': sites,
-                'scope': quote(settings.MASTODON_CLIENT_SCOPE),
-                'selected_site': selected_site,
-                'allow_any_site': settings.MASTODON_ALLOW_ANY_SITE,
-            }
+                "sites": sites,
+                "scope": quote(settings.MASTODON_CLIENT_SCOPE),
+                "selected_site": selected_site,
+                "allow_any_site": settings.MASTODON_ALLOW_ANY_SITE,
+            },
         )
     else:
         return HttpResponseBadRequest()
@@ -70,70 +54,80 @@ def login(request):
 
 # connect will redirect to mastodon server
 def connect(request):
-    login_domain = request.session['swap_domain'] if request.session.get('swap_login') else request.GET.get('domain')
+    login_domain = (
+        request.session["swap_domain"]
+        if request.session.get("swap_login")
+        else request.GET.get("domain")
+    )
     if not login_domain:
-        return render(request, 'common/error.html', {'msg': '未指定实例域名', 'secondary_msg': "", })
-    login_domain = login_domain.strip().lower().split('//')[-1].split('/')[0].split('@')[-1]
+        return render(
+            request,
+            "common/error.html",
+            {
+                "msg": "未指定实例域名",
+                "secondary_msg": "",
+            },
+        )
+    login_domain = (
+        login_domain.strip().lower().split("//")[-1].split("/")[0].split("@")[-1]
+    )
     domain, version = get_instance_info(login_domain)
     app, error_msg = get_mastodon_application(domain)
     if app is None:
-        return render(request, 'common/error.html', {'msg': error_msg, 'secondary_msg': "", })
+        return render(
+            request,
+            "common/error.html",
+            {
+                "msg": error_msg,
+                "secondary_msg": "",
+            },
+        )
     else:
         login_url = get_mastodon_login_url(app, login_domain, version, request)
         resp = redirect(login_url)
-        resp.set_cookie('mastodon_domain', domain)
+        resp.set_cookie("mastodon_domain", domain)
         return resp
 
 
 # mastodon server redirect back to here
 @mastodon_request_included
 def OAuth2_login(request):
-    if request.method != 'GET':
+    if request.method != "GET":
         return HttpResponseBadRequest()
 
-    code = request.GET.get('code')
-    site = request.COOKIES.get('mastodon_domain')
+    code = request.GET.get("code")
+    site = request.COOKIES.get("mastodon_domain")
     try:
         token, refresh_token = obtain_token(site, request, code)
     except ObjectDoesNotExist:
         return HttpResponseBadRequest("Mastodon site not registered")
     if not token:
-        return render(
-            request,
-            'common/error.html',
-            {
-                'msg': _("认证失败😫")
-            }
-        )
+        return render(request, "common/error.html", {"msg": _("认证失败😫")})
 
-    if request.session.get('swap_login', False) and request.user.is_authenticated:  # swap login for existing user
+    if (
+        request.session.get("swap_login", False) and request.user.is_authenticated
+    ):  # swap login for existing user
         return swap_login(request, token, site, refresh_token)
 
     user = authenticate(request, token=token, site=site)
     if user:  # existing user
         user.mastodon_token = token
         user.mastodon_refresh_token = refresh_token
-        user.save(update_fields=['mastodon_token', 'mastodon_refresh_token'])
+        user.save(update_fields=["mastodon_token", "mastodon_refresh_token"])
         auth_login(request, user)
-        if request.session.get('next_url') is not None:
-            response = redirect(request.session.get('next_url'))
-            del request.session['next_url']
+        if request.session.get("next_url") is not None:
+            response = redirect(request.session.get("next_url"))
+            del request.session["next_url"]
         else:
-            response = redirect(reverse('common:home'))
+            response = redirect(reverse("common:home"))
         return response
     else:  # newly registered user
         code, user_data = verify_account(site, token)
         if code != 200 or user_data is None:
-            return render(
-                request,
-                'common/error.html',
-                {
-                    'msg': _("联邦网络访问失败😫")
-                }
-            )
+            return render(request, "common/error.html", {"msg": _("联邦网络访问失败😫")})
         new_user = User(
-            username=user_data['username'],
-            mastodon_id=user_data['id'],
+            username=user_data["username"],
+            mastodon_id=user_data["id"],
             mastodon_site=site,
             mastodon_token=token,
             mastodon_refresh_token=refresh_token,
@@ -141,15 +135,15 @@ def OAuth2_login(request):
         )
         new_user.save()
         Preference.objects.create(user=new_user)
-        request.session['new_user'] = True
+        request.session["new_user"] = True
         auth_login(request, new_user)
-        return redirect(reverse('users:register'))
+        return redirect(reverse("users:register"))
 
 
 @mastodon_request_included
 @login_required
 def logout(request):
-    if request.method == 'GET':
+    if request.method == "GET":
         # revoke_token(request.user.mastodon_site, request.user.mastodon_token)
         auth_logout(request)
         return redirect(reverse("users:login"))
@@ -160,9 +154,9 @@ def logout(request):
 @mastodon_request_included
 @login_required
 def reconnect(request):
-    if request.method == 'POST':
-        request.session['swap_login'] = True
-        request.session['swap_domain'] = request.POST['domain']
+    if request.method == "POST":
+        request.session["swap_login"] = True
+        request.session["swap_domain"] = request.POST["domain"]
         return connect(request)
     else:
         return HttpResponseBadRequest()
@@ -170,76 +164,85 @@ def reconnect(request):
 
 @mastodon_request_included
 def register(request):
-    if request.session.get('new_user'):
-        del request.session['new_user']
-        return render(request, 'users/register.html')
+    if request.session.get("new_user"):
+        del request.session["new_user"]
+        return render(request, "users/register.html")
     else:
-        return redirect(reverse('common:home'))
+        return redirect(reverse("common:home"))
 
 
 def swap_login(request, token, site, refresh_token):
-    del request.session['swap_login']
-    del request.session['swap_domain']
+    del request.session["swap_login"]
+    del request.session["swap_domain"]
     code, data = verify_account(site, token)
     current_user = request.user
     if code == 200 and data is not None:
-        username = data['username']
+        username = data["username"]
         if username == current_user.username and site == current_user.mastodon_site:
-            messages.add_message(request, messages.ERROR, _(f'该身份 {username}@{site} 与当前账号相同。'))
+            messages.add_message(
+                request, messages.ERROR, _(f"该身份 {username}@{site} 与当前账号相同。")
+            )
         else:
             try:
                 existing_user = User.objects.get(username=username, mastodon_site=site)
-                messages.add_message(request, messages.ERROR, _(f'该身份 {username}@{site} 已被用于其它账号。'))
+                messages.add_message(
+                    request, messages.ERROR, _(f"该身份 {username}@{site} 已被用于其它账号。")
+                )
             except ObjectDoesNotExist:
                 current_user.username = username
-                current_user.mastodon_id = data['id']
+                current_user.mastodon_id = data["id"]
                 current_user.mastodon_site = site
                 current_user.mastodon_token = token
                 current_user.mastodon_refresh_token = refresh_token
                 current_user.mastodon_account = data
-                current_user.save(update_fields=['username', 'mastodon_id', 'mastodon_site', 'mastodon_token', 'mastodon_refresh_token', 'mastodon_account'])
-                django_rq.get_queue('mastodon').enqueue(refresh_mastodon_data_task, current_user, token)
-                messages.add_message(request, messages.INFO, _(f'账号身份已更新为 {username}@{site}。'))
+                current_user.save(
+                    update_fields=[
+                        "username",
+                        "mastodon_id",
+                        "mastodon_site",
+                        "mastodon_token",
+                        "mastodon_refresh_token",
+                        "mastodon_account",
+                    ]
+                )
+                django_rq.get_queue("mastodon").enqueue(
+                    refresh_mastodon_data_task, current_user, token
+                )
+                messages.add_message(
+                    request, messages.INFO, _(f"账号身份已更新为 {username}@{site}。")
+                )
     else:
-        messages.add_message(request, messages.ERROR, _('连接联邦网络获取身份信息失败。'))
-    return redirect(reverse('users:data'))
+        messages.add_message(request, messages.ERROR, _("连接联邦网络获取身份信息失败。"))
+    return redirect(reverse("users:data"))
 
 
 def auth_login(request, user):
-    """ Decorates django ``login()``. Attach token to session."""
+    """Decorates django ``login()``. Attach token to session."""
     auth.login(request, user)
-    if user.mastodon_last_refresh < timezone.now() - timedelta(hours=1) or user.mastodon_account == {}:
-        django_rq.get_queue('mastodon').enqueue(refresh_mastodon_data_task, user)
+    if (
+        user.mastodon_last_refresh < timezone.now() - timedelta(hours=1)
+        or user.mastodon_account == {}
+    ):
+        django_rq.get_queue("mastodon").enqueue(refresh_mastodon_data_task, user)
 
 
 def auth_logout(request):
-    """ Decorates django ``logout()``. Release token in session."""
+    """Decorates django ``logout()``. Release token in session."""
     auth.logout(request)
 
 
 @login_required
 def clear_data(request):
-    if request.method == 'POST':
-        if request.POST.get('verification') == request.user.mastodon_username:
-            BookMark.objects.filter(owner=request.user).delete()
-            MovieMark.objects.filter(owner=request.user).delete()
-            GameMark.objects.filter(owner=request.user).delete()
-            AlbumMark.objects.filter(owner=request.user).delete()
-            SongMark.objects.filter(owner=request.user).delete()
-            BookReview.objects.filter(owner=request.user).delete()
-            MovieReview.objects.filter(owner=request.user).delete()
-            GameReview.objects.filter(owner=request.user).delete()
-            AlbumReview.objects.filter(owner=request.user).delete()
-            SongReview.objects.filter(owner=request.user).delete()
-            CollectionMark.objects.filter(owner=request.user).delete()
-            Collection.objects.filter(owner=request.user).delete()
+    if request.method == "POST":
+        if request.POST.get("verification") == request.user.mastodon_username:
+            remove_data_by_user(request.user)
             request.user.first_name = request.user.username
             request.user.last_name = request.user.mastodon_site
             request.user.is_active = False
-            request.user.username = 'removed_' + str(request.user.id)
+            request.user.username = "removed_" + str(request.user.id)
             request.user.mastodon_id = 0
-            request.user.mastodon_site = 'removed'
-            request.user.mastodon_token = ''
+            request.user.mastodon_site = "removed"
+            request.user.mastodon_token = ""
             request.user.mastodon_locked = False
             request.user.mastodon_followers = []
             request.user.mastodon_following = []
@@ -251,5 +254,5 @@ def clear_data(request):
             auth_logout(request)
             return redirect(reverse("users:login"))
         else:
-            messages.add_message(request, messages.ERROR, _('验证信息不符。'))
+            messages.add_message(request, messages.ERROR, _("验证信息不符。"))
     return redirect(reverse("users:data"))

users/data.py

@@ -12,17 +12,8 @@ from .forms import ReportForm
 from mastodon.api import *
 from mastodon import mastodon_request_included
 from common.config import *
-from common.models import MarkStatusEnum
 from common.utils import PageLinksGenerator
 from management.models import Announcement
-from books.models import *
-from movies.models import *
-from music.models import *
-from games.models import *
-from books.forms import BookMarkStatusTranslator
-from movies.forms import MovieMarkStatusTranslator
-from music.forms import MusicMarkStatusTranslator
-from games.forms import GameMarkStatusTranslator
 from mastodon.models import MastodonApplication
 from mastodon.api import verify_account
 from django.conf import settings
@@ -34,20 +25,10 @@ from datetime import timedelta
 from django.utils import timezone
 import json
 from django.contrib import messages
-from books.models import BookMark, BookReview
-from movies.models import MovieMark, MovieReview
-from games.models import GameMark, GameReview
-from music.models import AlbumMark, SongMark, AlbumReview, SongReview
-from timeline.models import Activity
-from collection.models import Collection
 
-if settings.ENABLE_NEW_MODEL:
-    from journal.importers.douban import DoubanImporter
-    from journal.importers.goodreads import GoodreadsImporter
-    from journal.models import reset_visibility_for_user
-else:
-    from common.importers.douban import DoubanImporter
-    from common.importers.goodreads import GoodreadsImporter
+from journal.importers.douban import DoubanImporter
+from journal.importers.goodreads import GoodreadsImporter
+from journal.models import reset_visibility_for_user
 
 
 @mastodon_request_included
@@ -84,7 +65,6 @@ def data(request):
         "users/data.html",
         {
             "allow_any_site": settings.MASTODON_ALLOW_ANY_SITE,
-            "latest_task": request.user.user_synctasks.order_by("-id").first(),
             "import_status": request.user.get_preference().import_status,
             "export_status": request.user.get_preference().export_status,
         },

users/models.py

@@ -8,27 +8,31 @@ from django.utils.translation import gettext_lazy as _
 from common.utils import GenerateDateUUIDMediaFilePath
 from django.conf import settings
 from mastodon.api import *
-from django.shortcuts import reverse
+from django.urls import reverse
 
 
 def report_image_path(instance, filename):
-    return GenerateDateUUIDMediaFilePath(instance, filename, settings.REPORT_MEDIA_PATH_ROOT)
+    return GenerateDateUUIDMediaFilePath(
+        instance, filename, settings.REPORT_MEDIA_PATH_ROOT
+    )
 
 
 class User(AbstractUser):
     if settings.MASTODON_ALLOW_ANY_SITE:
         username = models.CharField(
-            _('username'),
+            _("username"),
             max_length=150,
             unique=False,
-            help_text=_('Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.'),
+            help_text=_(
+                "Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only."
+            ),
         )
     following = models.JSONField(default=list)
     mastodon_id = models.CharField(max_length=100, blank=False)
     # mastodon domain name, eg donotban.com
     mastodon_site = models.CharField(max_length=100, blank=False)
-    mastodon_token = models.CharField(max_length=2048, default='')
-    mastodon_refresh_token = models.CharField(max_length=2048, default='')
+    mastodon_token = models.CharField(max_length=2048, default="")
+    mastodon_refresh_token = models.CharField(max_length=2048, default="")
     mastodon_locked = models.BooleanField(default=False)
     mastodon_followers = models.JSONField(default=list)
     mastodon_following = models.JSONField(default=list)
@@ -44,7 +48,8 @@ class User(AbstractUser):
     class Meta:
         constraints = [
             models.UniqueConstraint(
-                fields=['username', 'mastodon_site'], name="unique_user_identity")
+                fields=["username", "mastodon_site"], name="unique_user_identity"
+            )
         ]
 
     # def save(self, *args, **kwargs):
@@ -54,15 +59,21 @@ class User(AbstractUser):
 
     @property
     def mastodon_username(self):
-        return self.username + '@' + self.mastodon_site
+        return self.username + "@" + self.mastodon_site
 
     @property
     def display_name(self):
-        return self.mastodon_account['display_name'] if self.mastodon_account and 'display_name' in self.mastodon_account and self.mastodon_account['display_name'] else self.mastodon_username
+        return (
+            self.mastodon_account["display_name"]
+            if self.mastodon_account
+            and "display_name" in self.mastodon_account
+            and self.mastodon_account["display_name"]
+            else self.mastodon_username
+        )
 
     @property
     def url(self):
-        return reverse("users:home", args=[self.mastodon_username])
+        return reverse("journal:user_profile", args=[self.mastodon_username])
 
     def __str__(self):
         return self.mastodon_username
@@ -74,59 +85,92 @@ class User(AbstractUser):
         return pref
 
     def refresh_mastodon_data(self):
-        """ Try refresh account data from mastodon server, return true if refreshed successfully, note it will not save to db """
+        """Try refresh account data from mastodon server, return true if refreshed successfully, note it will not save to db"""
         self.mastodon_last_refresh = timezone.now()
         code, mastodon_account = verify_account(self.mastodon_site, self.mastodon_token)
         if code == 401 and self.mastodon_refresh_token:
-            self.mastodon_token = refresh_access_token(self.mastodon_site, self.mastodon_refresh_token)
+            self.mastodon_token = refresh_access_token(
+                self.mastodon_site, self.mastodon_refresh_token
+            )
             if self.mastodon_token:
-                code, mastodon_account = verify_account(self.mastodon_site, self.mastodon_token)
+                code, mastodon_account = verify_account(
+                    self.mastodon_site, self.mastodon_token
+                )
         updated = False
         if mastodon_account:
             self.mastodon_account = mastodon_account
-            self.mastodon_locked = mastodon_account['locked']
-            if self.username != mastodon_account['username']:
+            self.mastodon_locked = mastodon_account["locked"]
+            if self.username != mastodon_account["username"]:
                 print(f"username changed from {self} to {mastodon_account['username']}")
-                self.username = mastodon_account['username']
+                self.username = mastodon_account["username"]
             # self.mastodon_token = token
             # user.mastodon_id  = mastodon_account['id']
-            self.mastodon_followers = get_related_acct_list(self.mastodon_site, self.mastodon_token, f'/api/v1/accounts/{self.mastodon_id}/followers')
-            self.mastodon_following = get_related_acct_list(self.mastodon_site, self.mastodon_token, f'/api/v1/accounts/{self.mastodon_id}/following')
-            self.mastodon_mutes = get_related_acct_list(self.mastodon_site, self.mastodon_token, '/api/v1/mutes')
-            self.mastodon_blocks = get_related_acct_list(self.mastodon_site, self.mastodon_token, '/api/v1/blocks')
-            self.mastodon_domain_blocks = get_related_acct_list(self.mastodon_site, self.mastodon_token, '/api/v1/domain_blocks')
+            self.mastodon_followers = get_related_acct_list(
+                self.mastodon_site,
+                self.mastodon_token,
+                f"/api/v1/accounts/{self.mastodon_id}/followers",
+            )
+            self.mastodon_following = get_related_acct_list(
+                self.mastodon_site,
+                self.mastodon_token,
+                f"/api/v1/accounts/{self.mastodon_id}/following",
+            )
+            self.mastodon_mutes = get_related_acct_list(
+                self.mastodon_site, self.mastodon_token, "/api/v1/mutes"
+            )
+            self.mastodon_blocks = get_related_acct_list(
+                self.mastodon_site, self.mastodon_token, "/api/v1/blocks"
+            )
+            self.mastodon_domain_blocks = get_related_acct_list(
+                self.mastodon_site, self.mastodon_token, "/api/v1/domain_blocks"
+            )
             self.following = self.get_following_ids()
             updated = True
         elif code == 401:
-            print(f'401 {self}')
-            self.mastodon_token = ''
+            print(f"401 {self}")
+            self.mastodon_token = ""
         return updated
 
     def get_following_ids(self):
         fl = []
         for m in self.mastodon_following:
             target = User.get(m)
-            if target and ((not target.mastodon_locked) or self.mastodon_username in target.mastodon_followers):
-                fl.append(target.id)
+            if target and (
+                (not target.mastodon_locked)
+                or self.mastodon_username in target.mastodon_followers
+            ):
+                fl.append(target.pk)
         return fl
 
     def is_blocking(self, target):
-        return target.mastodon_username in self.mastodon_blocks or target.mastodon_site in self.mastodon_domain_blocks
+        return (
+            (
+                target.mastodon_username in self.mastodon_blocks
+                or target.mastodon_site in self.mastodon_domain_blocks
+            )
+            if target.is_authenticated
+            else self.preference.no_anonymous_view
+        )
 
     def is_blocked_by(self, target):
-        return target.is_blocking(self)
+        return target.is_authenticated and target.is_blocking(self)
 
     def is_muting(self, target):
         return target.mastodon_username in self.mastodon_mutes
 
     def is_following(self, target):
-        return self.mastodon_username in target.mastodon_followers if target.mastodon_locked else self.mastodon_username in target.mastodon_followers or target.mastodon_username in self.mastodon_following
+        return (
+            self.mastodon_username in target.mastodon_followers
+            if target.mastodon_locked
+            else self.mastodon_username in target.mastodon_followers
+            or target.mastodon_username in self.mastodon_following
+        )
 
     def is_followed_by(self, target):
         return target.is_following(self)
 
     def get_mark_for_item(self, item):
-        params = {item.__class__.__name__.lower() + '_id': item.id, 'owner': self}
+        params = {item.__class__.__name__.lower() + "_id": item.id, "owner": self}
         mark = item.mark_class.objects.filter(**params).first()
         return mark
 
@@ -143,16 +187,16 @@ class User(AbstractUser):
             return 0
 
     @classmethod
-    def get(self, id):
+    def get(cls, id):
         if isinstance(id, str):
             try:
-                username = id.split('@')[0]
-                site = id.split('@')[1]
-            except IndexError as e:
+                username = id.split("@")[0]
+                site = id.split("@")[1]
+            except IndexError:
                 return None
-            query_kwargs = {'username': username, 'mastodon_site': site}
+            query_kwargs = {"username": username, "mastodon_site": site}
         elif isinstance(id, int):
-            query_kwargs = {'pk': id}
+            query_kwargs = {"pk": id}
         else:
             return None
         return User.objects.filter(**query_kwargs).first()
@@ -160,30 +204,44 @@ class User(AbstractUser):
 
 class Preference(models.Model):
     user = models.OneToOneField(User, models.CASCADE, primary_key=True)
-    home_layout = postgres.ArrayField(
-        postgres.HStoreField(),
+    profile_layout = models.JSONField(
         blank=True,
         default=list,
     )
-    export_status = models.JSONField(blank=True, null=True, encoder=DjangoJSONEncoder, default=dict)
-    import_status = models.JSONField(blank=True, null=True, encoder=DjangoJSONEncoder, default=dict)
+    export_status = models.JSONField(
+        blank=True, null=True, encoder=DjangoJSONEncoder, default=dict
+    )
+    import_status = models.JSONField(
+        blank=True, null=True, encoder=DjangoJSONEncoder, default=dict
+    )
     default_visibility = models.PositiveSmallIntegerField(default=0)
     classic_homepage = models.BooleanField(null=False, default=False)
     mastodon_publish_public = models.BooleanField(null=False, default=False)
-    mastodon_append_tag = models.CharField(max_length=2048, default='')
+    mastodon_append_tag = models.CharField(max_length=2048, default="")
     show_last_edit = models.PositiveSmallIntegerField(default=0)
+    no_anonymous_view = models.PositiveSmallIntegerField(default=0)
 
-    def get_serialized_home_layout(self):
-        return str(self.home_layout).replace("\'", "\"")
+    def get_serialized_profile_layout(self):
+        return str(self.profile_layout).replace("'", '"')
 
     def __str__(self):
         return str(self.user)
 
 
 class Report(models.Model):
-    submit_user = models.ForeignKey(User, on_delete=models.SET_NULL, related_name='sumbitted_reports', null=True)
-    reported_user = models.ForeignKey(User, on_delete=models.SET_NULL, related_name='accused_reports', null=True)
-    image = models.ImageField(upload_to=report_image_path, height_field=None, width_field=None, blank=True, default='')
+    submit_user = models.ForeignKey(
+        User, on_delete=models.SET_NULL, related_name="sumbitted_reports", null=True
+    )
+    reported_user = models.ForeignKey(
+        User, on_delete=models.SET_NULL, related_name="accused_reports", null=True
+    )
+    image = models.ImageField(
+        upload_to=report_image_path,
+        height_field=None,
+        width_field=None,
+        blank=True,
+        default="",
+    )
     is_read = models.BooleanField(default=False)
     submitted_time = models.DateTimeField(auto_now_add=True)
     message = models.CharField(max_length=1000)

users/tasks.py

@@ -12,17 +12,8 @@ from .forms import ReportForm
 from mastodon.api import *
 from mastodon import mastodon_request_included
 from common.config import *
-from common.models import MarkStatusEnum
 from common.utils import PageLinksGenerator
 from management.models import Announcement
-from books.models import *
-from movies.models import *
-from music.models import *
-from games.models import *
-from books.forms import BookMarkStatusTranslator
-from movies.forms import MovieMarkStatusTranslator
-from music.forms import MusicMarkStatusTranslator
-from games.forms import GameMarkStatusTranslator
 from mastodon.models import MastodonApplication
 from django.conf import settings
 from urllib.parse import quote
@@ -40,216 +31,3 @@ def refresh_mastodon_data_task(user, token=None):
         print(f"{user} mastodon data refreshed")
     else:
         print(f"{user} mastodon data refresh failed")
-
-
-def export_marks_task(user):
-    user.preference.export_status["marks_pending"] = True
-    user.preference.save(update_fields=["export_status"])
-    filename = GenerateDateUUIDMediaFilePath(
-        None, "f.xlsx", settings.MEDIA_ROOT + settings.EXPORT_FILE_PATH_ROOT
-    )
-    if not os.path.exists(os.path.dirname(filename)):
-        os.makedirs(os.path.dirname(filename))
-    heading = ["标题", "简介", "豆瓣评分", "链接", "创建时间", "我的评分", "标签", "评论", "NeoDB链接", "其它ID"]
-    wb = (
-        Workbook()
-    )  # adding write_only=True will speed up but corrupt the xlsx and won't be importable
-    for status, label in [("collect", "看过"), ("do", "在看"), ("wish", "想看")]:
-        ws = wb.create_sheet(title=label)
-        marks = MovieMark.objects.filter(owner=user, status=status).order_by(
-            "-edited_time"
-        )
-        ws.append(heading)
-        for mark in marks:
-            movie = mark.movie
-            title = movie.title
-            summary = (
-                str(movie.year)
-                + " / "
-                + ",".join(movie.area)
-                + " / "
-                + ",".join(map(lambda x: str(MovieGenreTranslator[x]), movie.genre))
-                + " / "
-                + ",".join(movie.director)
-                + " / "
-                + ",".join(movie.actor)
-            )
-            tags = ",".join(list(map(lambda m: m.content, mark.tags)))
-            world_rating = (movie.rating / 2) if movie.rating else None
-            timestamp = mark.edited_time.strftime("%Y-%m-%d %H:%M:%S")
-            my_rating = (mark.rating / 2) if mark.rating else None
-            text = mark.text
-            source_url = movie.source_url
-            url = settings.APP_WEBSITE + movie.get_absolute_url()
-            line = [
-                title,
-                summary,
-                world_rating,
-                source_url,
-                timestamp,
-                my_rating,
-                tags,
-                text,
-                url,
-                movie.imdb_code,
-            ]
-            ws.append(line)
-
-    for status, label in [("collect", "听过"), ("do", "在听"), ("wish", "想听")]:
-        ws = wb.create_sheet(title=label)
-        marks = AlbumMark.objects.filter(owner=user, status=status).order_by(
-            "-edited_time"
-        )
-        ws.append(heading)
-        for mark in marks:
-            album = mark.album
-            title = album.title
-            summary = (
-                ",".join(album.artist)
-                + " / "
-                + (album.release_date.strftime("%Y") if album.release_date else "")
-            )
-            tags = ",".join(list(map(lambda m: m.content, mark.tags)))
-            world_rating = (album.rating / 2) if album.rating else None
-            timestamp = mark.edited_time.strftime("%Y-%m-%d %H:%M:%S")
-            my_rating = (mark.rating / 2) if mark.rating else None
-            text = mark.text
-            source_url = album.source_url
-            url = settings.APP_WEBSITE + album.get_absolute_url()
-            line = [
-                title,
-                summary,
-                world_rating,
-                source_url,
-                timestamp,
-                my_rating,
-                tags,
-                text,
-                url,
-                "",
-            ]
-            ws.append(line)
-
-    for status, label in [("collect", "读过"), ("do", "在读"), ("wish", "想读")]:
-        ws = wb.create_sheet(title=label)
-        marks = BookMark.objects.filter(owner=user, status=status).order_by(
-            "-edited_time"
-        )
-        ws.append(heading)
-        for mark in marks:
-            book = mark.book
-            title = book.title
-            summary = (
-                ",".join(book.author)
-                + " / "
-                + str(book.pub_year)
-                + " / "
-                + book.pub_house
-            )
-            tags = ",".join(list(map(lambda m: m.content, mark.tags)))
-            world_rating = (book.rating / 2) if book.rating else None
-            timestamp = mark.edited_time.strftime("%Y-%m-%d %H:%M:%S")
-            my_rating = (mark.rating / 2) if mark.rating else None
-            text = mark.text
-            source_url = book.source_url
-            url = settings.APP_WEBSITE + book.get_absolute_url()
-            line = [
-                title,
-                summary,
-                world_rating,
-                source_url,
-                timestamp,
-                my_rating,
-                tags,
-                text,
-                url,
-                book.isbn,
-            ]
-            ws.append(line)
-
-    for status, label in [("collect", "玩过"), ("do", "在玩"), ("wish", "想玩")]:
-        ws = wb.create_sheet(title=label)
-        marks = GameMark.objects.filter(owner=user, status=status).order_by(
-            "-edited_time"
-        )
-        ws.append(heading)
-        for mark in marks:
-            game = mark.game
-            title = game.title
-            summary = (
-                ",".join(game.genre)
-                + " / "
-                + ",".join(game.platform)
-                + " / "
-                + (game.release_date.strftime("%Y-%m-%d") if game.release_date else "")
-            )
-            tags = ",".join(list(map(lambda m: m.content, mark.tags)))
-            world_rating = (game.rating / 2) if game.rating else None
-            timestamp = mark.edited_time.strftime("%Y-%m-%d %H:%M:%S")
-            my_rating = (mark.rating / 2) if mark.rating else None
-            text = mark.text
-            source_url = game.source_url
-            url = settings.APP_WEBSITE + game.get_absolute_url()
-            line = [
-                title,
-                summary,
-                world_rating,
-                source_url,
-                timestamp,
-                my_rating,
-                tags,
-                text,
-                url,
-                "",
-            ]
-            ws.append(line)
-
-    review_heading = [
-        "标题",
-        "评论对象",
-        "链接",
-        "创建时间",
-        "我的评分",
-        "类型",
-        "内容",
-        "评论对象原始链接",
-        "评论对象NeoDB链接",
-    ]
-    for ReviewModel, label in [
-        (MovieReview, "影评"),
-        (BookReview, "书评"),
-        (AlbumReview, "乐评"),
-        (GameReview, "游戏评论"),
-    ]:
-        ws = wb.create_sheet(title=label)
-        reviews = ReviewModel.objects.filter(owner=user).order_by("-edited_time")
-        ws.append(review_heading)
-        for review in reviews:
-            title = review.title
-            target = "《" + review.item.title + "》"
-            url = review.url
-            timestamp = review.edited_time.strftime("%Y-%m-%d %H:%M:%S")
-            my_rating = None  # (mark.rating / 2) if mark.rating else None
-            content = review.content
-            target_source_url = review.item.source_url
-            target_url = review.item.absolute_url
-            line = [
-                title,
-                target,
-                url,
-                timestamp,
-                my_rating,
-                label,
-                content,
-                target_source_url,
-                target_url,
-            ]
-            ws.append(line)
-
-    wb.save(filename=filename)
-    user.preference.export_status["marks_pending"] = False
-    user.preference.export_status["marks_file"] = filename
-    user.preference.export_status["marks_date"] = datetime.now().strftime(
-        "%Y-%m-%d %H:%M"
-    )
-    user.preference.save(update_fields=["export_status"])

users/templates/users/data.html

@@ -25,131 +25,25 @@
             <section id="content">
                 <div class="grid grid--reverse-order">
                     <div class="grid__main grid__main--reverse-order">
+                        {% if messages %}
                         <div class="main-section-wrapper">
-                            {% if messages %}
                             <ul class="messages">
                                 {% for message in messages %}
                                 <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message }}</li>
                                 {% endfor %}
                             </ul>
-                            {% endif %}
                         </div>
+                        {% endif %}
 
                         <div class="main-section-wrapper">
                             <div class="tools-section-wrapper">
                                 <div class="import-panel">
-                                    <h5 class="import-panel__label">{% trans '导入豆瓣标记和短评' %}</h5>
-                                    <div class="import-panel__body">
-                                        <form action="{% url 'sync:douban' %}" method="POST" enctype="multipart/form-data" >
-                                            {% csrf_token %}
-                                            <input type="hidden" name="user" value="{{ request.user.id }}">
-                                            <span>{% trans '导入：' %}</span>
-                                            <div class="import-panel__checkbox">
-                                                <input type="checkbox" name="sync_book" id="syncBook" checked>
-                                                <label for="syncBook">{% trans '书' %}</label>
-                                            </div>
-                                            <div class="import-panel__checkbox">
-                                                <input type="checkbox" name="sync_movie" id="syncMovie" checked>
-                                                <label for="syncMovie">{% trans '电影' %}</label>
-                                            </div>
-                                            <div class="import-panel__checkbox">
-                                                <input type="checkbox" name="sync_music" id="syncMusic" checked>
-                                                <label for="syncMusic">{% trans '音乐' %}</label>
-                                            </div>
-                                            <div class="import-panel__checkbox">
-                                                <input type="checkbox" name="sync_game" id="syncGame" checked>
-                                                <label for="syncGame">{% trans '游戏' %}</label>
-                                            </div>
-                                            <div></div>
-                                            <span>{% trans '覆盖：' %}</span>
-                                            <div class="import-panel__checkbox import-panel__checkbox--last">
-                                                <input type="checkbox" name="overwrite" id="overwrite">
-                                                <label for="overwrite">{% trans '选中会覆盖现有标记' %}</label>
-                                            </div>
-                                            <div></div>
-                                            <span>{% trans '可见性：' %}</span>
-                                            <div class="import-panel__checkbox import-panel__checkbox--last">
-                                                <input type="checkbox" name="default_public" id="visibility" checked>
-                                                <label for="visibility">{% trans '选中后导入标记对其他用户可见；标记可见性在导入后也可更改。' %}</label>
-                                            </div>
-                                            <div></div>
-                                            <div class="import-panel__file-input">
-                                                从<a href="https://doufen.org" target="_blank">豆伴(豆坟)</a>备份导出的.xlsx文件，<strong>请勿手动修改该文件</strong>:
-                                                <input type="file" name="file" id="excelFile" required accept=".xlsx"> 
-                                            </div>
-                                            <input type="submit" class="import-panel__button" value="{% trans '导入' %}" id="uploadBtn"
-                                                {% if not latest_task is None and not latest_task.is_finished %}
-                                                disabled
-                                                {% endif %}
-                                            >
-                                        </form>
-                                        <div class="import-panel__progress"
-                                            {% if latest_task.is_finished or latest_task is None %}
-                                            style="display: none;"
-                                            {% endif %}
-                                        >
-                                            <label for="importProgress">{% trans '进度' %}</label>
-                                            <progress id="importProgress" value="{{ latest_task.finished_items }}" max="{{ latest_task.total_items }}"></progress>
-                                            <span class="float-right" id="progressPercent">{{ latest_task.get_progress | floatformat:"0" }}%</span>
-                                            <span class="clearfix"></span>
-                                        </div>
-                                        <div class="import-panel__last-task"
-                                            {% if not latest_task.is_finished %}`
-                                            style="display: none;"
-                                            {% endif %}
-                                        >
-                                            {% trans '上次导入：' %}
-                                            <span class="index">{% trans '总数' %} <span id="lastTaskTotalItems">{{ latest_task.total_items }}</span></span>
-                                            <span class="index">{% trans '同步' %} <span id="lastTaskSuccessItems">{{ latest_task.success_items }}</span></span>
-                                            <span class="index">{% trans '状态' %} <span id="lastTaskStatus">{{ latest_task.get_status_emoji }}</span></span>
-                                            <div class="import-panel__fail-urls"
-                                            {% if not latest_task.failed_urls %}
-                                                style="display: none;"
-                                            {% endif %}
-                                            >
-                                                <span>
-                                                    {% trans '失败条目链接' %}
-                                                </span>
-                                                <a class="float-right" style="cursor: pointer;" id="failedUrlsBtn">
-                                                    ▶
-                                                </a>
-                                                <script>
-                                                    $("#failedUrlsBtn").data("collapse", true);
-                                                    $("#failedUrlsBtn").on('click', ()=>{
-                                                        const btn = $("#failedUrlsBtn");
-                                                        if(btn.data("collapse") == true) {
-                                                            btn.data("collapse", false);
-                                                            btn.text("▼");
-                                                            $("#failedUrls").show();
-                                                        } else {
-                                                            btn.data("collapse", true);
-                                                            btn.text("▶");
-                                                            $("#failedUrls").hide();
-                                                        }
-                                                    });
-                                                </script>
-                                                <span class="clearfix"></span>
-                                                <ul id="failedUrls" style="display: none;">
-                                                    {% for url in latest_task.failed_urls %}
-                                                    <li>{{ url }}</li>
-                                                    {% endfor %}
-                                                </ul>
-                                            </div>
-                                        </div>
-                                    </div>
-                                </div>
-                            </div>
-                        </div>
-
-                        <div class="main-section-wrapper">
-                            <div class="tools-section-wrapper">
-                                <div class="import-panel">
-                                    <h5 class="import-panel__label">{% trans '导入豆瓣评论' %}</h5>
+                                    <h5 class="import-panel__label">{% trans '导入豆瓣标记和评论' %}</h5>
                                     <div class="import-panel__body">
                                         <form action="{% url 'users:import_douban' %}" method="POST" enctype="multipart/form-data" >
                                             {% csrf_token %}
                                             <div>
-                                                请在豆伴(豆坟)导出时勾选「书影音游剧」和「评论」；已经存在的评论不会被覆盖。
+                                                请在豆伴(豆坟)导出时勾选「书影音游剧」和「评论」。正向变化（想读->在读->已读）的标记会更新，其它已经存在的标记和评论不会被覆盖。
                                             </div>
                                             <div class="import-panel__checkbox">
                                                 <p>从<a href="https://doufen.org" target="_blank">豆伴(豆坟)</a>备份导出的.xlsx文件:
@@ -323,8 +217,6 @@
         {% include "partial/_footer.html" %}
     </div>
 
-    <div id="queryProgressURL" data-url="{% url 'sync:progress' %}"></div>
-    <div id="querySyncInfoURL" data-url="{% url 'sync:last' %}"></div>
 </body>
 
 

users/templates/users/home.html

@@ -1,674 +0,0 @@
-{% load static %}
-{% load i18n %}
-{% load admin_url %}
-{% load mastodon %}
-{% load oauth_token %}
-{% load truncate %}
-{% load thumb %}
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    {% if user == request.user %}
-        <title>{{ site_name }} - {% trans '我的个人主页' %}</title>
-    {% else %}
-        <title>{{ site_name }} - {{user.display_name}}</title>
-    {% endif %}
-    <link rel="alternate" type="application/rss+xml" title="{{ site_name }} - {{ user.mastodon_username }}的评论" href="{{ request.build_absolute_uri }}feed/reviews/">
-
-    {% include "partial/_common_libs.html" with jquery=1 %}
-
-    <script src="{% static 'js/mastodon.js' %}" defer></script>
-    <script src="{% static 'js/home.js' %}" defer></script>
-</head>
-
-<body>
-    <div id="page-wrapper">
-        <div id="content-wrapper">
-            {% include "partial/_navbar.html" with current="home" %}
-
-            <section id="content">
-                <div class="grid grid--reverse-order">
-                    <div class="grid__main grid__main--reverse-order">
-
-                        <div class="main-section-wrapper sortable">
-
-                            <div class="entity-sort" id="bookWish">
-                                <h5 class="entity-sort__label">
-                                    {% trans '想读的书' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ wish_book_count }}
-                                </span>
-                                {% if wish_book_more %}
-                                <a href="{% url 'users:book_list' user.mastodon_username 'wish' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-                                <ul class="entity-sort__entity-list">
-                                    {% for wish_book_mark in wish_book_marks %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'books:retrieve' wish_book_mark.book.id %}">
-                                            <img src="{{ wish_book_mark.book.cover|thumb:'normal' }}"
-                                                alt="{{wish_book_mark.book.title}}" class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{ wish_book_mark.book.title }}">
-                                                {{ wish_book_mark.book.title }}</div>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="bookDo">
-                                <h5 class="entity-sort__label">
-                                    {% trans '在读的书' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ do_book_count }}
-                                </span>
-                                {% if do_book_more %}
-                                <a href="{% url 'users:book_list' user.mastodon_username 'do' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for do_book_mark in do_book_marks %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'books:retrieve' do_book_mark.book.id %}">
-                                            <img src="{{ do_book_mark.book.cover|thumb:'normal' }}"
-                                                alt="{{do_book_mark.book.title}}" class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{do_book_mark.book.title}}">
-                                                {{ do_book_mark.book.title }}</div>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="bookCollect">
-                                <h5 class="entity-sort__label">
-                                    {% trans '读过的书' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ collect_book_count }}
-                                </span>
-                                {% if collect_book_more %}
-                                <a href="{% url 'users:book_list' user.mastodon_username 'collect' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for collect_book_mark in collect_book_marks %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'books:retrieve' collect_book_mark.book.id %}">
-                                            <img src="{{ collect_book_mark.book.cover|thumb:'normal' }}"
-                                                alt="{{collect_book_mark.book.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{collect_book_mark.book.title}}">{{ collect_book_mark.book.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="bookReviewed">
-                                <h5 class="entity-sort__label">
-                                    {% trans '评论过的书籍' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ book_reviews_count }}
-                                </span>
-                                {% if book_reviews_more %}
-                                <a href="{% url 'users:book_list' user.mastodon_username 'reviewed' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for book_review in book_reviews %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'books:retrieve' book_review.book.id %}">
-                                            <img src="{{ book_review.book.cover|thumb:'normal' }}"
-                                                alt="{{book_review.book.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{book_review.book.title}}">{{ book_review.book.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="movieWish">
-                                <h5 class="entity-sort__label">
-                                    {% trans '想看的电影/剧集' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ wish_movie_count }}
-                                </span>
-                                {% if wish_movie_more %}
-                                <a href="{% url 'users:movie_list' user.mastodon_username 'wish' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for wish_movie_mark in wish_movie_marks %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'movies:retrieve' wish_movie_mark.movie.id %}">
-                                            <img src="{{ wish_movie_mark.movie.cover|thumb:'normal' }}"
-                                                alt="{{wish_movie_mark.movie.title}}" class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{wish_movie_mark.movie.title}}">
-                                                {{ wish_movie_mark.movie.title }}</div>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="movieDo">
-                                <h5 class="entity-sort__label">
-                                    {% trans '在看的电影/剧集' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ do_movie_count }}
-                                </span>
-                                {% if do_movie_more %}
-                                <a href="{% url 'users:movie_list' user.mastodon_username 'do' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for do_movie_mark in do_movie_marks %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'movies:retrieve' do_movie_mark.movie.id %}">
-                                            <img src="{{ do_movie_mark.movie.cover|thumb:'normal' }}"
-                                                alt="{{do_movie_mark.movie.title}}" class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{do_movie_mark.movie.title}}">
-                                                {{ do_movie_mark.movie.title }}</div>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="movieCollect">
-                                <h5 class="entity-sort__label">
-                                    {% trans '看过的电影/剧集' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ collect_movie_count }}
-                                </span>
-                                {% if collect_movie_more %}
-                                <a href="{% url 'users:movie_list' user.mastodon_username 'collect' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for collect_movie_mark in collect_movie_marks %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'movies:retrieve' collect_movie_mark.movie.id %}">
-                                            <img src="{{ collect_movie_mark.movie.cover|thumb:'normal' }}"
-                                                alt="{{collect_movie_mark.movie.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{collect_movie_mark.movie.title}}">{{ collect_movie_mark.movie.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="movieReviewed">
-                                <h5 class="entity-sort__label">
-                                    {% trans '评论过的电影/剧集' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ movie_reviews_count }}
-                                </span>
-                                {% if movie_reviews_more %}
-                                <a href="{% url 'users:movie_list' user.mastodon_username 'reviewed' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for movie_review in movie_reviews %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'movies:retrieve' movie_review.movie.id %}">
-                                            <img src="{{ movie_review.movie.cover|thumb:'normal' }}"
-                                                alt="{{movie_review.movie.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{movie_review.movie.title}}">{{ movie_review.movie.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="musicWish">
-                                <h5 class="entity-sort__label">
-                                    {% trans '想听的音乐' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ wish_music_count }}
-                                </span>
-                                {% if wish_music_more %}
-                                <a href="{% url 'users:music_list' user.mastodon_username 'wish' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for wish_music_mark in wish_music_marks %}
-                                    <li class="entity-sort__entity">
-                    
-                                        
-                                        {% if wish_music_mark.type == 'album' %}
-                                        <a href="{% url 'music:retrieve_album' wish_music_mark.album.id %}">
-                                            <img src="{{ wish_music_mark.album.cover|thumb:'normal' }}"
-                                                alt="{{wish_music_mark.album.title}}" class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{wish_music_mark.album.title}}">
-                                                {{ wish_music_mark.album.title }}</div>
-                                        </a>
-                                        {% else %}
-                                        <a href="{% url 'music:retrieve_song' wish_music_mark.song.id %}">
-                                            <img src="{{ wish_music_mark.song.cover|thumb:'normal' }}" alt="{{wish_music_mark.song.title}}"
-                                                class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{wish_music_mark.song.title}}">
-                                                {{ wish_music_mark.song.title }}</div>
-                                        </a>
-                                        {% endif %}
-                                            
-    
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="musicDo">
-                                <h5 class="entity-sort__label">
-                                    {% trans '在听的音乐' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ do_music_count }}
-                                </span>
-                                {% if do_music_more %}
-                                <a href="{% url 'users:music_list' user.mastodon_username 'do' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for do_music_mark in do_music_marks %}
-                                    <li class="entity-sort__entity">
-                                        {% if do_music_mark.type == 'album' %}
-                                        <a href="{% url 'music:retrieve_album' do_music_mark.album.id %}">
-                                            <img src="{{ do_music_mark.album.cover|thumb:'normal' }}"
-                                                alt="{{do_music_mark.album.title}}" class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{do_music_mark.album.title}}">
-                                                {{ do_music_mark.album.title }}</div>
-                                        </a>
-                                        {% else %}
-                                        <a href="{% url 'music:retrieve_song' do_music_mark.song.id %}">
-                                            <img src="{{ do_music_mark.song.cover|thumb:'normal' }}"
-                                                alt="{{do_music_mark.song.title}}" class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{do_music_mark.song.title}}">
-                                                {{ do_music_mark.song.title }}</div>
-                                        </a>
-                                        {% endif %}
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="musicCollect">
-                                <h5 class="entity-sort__label">
-                                    {% trans '听过的音乐' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ collect_music_count }}
-                                </span>
-                                {% if collect_music_more %}
-                                <a href="{% url 'users:music_list' user.mastodon_username 'collect' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for collect_music_mark in collect_music_marks %}
-                                    <li class="entity-sort__entity">
-                                        {% if collect_music_mark.type == 'album' %}
-                                        <a href="{% url 'music:retrieve_album' collect_music_mark.album.id %}">
-                                            <img src="{{ collect_music_mark.album.cover|thumb:'normal' }}"
-                                                alt="{{collect_music_mark.album.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{collect_music_mark.album.title}}">{{ collect_music_mark.album.title }}</span>
-                                        </a>
-                                        {% else %}
-                                        <a href="{% url 'music:retrieve_song' collect_music_mark.song.id %}">
-                                            <img src="{{ collect_music_mark.song.cover|thumb:'normal' }}"
-                                                alt="{{collect_music_mark.song.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{collect_music_mark.song.title}}">{{ collect_music_mark.song.title }}</span>
-                                        </a>
-                                        {% endif %}
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="musicReviewed">
-                                <h5 class="entity-sort__label">
-                                    {% trans '评论过的音乐' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ music_reviews_count }}
-                                </span>
-                                {% if music_reviews_more %}
-                                <a href="{% url 'users:music_list' user.mastodon_username 'reviewed' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for music_review in music_reviews %}
-                                    <li class="entity-sort__entity">
-                                        {% if music_review.type == 'album' %}
-                                        <a href="{% url 'music:retrieve_album' music_review.album.id %}">
-                                            <img src="{{ music_review.album.cover|thumb:'normal' }}"
-                                                alt="{{music_review.album.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{music_review.album.title}}">{{ music_review.album.title }}</span>
-                                        </a>
-                                        {% else %}
-                                        <a href="{% url 'music:retrieve_song' music_review.song.id %}">
-                                            <img src="{{ music_review.song.cover|thumb:'normal' }}"
-                                                alt="{{music_review.song.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{music_review.song.title}}">{{ music_review.song.title }}</span>
-                                        </a>
-                                        {% endif %}
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="gameWish">
-                                <h5 class="entity-sort__label">
-                                    {% trans '想玩的游戏' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ wish_game_count }}
-                                </span>
-                                {% if wish_game_more %}
-                                <a href="{% url 'users:game_list' user.mastodon_username 'wish' %}" class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-                            
-                                <ul class="entity-sort__entity-list">
-                                    {% for wish_game_mark in wish_game_marks %}
-                                    <li class="entity-sort__entity">
-                            
-                                        <a href="{% url 'games:retrieve' wish_game_mark.game.id %}">
-                                            <img src="{{ wish_game_mark.game.cover|thumb:'normal' }}" alt="{{wish_game_mark.game.title}}"
-                                                class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{wish_game_mark.game.title}}">
-                                                {{ wish_game_mark.game.title }}</div>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-                            
-                            <div class="entity-sort" id="gameDo">
-                                <h5 class="entity-sort__label">
-                                    {% trans '在玩的游戏' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ do_game_count }}
-                                </span>
-                                {% if do_game_more %}
-                                <a href="{% url 'users:game_list' user.mastodon_username 'do' %}" class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-                            
-                                <ul class="entity-sort__entity-list">
-                                    {% for do_game_mark in do_game_marks %}
-                                    <li class="entity-sort__entity">
-                            
-                                        <a href="{% url 'games:retrieve' do_game_mark.game.id %}">
-                                            <img src="{{ do_game_mark.game.cover|thumb:'normal' }}" alt="{{do_game_mark.game.title}}"
-                                                class="entity-sort__entity-img">
-                                            <div class="entity-sort__entity-name" title="{{do_game_mark.game.title}}">
-                                                {{ do_game_mark.game.title }}</div>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-                            
-                            <div class="entity-sort" id="gameCollect">
-                                <h5 class="entity-sort__label">
-                                    {% trans '玩过的游戏' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ collect_game_count }}
-                                </span>
-                                {% if collect_game_more %}
-                                <a href="{% url 'users:game_list' user.mastodon_username 'collect' %}" class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-                            
-                                <ul class="entity-sort__entity-list">
-                                    {% for collect_game_mark in collect_game_marks %}
-                                    <li class="entity-sort__entity">
-                            
-                                        <a href="{% url 'games:retrieve' collect_game_mark.game.id %}">
-                                            <img src="{{ collect_game_mark.game.cover|thumb:'normal' }}" alt="{{collect_game_mark.game.title}}"
-                                                class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name" title="{{collect_game_mark.game.title}}">{{collect_game_mark.game.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="gameReviewed">
-                                <h5 class="entity-sort__label">
-                                    {% trans '评论过的游戏' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ game_reviews_count }}
-                                </span>
-                                {% if game_reviews_more %}
-                                <a href="{% url 'users:game_list' user.mastodon_username 'reviewed' %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for game_review in game_reviews %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'games:retrieve' game_review.game.id %}">
-                                            <img src="{{ game_review.game.cover|thumb:'normal' }}"
-                                                alt="{{game_review.game.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{game_review.game.title}}">{{ game_review.game.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="collectionCreated">
-                                <h5 class="entity-sort__label">
-                                    {% trans '创建的收藏单' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ collections_count }}
-                                </span>
-                                {% if collections_more %}
-                                <a href="{% url 'users:collection_list' user.mastodon_username %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-                                {% if user == request.user %}
-                                <a href="{% url 'collection:create' %}"class="entity-sort__more-link">{% trans '新建' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for collection in collections %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'collection:retrieve' collection.id %}">
-                                            <img src="{{ collection.cover|thumb:'normal' }}"
-                                                alt="{{collection.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{collection.title}}">{{ collection.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                            <div class="entity-sort" id="collectionMarked">
-                                <h5 class="entity-sort__label">
-                                    {% trans '关注的收藏单' %}
-                                </h5>
-                                <span class="entity-sort__count">
-                                    {{ marked_collections_count }}
-                                </span>
-                                {% if marked_collections_more %}
-                                <a href="{% url 'users:marked_collection_list' user.mastodon_username %}"
-                                    class="entity-sort__more-link">{% trans '更多' %}</a>
-                                {% endif %}
-
-                                <ul class="entity-sort__entity-list">
-                                    {% for collection in marked_collections %}
-                                    <li class="entity-sort__entity">
-
-                                        <a href="{% url 'collection:retrieve' collection.id %}">
-                                            <img src="{{ collection.cover|thumb:'normal' }}"
-                                                alt="{{collection.title}}" class="entity-sort__entity-img">
-                                            <span class="entity-sort__entity-name"
-                                                title="{{collection.title}}">{{ collection.title }}</span>
-                                        </a>
-                                    </li>
-                                    {% empty %}
-                                    <div>暂无记录</div>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-
-                        </div>
-                            
-                        {% if user == request.user %}
-                            
-                        <div class="entity-sort-control">
-                            <div class="entity-sort-control__button" id="sortEditButton">
-                                <span class="entity-sort-control__text" id="sortEditText">
-                                    {% trans '编辑布局' %}
-                                </span>
-                                <span class="entity-sort-control__text" id="sortSaveText" style="display: none;">
-                                    {% trans '保存' %}
-                                </span>
-                                <span class="icon-edit" id="sortEditIcon">
-                                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 383.947 383.947">
-                                        <polygon points="0,303.947 0,383.947 80,383.947 316.053,147.893 236.053,67.893    " />
-                                        <path
-                                            d="M377.707,56.053L327.893,6.24c-8.32-8.32-21.867-8.32-30.187,0l-39.04,39.04l80,80l39.04-39.04     C386.027,77.92,386.027,64.373,377.707,56.053z" />
-                                    </svg>
-                                </span>
-                                <span class="icon-save" id="sortSaveIcon" style="display: none;">
-                                    <svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 384 384" >
-                                        <path
-                                            d="M298.667,0h-256C19.093,0,0,19.093,0,42.667v298.667C0,364.907,19.093,384,42.667,384h298.667    C364.907,384,384,364.907,384,341.333v-256L298.667,0z M192,341.333c-35.307,0-64-28.693-64-64c0-35.307,28.693-64,64-64    s64,28.693,64,64C256,312.64,227.307,341.333,192,341.333z M256,128H42.667V42.667H256V128z" />
-                                    </svg>
-                                </span>
-                            </div>
-                            <div class="entity-sort-control__button" id="sortExitButton" style="display: none;">
-                                <span class="entity-sort-control__text">
-                                    {% trans '取消' %}
-                                </span>
-                            </div>
-                        </div>
-                        <div class="entity-sort-control__button entity-sort-control__button--float-right" id="toggleDisplayButtonTemplate" style="display: none;">
-                            <span class="showText" style="display: none;">
-                                {% trans '显示' %}
-                            </span>
-                            <span class="hideText" style="display: none;">
-                                {% trans '隐藏' %}
-                            </span>
-                        </div>
-                        <form action="{% url 'users:set_layout' %}" method="post" id="sortForm">
-                            {% csrf_token %}
-                            <input type="hidden" name="layout">
-                        </form>
-                        <script src="https://cdn.staticfile.org/html5sortable/0.13.3/html5sortable.min.js" crossorigin="anonymous"></script>
-                        <script src="{% static 'js/sort_layout.js' %}"></script>
-                        {% endif %}
-                        <script>
-                            const initialLayoutData = JSON.parse("{{ layout|escapejs }}");
-                            // initialize sort element visibility and order
-                                initialLayoutData.forEach(elem => {
-                                    // False to false, True to true
-                                    if (elem.visibility === "False") {
-                                        elem.visibility = false;
-                                    } else {
-                                        elem.visibility = true;
-                                    }
-                                    // set visiblity
-                                    $('#' + elem.id).data('visibility', elem.visibility);
-                                    if (!elem.visibility) {
-                                        $('#' + elem.id).hide();
-                                    }
-                                    // order
-                                    $('#' + elem.id).appendTo('.main-section-wrapper');
-                                });
-                        </script>
-
-                    </div>
-
-                    {% include "partial/_sidebar.html" %}
-                </div>
-            </section>
-        </div>
-        {% include "partial/_footer.html" %}
-    </div>
-
-    {% if unread_announcements %}
-    {% include "partial/_announcement.html" %}
-    {% endif %}
-</body>
-</html>

users/templates/users/item_list.html

@@ -1,94 +0,0 @@
-{% load static %}
-{% load i18n %}
-{% load l10n %}
-{% load admin_url %}
-{% load mastodon %}
-{% load oauth_token %}
-{% load truncate %}
-{% load thumb %}
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{{ site_name }} - {{ user.mastodon_username }} {{ list_title }}</title>
-    <script src="https://cdn.staticfile.org/jquery/3.6.1/jquery.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/htmx/1.8.4/htmx.min.js"></script>
-    <script src="{% static 'lib/js/rating-star.js' %}"></script>
-    <script src="{% static 'js/rating-star-readonly.js' %}"></script>
-    <script src="{% static 'js/mastodon.js' %}"></script>
-    <script src="{% static 'js/home.js' %}"></script>
-    <link rel="stylesheet" href="{% static 'lib/css/rating-star.css' %}">
-    <link rel="stylesheet" href="{% static 'css/boofilsic.min.css' %}">
-    <link rel="stylesheet" href="{% static 'lib/css/collection.css' %}">
-</head>
-
-<body>
-    <div id="page-wrapper">
-        <div id="content-wrapper">
-            {% include "partial/_navbar.html" %}
-        
-            <section id="content" class="container">
-                <div class="grid grid--reverse-order">
-                    <div class="grid__main grid__main--reverse-order">
-                        <div class="main-section-wrapper">
-                            <div class="entity-list">
-
-                                <div class="set">
-                                    <h5 class="entity-list__title">
-                                        {{ user.mastodon_username }} {{ list_title }}
-                                    </h5>
-                                </div>
-                                <ul class="entity-list__entities">
-                                    {% for mark in marks %}
-                                    {% include "partial/list_item.html" with item=mark.item hide_category=True %}
-                                    {% empty %}
-                                    <div>{% trans '无结果' %}</div>
-                                    {% endfor %} 
-                                </ul>
-                            </div>
-                            <div class="pagination">
-                            
-                                {% if marks.pagination.has_prev %}
-                                <a href="?{% if request.GET.t %}t={{ request.GET.t }}&{% endif %}page=1" class="pagination__nav-link pagination__nav-link">&laquo;</a>
-                                <a href="?{% if request.GET.t %}t={{ request.GET.t }}&{% endif %}page={{ marks.previous_page_number }}"
-                                    class="pagination__nav-link pagination__nav-link--right-margin pagination__nav-link">&lsaquo;</a>
-                                {% endif %}
-                            
-                                {% for page in marks.pagination.page_range %}
-                            
-                                {% if page == marks.pagination.current_page %}
-                                <a href="?{% if request.GET.t %}t={{ request.GET.t }}&{% endif %}page={{ page }}" class="pagination__page-link pagination__page-link--current">{{ page }}</a>
-                                {% else %}
-                                <a href="?{% if request.GET.t %}t={{ request.GET.t }}&{% endif %}page={{ page }}" class="pagination__page-link">{{ page }}</a>
-                                {% endif %}
-                            
-                                {% endfor %}
-                            
-                                {% if marks.pagination.has_next %}
-                                <a href="?{% if request.GET.t %}t={{ request.GET.t }}&{% endif %}page={{ marks.next_page_number }}"
-                                    class="pagination__nav-link pagination__nav-link--left-margin">&rsaquo;</a>
-                                <a href="?{% if request.GET.t %}t={{ request.GET.t }}&{% endif %}page={{ marks.pagination.last_page }}" class="pagination__nav-link">&raquo;</a>
-                                {% endif %}
-                            
-                            </div>
-                        </div>
-                    </div>
-
-                    {% include "partial/_sidebar.html" %}
-                </div>
-            </section>
-        </div>
-        {% include "partial/_footer.html" %}
-    </div>
-
-    <script>
-      document.body.addEventListener('htmx:configRequest', (event) => {
-        event.detail.headers['X-CSRFToken'] = '{{ csrf_token }}';
-      })
-    </script>
-</body>
-
-
-</html>

users/templates/users/manage_report.html

@@ -27,9 +27,9 @@
                         
                         {% for report in reports %}
                             <div class="report">
-                                <a href="{% url 'users:home' report.submit_user.mastodon_username %}">{{ report.submit_user.username }}</a>
+                                <a href="{% url 'journal:user_profile' report.submit_user.mastodon_username %}">{{ report.submit_user.username }}</a>
                                 {% trans '举报了' %}
-                                <a href="{% url 'users:home' report.reported_user.mastodon_username %}">{{ report.reported_user.username }}</a>
+                                <a href="{% url 'journal:user_profile' report.reported_user.mastodon_username %}">{{ report.reported_user.username }}</a>
                                 @{{ report.submitted_time }}
                                 
                                 {% if report.image %}

users/templates/users/preferences.html

@@ -45,7 +45,13 @@
                                             <span>{% trans '登录后显示个人主页：' %}</span>
                                             <div class="import-panel__checkbox import-panel__checkbox--last">
                                                 <input type="checkbox" name="classic_homepage" id="classic_homepage" {%if request.user.preference.classic_homepage %}checked{% endif %} style="margin-bottom:1.5em">
-                                                <label for="classic_homepage">{% trans '默认登录后显示好友动态，如果希望登录后显示原版风格个人主页可选中此处' %}</label>
+                                                <label for="classic_homepage">{% trans '默认登录后显示好友动态，如果希望登录后显示本人主页可选中此处' %}</label>
+                                            </div>
+                                            <br>
+                                            <span>{% trans '不允许未登录用户访问个人主页和RSS：' %}</span>
+                                            <div class="import-panel__checkbox import-panel__checkbox--last">
+                                                <input type="checkbox" name="no_anonymous_view" id="no_anonymous_view" {%if request.user.preference.no_anonymous_view %}checked{% endif %}>
+                                                <label for="no_anonymous_view">{% trans '选中此选项后，未登录访客不能看到你的个人主页，或通过RSS订阅你的评论' %}</label>
                                             </div>
                                             <br>
                                             <span>{% trans '显示最近编辑者：' %}</span>

users/templates/users/relation_list.html

@@ -64,7 +64,7 @@
                                 <div class="user-profile__header">
                                     <!-- <img src="" class="user-profile__avatar mast-avatar" alt="{{ user.username }}"> -->
                                     <img src="" class="user-profile__avatar mast-avatar">
-                                    <a href="{% url 'users:home' user.mastodon_username %}">
+                                    <a href="{% url 'journal:user_profile' user.mastodon_username %}">
                                         <h5 class="user-profile__username mast-displayname"></h5>
                                     </a>
                                 </div>
@@ -142,7 +142,7 @@
     {% else %}
     <div id="userMastodonID" hidden="true"></div>
     {% endif %}
-    <div id="userPageURL" hidden="true">{% url 'users:home' 0 %}</div>
+    <div id="userPageURL" hidden="true">{% url 'journal:user_profile' 0 %}</div>
     <div id="spinner" hidden>
         <div class="spinner">
             <div></div>

users/templates/users/tags.html

@@ -1,110 +0,0 @@
-{% load static %}
-{% load i18n %}
-{% load l10n %}
-{% load humanize %}
-{% load admin_url %}
-{% load mastodon %}
-{% load oauth_token %}
-{% load truncate %}
-{% load highlight %}
-{% load thumb %}
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{{ site_name }} - 我的标签</title>
-    <script src="https://cdn.staticfile.org/jquery/3.6.1/jquery.min.js"></script>
-    <script src="{% static 'lib/js/rating-star.js' %}"></script>
-    <script src="{% static 'js/rating-star-readonly.js' %}"></script>
-    <link rel="stylesheet" href="{% static 'lib/css/rating-star.css' %}">
-    <link rel="stylesheet" href="{% static 'css/boofilsic.min.css' %}">
-    <script src="{% static 'js/mastodon.js' %}"></script>
-    <script src="{% static 'js/home.js' %}"></script>
-</head>
-
-<body>
-    <div id="page-wrapper">
-        <div id="content-wrapper">
-            {% include "partial/_navbar.html" %}
-
-            <section id="content">
-                <div class="grid">
-                    <div class="grid__main" id="main">
-                        <div class="main-section-wrapper">
-                            <div class="entity-reviews">                                
-                                <div class="tag-collection entity-reviews__review-list">
-                                    <h5>{% trans '书籍' %}</h5>
-                                    {% for v in book_tags %}
-				    <span style="display: inline-block;margin: 4px;">
-				    <span class="tag-collection__tag" style="display:inline;float:none;">
-                                        <a href="{% url 'users:book_list' user.mastodon_username 'tagged' %}?t={{ v.content }}">{{ v.content }}</a>
-                                    </span>
-				    <span class="entity-reviews__review-time">({{ v.total }})</span>
-				    </span>
-                                    {% empty %}
-                                        {% trans '暂无标签' %}
-                                    {% endfor %}
-                                    <div class="clearfix" style="margin-bottom: 16px;"></div>
-
-                                    <h5>{% trans '电影和剧集' %}</h5>
-                                    {% for v in movie_tags %}
-				    <span style="display: inline-block;margin: 4px;">
-				    <span class="tag-collection__tag" style="display:inline;float:none;">
-                                        <a href="{% url 'users:movie_list' user.mastodon_username 'tagged' %}?t={{ v.content }}">{{ v.content }}</a>
-                                    </span>
-                                    <span class="entity-reviews__review-time">({{ v.total }})</span>
-				    </span>
-                                    {% empty %}
-                                        {% trans '暂无标签' %}
-                                    {% endfor %}
-                                    <div class="clearfix" style="margin-bottom: 16px;"></div>
-
-                                    <h5>{% trans '音乐' %}</h5>
-                                    {% for v in music_tags %}
-				    <span style="display: inline-block;margin: 4px;">
-				    <span class="tag-collection__tag" style="display:inline;float:none;">
-                                        <a href="{% url 'users:music_list' user.mastodon_username 'tagged' %}?t={{ v.content }}">{{ v.content }}</a>
-                                    </span>
-                                    <span class="entity-reviews__review-time">({{ v.total }})</span>
-				    </span>
-                                    {% empty %}
-                                        {% trans '暂无标签' %}
-                                    {% endfor %}
-                                    <div class="clearfix" style="margin-bottom: 16px;"></div>
-
-                                    <h5>{% trans '游戏' %}</h5>
-                                    {% for v in game_tags %}
-				    <span style="display: inline-block;margin: 4px;">
-				    <span class="tag-collection__tag" style="display:inline;float:none;">
-                                        <a href="{% url 'users:game_list' user.mastodon_username 'tagged' %}?t={{ v.content }}">{{ v.content }}</a>
-                                    </span>
-                                    <span class="entity-reviews__review-time">({{ v.total }})</span>
-				    </span>
-                                    {% empty %}
-                                        {% trans '暂无标签' %}
-                                    {% endfor %}
-                                    <div class="clearfix" style="margin-bottom: 16px;"></div>
-                                </div>
-                            </div>
-                        </div>
-                    </div>
-
-                    {% include "partial/_sidebar.html" %}
-
-                </div>
-            </section>
-        </div>
-        {% include "partial/_footer.html" %}
-    </div>
-
-    
-
-    <script>
-
-    </script>
-</body>
-
-
-</html>

users/urls.py

@@ -1,6 +1,5 @@
 from django.urls import path
 from .views import *
-from .feeds import ReviewFeed
 
 app_name = "users"
 urlpatterns = [
@@ -9,7 +8,7 @@ urlpatterns = [
     path("connect/", connect, name="connect"),
     path("reconnect/", reconnect, name="reconnect"),
     path("data/", data, name="data"),
-    path("data/import_status/", data_import_status, name="import_status"),
+    path("data/import_status", data_import_status, name="import_status"),
     path("data/import_goodreads", import_goodreads, name="import_goodreads"),
     path("data/import_douban", import_douban, name="import_douban"),
     path("data/export_reviews", export_reviews, name="export_reviews"),
@@ -21,29 +20,8 @@ urlpatterns = [
     path("logout/", logout, name="logout"),
     path("layout/", set_layout, name="set_layout"),
     path("OAuth2_login/", OAuth2_login, name="OAuth2_login"),
-    path("<int:id>/", home_redirect, name="home_redirect"),
-    # path('<int:id>/followers/', followers, name='followers'),
-    # path('<int:id>/following/', following, name='following'),
-    # path('<int:id>/collections/', collection_list, name='collection_list'),
-    # path('<int:id>/book/<str:status>/', book_list, name='book_list'),
-    # path('<int:id>/movie/<str:status>/', movie_list, name='movie_list'),
-    # path('<int:id>/music/<str:status>/', music_list, name='music_list'),
-    # path('<int:id>/game/<str:status>/', game_list, name='game_list'),
-    path("<str:id>/", home, name="home"),
     path("<str:id>/followers/", followers, name="followers"),
     path("<str:id>/following/", following, name="following"),
-    path("<str:id>/tags/", tag_list, name="tag_list"),
-    path("<str:id>/collections/", collection_list, name="collection_list"),
-    path(
-        "<str:id>/collections/marked/",
-        marked_collection_list,
-        name="marked_collection_list",
-    ),
-    path("<str:id>/book/<str:status>/", book_list, name="book_list"),
-    path("<str:id>/movie/<str:status>/", movie_list, name="movie_list"),
-    path("<str:id>/music/<str:status>/", music_list, name="music_list"),
-    path("<str:id>/game/<str:status>/", game_list, name="game_list"),
-    path("<str:id>/feed/reviews/", ReviewFeed(), name="review_feed"),
     path("report/", report, name="report"),
     path("manage_report/", manage_report, name="manage_report"),
 ]

users/views.py

@@ -1,46 +1,16 @@
-from django.shortcuts import reverse, redirect, render, get_object_or_404
-from django.http import HttpResponseBadRequest, HttpResponse
+from django.shortcuts import redirect, render, get_object_or_404
+from django.urls import reverse
+from django.http import HttpResponseBadRequest
 from django.contrib.auth.decorators import login_required
-from django.contrib import auth
-from django.contrib.auth import authenticate
-from django.core.paginator import Paginator
 from django.utils.translation import gettext_lazy as _
-from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
-from django.db.models import Count
 from .models import User, Report, Preference
 from .forms import ReportForm
 from mastodon.api import *
 from mastodon import mastodon_request_included
 from common.config import *
-from common.models import MarkStatusEnum
-from common.utils import PageLinksGenerator
-from management.models import Announcement
-from books.models import *
-from movies.models import *
-from music.models import *
-from games.models import *
-from books.forms import BookMarkStatusTranslator
-from movies.forms import MovieMarkStatusTranslator
-from music.forms import MusicMarkStatusTranslator
-from games.forms import GameMarkStatusTranslator
-from mastodon.models import MastodonApplication
-from mastodon.api import verify_account
-from django.conf import settings
-from urllib.parse import quote
-import django_rq
 from .account import *
 from .data import *
-from datetime import timedelta
-from django.utils import timezone
 import json
-from django.contrib import messages
-from books.models import BookMark, BookReview
-from movies.models import MovieMark, MovieReview
-from games.models import GameMark, GameReview
-from music.models import AlbumMark, SongMark, AlbumReview, SongReview
-from collection.models import Collection
-from common.importers.goodreads import GoodreadsImporter
-from common.importers.douban import DoubanImporter
 
 
 def render_user_not_found(request):
@@ -48,11 +18,11 @@ def render_user_not_found(request):
     sec_msg = _("")
     return render(
         request,
-        'common/error.html',
+        "common/error.html",
         {
-            'msg': msg,
-            'secondary_msg': sec_msg,
-        }
+            "msg": msg,
+            "secondary_msg": sec_msg,
+        },
     )
 
 
@@ -60,518 +30,44 @@ def render_user_blocked(request):
     msg = _("你没有访问TA主页的权限😥")
     return render(
         request,
-        'common/error.html',
+        "common/error.html",
         {
-            'msg': msg,
-        }
+            "msg": msg,
+        },
     )
 
 
-def home_redirect(request, id):
-    try:
-        query_kwargs = {'pk': id}
-        user = User.objects.get(**query_kwargs)
-        return redirect(reverse("users:home", args=[user.mastodon_username]))
-    except Exception:
-        return redirect(settings.LOGIN_URL)
-
-
-def home_anonymous(request, id):
-    login_url = settings.LOGIN_URL + "?next=" + request.get_full_path()
-    try:
-        username = id.split('@')[0]
-        site = id.split('@')[1]
-        return render(request, 'users/home_anonymous.html', {
-                      'login_url': login_url,
-                      'username': username,
-                      'site': site,
-                      })
-    except Exception:
-        return redirect(login_url)
-
-
-@mastodon_request_included
-def home(request, id):
-    if not request.user.is_authenticated:
-        return home_anonymous(request, id)
-    if request.method == 'GET':
-        user = User.get(id)
-        if user is None:
-            return render_user_not_found(request)
-
-        # access one's own home page
-        if user == request.user:
-            reports = Report.objects.order_by(
-                '-submitted_time').filter(is_read=False)
-            unread_announcements = Announcement.objects.filter(
-                pk__gt=request.user.read_announcement_index).order_by('-pk')
-            try:
-                request.user.read_announcement_index = Announcement.objects.latest(
-                    'pk').pk
-                request.user.save(update_fields=['read_announcement_index'])
-            except ObjectDoesNotExist as e:
-                # when there is no annoucenment
-                pass
-            book_marks = request.user.user_bookmarks.all()
-            movie_marks = request.user.user_moviemarks.all()
-            album_marks = request.user.user_albummarks.all()
-            song_marks = request.user.user_songmarks.all()
-            game_marks = request.user.user_gamemarks.all()
-            book_reviews = request.user.user_bookreviews.all()
-            movie_reviews = request.user.user_moviereviews.all()
-            album_reviews = request.user.user_albumreviews.all()
-            song_reviews = request.user.user_songreviews.all()
-            game_reviews = request.user.user_gamereviews.all()
-
-        # visit other's home page
-        else:
-            # no these value on other's home page
-            reports = None
-            unread_announcements = None
-
-            if request.user.is_blocked_by(user) or request.user.is_blocking(user):
-                msg = _("你没有访问TA主页的权限😥")
-                return render(
-                    request,
-                    'common/error.html',
-                    {
-                        'msg': msg,
-                    }
-                )
-            is_following = request.user.is_following(user)
-            book_marks = BookMark.get_available_by_user(user, is_following)
-            movie_marks = MovieMark.get_available_by_user(user, is_following)
-            song_marks = SongMark.get_available_by_user(user, is_following)
-            album_marks = AlbumMark.get_available_by_user(user, is_following)
-            game_marks = GameMark.get_available_by_user(user, is_following)
-            book_reviews = BookReview.get_available_by_user(user, is_following)
-            movie_reviews = MovieReview.get_available_by_user(user, is_following)
-            song_reviews = SongReview.get_available_by_user(user, is_following)
-            album_reviews = AlbumReview.get_available_by_user(user, is_following)
-            game_reviews = GameReview.get_available_by_user(user, is_following)
-
-        collections = Collection.objects.filter(owner=user)
-        marked_collections = Collection.objects.filter(pk__in=CollectionMark.objects.filter(owner=user).values_list('collection', flat=True))
-
-        # book marks
-        filtered_book_marks = filter_marks(book_marks, BOOKS_PER_SET, 'book')
-        book_marks_count = count_marks(book_marks, "book")
-
-        # movie marks
-        filtered_movie_marks = filter_marks(movie_marks, MOVIES_PER_SET, 'movie')
-        movie_marks_count = count_marks(movie_marks, "movie")
-
-        # game marks
-        filtered_game_marks = filter_marks(game_marks, GAMES_PER_SET, 'game')
-        game_marks_count = count_marks(game_marks, "game")
-
-        # music marks
-        filtered_music_marks = filter_marks([song_marks, album_marks], MUSIC_PER_SET, 'music')
-        music_marks_count = count_marks([song_marks, album_marks], "music")
-
-        for mark in filtered_music_marks["do_music_marks"] +\
-            filtered_music_marks["wish_music_marks"] +\
-                filtered_music_marks["collect_music_marks"]:
-            # for template convenience
-            if mark.__class__ == AlbumMark:
-                mark.type = "album"
-            else:
-                mark.type = "song"
-
-        music_reviews = list(album_reviews.order_by("-edited_time")) + list(song_reviews.order_by("-edited_time"))
-        for review in music_reviews:
-            review.type = 'album' if review.__class__ == AlbumReview else 'song'
-
-        layout = user.get_preference().get_serialized_home_layout()
-
-        return render(
-            request,
-            'users/home.html',
-            {
-                'user': user,
-                **filtered_book_marks,
-                **filtered_movie_marks,
-                **filtered_game_marks,
-                **filtered_music_marks,
-                **book_marks_count,
-                **movie_marks_count,
-                **music_marks_count,
-                **game_marks_count,
-
-                'book_tags': BookTag.all_by_user(user)[:10] if user == request.user else [],
-                'movie_tags': MovieTag.all_by_user(user)[:10] if user == request.user else [],
-                'music_tags': AlbumTag.all_by_user(user)[:10] if user == request.user else [],
-                'game_tags': GameTag.all_by_user(user)[:10] if user == request.user else [],
-
-                'book_reviews': book_reviews.order_by("-edited_time")[:BOOKS_PER_SET],
-                'movie_reviews': movie_reviews.order_by("-edited_time")[:MOVIES_PER_SET],
-                'music_reviews': music_reviews[:MUSIC_PER_SET],
-                'game_reviews': game_reviews[:GAMES_PER_SET],
-                'book_reviews_more': book_reviews.count() > BOOKS_PER_SET,
-                'movie_reviews_more': movie_reviews.count() > MOVIES_PER_SET,
-                'music_reviews_more': len(music_reviews) > MUSIC_PER_SET,
-                'game_reviews_more': game_reviews.count() > GAMES_PER_SET,
-                'book_reviews_count': book_reviews.count(),
-                'movie_reviews_count': movie_reviews.count(),
-                'music_reviews_count': len(music_reviews),
-                'game_reviews_count': game_reviews.count(),
-
-                'collections': collections.order_by("-edited_time")[:BOOKS_PER_SET],
-                'collections_count': collections.count(),
-                'collections_more': collections.count() > BOOKS_PER_SET,
-
-                'marked_collections': marked_collections.order_by("-edited_time")[:BOOKS_PER_SET],
-                'marked_collections_count': marked_collections.count(),
-                'marked_collections_more': marked_collections.count() > BOOKS_PER_SET,
-
-                'layout': layout,
-                'reports': reports,
-                'unread_announcements': unread_announcements,
-            }
-        )
-    else:
-        return HttpResponseBadRequest()
-
-
-def filter_marks(querysets, maximum, type_name):
-    """
-    Filter marks by amount limits and order them edited time, store results in a dict,
-    which could be directly used in template.
-    @param querysets: one queryset or multiple querysets as a list
-    """
-    result = {}
-    if not isinstance(querysets, list):
-        querysets = [querysets]
-
-    for status in MarkStatusEnum.values:
-        marks = []
-        count = 0
-        for queryset in querysets:
-            marks += list(queryset.filter(status=MarkStatusEnum[status.upper()]).order_by("-created_time")[:maximum])
-            count += queryset.filter(status=MarkStatusEnum[status.upper()]).count()
-
-        # marks
-        marks = sorted(marks, key=lambda e: e.edited_time, reverse=True)[:maximum]
-        result[f"{status}_{type_name}_marks"] = marks
-        # flag indicates if marks are more than `maximun`
-        if count > maximum:
-            result[f"{status}_{type_name}_more"] = True
-        else:
-            result[f"{status}_{type_name}_more"] = False
-
-    return result
-
-
-def count_marks(querysets, type_name):
-    """
-    Count all available marks, then assembly a dict to be used in template
-    @param querysets: one queryset or multiple querysets as a list
-    """
-    result = {}
-    if not isinstance(querysets, list):
-        querysets = [querysets]
-    for status in MarkStatusEnum.values:
-        count = 0
-        for queryset in querysets:
-            count += queryset.filter(status=MarkStatusEnum[status.upper()]).count()
-        result[f"{status}_{type_name}_count"] = count
-    return result
-
-
-@mastodon_request_included
 @login_required
 def followers(request, id):
-    if request.method == 'GET':
+    if request.method == "GET":
         user = User.get(id)
         if user is None or user != request.user:
             return render_user_not_found(request)
         return render(
             request,
-            'users/relation_list.html',
+            "users/relation_list.html",
             {
-                'user': user,
-                'is_followers_page': True,
-            }
+                "user": user,
+                "is_followers_page": True,
+            },
         )
     else:
         return HttpResponseBadRequest()
 
 
-@mastodon_request_included
 @login_required
 def following(request, id):
-    if request.method == 'GET':
+    if request.method == "GET":
         user = User.get(id)
         if user is None or user != request.user:
             return render_user_not_found(request)
         return render(
             request,
-            'users/relation_list.html',
+            "users/relation_list.html",
             {
-                'user': user,
-                'page_type': 'followers',
-            }
-        )
-    else:
-        return HttpResponseBadRequest()
-
-
-@mastodon_request_included
-@login_required
-def book_list(request, id, status):
-    if request.method == 'GET':
-        if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']:
-            return HttpResponseBadRequest()
-
-        user = User.get(id)
-        if user is None:
-            return render_user_not_found(request)
-        tag = request.GET.get('t', default='')
-        if user != request.user:
-            if request.user.is_blocked_by(user) or request.user.is_blocking(user):
-                msg = _("你没有访问TA主页的权限😥")
-                return render(
-                    request,
-                    'common/error.html',
-                    {
-                        'msg': msg,
-                    }
-                )
-            is_following = request.user.is_following(user)
-            if status == 'reviewed':
-                queryset = BookReview.get_available_by_user(user, is_following).order_by("-edited_time")
-            elif status == 'tagged':
-                queryset = BookTag.find_by_user(tag, user, request.user).order_by("-mark__created_time")
-            else:
-                queryset = BookMark.get_available_by_user(user, is_following).filter(
-                    status=MarkStatusEnum[status.upper()]).order_by("-created_time")
-        else:
-            if status == 'reviewed':
-                queryset = BookReview.objects.filter(owner=user).order_by("-edited_time")
-            elif status == 'tagged':
-                queryset = BookTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time")
-            else:
-                queryset = BookMark.objects.filter(
-                    owner=user, status=MarkStatusEnum[status.upper()]).order_by("-created_time")
-        paginator = Paginator(queryset, ITEMS_PER_PAGE)
-        page_number = request.GET.get('page', default=1)
-        marks = paginator.get_page(page_number)
-        for mark in marks:
-            mark.book.tag_list = mark.book.get_tags_manager().values('content').annotate(
-                tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST]
-        marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages)
-        if status == 'reviewed':
-            list_title = str(_("评论过的书"))
-        elif status == 'tagged':
-            list_title = str(_(f"标记为「{tag}」的书"))
-        else:
-            list_title = str(BookMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的书"))
-        return render(
-            request,
-            'users/item_list.html',
-            {
-                'marks': marks,
-                'user': user,
-                'status': status,
-                'list_title': list_title,
-            }
-        )
-    else:
-        return HttpResponseBadRequest()
-
-
-@mastodon_request_included
-@login_required
-def movie_list(request, id, status):
-    if request.method == 'GET':
-        if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']:
-            return HttpResponseBadRequest()
-
-        user = User.get(id)
-        if user is None:
-            return render_user_not_found(request)
-        tag = request.GET.get('t', default='')
-        if user != request.user:
-            if request.user.is_blocked_by(user) or request.user.is_blocking(user):
-                msg = _("你没有访问TA主页的权限😥")
-                return render(
-                    request,
-                    'common/error.html',
-                    {
-                        'msg': msg,
-                    }
-                )
-            is_following = request.user.is_following(user)
-            if status == 'reviewed':
-                queryset = MovieReview.get_available_by_user(user, is_following).order_by("-edited_time")
-            elif status == 'tagged':
-                queryset = MovieTag.find_by_user(tag, user, request.user).order_by("-mark__created_time")
-            else:
-                queryset = MovieMark.get_available_by_user(user, is_following).filter(
-                    status=MarkStatusEnum[status.upper()]).order_by("-created_time")
-        else:
-            if status == 'reviewed':
-                queryset = MovieReview.objects.filter(owner=user).order_by("-edited_time")
-            elif status == 'tagged':
-                queryset = MovieTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time")
-            else:
-                queryset = MovieMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()]).order_by("-created_time")
-        paginator = Paginator(queryset, ITEMS_PER_PAGE)
-        page_number = request.GET.get('page', default=1)
-        marks = paginator.get_page(page_number)
-        for mark in marks:
-            mark.movie.tag_list = mark.movie.get_tags_manager().values('content').annotate(
-                tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST]
-        marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages)
-        if status == 'reviewed':
-            list_title = str(_("评论过的电影和剧集"))
-        elif status == 'tagged':
-            list_title = str(_(f"标记为「{tag}」的电影和剧集"))
-        else:
-            list_title = str(MovieMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的电影和剧集"))
-
-        return render(
-            request,
-            'users/item_list.html',
-            {
-                'marks': marks,
-                'user': user,
-                'status': status,
-                'list_title': list_title,
-            }
-        )
-    else:
-        return HttpResponseBadRequest()
-
-
-@mastodon_request_included
-@login_required
-def game_list(request, id, status):
-    if request.method == 'GET':
-        if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']:
-            return HttpResponseBadRequest()
-
-        user = User.get(id)
-        if user is None:
-            return render_user_not_found(request)
-        tag = request.GET.get('t', default='')
-        if user != request.user:
-            if request.user.is_blocked_by(user) or request.user.is_blocking(user):
-                msg = _("你没有访问TA主页的权限😥")
-                return render(
-                    request,
-                    'common/error.html',
-                    {
-                        'msg': msg,
-                    }
-                )
-            is_following = request.user.is_following(user)
-            if status == 'reviewed':
-                queryset = GameReview.get_available_by_user(user, is_following).order_by("-edited_time")
-            elif status == 'tagged':
-                queryset = GameTag.find_by_user(tag, user, request.user).order_by("-mark__created_time")
-            else:
-                queryset = GameMark.get_available_by_user(user, is_following).filter(
-                    status=MarkStatusEnum[status.upper()]).order_by("-created_time")
-        else:
-            if status == 'reviewed':
-                queryset = GameReview.objects.filter(owner=user).order_by("-edited_time")
-            elif status == 'tagged':
-                queryset = GameTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time")
-            else:
-                queryset = GameMark.objects.filter(
-                    owner=user, status=MarkStatusEnum[status.upper()]).order_by("-created_time")
-        paginator = Paginator(queryset, ITEMS_PER_PAGE)
-        page_number = request.GET.get('page', default=1)
-        marks = paginator.get_page(page_number)
-        for mark in marks:
-            mark.game.tag_list = mark.game.get_tags_manager().values('content').annotate(
-                tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST]
-        marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages)
-        if status == 'reviewed':
-            list_title = str(_("评论过的游戏"))
-        elif status == 'tagged':
-            list_title = str(_(f"标记为「{tag}」的游戏"))
-        else:
-            list_title = str(GameMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的游戏"))
-        return render(
-            request,
-            'users/item_list.html',
-            {
-                'marks': marks,
-                'user': user,
-                'status': status,
-                'list_title': list_title,
-            }
-        )
-    else:
-        return HttpResponseBadRequest()
-
-
-@mastodon_request_included
-@login_required
-def music_list(request, id, status):
-    if request.method == 'GET':
-        if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']:
-            return HttpResponseBadRequest()
-
-        user = User.get(id)
-        if user is None:
-            return render_user_not_found(request)
-        tag = request.GET.get('t', default='')
-        if not user == request.user:
-            if request.user.is_blocked_by(user) or request.user.is_blocking(user):
-                return render_user_blocked(request)
-            is_following = request.user.is_following(user)
-            if status == 'reviewed':
-                queryset = list(AlbumReview.get_available_by_user(user, is_following).order_by("-edited_time")) + \
-                    list(SongReview.get_available_by_user(user, is_following).order_by("-edited_time"))
-            elif status == 'tagged':
-                queryset = list(AlbumTag.find_by_user(tag, user, request.user).order_by("-mark__created_time"))
-            else:
-                queryset = list(AlbumMark.get_available_by_user(user, is_following).filter(
-                    status=MarkStatusEnum[status.upper()])) \
-                    + list(SongMark.get_available_by_user(user, is_following).filter(
-                        status=MarkStatusEnum[status.upper()]))
-        else:
-            if status == 'reviewed':
-                queryset = list(AlbumReview.objects.filter(owner=user).order_by("-edited_time")) + \
-                    list(SongReview.objects.filter(owner=user).order_by("-edited_time"))
-            elif status == 'tagged':
-                queryset = list(AlbumTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time"))
-            else:
-                queryset = list(AlbumMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()])) \
-                    + list(SongMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()]))
-        queryset = sorted(queryset, key=lambda e: e.edited_time, reverse=True)
-        paginator = Paginator(queryset, ITEMS_PER_PAGE)
-        page_number = request.GET.get('page', default=1)
-        marks = paginator.get_page(page_number)
-        for mark in marks:
-            if mark.__class__ in [AlbumMark, AlbumReview, AlbumTag]:
-                mark.music = mark.album
-                mark.music.tag_list = mark.album.get_tags_manager().values('content').annotate(
-                    tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST]
-            elif mark.__class__ == SongMark or mark.__class__ == SongReview:
-                mark.music = mark.song
-                mark.music.tag_list = mark.song.get_tags_manager().values('content').annotate(
-                    tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST]
-
-        marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages)
-        if status == 'reviewed':
-            list_title = str(_("评论过的音乐"))
-        elif status == 'tagged':
-            list_title = str(_(f"标记为「{tag}」的音乐"))
-        else:
-            list_title = str(MusicMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的音乐"))
-        return render(
-            request,
-            'users/item_list.html',
-            {
-                'marks': marks,
-                'user': user,
-                'status': status,
-                'list_title': list_title,
-            }
+                "user": user,
+                "page_type": "followers",
+            },
         )
     else:
         return HttpResponseBadRequest()
@@ -579,45 +75,52 @@ def music_list(request, id, status):
 
 @login_required
 def set_layout(request):
-    if request.method == 'POST':
-        layout = json.loads(request.POST.get('layout'))
-        request.user.preference.home_layout = layout
+    if request.method == "POST":
+        layout = json.loads(request.POST.get("layout"))
+        request.user.preference.profile_layout = layout
         request.user.preference.save()
-        return redirect(reverse("users:home", args=[request.user.mastodon_username]))
+        return redirect(
+            reverse("journal:user_profile", args=[request.user.mastodon_username])
+        )
     else:
         return HttpResponseBadRequest()
 
 
 @login_required
 def report(request):
-    if request.method == 'GET':
-        user_id = request.GET.get('user_id')
+    if request.method == "GET":
+        user_id = request.GET.get("user_id")
         if user_id:
             user = get_object_or_404(User, pk=user_id)
-            form = ReportForm(initial={'reported_user': user})
+            form = ReportForm(initial={"reported_user": user})
         else:
             form = ReportForm()
         return render(
             request,
-            'users/report.html',
+            "users/report.html",
             {
-                'form': form,
-            }
+                "form": form,
+            },
         )
-    elif request.method == 'POST':
+    elif request.method == "POST":
         form = ReportForm(request.POST)
         if form.is_valid():
             form.instance.is_read = False
             form.instance.submit_user = request.user
             form.save()
-            return redirect(reverse("users:home", args=[form.instance.reported_user.mastodon_username]))
+            return redirect(
+                reverse(
+                    "journal:user_profile",
+                    args=[form.instance.reported_user.mastodon_username],
+                )
+            )
         else:
             return render(
                 request,
-                'users/report.html',
+                "users/report.html",
                 {
-                    'form': form,
-                }
+                    "form": form,
+                },
             )
     else:
         return HttpResponseBadRequest()
@@ -625,53 +128,17 @@ def report(request):
 
 @login_required
 def manage_report(request):
-    if request.method == 'GET':
+    if request.method == "GET":
         reports = Report.objects.all()
         for r in reports.filter(is_read=False):
             r.is_read = True
             r.save()
         return render(
             request,
-            'users/manage_report.html',
+            "users/manage_report.html",
             {
-                'reports': reports,
-            }
+                "reports": reports,
+            },
         )
     else:
         return HttpResponseBadRequest()
-
-
-@login_required
-def collection_list(request, id):
-    from collection.views import list
-    user = User.get(id)
-    if user is None:
-        return render_user_not_found(request)
-    return list(request, user.id)
-
-
-@login_required
-def marked_collection_list(request, id):
-    from collection.views import list
-    user = User.get(id)
-    if user is None:
-        return render_user_not_found(request)
-    return list(request, user.id, True)
-
-
-@login_required
-def tag_list(request, id):
-    user = User.get(id)
-    if user is None:
-        return render_user_not_found(request)
-    if user != request.user:
-        raise PermissionDenied()  # tag list is for user's own view only, for now
-    return render(
-        request,
-        'users/tags.html', {
-            'book_tags': BookTag.all_by_user(user),
-            'movie_tags': MovieTag.all_by_user(user),
-            'music_tags': AlbumTag.all_by_user(user),
-            'game_tags': GameTag.all_by_user(user),
-        }
-    )