289 lines
11 KiB
Python
289 lines
11 KiB
Python
from catalog.book.models import *
|
|
from catalog.book.utils import *
|
|
from catalog.common import *
|
|
from common.models.lang import detect_language
|
|
|
|
from .douban import RE_NUMBERS, RE_WHITESPACES, DoubanDownloader, DoubanSearcher
|
|
|
|
|
|
@SiteManager.register
|
|
class DoubanBook(AbstractSite):
|
|
SITE_NAME = SiteName.Douban
|
|
ID_TYPE = IdType.DoubanBook
|
|
URL_PATTERNS = [
|
|
r"\w+://book\.douban\.com/subject/(\d+)/{0,1}",
|
|
r"\w+://m.douban.com/book/subject/(\d+)/{0,1}",
|
|
r"\w+://www.douban.com/doubanapp/dispatch\?uri=/book/(\d+)/",
|
|
r"\w+://www.douban.com/doubanapp/dispatch/book/(\d+)",
|
|
]
|
|
WIKI_PROPERTY_ID = "?"
|
|
DEFAULT_MODEL = Edition
|
|
|
|
@classmethod
|
|
def id_to_url(cls, id_value):
|
|
return "https://book.douban.com/subject/" + id_value + "/"
|
|
|
|
@classmethod
|
|
def search(cls, q: str, p: int = 1):
|
|
return DoubanSearcher.search(ItemCategory.Book, "book", q, p)
|
|
|
|
def scrape(self):
|
|
content = DoubanDownloader(self.url).download().html()
|
|
|
|
isbn_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='ISBN:']/following::text()"
|
|
)
|
|
isbn = isbn_elem[0].strip() if isbn_elem else None
|
|
|
|
title_elem = self.query_list(content, "/html/body//h1/span/text()")
|
|
title = (
|
|
title_elem[0].strip() if title_elem else f"Unknown Title {self.id_value}"
|
|
)
|
|
|
|
subtitle_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='副标题:']/following::text()"
|
|
)
|
|
subtitle = subtitle_elem[0].strip()[:500] if subtitle_elem else None
|
|
|
|
orig_title_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='原作名:']/following::text()"
|
|
)
|
|
orig_title = orig_title_elem[0].strip()[:500] if orig_title_elem else None
|
|
|
|
language_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='语言:']/following::text()"
|
|
)
|
|
language = [language_elem[0].strip()] if language_elem else []
|
|
|
|
pub_house_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='出版社:']/following::text()"
|
|
)
|
|
pub_house = pub_house_elem[0].strip() if pub_house_elem else None
|
|
if not pub_house:
|
|
pub_house_elem = self.query_list(
|
|
content,
|
|
"//div[@id='info']//span[text()='出版社:']/following-sibling::a/text()",
|
|
)
|
|
pub_house = pub_house_elem[0].strip() if pub_house_elem else None
|
|
|
|
pub_date_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='出版年:']/following::text()"
|
|
)
|
|
pub_date = pub_date_elem[0].strip() if pub_date_elem else ""
|
|
year_month_day = RE_NUMBERS.findall(pub_date)
|
|
if len(year_month_day) in (2, 3):
|
|
pub_year = int(year_month_day[0])
|
|
pub_month = int(year_month_day[1])
|
|
elif len(year_month_day) == 1:
|
|
pub_year = int(year_month_day[0])
|
|
pub_month = None
|
|
else:
|
|
pub_year = None
|
|
pub_month = None
|
|
if pub_year and pub_month and pub_year < pub_month:
|
|
pub_year, pub_month = pub_month, pub_year
|
|
pub_year = (
|
|
None
|
|
if pub_year is not None and pub_year not in range(0, 3000)
|
|
else pub_year
|
|
)
|
|
pub_month = (
|
|
None
|
|
if pub_month is not None and pub_month not in range(1, 12)
|
|
else pub_month
|
|
)
|
|
|
|
binding_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='装帧:']/following::text()"
|
|
)
|
|
binding = binding_elem[0].strip() if binding_elem else None
|
|
|
|
price_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='定价:']/following::text()"
|
|
)
|
|
price = price_elem[0].strip() if price_elem else None
|
|
|
|
pages_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='页数:']/following::text()"
|
|
)
|
|
pages = pages_elem[0].strip() if pages_elem else None
|
|
if pages is not None:
|
|
pages = (
|
|
int(RE_NUMBERS.findall(pages)[0]) if RE_NUMBERS.findall(pages) else None
|
|
)
|
|
if pages and (pages > 999999 or pages < 1):
|
|
pages = None
|
|
|
|
brief_elem = self.query_list(
|
|
content,
|
|
"//h2/span[text()='内容简介']/../following-sibling::div[1]//div[@class='intro'][not(ancestor::span[@class='short'])]/p/text()",
|
|
)
|
|
brief = "\n".join(p.strip() for p in brief_elem) if brief_elem else None
|
|
|
|
contents = None
|
|
try:
|
|
contents_elem = self.query_list(
|
|
content, "//h2/span[text()='目录']/../following-sibling::div[1]"
|
|
)[0]
|
|
# if next the id of next sibling contains `dir`, that would be the full contents
|
|
if "dir" in contents_elem.getnext().xpath("@id")[0]:
|
|
contents_elem = contents_elem.getnext()
|
|
contents = (
|
|
"\n".join(p.strip() for p in contents_elem.xpath("text()")[:-2])
|
|
if contents_elem is not None
|
|
else None
|
|
)
|
|
else:
|
|
contents = (
|
|
"\n".join(
|
|
p.strip() for p in self.query_list(contents_elem, "text()")
|
|
)
|
|
if contents_elem is not None
|
|
else None
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
img_url_elem = self.query_list(content, "//*[@id='mainpic']/a/img/@src")
|
|
img_url = img_url_elem[0].strip() if img_url_elem else None
|
|
|
|
# there are two html formats for authors and translators
|
|
authors_elem = self.query_list(
|
|
content,
|
|
"""//div[@id='info']//span[text()='作者:']/following-sibling::br[1]/
|
|
preceding-sibling::a[preceding-sibling::span[text()='作者:']]/text()""",
|
|
)
|
|
if not authors_elem:
|
|
authors_elem = self.query_list(
|
|
content,
|
|
"""//div[@id='info']//span[text()=' 作者']/following-sibling::a/text()""",
|
|
)
|
|
if authors_elem:
|
|
authors = []
|
|
for author in authors_elem:
|
|
authors.append(RE_WHITESPACES.sub(" ", author.strip())[:200])
|
|
else:
|
|
authors = None
|
|
|
|
translators_elem = self.query_list(
|
|
content,
|
|
"""//div[@id='info']//span[text()='译者:']/following-sibling::br[1]/
|
|
preceding-sibling::a[preceding-sibling::span[text()='译者:']]/text()""",
|
|
)
|
|
if not translators_elem:
|
|
translators_elem = self.query_list(
|
|
content,
|
|
"""//div[@id='info']//span[text()=' 译者']/following-sibling::a/text()""",
|
|
)
|
|
if translators_elem:
|
|
translators = []
|
|
for translator in translators_elem:
|
|
translators.append(RE_WHITESPACES.sub(" ", translator.strip()))
|
|
else:
|
|
translators = None
|
|
|
|
cncode_elem = self.query_list(
|
|
content, "//div[@id='info']//span[text()='统一书号:']/following::text()"
|
|
)
|
|
cubn = cncode_elem[0].strip() if cncode_elem else None
|
|
|
|
series_elem = self.query_list(
|
|
content,
|
|
"//div[@id='info']//span[text()='丛书:']/following-sibling::a[1]/text()",
|
|
)
|
|
series = series_elem[0].strip() if series_elem else None
|
|
|
|
imprint_elem = self.query_list(
|
|
content,
|
|
"//div[@id='info']//span[text()='出品方:']/following-sibling::a[1]/text()",
|
|
)
|
|
imprint = imprint_elem[0].strip() if imprint_elem else None
|
|
|
|
lang = detect_language(title + " " + (brief or ""))
|
|
data = {
|
|
"title": title,
|
|
"subtitle": subtitle,
|
|
"localized_title": [{"lang": lang, "text": title}],
|
|
"localized_subtitle": [{"lang": lang, "text": subtitle}],
|
|
"localized_description": [{"lang": lang, "text": brief}] if brief else [],
|
|
"orig_title": orig_title,
|
|
"author": authors,
|
|
"translator": translators,
|
|
"language": language,
|
|
"pub_house": pub_house,
|
|
"pub_year": pub_year,
|
|
"pub_month": pub_month,
|
|
"binding": binding,
|
|
"format": binding_to_format(binding),
|
|
"price": price,
|
|
"pages": pages,
|
|
"isbn": isbn,
|
|
"cubn": cubn,
|
|
"brief": brief,
|
|
"contents": contents,
|
|
"series": series,
|
|
"imprint": imprint,
|
|
"cover_image_url": img_url,
|
|
}
|
|
|
|
works_element = self.query_list(
|
|
content,
|
|
'//h2/span[text()="这本书的其他版本"]/following-sibling::span[@class="pl"]/a/@href',
|
|
)
|
|
if works_element:
|
|
r = re.match(r"\w+://book.douban.com/works/(\d+)", works_element[0])
|
|
data["required_resources"] = [
|
|
{
|
|
"model": "Work",
|
|
"id_type": IdType.DoubanBook_Work,
|
|
"id_value": r[1] if r else None,
|
|
"title": data["title"],
|
|
"url": works_element[0],
|
|
"content": {
|
|
"metadata": {
|
|
"title": data["title"],
|
|
"localized_title": data["localized_title"],
|
|
}
|
|
},
|
|
}
|
|
]
|
|
|
|
pd = ResourceContent(metadata=data)
|
|
t, n = detect_isbn_asin(isbn or "")
|
|
if t:
|
|
pd.lookup_ids[t] = n
|
|
pd.lookup_ids[IdType.CUBN] = cubn
|
|
return pd
|
|
|
|
|
|
@SiteManager.register
|
|
class DoubanBook_Work(AbstractSite):
|
|
SITE_NAME = SiteName.Douban
|
|
ID_TYPE = IdType.DoubanBook_Work
|
|
URL_PATTERNS = [r"\w+://book\.douban\.com/works/(\d+)"]
|
|
WIKI_PROPERTY_ID = "?"
|
|
DEFAULT_MODEL = Work
|
|
|
|
@classmethod
|
|
def id_to_url(cls, id_value):
|
|
return "https://book.douban.com/works/" + id_value + "/"
|
|
|
|
def scrape(self):
|
|
content = DoubanDownloader(self.url).download().html()
|
|
title_elem = self.query_list(content, "//h1/text()")
|
|
title = title_elem[0].split("全部版本(")[0].strip() if title_elem else None
|
|
if not title:
|
|
raise ParseError(self, "title")
|
|
book_urls = self.query_list(content, '//a[@class="pl2"]/@href')
|
|
related_resources = []
|
|
for url in book_urls:
|
|
site = SiteManager.get_site_by_url(url)
|
|
if site:
|
|
related_resources.append({"url": url})
|
|
d = {
|
|
"title": title,
|
|
"localized_title": [{"lang": "zh-cn", "text": title}],
|
|
"related_resources": related_resources,
|
|
}
|
|
pd = ResourceContent(metadata=d)
|
|
return pd
|