2022-12-08 23:58:44 +00:00
|
|
|
"""
|
|
|
|
IGDB
|
|
|
|
|
|
|
|
use (e.g. "portal-2") as id, which is different from real id in IGDB API
|
|
|
|
"""
|
|
|
|
|
|
|
|
import datetime
|
|
|
|
import json
|
2024-12-08 19:33:05 +00:00
|
|
|
from urllib.parse import quote_plus
|
2022-12-08 23:58:44 +00:00
|
|
|
|
2023-08-10 11:27:31 -04:00
|
|
|
import requests
|
|
|
|
from django.conf import settings
|
2023-11-19 10:52:02 -05:00
|
|
|
from django.core.cache import cache
|
2023-08-10 11:27:31 -04:00
|
|
|
from igdb.wrapper import IGDBWrapper
|
2024-05-25 23:38:11 -04:00
|
|
|
from loguru import logger
|
2023-08-10 11:27:31 -04:00
|
|
|
|
|
|
|
from catalog.common import *
|
|
|
|
from catalog.models import *
|
2024-12-08 19:33:05 +00:00
|
|
|
from catalog.search.models import ExternalSearchResultItem
|
2022-12-08 23:58:44 +00:00
|
|
|
|
2023-11-19 10:52:02 -05:00
|
|
|
_cache_key = "igdb_access_token"
|
2022-12-08 23:58:44 +00:00
|
|
|
|
|
|
|
|
|
|
|
def _igdb_access_token():
|
2023-08-17 02:09:36 -04:00
|
|
|
if not settings.IGDB_CLIENT_SECRET:
|
|
|
|
return "<missing>"
|
2022-12-08 23:58:44 +00:00
|
|
|
try:
|
2023-11-19 10:52:02 -05:00
|
|
|
token = cache.get(_cache_key)
|
|
|
|
if not token:
|
|
|
|
j = requests.post(
|
|
|
|
f"https://id.twitch.tv/oauth2/token?client_id={settings.IGDB_CLIENT_ID}&client_secret={settings.IGDB_CLIENT_SECRET}&grant_type=client_credentials"
|
|
|
|
).json()
|
|
|
|
token = j["access_token"]
|
|
|
|
ttl = j["expires_in"] - 60
|
|
|
|
cache.set(_cache_key, token, ttl)
|
2024-05-25 23:38:11 -04:00
|
|
|
except Exception as e:
|
|
|
|
logger.error("unable to obtain IGDB token", extra={"exception": e})
|
2022-12-29 23:57:02 -05:00
|
|
|
token = "<invalid>"
|
2022-12-08 23:58:44 +00:00
|
|
|
return token
|
|
|
|
|
|
|
|
|
|
|
|
def search_igdb_by_3p_url(steam_url):
|
2022-12-29 23:57:02 -05:00
|
|
|
r = IGDB.api_query("websites", f'fields *, game.*; where url = "{steam_url}";')
|
2022-12-08 23:58:44 +00:00
|
|
|
if not r:
|
|
|
|
return None
|
2022-12-29 23:57:02 -05:00
|
|
|
r = sorted(r, key=lambda w: w["game"]["id"])
|
|
|
|
return IGDB(url=r[0]["game"]["url"])
|
2022-12-08 23:58:44 +00:00
|
|
|
|
|
|
|
|
2022-12-15 17:29:35 -05:00
|
|
|
@SiteManager.register
|
2022-12-08 23:58:44 +00:00
|
|
|
class IGDB(AbstractSite):
|
2022-12-16 01:08:10 -05:00
|
|
|
SITE_NAME = SiteName.IGDB
|
2022-12-08 23:58:44 +00:00
|
|
|
ID_TYPE = IdType.IGDB
|
2023-05-17 10:32:12 -04:00
|
|
|
URL_PATTERNS = [
|
|
|
|
r"\w+://www\.igdb\.com/games/([a-zA-Z0-9\-_]+)",
|
|
|
|
r"\w+://m\.igdb\.com/games/([a-zA-Z0-9\-_]+)",
|
|
|
|
]
|
2022-12-29 23:57:02 -05:00
|
|
|
WIKI_PROPERTY_ID = "?"
|
2022-12-08 23:58:44 +00:00
|
|
|
DEFAULT_MODEL = Game
|
|
|
|
|
|
|
|
@classmethod
|
2023-06-05 10:06:16 -04:00
|
|
|
def id_to_url(cls, id_value):
|
2022-12-08 23:58:44 +00:00
|
|
|
return "https://www.igdb.com/games/" + id_value
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def api_query(cls, p, q):
|
2022-12-29 23:57:02 -05:00
|
|
|
key = "igdb:" + p + "/" + q
|
2022-12-08 23:58:44 +00:00
|
|
|
if get_mock_mode():
|
|
|
|
r = BasicDownloader(key).download().json()
|
|
|
|
else:
|
2023-11-19 10:52:02 -05:00
|
|
|
_wrapper = IGDBWrapper(settings.IGDB_CLIENT_ID, _igdb_access_token())
|
2024-12-25 16:02:34 -05:00
|
|
|
try:
|
|
|
|
r = json.loads(_wrapper.api_request(p, q)) # type: ignore
|
|
|
|
except requests.HTTPError as e:
|
|
|
|
logger.error("IGDB API: {e}", extra={"exception": e})
|
|
|
|
return []
|
2022-12-08 23:58:44 +00:00
|
|
|
if settings.DOWNLOADER_SAVEDIR:
|
2022-12-29 23:57:02 -05:00
|
|
|
with open(
|
|
|
|
settings.DOWNLOADER_SAVEDIR + "/" + get_mock_file(key),
|
|
|
|
"w",
|
|
|
|
encoding="utf-8",
|
|
|
|
) as fp:
|
2022-12-08 23:58:44 +00:00
|
|
|
fp.write(json.dumps(r))
|
|
|
|
return r
|
|
|
|
|
2024-12-08 19:33:05 +00:00
|
|
|
@classmethod
|
|
|
|
def search(cls, q, limit: int, offset: int = 0):
|
|
|
|
rs = cls.api_query(
|
|
|
|
"games",
|
2024-12-09 06:56:29 +00:00
|
|
|
f'fields *, cover.url, genres.name, platforms.name, involved_companies.*, involved_companies.company.name; search "{quote_plus(q)}"; limit {limit}; offset {offset};',
|
2024-12-08 19:33:05 +00:00
|
|
|
)
|
|
|
|
result = []
|
|
|
|
for r in rs:
|
|
|
|
subtitle = ""
|
|
|
|
if "first_release_date" in r:
|
|
|
|
subtitle = datetime.datetime.fromtimestamp(
|
|
|
|
r["first_release_date"], datetime.timezone.utc
|
|
|
|
).strftime("%Y-%m-%d ")
|
|
|
|
if "platforms" in r:
|
|
|
|
ps = sorted(r["platforms"], key=lambda p: p["id"])
|
|
|
|
subtitle += ",".join(
|
|
|
|
[(p["name"] if p["id"] != 6 else "Windows") for p in ps]
|
|
|
|
)
|
|
|
|
brief = r["summary"] if "summary" in r else ""
|
|
|
|
brief += "\n\n" + r["storyline"] if "storyline" in r else ""
|
2024-12-09 06:56:29 +00:00
|
|
|
cover = (
|
|
|
|
"https:" + r["cover"]["url"].replace("t_thumb", "t_cover_big")
|
|
|
|
if r.get("cover")
|
|
|
|
else ""
|
|
|
|
)
|
2024-12-08 19:33:05 +00:00
|
|
|
result.append(
|
|
|
|
ExternalSearchResultItem(
|
|
|
|
ItemCategory.Game,
|
|
|
|
SiteName.IGDB,
|
|
|
|
r["url"],
|
|
|
|
r["name"],
|
|
|
|
subtitle,
|
|
|
|
brief,
|
|
|
|
cover,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
return result
|
|
|
|
|
2022-12-08 23:58:44 +00:00
|
|
|
def scrape(self):
|
2022-12-29 23:57:02 -05:00
|
|
|
fields = "*, cover.url, genres.name, platforms.name, involved_companies.*, involved_companies.company.name"
|
2024-12-25 16:02:34 -05:00
|
|
|
r = self.api_query("games", f'fields {fields}; where url = "{self.url}";')
|
|
|
|
if not r:
|
|
|
|
raise ParseError(self, "no data")
|
|
|
|
r = r[0]
|
2022-12-29 23:57:02 -05:00
|
|
|
brief = r["summary"] if "summary" in r else ""
|
|
|
|
brief += "\n\n" + r["storyline"] if "storyline" in r else ""
|
2022-12-08 23:58:44 +00:00
|
|
|
developer = None
|
|
|
|
publisher = None
|
|
|
|
release_date = None
|
|
|
|
genre = None
|
|
|
|
platform = None
|
2022-12-29 23:57:02 -05:00
|
|
|
if "involved_companies" in r:
|
|
|
|
developer = next(
|
|
|
|
iter(
|
|
|
|
[
|
|
|
|
c["company"]["name"]
|
|
|
|
for c in r["involved_companies"]
|
|
|
|
if c["developer"]
|
|
|
|
]
|
|
|
|
),
|
|
|
|
None,
|
|
|
|
)
|
|
|
|
publisher = next(
|
|
|
|
iter(
|
|
|
|
[
|
|
|
|
c["company"]["name"]
|
|
|
|
for c in r["involved_companies"]
|
|
|
|
if c["publisher"]
|
|
|
|
]
|
|
|
|
),
|
|
|
|
None,
|
|
|
|
)
|
|
|
|
if "platforms" in r:
|
|
|
|
ps = sorted(r["platforms"], key=lambda p: p["id"])
|
|
|
|
platform = [(p["name"] if p["id"] != 6 else "Windows") for p in ps]
|
|
|
|
if "first_release_date" in r:
|
|
|
|
release_date = datetime.datetime.fromtimestamp(
|
|
|
|
r["first_release_date"], datetime.timezone.utc
|
|
|
|
).strftime("%Y-%m-%d")
|
|
|
|
if "genres" in r:
|
|
|
|
genre = [g["name"] for g in r["genres"]]
|
|
|
|
websites = self.api_query(
|
|
|
|
"websites", f'fields *; where game.url = "{self.url}";'
|
|
|
|
)
|
2022-12-08 23:58:44 +00:00
|
|
|
steam_url = None
|
|
|
|
official_site = None
|
|
|
|
for website in websites:
|
2022-12-29 23:57:02 -05:00
|
|
|
if website["category"] == 1:
|
|
|
|
official_site = website["url"]
|
|
|
|
elif website["category"] == 13:
|
|
|
|
steam_url = website["url"]
|
|
|
|
pd = ResourceContent(
|
|
|
|
metadata={
|
2024-07-13 00:16:47 -04:00
|
|
|
"localized_title": [{"lang": "en", "text": r["name"]}],
|
|
|
|
"localized_description": [{"lang": "en", "text": brief}],
|
2022-12-29 23:57:02 -05:00
|
|
|
"title": r["name"],
|
|
|
|
"other_title": [],
|
2023-11-11 14:01:23 -05:00
|
|
|
"developer": [developer] if developer else [],
|
|
|
|
"publisher": [publisher] if publisher else [],
|
2022-12-29 23:57:02 -05:00
|
|
|
"release_date": release_date,
|
|
|
|
"genre": genre,
|
|
|
|
"platform": platform,
|
|
|
|
"brief": brief,
|
|
|
|
"official_site": official_site,
|
|
|
|
"igdb_id": r["id"],
|
2024-06-02 14:50:07 -04:00
|
|
|
"cover_image_url": (
|
|
|
|
"https:" + r["cover"]["url"].replace("t_thumb", "t_cover_big")
|
|
|
|
if r.get("cover")
|
|
|
|
else None
|
|
|
|
),
|
2022-12-29 23:57:02 -05:00
|
|
|
}
|
|
|
|
)
|
2022-12-08 23:58:44 +00:00
|
|
|
if steam_url:
|
2023-06-05 10:06:16 -04:00
|
|
|
pd.lookup_ids[IdType.Steam] = SiteManager.get_site_cls_by_id_type(
|
2022-12-29 23:57:02 -05:00
|
|
|
IdType.Steam
|
|
|
|
).url_to_id(steam_url)
|
2022-12-08 23:58:44 +00:00
|
|
|
return pd
|