lib.itmens/common/scrapers/igdb.py

import requests
import re
from common.models import SourceSiteEnum
from games.models import Game
from games.forms import GameForm
from django.conf import settings
from common.scraper import *
from igdb.wrapper import IGDBWrapper
import json
import datetime


wrapper = IGDBWrapper(settings.IGDB_CLIENT_ID, settings.IGDB_ACCESS_TOKEN)


class IgdbGameScraper(AbstractScraper):
    site_name = SourceSiteEnum.IGDB.value
    host = 'https://www.igdb.com/'
    data_class = Game
    form_class = GameForm
    regex = re.compile(r"https://www\.igdb\.com/games/([a-zA-Z0-9\-_]+)")

    def scrape_steam(self, steam_url):
        r = json.loads(wrapper.api_request('websites', f'fields *, game.*; where url = "{steam_url}";'))
        if not r:
            raise ValueError("Cannot find steam url in IGDB")
        r = sorted(r, key=lambda w: w['game']['id'])
        return self.scrape(r[0]['game']['url'])

    def scrape(self, url):
        m = self.regex.match(url)
        if m:
            effective_url = m[0]
        else:
            raise ValueError("not valid url")
        effective_url = m[0]
        slug = m[1]
        fields = '*, cover.url, genres.name, platforms.name, involved_companies.*, involved_companies.company.name'
        r = json.loads(wrapper.api_request('games', f'fields {fields}; where url = "{effective_url}";'))[0]
        brief = r['summary'] if 'summary' in r else ''
        brief += "\n\n" + r['storyline'] if 'storyline' in r else ''
        developer = None
        publisher = None
        release_date = None
        genre = None
        platform = None
        if 'involved_companies' in r:
            developer = next(iter([c['company']['name'] for c in r['involved_companies'] if c['developer'] == True]), None)
            publisher = next(iter([c['company']['name'] for c in r['involved_companies'] if c['publisher'] == True]), None)
        if 'platforms' in r:
            ps = sorted(r['platforms'], key=lambda p: p['id'])
            platform = [(p['name'] if p['id'] != 6 else 'Windows') for p in ps]
        if 'first_release_date' in r:
            release_date = datetime.datetime.fromtimestamp(r['first_release_date'], datetime.timezone.utc)
        if 'genres' in r:
            genre = [g['name'] for g in r['genres']]
        other_info = {'igdb_id': r['id']}
        websites = json.loads(wrapper.api_request('websites', f'fields *; where game.url = "{effective_url}";'))
        for website in websites:
            if website['category'] == 1:
                other_info['official_site'] = website['url']
            elif website['category'] == 13:
                other_info['steam_url'] = website['url']
        data = {
            'title': r['name'],
            'other_title': None,
            'developer': developer,
            'publisher': publisher,
            'release_date': release_date,
            'genre': genre,
            'platform': platform,
            'brief': brief,
            'other_info': other_info,
            'source_site': self.site_name,
            'source_url': self.get_effective_url(url),
        }
        raw_img, ext = self.download_image('https:' + r['cover']['url'].replace('t_thumb', 't_cover_big'), url)

        self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext
        return data, raw_img

    @classmethod
    def get_effective_url(cls, raw_url):
        m = cls.regex.match(raw_url)
        if m:
            return m[0]
        else:
            return None