workaround race condition
This commit is contained in:
parent
b79971f0e0
commit
3164544f95
2 changed files with 10 additions and 2 deletions
|
@ -84,6 +84,9 @@ class AbstractSite:
|
||||||
self.url = self.id_to_url(self.id_value) if self.id_value else None
|
self.url = self.id_to_url(self.id_value) if self.id_value else None
|
||||||
self.resource = None
|
self.resource = None
|
||||||
|
|
||||||
|
def clear_cache(self):
|
||||||
|
self.resource = None
|
||||||
|
|
||||||
def get_resource(self) -> ExternalResource:
|
def get_resource(self) -> ExternalResource:
|
||||||
if not self.resource:
|
if not self.resource:
|
||||||
self.resource = ExternalResource.objects.filter(url=self.url).first()
|
self.resource = ExternalResource.objects.filter(url=self.url).first()
|
||||||
|
|
|
@ -86,10 +86,15 @@ def _get_or_create_item(item_obj) -> Item | None:
|
||||||
if not site:
|
if not site:
|
||||||
logger.warning(f"Site not found for {url}")
|
logger.warning(f"Site not found for {url}")
|
||||||
return None
|
return None
|
||||||
site.get_resource_ready()
|
try:
|
||||||
|
site.get_resource_ready()
|
||||||
|
except Exception:
|
||||||
|
# occationally race condition happens and resource is fetched by another process,
|
||||||
|
# so we clear cache to retry matching the resource
|
||||||
|
site.clear_cache()
|
||||||
item = site.get_item()
|
item = site.get_item()
|
||||||
if not item:
|
if not item:
|
||||||
logger.warning(f"Item not fetched for {url}")
|
logger.error(f"Item not fetched for {url}")
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Add table
Reference in a new issue