2021-08-04 01:39:19 +00:00
|
|
|
import logging
|
2021-08-23 11:36:19 +00:00
|
|
|
import multiprocessing
|
2021-08-04 01:39:19 +00:00
|
|
|
|
|
|
|
import requests
|
2021-08-04 01:41:14 +00:00
|
|
|
from PIL import Image
|
2021-08-04 01:39:19 +00:00
|
|
|
|
2021-08-24 15:29:12 +00:00
|
|
|
from .language import Language
|
2021-08-23 14:55:41 +00:00
|
|
|
from .utils import RESOLUTION
|
2021-08-17 15:37:28 +00:00
|
|
|
|
2021-08-24 15:29:12 +00:00
|
|
|
# constants
|
|
|
|
FALLBACK_LANGUAGE = Language("en")
|
|
|
|
|
2021-08-04 01:39:19 +00:00
|
|
|
|
2021-08-23 11:36:19 +00:00
|
|
|
class ImageLoader:
|
|
|
|
@classmethod
|
2021-08-24 15:29:12 +00:00
|
|
|
def _load_inner(cls, url_parts: tuple[str, str, str]) -> Image.Image:
|
2021-08-04 01:39:19 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
2021-08-24 15:29:12 +00:00
|
|
|
base_url, code, lang_suffix = url_parts
|
|
|
|
|
|
|
|
# put together image url
|
|
|
|
url = base_url.format(code, lang_suffix)
|
|
|
|
logger.info(f"trying image {url}")
|
2021-08-04 16:36:23 +00:00
|
|
|
|
2021-08-23 11:36:19 +00:00
|
|
|
# fetch image (retry on fail)
|
|
|
|
while True:
|
|
|
|
try:
|
2021-08-24 15:29:12 +00:00
|
|
|
res = requests.get(url, stream=True)
|
|
|
|
break
|
2021-08-04 18:15:38 +00:00
|
|
|
|
2021-08-24 15:29:12 +00:00
|
|
|
except requests.RequestException:
|
2021-08-23 11:36:19 +00:00
|
|
|
pass
|
2021-08-04 01:39:19 +00:00
|
|
|
|
2021-08-24 15:29:12 +00:00
|
|
|
# if rejected, substitute the english version
|
|
|
|
if not res.status_code == 200:
|
|
|
|
logger.warning(f"falling back to english version of {url}")
|
|
|
|
return cls._load_inner((base_url, code, FALLBACK_LANGUAGE.image_suffix))
|
|
|
|
|
|
|
|
# unify images
|
|
|
|
image = Image.open(res.raw)
|
|
|
|
image.convert(mode="RGB")
|
|
|
|
return image.resize(RESOLUTION, Image.BICUBIC)
|
|
|
|
|
2021-08-04 01:39:19 +00:00
|
|
|
@classmethod
|
2021-08-24 15:29:12 +00:00
|
|
|
def load(cls, urls_parts: list[tuple[str, str, str]], num_threads: int) -> list[Image.Image]:
|
2021-08-23 11:36:19 +00:00
|
|
|
with multiprocessing.Pool(num_threads) as p:
|
2021-08-24 15:29:12 +00:00
|
|
|
return p.map(cls._load_inner, urls_parts)
|