From 03556edb1d533739fb60345e2026b3bf804e484d Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Fri, 18 Jan 2019 14:42:46 +0100 Subject: [PATCH] Catch URLError and HTTPError exception for timed out and non-existing tiles Signed-off-by: Adeel Ahmad --- hips/tiles/fetch.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hips/tiles/fetch.py b/hips/tiles/fetch.py index cdd429b..0382dd5 100644 --- a/hips/tiles/fetch.py +++ b/hips/tiles/fetch.py @@ -1,4 +1,5 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import socket import asyncio import urllib.request import concurrent.futures @@ -84,10 +85,19 @@ def fetch_tiles(tile_metas: List[HipsTileMeta], hips_survey: HipsSurveyPropertie return tiles + def fetch_tile_urllib(url: str, timeout: float) -> dict: """Fetch a HiPS tile asynchronously.""" - with urllib.request.urlopen(url, timeout=timeout) as conn: - return {'raw_data': conn.read(), 'url': url} + try: + with urllib.request.urlopen(url, timeout=timeout) as conn: + return {'raw_data': conn.read(), 'url': url} + except urllib.error.HTTPError as error: + if error.code == 404: + error.msg = f'Tile not found at:\n{url}' + raise + except urllib.error.URLError as error: + if isinstance(error.reason, socket.timeout): + raise TimeoutError(f'The server timed out while fetching the tile at:\n{url}') def tiles_urllib(tile_urls: List[str], hips_survey: HipsSurveyProperties, @@ -96,6 +106,7 @@ def tiles_urllib(tile_urls: List[str], hips_survey: HipsSurveyProperties, with concurrent.futures.ThreadPoolExecutor(max_workers=n_parallel) as executor: return list(executor.map(fetch_tile_urllib, tile_urls, [timeout] * len(tile_urls))) + async def fetch_tile_aiohttp(url: str, session, timeout: float) -> dict: """Fetch a HiPS tile asynchronously using aiohttp.""" async with session.get(url, timeout=timeout) as response: