Skip to content

Commit

Permalink
Catch URLError and HTTPError exception for timed out and non-existing…
Browse files Browse the repository at this point in the history
… tiles

Signed-off-by: Adeel Ahmad <[email protected]>
  • Loading branch information
Adeel Ahmad authored and adl1995 committed Jan 18, 2019
1 parent 6c8874c commit 03556ed
Showing 1 changed file with 13 additions and 2 deletions.
15 changes: 13 additions & 2 deletions hips/tiles/fetch.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Licensed under a 3-clause BSD style license - see LICENSE.rst
import socket
import asyncio
import urllib.request
import concurrent.futures
Expand Down Expand Up @@ -84,10 +85,19 @@ def fetch_tiles(tile_metas: List[HipsTileMeta], hips_survey: HipsSurveyPropertie

return tiles


def fetch_tile_urllib(url: str, timeout: float) -> dict:
"""Fetch a HiPS tile asynchronously."""
with urllib.request.urlopen(url, timeout=timeout) as conn:
return {'raw_data': conn.read(), 'url': url}
try:
with urllib.request.urlopen(url, timeout=timeout) as conn:
return {'raw_data': conn.read(), 'url': url}
except urllib.error.HTTPError as error:
if error.code == 404:
error.msg = f'Tile not found at:\n{url}'
raise
except urllib.error.URLError as error:
if isinstance(error.reason, socket.timeout):
raise TimeoutError(f'The server timed out while fetching the tile at:\n{url}')


def tiles_urllib(tile_urls: List[str], hips_survey: HipsSurveyProperties,
Expand All @@ -96,6 +106,7 @@ def tiles_urllib(tile_urls: List[str], hips_survey: HipsSurveyProperties,
with concurrent.futures.ThreadPoolExecutor(max_workers=n_parallel) as executor:
return list(executor.map(fetch_tile_urllib, tile_urls, [timeout] * len(tile_urls)))


async def fetch_tile_aiohttp(url: str, session, timeout: float) -> dict:
"""Fetch a HiPS tile asynchronously using aiohttp."""
async with session.get(url, timeout=timeout) as response:
Expand Down

0 comments on commit 03556ed

Please sign in to comment.