Skip to content

Commit

Permalink
Handle case for missing HiPS tiles
Browse files Browse the repository at this point in the history
Signed-off-by: Adeel Ahmad <[email protected]>
  • Loading branch information
adl1995 committed Feb 6, 2019
1 parent 030aa50 commit 3d7f49e
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 13 deletions.
2 changes: 1 addition & 1 deletion hips/draw/ui.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def plot(self, show_grid: bool = False) -> None:
def report(self) -> None:
"""Print a brief report for the fetched data."""

print (
print(
f"Time for fetching tiles = {self.stats['fetch_time']} seconds\n"
f"Time for drawing tiles = {self.stats['draw_time']} seconds\n"
f"Total memory consumed = {self.stats['consumed_memory'] / 1e6} MB\n"
Expand Down
14 changes: 10 additions & 4 deletions hips/tiles/fetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,11 @@ def fetch_tiles(tile_metas: List[HipsTileMeta], hips_survey: HipsSurveyPropertie

tiles = []
for idx, response in enumerate(response_all):
tiles.append(HipsTile(tile_metas[idx], response['raw_data']))
try:
response['raw_data']
tiles.append(HipsTile(tile_metas[idx], response['raw_data']))
except KeyError:
tiles.append(HipsTile(tile_metas[idx], b'', is_missing=True))

return tiles

Expand All @@ -92,12 +96,14 @@ def fetch_tile_urllib(url: str, timeout: float) -> dict:
with urllib.request.urlopen(url, timeout=timeout) as conn:
return {'raw_data': conn.read(), 'url': url}
except urllib.error.HTTPError as error:
# If the tile is missing, enable the `is_missing` flag in HipsTile.
if error.code == 404:
error.msg = f'Tile not found at:\n{url}'
raise
print(f'Tile not found at:\n{url}')
return {'is_missing': True}
except urllib.error.URLError as error:
if isinstance(error.reason, socket.timeout):
raise TimeoutError(f'The server timed out while fetching the tile at:\n{url}')
print(f'The server timed out while fetching the tile at:\n{url}')
return {'is_missing': True}


def tiles_urllib(tile_urls: List[str], hips_survey: HipsSurveyProperties,
Expand Down
36 changes: 28 additions & 8 deletions hips/tiles/tile.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Licensed under a 3-clause BSD style license - see LICENSE.rst
from typing import List, Tuple
from copy import deepcopy
import socket
import warnings
import urllib.request
from io import BytesIO
Expand Down Expand Up @@ -154,6 +155,8 @@ class HipsTile:
Metadata of HiPS tile
raw_data : `bytes`
Raw data (copy of bytes from file)
is_missing : `bool`
To check whether the tile is missing or not
Examples
--------
Expand All @@ -175,9 +178,10 @@ class HipsTile:
int16
"""

def __init__(self, meta: HipsTileMeta, raw_data: bytes) -> None:
def __init__(self, meta: HipsTileMeta, raw_data: bytes, is_missing: bool = False) -> None:
self.meta = meta
self.raw_data = raw_data
self.is_missing = is_missing
self._data = None

def __eq__(self, other: "HipsTile") -> bool:
Expand Down Expand Up @@ -259,8 +263,11 @@ def data(self) -> np.ndarray:
See the `to_numpy` function.
"""
if self._data is None:
if self._data is None and not self.is_missing:
self._data = self.to_numpy(self.raw_data, self.meta.file_format)
elif self.is_missing:
self._data = np.zeros((compute_image_shape(self.meta.width, self.meta.width, self.meta.file_format)),
dtype=np.uint8)

return self._data

Expand Down Expand Up @@ -295,6 +302,7 @@ def to_numpy(raw_data: bytes, fmt: str) -> np.ndarray:
elif fmt in {"jpg", "png"}:
with Image.open(bio) as image:
data = np.array(image)

# Flip tile to be consistent with FITS orientation
data = np.flipud(data)
else:
Expand All @@ -316,8 +324,12 @@ def read(cls, meta: HipsTileMeta, filename: str = None) -> "HipsTile":
filename : str
Filename
"""
raw_data = Path(filename).read_bytes()
return cls(meta, raw_data)
if Path(filename).exists():
return cls(meta, Path(filename).read_bytes())
else:
print(f'Tile not found at:\n{filename}')
return cls(meta, b'', is_missing=True)


@classmethod
def fetch(cls, meta: HipsTileMeta, url: str) -> "HipsTile":
Expand All @@ -330,10 +342,18 @@ def fetch(cls, meta: HipsTileMeta, url: str) -> "HipsTile":
url : str
URL containing HiPS tile
"""
with urllib.request.urlopen(url) as response:
raw_data = response.read()

return cls(meta, raw_data)
try:
with urllib.request.urlopen(url, timeout=10) as response:
raw_data = response.read()
return cls(meta, raw_data)
except urllib.error.HTTPError as error:
if error.code == 404:
print(f'Tile not found at:\n{url}')
return cls(meta, b'', is_missing=True)
except urllib.error.URLError as error:
if isinstance(error.reason, socket.timeout):
print(f'The server timed out while fetching the tile at:\n{url}')
return cls(meta, b'', is_missing=True)

def write(self, filename: str) -> None:
"""Write to file.
Expand Down

0 comments on commit 3d7f49e

Please sign in to comment.