Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multi-extension data format identification #621

Merged
merged 2 commits into from
Aug 30, 2021
Merged

Multi-extension data format identification #621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ea76688
making data_formats more robust; adding support for xx multi format i…
havok2063 May 17, 2021
a3e5d43
fixing linting issues
havok2063 May 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions jdaviz/core/data_formats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,21 @@

import astropy.io
from specutils.io.registers import identify_spectrum_format
from specutils import Spectrum1D
from specutils import SpectrumList

from jdaviz.core.config import list_configurations


# create a default file format to configuration mapping
default_mapping = {'JWST x1d': 'specviz', 'JWST s2d': 'specviz2d',
'JWST s3d': 'cubeviz', 'MaNGA cube': 'cubeviz',
'MaNGA rss': 'imviz'}
'MaNGA rss': 'specviz'}

formats_table = astropy.io.registry.get_formats(data_class=Spectrum1D,
readwrite='Read')
# get formats table for specutils objects
formats_table = astropy.io.registry.get_formats(readwrite='Read')
formats_table.add_index('Data class')
formats_table = formats_table.loc[['Spectrum1D', 'SpectrumList']]
formats_table.sort(['Data class', 'Format'])

file_to_config_mapping = {i: default_mapping.get(
i, 'specviz') for i in formats_table['Format']}
Expand Down Expand Up @@ -63,7 +66,7 @@ def get_valid_format(filename):
The recommended application configuration
"""

valid_file_format = identify_spectrum_format(filename)
valid_file_format = identify_spectrum_format(filename, SpectrumList)
ndim = guess_dimensionality(filename)

if valid_file_format:
Expand Down
57 changes: 38 additions & 19 deletions jdaviz/tests/test_data_formats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,54 +6,69 @@
from jdaviz.core.config import list_configurations


def create_sdss(n_dim=2):
def create_sdss(n_dim=2, **kwargs):
""" create a fake SDSS single fiber spectral fits file """
primary = fits.PrimaryHDU(header=fits.Header({'TELESCOP': "SDSS 2.5-M", 'FIBERID': 555}))
ext = fits.BinTableHDU(name='DATA', header=fits.Header({'TTYPE3': "ivar"}))
hdulist = fits.HDUList([primary, ext])
return hdulist
return fits.HDUList([primary, ext])


def create_manga(n_dim):
def create_manga(n_dim=None, **kwargs):
""" create a fake SDSS MaNGA fits file """
primary = fits.PrimaryHDU(header=fits.Header({'TELESCOP': "SDSS 2.5-M"}))
ext = fits.ImageHDU(name='FLUX', header=fits.Header({'INSTRUME': "MaNGA"}))
shape = (1,) if n_dim == 1 else (1, 2,) if n_dim == 2 else (1, 2, 3) if n_dim == 3 else (1,)
ext.data = np.empty(shape)
hdulist = fits.HDUList([primary, ext])
return hdulist
return fits.HDUList([primary, ext])


def create_jwst(n_dim):
def _create_bintable(name, ver=1):
return fits.BinTableHDU.from_columns(
[fits.Column(name='target', format='20A', array=np.ones(3)),
fits.Column(name='V_mag', format='E', array=np.ones(3))],
name=name, ver=ver
)


def create_jwst(n_dim=None, multi=None, ext=None):
""" create a fake JWST fits file """
primary = fits.PrimaryHDU(header=fits.Header({'TELESCOP': "JWST"}))
ext = fits.ImageHDU(name='ASDF')
asdf = fits.ImageHDU(name='ASDF')
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure what this does, but it doesn't seem right to create ASDF without using asdf or gwcs.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function creates a fake JWST file with mostly empty data extensions, just enough to get the format and structure right so I can test the various data format identification functions. Since the fake ASDF extension isn't really being used, tested or populated, I think it's ok to not make a fully realized one using asdf or gwcs.

I think if we wanted to create a more robust fake file generator to be used more generally down the line, then, yeah, we'd certainly want more realistic fake extensions.

sci = fits.ImageHDU(name='SCI')
exts = [primary, ext, sci]
exts = [primary]
if n_dim == 1:
ex1d = fits.ImageHDU(name='EXTRACT1D')
ex1d = _create_bintable(ext, ver=1)
exts.append(ex1d)
shape = (1,) if n_dim == 1 else (1, 2,) if n_dim == 2 else (1, 2, 3) if n_dim == 3 else (1,)
# create two more extensions
if multi:
for i in range(1, 3):
ex1d = _create_bintable(ext, ver=i)
exts.append(ex1d)
shape = (1,) if n_dim == 1 else (1, 2,) if n_dim == 2 else (1, 2, 3) if n_dim == 3 else (1, )
sci.data = np.empty(shape)
hdulist = fits.HDUList(exts)
return hdulist
if n_dim > 1:
exts.append(sci)
exts.append(asdf)
return fits.HDUList(exts)


def create_generic(n_dim=1):
def create_generic(n_dim=1, **kwargs):
""" create a generic fits file of dimension N """
primary = fits.PrimaryHDU()
ext = fits.ImageHDU(name='DATA')
shape = (1,) if n_dim == 1 else (1, 2,) if n_dim == 2 else (1, 2, 3) if n_dim == 3 else (1,)
ext.data = np.empty(shape)
hdulist = fits.HDUList([primary, ext])
return hdulist
return fits.HDUList([primary, ext])


data = {'MaNGA cube': {'ndim': 3, 'fxn': create_manga},
'MaNGA rss': {'ndim': 2, 'fxn': create_manga},
'JWST s3d': {'ndim': 3, 'fxn': create_jwst},
'JWST s2d': {'ndim': 2, 'fxn': create_jwst},
'JWST x1d': {'ndim': 1, 'fxn': create_jwst},
'JWST c1d': {'ndim': 1, 'fxn': create_jwst, 'ext': 'COMBINE1D'},
'JWST x1d multi': {'ndim': 1, 'fxn': create_jwst, 'multi': True},
'JWST c1d multi': {'ndim': 1, 'fxn': create_jwst, 'ext': 'COMBINE1D', 'multi': True},
'SDSS-III/IV spec': {'ndim': 2, 'fxn': create_sdss},
'generic 3d': {'ndim': 3, 'fxn': create_generic},
'generic 1d': {'ndim': 1, 'fxn': create_generic}
Expand All @@ -72,7 +87,9 @@ def _create_fits(name):
# generate fake fits.HDUList
ndim = data[name]['ndim']
fxn = data[name]['fxn']
hdulist = fxn(n_dim=ndim)
multi = data[name].get('multi', None)
ext = data[name].get('ext', 'EXTRACT1D')
hdulist = fxn(n_dim=ndim, ext=ext, multi=multi)
hdulist.writeto(filepath)
return str(filepath)

Expand All @@ -81,10 +98,13 @@ def _create_fits(name):

@pytest.mark.parametrize('name, expconf',
[('MaNGA cube', 'cubeviz'),
('MaNGA rss', 'imviz'),
('MaNGA rss', 'specviz'),
('JWST s3d', 'cubeviz'),
('JWST s2d', 'specviz2d'),
('JWST x1d', 'specviz'),
('JWST c1d', 'specviz'),
('JWST x1d multi', 'specviz'),
('JWST c1d multi', 'specviz'),
('SDSS-III/IV spec', 'specviz'),
('generic 3d', 'cubeviz'),
('generic 1d', 'specviz')])
Expand All @@ -108,7 +128,6 @@ def test_list_configurations():

@pytest.mark.parametrize('name, expconf, expstat',
[('MaNGA cube', 'cubeviz', 'Success: Valid Format'),
('MaNGA rss', 'imviz', 'Success: Valid Format'),
('generic 1d', 'specviz', 'Error: Cannot determine format of '
'the file to load.')])
def test_identify_data(create_fake_fits, name, expconf, expstat):
Expand Down