From 1a8e6f3c4f22a5fe0a90c127e505dd31614dee9f Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 15 May 2018 13:10:08 +0200 Subject: [PATCH] Remove telescope specific code from protozfits (#39) * try to execute these tests first * remove number of tests for some time * these tests were duplicate * rename file to execute these tests 2nd * do not test Digicamfile anymore - prior to remove the file * remove DigicamFile from __init__ - prior to remove the file * just import digicam ... not doing anything with it * made a mistake in importing digicam * I try to import, what digicam.py imported * remove imports, I assume are harmless * binary search ... I think these can go * okay I guess this is it now * add some explanation * remove digicam specific files. * re-add all the tests for linux and OSX Py3.6 again * rename SimpleFile back into File * bump version * remove old __init__ * simple.py *is* __init__.py * re-add the few parts from old init, which were missing in simply.py * pep8 * try if it helps to import rawzfitsreader as the very first thing * maybe here is enough? * some explanation * this test was failing for some time ... maybe it runs now? * this is still seg faulting ... not soo nice * turning around imports to prove my comment in the PR discussion * add more fine grained tests * swapping import statements again. so everything works --- protozfits/VERSION | 2 +- protozfits/__init__.py | 237 +++++++++++++- protozfits/digicam.py | 138 -------- protozfits/patch_ids.py | 67 ---- protozfits/simple.py | 229 -------------- ...fitsreader.py => test_1_rawzfitsreader.py} | 39 ++- protozfits/tests/test_R1_example_file.py | 4 +- protozfits/tests/test_read_raw_events.py | 297 ------------------ 8 files changed, 271 insertions(+), 742 deletions(-) delete mode 100644 protozfits/digicam.py delete mode 100644 protozfits/patch_ids.py delete mode 100644 protozfits/simple.py rename protozfits/tests/{test_rawzfitsreader.py => test_1_rawzfitsreader.py} (88%) delete mode 100644 protozfits/tests/test_read_raw_events.py diff --git a/protozfits/VERSION b/protozfits/VERSION index a315b1b..6d7de6e 100644 --- a/protozfits/VERSION +++ b/protozfits/VERSION @@ -1 +1 @@ -1.0.1a0 +1.0.2 diff --git a/protozfits/__init__.py b/protozfits/__init__.py index 58f5618..86a31c5 100644 --- a/protozfits/__init__.py +++ b/protozfits/__init__.py @@ -1,15 +1,240 @@ from pkg_resources import resource_string -from .digicam import File as DigicamFile -from .simple import File as SimpleFile +from enum import Enum +from collections import namedtuple +from warnings import warn +import numpy as np +from astropy.io import fits + +# Beware: +# for some reason rawzfitsreader needs to be imported before +# GeneratedProtocolMessageType +from . import rawzfitsreader +from google.protobuf.pyext.cpp_message import GeneratedProtocolMessageType +from .CoreMessages_pb2 import AnyArray from .any_array_to_numpy import any_array_to_numpy -from .simple import make_namedtuple -__version__ = resource_string('protozfits', 'VERSION').decode().strip() +from . import L0_pb2 +from . import R1_pb2 +from . import R1_LSTCam_pb2 +from . import R1_NectarCam_pb2 +from . import R1_DigiCam_pb2 + +__version__ = resource_string('protozfits', 'VERSION').decode().strip() __all__ = [ - 'DigicamFile', - 'SimpleFile', + 'File', 'make_namedtuple', 'any_array_to_numpy', ] + +pb2_modules = { + 'L0': L0_pb2, + 'R1': R1_pb2, + 'R1_DigiCam': R1_DigiCam_pb2, + 'R1_NectarCam': R1_NectarCam_pb2, + 'R1_LSTCam': R1_LSTCam_pb2, +} + + +def get_class_from_PBFHEAD(pbfhead): + module_name, class_name = pbfhead.split('.') + return getattr(pb2_modules[module_name], class_name) + + +class File: + instances = 0 + + def __init__(self, path, pure_protobuf=False): + File.instances += 1 + if File.instances > 1: + warn('''\ + Multiple open zfits files at the same time are not supported. + Reading from mutliple open tables at the same time will reset these + tables continously and you will read always the same events. + ''') + Table._Table__last_opened = None + bintable_descriptions = detect_bintables(path) + for btd in bintable_descriptions: + self.__dict__[btd.extname] = Table(btd, pure_protobuf) + + def __repr__(self): + return "%s(%r)" % ( + self.__class__.__name__, + self.__dict__ + ) + + def __enter__(self): + return self + + def __exit__(self, type, value, tb): + self.close() + + def close(self): + File.instances -= 1 + + def __del__(self): + self.close() + + +BinTableDescription = namedtuple( + 'BinTableDescription', + [ + 'path', + 'index', + 'extname', + 'pbfhead', + 'znaxis2', + 'header', + ] +) + + +def detect_bintables(path): + fitsfile = fits.open(path) + bintables = [ + BinTableDescription( + path=path, + index=hdu_id, + extname=hdu.header['EXTNAME'], + pbfhead=hdu.header['PBFHEAD'], + znaxis2=hdu.header['ZNAXIS2'], + header=hdu.header + ) + for hdu_id, hdu in enumerate(fitsfile) + if 'XTENSION' in hdu.header and hdu.header['XTENSION'] == 'BINTABLE' + ] + fitsfile.close() + return bintables + + +class Table: + '''Iterable Table + ''' + __last_opened = None + '''the rawzfitsreader has a "bug" which is: It cannot have two open + hdus. So when the File would open all N tables at construction time, + every `rawzfitsreader.readEvent()` would act on the last opened table. + + So the Tables remember which hdu was opened last, and if it was not them. + They open it. + ''' + + def __init__(self, desc, pure_protobuf=False): + ''' + desc: BinTableDescription + ''' + self.__desc = desc + self.__pbuf_class = get_class_from_PBFHEAD(desc.pbfhead) + self.header = self.__desc.header + self.pure_protobuf = pure_protobuf + + def __len__(self): + return self.__desc.znaxis2 + + def __iter__(self): + return self + + def __next__(self): + if not Table.__last_opened == self.__desc: + rawzfitsreader.open(self.__desc.path+":"+self.__desc.extname) + Table.__last_opened = self.__desc + row = self.__pbuf_class() + try: + row.ParseFromString(rawzfitsreader.readEvent()) + except EOFError: + raise StopIteration + + if not self.pure_protobuf: + return make_namedtuple(row) + else: + return row + + def __repr__(self): + return '{cn}({d.znaxis2}x{d.pbfhead})'.format( + cn=self.__class__.__name__, + d=self.__desc + ) + + +def make_namedtuple(message): + namedtuple_class = named_tuples[message.__class__] + return namedtuple_class._make( + message_getitem(message, name) + for name in namedtuple_class._fields + ) + + +def message_getitem(msg, name): + value = msg.__getattribute__(name) + if isinstance(value, AnyArray): + value = any_array_to_numpy(value) + elif (msg.__class__, name) in enum_types: + value = enum_types[(msg.__class__, name)](value) + elif type(value) in named_tuples: + value = make_namedtuple(value) + return value + + +messages = set() +for module in pb2_modules.values(): + for name in dir(module): + thing = getattr(module, name) + if isinstance(thing, GeneratedProtocolMessageType): + messages.add(thing) + + +def namedtuple_repr2(self): + '''a nicer repr for big namedtuples containing big numpy arrays''' + old_print_options = np.get_printoptions() + np.set_printoptions(precision=3, threshold=50, edgeitems=2) + delim = '\n ' + s = self.__class__.__name__ + '(' + delim + + s += delim.join([ + '{0}={1}'.format( + key, + repr( + getattr(self, key) + ).replace('\n', delim) + ) + for key in self._fields + ]) + s += ')' + np.set_printoptions(**old_print_options) + return s + + +def nt(m): + '''create namedtuple class from protobuf.message type''' + _nt = namedtuple( + m.__name__, + list(m.DESCRIPTOR.fields_by_name) + ) + _nt.__repr__ = namedtuple_repr2 + return _nt + + +named_tuples = {m: nt(m) for m in messages} + +enum_types = {} +for m in messages: + d = m.DESCRIPTOR + for field in d.fields: + if field.enum_type is not None: + et = field.enum_type + enum = Enum( + field.name, + zip(et.values_by_name, et.values_by_number) + ) + enum_types[(m, field.name)] = enum + + +def rewind_table(): + # rawzfitsreader.rewindTable() has a bug at the moment, + # it always throws a SystemError + # we let that one pass + try: + rawzfitsreader.rewindTable() + except SystemError: + pass diff --git a/protozfits/digicam.py b/protozfits/digicam.py deleted file mode 100644 index f62f61d..0000000 --- a/protozfits/digicam.py +++ /dev/null @@ -1,138 +0,0 @@ -from os.path import isfile -import warnings -import numpy as np -from astropy.utils.decorators import lazyproperty - -from . import rawzfitsreader -from google.protobuf.pyext.cpp_message import GeneratedProtocolMessageType -from .patch_ids import PATCH_ID_INPUT_SORT_IDS, PATCH_ID_OUTPUT_SORT_IDS -from .any_array_to_numpy import any_array_to_numpy -from .L0_pb2 import CameraEvent - - -class File: - - def __init__(self, fname): - if not isfile(fname): - raise FileNotFoundError(fname) - self.fname = fname - rawzfitsreader.open(self.fname + ":Events") - self.numrows = rawzfitsreader.getNumRows() - self.run_id = 0 - - def __iter__(self): - return self - - def __next__(self): - event = CameraEvent() - try: - event.ParseFromString(rawzfitsreader.readEvent()) - return Event(event, self.run_id) - except EOFError: - raise StopIteration - - def list_tables(self): - return rawzfitsreader.listAllTables(self.fname) - - def rewind_table(self): - # Rewind the current reader. Go to the beginning of the table. - rawzfitsreader.rewindTable() - - -class Event: - _sort_ids = None - - def __init__(self, event, run_id): - if type(type(event)) is GeneratedProtocolMessageType: - trafo = any_array_to_numpy - else: - trafo = no_trafo - - self.run_id = run_id - self._event = event - - self.pixel_ids = trafo( - self._event.hiGain.waveforms.pixelsIndices) - if Event._sort_ids is None: - Event._sort_ids = np.argsort(self.pixel_ids) - self.n_pixels = len(self.pixel_ids) - self._samples = ( - trafo(self._event.hiGain.waveforms.samples) - ).reshape(self.n_pixels, -1) - self.baseline = self.unsorted_baseline[Event._sort_ids] - self.telescope_id = self._event.telescopeID - self.event_number = self._event.eventNumber - self.central_event_gps_time = ( - self._event.trig.timeSec * 1E9 + self._event.trig.timeNanoSec - ) - self.local_time = ( - self._event.local_time_sec * 1E9 + self._event.local_time_nanosec - ) - self.event_number_array = self._event.arrayEvtNum - self.camera_event_type = self._event.event_type - self.array_event_type = self._event.eventType - self.num_gains = self._event.num_gains - self.num_channels = self._event.head.numGainChannels - self.num_samples = self._samples.shape[1] - self.pixel_flags = trafo( - self._event.pixels_flags)[Event._sort_ids] - self.adc_samples = self._samples[Event._sort_ids] - - top7 = trafo(self._event.trigger_output_patch7) - if len(top7) > 0: - self.trigger_output_patch7 = _prepare_trigger_output(top7) - else: - warnings.warn('trigger_output_patch7 does not exist: --> nan') - self.trigger_output_patch7 = np.zeros( - (432, self.num_samples)) * np.nan - - top19 = trafo(self._event.trigger_output_patch19) - if len(top19) > 0: - self.trigger_output_patch19 = _prepare_trigger_output(top19) - else: - warnings.warn('trigger_output_patch19 does not exist: --> nan') - self.trigger_output_patch19 = np.zeros( - (432, self.num_samples)) * np.nan - - tit = trafo(self._event.trigger_input_traces) - if len(tit) > 0: - self.trigger_input_traces = _prepare_trigger_input(tit) - else: - warnings.warn('trigger_input_traces does not exist: --> nan') - self.trigger_input_traces = np.zeros( - (432, self.num_samples)) * np.nan - - @lazyproperty - def unsorted_baseline(self): - try: - return any_array_to_numpy(self._event.hiGain.waveforms.baselines) - except ValueError: - warnings.warn(( - "Could not read `hiGain.waveforms.baselines` for event:{0}" - "of run_id:{1}".format(self.event_number, self.run_id) - )) - return np.ones(len(self.pixel_ids)) * np.nan - - -def _prepare_trigger_input(_a): - A, B = 3, 192 - cut = 144 - _a = _a.reshape(-1, A) - _a = _a.reshape(-1, A, B) - _a = _a[..., :cut] - _a = _a.reshape(_a.shape[0], -1) - _a = _a.T - _a = _a[PATCH_ID_INPUT_SORT_IDS] - return _a - - -def _prepare_trigger_output(_a): - A, B, C = 3, 18, 8 - _a = np.unpackbits(_a.reshape(-1, A, B, 1), axis=-1) - _a = _a[..., ::-1] - _a = _a.reshape(-1, A*B*C).T - return _a[PATCH_ID_OUTPUT_SORT_IDS] - - -def no_trafo(value): - return value diff --git a/protozfits/patch_ids.py b/protozfits/patch_ids.py deleted file mode 100644 index 4607cf9..0000000 --- a/protozfits/patch_ids.py +++ /dev/null @@ -1,67 +0,0 @@ -import numpy as np - -PATCH_ID_INPUT = [ - 204, 216, 180, 192, 229, 241, 205, 217, 254, 266, 230, 242, - 279, 291, 255, 267, 304, 316, 280, 292, 329, 341, 305, 317, 156, 168, 132, - 144, 181, 193, 157, 169, 206, 218, 182, 194, 231, 243, 207, 219, 256, 268, - 232, 244, 281, 293, 257, 269, 108, 120, 84, 96, 133, 145, 109, 121, 158, - 170, 134, 146, 183, 195, 159, 171, 208, 220, 184, 196, 233, 245, 209, 221, - 60, 72, 40, 50, 85, 97, 61, 73, 110, 122, 86, 98, 135, 147, 111, 123, 160, - 172, 136, 148, 185, 197, 161, 173, 24, 32, 12, 18, 41, 51, 25, 33, 62, 74, - 42, 52, 87, 99, 63, 75, 112, 124, 88, 100, 137, 149, 113, 125, 4, 8, 0, 2, - 13, 19, 5, 9, 26, 34, 14, 20, 43, 53, 27, 35, 64, 76, 44, 54, 89, 101, 65, - 77, 228, 239, 240, 252, 251, 262, 263, 275, 274, 285, 286, 298, 297, 308, - 309, 321, 320, 331, 332, 344, 343, 354, 355, 366, 253, 264, 265, 277, 276, - 287, 288, 300, 299, 310, 311, 323, 322, 333, 334, 346, 345, 356, 357, 368, - 367, 377, 378, 387, 278, 289, 290, 302, 301, 312, 313, 325, 324, 335, 336, - 348, 347, 358, 359, 370, 369, 379, 380, 389, 388, 396, 397, 404, 303, 314, - 315, 327, 326, 337, 338, 350, 349, 360, 361, 372, 371, 381, 382, 391, 390, - 398, 399, 406, 405, 411, 412, 417, 328, 339, 340, 352, 351, 362, 363, 374, - 373, 383, 384, 393, 392, 400, 401, 408, 407, 413, 414, 419, 418, 422, 423, - 426, 353, 364, 365, 376, 375, 385, 386, 395, 394, 402, 403, 410, 409, 415, - 416, 421, 420, 424, 425, 428, 427, 429, 430, 431, 215, 191, 227, 203, 167, - 143, 179, 155, 119, 95, 131, 107, 71, 49, 83, 59, 31, 17, 39, 23, 7, 1, - 11, 3, 238, 214, 250, 226, 190, 166, 202, 178, 142, 118, 154, 130, 94, 70, - 106, 82, 48, 30, 58, 38, 16, 6, 22, 10, 261, 237, 273, 249, 213, 189, 225, - 201, 165, 141, 177, 153, 117, 93, 129, 105, 69, 47, 81, 57, 29, 15, 37, - 21, 284, 260, 296, 272, 236, 212, 248, 224, 188, 164, 200, 176, 140, 116, - 152, 128, 92, 68, 104, 80, 46, 28, 56, 36, 307, 283, 319, 295, 259, 235, - 271, 247, 211, 187, 223, 199, 163, 139, 175, 151, 115, 91, 127, 103, 67, - 45, 79, 55, 330, 306, 342, 318, 282, 258, 294, 270, 234, 210, 246, 222, - 186, 162, 198, 174, 138, 114, 150, 126, 90, 66, 102, 78 -] - -PATCH_ID_INPUT_SORT_IDS = np.argsort(PATCH_ID_INPUT) - -PATCH_ID_OUTPUT = [ - 204, 216, 229, 241, 254, 266, 279, 291, 304, 316, 329, - 341, 180, 192, 205, 217, 230, 242, 255, 267, 280, 292, 305, 317, 156, 168, - 181, 193, 206, 218, 231, 243, 256, 268, 281, 293, 132, 144, 157, 169, 182, - 194, 207, 219, 232, 244, 257, 269, 108, 120, 133, 145, 158, 170, 183, 195, - 208, 220, 233, 245, 84, 96, 109, 121, 134, 146, 159, 171, 184, 196, 209, - 221, 60, 72, 85, 97, 110, 122, 135, 147, 160, 172, 185, 197, 40, 50, 61, - 73, 86, 98, 111, 123, 136, 148, 161, 173, 24, 32, 41, 51, 62, 74, 87, 99, - 112, 124, 137, 149, 12, 18, 25, 33, 42, 52, 63, 75, 88, 100, 113, 125, 4, - 8, 13, 19, 26, 34, 43, 53, 64, 76, 89, 101, 0, 2, 5, 9, 14, 20, 27, 35, - 44, 54, 65, 77, 228, 239, 251, 262, 274, 285, 297, 308, 320, 331, 343, - 354, 240, 252, 263, 275, 286, 298, 309, 321, 332, 344, 355, 366, 253, 264, - 276, 287, 299, 310, 322, 333, 345, 356, 367, 377, 265, 277, 288, 300, 311, - 323, 334, 346, 357, 368, 378, 387, 278, 289, 301, 312, 324, 335, 347, 358, - 369, 379, 388, 396, 290, 302, 313, 325, 336, 348, 359, 370, 380, 389, 397, - 404, 303, 314, 326, 337, 349, 360, 371, 381, 390, 398, 405, 411, 315, 327, - 338, 350, 361, 372, 382, 391, 399, 406, 412, 417, 328, 339, 351, 362, 373, - 383, 392, 400, 407, 413, 418, 422, 340, 352, 363, 374, 384, 393, 401, 408, - 414, 419, 423, 426, 353, 364, 375, 385, 394, 402, 409, 415, 420, 424, 427, - 429, 365, 376, 386, 395, 403, 410, 416, 421, 425, 428, 430, 431, 215, 191, - 167, 143, 119, 95, 71, 49, 31, 17, 7, 1, 227, 203, 179, 155, 131, 107, 83, - 59, 39, 23, 11, 3, 238, 214, 190, 166, 142, 118, 94, 70, 48, 30, 16, 6, - 250, 226, 202, 178, 154, 130, 106, 82, 58, 38, 22, 10, 261, 237, 213, 189, - 165, 141, 117, 93, 69, 47, 29, 15, 273, 249, 225, 201, 177, 153, 129, 105, - 81, 57, 37, 21, 284, 260, 236, 212, 188, 164, 140, 116, 92, 68, 46, 28, - 296, 272, 248, 224, 200, 176, 152, 128, 104, 80, 56, 36, 307, 283, 259, - 235, 211, 187, 163, 139, 115, 91, 67, 45, 319, 295, 271, 247, 223, 199, - 175, 151, 127, 103, 79, 55, 330, 306, 282, 258, 234, 210, 186, 162, 138, - 114, 90, 66, 342, 318, 294, 270, 246, 222, 198, 174, 150, 126, 102, 78 -] - -PATCH_ID_OUTPUT_SORT_IDS = np.argsort(PATCH_ID_OUTPUT) diff --git a/protozfits/simple.py b/protozfits/simple.py deleted file mode 100644 index 2ff10cb..0000000 --- a/protozfits/simple.py +++ /dev/null @@ -1,229 +0,0 @@ -from enum import Enum -from collections import namedtuple -from warnings import warn -import numpy as np -from astropy.io import fits - -from google.protobuf.pyext.cpp_message import GeneratedProtocolMessageType -from . import rawzfitsreader -from .CoreMessages_pb2 import AnyArray -from .any_array_to_numpy import any_array_to_numpy - - -from . import L0_pb2 -from . import R1_pb2 -from . import R1_LSTCam_pb2 -from . import R1_NectarCam_pb2 -from . import R1_DigiCam_pb2 - -pb2_modules = { - 'L0': L0_pb2, - 'R1': R1_pb2, - 'R1_DigiCam': R1_DigiCam_pb2, - 'R1_NectarCam': R1_NectarCam_pb2, - 'R1_LSTCam': R1_LSTCam_pb2, -} - - -def get_class_from_PBFHEAD(pbfhead): - module_name, class_name = pbfhead.split('.') - return getattr(pb2_modules[module_name], class_name) - - -class File: - instances = 0 - - def __init__(self, path, pure_protobuf=False): - File.instances += 1 - if File.instances > 1: - warn('''\ - Multiple open zfits files at the same time are not supported. - Reading from mutliple open tables at the same time will reset these - tables continously and you will read always the same events. - ''') - Table._Table__last_opened = None - bintable_descriptions = detect_bintables(path) - for btd in bintable_descriptions: - self.__dict__[btd.extname] = Table(btd, pure_protobuf) - - def __repr__(self): - return "%s(%r)" % ( - self.__class__.__name__, - self.__dict__ - ) - - def __enter__(self): - return self - - def __exit__(self, type, value, tb): - self.close() - - def close(self): - File.instances -= 1 - - def __del__(self): - self.close() - - -BinTableDescription = namedtuple( - 'BinTableDescription', - [ - 'path', - 'index', - 'extname', - 'pbfhead', - 'znaxis2', - 'header', - ] -) - - -def detect_bintables(path): - fitsfile = fits.open(path) - bintables = [ - BinTableDescription( - path=path, - index=hdu_id, - extname=hdu.header['EXTNAME'], - pbfhead=hdu.header['PBFHEAD'], - znaxis2=hdu.header['ZNAXIS2'], - header=hdu.header - ) - for hdu_id, hdu in enumerate(fitsfile) - if 'XTENSION' in hdu.header and hdu.header['XTENSION'] == 'BINTABLE' - ] - fitsfile.close() - return bintables - - -class Table: - '''Iterable Table - ''' - __last_opened = None - '''the rawzfitsreader has a "bug" which is: It cannot have two open - hdus. So when the File would open all N tables at construction time, - every `rawzfitsreader.readEvent()` would act on the last opened table. - - So the Tables remember which hdu was opened last, and if it was not them. - They open it. - ''' - - def __init__(self, desc, pure_protobuf=False): - ''' - desc: BinTableDescription - ''' - self.__desc = desc - self.__pbuf_class = get_class_from_PBFHEAD(desc.pbfhead) - self.header = self.__desc.header - self.pure_protobuf = pure_protobuf - - def __len__(self): - return self.__desc.znaxis2 - - def __iter__(self): - return self - - def __next__(self): - if not Table.__last_opened == self.__desc: - rawzfitsreader.open(self.__desc.path+":"+self.__desc.extname) - Table.__last_opened = self.__desc - row = self.__pbuf_class() - try: - row.ParseFromString(rawzfitsreader.readEvent()) - except EOFError: - raise StopIteration - - if not self.pure_protobuf: - return make_namedtuple(row) - else: - return row - - def __repr__(self): - return '{cn}({d.znaxis2}x{d.pbfhead})'.format( - cn=self.__class__.__name__, - d=self.__desc - ) - - -def make_namedtuple(message): - namedtuple_class = named_tuples[message.__class__] - return namedtuple_class._make( - message_getitem(message, name) - for name in namedtuple_class._fields - ) - - -def message_getitem(msg, name): - value = msg.__getattribute__(name) - if isinstance(value, AnyArray): - value = any_array_to_numpy(value) - elif (msg.__class__, name) in enum_types: - value = enum_types[(msg.__class__, name)](value) - elif type(value) in named_tuples: - value = make_namedtuple(value) - return value - - -messages = set() -for module in pb2_modules.values(): - for name in dir(module): - thing = getattr(module, name) - if isinstance(thing, GeneratedProtocolMessageType): - messages.add(thing) - - - -def namedtuple_repr2(self): - '''a nicer repr for big namedtuples containing big numpy arrays''' - old_print_options = np.get_printoptions() - np.set_printoptions(precision=3, threshold=50, edgeitems=2) - delim = '\n ' - s = self.__class__.__name__ + '(' + delim - - s += delim.join([ - '{0}={1}'.format( - key, - repr( - getattr(self, key) - ).replace('\n', delim) - ) - for key in self._fields - ]) - s += ')' - np.set_printoptions(**old_print_options) - return s - - -def nt(m): - '''create namedtuple class from protobuf.message type''' - _nt = namedtuple( - m.__name__, - list(m.DESCRIPTOR.fields_by_name) - ) - _nt.__repr__ = namedtuple_repr2 - return _nt - - -named_tuples = {m: nt(m) for m in messages} - -enum_types = {} -for m in messages: - d = m.DESCRIPTOR - for field in d.fields: - if field.enum_type is not None: - et = field.enum_type - enum = Enum( - field.name, - zip(et.values_by_name, et.values_by_number) - ) - enum_types[(m, field.name)] = enum - - -def rewind_table(): - # rawzfitsreader.rewindTable() has a bug at the moment, - # it always throws a SystemError - # we let that one pass - try: - rawzfitsreader.rewindTable() - except SystemError: - pass diff --git a/protozfits/tests/test_rawzfitsreader.py b/protozfits/tests/test_1_rawzfitsreader.py similarity index 88% rename from protozfits/tests/test_rawzfitsreader.py rename to protozfits/tests/test_1_rawzfitsreader.py index 0f88dc9..da83c09 100644 --- a/protozfits/tests/test_rawzfitsreader.py +++ b/protozfits/tests/test_1_rawzfitsreader.py @@ -45,6 +45,40 @@ def to_numpy(a): any_array_type_cannot_convert_exception_text[a.type]) +def test_import_only(): + from protozfits import rawzfitsreader + from protozfits import L0_pb2 + + +def test_import_and_open(): + from protozfits import rawzfitsreader + from protozfits import L0_pb2 + + relative_test_file_path = os.path.relpath(example_file_path) + rawzfitsreader.open(relative_test_file_path + ':Events') + + +def test_import_open_and_read(): + from protozfits import rawzfitsreader + from protozfits import L0_pb2 + + relative_test_file_path = os.path.relpath(example_file_path) + rawzfitsreader.open(relative_test_file_path + ':Events') + raw = rawzfitsreader.readEvent() + + +def test_import_open_read_and_parse(): + from protozfits import rawzfitsreader + from protozfits import L0_pb2 + + relative_test_file_path = os.path.relpath(example_file_path) + rawzfitsreader.open(relative_test_file_path + ':Events') + raw = rawzfitsreader.readEvent() + + event = L0_pb2.CameraEvent() + event.ParseFromString(raw) + + def test_rawreader_can_work_with_relative_path(): from protozfits import rawzfitsreader from protozfits import L0_pb2 @@ -57,7 +91,8 @@ def test_rawreader_can_work_with_relative_path(): event = L0_pb2.CameraEvent() event.ParseFromString(raw) -""" + +@pytest.mark.skip(reason="This is currently SegFaulting") def test_examplefile_has_no_runheader(): from protozfits import rawzfitsreader from protozfits import L0_pb2 @@ -70,7 +105,7 @@ def test_examplefile_has_no_runheader(): header = L0_pb2.CameraRunHeader() with pytest.raises(TypeError): header.ParseFromString(raw) -""" + def test_rawreader_can_work_with_absolute_path(): from protozfits import rawzfitsreader diff --git a/protozfits/tests/test_R1_example_file.py b/protozfits/tests/test_R1_example_file.py index 778a314..c7bf6be 100644 --- a/protozfits/tests/test_R1_example_file.py +++ b/protozfits/tests/test_R1_example_file.py @@ -2,7 +2,7 @@ import pkg_resources import os -from protozfits import SimpleFile +from protozfits import File example_file_path = pkg_resources.resource_filename( 'protozfits', @@ -16,7 +16,7 @@ def test_can_iterate_over_events_and_run_header(): - with SimpleFile(example_file_path) as f: + with File(example_file_path) as f: camera_config = next(f.CameraConfig) assert (camera_config.expected_pixels_id == np.arange(14)).all() diff --git a/protozfits/tests/test_read_raw_events.py b/protozfits/tests/test_read_raw_events.py deleted file mode 100644 index 11521b6..0000000 --- a/protozfits/tests/test_read_raw_events.py +++ /dev/null @@ -1,297 +0,0 @@ -import pytest - -import pkg_resources -import os -import numpy as np - -example_file_path = pkg_resources.resource_filename( - 'protozfits', - os.path.join( - 'tests', - 'resources', - 'example_10evts.fits.fz' - ) -) - -FIRST_EVENT_IN_EXAMPLE_FILE = 97750287 -TELESCOPE_ID_IN_EXAMPLE_FILE = 1 -EVENTS_IN_EXAMPLE_FILE = 10 -EXPECTED_LOCAL_TIME = [ - 1.5094154944067896e+18, - 1.509415494408104e+18, - 1.509415494408684e+18, - 1.509415494415717e+18, - 1.5094154944180828e+18, - 1.5094154944218719e+18, - 1.5094154944245553e+18, - 1.5094154944267853e+18, - 1.509415494438982e+18, - 1.5094154944452902e+18 -] -EXPECTED_GPS_TIME = [0] * EVENTS_IN_EXAMPLE_FILE - - -def test_zfile_raises_on_wrong_path(): - from protozfits import DigicamFile - with pytest.raises(FileNotFoundError): - DigicamFile('foo.bar') - - -def test_zfile_opens_correct_path(): - from protozfits import DigicamFile - DigicamFile(example_file_path) - - -def test_rawreader_can_work_with_relative_path(): - from protozfits import rawzfitsreader - from protozfits import L0_pb2 - - relative_test_file_path = os.path.relpath(example_file_path) - rawzfitsreader.open(relative_test_file_path + ':Events') - raw = rawzfitsreader.readEvent() - assert rawzfitsreader.getNumRows() == EVENTS_IN_EXAMPLE_FILE - - event = L0_pb2.CameraEvent() - event.ParseFromString(raw) - -""" -def test_examplefile_has_no_runheader(): - from protozfits import rawzfitsreader - from protozfits import L0_pb2 - - rawzfitsreader.open(example_file_path + ':RunHeader') - - raw = rawzfitsreader.readEvent() - assert raw < 0 - - header = L0_pb2.CameraRunHeader() - with pytest.raises(TypeError): - header.ParseFromString(raw) -""" - -def test_rawreader_can_work_with_absolute_path(): - from protozfits import rawzfitsreader - from protozfits import L0_pb2 - - rawzfitsreader.open(example_file_path + ':Events') - raw = rawzfitsreader.readEvent() - assert rawzfitsreader.getNumRows() == EVENTS_IN_EXAMPLE_FILE - - event = L0_pb2.CameraEvent() - event.ParseFromString(raw) - - -def test_can_iterate_over_events(): - from protozfits import DigicamFile - - for __ in DigicamFile(example_file_path): - pass - - -def test_iteration_yield_expected_fields(): - from protozfits import DigicamFile - - for event in DigicamFile(example_file_path): - # we just want to see, that the zfits file has all these - # fields and we can access them - event.telescope_id - event.num_gains - event.n_pixels - event.event_number - event.pixel_flags - - event.local_time - event.central_event_gps_time - event.camera_event_type - event.array_event_type - event.num_samples - event.adc_samples - - # expert mode fields - event.trigger_input_traces - event.trigger_output_patch7 - event.trigger_output_patch19 - event.baseline - - -def test_event_number(): - from protozfits import DigicamFile - - event_numbers = [ - event.event_number - for event in DigicamFile(example_file_path) - ] - expected_event_numbers = [ - FIRST_EVENT_IN_EXAMPLE_FILE + i - for i in range(EVENTS_IN_EXAMPLE_FILE) - ] - assert event_numbers == expected_event_numbers - - -def test_telescope_ids(): - from protozfits import DigicamFile - telescope_ids = [ - event.telescope_id - for event in DigicamFile(example_file_path) - ] - expected_ids = [TELESCOPE_ID_IN_EXAMPLE_FILE] * EVENTS_IN_EXAMPLE_FILE - assert telescope_ids == expected_ids - - -def test_num_gains(): - from protozfits import DigicamFile - num_gains = [ - event.num_gains - for event in DigicamFile(example_file_path) - ] - expected_num_gains = [0] * EVENTS_IN_EXAMPLE_FILE - assert num_gains == expected_num_gains - - -def test_num_channels(): - from protozfits import DigicamFile - for event in DigicamFile(example_file_path): - assert -1 == event.num_channels - - -def test_n_pixel(): - from protozfits import DigicamFile - n_pixel = [ - event.n_pixels - for event in DigicamFile(example_file_path) - ] - assert n_pixel == [1296] * EVENTS_IN_EXAMPLE_FILE - - -def test_pixel_flags(): - from protozfits import DigicamFile - pixel_flags = [ - event.pixel_flags - for event in DigicamFile(example_file_path) - ] - expected_pixel_flags = [ - np.ones(1296, dtype=np.bool) - ] * EVENTS_IN_EXAMPLE_FILE - - for actual, expected in zip(pixel_flags, expected_pixel_flags): - assert (actual == expected).all() - - -def test_local_time(): - from protozfits import DigicamFile - local_time = [ - event.local_time - for event in DigicamFile(example_file_path) - ] - assert local_time == EXPECTED_LOCAL_TIME - - -def test_gps_time(): - from protozfits import DigicamFile - gps_time = [ - event.central_event_gps_time - for event in DigicamFile(example_file_path) - ] - assert gps_time == EXPECTED_GPS_TIME - - -def test_camera_event_type(): - from protozfits import DigicamFile - camera_event_type = [ - event.camera_event_type - for event in DigicamFile(example_file_path) - ] - assert camera_event_type == [1, 1, 1, 1, 1, 8, 1, 1, 1, 1] - - -def test_array_event_type(): - from protozfits import DigicamFile - array_event_type = [ - event.array_event_type - for event in DigicamFile(example_file_path) - ] - assert array_event_type == [0] * EVENTS_IN_EXAMPLE_FILE - - -def test_num_samples(): - from protozfits import DigicamFile - num_samples = [ - event.num_samples - for event in DigicamFile(example_file_path) - ] - assert num_samples == [50] * EVENTS_IN_EXAMPLE_FILE - - -def test_adc_samples(): - from protozfits import DigicamFile - adc_samples = [ - event.adc_samples - for event in DigicamFile(example_file_path) - ] - - for actual in adc_samples: - assert actual.dtype == np.int16 - assert actual.shape == (1296, 50) - - adc_samples = np.array(adc_samples) - - # these are 12 bit ADC values, so the range must - # can at least be asserted - assert adc_samples.min() == 0 - assert adc_samples.max() == (2**12) - 1 - - -def test_trigger_input_traces(): - from protozfits import DigicamFile - trigger_input_traces = [ - event.trigger_input_traces - for event in DigicamFile(example_file_path) - ] - - for actual in trigger_input_traces: - assert actual.dtype == np.uint8 - assert actual.shape == (432, 50) - - -def test_trigger_output_patch7(): - from protozfits import DigicamFile - trigger_output_patch7 = [ - event.trigger_output_patch7 - for event in DigicamFile(example_file_path) - ] - - for actual in trigger_output_patch7: - assert actual.dtype == np.uint8 - assert actual.shape == (432, 50) - - -def test_trigger_output_patch19(): - from protozfits import DigicamFile - trigger_output_patch19 = [ - event.trigger_output_patch19 - for event in DigicamFile(example_file_path) - ] - - for actual in trigger_output_patch19: - assert actual.dtype == np.uint8 - assert actual.shape == (432, 50) - - -def test_baseline(): - from protozfits import DigicamFile - baseline = [ - event.baseline - for event in DigicamFile(example_file_path) - ] - - for actual in baseline: - assert actual.dtype == np.int16 - assert actual.shape == (1296,) - - baseline = np.array(baseline) - - baseline_deviation_between_events = baseline.std(axis=0) - # I don't know if this is a good test, but I assume baseline should - # not vary too much between events, so I had a look at these. - assert baseline_deviation_between_events.max() < 60 - assert baseline_deviation_between_events.mean() < 2