From 88623ea9432118ea316d8e321100ef38bbc006fd Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 16:29:06 +0100 Subject: [PATCH 01/14] copy protozfitsreader.py from digicampipe here. --- protozfitsreader/__init__.py | 534 +++++++++++++++++++---------------- 1 file changed, 294 insertions(+), 240 deletions(-) diff --git a/protozfitsreader/__init__.py b/protozfitsreader/__init__.py index 1db7ea8..4c698d3 100644 --- a/protozfitsreader/__init__.py +++ b/protozfitsreader/__init__.py @@ -2,261 +2,315 @@ # this code should run in python3. # Zfits/protobuf loader. # import protozfitsreader +from os.path import isfile import numpy as np -from . import rawzfitsreader -from . import L0_pb2 +from protozfitsreader import rawzfitsreader +from protozfitsreader import L0_pb2 +import warnings + +pixel_remap = [ + 425, 461, 353, 389, 352, 388, 424, 460, 315, 351, 387, 423, + 281, 317, 209, 245, 208, 244, 280, 316, 175, 207, 243, 279, 350, 386, 278, + 314, 277, 313, 349, 385, 240, 276, 312, 348, 206, 242, 144, 174, 143, + 173, 205, 241, 116, 142, 172, 204, 713, 749, 641, 677, 640, 676, 712, 748, + 603, 639, 675, 711, 569, 605, 497, 533, 496, 532, 568, 604, 459, 495, 531, + 567, 638, 674, 566, 602, 565, 601, 637, 673, 528, 564, 600, 636, 494, 530, + 422, 458, 421, 457, 493, 529, 384, 420, 456, 492, 1001, 1037, 929, 965, + 928, 964, 1000, 1036, 891, 927, 963, 999, 857, 893, 785, 821, 784, 820, + 856, 892, 747, 783, 819, 855, 926, 962, 854, 890, 853, 889, 925, 961, 816, + 852, 888, 924, 782, 818, 710, 746, 709, 745, 781, 817, 672, 708, 744, 780, + 275, 311, 203, 239, 202, 238, 274, 310, 169, 201, 237, 273, 141, 171, 91, + 115, 90, 114, 140, 170, 69, 89, 113, 139, 200, 236, 138, 168, 137, 167, + 199, 235, 110, 136, 166, 198, 88, 112, 50, 68, 49, 67, 87, 111, 34, 48, + 66, 86, 563, 599, 491, 527, 490, 526, 562, 598, 453, 489, 525, 561, 419, + 455, 347, 383, 346, 382, 418, 454, 309, 345, 381, 417, 488, 524, 416, 452, + 415, 451, 487, 523, 378, 414, 450, 486, 344, 380, 272, 308, 271, 307, 343, + 379, 234, 270, 306, 342, 851, 887, 779, 815, 778, 814, 850, 886, 741, 777, + 813, 849, 707, 743, 635, 671, 634, 670, 706, 742, 597, 633, 669, 705, 776, + 812, 704, 740, 703, 739, 775, 811, 666, 702, 738, 774, 632, 668, 560, 596, + 559, 595, 631, 667, 522, 558, 594, 630, 135, 165, 85, 109, 84, 108, 134, + 164, 63, 83, 107, 133, 47, 65, 21, 33, 20, 32, 46, 64, 11, 19, 31, 45, 82, + 106, 44, 62, 43, 61, 81, 105, 28, 42, 60, 80, 18, 30, 4, 10, 3, 9, 17, 29, + 0, 2, 8, 16, 413, 449, 341, 377, 340, 376, 412, 448, 303, 339, 375, 411, + 269, 305, 197, 233, 196, 232, 268, 304, 163, 195, 231, 267, 338, 374, 266, + 302, 265, 301, 337, 373, 228, 264, 300, 336, 194, 230, 132, 162, 131, 161, + 193, 229, 104, 130, 160, 192, 701, 737, 629, 665, 628, 664, 700, 736, 591, + 627, 663, 699, 557, 593, 485, 521, 484, 520, 556, 592, 447, 483, 519, 555, + 626, 662, 554, 590, 553, 589, 625, 661, 516, 552, 588, 624, 482, 518, 410, + 446, 409, 445, 481, 517, 372, 408, 444, 480, 1271, 1270, 1282, 1281, 1273, + 1272, 1259, 1258, 1274, 1261, 1260, 1244, 1290, 1289, 1295, 1294, 1292, + 1291, 1284, 1283, 1293, 1286, 1285, 1275, 1246, 1245, 1263, 1262, 1248, + 1247, 1228, 1227, 1249, 1230, 1229, 1207, 1277, 1276, 1288, 1287, 1279, + 1278, 1265, 1264, 1280, 1267, 1266, 1250, 1197, 1196, 1220, 1219, 1199, + 1198, 1173, 1172, 1200, 1175, 1174, 1146, 1240, 1239, 1257, 1256, 1242, + 1241, 1222, 1221, 1243, 1224, 1223, 1201, 1148, 1147, 1177, 1176, 1150, + 1149, 1118, 1117, 1151, 1120, 1119, 1085, 1203, 1202, 1226, 1225, 1205, + 1204, 1179, 1178, 1206, 1181, 1180, 1152, 1075, 1074, 1110, 1109, + 1077, 1076, 1041, 1040, 1078, 1043, 1042, 1005, 1142, 1141, 1171, 1170, + 1144, 1143, 1112, 1111, 1145, 1114, 1113, 1079, 1007, 1006, 1045, 1044, + 1009, 1008, 972, 971, 1010, 974, 973, 936, 1081, 1080, 1116, 1115, 1083, + 1082, 1047, 1046, 1084, 1049, 1048, 1011, 1209, 1208, 1232, 1231, 1211, + 1210, 1185, 1184, 1212, 1187, 1186, 1158, 1252, 1251, 1269, 1268, 1254, + 1253, 1234, 1233, 1255, 1236, 1235, 1213, 1160, 1159, 1189, 1188, 1162, + 1161, 1130, 1129, 1163, 1132, 1131, 1097, 1215, 1214, 1238, 1237, 1217, + 1216, 1191, 1190, 1218, 1193, 1192, 1164, 1087, 1086, 1122, 1121, 1089, + 1088, 1053, 1052, 1090, 1055, 1054, 1017, 1154, 1153, 1183, 1182, 1156, + 1155, 1124, 1123, 1157, 1126, 1125, 1091, 1019, 1018, 1057, 1056, + 1021, 1020, 984, 983, 1022, 986, 985, 948, 1093, 1092, 1128, 1127, 1095, + 1094, 1059, 1058, 1096, 1061, 1060, 1023, 938, 937, 976, 975, 940, 939, + 903, 902, 941, 905, 904, 867, 1013, 1012, 1051, 1050, 1015, 1014, 978, 977, + 1016, 980, 979, 942, 869, 868, 907, 906, 871, 870, 834, 833, 872, 836, + 835, 798, 944, 943, 982, 981, 946, 945, 909, 908, 947, 911, 910, 873, 1099, + 1098, 1134, 1133, 1101, 1100, 1065, 1064, 1102, 1067, 1066, 1029, 1166, + 1165, 1195, 1194, 1168, 1167, 1136, 1135, 1169, 1138, 1137, 1103, 1031, + 1030, 1069, 1068, 1033, 1032, 996, 995, 1034, 998, 997, 960, 1105, 1104, + 1140, 1139, 1107, 1106, 1071, 1070, 1108, 1073, 1072, 1035, 950, 949, 988, + 987, 952, 951, 915, 914, 953, 917, 916, 879, 1025, 1024, 1063, 1062, 1027, + 1026, 990, 989, 1028, 992, 991, 954, 881, 880, 919, 918, 883, 882, 846, + 845, 884, 848, 847, 810, 956, 955, 994, 993, 958, 957, 921, 920, 959, 923, + 922, 885, 800, 799, 838, 837, 802, 801, 765, 764, 803, 767, 766, 729, 875, + 874, 913, 912, 877, 876, 840, 839, 878, 842, 841, 804, 731, 730, 769, 768, + 733, 732, 696, 695, 734, 698, 697, 660, 806, 805, 844, 843, 808, 807, 771, + 770, 809, 773, 772, 735, 146, 117, 177, 145, 213, 178, 179, 147, 249, 250, + 214, 215, 211, 176, 246, 210, 282, 247, 248, 212, 318, 319, 283, 284, 286, + 251, 321, 285, 357, 322, 323, 287, 393, 394, 358, 359, 355, 320, 390, 354, + 426, 391, 392, 356, 462, 463, 427, 428, 52, 35, 71, 51, 95, 72, 73, 53, + 121, 122, 96, 97, 93, 70, 118, 92, 148, 119, 120, 94, 180, 181, 149, 150, + 152, 123, 183, 151, 219, 184, 185, 153, 255, 256, 220, 221, 217, 182, 252, + 216, 288, 253, 254, 218, 324, 325, 289, 290, 6, 1, 13, 5, 25, 14, 15, 7, + 39, 40, 26, 27, 23, 12, 36, 22, 54, 37, 38, 24, 74, 75, 55, 56, 58, 41, 77, + 57, 101, 78, 79, 59, 127, 128, 102, 103, 99, 76, 124, 98, 154, 125, 126, + 100, 186, 187, 155, 156, 430, 395, 465, 429, 501, 466, 467, 431, 537, 538, + 502, 503, 499, 464, 534, 498, 570, 535, 536, 500, 606, 607, 571, 572, 574, + 539, 609, 573, 645, 610, 611, 575, 681, 682, 646, 647, 643, 608, 678, 642, + 714, 679, 680, 644, 750, 751, 715, 716, 292, 257, 327, 291, 363, 328, 329, + 293, 399, 400, 364, 365, 361, 326, 396, 360, 432, 397, 398, 362, 468, 469, + 433, 434, 436, 401, 471, 435, 507, 472, 473, 437, 543, 544, 508, 509, 505, + 470, 540, 504, 576, 541, 542, 506, 612, 613, 577, 578, 158, 129, 189, 157, + 225, 190, 191, 159, 261, 262, 226, 227, 223, 188, 258, 222, 294, 259, 260, + 224, 330, 331, 295, 296, 298, 263, 333, 297, 369, 334, 335, 299, 405, 406, + 370, 371, 367, 332, 402, 366, 438, 403, 404, 368, 474, 475, 439, 440, 718, + 683, 753, 717, 789, 754, 755, 719, 825, 826, 790, 791, 787, 752, 822, 786, + 858, 823, 824, 788, 894, 895, 859, 860, 862, 827, 897, 861, 933, 898, 899, + 863, 969, 970, 934, 935, 931, 896, 966, 930, 1002, 967, 968, 932, 1038, + 1039, 1003, 1004, 580, 545, 615, 579, 651, 616, 617, 581, 687, 688, 652, + 653, 649, 614, 684, 648, 720, 685, 686, 650, 756, 757, 721, 722, 724, 689, + 759, 723, 795, 760, 761, 725, 831, 832, 796, 797, 793, 758, 828, 792, 864, + 829, 830, 794, 900, 901, 865, 866, 442, 407, 477, 441, 513, 478, 479, 443, + 549, 550, 514, 515, 511, 476, 546, 510, 582, 547, 548, 512, 618, 619, 583, + 584, 586, 551, 621, 585, 657, 622, 623, 587, 693, 694, 658, 659, 655, 620, + 690, 654, 726, 691, 692, 656, 762, 763, 727, 728 +] + +PATCH_ID_INPUT = [ + 204, 216, 180, 192, 229, 241, 205, 217, 254, 266, 230, 242, + 279, 291, 255, 267, 304, 316, 280, 292, 329, 341, 305, 317, 156, 168, 132, + 144, 181, 193, 157, 169, 206, 218, 182, 194, 231, 243, 207, 219, 256, 268, + 232, 244, 281, 293, 257, 269, 108, 120, 84, 96, 133, 145, 109, 121, 158, + 170, 134, 146, 183, 195, 159, 171, 208, 220, 184, 196, 233, 245, 209, 221, + 60, 72, 40, 50, 85, 97, 61, 73, 110, 122, 86, 98, 135, 147, 111, 123, 160, + 172, 136, 148, 185, 197, 161, 173, 24, 32, 12, 18, 41, 51, 25, 33, 62, 74, + 42, 52, 87, 99, 63, 75, 112, 124, 88, 100, 137, 149, 113, 125, 4, 8, 0, 2, + 13, 19, 5, 9, 26, 34, 14, 20, 43, 53, 27, 35, 64, 76, 44, 54, 89, 101, 65, + 77, 228, 239, 240, 252, 251, 262, 263, 275, 274, 285, 286, 298, 297, 308, + 309, 321, 320, 331, 332, 344, 343, 354, 355, 366, 253, 264, 265, 277, 276, + 287, 288, 300, 299, 310, 311, 323, 322, 333, 334, 346, 345, 356, 357, 368, + 367, 377, 378, 387, 278, 289, 290, 302, 301, 312, 313, 325, 324, 335, 336, + 348, 347, 358, 359, 370, 369, 379, 380, 389, 388, 396, 397, 404, 303, 314, + 315, 327, 326, 337, 338, 350, 349, 360, 361, 372, 371, 381, 382, 391, 390, + 398, 399, 406, 405, 411, 412, 417, 328, 339, 340, 352, 351, 362, 363, 374, + 373, 383, 384, 393, 392, 400, 401, 408, 407, 413, 414, 419, 418, 422, 423, + 426, 353, 364, 365, 376, 375, 385, 386, 395, 394, 402, 403, 410, 409, 415, + 416, 421, 420, 424, 425, 428, 427, 429, 430, 431, 215, 191, 227, 203, 167, + 143, 179, 155, 119, 95, 131, 107, 71, 49, 83, 59, 31, 17, 39, 23, 7, 1, + 11, 3, 238, 214, 250, 226, 190, 166, 202, 178, 142, 118, 154, 130, 94, 70, + 106, 82, 48, 30, 58, 38, 16, 6, 22, 10, 261, 237, 273, 249, 213, 189, 225, + 201, 165, 141, 177, 153, 117, 93, 129, 105, 69, 47, 81, 57, 29, 15, 37, + 21, 284, 260, 296, 272, 236, 212, 248, 224, 188, 164, 200, 176, 140, 116, + 152, 128, 92, 68, 104, 80, 46, 28, 56, 36, 307, 283, 319, 295, 259, 235, + 271, 247, 211, 187, 223, 199, 163, 139, 175, 151, 115, 91, 127, 103, 67, + 45, 79, 55, 330, 306, 342, 318, 282, 258, 294, 270, 234, 210, 246, 222, + 186, 162, 198, 174, 138, 114, 150, 126, 90, 66, 102, 78 +] + +PATCH_ID_OUTPUT = [ + 204, 216, 229, 241, 254, 266, 279, 291, 304, 316, 329, + 341, 180, 192, 205, 217, 230, 242, 255, 267, 280, 292, 305, 317, 156, 168, + 181, 193, 206, 218, 231, 243, 256, 268, 281, 293, 132, 144, 157, 169, 182, + 194, 207, 219, 232, 244, 257, 269, 108, 120, 133, 145, 158, 170, 183, 195, + 208, 220, 233, 245, 84, 96, 109, 121, 134, 146, 159, 171, 184, 196, 209, + 221, 60, 72, 85, 97, 110, 122, 135, 147, 160, 172, 185, 197, 40, 50, 61, + 73, 86, 98, 111, 123, 136, 148, 161, 173, 24, 32, 41, 51, 62, 74, 87, 99, + 112, 124, 137, 149, 12, 18, 25, 33, 42, 52, 63, 75, 88, 100, 113, 125, 4, + 8, 13, 19, 26, 34, 43, 53, 64, 76, 89, 101, 0, 2, 5, 9, 14, 20, 27, 35, + 44, 54, 65, 77, 228, 239, 251, 262, 274, 285, 297, 308, 320, 331, 343, + 354, 240, 252, 263, 275, 286, 298, 309, 321, 332, 344, 355, 366, 253, 264, + 276, 287, 299, 310, 322, 333, 345, 356, 367, 377, 265, 277, 288, 300, 311, + 323, 334, 346, 357, 368, 378, 387, 278, 289, 301, 312, 324, 335, 347, 358, + 369, 379, 388, 396, 290, 302, 313, 325, 336, 348, 359, 370, 380, 389, 397, + 404, 303, 314, 326, 337, 349, 360, 371, 381, 390, 398, 405, 411, 315, 327, + 338, 350, 361, 372, 382, 391, 399, 406, 412, 417, 328, 339, 351, 362, 373, + 383, 392, 400, 407, 413, 418, 422, 340, 352, 363, 374, 384, 393, 401, 408, + 414, 419, 423, 426, 353, 364, 375, 385, 394, 402, 409, 415, 420, 424, 427, + 429, 365, 376, 386, 395, 403, 410, 416, 421, 425, 428, 430, 431, 215, 191, + 167, 143, 119, 95, 71, 49, 31, 17, 7, 1, 227, 203, 179, 155, 131, 107, 83, + 59, 39, 23, 11, 3, 238, 214, 190, 166, 142, 118, 94, 70, 48, 30, 16, 6, + 250, 226, 202, 178, 154, 130, 106, 82, 58, 38, 22, 10, 261, 237, 213, 189, + 165, 141, 117, 93, 69, 47, 29, 15, 273, 249, 225, 201, 177, 153, 129, 105, + 81, 57, 37, 21, 284, 260, 236, 212, 188, 164, 140, 116, 92, 68, 46, 28, + 296, 272, 248, 224, 200, 176, 152, 128, 104, 80, 56, 36, 307, 283, 259, + 235, 211, 187, 163, 139, 115, 91, 67, 45, 319, 295, 271, 247, 223, 199, + 175, 151, 127, 103, 79, 55, 330, 306, 282, 258, 234, 210, 186, 162, 138, + 114, 90, 66, 342, 318, 294, 270, 246, 222, 198, 174, 150, 126, 102, 78 +] + + +class ZFile: - -class ZFile(object): def __init__(self, fname): + if not isfile(fname): + raise FileNotFoundError(fname) self.fname = fname + self.eventnumber = 0 + self.is_events_table_open = False + self.__open_events() + self.numrows = rawzfitsreader.getNumRows() + self.run_id = 0 - def _read_file(self): - if self.ttype not in ["RunHeader", "Events", "RunTails"]: - print("Error: Table type not RunHeader, Events or RunTails") + def __next__(self): + if self.eventnumber < self.numrows: + self.__open_events() + event = L0_pb2.CameraEvent() + event.ParseFromString(rawzfitsreader.readEvent()) + self.eventnumber += 1 + return Event(event, self.run_id, self.eventnumber) else: - rawzfitsreader.open("%s:%s" % (self.fname, self.ttype)) + raise StopIteration - def _read_message(self): - '''Read next message. - Fills property self.rawmessage and self.numrows - ''' - self.rawmessage = rawzfitsreader.readEvent() - self.numrows = rawzfitsreader.getNumRows() + def __iter__(self): + return self - def _extract_field(self, obj, field): - # Read a specific field in object 'obj' given as input 'field' - if not obj.HasField(field): - raise KeyError(str(field)) - return getattr(obj, field) + def __open_events(self): + if not self.is_events_table_open: + rawzfitsreader.open(self.fname + ":Events") + self.is_events_table_open = True def list_tables(self): return rawzfitsreader.listAllTables(self.fname) - def read_runheader(self): - if not self.ttype == "RunHeader": - self.ttype = "RunHeader" - self._read_file() - - self._read_message() - self.header = L0_pb2.CameraRunHeader() - self.header.ParseFromString(self.rawmessage) - - def read_event(self): - if not self.ttype == "Events": - self.ttype = "Events" - self._read_file() - self.eventnumber = 1 - else: - self.eventnumber += 1 - - self._read_message() - self.event = L0_pb2.CameraEvent() - self.event.ParseFromString(self.rawmessage) - def rewind_table(self): # Rewind the current reader. Go to the beginning of the table. rawzfitsreader.rewindTable() - def move_to_next_event(self): - # Iterate over events - i = 0 - numrows = i+2 - # Hook to deal with file with no header (1) - if hasattr(self, 'numrows'): - numrows = self.numrows - # End - Hook to deal with file with no header (1) - - while i < numrows: - self.read_event() - # Hook to deal with file with no header (2) - if hasattr(self, 'numrows'): - numrows = self.numrows - # End - Hook to deal with file with no header (2) - # Hook to deal with file with no header (3) +class Event: + def __init__(self, event, run_id, event_id): + self.event_id = event_id + self.run_id = run_id + self._event = event + + _e = self._event # just to make lines shorter + _w = self._event.hiGain.waveforms # just to make lines shorter + + self.pixel_ids = toNumPyArray(_w.pixelsIndices) + self._sort_ids = np.argsort(self.pixel_ids) + self.n_pixels = len(self.pixel_ids) + self._samples = toNumPyArray(_w.samples).reshape(self.n_pixels, -1) + self.baseline = self.unsorted_baseline[self._sort_ids] + self.telescope_id = _e.telescopeID + self.event_number = _e.eventNumber + self.central_event_gps_time = self.__calc_central_event_gps_time() + self.local_time = self.__calc_local_time() + self.event_number_array = _e.arrayEvtNum + self.camera_event_type = _e.event_type + self.array_event_type = _e.eventType + self.num_gains = _e.num_gains + self.num_channels = _e.head.numGainChannels + self.num_samples = self._samples.shape[1] + self.pixel_flags = toNumPyArray(_e.pixels_flags)[self._sort_ids] + self.adc_samples = self._samples[self._sort_ids] + self.trigger_output_patch7 = _prepare_trigger_output( + _e.trigger_output_patch7) + self.trigger_output_patch19 = _prepare_trigger_output( + _e.trigger_output_patch19) + self.trigger_input_traces = _prepare_trigger_input( + _e.trigger_input_traces) + + @property + def unsorted_baseline(self): + if not hasattr(self, '__unsorted_baseline'): try: - run_id = self.get_run_id() - event_id = self.get_event_id() - except: - run_id = 0 - event_id = self.eventnumber - # Hook to deal with file with no header (3) - - yield run_id, event_id - i += 1 - - def get_telescope_id(self): - return toNumPyArray(self.event.telescopeID) - - def get_event_number(self): - return toNumPyArray(self.event.eventNumber) - - def get_run_id(self): - return toNumPyArray(self.header.runNumber) - - def get_central_event_gps_time(self): - timeSec = self.event.trig.timeSec - timeNanoSec = self.event.trig.timeNanoSec - return timeSec, timeNanoSec - - def get_local_time(self): - timeSec = self.event.local_time_sec - timeNanoSec = self.event.local_time_nanosec - return timeSec, timeNanoSec - - def get_event_type(self): - return self.event.event_type - - def get_eventType(self): - return self.event.eventType - - def get_num_channels(self): - return toNumPyArray(self.event.head.numGainChannels) - - def _get_adc(self, channel, telescope_id=None): - # Expect hi/lo -> Will append Gain at the end -> hiGain/loGain - sel_channel = self._extract_field(self.event, "%sGain" % channel) - return sel_channel - - def get_pixel_position(self, telescope_id=None): - return None - - def get_number_of_pixels(self, telescope_id=None): - return None - - def get_adc_sum(self, channel, telescope_id=None): - sel_channel = self._get_adc(channel, telescope_id) - integrals = sel_channel.integrals - - pixelsIndices = toNumPyArray(integrals.pixelsIndices) - - # Structured array (dict of dicts) - properties = dict() - for par in [ - "gains", - "maximumTimes", - "raiseTimes", - "tailTimes", - "firstSplIdx" - ]: - properties[par] = dict( - zip( - pixelsIndices, - toNumPyArray(self._extract_field(par)) - ) - ) - return properties - - def get_adc_samples(self, channel, telescope_id=None): - sel_channel = self._get_adc(channel, telescope_id) - waveforms = sel_channel.waveforms - samples = toNumPyArray(waveforms.samples) - pixels = toNumPyArray(waveforms.pixelsIndices) - npixels = len(pixels) - samples = samples.reshape(npixels, -1) - properties = dict(zip(pixels, samples)) - - return(properties) - - def get_adcs_samples(self, telescope_id=None): - ''' - Get the samples for all channels - - :param telescope_id: id of the telescopeof interest - :return: dictionnary of samples (value) per pixel indices (key) - ''' - waveforms = self.event.hiGain.waveforms - samples = toNumPyArray(waveforms.samples) - pixels = toNumPyArray(waveforms.pixelsIndices) - npixels = len(pixels) - samples = samples.reshape(npixels, -1) - properties = dict(zip(pixels, samples)) - return(properties) - - def get_trigger_input_traces(self, telescope_id=None): - ''' - Get the samples for all channels - - :param telescope_id: id of the telescopeof interest - :return: dictionnary of samples (value) per pixel indices (key) - ''' - patch_traces = toNumPyArray(self.event.trigger_input_traces) - patches = np.arange(0, 192, 1) - patch_traces = patch_traces.reshape(patches.shape[0], -1) - properties = dict(zip(patches, patch_traces)) - - return(properties) - - def get_trigger_output_patch7(self, telescope_id=None): - ''' - Get the samples for all channels - - :param telescope_id: id of the telescopeof interest - :return: dictionnary of samples (value) per pixel indices (key) - ''' - frames = toNumPyArray(self.event.trigger_output_patch7) - n_samples = frames.shape[0] / 18 / 3 - frames = np.unpackbits( - frames.reshape(n_samples, 3, 18, 1), axis=-1 - )[..., ::-1].reshape(n_samples, 3, 144).reshape(n_samples, 432).T - patches = np.arange(0, 432) - properties = dict(zip(patches, frames)) - return(properties) - - def get_trigger_output_patch19(self, telescope_id=None): - ''' - Get the samples for all channels - - :param telescope_id: id of the telescopeof interest - :return: dictionnary of samples (value) per pixel indices (key) - ''' - frames = toNumPyArray(self.event.trigger_output_patch19) - n_samples = frames.shape[0] / 18 / 3 - frames = np.unpackbits( - frames.reshape(n_samples, 3, 18, 1), axis=-1 - )[..., ::-1].reshape(n_samples, 3, 144).reshape(n_samples, 432).T - patches = np.arange(0, 432) - properties = dict(zip(patches, frames)) - - return(properties) - - def get_pixel_flags(self, telescope_id = None): - ''' - Get the flag of pixels - :param id of the telescopeof interest - :return: dictionnary of flags (value) per pixel indices (key) - ''' - waveforms = self.event.hiGain.waveforms - flags = toNumPyArray(self.event.pixels_flags) - pixels = toNumPyArray(waveforms.pixelsIndices) - # Structured array (dict) - properties = dict(zip(pixels, flags)) - return(properties) - - def listof_fields(self, obj): - return [f.name for f in obj.DESCRIPTOR.fields] - - -def toNumPyArray(any_array): - any_array_type_to_numpy_type = { - 1: np.int8, - 2: np.uint8, - 3: np.int16, - 4: np.uint16, - 5: np.int32, - 6: np.uint32, - 7: np.int64, - 8: np.uint64, - 9: np.float, - 10: np.double, - } - if any_array.type == 0: - raise Exception("any_array has no type") - if any_array.type == 11: - raise Exception("I have no idea if the boolean representation of the anyarray is the same as the numpy one") - - return np.fromstring( - any_array.data, - any_array_type_to_numpy_type[any_array.type] - ) + self.__unsorted_baseline = toNumPyArray( + self._event.hiGain.waveforms.baselines) + except ValueError: + warnings.warn(( + "Could not read `hiGain.waveforms.baselines` for event:{0}" + "of run_id:{1}".format(self.event_id, self.run_id) + )) + self.__unsorted_baseline = np.ones( + len(self.pixel_ids) + ) * np.nan + return self.__unsorted_baseline + + def __calc_central_event_gps_time(self): + time_second = self._event.trig.timeSec + time_nanosecond = self._event.trig.timeNanoSec + return time_second * 1E9 + time_nanosecond + + def __calc_local_time(self): + time_second = self._event.local_time_sec + time_nanosecond = self._event.local_time_nanosec + return time_second * 1E9 + time_nanosecond + + +def _prepare_trigger_input(_a): + _a = toNumPyArray(_a) + A, B = 3, 192 + cut = 144 + _a = _a.reshape(-1, A) + _a = _a.reshape(-1, A, B) + _a = _a[..., :cut] + _a = _a.reshape(_a.shape[0], -1) + _a = _a.T + _a = _a[np.argsort(PATCH_ID_INPUT)] + return _a + + +def _prepare_trigger_output(_a): + _a = toNumPyArray(_a) + A, B, C = 3, 18, 8 + + _a = np.unpackbits(_a.reshape(-1, A, B, 1), axis=-1) + _a = _a[..., ::-1] + _a = _a.reshape(-1, A*B*C).T + return _a[np.argsort(PATCH_ID_OUTPUT)] + + +any_array_type_to_npdtype = { + 1: 'i1', + 2: 'u1', + 3: 'i2', + 4: 'u2', + 5: 'i4', + 6: 'u4', + 7: 'i8', + 8: 'u8', + 9: 'f4', + 10: 'f8', +} + +any_array_type_cannot_convert_exception_text = { + 0: "This any array has no defined type", + 11: """I have no idea if the boolean representation + of the anyarray is the same as the numpy one""" +} + + +def toNumPyArray(a): + if a.type in any_array_type_to_npdtype: + return np.frombuffer( + a.data, any_array_type_to_npdtype[a.type]) + else: + raise ValueError( + "Conversion to NumpyArray failed with error:\n%s", + any_array_type_cannot_convert_exception_text[a.type]) From a451f869e796d3bc8a8260b73692fdcd959a0f6b Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 16:29:40 +0100 Subject: [PATCH 02/14] remove unneeded comment and adjust import statements --- protozfitsreader/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/protozfitsreader/__init__.py b/protozfitsreader/__init__.py index 4c698d3..5cba428 100644 --- a/protozfitsreader/__init__.py +++ b/protozfitsreader/__init__.py @@ -1,11 +1,8 @@ #!/bin/env python -# this code should run in python3. -# Zfits/protobuf loader. -# import protozfitsreader from os.path import isfile import numpy as np -from protozfitsreader import rawzfitsreader -from protozfitsreader import L0_pb2 +from . import rawzfitsreader +from . import L0_pb2 import warnings pixel_remap = [ From d9fc3d589d1458597e6344545fd4386efb142f54 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 16:39:22 +0100 Subject: [PATCH 03/14] we should not have pyc files under version control --- .../test_rawzfitsreader.cpython-36-PYTEST.pyc | Bin 3117 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 protozfitsreader/tests/__pycache__/test_rawzfitsreader.cpython-36-PYTEST.pyc diff --git a/protozfitsreader/tests/__pycache__/test_rawzfitsreader.cpython-36-PYTEST.pyc b/protozfitsreader/tests/__pycache__/test_rawzfitsreader.cpython-36-PYTEST.pyc deleted file mode 100644 index 3cf69f1bc4595dceb7a6045cfc3325564008ccc8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3117 zcmb_e&2Jk;6yI5|ch~E$I7yR!QBt5#7f8~iNlVp0+th_fX&Racs=cf>nT_p@?X_km zNfXuwFmOW%A#th@LgE0bxWEC4KY%Nuw-pz}sn;HJK)g5WI8OMMu(fY~`{vE=eEfdz z?PNA$Wai!!~4V+NNw%cFIfn z7FFAXD4yY`X~Hx8jGv_%O;Wv`XsdRP8g`zVb{|dI1!~bW&Co2((LC*=1=>#s%9h2l(P3zh(4+Jiv_q6EDaGTDV3x{4q*u$0fO%10lLaE!S*r@p zJePVbN(zq)Zl{>XgAKOo@tq;>o?CC$Jm>tlw<-8229K`oRP!G^OgVWItbil`_PQ+agu6IBy?112l4%qym13r5DWE}kPml&+#l<|kJ(czkf zi~k0O5~|jeI%z2caRRW~f{hfg@qSY1ZPHRh#WzBgYQqX{G2EGLrImm-)zU%@a578^ zYh7*Wq0WC6X&*K>)Yk~jp?^(nYt#@~Ki5{)bsj>0K7=Q+_W|`GYN*hUSZfp6CfkIX z-4S2o2w}m9@30pUj4)~HzBZ(^3^>G0XjBPcQxqVgKQ!BkYuk|@XAt)4>!k3K){G*>2ejLW4>uOaxZIy197cjjz!`$Cah zZ*{=P%m|{1Jr{tEYrM!u%br-+sNV^;xYasav|1xGarktRGmwE24VB&~Ei5lNa|=%C z?zLNs^Cjoz+fvwJK0ky0oC_H6mcHOXZ>?YkvhSjnA>JHn)-Aj?1Eg zv+CAr4)dC9HK;dTP(uc~ot2GhO;j5^(rW=k@aU++E5SyM%220V3s&5k({SrvlybPc z=HU~f+%>thYS1V#7O-f*SqoU*6^?hWS#uk%fSxEl3!|7DPcFacGVa}EL48TEYNK4V z_7`jq!!lvarsv4%?yoTcCmp46p7CCu@4cDqy_t${;%Qv$y?H50dPs^d74|&r%I?xw zC8&F2wBc2`H?}HV9;}J6!Eu zz6ugfLRL&hMFZY=Qp zp1>tDKqdv~&t9*Zy1 znF~bYQEGX+=}Eb;r(rh-7qZUVs77X-UIQ zu8=aSxZG(34%=wNBhEt@$CbqF%K_6g$-JAAc_!Rv2%CcbF87dk-9q#jZvGqhfRQ>d zAptx9?g;~m4-_H(qX4!6`;x%H9Qr#PwEVQ>U=#W?viR%(%65wnV&I_kK@o!Dvp3>% z93c!~?E~yZ)Kh$PS$uTKRzu_I>_Kp8fhVjd2&z)&CC+KZcQ5F#avq*;5!SNt`-F zV#!%v+qeeo`!PNHmcy$s2_^J&Ni&}VMBtN{g#M;Ttl4@Hof zfG^3yUYvuv!-B~MF<$e7YD4A|`^3$;J4?%s{ERsK6=9!TF3p#gW^XT+oVgo^d*$)i z1Esr*rP<}u4QKxL?6rAkdG1!pE_9u*E-p!ztQ=Cp4ELjF)8+QdL}6a32Xv$6y^Nn! k{4y95^pEO?ziO!(QHc(hN=!{7@K+HZ{OhWfFtcXvciMO($p8QV From 855ec566218d891413f1c6767e47cb945dcafee8 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 16:40:25 +0100 Subject: [PATCH 04/14] add new tests from digicampipe for ZFile --- .../tests/test_read_raw_events.py | 303 ++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 protozfitsreader/tests/test_read_raw_events.py diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py new file mode 100644 index 0000000..7a4c5cc --- /dev/null +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -0,0 +1,303 @@ +import pytest + +import pkg_resources +import os +import numpy as np + +example_file_path = pkg_resources.resource_filename( + 'digicampipe', + os.path.join( + 'tests', + 'resources', + 'example_10_evts.000.fits.fz' + ) +) + +FIRST_EVENT_IN_EXAMPLE_FILE = 97750287 +TELESCOPE_ID_IN_EXAMPLE_FILE = 1 +EVENTS_IN_EXAMPLE_FILE = 10 +EXPECTED_LOCAL_TIME = [ + 1.5094154944067896e+18, + 1.509415494408104e+18, + 1.509415494408684e+18, + 1.509415494415717e+18, + 1.5094154944180828e+18, + 1.5094154944218719e+18, + 1.5094154944245553e+18, + 1.5094154944267853e+18, + 1.509415494438982e+18, + 1.5094154944452902e+18 +] +EXPECTED_GPS_TIME = [0] * EVENTS_IN_EXAMPLE_FILE + + +def test_zfile_raises_on_wrong_path(): + from digicampipe.io.protozfitsreader import ZFile + with pytest.raises(FileNotFoundError): + ZFile('foo.bar') + + +def test_zfile_opens_correct_path(): + from digicampipe.io.protozfitsreader import ZFile + ZFile(example_file_path) + + +def test_rawreader_can_work_with_relative_path(): + from protozfitsreader import rawzfitsreader + from protozfitsreader import L0_pb2 + + relative_test_file_path = os.path.relpath(example_file_path) + rawzfitsreader.open(relative_test_file_path + ':Events') + raw = rawzfitsreader.readEvent() + assert rawzfitsreader.getNumRows() == EVENTS_IN_EXAMPLE_FILE + + event = L0_pb2.CameraEvent() + event.ParseFromString(raw) + + +def test_examplefile_has_no_runheader(): + from protozfitsreader import rawzfitsreader + from protozfitsreader import L0_pb2 + + rawzfitsreader.open(example_file_path + ':RunHeader') + + raw = rawzfitsreader.readEvent() + assert raw < 0 + + header = L0_pb2.CameraRunHeader() + with pytest.raises(TypeError): + header.ParseFromString(raw) + + +def test_rawreader_can_work_with_absolute_path(): + from protozfitsreader import rawzfitsreader + from protozfitsreader import L0_pb2 + + rawzfitsreader.open(example_file_path + ':Events') + raw = rawzfitsreader.readEvent() + assert rawzfitsreader.getNumRows() == EVENTS_IN_EXAMPLE_FILE + + event = L0_pb2.CameraEvent() + event.ParseFromString(raw) + + +def test_can_iterate_over_events(): + from digicampipe.io.protozfitsreader import ZFile + + for __ in ZFile(example_file_path): + pass + + +def test_iteration_yield_expected_fields(): + from digicampipe.io.protozfitsreader import ZFile + + for event in ZFile(example_file_path): + # we just want to see, that the zfits file has all these + # fields and we can access them + event.event_id + event.telescope_id + event.num_gains + event.n_pixels + event.event_number + event.pixel_flags + + event.local_time + event.central_event_gps_time + event.camera_event_type + event.array_event_type + event.num_samples + event.adc_samples + + # expert mode fields + event.trigger_input_traces + event.trigger_output_patch7 + event.trigger_output_patch19 + event.baseline + + +def test_event_number(): + from digicampipe.io.protozfitsreader import ZFile + + event_numbers = [ + event.event_number + for event in ZFile(example_file_path) + ] + expected_event_numbers = [ + FIRST_EVENT_IN_EXAMPLE_FILE + i + for i in range(EVENTS_IN_EXAMPLE_FILE) + ] + assert event_numbers == expected_event_numbers + + +def test_telescope_ids(): + from digicampipe.io.protozfitsreader import ZFile + telescope_ids = [ + event.telescope_id + for event in ZFile(example_file_path) + ] + expected_ids = [TELESCOPE_ID_IN_EXAMPLE_FILE] * EVENTS_IN_EXAMPLE_FILE + assert telescope_ids == expected_ids + + +def test_num_gains(): + from digicampipe.io.protozfitsreader import ZFile + num_gains = [ + event.num_gains + for event in ZFile(example_file_path) + ] + expected_num_gains = [0] * EVENTS_IN_EXAMPLE_FILE + assert num_gains == expected_num_gains + + +def test_n_pixel(): + from digicampipe.io.protozfitsreader import ZFile + n_pixel = [ + event.n_pixels + for event in ZFile(example_file_path) + ] + assert n_pixel == [1296] * EVENTS_IN_EXAMPLE_FILE + + +def test_pixel_flags(): + from digicampipe.io.protozfitsreader import ZFile + pixel_flags = [ + event.pixel_flags + for event in ZFile(example_file_path) + ] + expected_pixel_flags = [ + np.ones(1296, dtype=np.bool) + ] * EVENTS_IN_EXAMPLE_FILE + + for actual, expected in zip(pixel_flags, expected_pixel_flags): + assert (actual == expected).all() + + +def test_local_time(): + from digicampipe.io.protozfitsreader import ZFile + local_time = [ + event.local_time + for event in ZFile(example_file_path) + ] + assert local_time == EXPECTED_LOCAL_TIME + + +def test_gps_time(): + from digicampipe.io.protozfitsreader import ZFile + gps_time = [ + event.central_event_gps_time + for event in ZFile(example_file_path) + ] + assert gps_time == EXPECTED_GPS_TIME + + +def test_camera_event_type(): + from digicampipe.io.protozfitsreader import ZFile + camera_event_type = [ + event.camera_event_type + for event in ZFile(example_file_path) + ] + assert camera_event_type == [1, 1, 1, 1, 1, 8, 1, 1, 1, 1] + + +def test_array_event_type(): + from digicampipe.io.protozfitsreader import ZFile + array_event_type = [ + event.array_event_type + for event in ZFile(example_file_path) + ] + assert array_event_type == [0] * EVENTS_IN_EXAMPLE_FILE + + +def test_num_samples(): + from digicampipe.io.protozfitsreader import ZFile + num_samples = [ + event.num_samples + for event in ZFile(example_file_path) + ] + assert num_samples == [50] * EVENTS_IN_EXAMPLE_FILE + + +def test_adc_samples(): + from digicampipe.io.protozfitsreader import ZFile + adc_samples = [ + event.adc_samples + for event in ZFile(example_file_path) + ] + + for actual in adc_samples: + assert actual.dtype == np.int16 + assert actual.shape == (1296, 50) + + adc_samples = np.array(adc_samples) + + # these are 12 bit ADC values, so the range must + # can at least be asserted + assert adc_samples.min() == 0 + assert adc_samples.max() == (2**12) - 1 + + +def test_trigger_input_traces(): + from digicampipe.io.protozfitsreader import ZFile + trigger_input_traces = [ + event.trigger_input_traces + for event in ZFile(example_file_path) + ] + + for actual in trigger_input_traces: + assert actual.dtype == np.uint8 + assert actual.shape == (432, 50) + + +def test_trigger_output_patch7(): + from digicampipe.io.protozfitsreader import ZFile + trigger_output_patch7 = [ + event.trigger_output_patch7 + for event in ZFile(example_file_path) + ] + + for actual in trigger_output_patch7: + assert actual.dtype == np.uint8 + assert actual.shape == (432, 50) + + +def test_trigger_output_patch19(): + from digicampipe.io.protozfitsreader import ZFile + trigger_output_patch19 = [ + event.trigger_output_patch19 + for event in ZFile(example_file_path) + ] + + for actual in trigger_output_patch19: + assert actual.dtype == np.uint8 + assert actual.shape == (432, 50) + + +def test_baseline(): + from digicampipe.io.protozfitsreader import ZFile + baseline = [ + event.baseline + for event in ZFile(example_file_path) + ] + + for actual in baseline: + assert actual.dtype == np.int16 + assert actual.shape == (1296,) + + baseline = np.array(baseline) + + baseline_deviation_between_events = baseline.std(axis=0) + # I don't know if this is a good test, but I assume baseline should + # not vary too much between events, so I had a look at these. + assert baseline_deviation_between_events.max() < 60 + assert baseline_deviation_between_events.mean() < 2 + + +def test_count_number_event(): + + from digicampipe.io.zfits import count_number_events + n_files = 10 + files = [example_file_path] * n_files # create a list of files + + assert count_number_events(files) == n_files * EVENTS_IN_EXAMPLE_FILE + + From 4420400bdd6b4fc8f0774b1613a03c6c061b7520 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 16:52:29 +0100 Subject: [PATCH 05/14] fix import statement --- .../tests/test_read_raw_events.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py index 7a4c5cc..208571d 100644 --- a/protozfitsreader/tests/test_read_raw_events.py +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -32,13 +32,13 @@ def test_zfile_raises_on_wrong_path(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile with pytest.raises(FileNotFoundError): ZFile('foo.bar') def test_zfile_opens_correct_path(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile ZFile(example_file_path) @@ -82,14 +82,14 @@ def test_rawreader_can_work_with_absolute_path(): def test_can_iterate_over_events(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile for __ in ZFile(example_file_path): pass def test_iteration_yield_expected_fields(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile for event in ZFile(example_file_path): # we just want to see, that the zfits file has all these @@ -116,7 +116,7 @@ def test_iteration_yield_expected_fields(): def test_event_number(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile event_numbers = [ event.event_number @@ -130,7 +130,7 @@ def test_event_number(): def test_telescope_ids(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile telescope_ids = [ event.telescope_id for event in ZFile(example_file_path) @@ -140,7 +140,7 @@ def test_telescope_ids(): def test_num_gains(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile num_gains = [ event.num_gains for event in ZFile(example_file_path) @@ -150,7 +150,7 @@ def test_num_gains(): def test_n_pixel(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile n_pixel = [ event.n_pixels for event in ZFile(example_file_path) @@ -159,7 +159,7 @@ def test_n_pixel(): def test_pixel_flags(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile pixel_flags = [ event.pixel_flags for event in ZFile(example_file_path) @@ -173,7 +173,7 @@ def test_pixel_flags(): def test_local_time(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile local_time = [ event.local_time for event in ZFile(example_file_path) @@ -182,7 +182,7 @@ def test_local_time(): def test_gps_time(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile gps_time = [ event.central_event_gps_time for event in ZFile(example_file_path) @@ -191,7 +191,7 @@ def test_gps_time(): def test_camera_event_type(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile camera_event_type = [ event.camera_event_type for event in ZFile(example_file_path) @@ -200,7 +200,7 @@ def test_camera_event_type(): def test_array_event_type(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile array_event_type = [ event.array_event_type for event in ZFile(example_file_path) @@ -209,7 +209,7 @@ def test_array_event_type(): def test_num_samples(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile num_samples = [ event.num_samples for event in ZFile(example_file_path) @@ -218,7 +218,7 @@ def test_num_samples(): def test_adc_samples(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile adc_samples = [ event.adc_samples for event in ZFile(example_file_path) @@ -237,7 +237,7 @@ def test_adc_samples(): def test_trigger_input_traces(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile trigger_input_traces = [ event.trigger_input_traces for event in ZFile(example_file_path) @@ -249,7 +249,7 @@ def test_trigger_input_traces(): def test_trigger_output_patch7(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile trigger_output_patch7 = [ event.trigger_output_patch7 for event in ZFile(example_file_path) @@ -261,7 +261,7 @@ def test_trigger_output_patch7(): def test_trigger_output_patch19(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile trigger_output_patch19 = [ event.trigger_output_patch19 for event in ZFile(example_file_path) @@ -273,7 +273,7 @@ def test_trigger_output_patch19(): def test_baseline(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile baseline = [ event.baseline for event in ZFile(example_file_path) From 68c58ebab5a7a57340efc210a8526c1f1e9948d4 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 16:54:17 +0100 Subject: [PATCH 06/14] bump minor version up --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8346e88..89c6258 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='protozfitsreader', packages=['protozfitsreader'], - version='0.43.1', + version='0.43.2', description='Basic python bindings for protobuf zfits reader', author="Etienne Lyard et al.", author_email="etienne.lyard@unige.ch", From 499ea8bc0dfddcb806949cb2278051ed74345158 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 17:23:41 +0100 Subject: [PATCH 07/14] remove a test that should be in digicampipe --- protozfitsreader/tests/test_read_raw_events.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py index 208571d..f9dad52 100644 --- a/protozfitsreader/tests/test_read_raw_events.py +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -290,14 +290,3 @@ def test_baseline(): # not vary too much between events, so I had a look at these. assert baseline_deviation_between_events.max() < 60 assert baseline_deviation_between_events.mean() < 2 - - -def test_count_number_event(): - - from digicampipe.io.zfits import count_number_events - n_files = 10 - files = [example_file_path] * n_files # create a list of files - - assert count_number_events(files) == n_files * EVENTS_IN_EXAMPLE_FILE - - From a8dded1bfbf1e810730c086dfdcc4a7c055c6278 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 19:41:04 +0100 Subject: [PATCH 08/14] wrong package name .. --- protozfitsreader/tests/test_read_raw_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py index f9dad52..f98d186 100644 --- a/protozfitsreader/tests/test_read_raw_events.py +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -5,7 +5,7 @@ import numpy as np example_file_path = pkg_resources.resource_filename( - 'digicampipe', + 'protozfitsreader', os.path.join( 'tests', 'resources', From fff52cdfdf48d850a2c3526265ff433b4ea8f3f7 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 25 Jan 2018 19:47:39 +0100 Subject: [PATCH 09/14] wrong test filename --- protozfitsreader/tests/test_read_raw_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py index f98d186..7c24fef 100644 --- a/protozfitsreader/tests/test_read_raw_events.py +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -9,7 +9,7 @@ os.path.join( 'tests', 'resources', - 'example_10_evts.000.fits.fz' + 'example_10evts.fits.fz' ) ) From 6dcd55671fcfeddd522be8546b3c7de8a69e17a2 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 6 Feb 2018 09:45:28 +0100 Subject: [PATCH 10/14] use the protozfitsreader.py from the current master of digicampipe --- protozfitsreader/__init__.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/protozfitsreader/__init__.py b/protozfitsreader/__init__.py index 5cba428..20240f9 100644 --- a/protozfitsreader/__init__.py +++ b/protozfitsreader/__init__.py @@ -1,4 +1,3 @@ -#!/bin/env python from os.path import isfile import numpy as np from . import rawzfitsreader @@ -168,29 +167,19 @@ def __init__(self, fname): if not isfile(fname): raise FileNotFoundError(fname) self.fname = fname - self.eventnumber = 0 - self.is_events_table_open = False - self.__open_events() + rawzfitsreader.open(self.fname + ":Events") self.numrows = rawzfitsreader.getNumRows() self.run_id = 0 + self.generator = self.__generator() - def __next__(self): - if self.eventnumber < self.numrows: - self.__open_events() + def __generator(self): + for _ in range(self.numrows): event = L0_pb2.CameraEvent() event.ParseFromString(rawzfitsreader.readEvent()) - self.eventnumber += 1 - return Event(event, self.run_id, self.eventnumber) - else: - raise StopIteration + yield Event(event, self.run_id) def __iter__(self): - return self - - def __open_events(self): - if not self.is_events_table_open: - rawzfitsreader.open(self.fname + ":Events") - self.is_events_table_open = True + return self.generator def list_tables(self): return rawzfitsreader.listAllTables(self.fname) @@ -201,8 +190,7 @@ def rewind_table(self): class Event: - def __init__(self, event, run_id, event_id): - self.event_id = event_id + def __init__(self, event, run_id): self.run_id = run_id self._event = event @@ -242,7 +230,7 @@ def unsorted_baseline(self): except ValueError: warnings.warn(( "Could not read `hiGain.waveforms.baselines` for event:{0}" - "of run_id:{1}".format(self.event_id, self.run_id) + "of run_id:{1}".format(self.event_number, self.run_id) )) self.__unsorted_baseline = np.ones( len(self.pixel_ids) From b445b77622c2578ccfeaa119dad4aee412935a51 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 6 Feb 2018 09:50:18 +0100 Subject: [PATCH 11/14] this does not exist anymore --- protozfitsreader/tests/test_read_raw_events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py index 7c24fef..d920b69 100644 --- a/protozfitsreader/tests/test_read_raw_events.py +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -94,7 +94,6 @@ def test_iteration_yield_expected_fields(): for event in ZFile(example_file_path): # we just want to see, that the zfits file has all these # fields and we can access them - event.event_id event.telescope_id event.num_gains event.n_pixels From 45bab48524094246e127fa520a78eeba71131725 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 6 Feb 2018 09:53:06 +0100 Subject: [PATCH 12/14] put new test from digicampipe also here --- protozfitsreader/tests/test_read_raw_events.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py index d920b69..5b53029 100644 --- a/protozfitsreader/tests/test_read_raw_events.py +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -148,6 +148,12 @@ def test_num_gains(): assert num_gains == expected_num_gains +def test_num_channels(): + from digicampipe.io.protozfitsreader import ZFile + for event in ZFile(example_file_path): + assert -1 == event.num_channels + + def test_n_pixel(): from protozfitsreader import ZFile n_pixel = [ From db74f60710cb184b57807f53b8d9288fc68ab620 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Mon, 12 Feb 2018 09:15:28 +0100 Subject: [PATCH 13/14] fix import --- protozfitsreader/tests/test_read_raw_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protozfitsreader/tests/test_read_raw_events.py b/protozfitsreader/tests/test_read_raw_events.py index 5b53029..eba8138 100644 --- a/protozfitsreader/tests/test_read_raw_events.py +++ b/protozfitsreader/tests/test_read_raw_events.py @@ -149,7 +149,7 @@ def test_num_gains(): def test_num_channels(): - from digicampipe.io.protozfitsreader import ZFile + from protozfitsreader import ZFile for event in ZFile(example_file_path): assert -1 == event.num_channels From a588a329c32615bb73e240a1dca7b026554c7f83 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Mon, 12 Feb 2018 09:43:05 +0100 Subject: [PATCH 14/14] remove ugly generator function --- protozfitsreader/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/protozfitsreader/__init__.py b/protozfitsreader/__init__.py index 20240f9..4b69078 100644 --- a/protozfitsreader/__init__.py +++ b/protozfitsreader/__init__.py @@ -170,16 +170,17 @@ def __init__(self, fname): rawzfitsreader.open(self.fname + ":Events") self.numrows = rawzfitsreader.getNumRows() self.run_id = 0 - self.generator = self.__generator() - - def __generator(self): - for _ in range(self.numrows): - event = L0_pb2.CameraEvent() - event.ParseFromString(rawzfitsreader.readEvent()) - yield Event(event, self.run_id) def __iter__(self): - return self.generator + return self + + def __next__(self): + event = L0_pb2.CameraEvent() + try: + event.ParseFromString(rawzfitsreader.readEvent()) + return Event(event, self.run_id) + except EOFError: + raise StopIteration def list_tables(self): return rawzfitsreader.listAllTables(self.fname)