diff --git a/src/pyfaradaycup/decommutator/README.md b/src/pyfaradaycup/decommutator/README.md index 6dfd90b..de90988 100644 --- a/src/pyfaradaycup/decommutator/README.md +++ b/src/pyfaradaycup/decommutator/README.md @@ -21,10 +21,6 @@ Dagster decides whether a step succeeded from the exit code of a subprocess, or - **Stop skipping APIDs that fail.** When a skeleton file is missing, or when writing a CDF fails, `main` logs a message and moves to the next APID, and the run still ends with "Script complete". Collect these failures and raise an exception at the end if there were any. -- **Remove the bare `except:` blocks**, or narrow them to the exceptions that are expected. - Several of them hide the original error. -- **Raise exceptions with messages.** - For example, an unreadable L0 file currently gives a `RuntimeError` with no message. ### Return what was written diff --git a/src/pyfaradaycup/decommutator/ccsds_reader_pipeline.py b/src/pyfaradaycup/decommutator/ccsds_reader_pipeline.py index 581a429..da21450 100644 --- a/src/pyfaradaycup/decommutator/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/decommutator/ccsds_reader_pipeline.py @@ -181,10 +181,9 @@ def _wrapper_status( pkt_inds = np.array( [(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)] ) - try: - pkt_starts = pkt_inds[:, 0] - except: # ruff:ignore[E722] + if pkt_inds.size == 0: return data + pkt_starts = pkt_inds[:, 0] # Loop through each packet beginning and decommutate it for i_pointer, pointer in enumerate(pkt_starts): # ruff:ignore[B007] @@ -254,10 +253,9 @@ def read_file( # ruff:ignore[C901] pkt_inds = np.array( [(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)] ) - try: - pkt_starts = pkt_inds[:, 0] - except: # ruff:ignore[E722] + if pkt_inds.size == 0: return data + pkt_starts = pkt_inds[:, 0] npackets = len(pkt_starts) @@ -341,7 +339,7 @@ def read_file_sc( # ruff:ignore[C901, PLR0912, PLR0915] | (cchead["CCSDS_PacketType"] != 0) | (cchead["CCSDS_SecHdrFlag"] != 1) ): - raise ValueError("CCSDS header values not as expected") # ruff:ignore[EM101, TRY003] + raise ValueError("CCSDS header values not as expected") # ruff:ignore[EM101, TRY003, TRY301] file_dt = datetime.datetime(2010, 1, 1) + datetime.timedelta( # ruff:ignore[DTZ001] seconds=cchead["CCSDS_MET"] ) @@ -350,9 +348,10 @@ def read_file_sc( # ruff:ignore[C901, PLR0912, PLR0915] except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: # ruff:ignore[E722] - print(sys.exc_info()) # ruff:ignore[T201] - print("Could not find which SC_HK file to use based on packet header") # ruff:ignore[T201] + except (ValueError, KeyError) as exc: + print( # ruff:ignore[T201] + f"Could not find which SC_HK file to use based on packet header: {exc!r}" + ) print("Attempting to find correct date based on filename/path") # ruff:ignore[T201] try: # Look for ...///... in the path @@ -371,7 +370,9 @@ def read_file_sc( # ruff:ignore[C901, PLR0912, PLR0915] except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: # ruff:ignore[E722] + except (StopIteration, KeyError): + # StopIteration: no / in the path + # KeyError: no SC_HK file for that flight software version print( # ruff:ignore[T201] "***WARNING*** Could not find date based on filename...using most recent" ) @@ -452,10 +453,9 @@ def read_file_sc( # ruff:ignore[C901, PLR0912, PLR0915] pkt_inds = np.array( [(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)] ) - try: - pkt_starts = pkt_inds[:, 0] - except: # ruff:ignore[E722] + if pkt_inds.size == 0: return data + pkt_starts = pkt_inds[:, 0] npackets = len(pkt_starts) # Some variables so we can display progress @@ -655,8 +655,8 @@ def _parse_pkt( # ruff:ignore[C901, PLR0912] thisbin = str_bin[startbit:endbit] try: thisval = int(thisbin, 2) - except: # ruff:ignore[E722] - # print(sys.exc_info()) + except ValueError: + # The packet ends before this mnemonic, so thisbin is empty thisval = -999 thisdat[thisname].append(thisval) return @@ -668,11 +668,12 @@ def _parse_pkt( # ruff:ignore[C901, PLR0912] for i_bit, bit in enumerate(form.bits[0 : len(form.bits) - sw_data_vars_len]): thisbin = str_bin[pointer : pointer + bit] + thisname = form.names[i_bit] try: thisval = int(thisbin, 2) - except: # ruff:ignore[E722] - raise RuntimeError # ruff:ignore[B904] - thisname = form.names[i_bit] + except ValueError as exc: + msg = f"Could not decode {thisname} in an APID {hex(apid)} packet" + raise RuntimeError(msg) from exc # store in our data variable thisdat[thisname].append(thisval) @@ -694,12 +695,12 @@ def _parse_pkt( # ruff:ignore[C901, PLR0912] while (pointer + total_sw_data_length) <= len(str_bin): for i in range(n_vars): thisbin = str_bin[pointer : pointer + form.bits[-n_vars + i]] + thisname = form.sw_data_vars[i] try: thisval = int(thisbin, 2) - except ValueError: - raise ValueError # ruff:ignore[B904] - - thisname = form.sw_data_vars[i] + except ValueError as exc: + msg = f"Could not decode {thisname} in an APID {hex(apid)} packet" + raise RuntimeError(msg) from exc newdat[thisname].append(thisval) pointer += form.bits[-n_vars + i] @@ -778,28 +779,24 @@ def _get_layout(apid: int, verbose: bool = False) -> apid_obj | None: # ruff:ig The layout of each field in the packet, or `None` if the APID is not found in the file. + Raises + ------ + RuntimeError + If ``sweap_tlm.blk`` cannot be read, or if a line in the + section for this APID cannot be parsed. + Notes ----- The file ``sweap_tlm.blk`` is read from the package data directory, - ``src/pyfaradaycup/data/``. If it cannot be opened, a - `RuntimeError` is raised. + ``src/pyfaradaycup/data/``. """ + blk_path = data_dir / "sweap_tlm.blk" try: - # It appears that there is a unique sweap_tlm.blk - file = (data_dir / "sweap_tlm.blk").open() - except: # ruff:ignore[E722] - if verbose: - print( # ruff:ignore[T201] - "***INFO*** 'sweap_tlm.blk' not found...using the one in src/pyfaradaycup/data" - ) - try: - file = (data_dir / "sweap_tlm.blk").open() - except: # ruff:ignore[E722] - # print(here) - print(sys.exc_info()) # ruff:ignore[T201] - raise RuntimeError(f"Unable to open {data_dir}/sweap_tlm.blk") # ruff:ignore[B904, EM102, TRY003] - lines = file.readlines() - file.close() + with blk_path.open() as file: + lines = file.readlines() + except OSError as exc: + msg = f"Unable to read {blk_path}" + raise RuntimeError(msg) from exc for i, line in enumerate(lines): if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): # ruff:ignore[FURB116] if verbose: @@ -826,9 +823,11 @@ def _get_layout(apid: int, verbose: bool = False) -> apid_obj | None: # ruff:ig thisapid.sw_data_vars = [] except IndexError: break - except: # ruff:ignore[E722] - print(sys.exc_info()) # ruff:ignore[T201] - raise RuntimeError # ruff:ignore[B904] + except ValueError as exc: + msg = ( + f"Could not parse line {i + 1} of {blk_path}: {line.strip()!r}" + ) + raise RuntimeError(msg) from exc start = np.array( [0] + [sum(thisapid.bits[0:i]) for i in range(1, len(thisapid.bits))] @@ -878,26 +877,31 @@ def _get_layout_sc( # ruff:ignore[C901] ``Block[...]`` line (an `int`), or `None` if the APID is not found in the file. + Raises + ------ + ValueError + If ``filename`` is not provided. + + RuntimeError + If the file cannot be read, or if a line in the section for + this APID cannot be parsed. + Notes ----- The APID section in the file starts with a line like ``SC_HK_0x``. Fields written as ``mnemonic[N]`` are treated - as ``N`` bytes long (``8 * N`` bits). If ``filename`` is not - provided, a `ValueError` is raised. If the file cannot be opened, a - `RuntimeError` is raised. + as ``N`` bytes long (``8 * N`` bits). """ if filename is None: raise ValueError("Please supply filename") # ruff:ignore[EM101, TRY003] try: - file = pathlib.Path(filename).open() # ruff:ignore[SIM115] - print(f"using sc_hk file: {filename}") # ruff:ignore[T201] - except: # ruff:ignore[E722] - print("could not open SC HK BLK file") # ruff:ignore[T201] - print(sys.exc_info()) # ruff:ignore[T201] - raise RuntimeError # ruff:ignore[B904] - - lines = file.readlines() - file.close() + with pathlib.Path(filename).open() as file: + lines = file.readlines() + except OSError as exc: + msg = f"Unable to read the spacecraft housekeeping definition file {filename}" + raise RuntimeError(msg) from exc + print(f"using sc_hk file: {filename}") # ruff:ignore[T201] + for i, line in enumerate(lines): if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": # ruff:ignore[FURB116] if verbose: @@ -925,9 +929,9 @@ def _get_layout_sc( # ruff:ignore[C901] thisapid.bits.append(int(pieces[3].strip())) except IndexError: break - except: # ruff:ignore[E722] - print(sys.exc_info()) # ruff:ignore[T201] - raise RuntimeError # ruff:ignore[B904] + except ValueError as exc: + msg = f"Could not parse line {i + 1} of {filename}: {line!r}" + raise RuntimeError(msg) from exc return (thisapid, length) # if we didn't find that APID print( # ruff:ignore[T201] diff --git a/src/pyfaradaycup/decommutator/swp_spc_l02l1.py b/src/pyfaradaycup/decommutator/swp_spc_l02l1.py index bde2260..5ac0146 100644 --- a/src/pyfaradaycup/decommutator/swp_spc_l02l1.py +++ b/src/pyfaradaycup/decommutator/swp_spc_l02l1.py @@ -14,7 +14,6 @@ import datetime import math import pathlib -import sys import warnings from typing import TYPE_CHECKING, TextIO @@ -103,6 +102,25 @@ def main( # ruff:ignore[C901, PLR0912, PLR0913, PLR0915, PLR0917] verbose : bool, optional If `True`, print messages to the screen as well as to the log file. + + Raises + ------ + ValueError + If ``l0file`` is not provided. + + FileExistsError + If an L1 CDF file already exists and ``overwrite`` is `False`. + + RuntimeError + If the log file cannot be opened, the SPICE kernels cannot be + loaded, or the L0 file cannot be read. + + Notes + ----- + Errors for a single APID do not stop the conversion. If the + skeleton file cannot be read, the CDF cannot be created, or the CDF + cannot be filled, the error is logged and the next APID is + processed. A CDF that could not be filled may be left behind. """ if not l0file: raise ValueError("Please supply l0file") # ruff:ignore[EM101, TRY003] @@ -145,13 +163,14 @@ def main( # ruff:ignore[C901, PLR0912, PLR0913, PLR0915, PLR0917] try: l0file.open().close() _statusmsg("L0 file exists and is readable") - except OSError: + except OSError as exc: + msg = f"Could not read L0 file: {l0file}" _statusmsg( - "***ERROR*** [swp_spc_l02l1.py] Input L0 file could not be read...exiting", + f"***ERROR*** [swp_spc_l02l1.py] {msg}...exiting", screen=True, verbose=verbose, ) - raise RuntimeError # ruff:ignore[B904] + raise RuntimeError(msg) from exc # Load in Leap Second Kernel _statusmsg("***INFO*** [swp_spc_l02l1.py] Finding newest leap second kernel...") @@ -286,13 +305,12 @@ def main( # ruff:ignore[C901, PLR0912, PLR0913, PLR0915, PLR0917] l1path.parent.mkdir(parents=True, exist_ok=True) try: cdf = pycdf.CDF(str(l1path), skeleton_filename) - except "CDFError": # ruff:ignore[B030] # ty: ignore[invalid-exception-caught] + except pycdf.CDFError as exc: _statusmsg( - f"\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={apid})...continuing to next APID\n).", + f"***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={hex(apid)}): {exc!r}...continuing to next APID", screen=True, verbose=verbose, ) - _statusmsg(sys.exc_info(), screen=True, verbose=verbose) # ty: ignore[invalid-argument-type] continue # Run a different procedure to put data into CDF file depending on APID @@ -312,10 +330,9 @@ def main( # ruff:ignore[C901, PLR0912, PLR0913, PLR0915, PLR0917] } try: cdfproc[apid](cdf, l0data[apid], verbose=verbose) - except: # ruff:ignore[E722] - _statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + except Exception as exc: # ruff:ignore[BLE001] _statusmsg( - f"***WARNING*** [swp_spc_l02l1] CDF not processed for APID={hex(apid)}", + f"***WARNING*** [swp_spc_l02l1] CDF not processed for APID={hex(apid)}: {exc!r}", screen=True, verbose=verbose, ) @@ -359,6 +376,12 @@ def _cdf35e_35f(cdf: pycdf.CDF, dat: dict[str, list], verbose: bool = False) -> Not currently used. Accepted so that `main` can call each CDF writer with the same arguments. + Raises + ------ + RuntimeError + If a variable cannot be written to the CDF, such as when the + data has the wrong shape or type for that variable. + Notes ----- Unlike `_cdf352` and `_cdf351_353_354`, the data is not expanded: @@ -372,8 +395,7 @@ def _cdf35e_35f(cdf: pycdf.CDF, dat: dict[str, list], verbose: bool = False) -> Each variable in the CDF is filled from the matching key in ``dat``. Variables with no matching key are filled with the - variable's ``FILLVAL``. If an unexpected error occurs, a - `RuntimeError` is raised. + variable's ``FILLVAL``. """ # Calculate MET from the variables in the L0 data # MET of each NYS @@ -424,9 +446,9 @@ def _cdf35e_35f(cdf: pycdf.CDF, dat: dict[str, list], verbose: bool = False) -> except KeyError: # ruff:ignore[PERF203] if key not in dat.keys(): # ruff:ignore[SIM118] cdf[key] = np.ones(len(dat["Epoch"])) * cdf[key].attrs["FILLVAL"] - except: # ruff:ignore[E722] - msg = f"Failed : Key:{key} failed insert into CDF" - raise RuntimeError(msg) # ruff:ignore[B904] + except (pycdf.CDFError, TypeError, ValueError) as exc: + msg = f"Could not write variable {key!r} to the CDF: {exc}" + raise RuntimeError(msg) from exc def _cdf351_353_354( # ruff:ignore[C901, PLR0912, PLR0915, RET503] @@ -471,6 +493,12 @@ def _cdf351_353_354( # ruff:ignore[C901, PLR0912, PLR0915, RET503] If ``nocdf`` is `True`, the expanded data, with one value per measurement for each key. Otherwise, `None`. + Raises + ------ + RuntimeError + If a variable in the CDF skeleton is not in the L0 data, or if + a variable cannot be written to the CDF. + Notes ----- ``"Epoch"`` is in nanoseconds past J2000. Measurements are spaced @@ -615,15 +643,14 @@ def _cdf351_353_354( # ruff:ignore[C901, PLR0912, PLR0915, RET503] try: # insert data cdf[key] = dat_exp[key] - except: # ruff:ignore[E722, PERF203] - _statusmsg( - f"Failed : Key:{key} failed insert into CDF", - screen=True, - verbose=verbose, - ) - _statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - - raise RuntimeError # ruff:ignore[B904] + except KeyError as exc: # ruff:ignore[PERF203] + msg = f"The CDF skeleton has variable {key!r}, but the L0 data does not" + _statusmsg(f"Failed : {msg}", screen=True, verbose=verbose) + raise RuntimeError(msg) from exc + except (pycdf.CDFError, TypeError, ValueError) as exc: + msg = f"Could not write variable {key!r} to the CDF: {exc}" + _statusmsg(f"Failed : {msg}", screen=True, verbose=verbose) + raise RuntimeError(msg) from exc def _cdf352( # ruff:ignore[C901, PLR0912, PLR0915] @@ -665,6 +692,11 @@ def _cdf352( # ruff:ignore[C901, PLR0912, PLR0915] If ``nocdf`` is `True`, the expanded data, with one value per measurement for each key. Otherwise, an empty tuple. + Raises + ------ + RuntimeError + If an unexpected error occurs while expanding the packets. + Notes ----- ``"SPC_TIMESERCOLL"`` sets which four channels a packet contains: @@ -674,6 +706,10 @@ def _cdf352( # ruff:ignore[C901, PLR0912, PLR0915] channel names go into ``"VAR0_NAME"`` through ``"VAR3_NAME"``. Packets with any other value are logged as errors and skipped. + Unlike `_cdf351_353_354`, if a variable cannot be written to the + CDF, the error is logged and the remaining variables are still + written. + ``"Epoch"`` is in nanoseconds past J2000. Each measurement is spaced ``1 / (32 * 1171.875)`` seconds apart, starting at the packet's start tick. Values that appear once per packet are @@ -739,7 +775,7 @@ def _cdf352( # ruff:ignore[C901, PLR0912, PLR0915] try: if coll_used not in coll2var: - raise ValueError( # ruff:ignore[TRY003] + raise ValueError( # ruff:ignore[TRY003, TRY301] f"Value: {coll_used} not in coll2var.keys()" # ruff:ignore[EM102] ) # probably a corrupt packet @@ -761,11 +797,12 @@ def _cdf352( # ruff:ignore[C901, PLR0912, PLR0915] dat_exp["VAR2"].extend(dat["G2_000"][i]) dat_exp["VAR3"].extend(dat["G3_000"][i]) - except: # ruff:ignore[E722] + except ValueError as exc: _statusmsg( - "***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?)" + f"***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?): {exc}", + screen=True, + verbose=verbose, ) - _statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) continue # Extend the expanded dt @@ -795,16 +832,21 @@ def _cdf352( # ruff:ignore[C901, PLR0912, PLR0915] try: # insert data cdf[key] = dat_exp[key] - except: # ruff:ignore[E722, PERF203] + except KeyError: # ruff:ignore[PERF203] + _statusmsg( + f"Failed : The CDF skeleton has variable {key!r}, but the L0 data does not", + screen=True, + verbose=verbose, + ) + except (pycdf.CDFError, TypeError, ValueError) as exc: _statusmsg( - f"Failed : Key:{key} failed insert into CDF", + f"Failed : Could not write variable {key!r} to the CDF: {exc}", screen=True, verbose=verbose, ) - _statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - except: # ruff:ignore[E722] - print(sys.exc_info()) # ruff:ignore[T201] - raise RuntimeError # ruff:ignore[B904] + except Exception as exc: + msg = "Could not process the APID 0x352 packets" + raise RuntimeError(msg) from exc return () @@ -963,11 +1005,10 @@ def _get_newest_kernel( versions = [int(file.stem[-ndigits:]) for file in files] try: maxind = np.argmax(versions) - except ValueError: - _statusmsg("***ERROR*** Could not find kernel versions") - print(sys.exc_info()) # ruff:ignore[T201] - - raise RuntimeError("Could not find kernel versions") # ruff:ignore[B904, EM101, TRY003] + except ValueError as exc: + msg = f"Could not find any kernel files matching {globstr} in {globdir}" + _statusmsg(f"***ERROR*** {msg}") + raise RuntimeError(msg) from exc # return path to newest file return str(files[maxind])