JP-3908: Incorporate stfitsdiff into regression tests

changes/9414.general.rst

@@ -0,0 +1 @@
+Implementing STFITSDiff for our regression tests instead of FITSDiff to improve reports.

jwst/regtest/regtestdata.py

@@ -9,7 +9,7 @@
 import sys
 
 import asdf
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from ci_watson.artifactory_helpers import (
     check_url,
     get_bigdata_root,

jwst/regtest/st_fitsdiff.py

@@ -1202,6 +1202,7 @@ def __init__(
         self.report_table = Table(
             names=(
                 "col_name",
+                "dtype",
                 "abs_diffs",
                 "abs_max",
                 "abs_mean",
@@ -1212,12 +1213,13 @@ def __init__(
                 "rel_std",
             ),
             dtype=(
+                "str",
                 "str",
                 "int32",
                 "float64",
                 "float64",
                 "float64",
-                "int32",
+                "float64",
                 "float64",
                 "float64",
                 "float64",
@@ -1404,49 +1406,102 @@ def _diff(self):
                 #   it is not important where they come from
 
                 # Calculate the absolute and relative differences
-                nan_idx = np.isnan(arra) | np.isnan(arrb)
-                anonan = arra[~nan_idx]
-                bnonan = arrb[~nan_idx]
-                diffs = np.abs(anonan - bnonan)
-                abs_diffs = diffs[diffs > self.atol].size
-                nozeros = (diffs != 0.0) & (bnonan != 0.0)
-                rel_values = diffs[nozeros] / np.abs(bnonan[nozeros])
-                rel_diffs = rel_values[rel_values > self.rtol].size
-
-                sum_diffs = np.any(diffs > (self.atol + self.rtol * np.abs(bnonan)))
-                if sum_diffs:
-                    # Report the total number of zeros, nans, and no-nan values
-                    self.report_zeros_nan.add_row(
-                        (
-                            col.name,
-                            arra[arra == 0.0].size,
-                            arrb[arrb == 0.0].size,
-                            arra[np.isnan(arra)].size,
-                            arrb[np.isnan(arrb)].size,
-                            arra[~np.isnan(arra)].size,
-                            arrb[~np.isnan(arrb)].size,
-                            np.mean(anonan),
-                            np.mean(bnonan),
+                get_stats = False
+                if np.issubdtype(arra.dtype, np.floating) and np.issubdtype(
+                    arrb.dtype, np.floating
+                ):
+                    nan_idx = np.isnan(arra) | np.isnan(arrb)
+                    anonan = arra[~nan_idx]
+                    bnonan = arrb[~nan_idx]
+                    diffs = np.abs(anonan - bnonan)
+                    abs_diffs = diffs[diffs > self.atol].size
+                    nozeros = (diffs != 0.0) & (bnonan != 0.0)
+                    rel_values = diffs[nozeros] / np.abs(bnonan[nozeros])
+                    rel_diffs = rel_values[rel_values > self.rtol].size
+                    get_stats = True
+
+                elif "P" in col.format or "Q" in col.format:
+                    diffs = []
+                    rel_values = []
+                    zeros_idx = []
+                    nan_idx = []
+                    for idx in range(len(arra)):
+                        if arra[idx] == 0 or arrb[idx] == 0:
+                            zeros_idx.append(idx)
+                        elif np.isnan(arra[idx]) or np.isnan(arrb[idx]):
+                            nan_idx.append(idx)
+                        else:
+                            df = np.abs(arra[idx] - arrb[idx])
+                            if not df <= self.atol:
+                                diffs.append(df)
+                            dfr = df / np.abs(arrb[idx])
+                            if not dfr <= self.rtol:
+                                rel_values.append(dfr)
+                    get_stats = True
+                    diffs = np.array(diffs)
+                    abs_diffs = diffs.size
+                    rel_values = np.array(rel_values)
+                    rel_diffs = rel_values.size
+                    anonan = arra[~np.array(nan_idx)]
+                    bnonan = arrb[~np.array(nan_idx)]
+
+                if get_stats:
+                    sum_diffs = np.any(diffs > (self.atol + self.rtol * np.abs(bnonan)))
+                    if sum_diffs:
+                        # Report the total number of zeros, nans, and no-nan values
+                        self.report_zeros_nan.add_row(
+                            (
+                                col.name,
+                                arra[arra == 0.0].size,
+                                arrb[arrb == 0.0].size,
+                                arra[np.isnan(arra)].size,
+                                arrb[np.isnan(arrb)].size,
+                                arra[~np.isnan(arra)].size,
+                                arrb[~np.isnan(arrb)].size,
+                                np.mean(anonan),
+                                np.mean(bnonan),
+                            )
                         )
-                    )
 
-                    # Report the differences per column
-                    self.report_table.add_row(
-                        (
-                            col.name,
-                            abs_diffs,
-                            np.max(diffs),
-                            np.mean(diffs),
-                            np.std(diffs),
-                            rel_diffs,
-                            np.max(rel_values),
-                            np.mean(rel_values),
-                            np.std(rel_values),
+                        # Report the differences per column
+                        self.report_table.add_row(
+                            (
+                                col.name,
+                                str(arra.dtype).replace(">", ""),
+                                abs_diffs,
+                                np.max(diffs),
+                                np.mean(diffs),
+                                np.std(diffs),
+                                rel_diffs,
+                                np.max(rel_values),
+                                np.mean(rel_values),
+                                np.std(rel_values),
+                            )
                         )
-                    )
 
-                    self.diff_total += abs_diffs
-                    self.rel_diffs += rel_diffs
+                        self.diff_total += abs_diffs
+                        self.rel_diffs += rel_diffs
+
+                else:
+                    diffs = arra[arra != arrb].size
+                    if diffs > 0:
+                        # Report the differences per column
+                        self.report_table.add_row(
+                            (
+                                col.name,
+                                str(arra.dtype).replace(">", ""),
+                                diffs,
+                                0,
+                                0,
+                                0,
+                                0,
+                                0,
+                                0,
+                                0,
+                            )
+                        )
+                        self.diff_total += diffs
+                        self.rel_diffs = np.nan
 
         # Calculate the absolute difference
         total_values = len(self.a) * len(self.a.dtype.fields)
@@ -1537,23 +1592,29 @@ def _report(self):
                 f"Found {self.diff_total} different table data element(s). "
                 "Reporting percentages above respective tolerances: "
                 f"\n    - absolute .... {self.diff_ratio:.4g}%"
-                f"\n    - relative .... {self.diff_ratio_rel:.4g}%"
             )
+            if np.isnan(self.diff_ratio_rel):
+                self._writeln(
+                    "\n * Unable to calculate relative differences and stats due to data types"
+                )
+            else:
+                self._writeln(f"    - relative .... {self.diff_ratio_rel:.4g}%")
 
-            # Print differences in zeros and nans per column
-            self._writeln("\nValues in a and b")
-            self.report_zeros_nan["mean_a"].format = ".4g"
-            self.report_zeros_nan["mean_b"].format = ".4g"
-            tlines = self.report_zeros_nan.pformat()
-            for tline in tlines:
-                self._writeln(tline)
+                # Print differences in zeros and nans per column
+                self._writeln("\nValues in a and b")
+                self.report_zeros_nan["mean_a"].format = ".4g"
+                self.report_zeros_nan["mean_b"].format = ".4g"
+                tlines = self.report_zeros_nan.pformat()
+                for tline in tlines:
+                    self._writeln(tline)
 
             # Print the difference (a-b) stats
             self._writeln("\nDifference stats: abs(a - b) ")
             # make sure the format is acceptable
             for colname in self.report_table.columns:
-                if colname != "col_name":
-                    self.report_table[colname].format = ".4g"
+                if colname in ["col_name", "dtype"]:
+                    continue
+                self.report_table[colname].format = ".4g"
             tlines = self.report_table.pformat()
             for tline in tlines:
                 self._writeln(tline)

jwst/regtest/test_fgs_image2.py

@@ -1,5 +1,5 @@
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_fgs_image3.py

@@ -1,5 +1,5 @@
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_coron3.py

@@ -1,5 +1,5 @@
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_cubebuild.py

@@ -1,6 +1,6 @@
 """Test CubeBuildStep on MIRI MRS"""
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from jwst.stpipe import Step
 
 

jwst/regtest/test_miri_dark.py

@@ -1,5 +1,5 @@
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_image.py

@@ -3,7 +3,7 @@
 
 import numpy as np
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from numpy.testing import assert_allclose
 from gwcs.wcstools import grid_from_bounding_box
 from stdatamodels.jwst import datamodels

jwst/regtest/test_miri_lrs_dedicated_mbkg.py

@@ -1,6 +1,6 @@
 import os
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.master_background import MasterBackgroundStep
 

jwst/regtest/test_miri_lrs_masterbg_user.py

@@ -1,6 +1,6 @@
 import os
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.master_background import MasterBackgroundStep
 

jwst/regtest/test_miri_lrs_nod_masterbg.py

@@ -1,6 +1,6 @@
 import os
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_lrs_optimal_extraction.py

@@ -1,4 +1,4 @@
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 import pytest
 
 from jwst.stpipe import Step

jwst/regtest/test_miri_lrs_slit_spec2.py

@@ -1,4 +1,4 @@
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from gwcs.wcstools import grid_from_bounding_box
 from numpy.testing import assert_allclose
 import pytest

jwst/regtest/test_miri_lrs_slit_spec3.py

@@ -6,7 +6,7 @@
 
 from gwcs import wcstools
 import asdf
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 from jwst import datamodels

jwst/regtest/test_miri_lrs_slitless.py

@@ -1,7 +1,7 @@
 import os
 
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from numpy.testing import assert_allclose
 from gwcs.wcstools import grid_from_bounding_box
 from stdatamodels.jwst import datamodels

jwst/regtest/test_miri_mrs_badpix_selfcal.py

@@ -1,7 +1,7 @@
 import os
 
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_mrs_dedicated_mbkg.py

@@ -1,6 +1,6 @@
 import os
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.master_background import MasterBackgroundStep
 

jwst/regtest/test_miri_mrs_extract1d.py

@@ -1,7 +1,7 @@
 """Test Extract1dStep on MIRI data running the step various ways
 """
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_mrs_spec3.py

@@ -2,7 +2,7 @@
 import os
 
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_mrs_spec3_moving_target.py

@@ -1,7 +1,7 @@
 """Regression tests for Moving Target MIRI MRS mode"""
 import os
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from jwst.stpipe import Step
 
 # Define artifactory source and truth

jwst/regtest/test_miri_mrs_straylight.py

@@ -1,7 +1,7 @@
 """Regression tests for MIRI MRS straylight step"""
 import pytest
 
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from jwst.stpipe import Step
 
 # Define artifactory source and truth file paths

jwst/regtest/test_miri_mrs_tso.py

@@ -1,6 +1,6 @@
 """Regression test for MIRI MRS TSO mode"""
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_residual_fringe.py

@@ -1,6 +1,6 @@
 """Test ResidualFringeStep on MIRI MRS"""
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 
 from jwst.stpipe import Step
 

jwst/regtest/test_miri_setpointing.py

@@ -1,5 +1,5 @@
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from jwst.lib.set_telescope_pointing import add_wcs
 
 

jwst/regtest/test_miri_spectral_leak.py

@@ -1,6 +1,6 @@
 """Test SpectralLeakStep on MIRI MRS"""
 import pytest
-from astropy.io.fits.diff import FITSDiff
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
 from jwst.stpipe import Step
 
 @pytest.mark.bigdata