Skip to content

Commit 81c9554

Browse files
melanieclarkemelanieclarke
authored
Separate nsclean tests from standard runs (#9453)
2 parents ab86dd6 + 9c837cd commit 81c9554

File tree

3 files changed

+133
-17
lines changed

3 files changed

+133
-17
lines changed

‎jwst/regtest/test_nirspec_fs_spec2.py

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,6 @@ def run_pipeline(rtdata_module, request, resource_tracker):
5252
# Run the calwebb_spec2 pipeline; save results from intermediate steps
5353
args = ["calwebb_spec2", rtdata.input,
5454
"--steps.assign_wcs.save_results=true",
55-
"--steps.nsclean.skip=False",
56-
"--steps.nsclean.save_results=true",
5755
"--steps.extract_2d.save_results=true",
5856
"--steps.wavecorr.save_results=true",
5957
"--steps.srctype.save_results=true",
@@ -67,6 +65,28 @@ def run_pipeline(rtdata_module, request, resource_tracker):
6765
return rtdata
6866

6967

68+
@pytest.fixture(scope="module")
69+
def run_pipeline_nsclean(rtdata_module):
70+
"""Run the calwebb_spec2 pipeline on NIRSpec Fixed-Slit exposures with nsclean."""
71+
72+
rtdata = rtdata_module
73+
74+
filename = "jw01245-o002_20230107t223023_spec2_00001_asn.json"
75+
rtdata.get_asn('nirspec/fs/' + filename)
76+
77+
# Run the calwebb_spec2 pipeline with nsclean
78+
args = ["calwebb_spec2", rtdata.input,
79+
"--output_file=jw01245002001_04102_00002_nrs1_nsc",
80+
"--steps.nsclean.skip=False",
81+
"--steps.nsclean.save_results=true"]
82+
83+
# FIXME: Handle warnings properly.
84+
# Example: RuntimeWarning: overflow encountered in square
85+
Step.from_cmdline(args)
86+
87+
return rtdata
88+
89+
7090
@pytest.fixture(scope="module")
7191
def run_pipeline_pixel_replace(rtdata_module):
7292
"""Run the calwebb_spec2 pipeline on NIRSpec Fixed-Slit exposures including pixel replacement.
@@ -93,7 +113,7 @@ def test_log_tracked_resources_spec2(log_tracked_resources, run_pipeline):
93113

94114

95115
@pytest.mark.parametrize("suffix", [
96-
"assign_wcs", "nsclean", "extract_2d", "wavecorr", "flat_field", "pathloss", "srctype",
116+
"assign_wcs", "extract_2d", "wavecorr", "flat_field", "pathloss", "srctype",
97117
"cal", "s2d", "x1d"])
98118
def test_nirspec_fs_spec2(run_pipeline, fitsdiff_default_kwargs, suffix):
99119
"""Regression test of the calwebb_spec2 pipeline on a
@@ -113,6 +133,24 @@ def test_nirspec_fs_spec2(run_pipeline, fitsdiff_default_kwargs, suffix):
113133
assert diff.identical, diff.report()
114134

115135

136+
@pytest.mark.parametrize("suffix", ["nsclean", "cal", "s2d", "x1d"])
137+
def test_nirspec_fs_nsclean(run_pipeline_nsclean, fitsdiff_default_kwargs, suffix):
138+
"""Regression test of the calwebb_spec2 pipeline with nsclean."""
139+
140+
# Run the pipeline and retrieve outputs
141+
rtdata = run_pipeline_nsclean
142+
basename = "jw01245002001_04102_00002_nrs1_nsc"
143+
output = f"{basename}_{suffix}.fits"
144+
rtdata.output = output
145+
146+
# Get the truth files
147+
rtdata.get_truth(os.path.join("truth/test_nirspec_fs_spec2", output))
148+
149+
# Compare the results
150+
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
151+
assert diff.identical, diff.report()
152+
153+
116154
@pytest.mark.parametrize(
117155
'output',
118156
['jw013090_prtest_04102_00004_nrs2_pixel_replace.fits',

‎jwst/regtest/test_nirspec_ifu_spec2.py

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import pytest
33

44
from jwst.regtest import regtestdata as rt
5+
from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
56

67
# Define artifactory source and truth
78
INPUT_PATH = 'nirspec/ifu'
@@ -27,20 +28,43 @@ def run_spec2(rtdata_module, resource_tracker):
2728
'args': [
2829
'--steps.assign_wcs.save_results=true',
2930
'--steps.msa_flagging.save_results=true',
30-
'--steps.nsclean.skip=False',
31-
'--steps.nsclean.save_results=true',
3231
'--steps.srctype.save_results=true',
3332
'--steps.flat_field.save_results=true',
3433
'--steps.pathloss.save_results=true',
3534
]
3635
}
36+
3737
# FIXME: Handle warnings properly.
38-
# Example: RuntimeWarning: Invalid interval: upper bound XXX is strictly less than lower bound XXX
38+
# Example: RuntimeWarning: overflow encountered in multiply
3939
with resource_tracker.track():
4040
rtdata = rt.run_step_from_dict(rtdata, **step_params)
4141
return rtdata
4242

4343

44+
@pytest.fixture(scope='module')
45+
def run_spec2_nsclean(rtdata_module, resource_tracker):
46+
"""Run the Spec2Pipeline on a spec2 ASN containing a single exposure"""
47+
rtdata = rtdata_module
48+
49+
# Set up the inputs
50+
asn_name = 'jw01251-o004_20221105t023552_spec2_031_asn.json'
51+
asn_path = INPUT_PATH + '/' + asn_name
52+
53+
# Run the pipeline
54+
step_params = {
55+
'input_path': asn_path,
56+
'step': 'calwebb_spec2',
57+
'args': [
58+
'--output_file=jw01251004001_03107_00002_nrs1_nsc',
59+
'--steps.nsclean.skip=False',
60+
'--steps.nsclean.save_results=true',
61+
]
62+
}
63+
64+
rtdata = rt.run_step_from_dict(rtdata, **step_params)
65+
return rtdata
66+
67+
4468
@pytest.mark.slow
4569
def test_log_tracked_resources_spec2(log_tracked_resources, run_spec2):
4670
log_tracked_resources()
@@ -50,14 +74,32 @@ def test_log_tracked_resources_spec2(log_tracked_resources, run_spec2):
5074
@pytest.mark.parametrize(
5175
'suffix',
5276
['assign_wcs', 'cal', 'flat_field', 'msa_flagging',
53-
'nsclean', 'pathloss', 's3d', 'srctype', 'x1d']
77+
'pathloss', 's3d', 'srctype', 'x1d']
5478
)
5579
def test_spec2(run_spec2, fitsdiff_default_kwargs, suffix):
5680
"""Regression test matching output files"""
5781
rt.is_like_truth(run_spec2, fitsdiff_default_kwargs, suffix,
5882
truth_path=TRUTH_PATH)
5983

6084

85+
@pytest.mark.slow
86+
@pytest.mark.parametrize('suffix', ['nsclean', 'cal', 's3d', 'x1d'])
87+
def test_spec2_nsclean(run_spec2_nsclean, fitsdiff_default_kwargs, suffix):
88+
"""Regression test matching output files"""
89+
90+
# Run the pipeline and retrieve outputs
91+
rtdata = run_spec2_nsclean
92+
output = f"jw01251004001_03107_00002_nrs1_nsc_{suffix}.fits"
93+
rtdata.output = output
94+
95+
# Get the truth files
96+
rtdata.get_truth(f"{TRUTH_PATH}/{output}")
97+
98+
# Compare the results
99+
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
100+
assert diff.identical, diff.report()
101+
102+
61103
@pytest.fixture
62104
def run_photom(rtdata):
63105
"""Run the photom step on an NRS IFU exposure with SRCTYPE=POINT"""

‎jwst/regtest/test_nirspec_mos_spec2.py

Lines changed: 46 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
2-
from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
3-
from astropy.utils.exceptions import AstropyUserWarning
42

3+
from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
54
from jwst.stpipe import Step
65

76
# Mark all tests in this module
@@ -24,20 +23,40 @@ def run_pipeline(rtdata_module, resource_tracker):
2423
args = ["calwebb_spec2", rtdata.input,
2524
"--steps.assign_wcs.save_results=true",
2625
"--steps.msa_flagging.save_results=true",
27-
"--steps.nsclean.skip=False",
28-
"--steps.nsclean.save_results=true",
2926
"--steps.bkg_subtract.save_results=true",
3027
"--steps.extract_2d.save_results=true",
3128
"--steps.srctype.save_results=true",
3229
"--steps.wavecorr.save_results=true",
3330
"--steps.flat_field.save_results=true",
3431
"--steps.pathloss.save_results=true",
3532
"--steps.barshadow.save_results=true"]
36-
# FIXME: Handle warnings properly.
37-
# Example: RuntimeWarning: Invalid interval: upper bound XXX is strictly less than lower bound XXX
38-
with pytest.warns(AstropyUserWarning, match="Input data contains invalid values"):
39-
with resource_tracker.track():
40-
Step.from_cmdline(args)
33+
34+
with resource_tracker.track():
35+
Step.from_cmdline(args)
36+
37+
return rtdata
38+
39+
40+
@pytest.fixture(scope="module")
41+
def run_pipeline_nsclean(rtdata_module, resource_tracker):
42+
"""Run the calwebb_spec2 pipeline on NIRSpec MOS with nsclean."""
43+
44+
rtdata = rtdata_module
45+
46+
# Get the MSA metadata file referenced in the input exposure
47+
rtdata.get_data("nirspec/mos/jw01345066001_01_msa.fits")
48+
49+
# Get the input ASN file and exposures
50+
rtdata.get_asn("nirspec/mos/jw01345-o066_20230831t181155_spec2_00010_asn.json")
51+
52+
# Run the calwebb_spec2 pipeline; save results from intermediate steps
53+
args = ["calwebb_spec2", rtdata.input,
54+
"--output_file=jw01345066001_05101_00003_nrs1_nsc",
55+
"--steps.nsclean.skip=False",
56+
"--steps.nsclean.save_results=True"]
57+
58+
with resource_tracker.track():
59+
Step.from_cmdline(args)
4160

4261
return rtdata
4362

@@ -47,7 +66,7 @@ def test_log_tracked_resources(log_tracked_resources, run_pipeline):
4766

4867

4968
@pytest.mark.parametrize("suffix", [
50-
"assign_wcs", "msa_flagging", "nsclean", "bsub", "extract_2d", "wavecorr", "flat_field",
69+
"assign_wcs", "msa_flagging", "bsub", "extract_2d", "wavecorr", "flat_field",
5170
"srctype", "pathloss", "barshadow", "cal", "s2d", "x1d"])
5271
def test_nirspec_mos_spec2(run_pipeline, fitsdiff_default_kwargs, suffix):
5372
"""Regression test of the calwebb_spec2 pipeline on a
@@ -66,3 +85,20 @@ def test_nirspec_mos_spec2(run_pipeline, fitsdiff_default_kwargs, suffix):
6685
# Compare the results
6786
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
6887
assert diff.identical, diff.report()
88+
89+
90+
@pytest.mark.parametrize("suffix", ["nsclean", "cal", "s2d", "x1d"])
91+
def test_nirspec_mos_spec2_nsclean(run_pipeline_nsclean, fitsdiff_default_kwargs, suffix):
92+
"""Regression test of the calwebb_spec2 pipeline with nsclean."""
93+
94+
# Run the pipeline and retrieve outputs
95+
rtdata = run_pipeline_nsclean
96+
output = f"jw01345066001_05101_00003_nrs1_nsc_{suffix}.fits"
97+
rtdata.output = output
98+
99+
# Get the truth files
100+
rtdata.get_truth("truth/test_nirspec_mos_spec2/" + output)
101+
102+
# Compare the results
103+
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
104+
assert diff.identical, diff.report()

0 commit comments

Comments
 (0)