finalized calib diagnostics by finishing the offset outlier detection (c413e59d) · Commits · sns-hfir-scse / diffraction / pg3_calibration

cal_config.json

+12 −6

Original line number	Diff line number	Diff line
		@@ -21,27 +21,33 @@
		"Diagnostics": {
		"PAC": {
		"NumMasks": [3790, 200],
		"NumBadUnmasked": 50
		"NumBadUnmasked": 50,
		"NumOutliers": 20
		},
		"MAG": {
		"NumMasks": [10400, 250],
		"NumBadUnmasked": 300
		"NumBadUnmasked": 300,
		"NumOutliers": 20
		},
		"OC": {
		"NumMasks": [4340, 200],
		"NumBadUnmasked": 50
		"NumBadUnmasked": 50,
		"NumOutliers": 20
		},
		"MICAS": {
		"NumMasks": [3250, 200],
		"NumBadUnmasked": 350
		"NumBadUnmasked": 350,
		"NumOutliers": 20
		},
		"LTJANIS": {
		"NumMasks": [4050, 200],
		"NumBadUnmasked": 50
		"NumBadUnmasked": 50,
		"NumOutliers": 20
		},
		"JANISGAS": {
		"NumMasks": [3600, 200],
		"NumBadUnmasked": 30
		"NumBadUnmasked": 30,
		"NumOutliers": 20
		}
		}
		}

utils/pg3_calib_doctor.py

+157 −19

Original line number	Diff line number	Diff line
		@@ -15,12 +15,14 @@ from slack_sdk import WebClient


		def plot_diagnostics(run_num, out_dir, num_masks, bad_diag_input, good_diag_input,
		rel_offset, diag_dict, output_html=None):
		rel_offset, diag_dict, outlier_pixel_indices=None, outlier_peak_map=None,
		output_html=None):
		"""
		Creates a combined diagnostic HTML with three subplots:
		Creates a combined diagnostic HTML with four subplots:
		1. Unmasked bad pixels spectra
		2. Masked good pixels spectra
		3. Relative d-spacing offset per pixel index
		4. Outlier pixel spectra

		Parameters
		----------
		@@ -38,6 +40,10 @@ def plot_diagnostics(run_num, out_dir, num_masks, bad_diag_input, good_diag_inpu
		Full array of relative d-spacing offsets for all pixels (0.0 for masked pixels).
		diag_dict : dict
		Diagnostic thresholds dictionary.
		outlier_pixel_indices : list or None
		Pixel indices identified as outliers in the offset plot.
		outlier_peak_map : dict or None
		Mapping {pixel_index: nominal_d} for each outlier pixel.
		output_html : str, optional
		Output HTML file path.
		"""
		@@ -57,14 +63,26 @@ def plot_diagnostics(run_num, out_dir, num_masks, bad_diag_input, good_diag_inpu
		_IQR = _Q3 - _Q1
		num_outliers = int(np.sum((_unmasked > _Q3 + 1.5 * _IQR) \| (_unmasked < _Q1 - 1.5 * _IQR)))

		# Build color palette: one color per unique nominal d-spacing group
		_outlier_count = len(outlier_pixel_indices) if outlier_pixel_indices else 0
		_group_colors = {}
		if outlier_peak_map:
		_unique_peaks = sorted(set(outlier_peak_map.values()))
		_palette = [
		'#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00',
		'#a65628', '#f781bf', '#00ced1', '#8b008b', '#2e8b57'
		]
		for _i, _peak in enumerate(_unique_peaks):
		_group_colors[_peak] = _palette[_i % len(_palette)]
		fig = make_subplots(
		rows=3, cols=1,
		rows=4, cols=1,
		subplot_titles=[
		f"Unmasked Bad Pixels (Count: {bad_count})",
		f"Masked Good Pixels (Count: {good_count})",
		f"Relative d-spacing Offset per Pixel (Outliers: {num_outliers})"
		f"Relative d-spacing Offset per Pixel (Outliers: {num_outliers})",
		f"Outlier Pixel Spectra (Count: {_outlier_count})"
		],
		vertical_spacing=0.08
		vertical_spacing=0.06
		)

		def add_spectra_traces(pattern, row):
		@@ -165,6 +183,29 @@ def plot_diagnostics(run_num, out_dir, num_masks, bad_diag_input, good_diag_inpu
		row=3, col=1
		)
		if outlier_mask.any():
		# Color each outlier marker by its nominal-peak group
		if outlier_peak_map and _group_colors:
		_shown_groups = set()
		for _pidx, _oidx in zip(unmasked_indices[outlier_mask], unmasked_offset[outlier_mask]):
		_d = outlier_peak_map.get(int(_pidx))
		_col = _group_colors.get(_d, 'red') if _d is not None else 'red'
		_gname = f'Outlier (ref {_d:.4f} Å)' if _d is not None else 'Outlier'
		_show = _gname not in _shown_groups
		_shown_groups.add(_gname)
		fig.add_trace(
		go.Scatter(
		x=[int(_pidx)],
		y=[float(_oidx)],
		mode='markers',
		name=_gname,
		showlegend=_show,
		legendgroup=_gname,
		marker=dict(size=9, color=_col, symbol='x'),
		hovertemplate=f'Pixel: %{{x}}<br>\|Rel. Offset\|: %{{y:.6f}}<extra></extra>'
		),
		row=3, col=1
		)
		else:
		fig.add_trace(
		go.Scatter(
		x=unmasked_indices[outlier_mask].tolist(),
		@@ -191,18 +232,77 @@ def plot_diagnostics(run_num, out_dir, num_masks, bad_diag_input, good_diag_inpu
		row=3, col=1
		)

		# Outlier spectra panel: load files per pixel, color by nominal-peak group
		if outlier_pixel_indices and outlier_peak_map:
		_shown_spec_groups = set()
		for _pix in outlier_pixel_indices:
		_path = os.path.join(out_dir, f"PG3_{run_num}_outlier_pixels_{_pix}.dat")
		if not os.path.exists(_path):
		continue
		_d = outlier_peak_map.get(_pix)
		_col = _group_colors.get(_d, 'grey') if _d is not None else 'grey'
		_gname = f'Outlier group {_d:.4f} Å' if _d is not None else 'Outlier'
		_show = _gname not in _shown_spec_groups
		_shown_spec_groups.add(_gname)
		_x_vals, _y_vals = [], []
		try:
		with open(_path, 'r', encoding='utf-8') as _f:
		_lines = _f.readlines()
		for _line in _lines[2:]:
		_line = _line.strip()
		if not _line:
		continue
		_parts = _line.split(',')
		if len(_parts) >= 2:
		try:
		_x_vals.append(float(_parts[0]))
		_y_vals.append(float(_parts[1]))
		except ValueError:
		continue
		if _x_vals:
		fig.add_trace(
		go.Scatter(
		x=_x_vals,
		y=_y_vals,
		mode='lines',
		name=_gname,
		showlegend=_show,
		legendgroup=_gname,
		line=dict(color=_col),
		hovertemplate='X: %{x}<br>Y: %{y}<extra></extra>'
		),
		row=4, col=1
		)
		except Exception as _e:
		print(f"Error reading {os.path.basename(_path)}: {_e}")
		fig.add_trace(
		go.Scatter(x=[], y=[], mode='lines', showlegend=False, hoverinfo='skip'),
		row=4, col=1
		)
		if _group_colors:
		for _d_val, _col in _group_colors.items():
		fig.add_vline(
		x=_d_val, row=4, col=1,
		line_dash='dash', line_color=_col, line_width=2.0,
		annotation_text=f'{_d_val:.4f} Å',
		annotation_position='top right',
		annotation_font_color=_col
		)

		fig.update_layout(
		title=f"PG3 Calibration Diagnostics — Run {run_num} \| Total Masks: {num_masks}",
		height=1800,
		height=2400,
		hovermode="closest",
		legend=dict(title="Spectrum / Offset", traceorder="normal")
		)
		fig.update_xaxes(title_text="d (Å)", row=1, col=1)
		fig.update_xaxes(title_text="d (Å)", row=2, col=1)
		fig.update_xaxes(title_text="Detector Pixel Index", row=3, col=1)
		fig.update_xaxes(title_text="d (Å)", row=4, col=1)
		fig.update_yaxes(title_text="Counts", row=1, col=1)
		fig.update_yaxes(title_text="Counts", row=2, col=1)
		fig.update_yaxes(title_text="\|Relative Offset\|", row=3, col=1)
		fig.update_yaxes(title_text="Counts", row=4, col=1)

		fig.write_html(output_html)
		print(f"[Info] Successfully saved combined diagnostics plot to {output_html}")
		@@ -212,8 +312,9 @@ def plot_diagnostics(run_num, out_dir, num_masks, bad_diag_input, good_diag_inpu
		condt_2 = num_masks > diag_dict["NumMasks"][0] + diag_dict["NumMasks"][1]
		condt_bad = bad_diag_input is not None and bad_diag_input[1] > diag_dict["NumBadUnmasked"]
		condt_good = good_diag_input is not None and good_diag_input[1] > 0
		condt_outliers = num_outliers > diag_dict["NumOutliers"]

		post_warning = condt_1 or condt_2 or condt_bad or condt_good
		post_warning = condt_1 or condt_2 or condt_bad or condt_good or condt_outliers

		if post_warning:
		trigger_msg = ""
		@@ -226,6 +327,13 @@ def plot_diagnostics(run_num, out_dir, num_masks, bad_diag_input, good_diag_inpu
		if trigger_msg:
		trigger_msg += "\n"
		trigger_msg += f"⚠️ Trigger: Number of masked good pixels ({good_diag_input[1]}) is non-zero."
		if condt_outliers:
		if trigger_msg:
		trigger_msg += "\n"
		trigger_msg += (
		f"⚠️ Trigger: Number of offset outliers ({num_outliers}) "
		f"exceeds expected threshold ({diag_dict['NumOutliers']})."
		)
		if condt_1 or condt_2:
		if trigger_msg:
		trigger_msg += "\n"
		@@ -445,11 +553,41 @@ def pg3_calib_doctor(run_num, cal_file, manual_mask, out_dir, diag_dict):

		rel_offset_arr = np.array(rel_offset)

		# Identify outlier pixels and extract their spectra
		_unmasked_idx = np.where(rel_offset_arr != 0)[0]
		_unmasked_vals = rel_offset_arr[_unmasked_idx]
		outlier_pixel_indices = []
		outlier_peak_map = {}
		if len(_unmasked_vals) > 0:
		_Q1 = np.percentile(_unmasked_vals, 5)
		_Q3 = np.percentile(_unmasked_vals, 95)
		_IQR = _Q3 - _Q1
		_out_mask = (_unmasked_vals > _Q3 + 1.5 * _IQR) \| (_unmasked_vals < _Q1 - 1.5 * _IQR)
		outlier_pixel_indices = _unmasked_idx[_out_mask].tolist()
		outlier_peak_map = {}
		for _pix in outlier_pixel_indices:
		_cidx = get_chunk_index(_pix)
		if _cidx >= 0:
		outlier_peak_map[_pix] = pg3_chunks[_cidx][2][0]
		if outlier_pixel_indices:
		ExtractSpectra(
		InputWorkspace="pg3_wksp_d",
		OutputWorkspace="pg3_wksp_d_outliers",
		WorkspaceIndexList=outlier_pixel_indices
		)
		SaveAscii(
		InputWorkspace="pg3_wksp_d",
		Filename=os.path.join(run_out_dir, f"PG3_{run_num}_outlier_pixels.dat"),
		SpectrumList=outlier_pixel_indices,
		OneSpectrumPerFile=True
		)
		print(f"[Info] # of outlier pixels: {len(outlier_pixel_indices)}")

		bad_diag_input = [num_masks, len(bad_pixels_unmasked)] if bad_pixels_unmasked else None
		good_diag_input = [num_masks, len(good_pixels_masked)] if good_pixels_masked else None

		plot_diagnostics(run_num, run_out_dir, num_masks, bad_diag_input, good_diag_input,
		rel_offset_arr, diag_dict)
		rel_offset_arr, diag_dict, outlier_pixel_indices, outlier_peak_map)

		return