MUSE Pipeline Reference Manual  1.0.2
muse_wavecalib.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #if HAVE_POPEN && HAVE_PCLOSE
30 #define _BSD_SOURCE /* force popen/pclose, mkdtemp definitions from stdio/stdlib */
31 #endif
32 #include <cpl.h>
33 #include <math.h>
34 #include <string.h>
35 
36 #include "muse_wavecalib.h"
37 #include "muse_instrument.h"
38 
39 #include "muse_combine.h"
40 #include "muse_cplwrappers.h"
41 #include "muse_data_format_z.h"
42 #include "muse_dfs.h"
43 #include "muse_pfits.h"
44 #include "muse_tracing.h"
45 #include "muse_utils.h"
46 
47 /*----------------------------------------------------------------------------*
48  * Debugging Macros *
49  * Set these to 1 or higher for (lots of) debugging output *
50  *----------------------------------------------------------------------------*/
51 #define SEARCH_DEBUG 0 /* debugging in muse_wave_lines_search(), 1 or higher */
52 #define SEARCH_DEBUG_FILES 0 /* save different versions of the columns into *
53  * FITS files in muse_wave_lines_search() */
54 #define DEBUG_GAUSSFIT 0 /* debugging the Gaussian fit in muse_wave_lines_search() */
55 #define MUSE_WAVE_LINES_SEARCH_SHIFT_WARN 3.0 /* [pix] output debug message if *
56  * a shift of more than this occurs */
57 #define MUSE_WAVE_LINE_FIT_MAXSHIFT 2.0 /* [pix] don't use the fit when a shift *
58  * of more than this is detected */
59 #define MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT 0.25 /* [pix] maximum shift between *
60  * adjacent CCD columns */
61 
62 /*----------------------------------------------------------------------------*/
66 /*----------------------------------------------------------------------------*/
67 
70 /*----------------------------------------------------------------------------*/
95 /*----------------------------------------------------------------------------*/
97  { "lampno", CPL_TYPE_INT, "", "%d", "Number of the lamp", CPL_TRUE },
98  { "lampname", CPL_TYPE_STRING, "", "%s", "Name of the lamp", CPL_TRUE },
99  { "x", CPL_TYPE_DOUBLE, "pix", "%.2f", "x-position on CCD", CPL_TRUE },
100  { "y", CPL_TYPE_DOUBLE, "pix", "%.2f", "first-guess y-position on CCD", CPL_TRUE },
101  { "peak", CPL_TYPE_DOUBLE, "pix", "%g", "Peak of line", CPL_TRUE },
102  { "center", CPL_TYPE_DOUBLE, "pix", "%.4f", "Gaussian line center", CPL_TRUE },
103  { "cerr", CPL_TYPE_DOUBLE, "pix", "%.4f", "error estimate of line center", CPL_TRUE },
104  { "sigma", CPL_TYPE_DOUBLE, "pix", "%.3f", "Gaussian sigma", CPL_TRUE },
105  { "fwhm", CPL_TYPE_DOUBLE, "pix", "%.3f", "Gaussian FWHM", CPL_TRUE },
106  { "flux", CPL_TYPE_DOUBLE, "count", "%g", "Gaussian area (flux)", CPL_TRUE },
107  { "bg", CPL_TYPE_DOUBLE, "count", "%.2f", "background level", CPL_TRUE },
108  { "mse", CPL_TYPE_DOUBLE, "", "%e", "mean squared error", CPL_TRUE },
109  { "lambda", CPL_TYPE_DOUBLE, "Angstrom", "%9.3f",
110  "identified wavelength of the line", CPL_TRUE },
111  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
112 };
113 
114 /*----------------------------------------------------------------------------*/
118 /*----------------------------------------------------------------------------*/
120  { "slice", CPL_TYPE_INT, "", "%02d", "slice number", CPL_TRUE},
121  { "iteration", CPL_TYPE_INT, "", "%d", "iteration", CPL_TRUE},
122  { "x", CPL_TYPE_INT, "pix", "%04d", "x-position on CCD", CPL_TRUE},
123  { "y", CPL_TYPE_FLOAT, "pix", "%8.3f", "y-position on CCD", CPL_TRUE},
124  { "lambda", CPL_TYPE_FLOAT, "Angstrom", "%8.3f", "wavelength", CPL_TRUE},
125  { "residual", CPL_TYPE_DOUBLE, "Angstrom", "%.4e", "residual at this point", CPL_TRUE},
126  { "rejlimit", CPL_TYPE_DOUBLE, "Angstrom", "%.4e",
127  "rejection limit for this iteration", CPL_TRUE},
128  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
129 };
130 
131 static void muse_wave_lines_add_flux_for_lsf(cpl_table *, cpl_table *);
132 
133 /* corresponds to muse_wave_weighting_type, *
134  * keep in sync with those values! */
135 const char *muse_wave_weighting_string[] = {
136  "uniform",
137  "centroid error",
138  "per-line RMS scatter",
139  "centroid error plus per-line RMS scatter"
140 };
141 
142 /*----------------------------------------------------------------------------*/
155 /*----------------------------------------------------------------------------*/
158 {
159  muse_wave_params *p = cpl_malloc(sizeof(muse_wave_params));
160  /* set defaults */
161  p->xorder = 2;
162  p->yorder = 6;
163  p->detsigma = 1.;
164  p->ddisp = 0.05; /* from 1.20...1.30 Angstrom/pixel */
165  p->tolerance = 0.1;
166  p->linesigma = -1.;
167  p->rflag = CPL_FALSE; /* do not create a residuals table by default */
168  p->residuals = NULL;
169  p->fitsigma = -1.;
171  p->targetrms = 0.03;
172  return p;
173 }
174 
175 /*----------------------------------------------------------------------------*/
185 /*----------------------------------------------------------------------------*/
186 void
188 {
189  if (!aParams) {
190  return;
191  }
192  cpl_table_delete(aParams->residuals);
193  aParams->residuals = NULL;
194  memset(aParams, 0, sizeof(muse_wave_params)); /* null out everything */
195  cpl_free(aParams);
196 }
197 
198 /*----------------------------------------------------------------------------*/
210 /*----------------------------------------------------------------------------*/
211 static void
212 muse_wave_calib_qc_peaks(cpl_table *aLines, cpl_propertylist *aHeader,
213  const unsigned short aSlice)
214 {
215  char keyword[KEYWORD_LENGTH];
216  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MEAN, aSlice);
217  cpl_propertylist_append_float(aHeader, keyword,
218  cpl_table_get_column_mean(aLines, "peak"));
219  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_STDEV, aSlice);
220  cpl_propertylist_append_float(aHeader, keyword,
221  cpl_table_get_column_stdev(aLines, "peak"));
222  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MIN, aSlice);
223  cpl_propertylist_append_float(aHeader, keyword,
224  cpl_table_get_column_min(aLines, "peak"));
225  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MAX, aSlice);
226  cpl_propertylist_append_float(aHeader, keyword,
227  cpl_table_get_column_max(aLines, "peak"));
228  if (aSlice != 20) {
229  return;
230  }
231  /* if we are in slice 20 (a slice that is near the middle of a stack, so
232  * should never be vignetted loop through all possible lamps */
233  int n, nlamps = muse_pfits_get_lampnum(aHeader);
234  for (n = 1; n < nlamps; n++) {
235  /* select and extract all table entries matching the lamp */
236  cpl_table_unselect_all(aLines);
237  cpl_table_or_selected_int(aLines, "lampno", CPL_EQUAL_TO, n);
238  cpl_table *lamplines = cpl_table_extract_selected(aLines);
239  if (cpl_table_get_nrow(lamplines) < 1) {
240  cpl_table_delete(lamplines);
241  continue;
242  }
243  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_MEAN,
244  aSlice, n);
245  cpl_propertylist_append_float(aHeader, keyword,
246  cpl_table_get_column_mean(lamplines, "peak"));
247  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_STDEV,
248  aSlice, n);
249  cpl_propertylist_append_float(aHeader, keyword,
250  cpl_table_get_column_stdev(lamplines, "peak"));
251  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_MAX,
252  aSlice, n);
253  cpl_propertylist_append_float(aHeader, keyword,
254  cpl_table_get_column_max(lamplines, "peak"));
255  cpl_table_delete(lamplines);
256  } /* for n (all lamps) */
257 } /* muse_wave_calib_qc_peaks() */
258 
259 /*----------------------------------------------------------------------------*/
268 /*----------------------------------------------------------------------------*/
269 static void
270 muse_wave_calib_qc_fwhm_old(cpl_table *aLines, cpl_propertylist *aHeader,
271  cpl_vector *aRefLam, const unsigned short aSlice)
272 {
273  int nlines = cpl_table_get_nrow(aLines);
274 
275  /* track the minimum and maximum resolutions and their wavelengths */
276  double Rmin = DBL_MAX, Rmax = -DBL_MAX, wlmin = DBL_MAX, wlmax = -DBL_MAX;
277 
278  /* convert the properties of the relevant lines from the respective *
279  * table columns into vectors for easy statistics computation */
280  cpl_vector *fwhms = cpl_vector_new(nlines),
281  *resol = cpl_vector_new(cpl_vector_get_size(aRefLam));
282  int i, iresol = 0;
283  for (i = 0; i < nlines; i++) {
284  /* FWHM is directly accessible, but needs to be converted to Angstrom */
285  cpl_errorstate prestate = cpl_errorstate_get();
286  double fwhm = cpl_table_get(aLines, "fwhm", i, NULL), /* in [pix] */
287  sampling = kMuseSpectralSamplingA, /* sensible default in [A/pix] */
288  lambda = cpl_table_get(aLines, "lambda", i, NULL),
289  s1 = (lambda - cpl_table_get(aLines, "lambda", i - 1, NULL))
290  / (cpl_table_get(aLines, "center", i, NULL)
291  - cpl_table_get(aLines, "center", i - 1, NULL)),
292  s2 = (cpl_table_get(aLines, "lambda", i + 1, NULL)
293  - cpl_table_get(aLines, "lambda", i, NULL))
294  / (cpl_table_get(aLines, "center", i + 1, NULL)
295  - cpl_table_get(aLines, "center", i, NULL));
296  if (!cpl_errorstate_is_equal(prestate)) {
297  cpl_errorstate_set(prestate); /* reset "Access beyond boundaries" errors */
298  }
299  if (i == 0) { /* no lower one */
300  sampling = s2;
301  } else if (i == nlines - 1) { /* no higher one */
302  sampling = s1;
303  } else {
304  sampling = (s1 + s2) / 2.;
305  }
306  fwhm *= sampling; /* now in [A] */
307  cpl_vector_set(fwhms, i, fwhm);
308  /* spectral resolution R for this line R = lambda / dlambda */
309  double R = lambda / fwhm;
310  /* compare wavelength to those lines in the FWHM reference list */
311  if (fabs(cpl_vector_get(aRefLam, cpl_vector_find(aRefLam, lambda)) - lambda)
312  < FLT_EPSILON) {
313  if (R < Rmin) {
314  Rmin = R;
315  wlmin = lambda;
316  }
317  if (R > Rmax) {
318  Rmax = R;
319  wlmax = lambda;
320  }
321  cpl_vector_set(resol, iresol++, R);
322  }
323  } /* for i (all arc lines) */
324  cpl_vector_set_size(resol, iresol);
325 
326  char keyword[KEYWORD_LENGTH];
327  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MEAN, aSlice);
328  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_mean(fwhms));
329  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_STDEV, aSlice);
330  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_stdev(fwhms));
331  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MIN, aSlice);
332  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_min(fwhms));
333  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MAX, aSlice);
334  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_max(fwhms));
335  cpl_vector_delete(fwhms);
336 
337  cpl_msg_debug(__func__, "Average spectral resolution in IFU %hhu is R=%4.0f, "
338  "ranging from %4.0f (at %6.1fA) to %4.0f (at %6.1fA)",
339  muse_utils_get_ifu(aHeader), cpl_vector_get_mean(resol),
340  Rmin, wlmin, Rmax, wlmax);
341  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, aSlice);
342  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_mean(resol));
343  cpl_vector_delete(resol);
344 } /* muse_wave_calib_qc_fwhm_old() */
345 
346 /*----------------------------------------------------------------------------*/
354 /*----------------------------------------------------------------------------*/
355 static void
356 muse_wave_calib_qc_fwhm(cpl_table *aFWHM, cpl_propertylist *aHeader,
357  const unsigned short aSlice)
358 {
359  /* compute the spectral resolution at each wavelength, as *
360  * R = lambda / fwhm *
361  * where both lambda and fwhm are [Angstrom] */
362  cpl_table_duplicate_column(aFWHM, "R", aFWHM, "lambda");
363  cpl_table_divide_columns(aFWHM, "R", "fwhm");
364  cpl_table_set_column_unit(aFWHM, "R", "");
365 
366 #if 0 /* AIT Gaussian FWHM */
367  /* range < 600 nm (as in PR5) */
368  cpl_table_unselect_all(aFWHM);
369  cpl_table_or_selected_double(aFWHM, "lambda", CPL_LESS_THAN, 6000.);
370  cpl_table *tblue = cpl_table_extract_selected(aFWHM);
371  /* range > 800 nm (as in PR5) */
372  cpl_table_unselect_all(aFWHM);
373  cpl_table_or_selected_double(aFWHM, "lambda", CPL_GREATER_THAN, 8000.);
374  cpl_table *tred = cpl_table_extract_selected(aFWHM);
375  /* range 600 - 800 nm (as in PR5) */
376  cpl_table_unselect_all(aFWHM);
377  cpl_table_or_selected_double(aFWHM, "lambda", CPL_NOT_LESS_THAN, 6000.);
378  cpl_table_and_selected_double(aFWHM, "lambda", CPL_NOT_GREATER_THAN, 8000.);
379  cpl_table *tgreen = cpl_table_extract_selected(aFWHM);
380 #if 0
381  printf("all:\n");
382  cpl_table_dump(aFWHM, 0, 100000, stdout);
383  fflush(stdout);
384  printf("blue:\n");
385  cpl_table_dump(tblue, 0, 100000, stdout);
386  fflush(stdout);
387  printf("green:\n");
388  cpl_table_dump(tgreen, 0, 100000, stdout);
389  fflush(stdout);
390  printf("red:\n");
391  cpl_table_dump(tred, 0, 100000, stdout);
392  fflush(stdout);
393 #endif
394 #endif /* AIT Gaussian FWHM */
395 
396  /* fill the QC parameters */
397  char keyword[KEYWORD_LENGTH];
398  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, aSlice);
399  double rmean = cpl_table_get_column_mean(aFWHM, "R");
400  cpl_propertylist_update_float(aHeader, keyword, rmean);
401 
402  double fmean = cpl_table_get_column_mean(aFWHM, "fwhm"),
403  fstdev = cpl_table_get_column_stdev(aFWHM, "fwhm"),
404  flo = cpl_table_get_column_min(aFWHM, "fwhm"),
405  fhi = cpl_table_get_column_max(aFWHM, "fwhm");
406  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MEAN, aSlice);
407  cpl_propertylist_update_float(aHeader, keyword, fmean);
408  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_STDEV, aSlice);
409  cpl_propertylist_update_float(aHeader, keyword, fstdev);
410  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MIN, aSlice);
411  cpl_propertylist_update_float(aHeader, keyword, flo);
412  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MAX, aSlice);
413  cpl_propertylist_update_float(aHeader, keyword, fhi);
414 
415 #if 0 /* AIT Gaussian FWHM */
416  /* output all the stuff as debug messages */
417  cpl_msg_debug(__func__, "Gaussian FWHM [%s]:\n"
418  "\tFWHM all (%d values)\t%.3f +/- %.3f (%.3f) %.3f...%.3f\n"
419  "\tFWHM blue (%d values)\t%.3f +/- %.3f (%.3f)\n"
420  "\tFWHM green (%d values)\t%.3f +/- %.3f (%.3f)\n"
421  "\tFWHM red (%d values)\t%.3f +/- %.3f (%.3f)",
422  cpl_table_get_column_unit(aFWHM, "fwhm"),
423  (int)cpl_table_get_nrow(aFWHM), fmean, fstdev, flo, fhi,
424  cpl_table_get_column_median(aFWHM, "fwhm"),
425  (int)cpl_table_get_nrow(tblue),
426  cpl_table_get_column_mean(tblue, "fwhm"),
427  cpl_table_get_column_stdev(tblue, "fwhm"),
428  cpl_table_get_column_median(tblue, "fwhm"),
429  (int)cpl_table_get_nrow(tgreen),
430  cpl_table_get_column_mean(tgreen, "fwhm"),
431  cpl_table_get_column_stdev(tgreen, "fwhm"),
432  cpl_table_get_column_median(tgreen, "fwhm"),
433  (int)cpl_table_get_nrow(tred),
434  cpl_table_get_column_mean(tred, "fwhm"),
435  cpl_table_get_column_stdev(tred, "fwhm"),
436  cpl_table_get_column_median(tred, "fwhm"));
437  cpl_msg_debug(__func__, "Gaussian spectral resolution R:\n"
438  "\tR all (%d values)\t%.1f +/- %.1f (%.1f) %.1f...%.1f\n"
439  "\tR blue (%d values)\t%.1f +/- %.1f (%.1f)\n"
440  "\tR green (%d values)\t%.1f +/- %.1f (%.1f)\n"
441  "\tR red (%d values)\t%.1f +/- %.1f (%.1f)",
442  (int)cpl_table_get_nrow(aFWHM), rmean,
443  cpl_table_get_column_stdev(aFWHM, "R"),
444  cpl_table_get_column_median(aFWHM, "R"),
445  cpl_table_get_column_min(aFWHM, "R"),
446  cpl_table_get_column_max(aFWHM, "R"),
447  (int)cpl_table_get_nrow(tblue),
448  cpl_table_get_column_mean(tblue, "R"),
449  cpl_table_get_column_stdev(tblue, "R"),
450  cpl_table_get_column_median(tblue, "R"),
451  (int)cpl_table_get_nrow(tgreen),
452  cpl_table_get_column_mean(tgreen, "R"),
453  cpl_table_get_column_stdev(tgreen, "R"),
454  cpl_table_get_column_median(tgreen, "R"),
455  (int)cpl_table_get_nrow(tred),
456  cpl_table_get_column_mean(tred, "R"),
457  cpl_table_get_column_stdev(tred, "R"),
458  cpl_table_get_column_median(tred, "R"));
459  cpl_table_delete(tblue);
460  cpl_table_delete(tgreen);
461  cpl_table_delete(tred);
462 #endif /* AIT Gaussian FWHM */
463  cpl_table_erase_column(aFWHM, "R");
464 } /* muse_wave_calib_qc_fwhm() */
465 
466 /*----------------------------------------------------------------------------*/
478 /*----------------------------------------------------------------------------*/
479 static void
480 muse_wave_calib_qc_lambda(muse_image *aImage, const unsigned short aSlice,
481  cpl_polynomial **aTrace, cpl_polynomial *aFit)
482 {
483  int ny = cpl_image_get_size_y(aImage->data);
484  unsigned char ifu = muse_utils_get_ifu(aImage->header);
485 
486  /* wavelength differences at bottom and top of each slice as QC parameters */
487  double xbotl = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT], 1, NULL),
488  xbotr = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT], 1, NULL),
489  xtopl = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT], ny, NULL),
490  xtopr = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT], ny, NULL);
491  cpl_vector *pos = cpl_vector_new(2);
492  cpl_vector_set(pos, 0, xbotl);
493  cpl_vector_set(pos, 1, 1);
494  double wlbotl = cpl_polynomial_eval(aFit, pos);
495  cpl_vector_set(pos, 0, xbotr);
496  cpl_vector_set(pos, 1, 1);
497  double wlbotr = cpl_polynomial_eval(aFit, pos);
498  cpl_vector_set(pos, 0, xtopl);
499  cpl_vector_set(pos, 1, ny);
500  double wltopl = cpl_polynomial_eval(aFit, pos);
501  cpl_vector_set(pos, 0, xtopr);
502  cpl_vector_set(pos, 1, ny);
503  double wltopr = cpl_polynomial_eval(aFit, pos);
504 #if 0
505  cpl_msg_debug(__func__, "Wavelengths at slice corners in slice %hu of IFU %hhu: "
506  "botl(%.2f,1)=%f, botr(%.2f,1)=%f, topl(%.2f,%d)=%f, topr(%.2f,%d)"
507  "=%f", aSlice, ifu, xbotl, wlbotl, xbotr, wlbotr,
508  xtopl, ny, wltopl, xtopr, ny, wltopr);
509 #endif
510  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_DWLEN_BOT, aSlice);
511  cpl_propertylist_append_float(aImage->header, keyword, wlbotl - wlbotr);
512  cpl_free(keyword);
513  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_DWLEN_TOP, aSlice);
514  cpl_propertylist_append_float(aImage->header, keyword, wltopl - wltopr);
515  cpl_free(keyword);
516 
517 #define DWLEN_WARN_LIMIT_N 6.5 /* 5.7 Angstrom differences can happen... */
518 #define DWLEN_WARN_LIMIT_E 5.9 /* ... at the bottom in both modes */
519  double limit = DWLEN_WARN_LIMIT_E;
520  cpl_errorstate prestate = cpl_errorstate_get();
521  if (muse_pfits_get_mode(aImage->header) != MUSE_MODE_WFM_NONAO_X) {
522  limit = DWLEN_WARN_LIMIT_N;
523  }
524  cpl_errorstate_set(prestate); /* swallow possible error about missing INS MODE */
525  if (fabs(wlbotl - wlbotr) > limit) {
526  cpl_msg_warning(__func__, "Wavelength differences at bottom of slice %hu "
527  "of IFU %hhu are large (%f Angstrom)!", aSlice, ifu,
528  wlbotl - wlbotr);
529  }
530  if (fabs(wltopl - wltopr) > DWLEN_WARN_LIMIT_E) { /* strict limit at red end */
531  cpl_msg_warning(__func__, "Wavelength differences at top of slice %hu of IFU "
532  "%hhu are large (%f Angstrom)!", aSlice, ifu, wltopl - wltopr);
533  }
534 
535  /* generate some arbitrary evaluation point, approx. at *
536  * the slice center, and compute the wavelength there. */
537  const double yc = (1. + ny) / 2.,
538  xc = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_CENTER], yc, NULL);
539  cpl_vector_set(pos, 0, xc);
540  cpl_vector_set(pos, 1, yc);
541  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_WLPOS, aSlice);
542  cpl_propertylist_append_float(aImage->header, keyword, yc);
543  cpl_free(keyword);
544  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_WLEN, aSlice);
545  cpl_propertylist_append_float(aImage->header, keyword,
546  cpl_polynomial_eval(aFit, pos));
547  cpl_free(keyword);
548  cpl_vector_delete(pos);
549 } /* muse_wave_calib_qc_lambda() */
550 
551 /*----------------------------------------------------------------------------*/
564 /*----------------------------------------------------------------------------*/
565 static void
566 muse_wave_calib_output_summary(const cpl_propertylist *aHeader,
567  const cpl_table *aWave)
568 {
569  unsigned char ifu = muse_utils_get_ifu(aHeader);
570  if (!aWave) {
571  cpl_msg_warning(__func__, "Wavelength solution missing for IFU %hhu, no "
572  "summary!", ifu);
573  return;
574  }
575  cpl_vector *found = cpl_vector_new(kMuseSlicesPerCCD),
576  *ident = cpl_vector_new(kMuseSlicesPerCCD),
577  *used = cpl_vector_new(kMuseSlicesPerCCD),
578  *resolution = cpl_vector_new(kMuseSlicesPerCCD);
579  unsigned short islice;
580  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
581  char keyword[KEYWORD_LENGTH];
582  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_NDET, islice + 1);
583  cpl_vector_set(found, islice, cpl_propertylist_get_int(aHeader, keyword));
584  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_NID, islice + 1);
585  cpl_vector_set(ident, islice, cpl_propertylist_get_int(aHeader, keyword));
586  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_FIT_NLINES, islice + 1);
587  cpl_vector_set(used, islice, cpl_propertylist_get_int(aHeader, keyword));
588  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, islice + 1);
589  cpl_vector_set(resolution, islice, cpl_propertylist_get_float(aHeader, keyword));
590  } /* for islice (all kMuseSlicesPerCCD) */
591  cpl_msg_info(__func__, "Summary of wavelength solution for IFU %hhu:\n"
592  "\tDetections per slice: %d ... %d\n"
593  "\tIdentified lines per slice: %d ... %d\n"
594  "\tLines per slice used in fit: %d ... %d\n"
595  "\tRMS of fit [Angstrom]: %5.3f +/- %5.3f (%5.3f ... %5.3f)\n"
596  "\tMean spectral resolution R: %6.1f +/- %5.1f", ifu,
597  (int)cpl_vector_get_min(found), (int)cpl_vector_get_max(found),
598  (int)cpl_vector_get_min(ident), (int)cpl_vector_get_max(ident),
599  (int)cpl_vector_get_min(used), (int)cpl_vector_get_max(used),
600  sqrt(cpl_table_get_column_mean(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
601  sqrt(cpl_table_get_column_stdev(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
602  sqrt(cpl_table_get_column_min(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
603  sqrt(cpl_table_get_column_max(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
604  cpl_vector_get_mean(resolution), cpl_vector_get_stdev(resolution));
605  cpl_vector_delete(found);
606  cpl_vector_delete(ident);
607  cpl_vector_delete(used);
608  cpl_vector_delete(resolution);
609 } /* muse_wave_calib_output_summary() */
610 
611 /*----------------------------------------------------------------------------*/
657 /*----------------------------------------------------------------------------*/
658 cpl_table *
659 muse_wave_calib(muse_image *aImage, cpl_table *aTrace, cpl_table *aLinelist,
660  muse_wave_params *aParams)
661 {
662  if (!aImage) {
663  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "arc image missing!");
664  return NULL;
665  }
666  unsigned char ifu = muse_utils_get_ifu(aImage->header);
667  /* variance always has to be larger than zero */
668  double minstat = cpl_image_get_min(aImage->stat);
669  if (minstat <= 0.) {
670  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "arc image %d does"
671  " not have valid STAT extension in IFU %hhu (minimum "
672  "is %e)!", 0, ifu, minstat);
673  return NULL;
674  }
675 
676  if (!aTrace) {
677  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "trace table missing "
678  "for IFU %hhu, cannot create wavelength calibration!",
679  ifu);
680  return NULL;
681  } else if (cpl_table_get_nrow(aTrace) != kMuseSlicesPerCCD) {
682  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "trace table "
683  "not valid for this dataset of IFU %hhu!", ifu);
684  return NULL;
685  }
686 
687  if (!aLinelist) {
688  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "no arc line list "
689  "supplied for IFU %hhu!", ifu);
690  return NULL;
691  }
692  if (!aParams) {
693  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "wavelength "
694  "calibration parameters missing for IFU %hhu!", ifu);
695  return NULL;
696  }
698  cpl_error_set_message(__func__, CPL_ERROR_UNSUPPORTED_MODE,
699  "unknown weighting scheme for IFU %hhu", ifu);
700  return NULL;
701  }
702 
703  /* do not support smaller mock datasets any more */
704  int nx = cpl_image_get_size_x(aImage->data),
705  ny = cpl_image_get_size_y(aImage->data);
706  if (ny < 4000) {
707  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "this dataset of "
708  "IFU %hhu is too small (%dx%d pix) to be supported",
709  ifu, nx, ny);
710  return NULL;
711  }
712 
713  cpl_msg_info(__func__, "Using polynomial orders %hu (x) and %hu (y), %s "
714  "weighting, assuming initial sampling of %.3f +/- %.3f Angstrom"
715  "/pix, in IFU %hhu", aParams->xorder, aParams->yorder,
716  muse_wave_weighting_string[aParams->fitweighting],
717  kMuseSpectralSamplingA, aParams->ddisp, ifu);
718 
719  cpl_vector *vreflam = muse_wave_lines_get(aLinelist, 5, 0.);
720  if (!vreflam) {
721  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "could not "
722  "create list of FWHM reference arc wavelengths for "
723  "IFU %hhu", ifu);
724  return NULL;
725  }
726 
727  cpl_propertylist_erase_regexp(aImage->header, "^ESO QC", 0);
728  cpl_table *wavecaltable = NULL;
729 
730  int debug = getenv("MUSE_DEBUG_WAVECAL")
731  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
732  char fn_debug[100];
733  FILE *fp_debug = NULL;
734  if (debug >= 3) {
735  snprintf(fn_debug, 99, "MUSE_DEBUG_WAVE_LINES-%02hhu.ascii", ifu);
736  cpl_msg_info(__func__, "Will write all single line fits to \"%s\"", fn_debug);
737  fp_debug = fopen(fn_debug, "w");
738  if (fp_debug) {
739  fprintf(fp_debug, "#slice x y lambda lambdaerr\n");
740  }
741  }
742 
743  /* loop over all slices */
744  unsigned short islice;
745  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
746  if (debug > 0) {
747  printf("\n\nSlice %hu of IFU %hhu\n", islice + 1, ifu);
748  fflush(stdout);
749  }
750  /* get the tracing polynomials for this slice */
751  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
752  islice + 1);
753  if (!ptrace) {
754  cpl_msg_warning(__func__, "slice %2hu of IFU %hhu: tracing polynomials "
755  "missing!", islice + 1, ifu);
756  continue;
757  }
758 
759  /* detect all lines in the center of the slice */
760  int imid = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
761  ny/2., NULL));
762  if (imid < 1 || imid > nx) {
763  cpl_msg_warning(__func__, "slice %2hu of IFU %hhu: faulty trace polynomial"
764  " detected", islice + 1, ifu);
765  muse_trace_polys_delete(ptrace);
766  continue; /* next slice */
767  }
768  /* create muse_image from the spectrum in which we want to search lines */
769  const int kWidth = 3;
770  int k, kcol1 = imid - kWidth/2, kcol2 = kcol1 + kWidth;
771  muse_imagelist *collist = muse_imagelist_new();
772  for (k = kcol1; k <= kcol2; k++) {
773  muse_image *column = muse_image_new();
774  column->data = cpl_image_extract(aImage->data, k, 1, k, ny);
775  column->dq = cpl_image_extract(aImage->dq, k, 1, k, ny);
776  column->stat = cpl_image_extract(aImage->stat, k, 1, k, ny);
777  column->header = cpl_propertylist_new();
778  cpl_propertylist_append_string(column->header, "BUNIT",
779  cpl_propertylist_get_string(aImage->header,
780  "BUNIT"));
781  muse_imagelist_set(collist, column, k - kcol1);
782  }
783  muse_image *column = muse_combine_median_create(collist);
784  muse_imagelist_delete(collist);
785 
786  /* now search, measure, and store the lines */
787  cpl_table *detlines = muse_wave_lines_search(column, aParams->detsigma,
788  islice + 1, ifu);
789  if (!detlines) {
790  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "problem when "
791  "searching for arc lines in slice %hu of IFU %hhu, "
792  "columns %d..%d", islice + 1, ifu, kcol1, kcol2);
793  muse_image_delete(column);
794  cpl_vector_delete(vreflam);
795  muse_trace_polys_delete(ptrace);
796  cpl_table_delete(wavecaltable);
797  cpl_table_delete(aParams->residuals);
798  aParams->residuals = NULL;
799  return NULL;
800  }
801  /* clean up only here, to not disturb error message above */
802  muse_image_delete(column);
803  if (debug >= 2) {
804  printf("Detected arc lines in slice %hu of IFU %hhu:\n", islice + 1, ifu);
805  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
806  fflush(stdout);
807  }
808 
809  /* identify the lines that we detected, and see if there are enough *
810  * for the polynomial order in y-direction to fit a polynomial */
811  int nfound = cpl_table_get_nrow(detlines);
812  /* get all arc lines, taking into account the detection limit */
813  double minflux = cpl_table_get_column_min(detlines, "flux") * 1.15;
814  cpl_vector *vlambda = muse_wave_lines_get(aLinelist, 1, minflux);
815  muse_wave_lines_identify(detlines, vlambda, aParams);
816  cpl_vector_delete(vlambda);
817  int nlines = cpl_table_get_nrow(detlines);
818  cpl_msg_debug(__func__, "Identified %d of %d arc lines in slice %hu of IFU "
819  "%hhu (column %d, %d..%d)", nlines, nfound, islice + 1, ifu,
820  imid, kcol1, kcol2);
821  if (debug >= 2) {
822  printf("Identified arc lines with wavelengths in slice %hu of IFU %hhu:\n",
823  islice + 1, ifu);
824  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
825  fflush(stdout);
826  }
827 
828  /* the first QC parameter does even make sense without lines */
829  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NDET, islice + 1);
830  cpl_propertylist_append_int(aImage->header, keyword, nfound);
831  cpl_free(keyword);
832 
833  if (nlines < aParams->yorder + 1) {
834  /* could apparently not identify enough lines, or an error occured */
835  cpl_msg_error(__func__, "Could not identify enough arc lines in slice %hu"
836  " of IFU %hhu (%d of %d, required %hu)", islice + 1, ifu,
837  nlines, nfound, aParams->yorder + 1);
838  muse_trace_polys_delete(ptrace);
839  cpl_table_delete(detlines);
840  continue; /* work on next slice immediately */
841  }
842 
843  /* the next set of QC parameters */
844  muse_wave_calib_qc_peaks(detlines, aImage->header, islice + 1);
845  muse_wave_calib_qc_fwhm_old(detlines, aImage->header, vreflam, islice + 1);
846  /* above we wrote the number of detected lines, now is the number of *
847  * arc lines really used in the fit, i.e. the identified ones */
848  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NID, islice + 1);
849  cpl_propertylist_append_int(aImage->header, keyword, nlines);
850  cpl_free(keyword);
851 
852  /* positions matrix and wavelengths vector to be used for the 2D polynomial *
853  * fit; just set and initial size of 1, it has to be resized with every line */
854  cpl_matrix *xypos = cpl_matrix_new(2, 1);
855  cpl_vector *lambdas = cpl_vector_new(1),
856  *dlambdas = NULL;
857  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
858  dlambdas = cpl_vector_new(1);
859  }
860  int ientry = 0; /* index for these two structures */
861 
862  /* for all identified arc lines, work on the original input data */
863  int j;
864  for (j = 0; j < nlines; j++) {
865  /* convenient access to some properties of this identified line */
866  double lambda = cpl_table_get(detlines, "lambda", j, NULL);
867  double ypos = cpl_table_get(detlines, "center", j, NULL);
868  /* better fix the Gaussian sigmas, otherwise *
869  * the fit might run astray in low S/N cases */
870  double sigma = cpl_table_get(detlines, "sigma", j, NULL);
871  int n = 0,
872  halfwidth = 2.*cpl_table_get(detlines, "fwhm", j, NULL); /* 2*FWHM */
873  /* get both slice edges and the center */
874  double dleft = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
875  ypos, NULL),
876  dright = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
877  ypos, NULL),
878  dmid = (dleft + dright) / 2.;
879  int ileft = ceil(dleft),
880  iright = floor(dright);
881 #if 0
882  cpl_msg_debug(__func__, "limits at y=%f: %f < %f < %f", ypos, dleft, dmid,
883  dright);
884 #endif
885 
886  /* table to store line fits for this one arc line */
887  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
888  (int)kMuseSliceHiLikelyWidth + 5);
889 
890  /* From the center of the slice move outwards and fit the line *
891  * until we have arrived at the edge of the slice */
892  int i;
893  for (i = dmid; i >= ileft; i--) {
894  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
895  sigma, fittable, ++n);
896  if (rc != CPL_ERROR_NONE) { /* do not count this line */
897  --n;
898  }
899  }
900 #if 0
901  printf("arc line j=%d, columns i=%d...", j + 1, i);
902 #endif
903  for (i = dmid + 1; i <= iright; i++) {
904  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
905  sigma, fittable, ++n);
906  if (rc != CPL_ERROR_NONE) { /* do not count this line */
907  --n;
908  }
909  }
910  /* now remove rows with invalid entries, i.e. those that were not *
911  * filled with the properties of the fit -- cpl_table_erase_invalid() *
912  * does not work, it deletes all columns */
913  cpl_table_select_all(fittable);
914  cpl_table_and_selected_invalid(fittable, "center");
915  cpl_table_erase_selected(fittable);
916 #if 0
917  printf("line %d, %d line fits\n", j + 1, i);
918  cpl_table_dump(fittable, 0, n, stdout);
919  fflush(stdout);
920 #endif
921  cpl_errorstate state = cpl_errorstate_get();
922  muse_wave_line_fit_iterate(fittable, -1, aParams);
923  int npos = cpl_table_get_nrow(fittable);
924  if (npos <= aParams->xorder) {
925  cpl_msg_debug(__func__, "Polynomial fit failed in slice %hu of IFU %hhu"
926  " for line at %.3fA (y-position near %.2f pix): %s",
927  islice + 1, ifu, lambda, ypos, cpl_error_get_message());
928  cpl_errorstate_set(state);
929  }
930 #if 0
931  else {
932  printf("%s: line %2d, %d line fits, %d with low residuals:\n", __func__,
933  j + 1, i, npos);
934  cpl_table_dump(fittable, 0, npos, stdout);
935  fflush(stdout);
936  }
937 #endif
938 
939  /* resize matrix/vector to be able to fit all new entries */
940  cpl_matrix_resize(xypos, 0, 0,
941  0, ientry + npos - cpl_matrix_get_ncol(xypos));
942  cpl_vector_set_size(lambdas, ientry + npos);
943  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
944  cpl_vector_set_size(dlambdas, ientry + npos);
945  }
946 
947  /* now add the final fit positions */
948  int ipos;
949  for (ipos = 0; ipos < npos; ipos++) {
950  /* set x-position (CCD column) in the first matrix row */
951  cpl_matrix_set(xypos, 0, ientry, cpl_table_get(fittable, "x", ipos, NULL));
952  /* y-position on CCD (center of Gauss fit) into second matrix row */
953  cpl_matrix_set(xypos, 1, ientry, cpl_table_get(fittable, "center", ipos, NULL));
954  /* the vector has to contain as many lambda entries */
955  cpl_vector_set(lambdas, ientry, lambda);
956  /* pretend that errors in Gaussian fit are errors in wavelength, *
957  * scale according to the nominal Angstrom/pix sampling of MUSE, *
958  * everything else would be too complicated for little gain */
959  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
960  cpl_vector_set(dlambdas, ientry, cpl_table_get(fittable, "cerr", ipos, NULL)
961  * kMuseSpectralSamplingA);
962  }
963 
964  ientry++; /* next position in this matrix/vector combo */
965  if (fp_debug) {
966  fprintf(fp_debug, " %02hu %04d %9.4f %10.4f %e\n", islice + 1,
967  (int)cpl_table_get(fittable, "x", ipos, NULL),
968  cpl_table_get(fittable, "center", ipos, NULL), lambda,
969  cpl_table_get(fittable, "cerr", ipos, NULL) * kMuseSpectralSamplingA);
970  }
971  }
972  cpl_table_delete(fittable);
973  } /* for j (all identified arc lines) */
974  muse_wave_lines_add_flux_for_lsf(aLinelist, detlines);
975  cpl_table_delete(detlines);
976 
977  /* Compute two-dimensional wavelength solution for each slice. */
978  cpl_polynomial *poly = NULL; /* polynomial for wavelength solution */
979  double mse = 0; /* mean squared error */
980  cpl_error_code rc = muse_wave_poly_fit(xypos, lambdas, dlambdas,
981  &poly, &mse, aParams, islice + 1);
982  cpl_matrix_delete(xypos);
983  int nfinal = muse_cplvector_count_unique(lambdas);
984  cpl_vector_delete(lambdas);
985  cpl_vector_delete(dlambdas);
986  /* above we wrote detected and identified lines, here *
987  * save the ones that actually survived the fit */
988  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_NLINES, islice + 1);
989  cpl_propertylist_append_int(aImage->header, keyword, nfinal);
990  cpl_free(keyword);
991  /* collect QC on wavelengths */
992  muse_wave_calib_qc_lambda(aImage, islice + 1, ptrace, poly);
993  /* trace polynomials are not needed further */
994  muse_trace_polys_delete(ptrace);
995 
996  if (rc != CPL_ERROR_NONE) { /* failure */
997  cpl_msg_warning(__func__, "Something went wrong while fitting in slice "
998  "%hu of IFU %hhu: %s", islice + 1, ifu,
999  cpl_error_get_message_default(rc));
1000  cpl_polynomial_delete(poly);
1001  continue; /* try next slice immediately */
1002  }
1003 
1004  if (!wavecaltable) {
1005  /* create output table with one row for each slice; no need to *
1006  * check return code, it can only fail for negative lengths */
1007  wavecaltable = muse_wave_table_create(kMuseSlicesPerCCD,
1008  aParams->xorder, aParams->yorder);
1009  }
1010  rc = muse_wave_table_add_poly(wavecaltable, poly, mse,
1011  aParams->xorder, aParams->yorder, islice);
1012 
1013  /* use the number of wavelengths as indicator of the lines used for the *
1014  * fit and record it as QC parameter */
1015  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_RMS, islice + 1);
1016  cpl_propertylist_append_float(aImage->header, keyword, sqrt(mse));
1017  cpl_free(keyword);
1018 
1019  if (rc != CPL_ERROR_NONE) {
1020  cpl_msg_warning(__func__, "Could not write polynomial to wavecal table "
1021  "for slice %hu of IFU %hhu: %s", islice + 1, ifu,
1022  cpl_error_get_message_default(rc));
1023  }
1024  /* clean up, ignore any failure with this */
1025  cpl_polynomial_delete(poly);
1026  } /* for islice (all kMuseSlicesPerCCD) */
1027  cpl_vector_delete(vreflam);
1028  if (fp_debug) {
1029  cpl_msg_info(__func__, "Done writing line fits to \"%s\"", fn_debug);
1030  fclose(fp_debug);
1031  }
1032 
1033  muse_wave_calib_output_summary(aImage->header, wavecaltable);
1034 #if 0
1035  if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
1036  cpl_table_dump(wavecaltable, 0, 3, stdout); fflush(stdout);
1037  }
1038 #endif
1039 
1040  return wavecaltable;
1041 } /* muse_wave_calib() */
1042 
1043 /*----------------------------------------------------------------------------*/
1107 /*----------------------------------------------------------------------------*/
1108 cpl_table *
1109 muse_wave_calib_lampwise(muse_imagelist *aImages, cpl_table *aTrace,
1110  cpl_table *aLinelist, muse_wave_params *aParams)
1111 {
1112  if (!aImages || muse_imagelist_get_size(aImages) < 1) {
1113  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "arc imagelist is "
1114  "missing or empty!");
1115  return NULL;
1116  }
1117  unsigned char ifu = muse_utils_get_ifu(muse_imagelist_get(aImages, 0)->header);
1118  if (!aTrace) {
1119  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "trace table missing "
1120  "for IFU %hhu, cannot create wavelength calibration!",
1121  ifu);
1122  return NULL;
1123  } else if (cpl_table_get_nrow(aTrace) != kMuseSlicesPerCCD) {
1124  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "trace table "
1125  "not valid for this dataset of IFU %hhu!", ifu);
1126  return NULL;
1127  }
1128  if (!aLinelist) {
1129  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "no arc line list "
1130  "supplied for IFU %hhu!", ifu);
1131  return NULL;
1132  }
1133  if (!aParams) {
1134  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "wavelength "
1135  "calibration parameters missing for IFU %hhu!", ifu);
1136  return NULL;
1137  }
1139  cpl_error_set_message(__func__, CPL_ERROR_UNSUPPORTED_MODE,
1140  "unknown weighting scheme for IFU %hhu", ifu);
1141  return NULL;
1142  }
1143 
1144  muse_image *firstimage = muse_imagelist_get(aImages, 0);
1145  int nx = cpl_image_get_size_x(firstimage->data),
1146  ny = cpl_image_get_size_y(firstimage->data);
1147  /* do not support binned exposures */
1148  if (ny < 4000) {
1149  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "this dataset of "
1150  "IFU %hhu is too small (%dx%d pix) to be supported",
1151  ifu, nx, ny);
1152  return NULL;
1153  }
1154 
1155  cpl_msg_info(__func__, "Using polynomial orders %hu (x) and %hu (y), %s "
1156  "weighting, assuming initial sampling of %.3f +/- %.3f Angstrom"
1157  "/pix, in IFU %hhu", aParams->xorder, aParams->yorder,
1158  muse_wave_weighting_string[aParams->fitweighting],
1159  kMuseSpectralSamplingA, aParams->ddisp, ifu);
1160 
1161  cpl_propertylist_erase_regexp(firstimage->header, "^ESO QC", 0);
1162  char *lamp = muse_utils_header_get_lamp_names(firstimage->header, ',');
1163  cpl_msg_debug(__func__, "Image 1 was taken with lamp %s", lamp);
1164  cpl_free(lamp);
1165  unsigned int k;
1166  for (k = 1; k < muse_imagelist_get_size(aImages); k++) {
1167  int nxk = cpl_image_get_size_x(muse_imagelist_get(aImages, k)->data),
1168  nyk = cpl_image_get_size_y(muse_imagelist_get(aImages, k)->data);
1169  if (nxk != nx || nyk != ny) {
1170  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "arc "
1171  "imagelist is not uniform (image %u is %dx%d, "
1172  "first image is %dx%d)", k + 1, nxk, nyk, nx, ny);
1173  return NULL;
1174  }
1175  cpl_propertylist_erase_regexp(muse_imagelist_get(aImages, k)->header,
1176  "^ESO QC", 0);
1177  lamp = muse_utils_header_get_lamp_names(muse_imagelist_get(aImages, k)->header, ',');
1178  cpl_msg_debug(__func__, "Image %d was taken with lamp %s", k + 1, lamp);
1179  cpl_free(lamp);
1180  } /* for k (all images in list except first) */
1181 
1182  cpl_vector *vreflam = muse_wave_lines_get(aLinelist, 5, 0.);
1183  if (!vreflam) {
1184  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "could not "
1185  "create list of FWHM reference arc wavelengths for "
1186  "IFU %hhu", ifu);
1187  return NULL;
1188  }
1189  cpl_table *wavecal = NULL;
1190  int debug = getenv("MUSE_DEBUG_WAVECAL")
1191  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
1192  char fn_debug[100];
1193  FILE *fp_debug = NULL;
1194  if (debug >= 3) {
1195  snprintf(fn_debug, 99, "MUSE_DEBUG_WAVE_LINES-%02hhu.ascii", ifu);
1196  cpl_msg_info(__func__, "Will write all single line fits to \"%s\"", fn_debug);
1197  fp_debug = fopen(fn_debug, "w");
1198  if (fp_debug) {
1199  fprintf(fp_debug, "#slice x y lambda lambdaerr\n");
1200  }
1201  }
1202 
1203  /* loop over all slices */
1204  unsigned short islice;
1205  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
1206  if (debug > 0) {
1207  printf("\n\nSlice %hu of IFU %hhu\n", islice + 1, ifu);
1208  fflush(stdout);
1209  }
1210  /* get the tracing polynomials for this slice */
1211  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
1212  islice + 1);
1213  if (!ptrace) {
1214  cpl_msg_warning(__func__, "slice %2hu of IFU %hhu: tracing polynomials "
1215  "missing!", islice + 1, ifu);
1216  continue;
1217  }
1218  /* detect all lines in the center of the slice */
1219  const int imid = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
1220  ny/2., NULL)),
1221  iWidth = 3,
1222  icol1 = imid - iWidth/2,
1223  icol2 = icol1 + iWidth;
1224  if (imid < 1 || imid > nx) {
1225  cpl_msg_warning(__func__, "slice %2hu of IFU %hhu: faulty trace polynomial"
1226  "detected", islice + 1, ifu);
1227  muse_trace_polys_delete(ptrace);
1228  continue; /* next slice */
1229  }
1230  cpl_table *detlines = NULL;
1231  int ndet = 0;
1232  for (k = 0; k < muse_imagelist_get_size(aImages); k++) {
1233  muse_image *arc = muse_imagelist_get(aImages, k);
1234  /* create muse_image from the spectrum in which we want to search lines */
1235  muse_imagelist *collist = muse_imagelist_new();
1236  int i;
1237  for (i = icol1; i <= icol2; i++) {
1238  muse_image *column = muse_image_new();
1239  column->data = cpl_image_extract(arc->data, i, 1, i, ny);
1240  column->dq = cpl_image_extract(arc->dq, i, 1, i, ny);
1241  column->stat = cpl_image_extract(arc->stat, i, 1, i, ny);
1242  column->header = cpl_propertylist_new();
1243  cpl_propertylist_append_string(column->header, "BUNIT",
1244  cpl_propertylist_get_string(arc->header,
1245  "BUNIT"));
1246  muse_imagelist_set(collist, column, i - icol1);
1247  } /* for i (neighboring image columns) */
1248  muse_image *column = muse_combine_median_create(collist);
1249  muse_imagelist_delete(collist);
1250 
1251  /* now search, measure, and store the lines */
1252  cpl_table *detections = muse_wave_lines_search(column, aParams->detsigma,
1253  islice + 1, ifu);
1254  muse_image_delete(column);
1255  int nlampdet = cpl_table_get_nrow(detections); /* det. with this lamp */
1256  ndet += nlampdet; /* detections of all lamps */
1257  char *lampname = muse_utils_header_get_lamp_names(arc->header, ',');
1258  cpl_table_fill_column_window_string(detections, "lampname", 0, nlampdet,
1259  lampname);
1260  cpl_array *lampnumbers = muse_utils_header_get_lamp_numbers(arc->header);
1261  cpl_table_fill_column_window_int(detections, "lampno", 0, nlampdet,
1262  cpl_array_get_int(lampnumbers, 0, NULL));
1263  cpl_array_delete(lampnumbers);
1264  /* get arc lines for this lamp from original list */
1265  double minflux = cpl_table_get_column_min(detections, "flux") * 1.15;
1266  cpl_vector *ionlambdas = muse_wave_lines_get_for_lamp(aLinelist, lampname,
1267  1, minflux);
1268  cpl_free(lampname);
1269  muse_wave_lines_identify(detections, ionlambdas, aParams);
1270  cpl_vector_delete(ionlambdas);
1271 
1272  if (!detlines) {
1273  detlines = detections;
1274  } else {
1275  cpl_table_insert(detlines, detections, cpl_table_get_nrow(detlines));
1276  cpl_table_delete(detections);
1277  }
1278  } /* for k (all images in list) */
1279  cpl_propertylist *order = cpl_propertylist_new();
1280  cpl_propertylist_append_bool(order, "y", CPL_FALSE);
1281  cpl_table_sort(detlines, order);
1282  cpl_propertylist_delete(order);
1283  int nlines = cpl_table_get_nrow(detlines);
1284  if (debug >= 2) {
1285  printf("Detected and identified %d arc lines in slice %hu of IFU %hhu:\n",
1286  nlines, islice + 1, ifu);
1287  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
1288  fflush(stdout);
1289  }
1290  /* the first QC parameter does even make sense without lines */
1291  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NDET, islice + 1);
1292  cpl_propertylist_append_int(firstimage->header, keyword, ndet);
1293  cpl_free(keyword);
1294 
1295  if (nlines < aParams->yorder + 1) {
1296  /* could apparently not identify enough lines, or an error occured */
1297  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "could not "
1298  "detect and/or identify enough arc lines in slice "
1299  "%hu of IFU %hhu (%d of %d, required %hu)",
1300  islice + 1, ifu, nlines, ndet, aParams->yorder + 1);
1301  muse_trace_polys_delete(ptrace);
1302  cpl_table_delete(detlines);
1303  continue; /* work on next slice immediately */
1304  }
1305  cpl_msg_info(__func__, "Identified %d of %d detected arc lines in slice %hu"
1306  " of IFU %hhu (column %d, %d..%d)", nlines, ndet, islice + 1,
1307  ifu, imid, icol1, icol2);
1308 
1309  /* the next set of QC parameters */
1310  muse_wave_calib_qc_peaks(detlines, firstimage->header, islice + 1);
1311  /* above we wrote the number of detected lines, now is the number of *
1312  * arc lines really used in the fit, i.e. the identified ones */
1313  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NID, islice + 1);
1314  cpl_propertylist_append_int(firstimage->header, keyword, nlines);
1315  cpl_free(keyword);
1316 
1317  /* convert y and lambda into matrix and vector for an initial, *
1318  * vertical, inverse 1D fit */
1319  cpl_vector *cen = cpl_vector_new(nlines);
1320  cpl_matrix *lbda = cpl_matrix_new(1, nlines);
1321  int idx;
1322  for (idx = 0; idx < nlines; idx++) {
1323  cpl_vector_set(cen, idx, cpl_table_get(detlines, "center", idx, NULL));
1324  cpl_matrix_set(lbda, 0, idx, cpl_table_get(detlines, "lambda", idx, NULL));
1325  }
1326 #if 0
1327  char *fn = cpl_sprintf("slice%02hu_lbda1.dat", islice + 1);
1328  FILE *file = fopen(fn, "w");
1329  cpl_matrix_dump(lbda, file);
1330  fclose(file);
1331  cpl_free(fn);
1332 #endif
1333  double mse1d, chisq1d;
1334  cpl_polynomial *fit
1335  = muse_utils_iterate_fit_polynomial(lbda, cen, NULL, detlines,
1336  aParams->yorder, 3.,
1337  &mse1d, &chisq1d);
1338 #if 0
1339  cpl_vector *res = cpl_vector_new(cpl_vector_get_size(cen));
1340  cpl_vector_fill_polynomial_fit_residual(res, cen, NULL, fit, lbda, NULL);
1341  cpl_bivector *biv = cpl_bivector_wrap_vectors(cen, res);
1342  cpl_plot_bivector(NULL, NULL, NULL, biv);
1343  cpl_bivector_unwrap_vectors(biv);
1344  cpl_vector_delete(res);
1345 #endif
1346  if (debug >= 1) {
1347  printf("Initial (inverse) polynomial fit in slice %2hu of IFU %hhu "
1348  "(RMS = %f, chisq = %e, %d of %d input points left)\n", islice + 1,
1349  ifu, sqrt(mse1d), chisq1d, (int)cpl_vector_get_size(cen), nlines);
1350  cpl_polynomial_dump(fit, stdout);
1351  fflush(stdout);
1352  }
1353  nlines = cpl_vector_get_size(cen);
1354 #if 0
1355  fn = cpl_sprintf("slice%02hu_lbda2.dat", islice + 1);
1356  file = fopen(fn, "w");
1357  cpl_matrix_dump(lbda, file);
1358  fclose(file);
1359  cpl_free(fn);
1360 #endif
1361  cpl_vector_delete(cen);
1362  cpl_matrix_delete(lbda);
1363 
1364  /* positions matrix and wavelengths vector to be used for the 2D polynomial *
1365  * fit; just set and initial size of 1, it has to be resized with every line */
1366  cpl_matrix *xypos = cpl_matrix_new(2, 1);
1367  cpl_vector *lambdas = cpl_vector_new(1),
1368  *dlambdas = NULL;
1369  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1370  dlambdas = cpl_vector_new(1);
1371  }
1372  int ientry = 0; /* index for these two structures */
1373 
1374  /* now measure all bright, single lines taken from the input linelist */
1375  cpl_table *fwhmtable = muse_cpltable_new(muse_wavelines_def, 0);
1376  cpl_table_set_column_unit(fwhmtable, "fwhm", "Angstrom");
1377  int j, narclines = cpl_table_get_nrow(aLinelist);
1378  for (j = 0; j < narclines; j++) {
1379  int quality = cpl_table_get_int(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, NULL);
1380  if (quality <= 1) {
1381  continue; /* skip unwanted line */
1382  }
1383 
1384  /* use the lamp name to find the corresponding exposure for arc line */
1385  const char *lampname = muse_wave_lines_get_lampname(aLinelist, j);
1386  muse_image *arc = NULL;
1387  for (k = 0; k < muse_imagelist_get_size(aImages); k++) {
1388  arc = muse_imagelist_get(aImages, k);
1389  char *thislamp = muse_utils_header_get_lamp_names(arc->header, ',');
1390  cpl_boolean match = CPL_FALSE;
1391  if (lampname && thislamp) {
1392  match = strncmp(lampname, thislamp, strlen(lampname)) == 0;
1393  }
1394  cpl_free(thislamp);
1395  if (match) {
1396  break;
1397  }
1398  } /* for k (all images in list) */
1399 
1400  cpl_table *fittable = NULL;
1401  if (quality == 2) { /* multiplet */
1402  fittable = muse_wave_line_handle_multiplet(arc, aLinelist, j, fit, ptrace,
1403  aParams, islice + 1, debug);
1404  } else {
1405  fittable = muse_wave_line_handle_singlet(arc, aLinelist, j, fit, ptrace,
1406  aParams, islice + 1, debug);
1407  }
1408  if (!fittable) {
1409  continue;
1410  }
1411  int nnew = cpl_table_get_nrow(fittable);
1412 
1413  /* resize matrix/vector to be able to fit all new entries */
1414  cpl_matrix_resize(xypos, 0, 0,
1415  0, ientry + nnew - cpl_matrix_get_ncol(xypos));
1416  cpl_vector_set_size(lambdas, ientry + nnew);
1417  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1418  cpl_vector_set_size(dlambdas, ientry + nnew);
1419  }
1420 
1421  /* now add the final fit positions */
1422  cpl_table_set_column_unit(fittable, "fwhm", ""); /* ongoing unit conversion */
1423  int ipos;
1424  for (ipos = 0; ipos < nnew; ipos++) {
1425  /* set x-position (CCD column) in the first matrix row */
1426  cpl_matrix_set(xypos, 0, ientry, cpl_table_get(fittable, "x", ipos, NULL));
1427  /* y-position on CCD (center of Gauss fit) into second matrix row */
1428  cpl_matrix_set(xypos, 1, ientry, cpl_table_get(fittable, "center", ipos, NULL));
1429  /* the vector has to contain as many lambda entries */
1430  double lambda = cpl_table_get(fittable, "lambda", ipos, NULL);
1431  cpl_vector_set(lambdas, ientry, lambda);
1432  /* Pretend that errors in Gaussian fit are errors in wavelength, *
1433  * use the first-guess 1D solution to convert from pix to Angstrom. */
1434  double dlbda;
1435  cpl_polynomial_eval_1d(fit, lambda, &dlbda); /* not interested in result */
1436  double cerr = 0.;
1437  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1438  cerr = cpl_table_get(fittable, "cerr", ipos, NULL) / dlbda;
1439  cpl_vector_set(dlambdas, ientry, cerr);
1440  }
1441  /* now that we have the actual sampling, also convert FWHM to Angstrom */
1442  int err;
1443  double fwhm = cpl_table_get(fittable, "fwhm", ipos, &err) / dlbda;
1444  if (!err) {
1445  cpl_table_set(fittable, "fwhm", ipos, fwhm);
1446  }
1447  ientry++; /* next position in this matrix/vector combo */
1448  if (fp_debug) {
1449  fprintf(fp_debug, " %02hu %04d %9.4f %10.4f %e\n", islice + 1,
1450  (int)cpl_table_get(fittable, "x", ipos, NULL),
1451  cpl_table_get(fittable, "center", ipos, NULL), lambda, cerr);
1452  }
1453  } /* for ipos (all fittable rows) */
1454  cpl_table_set_column_unit(fittable, "fwhm", "Angstrom");
1455  /* append relevant lines to extra table to compute FWHM and R later */
1456  cpl_table_select_all(fittable);
1457  cpl_table_and_selected_invalid(fittable, "fwhm");
1458  cpl_table_erase_selected(fittable);
1459  cpl_table_insert(fwhmtable, fittable, cpl_table_get_nrow(fwhmtable));
1460  cpl_table_delete(fittable);
1461  } /* for j (all arc lines) */
1462  /* reset original line quality info */
1463  cpl_table_abs_column(aLinelist, MUSE_LINE_CATALOG_QUALITY);
1464  muse_wave_lines_add_flux_for_lsf(aLinelist, detlines);
1465  cpl_table_delete(detlines);
1466  cpl_polynomial_delete(fit);
1467  muse_wave_calib_qc_fwhm(fwhmtable, firstimage->header, islice + 1);
1468  cpl_table_delete(fwhmtable);
1469 
1470  /* Compute two-dimensional wavelength solution for each slice. */
1471  cpl_polynomial *poly = NULL; /* polynomial for wavelength solution */
1472  double mse2d;
1473  cpl_error_code rc = muse_wave_poly_fit(xypos, lambdas, dlambdas,
1474  &poly, &mse2d, aParams, islice + 1);
1475  cpl_msg_info(__func__, "Polynomial fit of %"CPL_SIZE_FORMAT" of %d positions"
1476  " in slice %hu of IFU %hhu gave an RMS of %f Angstrom",
1477  cpl_vector_get_size(lambdas), ientry, islice + 1, ifu,
1478  sqrt(mse2d));
1479  cpl_matrix_delete(xypos);
1480  int nfinal = muse_cplvector_count_unique(lambdas);
1481  cpl_vector_delete(lambdas);
1482  cpl_vector_delete(dlambdas);
1483  /* above we wrote detected and identified lines, here *
1484  * save the ones that actually survived the fit */
1485  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_NLINES, islice + 1);
1486  cpl_propertylist_append_int(firstimage->header, keyword, nfinal);
1487  cpl_free(keyword);
1488  /* collect QC on wavelengths */
1489  muse_wave_calib_qc_lambda(firstimage, islice + 1, ptrace, poly);
1490  /* trace polynomials are not needed further */
1491  muse_trace_polys_delete(ptrace);
1492 
1493  if (rc != CPL_ERROR_NONE) { /* failure of the polynomial fit */
1494  cpl_error_set_message(__func__, rc, "a failure while computing the two-di"
1495  "mensional polynomial fit in slice %hu of IFU %hhu",
1496  islice + 1, ifu);
1497  cpl_polynomial_delete(poly);
1498  continue; /* try next slice immediately */
1499  }
1500 
1501  if (!wavecal) {
1502  /* create output table with one row for each slice; no need to *
1503  * check return code, it can only fail for negative lengths */
1504  wavecal = muse_wave_table_create(kMuseSlicesPerCCD,
1505  aParams->xorder, aParams->yorder);
1506  }
1507  rc = muse_wave_table_add_poly(wavecal, poly, mse2d,
1508  aParams->xorder, aParams->yorder, islice);
1509 
1510  /* use the number of wavelengths as indicator of the lines used for the *
1511  * fit and record it as QC parameter */
1512  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_RMS, islice + 1);
1513  cpl_propertylist_append_float(firstimage->header, keyword, sqrt(mse2d));
1514  cpl_free(keyword);
1515 
1516  if (rc != CPL_ERROR_NONE) {
1517  cpl_msg_warning(__func__, "Could not write polynomial to wavecal table "
1518  "for slice %hu of IFU %hhu: %s", islice + 1, ifu,
1519  cpl_error_get_message_default(rc));
1520  }
1521  /* clean up, ignore any failure with this */
1522  cpl_polynomial_delete(poly);
1523  } /* for islice (all kMuseSlicesPerCCD) */
1524  cpl_vector_delete(vreflam);
1525  if (fp_debug) {
1526  cpl_msg_info(__func__, "Done writing line fits to \"%s\"", fn_debug);
1527  fclose(fp_debug);
1528  }
1529 
1530  muse_wave_calib_output_summary(firstimage->header, wavecal);
1531  /* copy the QC parameters to the other image headers */
1532  for (k = 1; k < muse_imagelist_get_size(aImages); k++) {
1533  cpl_propertylist *header = muse_imagelist_get(aImages, k)->header;
1534  cpl_propertylist_erase_regexp(header, "^ESO QC", 0);
1535  cpl_propertylist_copy_property_regexp(header, firstimage->header, "^ESO QC", 0);
1536  } /* for k (all images in list except first) */
1537 
1538  return wavecal;
1539 } /* muse_wave_calib_lampwise() */
1540 
1541 /*----------------------------------------------------------------------------*/
1559 /*----------------------------------------------------------------------------*/
1560 cpl_boolean
1561 muse_wave_lines_check(cpl_table *aTable, cpl_propertylist *aHeader)
1562 {
1563  cpl_ensure(aTable && aHeader, CPL_ERROR_NULL_INPUT, CPL_FALSE);
1564  /* check the table */
1565  int nrow = cpl_table_get_nrow(aTable);
1566  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, CPL_FALSE);
1567  cpl_ensure(muse_cpltable_check(aTable, muse_line_catalog_def) == CPL_ERROR_NONE,
1568  CPL_ERROR_DATA_NOT_FOUND, CPL_FALSE);
1569 
1570  /* check the version */
1571  if (!cpl_propertylist_has(aHeader, MUSE_HDR_LINE_CATALOG_VERSION)) {
1572  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "%s does not "
1573  "contain a VERSION header entry!", MUSE_TAG_LINE_CATALOG);
1574  return CPL_FALSE;
1575  }
1576  int version = cpl_propertylist_get_int(aHeader, MUSE_HDR_LINE_CATALOG_VERSION);
1577 #define LINE_CATALOG_EXPECTED_VERSION 3
1578  if (version != LINE_CATALOG_EXPECTED_VERSION) {
1579  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "VERSION = %d "
1580  "is wrong, we need a %s with VERSION = %d", version,
1581  MUSE_TAG_LINE_CATALOG, LINE_CATALOG_EXPECTED_VERSION);
1582  return CPL_FALSE;
1583  }
1584  return CPL_TRUE;
1585 } /* muse_wave_lines_check() */
1586 
1587 /*----------------------------------------------------------------------------*/
1609 /*----------------------------------------------------------------------------*/
1610 cpl_vector *
1611 muse_wave_lines_get(cpl_table *aTable, int aGoodnessLimit, double aFluxLimit)
1612 {
1613  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, NULL);
1614  int nrow = cpl_table_get_nrow(aTable);
1615  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, NULL);
1616 
1617  cpl_ensure(cpl_table_has_column(aTable, MUSE_LINE_CATALOG_LAMBDA) == 1
1618  && cpl_table_has_column(aTable, MUSE_LINE_CATALOG_QUALITY) == 1,
1619  CPL_ERROR_DATA_NOT_FOUND, NULL);
1620 
1621  /* convert the usable wavelengths into a vector */
1622  cpl_vector *lambdas = cpl_vector_new(nrow);
1623  int i, pos = 0;
1624  for (i = 0; i < nrow; i++) {
1625  double lambda = cpl_table_get(aTable, MUSE_LINE_CATALOG_LAMBDA, i, NULL),
1626  flux = cpl_table_get(aTable, MUSE_LINE_CATALOG_FLUX, i, NULL);
1627 
1628  /* make sure that the arc line table is sorted, to simplify debugging */
1629  if (i > 0 && lambda < cpl_table_get(aTable, MUSE_LINE_CATALOG_LAMBDA, i-1,
1630  NULL)) {
1631  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "%s is not "
1632  "sorted by increasing lambda (at row %d)!",
1633  MUSE_TAG_LINE_CATALOG, i+1);
1634  cpl_vector_delete(lambdas);
1635  return NULL;
1636  }
1637 
1638  if (cpl_table_get(aTable, MUSE_LINE_CATALOG_QUALITY, i, NULL)
1639  < aGoodnessLimit || flux < aFluxLimit) {
1640  /* this is apparently a line that should not be used */
1641  continue;
1642  }
1643  cpl_vector_set(lambdas, pos, lambda);
1644  pos++;
1645  } /* for i (all table rows) */
1646  if (!pos) { /* setting the size to zero does not work! */
1647  cpl_vector_delete(lambdas);
1648  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "No lines with "
1649  "%s >= %d found", MUSE_LINE_CATALOG_QUALITY,
1650  aGoodnessLimit);
1651  return NULL;
1652  }
1653  /* cut vector size to contain the usable wavelengths */
1654  cpl_vector_set_size(lambdas, pos);
1655 
1656  /* remove lines that are too close together, i.e. within 1.5 Angstrom */
1657  for (i = 0; i < cpl_vector_get_size(lambdas) - 1; i++) {
1658  double l1 = cpl_vector_get(lambdas, i),
1659  l2 = cpl_vector_get(lambdas, i + 1);
1660  if ((l2 - l1) < 1.5) {
1661  cpl_msg_debug(__func__, "Excluding lines at %.3f and %.3f (d = %.3f) "
1662  "Angstrom", l1, l2, l2 - l1);
1663  muse_cplvector_erase_element(lambdas, i + 1);
1664  muse_cplvector_erase_element(lambdas, i);
1665  i--; /* stay at this index to see what moved here */
1666  }
1667  } /* for i (all vector entries) */
1668 #if 0
1669  printf("%d arc lines in input list are usable:\n", pos);
1670  cpl_vector_dump(lambdas, stdout);
1671 #endif
1672  cpl_msg_debug(__func__, "Using a list of %d %s arc lines (from %6.1f to %6.1f "
1673  "Angstrom)", pos,
1674  aGoodnessLimit == 1 ? "good"
1675  : (aGoodnessLimit == 5 ? "FWHM reference"
1676  : "all"),
1677  cpl_vector_get(lambdas, 0),
1678  cpl_vector_get(lambdas, cpl_vector_get_size(lambdas) - 1));
1679 
1680  return lambdas;
1681 } /* muse_wave_lines_get() */
1682 
1683 /*----------------------------------------------------------------------------*/
1700 /*----------------------------------------------------------------------------*/
1701 cpl_vector *
1702 muse_wave_lines_get_for_lamp(cpl_table *aTable, const char *aLamp,
1703  int aGoodnessLimit, double aFluxLimit)
1704 {
1705  cpl_ensure(aTable && aLamp, CPL_ERROR_NULL_INPUT, NULL);
1706  int nrow = cpl_table_get_nrow(aTable);
1707  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, NULL);
1708 
1709  cpl_table_unselect_all(aTable);
1710  int i;
1711  for (i = 0; i < nrow; i++) {
1712  const char *lampname = muse_wave_lines_get_lampname(aTable, i);
1713  if (!strcmp(lampname, aLamp)) {
1714  cpl_table_select_row(aTable, i);
1715  }
1716  } /* for i (all table rows) */
1717  cpl_table *lamplines = cpl_table_extract_selected(aTable);
1718 #if 0
1719  printf("lamplines for %s\n", aLamp);
1720  cpl_table_dump(lamplines, 0, 10000, stdout);
1721  fflush(stdout);
1722 #endif
1723  cpl_vector *lamplambdas = muse_wave_lines_get(lamplines, aGoodnessLimit,
1724  aFluxLimit);
1725  cpl_table_delete(lamplines);
1726  return lamplambdas;
1727 } /* muse_wave_lines_get_for_lamp() */
1728 
1729 /*----------------------------------------------------------------------------*/
1743 /*----------------------------------------------------------------------------*/
1744 const char *
1745 muse_wave_lines_get_lampname(cpl_table *aTable, const int aIdx)
1746 {
1747  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, "Unknown_Lamp");
1748  const char *ion = cpl_table_get_string(aTable, "ion", aIdx);
1749  cpl_ensure(ion, CPL_ERROR_ILLEGAL_INPUT, "Unknown_Lamp");
1750 
1751  /* for the HgCd lamp we have "HgI", "CdI", and "Hg_or_Cd" */
1752  if (!strncmp(ion, "Hg", 2) || !strncmp(ion, "Cd", 2)) {
1753  return "HgCd";
1754  }
1755  /* for the Ne lamp we have only "NeI" */
1756  if (!strncmp(ion, "Ne", 2)) {
1757  return "Ne";
1758  }
1759  /* for the Xe lamp we have "XeI" and "XeII" */
1760  if (!strncmp(ion, "Xe", 2)) {
1761  return "Xe";
1762  }
1763  return "Unknown_Lamp";
1764 } /* muse_wave_lines_get_lampname() */
1765 
1766 /*----------------------------------------------------------------------------*/
1778 /*----------------------------------------------------------------------------*/
1779 static void
1780 muse_wave_lines_add_flux_for_lsf(cpl_table *aLineList, cpl_table *aDetlines)
1781 {
1782  if (!cpl_table_has_column(aLineList, "Flux_for_LSF")) {
1783  cpl_table_new_column(aLineList, "Flux_for_LSF",
1784  CPL_TYPE_DOUBLE);
1785  cpl_table_fill_column_window(aLineList, "Flux_for_LSF",
1786  0, cpl_table_get_nrow(aLineList), 0.);
1787  }
1788 
1789  const double sigma = 0.1;
1790  cpl_size i, n = cpl_table_get_nrow(aDetlines);
1791  for (i = 0; i < n; i++) {
1792  int status;
1793  double lbda = cpl_table_get(aDetlines, "lambda", i, &status);
1794  if (status == 1) {
1795  continue;
1796  }
1797  cpl_size i_l = muse_cpltable_find_sorted(aLineList, MUSE_LINE_CATALOG_LAMBDA,
1798  lbda + sigma);
1799  if (cpl_table_get(aLineList, MUSE_LINE_CATALOG_LAMBDA, i_l, &status)
1800  < lbda - sigma) {
1801  continue;
1802  }
1803  double a = cpl_table_get(aLineList, "Flux_for_LSF", i_l, &status);
1804  a += cpl_table_get(aDetlines, "flux", i, &status) / 300.;
1805  cpl_table_set(aLineList, "Flux_for_LSF", i_l, a);
1806  } /* for i (all rows in aDetlines) */
1807 } /* muse_wave_lines_add_flux_for_lsf() */
1808 
1809 /*----------------------------------------------------------------------------*/
1836 /*----------------------------------------------------------------------------*/
1837 cpl_table *
1838 muse_wave_lines_search(muse_image *aColumnImage, double aSigma,
1839  const unsigned short aSlice, const unsigned char aIFU)
1840 {
1841 #if !SEARCH_DEBUG_FILES
1842  UNUSED_ARGUMENT(aSlice);
1843 #endif
1844  cpl_ensure(aColumnImage, CPL_ERROR_NULL_INPUT, NULL);
1845  cpl_ensure(cpl_image_get_min(aColumnImage->stat) > 0.,
1846  CPL_ERROR_DATA_NOT_FOUND, NULL);
1847  cpl_ensure(aSigma > 0., CPL_ERROR_ILLEGAL_INPUT, NULL);
1848 
1849 #define SEARCH_SUBTRACT_BG 1
1850 #if SEARCH_SUBTRACT_BG
1851  /* Subtract any large scale background using a median-filtered spectrum. *
1852  * In the longer run, maybe a polynomial fit to the median-smoothed data *
1853  * should be used instead, because it does not create extra noise. */
1854  cpl_image *bgmedian = cpl_image_duplicate(aColumnImage->data);
1855  cpl_image_fill_noise_uniform(bgmedian, -FLT_MIN, FLT_MIN);
1856 #define BG_KERNEL_WIDTH 51
1857  cpl_mask *filter = cpl_mask_new(1, BG_KERNEL_WIDTH);
1858  cpl_mask_not(filter);
1859  cpl_image_filter_mask(bgmedian, aColumnImage->data, filter, CPL_FILTER_MEDIAN,
1860  CPL_BORDER_FILTER);
1861  cpl_mask_delete(filter);
1862 #if SEARCH_DEBUG_FILES
1863  char fn[FILENAME_MAX];
1864  sprintf(fn, "column_slice%02hu.fits", aSlice);
1865  muse_image_save(aColumnImage, fn);
1866  sprintf(fn, "column_slice%02hu_bgmedian.fits", aSlice);
1867  cpl_image_save(bgmedian, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
1868 #endif
1869  cpl_image *bgsub = cpl_image_subtract_create(aColumnImage->data, bgmedian);
1870  cpl_image_delete(bgmedian);
1871 #if SEARCH_DEBUG_FILES
1872  sprintf(fn, "column_slice%02hu_bgsub.fits", aSlice);
1873  cpl_image_save(bgsub, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
1874 #endif
1875 #endif
1876 
1877  /* create the S/N image */
1878  cpl_image *N = cpl_image_power_create(aColumnImage->stat, 0.5), /* noise */
1879 #if SEARCH_SUBTRACT_BG
1880  *SN = cpl_image_divide_create(bgsub, N); /* S/N image */
1881  cpl_image_delete(bgsub);
1882 #else
1883  *SN = cpl_image_divide_create(aColumnImage->data, N); /* S/N image */
1884 #endif
1885  cpl_image_delete(N);
1886 #if SEARCH_DEBUG_FILES
1887  sprintf(fn, "column_slice%02hu_SN.fits", aSlice);
1888  cpl_image_save(SN, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
1889 #endif
1890 
1891  /* use median statistics of the S/N image to determine the detection limit */
1892  double mdev, median = cpl_image_get_median_dev(SN, &mdev),
1893  limitSN = fmax(0.1, median + aSigma*mdev);
1894  cpl_mask *mask = cpl_mask_threshold_image_create(SN, limitSN, FLT_MAX);
1895  cpl_size nlines = 0; /* number of detected lines */
1896  cpl_image *label = cpl_image_labelise_mask_create(mask, &nlines);
1897  cpl_mask_delete(mask);
1898 #if SEARCH_DEBUG_FILES
1899  sprintf(fn, "column_slice%02hu_label.fits", aSlice);
1900  cpl_image_save(label, fn, CPL_TYPE_USHORT, NULL, CPL_IO_DEFAULT);
1901 #endif
1902 #if SEARCH_DEBUG || SEARCH_DEBUG_FILES
1903  double mean = cpl_image_get_mean(SN),
1904  stdev = cpl_image_get_stdev(SN),
1905  min = cpl_image_get_min(SN),
1906  max = cpl_image_get_max(SN);
1907  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" lines found; parameters: sigma=%f, "
1908  "med=%f+/-%f " "mean=%f+/-%f min/max=%f/%f limit(S/N)=%f", nlines,
1909  aSigma, median, mdev, mean, stdev, min, max, limitSN);
1910 #endif
1911  cpl_image_delete(SN);
1912 
1913  /* create table to store the fit results of all the lines */
1914  cpl_table *linesresult = muse_cpltable_new(muse_wavelines_def, nlines);
1915 
1916  /* now loop through all the detected lines and measure them */
1917  int i;
1918  for (i = 0; i < nlines; i++) {
1919  /* create mask for this label to isolate line (note that i starts at 0!) */
1920  cpl_mask *linemask = cpl_mask_threshold_image_create(label, i+0.5, i+1.5);
1921  int masksize = cpl_mask_get_size_y(linemask);
1922 
1923  /* determine the edges of the labeled region and the center of the line */
1924  int j = 1;
1925  while (j <= masksize && cpl_mask_get(linemask, 1, j) == CPL_BINARY_0) {
1926  j++;
1927  }
1928  int lopix = j;
1929  while (j <= masksize && cpl_mask_get(linemask, 1, j) == CPL_BINARY_1) {
1930  j++;
1931  }
1932  int hipix = j - 1;
1933  double cenpix = (hipix + lopix) / 2.;
1934 #if SEARCH_DEBUG
1935  int err;
1936  cpl_msg_debug(__func__, "lo/hi/cen=%d,%d,%f: values=%f,%f", lopix, hipix,
1937  cenpix, cpl_image_get(aColumnImage->data, 1, lopix, &err),
1938  cpl_image_get(aColumnImage->data, 1, hipix, &err));
1939 #endif
1940  cpl_mask_delete(linemask);
1941  if (lopix == hipix) {
1942  /* one pixel is not enough for a solid detection: *
1943  * set to some very low area / flux to be deleted below */
1944  cpl_table_set(linesresult, "flux", i, -1.);
1945  continue;
1946  }
1947 
1948  /* Enlarge region to be sure to measure the whole line (correct *
1949  * width!). As long as the pixel values are smaller than on the *
1950  * edges, and we don't go too far this should be OK. */
1951 #define MAXENLARGE 5
1952  /* reference pixel value (lower edge of mask) */
1953  int err0, err1 = 0, err2 = 0;
1954  double valref = cpl_image_get(aColumnImage->data, 1, lopix, &err0);
1955  cpl_errorstate prestate = cpl_errorstate_get();
1956  j = lopix;
1957  double value = 0;
1958  while (err1 == 0 && value < valref && (lopix - j) <= MAXENLARGE) {
1959  j--; /* enlarge downwards */
1960  value = cpl_image_get(aColumnImage->data, 1, j, &err1);
1961  }
1962  lopix = j + 1; /* set new lower edge */
1963 
1964  /* reference pixel value (lower edge of mask) */
1965  valref = cpl_image_get(aColumnImage->data, 1, hipix, &err2);
1966  j = hipix;
1967  value = 0;
1968  while (err2 == 0 && value < valref && (j - hipix) <= MAXENLARGE) {
1969  j++; /* enlarge upwards */
1970  value = cpl_image_get(aColumnImage->data, 1, j, &err2);
1971  }
1972  hipix = j - 1; /* set new upper edge */
1973 #if SEARCH_DEBUG
1974  cpl_msg_debug(__func__, "region=%d...%d, size=%d", lopix, hipix,
1975  hipix - lopix + 1);
1976 #endif
1977  if (err1 < 0 || err2 < 0) {
1978  cpl_errorstate_set(prestate); /* clean possible "Access beyond boundaries" */
1979  }
1980 
1981  /* fill vectors */
1982  cpl_vector *positions = cpl_vector_new(hipix - lopix + 1), /* for positions */
1983  *values = cpl_vector_new(hipix - lopix + 1), /* for pixel values */
1984  *valuessigma = cpl_vector_new(hipix - lopix + 1); /* sigma values */
1985  int k;
1986  for (j = lopix, k = 0; j <= hipix; j++, k++) {
1987  cpl_vector_set(positions, k, j);
1988  cpl_vector_set(values, k, cpl_image_get(aColumnImage->data, 1, j, &err0));
1989  /* when setting pixel value from stat extension, use *
1990  * square root, here we need the sigma not the variance: */
1991  cpl_vector_set(valuessigma, k, sqrt(cpl_image_get(aColumnImage->stat,
1992  1, j, &err0)));
1993  }
1994 
1995 #if SEARCH_DEBUG
1996  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
1997  cpl_bivector_dump(biv, stdout);
1998  cpl_bivector_unwrap_vectors(biv);
1999 #endif
2000 
2001  /* fit Gaussian */
2002  /* compute position, sigma, area, background offset, and mean squared error */
2003  double center, cerr, sigma, area, bglevel, mse;
2004  cpl_matrix *covariance;
2005  prestate = cpl_errorstate_get();
2006  cpl_error_code rc = cpl_vector_fit_gaussian(positions, NULL, values,
2007  valuessigma, CPL_FIT_ALL,
2008  &center, &sigma, &area, &bglevel,
2009  &mse, NULL, &covariance);
2010 
2011  /* try to treat some possible problems */
2012  if (rc == CPL_ERROR_CONTINUE) { /* fit didn't converge */
2013  /* estimate position error as sigma^2/area as CPL docs suggest *
2014  * no warning necessary in this case, print a debug message */
2015  cerr = sqrt(sigma * sigma / area);
2016  cpl_msg_debug(__func__, "Gaussian fit in slice %hu of IFU %hhu around "
2017  "position %6.1f: %s", aSlice, aIFU, cenpix,
2018  cpl_error_get_message());
2019 #if DEBUG_GAUSSFIT
2020  if (cpl_msg_get_log_level() == CPL_MSG_DEBUG) {
2021  /* CPL_ERROR_CONTINUE often occurs if there is another peak on the edge *
2022  * of the input positions/data array, so print it out in debug mode */
2023  cpl_bivector *bv = cpl_bivector_wrap_vectors(positions, values);
2024  printf("Gaussian fit: %f+/-%f, %f, %f, %f %f\nand the input data:\n",
2025  center, cerr, bglevel, area, sigma, mse);
2026  cpl_bivector_dump(bv, stdout);
2027  cpl_bivector_unwrap_vectors(bv);
2028  fflush(stdout);
2029  }
2030 #endif
2031  } else if (rc == CPL_ERROR_SINGULAR_MATRIX || !covariance) {
2032  cerr = sqrt(sigma * sigma / area);
2033  cpl_msg_debug(__func__, "Gaussian fit in slice %hu of IFU %hhu around "
2034  "position %6.1f: %s", aSlice, aIFU, cenpix,
2035  cpl_error_get_message());
2036  } else if (rc != CPL_ERROR_NONE) {
2037  /* although this is unlikely to occur, better print some warning *
2038  * and do something to the data to give this fit less weight in *
2039  * the final computation of the solution */
2040  cpl_msg_debug(__func__, "Gaussian fit in slice %hu of IFU %hhu around "
2041  "position %6.1f: %s", aSlice, aIFU, cenpix,
2042  cpl_error_get_message());
2043  cerr = 100.; /* set a high centering error */
2044  } else {
2045  /* no error returned, everything should have worked nicely *
2046  * take the centering error from the covariance matrix */
2047  cerr = sqrt(cpl_matrix_get(covariance, 0, 0));
2048 #if SEARCH_DEBUG
2049  cpl_matrix_dump(covariance, stdout);
2050 #endif
2051  cpl_matrix_delete(covariance);
2052  }
2053  cpl_errorstate_set(prestate); /* now we can reset the status */
2054  if (fabs(center - cenpix) > MUSE_WAVE_LINES_SEARCH_SHIFT_WARN) {
2055  cpl_msg_debug(__func__, "Large shift in Gaussian centering in slice %hu "
2056  "of IFU %hhu: initial %7.2f, fit %7.2f", aSlice, aIFU,
2057  cenpix, center);
2058  }
2059 
2060  /* save all results of this fit into the output table *
2061  * (no need to fill the x-position into the "x" column) */
2062  cpl_table_set(linesresult, "y", i, cenpix);
2063  /* set actual peak value in the table using cpl_vector_get_max() is safe, *
2064  * because the vector is constructed to be small enough to not include *
2065  * neighboring lines and so the likelihood of including cosmic rays is low */
2066  cpl_table_set(linesresult, "peak", i, cpl_vector_get_max(values));
2067  cpl_table_set(linesresult, "center", i, center);
2068  cpl_table_set(linesresult, "cerr", i, cerr);
2069  cpl_table_set(linesresult, "fwhm", i, CPL_MATH_FWHM_SIG * sigma);
2070  cpl_table_set(linesresult, "sigma", i, sigma);
2071  cpl_table_set(linesresult, "flux", i, area);
2072  cpl_table_set(linesresult, "bg", i, bglevel);
2073  cpl_table_set(linesresult, "mse", i, mse);
2074 
2075  cpl_vector_delete(positions);
2076  cpl_vector_delete(values);
2077  cpl_vector_delete(valuessigma);
2078 #if SEARCH_DEBUG
2079  printf("%s: results of fit stored in table row %d:\n", __func__, i + 1);
2080  cpl_table_dump(linesresult, i, 1, stdout);
2081  fflush(stdout);
2082 #endif
2083  } /* for i (all detected lines) */
2084  cpl_image_delete(label);
2085 
2086  /* again loop through all the table lines, and remove unlikely detections */
2087  cpl_table_unselect_all(linesresult);
2088  for (i = 0; i < cpl_table_get_nrow(linesresult); i++) {
2089  if (cpl_table_get(linesresult, "cerr", i, NULL) > kMuseArcMaxCenteringError ||
2090  cpl_table_get(linesresult, "fwhm", i, NULL) < kMuseArcMinFWHM ||
2091  cpl_table_get(linesresult, "fwhm", i, NULL) > kMuseArcMaxFWHM ||
2092  cpl_table_get(linesresult, "flux", i, NULL) < kMuseArcMinFlux) {
2093  cpl_table_select_row(linesresult, i);
2094  }
2095  } /* for i (all detected lines) */
2096  cpl_table_erase_selected(linesresult);
2097 
2098  return linesresult;
2099 } /* muse_wave_lines_search() */
2100 
2101 /*----------------------------------------------------------------------------*/
2121 /*----------------------------------------------------------------------------*/
2122 cpl_error_code
2123 muse_wave_lines_identify(cpl_table *aLines, cpl_vector *aLambdas,
2124  const muse_wave_params *aParams)
2125 {
2126  cpl_ensure_code(aLines && aLambdas, CPL_ERROR_NULL_INPUT);
2127 
2128  /* convert detected lines from the table into a simple vector */
2129  int i, nlines = cpl_table_get_nrow(aLines);
2130  cpl_vector *vcenter = cpl_vector_new(nlines);
2131  for (i = 0; i < nlines; i++) {
2132  cpl_vector_set(vcenter, i, cpl_table_get(aLines, "center", i, NULL));
2133  } /* for i (all lines) */
2134 #if 0
2135  cpl_vector_dump(vcenter, stdout);
2136 #endif
2137  double dmin = kMuseSpectralSamplingA - kMuseSpectralSamplingA * aParams->ddisp,
2138  dmax = kMuseSpectralSamplingA + kMuseSpectralSamplingA * aParams->ddisp;
2139  cpl_bivector *id_lines = cpl_ppm_match_positions(vcenter, aLambdas, dmin, dmax,
2140  aParams->tolerance, NULL, NULL);
2141 #if 0
2142  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" identified lines (of %"
2143  CPL_SIZE_FORMAT" detected lines and %"CPL_SIZE_FORMAT
2144  " from linelist)", cpl_bivector_get_size(id_lines),
2145  cpl_vector_get_size(vcenter), cpl_vector_get_size(aLambdas));
2146  cpl_bivector_dump(id_lines, stdout);
2147  fflush(stdout);
2148 #endif
2149  cpl_vector_delete(vcenter);
2150 
2151 #if 0
2152  printf("detected lines before cleanup:\n");
2153  cpl_table_dump(aLines, 0, nlines, stdout);
2154  fflush(stdout);
2155 #endif
2156  /* now go through the detected lines and remove all unidentified */
2157  const double *id_center = cpl_bivector_get_x_data_const(id_lines),
2158  *id_lambda = cpl_bivector_get_y_data_const(id_lines);
2159  cpl_table_unselect_all(aLines);
2160  int j, nid = cpl_bivector_get_size(id_lines);
2161  for (i = 0, j = 0; i < cpl_table_get_nrow(aLines) && id_center && id_lambda; i++) {
2162 #if 0
2163  cpl_msg_debug(__func__, "c=%f, l=%f, c_det=%f", id_center[j], id_lambda[j],
2164  cpl_table_get(aLines, "center", i, NULL));
2165 #endif
2166  if (j < nid && fabs(id_center[j] - cpl_table_get(aLines, "center", i, NULL))
2167  < DBL_EPSILON) {
2168  /* found the same line, append the wavelength */
2169  cpl_table_set(aLines, "lambda", i, id_lambda[j]);
2170  j++;
2171  continue;
2172  }
2173  cpl_table_select_row(aLines, i); /* not matching, delete this line */
2174  } /* for i (detected lines) and j (identified lines) */
2175  cpl_table_erase_selected(aLines);
2176  cpl_bivector_delete(id_lines);
2177  int debug = getenv("MUSE_DEBUG_WAVECAL")
2178  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
2179  if (debug >= 2) {
2180  printf("identified %d lines, %"CPL_SIZE_FORMAT" after cleanup:\n", nid,
2181  cpl_table_get_nrow(aLines));
2182  cpl_table_dump(aLines, 0, nid, stdout);
2183  fflush(stdout);
2184  }
2185  nid = cpl_table_get_nrow(aLines);
2186  /* check that after identification enough lines are still there */
2187  if (nid <= 0) {
2188  return CPL_ERROR_DATA_NOT_FOUND;
2189  } else if (nid < aParams->yorder + 1) {
2190  return CPL_ERROR_ILLEGAL_OUTPUT;
2191  }
2192  return CPL_ERROR_NONE;
2193 } /* muse_wave_lines_identify() */
2194 
2195 /*----------------------------------------------------------------------------*/
2227 /*----------------------------------------------------------------------------*/
2228 cpl_table *
2229 muse_wave_line_handle_singlet(muse_image *aImage, cpl_table *aLinelist,
2230  unsigned int aIdx, cpl_polynomial *aPoly,
2231  cpl_polynomial **aTrace,
2232  const muse_wave_params *aParams,
2233  const unsigned short aSlice, int aDebug)
2234 {
2235  cpl_ensure(aImage && aLinelist && aPoly && aTrace, CPL_ERROR_NULL_INPUT, NULL);
2236 
2237  /* Fix the Gaussian sigmas for lines that are not FWHM reference lines, *
2238  * otherwise the fit might run astray in low S/N cases. A negative sigma *
2239  * signifies constant value to fitting routine. */
2240  double sigma = kMuseSliceSlitWidthA / kMuseSpectralSamplingA / CPL_MATH_FWHM_SIG;
2241  /* if not reference line, set it negative, so that it is fixed in the fit */
2242  if (cpl_table_get(aLinelist, MUSE_LINE_CATALOG_QUALITY, aIdx, NULL) != 5) {
2243  sigma *= -1;
2244  }
2245  int halfwidth = 3.*kMuseSliceSlitWidthA / kMuseSpectralSamplingA; /* 3*FWHM */
2246 
2247  /* convenient access to some properties of the current line */
2248  double lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, aIdx, NULL),
2249  ypos = cpl_polynomial_eval_1d(aPoly, lambda, NULL);
2250  if ((ypos - halfwidth) < 1 ||
2251  (ypos + halfwidth) > cpl_image_get_size_y(aImage->data)) {
2252  if (aDebug >= 2) {
2253  cpl_msg_debug(__func__, "%f is supposed to lie near %.3f in slice %2hu of"
2254  " IFU %hhu, i.e. outside!", lambda, ypos, aSlice,
2255  muse_utils_get_ifu(aImage->header));
2256  }
2257  return NULL;
2258  }
2259  if (aDebug >= 2) {
2260  cpl_msg_debug(__func__, "%f is supposed to lie near %.3f in slice %2hu of "
2261  "IFU %hhu", lambda, ypos, aSlice,
2262  muse_utils_get_ifu(aImage->header));
2263  }
2264  /* get both slice edges and the center */
2265  double dleft = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT],
2266  ypos, NULL),
2267  dright = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT],
2268  ypos, NULL),
2269  dmid = (dleft + dright) / 2.;
2270  int ileft = ceil(dleft),
2271  iright = floor(dright);
2272 #if 0
2273  cpl_msg_debug(__func__, "limits at y=%f: %f < %f < %f", ypos, dleft, dmid,
2274  dright);
2275 #endif
2276 
2277  /* table to store line fits for this one arc line */
2278  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
2279  (int)kMuseSliceHiLikelyWidth + 5);
2280 
2281  /* From the center of the slice move outwards and fit the line *
2282  * until we have arrived at the edge of the slice */
2283  double y = ypos;
2284  int i, n = 0;
2285  for (i = dmid; i >= ileft; i--) {
2286  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, y, halfwidth,
2287  sigma, fittable, ++n);
2288  if (rc != CPL_ERROR_NONE) { /* do not count this line */
2289  --n;
2290  } else {
2291  double ynew = cpl_table_get(fittable, "center", n - 1, NULL);
2292  if (fabs(y - ynew) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2293  y = ynew;
2294  }
2295  }
2296  } /* for i (columns to left of slice middle) */
2297 #if 0
2298  printf("arc line aIdx=%u, columns i=%d...", aIdx + 1, i);
2299 #endif
2300  y = ypos; /* start at middle y position again */
2301  for (i = dmid + 1; i <= iright; i++) {
2302  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
2303  sigma, fittable, ++n);
2304  if (rc != CPL_ERROR_NONE) { /* do not count this line */
2305  --n;
2306  } else {
2307  double ynew = cpl_table_get(fittable, "center", n - 1, NULL);
2308  if (fabs(y - ynew) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2309  y = ynew;
2310  }
2311  }
2312  } /* for i (columns to right of slice middle) */
2313  /* now remove rows with invalid entries, i.e. those that were not *
2314  * filled with the properties of the fit -- cpl_table_erase_invalid() *
2315  * does not work, it deletes all columns */
2316  cpl_table_select_all(fittable);
2317  cpl_table_and_selected_invalid(fittable, "center");
2318  cpl_table_erase_selected(fittable);
2319  cpl_table_fill_column_window(fittable, "lambda", 0,
2320  cpl_table_get_nrow(fittable), lambda);
2321 #if 0
2322  printf("line %u, %d line fits\n", aIdx + 1, n);
2323  cpl_table_dump(fittable, 0, n, stdout);
2324  fflush(stdout);
2325 #endif
2326  /* and reject deviant fits */
2327  muse_wave_line_fit_iterate(fittable, lambda, aParams);
2328  int npos = cpl_table_get_nrow(fittable);
2329  if (npos <= 0) {
2330  cpl_msg_warning(__func__, "Polynomial fit failed in slice %hu of IFU %hhu "
2331  "for line %u (y-position near %.2f pix): %s", aSlice,
2332  muse_utils_get_ifu(aImage->header), aIdx + 1, ypos,
2333  cpl_error_get_message());
2334  }
2335 #if 0
2336  else {
2337  printf("%s: line %2u, %d line fits, %d with low residuals:\n", __func__,
2338  aIdx + 1, n, npos);
2339  cpl_table_dump(fittable, 0, npos, stdout);
2340  fflush(stdout);
2341  }
2342 #endif
2343  return fittable;
2344 } /* muse_wave_line_handle_singlet() */
2345 
2346 /*----------------------------------------------------------------------------*/
2382 /*----------------------------------------------------------------------------*/
2383 cpl_table *
2384 muse_wave_line_handle_multiplet(muse_image *aImage, cpl_table *aLinelist,
2385  unsigned int aIdx, cpl_polynomial *aPoly,
2386  cpl_polynomial **aTrace,
2387  const muse_wave_params *aParams,
2388  const unsigned short aSlice, int aDebug)
2389 {
2390  cpl_ensure(aImage && aLinelist && aPoly && aTrace, CPL_ERROR_NULL_INPUT, NULL);
2391 
2392  /* search for the other line(s) of the multiplet */
2393 #define MULTIPLET_SEARCH_RANGE 40. /* 40 Angstrom for NeI 7024...7059 */
2394  unsigned int nlines = 1; /* for the moment we know of one line of the multiplet */
2395  double lbda1 = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, aIdx, NULL);
2396  const char *lamp1 = muse_wave_lines_get_lampname(aLinelist, aIdx);
2397  cpl_vector *lambdas = cpl_vector_new(nlines),
2398  *fluxes = cpl_vector_new(nlines);
2399  cpl_vector_set(lambdas, 0, lbda1);
2400  cpl_vector_set(fluxes, 0, cpl_table_get(aLinelist, MUSE_LINE_CATALOG_FLUX,
2401  aIdx, NULL));
2402  double lambda;
2403  unsigned int j = aIdx;
2404  for (lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, ++j, NULL);
2405  fabs(lambda - lbda1) < MULTIPLET_SEARCH_RANGE;
2406  lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, ++j, NULL)) {
2407  int quality = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, NULL);
2408  const char *lamp = muse_wave_lines_get_lampname(aLinelist, j);
2409  if (quality == 2 && !strcmp(lamp1, lamp)) { /* duplet of the same lamp */
2410  nlines++;
2411  cpl_vector_set_size(lambdas, nlines);
2412  cpl_vector_set_size(fluxes, nlines);
2413  cpl_vector_set(lambdas, nlines - 1, lambda);
2414  double flux = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_FLUX, j, NULL);
2415  cpl_vector_set(fluxes, nlines - 1, flux);
2416  /* invert the sign of the quality, so that this line is not found again */
2417  cpl_table_set(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, -quality);
2418  }
2419  } /* for lambda */
2420 
2421  if (aDebug >= 2) {
2422  printf("found multiplet of lamp %s with %u lines:\n", lamp1, nlines);
2423  cpl_bivector *bv = cpl_bivector_wrap_vectors(lambdas, fluxes);
2424  cpl_bivector_dump(bv, stdout);
2425  cpl_bivector_unwrap_vectors(bv);
2426  fflush(stdout);
2427  }
2428 
2429  /* positive sigma: using negative value (= fixed parameter in fit) actually *
2430  * causes higher deviations, worse RMS in the global fit, so leave it free */
2431  double sigma = kMuseSliceSlitWidthA / kMuseSpectralSamplingA / CPL_MATH_FWHM_SIG;
2432  int halfwidth = 3.*kMuseSliceSlitWidthA / kMuseSpectralSamplingA; /* 3*FWHM */
2433 
2434  cpl_vector *ypos = cpl_vector_new(nlines);
2435  for (j = 0; j < nlines; j++) {
2436  double yp = cpl_polynomial_eval_1d(aPoly, cpl_vector_get(lambdas, j), NULL);
2437  cpl_vector_set(ypos, j, yp);
2438  } /* for j */
2439  double ypos1 = cpl_vector_get(ypos, 0),
2440  ypos2 = cpl_vector_get(ypos, nlines - 1);
2441  cpl_bivector *peaks = cpl_bivector_wrap_vectors(ypos, fluxes);
2442  if ((ypos1 - halfwidth) < 1 ||
2443  (ypos2 + halfwidth) > cpl_image_get_size_y(aImage->data)) {
2444  if (aDebug >= 2) {
2445  cpl_msg_debug(__func__, "%f is supposed to lie at %.3f..%.3f in slice "
2446  "%2hu of IFU %hhu, i.e. outside!", lambda, ypos1, ypos2,
2447  aSlice, muse_utils_get_ifu(aImage->header));
2448  }
2449  cpl_bivector_delete(peaks);
2450  cpl_vector_delete(lambdas);
2451  return NULL;
2452  }
2453  if (aDebug >= 2) {
2454  cpl_msg_debug(__func__, "%f is supposed to lie at %.3f..%.3f in slice %2hu "
2455  "of IFU %hhu", lambda, ypos1, ypos2, aSlice,
2456  muse_utils_get_ifu(aImage->header));
2457  }
2458  /* get both slice edges and the center */
2459  double dleft = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT],
2460  (ypos1 + ypos2)/2., NULL),
2461  dright = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT],
2462  (ypos1 + ypos2)/2., NULL),
2463  dmid = (dleft + dright) / 2.;
2464  int ileft = ceil(dleft),
2465  iright = floor(dright);
2466 
2467  /* table to store line fits for this one arc line */
2468  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
2469  ((int)kMuseSliceHiLikelyWidth + 5) * nlines);
2470 
2471  /* From the center of the slice move outwards and fit the *
2472  * line until we have arrived at the edge of the slice */
2473  cpl_bivector *bp = cpl_bivector_duplicate(peaks),
2474  *bpgood = cpl_bivector_duplicate(peaks);
2475  int i, n = 0;
2476  for (i = dmid; i >= ileft; i--) {
2477  n += nlines;
2478  cpl_error_code rc = muse_wave_line_fit_multiple(aImage, i, bp, lambdas,
2479  halfwidth, sigma, fittable, n);
2480  if (rc != CPL_ERROR_NONE) { /* do not count this fit */
2481  cpl_bivector_delete(bp);
2482  bp = cpl_bivector_duplicate(bpgood);
2483  n -= nlines;
2484  } else {
2485  cpl_vector *shifts = cpl_vector_duplicate(cpl_bivector_get_x(bp));
2486  cpl_vector_subtract(shifts, cpl_bivector_get_x(bpgood));
2487  double shift = cpl_vector_get_median(shifts); /* may be non-const! */
2488  cpl_vector_delete(shifts);
2489  if (fabs(shift) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2490  cpl_bivector_delete(bpgood);
2491  bpgood = cpl_bivector_duplicate(bp);
2492  } else {
2493  cpl_bivector_delete(bp);
2494  bp = cpl_bivector_duplicate(bpgood);
2495  }
2496  } /* else (good fit) */
2497  } /* for i (columns to left of slice middle) */
2498  cpl_bivector_delete(bp);
2499  cpl_bivector_delete(bpgood);
2500  bp = cpl_bivector_duplicate(peaks);
2501  bpgood = cpl_bivector_duplicate(peaks);
2502  for (i = dmid + 1; i <= iright; i++) {
2503  n += nlines;
2504  cpl_error_code rc = muse_wave_line_fit_multiple(aImage, i, bp, lambdas,
2505  halfwidth, sigma, fittable, n);
2506  if (rc != CPL_ERROR_NONE) { /* do not count this fit */
2507  cpl_bivector_delete(bp);
2508  bp = cpl_bivector_duplicate(bpgood);
2509  n -= nlines;
2510  } else {
2511  cpl_vector *shifts = cpl_vector_duplicate(cpl_bivector_get_x(bp));
2512  cpl_vector_subtract(shifts, cpl_bivector_get_x(bpgood));
2513  double shift = cpl_vector_get_median(shifts); /* may be non-const! */
2514  cpl_vector_delete(shifts);
2515  if (fabs(shift) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2516  cpl_bivector_delete(bpgood);
2517  bpgood = cpl_bivector_duplicate(bp);
2518  } else {
2519  cpl_bivector_delete(bp);
2520  bp = cpl_bivector_duplicate(bpgood);
2521  }
2522  } /* else (good fit) */
2523  } /* for i (columns to right of slice middle) */
2524  cpl_bivector_delete(bp);
2525  cpl_bivector_delete(bpgood);
2526  /* now remove rows with invalid entries, i.e. those that *
2527  * were not filled with the properties of the fit */
2528  cpl_table_select_all(fittable);
2529  cpl_table_and_selected_invalid(fittable, "center");
2530  cpl_table_erase_selected(fittable);
2531  cpl_bivector_delete(peaks);
2532  /* and reject deviant fits */
2533  for (j = 0; j < nlines; j++) {
2534  muse_wave_line_fit_iterate(fittable, cpl_vector_get(lambdas, j), aParams);
2535  } /* for i */
2536  cpl_vector_delete(lambdas);
2537 
2538  return fittable;
2539 } /* muse_wave_line_handle_multiplet() */
2540 
2541 /*----------------------------------------------------------------------------*/
2572 /*----------------------------------------------------------------------------*/
2573 cpl_error_code
2574 muse_wave_line_fit_single(muse_image *aImage, int aX, double aY, int aHalfWidth,
2575  double aSigma, cpl_table *aFitTable, int aRowsNeeded)
2576 {
2577  cpl_ensure_code(aImage && aImage->data && aImage->stat && aFitTable,
2578  CPL_ERROR_NULL_INPUT);
2579 #if 0
2580  cpl_msg_debug(__func__, "%d %d %d -> %d, %d", aX, (int)aY, aHalfWidth,
2581  2*aHalfWidth+1, aRowsNeeded);
2582 #endif
2583  /* determine area on the input image, copy area into the vectors */
2584  cpl_vector *positions = cpl_vector_new(2*aHalfWidth+1),
2585  *values = cpl_vector_new(2*aHalfWidth+1),
2586  *valuessigma = cpl_vector_new(2*aHalfWidth+1);
2587  int i, j, ny = cpl_image_get_size_y(aImage->data);
2588  for (i = (int)aY - aHalfWidth, j = 0; i <= (int)aY + aHalfWidth && i <= ny;
2589  i++, j++) {
2590  int err;
2591  cpl_vector_set(positions, j, i);
2592  cpl_vector_set(values, j, cpl_image_get(aImage->data, aX, i, &err));
2593  cpl_vector_set(valuessigma, j,
2594  sqrt(cpl_image_get(aImage->stat, aX, i, &err)));
2595  }
2596 #if 0
2597  printf("input vectors: positions, values\n");
2598  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
2599  cpl_bivector_dump(biv, stdout);
2600  fflush(stdout);
2601  cpl_bivector_unwrap_vectors(biv);
2602 #if 0
2603  printf("input vectors: values, valuessigma\n");
2604  biv = cpl_bivector_wrap_vectors(values, valuessigma);
2605  cpl_bivector_dump(biv, stdout);
2606  fflush(stdout);
2607  cpl_bivector_unwrap_vectors(biv);
2608 #endif
2609 #endif
2610 
2611  /* do the Gaussfit */
2612  cpl_errorstate prestate = cpl_errorstate_get();
2613  double center, cerror, area, bglevel, mse;
2614  cpl_matrix *covariance = NULL;
2615  cpl_fit_mode mode = CPL_FIT_ALL;
2616  double sigma = aSigma;
2617  if (sigma < 0) {
2618  mode = CPL_FIT_CENTROID | CPL_FIT_AREA | CPL_FIT_OFFSET;
2619  sigma *= -1;
2620  }
2621  cpl_error_code rc
2622  = cpl_vector_fit_gaussian(positions, NULL, values, valuessigma, mode,
2623  &center, &sigma, &area, &bglevel, &mse, NULL,
2624  &covariance);
2625 #if 0
2626  printf("--> rc=%d (%d) %s %#x %#x\n", rc, CPL_ERROR_NONE,
2627  cpl_error_get_message(), covariance, &covariance);
2628  fflush(stdout);
2629 #endif
2630  cpl_vector_delete(positions);
2631  cpl_vector_delete(values);
2632  cpl_vector_delete(valuessigma);
2633  /* Determine goodness of fit and try to treat some possible problems. *
2634  * (This is partly copied from muse_wave_lines_search().) */
2635  /* if no error happened, take the centering error from the covariance matrix */
2636  if (covariance) {
2637  cerror = sqrt(cpl_matrix_get(covariance, 0, 0));
2638  cpl_matrix_delete(covariance);
2639  } else { /* a missing covariance matrix is bad */
2640  cpl_msg_debug(__func__, "Gauss fit produced no covariance matrix (y=%.3f in "
2641  "column=%d): %s", aY, aX, cpl_error_get_message());
2642  cpl_errorstate_set(prestate);
2643  return rc == CPL_ERROR_NONE ? CPL_ERROR_ILLEGAL_OUTPUT : rc;
2644  }
2645  if (rc == CPL_ERROR_CONTINUE) { /* fit didn't converge */
2646  /* estimate position error as sigma^2/area as CPL docs *
2647  * suggest and pretend that everything worked fine */
2648  cerror = sqrt(sigma * sigma / area);
2649  cpl_errorstate_set(prestate);
2650  rc = CPL_ERROR_NONE;
2651  }
2652  if (rc != CPL_ERROR_NONE) {
2653  /* some other, probably more serious error, don't use this fit */
2654  cpl_msg_debug(__func__, "Gauss fit failed with some error (y=%.3f in "
2655  "column=%d): %s", aY, aX, cpl_error_get_message());
2656  cpl_errorstate_set(prestate);
2657  return rc;
2658  }
2659  if (fabs(center - aY) > MUSE_WAVE_LINE_FIT_MAXSHIFT) {
2660  /* too big shift, don't use this fit */
2661  cpl_msg_debug(__func__, "Gauss fit gave unexpectedly large offset "
2662  "(shifted %.3f pix from y=%.3f in column=%d)",
2663  center - aY, aY, aX);
2664  return CPL_ERROR_ACCESS_OUT_OF_RANGE;
2665  }
2666 
2667  /* store the results in the table, after making sure it is large enough */
2668  if (cpl_table_get_nrow(aFitTable) < aRowsNeeded) {
2669  cpl_table_set_size(aFitTable, aRowsNeeded);
2670  }
2671  cpl_table_set(aFitTable, "center", aRowsNeeded - 1, center);
2672  cpl_table_set(aFitTable, "cerr", aRowsNeeded - 1, cerror);
2673  cpl_table_set(aFitTable, "sigma", aRowsNeeded - 1, sigma);
2674  if (mode == CPL_FIT_ALL) { /* is FWHM reference line, store the FWHM */
2675  cpl_table_set(aFitTable, "fwhm", aRowsNeeded - 1, CPL_MATH_FWHM_SIG * sigma);
2676  }
2677  cpl_table_set(aFitTable, "flux", aRowsNeeded - 1, area);
2678  cpl_table_set(aFitTable, "bg", aRowsNeeded - 1, bglevel);
2679  cpl_table_set(aFitTable, "mse", aRowsNeeded - 1, mse);
2680  cpl_table_set(aFitTable, "x", aRowsNeeded - 1, aX);
2681  cpl_table_set(aFitTable, "y", aRowsNeeded - 1, aY);
2682 
2683 #if 0
2684  printf("--> Gaussian fit: %f +/- %f, %f\n", center, cerror, area);
2685  fflush(stdout);
2686 #endif
2687  return rc;
2688 } /* muse_wave_line_fit_single() */
2689 
2690 /*----------------------------------------------------------------------------*/
2728 /*----------------------------------------------------------------------------*/
2729 cpl_error_code
2730 muse_wave_line_fit_multiple(muse_image *aImage, int aX, cpl_bivector *aPeaks,
2731  cpl_vector *aLambdas, int aHalfWidth, double aSigma,
2732  cpl_table *aFitTable, int aRowsNeeded)
2733 {
2734  cpl_ensure_code(aImage && aImage->data && aImage->stat && aFitTable,
2735  CPL_ERROR_NULL_INPUT);
2736  cpl_vector *ypeaks = cpl_bivector_get_x(aPeaks),
2737  *fluxes = cpl_bivector_get_y(aPeaks);
2738  int np = cpl_vector_get_size(ypeaks);
2739  double yp1 = cpl_vector_get(ypeaks, 0),
2740  yp2 = cpl_vector_get(ypeaks, np - 1);
2741  int i1 = floor(yp1),
2742  i2 = ceil(yp2);
2743 #if 0
2744  cpl_msg_debug(__func__, "%d %d..%d %d -> %d", aX, i1, i2, aHalfWidth,
2745  aRowsNeeded);
2746 #endif
2747  /* determine area on the input image, copy area into the vectors */
2748  int npoints = (i2 + aHalfWidth) - (i1 - aHalfWidth) + 1;
2749  cpl_vector *positions = cpl_vector_new(npoints),
2750  *values = cpl_vector_new(npoints),
2751  *valuessigma = cpl_vector_new(npoints);
2752  double minval = DBL_MAX; /* 1st-guess fit value */
2753  int i, j, ny = cpl_image_get_size_y(aImage->data);
2754  for (i = i1 - aHalfWidth, j = 0; i <= i2 + aHalfWidth && i <= ny; i++, j++) {
2755  int err;
2756  cpl_vector_set(positions, j, i);
2757  double value = cpl_image_get(aImage->data, aX, i, &err);
2758  cpl_vector_set(values, j, value);
2759  if (value < minval) {
2760  minval = value;
2761  }
2762  cpl_vector_set(valuessigma, j,
2763  sqrt(cpl_image_get(aImage->stat, aX, i, &err)));
2764  }
2765  minval = minval > 0 ? minval : 0.;
2766 #if 0
2767  printf("input vectors: positions, values\n");
2768  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
2769  cpl_bivector_dump(biv, stdout);
2770  fflush(stdout);
2771  cpl_bivector_unwrap_vectors(biv);
2772 #endif
2773  cpl_bivector *bvalues = cpl_bivector_wrap_vectors(values, valuessigma);
2774 
2775  /* do the multi-Gauss fit */
2776  /* parameters: slope, background, sigma, np * center, np * flux */
2777  cpl_vector *poly = cpl_vector_new(2);
2778  cpl_vector_set(poly, 0, minval);
2779  cpl_vector_set(poly, 1, 0.);
2780  /* record the index of the expected strongest line, and its position */
2781  cpl_array *afluxes = cpl_array_wrap_double(cpl_vector_get_data(fluxes), np);
2782  cpl_size imaxflux;
2783  cpl_array_get_maxpos(afluxes, &imaxflux);
2784  double yposmax = cpl_vector_get(ypeaks, imaxflux);
2785  cpl_array_unwrap(afluxes);
2786  /* do the fit, store the errorstate before */
2787  cpl_errorstate prestate = cpl_errorstate_get();
2788  double sigma = aSigma,
2789  mse, chisq;
2790  cpl_matrix *covariance;
2791  cpl_error_code rc = muse_utils_fit_multigauss_1d(positions, bvalues, ypeaks,
2792  &sigma, fluxes, poly,
2793  &mse, &chisq, &covariance);
2794  cpl_vector_delete(positions);
2795  cpl_bivector_delete(bvalues);
2796 
2797  /* Determine goodness of fit and try to treat some possible problems. *
2798  * (This is partly copied from muse_wave_lines_search().) */
2799  /* if no error happened, take the centering error from the covariance matrix */
2800  if (!covariance) { /* a missing covariance matrix is bad */
2801  cpl_msg_debug(__func__, "Multi-Gauss fit produced no covariance matrix (y=%."
2802  "3f..%.3f in column=%d): %s", yp1, yp2, aX, cpl_error_get_message());
2803  cpl_errorstate_set(prestate);
2804  cpl_vector_delete(poly);
2805  return rc == CPL_ERROR_NONE ? CPL_ERROR_ILLEGAL_OUTPUT : rc;
2806  }
2807  if (rc != CPL_ERROR_NONE) {
2808  /* some other, probably more serious error, don't use this fit */
2809  cpl_msg_debug(__func__, "Multi-Gauss fit failed with some error (y=%.3f..%."
2810  "3f in column=%d): %s", yp1, yp2, aX, cpl_error_get_message());
2811  cpl_errorstate_set(prestate);
2812  cpl_matrix_delete(covariance);
2813  cpl_vector_delete(poly);
2814  return rc;
2815  }
2816  /* did the center of the brightest line shift too much? */
2817  double yfitmax = cpl_vector_get(ypeaks, imaxflux);
2818  if (fabs(yposmax - yfitmax) > MUSE_WAVE_LINE_FIT_MAXSHIFT) {
2819  cpl_msg_debug(__func__, "Multi-Gauss fit gave unexpectedly large offset "
2820  "(shifted %.3f pix from y=%.3f..%.3f in column=%d)",
2821  yposmax - yfitmax, yp1, yp2, aX);
2822  cpl_matrix_delete(covariance);
2823  cpl_vector_delete(poly);
2824  return CPL_ERROR_ACCESS_OUT_OF_RANGE;
2825  }
2826  /* are any of the fluxes negative? */
2827  double yfitmin = cpl_vector_get_min(fluxes);
2828  if (yfitmin < 0) {
2829  cpl_msg_debug(__func__, "Multi-Gauss fit gave negative flux (%e in "
2830  "multiplet from y=%.3f..%.3f in column=%d)", yfitmin,
2831  yp1, yp2, aX);
2832  cpl_matrix_delete(covariance);
2833  cpl_vector_delete(poly);
2834  return CPL_ERROR_ILLEGAL_OUTPUT;
2835  }
2836 
2837  /* store the results in the table, after making sure it is large enough */
2838  if (cpl_table_get_nrow(aFitTable) < aRowsNeeded) {
2839  cpl_table_set_size(aFitTable, aRowsNeeded);
2840  }
2841  /* first record the common values */
2842  cpl_table_fill_column_window(aFitTable, "mse", aRowsNeeded - np, np, mse);
2843  cpl_table_fill_column_window(aFitTable, "x", aRowsNeeded - np, np, aX);
2844  cpl_table_fill_column_window(aFitTable, "sigma", aRowsNeeded - np, np, sigma);
2845  /* multiplets are never FWHM reference lines, do not store FWHM */
2846  for (i = 0, j = aRowsNeeded - np; i < np; i++, j++) {
2847  cpl_table_set(aFitTable, "lambda", j, cpl_vector_get(aLambdas, i));
2848  cpl_table_set(aFitTable, "y", j, cpl_vector_get(ypeaks, i));
2849  double center = cpl_vector_get(ypeaks, i);
2850  cpl_table_set(aFitTable, "center", j, center);
2851  /* centroid errors are starting with row/column index 3 in the matrix, *
2852  * the first ones are the two polynomial coeffs and the sigma */
2853  double cerror = sqrt(cpl_matrix_get(covariance, 3 + i, 3 + i));
2854  cpl_table_set(aFitTable, "cerr", j, cerror);
2855  cpl_table_set(aFitTable, "flux", j, cpl_vector_get(fluxes, i));
2856  /* evaluate the linear background level at the fitted center */
2857  double bg = cpl_vector_get(poly, 0) + cpl_vector_get(poly, 1) * center;
2858  cpl_table_set(aFitTable, "bg", j, bg);
2859  } /* for i, j */
2860  cpl_vector_delete(poly);
2861  cpl_matrix_delete(covariance);
2862 #if 0
2863  printf("stored %d fitted multiplet values:\n", np);
2864  cpl_table_dump(aFitTable, aRowsNeeded - np - 2 < 0 ? 0 : aRowsNeeded - np - 2, np + 5, stdout);
2865  fflush(stdout);
2866 #endif
2867 
2868  return rc;
2869 } /* muse_wave_line_fit_multiple() */
2870 
2871 /*----------------------------------------------------------------------------*/
2910 /*----------------------------------------------------------------------------*/
2911 cpl_error_code
2912 muse_wave_line_fit_iterate(cpl_table *aFitTable, double aLambda,
2913  const muse_wave_params *aParams)
2914 {
2915  cpl_ensure_code(aFitTable, CPL_ERROR_NULL_INPUT);
2916  int npos = cpl_table_get_nrow(aFitTable);
2917  cpl_ensure_code(npos > 0, CPL_ERROR_ILLEGAL_INPUT);
2918  double rsigma = aParams->linesigma < 0 ? 2.5 : aParams->linesigma;
2919 
2920  /* select all entries with the requested wavelength */
2921  cpl_table *fittable = NULL;
2922  if (aLambda > 0) {
2923  cpl_table_unselect_all(aFitTable);
2924  cpl_table_or_selected_double(aFitTable, "lambda", CPL_EQUAL_TO, aLambda);
2925  npos = cpl_table_count_selected(aFitTable);
2926  cpl_ensure_code(npos > 0, CPL_ERROR_ILLEGAL_INPUT);
2927  fittable = cpl_table_extract_selected(aFitTable);
2928  cpl_table_erase_selected(aFitTable);
2929  } else {
2930  fittable = aFitTable;
2931  }
2932 
2933  /* convert "x" and "center" columns into matrix and vector for the fit */
2934  cpl_matrix *pos = cpl_matrix_new(1, npos);
2935  cpl_vector *val = cpl_vector_new(npos);
2936  int i;
2937  for (i = 0; i < npos; i++) {
2938  cpl_matrix_set(pos, 0, i, cpl_table_get(fittable, "x", i, NULL));
2939  cpl_vector_set(val, i, cpl_table_get(fittable, "center", i, NULL));
2940  }
2941 
2942  /* use the iterative polynomial field to throw out bad entries in fittable */
2943  cpl_errorstate state = cpl_errorstate_get();
2944  double mse;
2945  cpl_polynomial *fit = muse_utils_iterate_fit_polynomial(pos, val, NULL, fittable,
2946  aParams->xorder, rsigma,
2947  &mse, NULL);
2948  cpl_matrix_delete(pos);
2949  cpl_vector_delete(val);
2950  cpl_polynomial_delete(fit);
2951  if (!cpl_errorstate_is_equal(state)) {
2952  /* this implies less than xorder entries left, and is really bad; set the *
2953  * error to something high enough to basically exclude these points */
2954  cpl_table_fill_column_window(fittable, "cerr",
2955  0, cpl_table_get_nrow(fittable), 10.);
2956  } else if (aParams->fitweighting == MUSE_WAVE_WEIGHTING_SCATTER) {
2957  /* replace the original centroid error with the fit RMS */
2958  cpl_table_fill_column_window(fittable, "cerr",
2959  0, cpl_table_get_nrow(fittable), sqrt(mse));
2960  } else if (aParams->fitweighting == MUSE_WAVE_WEIGHTING_CERRSCATTER) {
2961  /* add RMS of fit to individual error in quadrature: *
2962  * cerr = sqrt(cerr^2 + mse) */
2963  cpl_table_power_column(fittable, "cerr", 2.); /* cerr^2 */
2964  cpl_table_add_scalar(fittable, "cerr", mse); /* + mse */
2965  cpl_table_power_column(fittable, "cerr", 0.5); /* sqrt() */
2966  }
2967 #if 0
2968  printf("line at %.3f Angstrom: rms = %f (mean error %f)\n", aLambda,
2969  sqrt(mse), cpl_table_get_column_mean(fittable, "cerr"));
2970  cpl_table_dump(fittable, 0, npos, stdout);
2971  fflush(stdout);
2972 #endif
2973  if (aLambda > 0) { /* need to copy results back into input table */
2974  cpl_table_insert(aFitTable, fittable, cpl_table_get_nrow(aFitTable));
2975  cpl_table_delete(fittable);
2976  }
2977  return CPL_ERROR_NONE;
2978 } /* muse_wave_line_fit_iterate() */
2979 
2980 /*----------------------------------------------------------------------------*/
2992 /*----------------------------------------------------------------------------*/
2993 static inline double
2994 muse_wave_ipow(double x, unsigned int y)
2995 {
2996  if (!y) {
2997  return 1;
2998  }
2999  if (y == 1) {
3000  return x;
3001  }
3002  double x2 = x*x;
3003  if (y == 2) {
3004  return x2;
3005  }
3006  if (y == 3) {
3007  return x2*x;
3008  }
3009  if (y == 4) {
3010  return x2*x2;
3011  }
3012  double result;
3013  /* Handle least significant bit in y here in order to avoid an unnecessary *
3014  * multiplication of x - which could cause an over- or underflow */
3015  result = y & 1 ? x : 1.; /* 0^0 is 1, while any other power of 0 is 0 */
3016  while (y >>= 1) {
3017  x *= x;
3018  if (y & 1) { /* Process least significant bit in y */
3019  result *= x;
3020  }
3021  }
3022  return result;
3023 } /* muse_wave_ipow() */
3024 
3025 /*----------------------------------------------------------------------------*/
3040 /*----------------------------------------------------------------------------*/
3041 static int
3042 muse_wave_poly_2d(const double xy[], const double p[], double *f)
3043 {
3044  unsigned short xorder = p[0],
3045  yorder = p[1];
3046  /* add up the polynomial orders */
3047  *f = 0;
3048  double x = xy[0],
3049  y = xy[1];
3050  unsigned int i, idx = 2;
3051  for (i = 0; i <= xorder; i++) {
3052  double xi = muse_wave_ipow(x, i);
3053  unsigned int j; /* y order */
3054  for (j = 0; j <= yorder; j++) {
3055  *f += p[idx++] * xi * muse_wave_ipow(y, j);
3056  } /* for j */
3057  } /* for i */
3058  return 0;
3059 } /* muse_wave_poly_2d() */
3060 
3061 /*----------------------------------------------------------------------------*/
3074 /*----------------------------------------------------------------------------*/
3075 static int
3076 muse_wave_dpoly_2d(const double xy[], const double p[], double f[])
3077 {
3078  unsigned short xorder = p[0],
3079  yorder = p[1];
3080  f[0] = f[1] = 0; /* derivatives regarding the polynomial orders are zero */
3081  /* now the real derivatives, they don't contain the coefficients any more */
3082  double x = xy[0],
3083  y = xy[1];
3084  unsigned int i, idx = 2;
3085  for (i = 0; i <= xorder; i++) {
3086  double xi = muse_wave_ipow(x, i);
3087  unsigned int j; /* y order */
3088  for (j = 0; j <= yorder; j++) {
3089  f[idx++] = xi * muse_wave_ipow(y, j);
3090  } /* for j */
3091  } /* for i */
3092  return 0;
3093 } /* muse_wave_poly_x2y5_deriv() */
3094 
3095 /*----------------------------------------------------------------------------*/
3130 /*----------------------------------------------------------------------------*/
3131 cpl_error_code
3132 muse_wave_poly_fit(cpl_matrix *aXYPos, cpl_vector *aLambdas, cpl_vector *aDLambdas,
3133  cpl_polynomial **aPoly, double *aMSE, muse_wave_params *aParams,
3134  const unsigned short aSlice)
3135 {
3136  cpl_ensure_code(aPoly && aXYPos && aLambdas, CPL_ERROR_NULL_INPUT);
3137 #if 0
3138  printf("aXYPos (%"CPL_SIZE_FORMAT"x%"CPL_SIZE_FORMAT"):\n",
3139  cpl_matrix_get_ncol(aXYPos), cpl_matrix_get_nrow(aXYPos));
3140  cpl_matrix_dump(aXYPos, stdout);
3141  printf("aLambdas (%"CPL_SIZE_FORMAT"):\n", cpl_vector_get_size(aLambdas));
3142  cpl_vector_dump(aLambdas, stdout);
3143  printf("aDLambdas (%"CPL_SIZE_FORMAT"):\n", cpl_vector_get_size(aDLambdas));
3144  cpl_vector_dump(aDLambdas, stdout);
3145  fflush(stdout);
3146 #endif
3147  double rsigma = aParams->fitsigma < 0 ? 3.0 : aParams->fitsigma;
3148  int debug = getenv("MUSE_DEBUG_WAVECAL")
3149  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
3150  /* prepare the polynomial for two dimensions */
3151  *aPoly = cpl_polynomial_new(2);
3152 
3153  double rms = -1; /* start with negative (= invalid) RMS */
3154  int large_residuals = 1, /* init to force a first fit */
3155  niter = 1;
3156  while (large_residuals > 0) {
3157  /* set some typical fit error by default */
3158  cpl_error_code errfit = CPL_ERROR_SINGULAR_MATRIX;
3159 
3160  if (aDLambdas) { /* use our own polynomial fitting with weighting */
3161  /* the lvmq function wants N rows of dimension D=2 *
3162  * instead of the other way around... */
3163  cpl_matrix *xypos = cpl_matrix_transpose_create(aXYPos);
3164 
3165  int npar = 2 + (aParams->xorder + 1) * (aParams->yorder + 1);
3166  cpl_vector *pars = cpl_vector_new(npar);
3167  /* pre-fill all but the zero-order with zero as the first guess value */
3168  cpl_vector_fill(pars, 0.);
3169  /* the polynomial orders go first into the array */
3170  cpl_vector_set(pars, 0, aParams->xorder);
3171  cpl_vector_set(pars, 1, aParams->yorder);
3172  cpl_vector_set(pars, 2, 5000.); /* near lower edge of lambda range */
3173  cpl_array *aflags = cpl_array_new(npar, CPL_TYPE_INT);
3174  /* most are real (free == 1) parameters, except the first two */
3175  cpl_array_set_int(aflags, 0, 0); /* fixed xorder */
3176  cpl_array_set_int(aflags, 1, 0); /* fixed yorder */
3177  cpl_array_fill_window_int(aflags, 2, npar - 2, 1);
3178  int *pflags = cpl_array_get_data_int(aflags);
3179  errfit = cpl_fit_lvmq(xypos, /* sigma_x (unsupported) */NULL,
3180  aLambdas, aDLambdas, pars, pflags,
3181  muse_wave_poly_2d, muse_wave_dpoly_2d,
3182  CPL_FIT_LVMQ_TOLERANCE, CPL_FIT_LVMQ_COUNT,
3183  CPL_FIT_LVMQ_MAXITER, NULL, NULL, NULL);
3184  cpl_array_delete(aflags);
3185  cpl_size k = 2; /* coefficients start at the 3rd vector entry */
3186  unsigned short i;
3187  for (i = 0; i <= aParams->xorder; i++) {
3188  unsigned short j;
3189  for (j = 0; j <= aParams->yorder; j++) {
3190  cpl_size pows[2] = { i, j }; /* trick to access the polynomial */
3191  cpl_polynomial_set_coeff(*aPoly, pows,
3192  cpl_vector_get(pars, k++));
3193  } /* for j */
3194  } /* for i */
3195  cpl_matrix_delete(xypos);
3196  cpl_vector_delete(pars);
3197  } else {
3198  /* use CPL polynomial fitting for everything else */
3199  const cpl_boolean sym = CPL_FALSE;
3200  const cpl_size mindeg2d[] = { 0, 0 },
3201  maxdeg2d[] = { aParams->xorder, aParams->yorder };
3202  errfit = cpl_polynomial_fit(*aPoly, aXYPos, &sym, aLambdas, NULL,
3203  CPL_TRUE, mindeg2d, maxdeg2d);
3204  } /* CPL polynomial fitting */
3205 #if 0
3206  printf("polynomial (orders=%hu/%hu, degree=%"CPL_SIZE_FORMAT"):\n",
3207  aParams->xorder, aParams->yorder, cpl_polynomial_get_degree(*aPoly));
3208  cpl_polynomial_dump(*aPoly, stdout);
3209  fflush(stdout);
3210 #endif
3211 
3212  if (errfit) {
3213  /* the fit failed, complain and abort */
3214  cpl_msg_error(__func__, "The polynomial fit in slice %hu failed: %s",
3215  aSlice, cpl_error_get_message());
3216 
3217 #if 0
3218  /* more fancy plotting of input datapoints *
3219  * (color coded lambda over x/y-position) */
3220  FILE *gp = popen("gnuplot -persist", "w");
3221  if (gp) {
3222  /* plot title with fit details */
3223  fprintf(gp, "set title \"2D polynomial fit residuals (failed fit: "
3224  "\'%s\')\n", cpl_error_get_message());
3225  /* set nice palette */
3226  fprintf(gp, "set palette defined ( 0 \"dark-violet\","
3227  "1 \"dark-blue\", 4 \"green\", 6 \"yellow\", 8 \"orange\","
3228  "9 \"red\", 10 \"dark-red\")\n");
3229  fprintf(gp, "unset key\n");
3230  cpl_matrix *xpos = cpl_matrix_extract_row(aXYPos, 0);
3231  fprintf(gp, "set xrange [%f:%f]\n", cpl_matrix_get_min(xpos),
3232  cpl_matrix_get_max(xpos));
3233  cpl_matrix_delete(xpos);
3234  fprintf(gp, "set cbrange [%f:%f]\n", cpl_vector_get_min(aLambdas),
3235  cpl_vector_get_max(aLambdas));
3236  fprintf(gp, "plot \"-\" w p pal\n");
3237 
3238  printf("# X Y lambda\n");
3239  int n;
3240  for (n = 0; n < cpl_vector_get_size(aLambdas); n++) {
3241  printf("%4d %7.2f %8.3f\n", (int)cpl_matrix_get(aXYPos, 0, n),
3242  cpl_matrix_get(aXYPos, 1, n), cpl_vector_get(aLambdas, n));
3243  fprintf(gp, "%f %f %f\n", cpl_matrix_get(aXYPos, 0, n),
3244  cpl_matrix_get(aXYPos, 1, n), cpl_vector_get(aLambdas, n));
3245  }
3246  fflush(stdout);
3247  fprintf(gp, "EOF\n");
3248  pclose(gp);
3249  }
3250 #endif
3251 
3252  /* make sure we delete the polynomial */
3253  cpl_polynomial_delete(*aPoly);
3254  *aPoly = NULL; /* make sure we can detect the failure */
3255  return CPL_ERROR_ILLEGAL_OUTPUT;
3256  } /* if failed fit */
3257 
3258  int npoints = cpl_vector_get_size(aLambdas);
3259  cpl_vector *res = cpl_vector_new(npoints);
3260  /* the aDLambdas parameter (the errors of the datapoints) do not *
3261  * change the filled vector, just the chi**2 which we don't need */
3262  cpl_vector_fill_polynomial_fit_residual(res, aLambdas, aDLambdas, *aPoly,
3263  aXYPos, NULL);
3264  /* compute the mean-squared error, without any weighting, in any case */
3265  double mse = 0;
3266  /* compute a (weighted) mean-squared error, if we know the error bars */
3267  if (aDLambdas) { /* no errors given, equal weights for all points */
3268  double wsum = 0;
3269  int i;
3270  for (i = 0; i < npoints; i++) {
3271  double weight = 1. / cpl_vector_get(aDLambdas, i);
3272  mse += pow(cpl_vector_get(res, i), 2) * weight;
3273  wsum += weight;
3274  } /* for i */
3275  mse /= wsum;
3276  } else {
3277  mse = cpl_vector_product(res, res) / npoints;
3278  }
3279  double rlimit = rsigma * sqrt(mse); /* limit using possibly weighted RMS */
3280  if (debug) {
3281  printf("Resulting 2D polynomial of slice %hu (%d points, RMS = %f dRMS = "
3282  "%f), %.1f-sigma limit now %f (target RMS = %f):\n", aSlice,
3283  npoints, sqrt(mse), rms < 0 ? 0. : rms - sqrt(mse), rsigma, rlimit,
3284  aParams->targetrms);
3285  if (debug > 1) {
3286  cpl_polynomial_dump(*aPoly, stdout);
3287  }
3288  fflush(stdout);
3289  }
3290 #if 0
3291  /* simple plotting of intermediate residuals *
3292  * (value over arbitrary x-position in vector) */
3293  cpl_plot_vector("set title \"res\"\n", "", "", res);
3294 #endif
3295  if (aParams->rflag) { /* dump debug information into table, if requested */
3296  int nnew = cpl_vector_get_size(res),
3297  nrow = aParams->residuals ? cpl_table_get_nrow(aParams->residuals) : 0;
3298  /* create the residuals table if wanted and not yet existing */
3299  if (!aParams->residuals) {
3300  aParams->residuals = muse_cpltable_new(muse_wavedebug_def, nnew);
3301  cpl_table_set_column_savetype(aParams->residuals, "slice", CPL_TYPE_UCHAR);
3302  } else { /* resize table as needed to fill in the new entries */
3303  cpl_table_set_size(aParams->residuals, nrow + nnew);
3304  }
3305  cpl_table_fill_column_window_int(aParams->residuals, "slice",
3306  nrow, nnew, aSlice);
3307  cpl_table_fill_column_window_int(aParams->residuals, "iteration",
3308  nrow, nnew, niter);
3309  cpl_table_fill_column_window_double(aParams->residuals, "rejlimit",
3310  nrow, nnew, rlimit);
3311  int n;
3312  for (n = 0; n < cpl_vector_get_size(res); n++) {
3313  cpl_table_set_int(aParams->residuals, "x", nrow + n,
3314  (int)cpl_matrix_get(aXYPos, 0, n));
3315  cpl_table_set_float(aParams->residuals, "y", nrow + n,
3316  cpl_matrix_get(aXYPos, 1, n));
3317  cpl_table_set_float(aParams->residuals, "lambda", nrow + n,
3318  cpl_vector_get(aLambdas, n));
3319  cpl_table_set_double(aParams->residuals, "residual", nrow + n,
3320  cpl_vector_get(res, n));
3321  } /* for n (all wavelengths) */
3322  if (debug) {
3323  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" entries in residuals table "
3324  "after iteration %d of slice %hu",
3325  cpl_table_get_nrow(aParams->residuals), niter, aSlice);
3326  }
3327  } /* if debug table */
3328  niter++;
3329 
3330  /* if the fit is already good enough, we don't *
3331  * need to go through point rejection any more */
3332  cpl_boolean isgoodenough = (rms < 0 ? CPL_FALSE : rms - sqrt(mse) < 0.001)
3333  || (sqrt(mse) <= aParams->targetrms);
3334  large_residuals = 0;
3335  int i;
3336  for (i = 0; !isgoodenough && i < cpl_vector_get_size(res); i++) {
3337  /* compare this residual value to the RMS value */
3338  if (fabs(cpl_vector_get(res, i)) < rlimit) {
3339  /* good fit at this point */
3340  continue;
3341  }
3342  /* bad residual, remove element, from residuals vector and the data vectors */
3343 #if 0
3344  cpl_msg_debug(__func__, "residual = %f (position %fx%f, lambda=%f+/-%f)",
3345  cpl_vector_get(res, i),
3346  cpl_matrix_get(aXYPos, 0, i), cpl_matrix_get(aXYPos, 1, i),
3347  cpl_vector_get(aLambdas, i),
3348  aDLambdas ? cpl_vector_get(aDLambdas, i) : 1.);
3349 #endif
3350  if (cpl_vector_get_size(res) == 1) {
3351  cpl_msg_debug(__func__, "trying to remove the last vector/matrix "
3352  "element when checking against fit sigma (slice %hu)",
3353  aSlice);
3354  break;
3355  }
3356  /* remove bad element from the fit structures... */
3358  cpl_matrix_erase_columns(aXYPos, i, 1);
3359  muse_cplvector_erase_element(aLambdas, i);
3360  if (aDLambdas) {
3361  muse_cplvector_erase_element(aDLambdas, i);
3362  }
3363 
3364  large_residuals++;
3365  i--; /* we stay at this position to see what moved here */
3366  } /* for i */
3367 
3368  rms = sqrt(mse);
3369  if (!large_residuals) {
3370  /* no large residuals (any more), so this is the final solution */
3371 
3372  /* save the mean squared error */
3373  if (aMSE) {
3374  *aMSE = mse;
3375  }
3376  } /* if !large_residuals */
3377 
3378  cpl_vector_delete(res);
3379  } /* while large_residuals */
3380 
3381  return CPL_ERROR_NONE;
3382 } /* muse_wave_poly_fit() */
3383 
3384 /*----------------------------------------------------------------------------*/
3394 /*----------------------------------------------------------------------------*/
3395 cpl_table *
3396 muse_wave_table_create(const unsigned short aNSlices, const unsigned short aXOrder,
3397  const unsigned short aYOrder)
3398 {
3399  cpl_table *table = cpl_table_new(aNSlices);
3400  cpl_ensure(table, CPL_ERROR_UNSPECIFIED, NULL);
3401 
3402  /* create column for slice number*/
3403  cpl_table_new_column(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, CPL_TYPE_INT);
3404  cpl_table_set_column_unit(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, "No");
3405  cpl_table_set_column_format(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, "%2d");
3406 
3407  /* create columns for all coefficients, so loop over fit orders */
3408  unsigned short i;
3409  for (i = 0; i <= aXOrder; i++) {
3410  unsigned short j;
3411  for (j = 0; j <= aYOrder; j++) {
3412  /* create column name, start coefficient names at 0 */
3413  char *colname = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
3414  cpl_table_new_column(table, colname, CPL_TYPE_DOUBLE);
3415  /* fit coeff are in wavelength space */
3416  cpl_table_set_column_unit(table, colname, "Angstrom");
3417  cpl_table_set_column_format(table, colname, "%12.5e");
3418  cpl_free(colname);
3419  }
3420  }
3421  /* create column for mean squared error of fitting parameters */
3422  cpl_table_new_column(table, MUSE_WAVECAL_TABLE_COL_MSE, CPL_TYPE_DOUBLE);
3423  cpl_table_set_column_format(table, MUSE_WAVECAL_TABLE_COL_MSE, "%f");
3424 
3425  return table;
3426 } /* muse_wave_table_create() */
3427 
3428 /*----------------------------------------------------------------------------*/
3442 /*----------------------------------------------------------------------------*/
3443 cpl_error_code
3444 muse_wave_table_add_poly(cpl_table *aTable, cpl_polynomial *aPoly,
3445  double aMSE, unsigned short aXOrder,
3446  unsigned short aYOrder, const unsigned short aRow)
3447 {
3448  cpl_ensure_code(aTable && aPoly, CPL_ERROR_NULL_INPUT);
3449  cpl_ensure_code(cpl_polynomial_get_dimension(aPoly) == 2,
3450  CPL_ERROR_ILLEGAL_INPUT);
3451 
3452  /* row numbers start at 0 but the numbers in the table should start with 1 */
3453  cpl_table_set_int(aTable, MUSE_WAVECAL_TABLE_COL_SLICE_NO, aRow, aRow + 1);
3454  cpl_table_set_double(aTable, MUSE_WAVECAL_TABLE_COL_MSE, aRow, aMSE);
3455 
3456  /* i and j loop over all orders of the polynomial */
3457  unsigned short i;
3458  for (i = 0; i <= aXOrder; i++) {
3459  unsigned short j;
3460  for (j = 0; j <= aYOrder; j++) {
3461  cpl_size pows[2] = { i, j }; /* trick to access the polynomial */
3462 
3463  char *colname = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
3464  cpl_error_code rc = cpl_table_set_double(aTable, colname, aRow,
3465  cpl_polynomial_get_coeff(aPoly,
3466  pows));
3467  if (rc != CPL_ERROR_NONE) {
3468  cpl_msg_warning(__func__, "Problem writing %f to field %s in "
3469  "wavelength table: %s",
3470  cpl_polynomial_get_coeff(aPoly, pows), colname,
3471  cpl_error_get_message());
3472  cpl_polynomial_dump(aPoly, stdout);
3473  cpl_table_dump(aTable, aRow, 1, stdout);
3474  fflush(stdout);
3475  }
3476  cpl_free(colname);
3477  } /* for j (polynomial orders in y) */
3478  } /* for i (polynomial orders in x) */
3479 
3480  return CPL_ERROR_NONE;
3481 } /* muse_wave_table_add_poly() */
3482 
3483 /*----------------------------------------------------------------------------*/
3498 /*----------------------------------------------------------------------------*/
3499 cpl_error_code
3500 muse_wave_table_get_orders(const cpl_table *aWave, unsigned short *aXOrder,
3501  unsigned short *aYOrder)
3502 {
3503  cpl_ensure_code(aWave && aXOrder && aYOrder, CPL_ERROR_NULL_INPUT);
3504  cpl_array *cols = cpl_table_get_column_names(aWave);
3505 
3506  /* second last entry will contain highest order (last is MSE) */
3507  const char *highcol = cpl_array_get_string(cols, cpl_array_get_size(cols) - 2);
3508  char *colname = cpl_strdup(highcol); /* duplicate it so that we can manipulate it */
3509  cpl_array_delete(cols);
3510 
3511  /* we can directly get the value of the last digit */
3512  *aYOrder = atoi(colname+4);
3513  /* now null out the last digit so that we can get the first one only */
3514  colname[4] = '\0';
3515  *aXOrder = atoi(colname+3);
3516  cpl_free(colname);
3517 
3518  return CPL_ERROR_NONE;
3519 } /* muse_wave_table_get_orders() */
3520 
3521 /*----------------------------------------------------------------------------*/
3542 /*----------------------------------------------------------------------------*/
3543 cpl_polynomial *
3544 muse_wave_table_get_poly_for_slice(const cpl_table *aTable,
3545  const unsigned short aSlice)
3546 {
3547  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, NULL);
3548  cpl_ensure(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
3549  CPL_ERROR_ILLEGAL_INPUT, NULL);
3550  /* search for row containing the requested slice, first to access it *
3551  * in a possibly incomplete table, and second to check its presence! */
3552  int irow, nrow = cpl_table_get_nrow(aTable);
3553  for (irow = 0; irow < nrow; irow++) {
3554  int err;
3555  unsigned short slice = cpl_table_get_int(aTable,
3556  MUSE_WAVECAL_TABLE_COL_SLICE_NO,
3557  irow, &err);
3558  if (slice == aSlice && !err) {
3559  break;
3560  }
3561  } /* for irow */
3562  cpl_ensure(irow < nrow, CPL_ERROR_DATA_NOT_FOUND, NULL);
3563 
3564  cpl_polynomial *pwave = cpl_polynomial_new(2);
3565  char colname[15]; /* "wlc" plus max 2x5 chars for the numbers */
3566  unsigned short wavexorder, waveyorder;
3567  muse_wave_table_get_orders(aTable, &wavexorder, &waveyorder);
3568  unsigned short l;
3569  for (l = 0; l <= wavexorder; l++) {
3570  unsigned short k;
3571  for (k = 0; k <= waveyorder; k++) {
3572  cpl_size pows[2] = { l, k }; /* trick to access the polynomial */
3573  sprintf(colname, MUSE_WAVECAL_TABLE_COL_COEFF, l, k);
3574  int err;
3575  cpl_polynomial_set_coeff(pwave, pows,
3576  cpl_table_get_double(aTable, colname, irow,
3577  &err));
3578  if (err != 0) { /* broken table entry */
3579  cpl_polynomial_delete(pwave);
3580  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT, "Wavelength "
3581  "calibration table broken in slice %hu (row index"
3582  " %d) column %s", aSlice, irow, colname);
3583  return NULL;
3584  } /* if */
3585  } /* for k */
3586  } /* for l */
3587  return pwave;
3588 } /* muse_wave_table_get_poly_for_slice() */
3589 
3590 /*----------------------------------------------------------------------------*/
3613 /*----------------------------------------------------------------------------*/
3614 cpl_image *
3615 muse_wave_map(muse_image *aImage, const cpl_table *aWave,
3616  const cpl_table *aTrace)
3617 {
3618  cpl_ensure(aImage && aWave && aTrace, CPL_ERROR_NULL_INPUT, NULL);
3619  int nx = cpl_image_get_size_x(aImage->data),
3620  ny = cpl_image_get_size_y(aImage->data);
3621  cpl_image *wavemap = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
3622  cpl_ensure(wavemap, cpl_error_get_code(), NULL);
3623  unsigned char ifu = muse_utils_get_ifu(aImage->header);
3624 
3625  /* get output image buffer as pointer */
3626  float *wdata = cpl_image_get_data_float(wavemap);
3627 
3628  unsigned short wavexorder, waveyorder;
3629  muse_wave_table_get_orders(aWave, &wavexorder, &waveyorder);
3630  cpl_msg_debug(__func__, "Order for trace solution is %d, for wavelength "
3631  "solution %hu/%hu, IFU %hhu", muse_trace_table_get_order(aTrace),
3632  wavexorder, waveyorder, ifu);
3633 
3634  /* loop through all slices */
3635  unsigned short islice;
3636  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
3637 #if 0
3638  cpl_msg_debug(__func__, "Starting to process slice %hu of IFU %hhu",
3639  islice + 1, ifu);
3640 #endif
3641 
3642  /* fill the wavelength calibration polynomial for this slice */
3643  cpl_polynomial *pwave = muse_wave_table_get_poly_for_slice(aWave, islice + 1);
3644  /* vector for the position within the slice (for evaluation *
3645  * of the wavelength solution in two dimensions */
3646  cpl_vector *pos = cpl_vector_new(2);
3647 
3648  /* get the tracing polynomials for this slice */
3649  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
3650  islice + 1);
3651  if (!ptrace) {
3652  cpl_msg_warning(__func__, "slice %2hu of IFU %hhu: tracing polynomials "
3653  "missing!", islice + 1, ifu);
3654  continue;
3655  }
3656 #if 0
3657  printf("polynomials for slice %hu:\n", islice + 1);
3658  cpl_polynomial_dump(ptrace[MUSE_TRACE_LEFT], stdout);
3659  cpl_polynomial_dump(ptrace[MUSE_TRACE_RIGHT], stdout);
3660  cpl_polynomial_dump(pwave, stdout);
3661  fflush(stdout);
3662 #endif
3663 
3664  /* within each slice, loop from bottom to top */
3665  int j;
3666  for (j = 1; j <= ny; j++) {
3667  /* determine the slice edges for this vertical *
3668  * position so that we can loop over those pixels */
3669  int ileft = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
3670  j, NULL)),
3671  iright = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
3672  j, NULL));
3673  /* try to detect faulty polynomials */
3674  if (ileft < 1 || iright > nx || ileft > iright) {
3675  cpl_msg_warning(__func__, "slice %2hu of IFU %hhu: faulty polynomial "
3676  "detected at y=%d", islice + 1, ifu, j);
3677  break; /* skip the rest of this slice */
3678  }
3679  cpl_vector_set(pos, 1, j); /* vertical pos. for wavelength evaluation */
3680 
3681  /* now loop over all pixels of this slice horizontally */
3682  int i;
3683  for (i = ileft; i <= iright; i++) {
3684  cpl_vector_set(pos, 0, i); /* horiz. pos. for wavelength evaluation */
3685 
3686  /* compute the wavelength of this pixel */
3687  wdata[(i-1) + (j-1)*nx] = cpl_polynomial_eval(pwave, pos);
3688  } /* for i (horizontal pixels) */
3689  } /* for j (vertical direction) */
3690 
3691  /* we are now done with this slice, clean up */
3692  muse_trace_polys_delete(ptrace);
3693  cpl_polynomial_delete(pwave);
3694  cpl_vector_delete(pos);
3695  } /* for islice */
3696 
3697  return wavemap;
3698 } /* muse_wave_map() */
3699 
3700 /*----------------------------------------------------------------------------*/
3724 /*----------------------------------------------------------------------------*/
3725 cpl_error_code
3726 muse_wave_plot_residuals(cpl_table *aTable, const unsigned short aSlice,
3727  unsigned int aIter, cpl_boolean aPlotLambda,
3728  cpl_vector *aCuts)
3729 {
3730 #if HAVE_POPEN && HAVE_PCLOSE
3731  cpl_ensure_code(aTable, CPL_ERROR_NULL_INPUT);
3732  cpl_error_code rc = muse_cpltable_check(aTable, muse_wavedebug_def);
3733  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
3734 
3735  FILE *gp = popen("gnuplot", "w");
3736  if (!gp) {
3737  return CPL_ERROR_ASSIGNING_STREAM;
3738  }
3739 
3740  /* select relevant rows in the table */
3741  cpl_table_unselect_all(aTable); /* start clean */
3742 
3743  int n, nrow = cpl_table_get_nrow(aTable),
3744  error = 0;
3745  if (aSlice == 0) {
3746  printf("Selecting data of all slices.\n");
3747 
3748  const int *slice = cpl_table_get_data_int_const(aTable, "slice"),
3749  *iter = cpl_table_get_data_int_const(aTable, "iteration");
3750  if (!aIter) { /* last iteration */
3751  fprintf(stderr, "Selecting data of last iteration of all slices\n");
3752  /* the last row of each slice number is the one with the last iteration */
3753  int sliceno = slice[nrow - 1],
3754  iterlast = iter[nrow - 1];
3755  for (n = nrow - 2; n >= 0; n--) {
3756  if (slice[n] == sliceno && iter[n] != iterlast) {
3757  cpl_table_select_row(aTable, n);
3758  }
3759  if (slice[n] != sliceno) { /* previous slice, reset comparison */
3760  sliceno = slice[n];
3761  iterlast = iter[n];
3762  }
3763  } /* for n (table rows) */
3764  cpl_table_erase_selected(aTable);
3765  /* plot title with some details */
3766  fprintf(gp, "set title \"slices %d..%d, iterations %d..%d: 2D polynomial "
3767  "fit residuals (limits %f..%f)\n",
3768  (int)cpl_table_get_column_min(aTable, "slice"),
3769  (int)cpl_table_get_column_max(aTable, "slice"),
3770  (int)cpl_table_get_column_min(aTable, "iteration"),
3771  (int)cpl_table_get_column_max(aTable, "iteration"),
3772  cpl_table_get_column_min(aTable, "rejlimit"),
3773  cpl_table_get_column_max(aTable, "rejlimit"));
3774  } else {
3775  printf("Selecting data of iteration %d.\n", aIter);
3776  for (n = 0; n < nrow; n++) {
3777  if (iter[n] != (int)aIter) {
3778  cpl_table_select_row(aTable, n);
3779  }
3780  } /* for n (table rows) */
3781  cpl_table_erase_selected(aTable);
3782  /* plot title with some details */
3783  fprintf(gp, "set title \"slices %d..%d, iteration %d: 2D polynomial fit "
3784  "residuals (limits %f..%f)\n",
3785  (int)cpl_table_get_column_min(aTable, "slice"),
3786  (int)cpl_table_get_column_max(aTable, "slice"), aIter,
3787  cpl_table_get_column_min(aTable, "rejlimit"),
3788  cpl_table_get_column_max(aTable, "rejlimit"));
3789  }
3790  } else {
3791  printf("Selecting data of slice %hu.\n", aSlice);
3792  const int *slice = cpl_table_get_data_int_const(aTable, "slice");
3793  for (n = 0; n < nrow; n++) {
3794  if (slice[n] != aSlice) {
3795  cpl_table_select_row(aTable, n);
3796  }
3797  } /* for n (table rows) */
3798  cpl_table_erase_selected(aTable);
3799  nrow = cpl_table_get_nrow(aTable);
3800  cpl_table_unselect_all(aTable);
3801 
3802  const int *iter = cpl_table_get_data_int_const(aTable, "iteration");
3803  if (!aIter) { /* last iteration */
3804  /* table is sorted, so last iteration is that of the last row */
3805  aIter = iter[nrow - 1];
3806  }
3807  printf("Selecting data of iteration %d.\n", aIter);
3808  for (n = 0; n < nrow; n++) {
3809  if (iter[n] != (int)aIter) {
3810  cpl_table_select_row(aTable, n);
3811  }
3812  } /* for n (table rows) */
3813  cpl_table_erase_selected(aTable);
3814 
3815  /* plot title with some details */
3816  fprintf(gp, "set title \"slice %hu, iteration %d: 2D polynomial fit residuals"
3817  " (limit=%f)\n", aSlice, aIter,
3818  cpl_table_get_double(aTable, "rejlimit", 0, &error));
3819  } /* single slice */
3820 
3821  /* data access */
3822  nrow = cpl_table_get_nrow(aTable);
3823  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
3824  printf("Plotting %d points.\n", nrow);
3825  const int *x = cpl_table_get_data_int_const(aTable, "x");
3826  const float *y = cpl_table_get_data_float_const(aTable, "y"),
3827  *lambda = cpl_table_get_data_float_const(aTable, "lambda");
3828  const double *r = cpl_table_get_data_double_const(aTable, "residual");
3829  /* *rlimit = cpl_table_get_data_double_const(aTable, "rejlimit"); */
3830 
3831  /* determine plotting limits */
3832  int xmin = cpl_table_get_column_min(aTable, "x") - 2,
3833  xmax = cpl_table_get_column_max(aTable, "x") + 2;
3834  float ymin = cpl_table_get_column_min(aTable, "y") - 2,
3835  ymax = cpl_table_get_column_max(aTable, "y") + 2,
3836  lmin = cpl_table_get_column_min(aTable, "lambda") - 2,
3837  lmax = cpl_table_get_column_max(aTable, "lambda") + 2;
3838  double rmin = cpl_table_get_column_min(aTable, "residual"),
3839  rmax = cpl_table_get_column_max(aTable, "residual");
3840  if (aCuts && cpl_vector_get_size(aCuts) == 2) {
3841  rmin = cpl_vector_get(aCuts, 0);
3842  rmax = cpl_vector_get(aCuts, 1);
3843  }
3844 
3845  /* set nice palette */
3846  fprintf(gp, "set palette defined ( 0 \"dark-violet\","
3847  "1 \"dark-blue\", 4 \"green\", 6 \"yellow\", 8 \"orange\","
3848  "9 \"red\", 10 \"dark-red\")\n");
3849  fprintf(gp, "unset key\n");
3850  printf("Setting plotting limits: [%d:%d][%.2f:%.2f][%.4f:%.4f]\n",
3851  xmin, xmax, aPlotLambda ? lmin : ymin, aPlotLambda ? lmax : ymax,
3852  rmin, rmax);
3853  fprintf(gp, "set xrange [%d:%d]\n", xmin, xmax);
3854  if (aPlotLambda) {
3855  fprintf(gp, "set yrange [%f:%f]\n", lmin, lmax);
3856  } else {
3857  fprintf(gp, "set yrange [%f:%f]\n", ymin, ymax);
3858  }
3859  fprintf(gp, "set cbrange [%f:%f]\n", rmin, rmax);
3860  fprintf(gp, "set view map\n");
3861  fprintf(gp, "splot \"-\" w p pal\n");
3862  for (n = 0; n < nrow; n++) {
3863  if (aPlotLambda) {
3864  fprintf(gp, "%d %f %e\n", x[n], lambda[n], r[n]);
3865  } else {
3866  fprintf(gp, "%d %f %e\n", x[n], y[n], r[n]);
3867  }
3868  }
3869  fprintf(gp, "EOF\n");
3870  fflush(gp);
3871  /* request keypress, so that working with the mouse keeps *
3872  * working and gnuplot has enough time to actually draw all *
3873  * the stuff before the files get removed */
3874  printf("Press ENTER to end program and close plot\n");
3875  getchar();
3876  pclose(gp);
3877  return CPL_ERROR_NONE;
3878 #else /* no HAVE_POPEN && HAVE_PCLOSE */
3879  return CPL_ERROR_UNSUPPORTED_MODE;
3880 #endif /* HAVE_POPEN && HAVE_PCLOSE */
3881 } /* muse_wave_plot_residuals() */
3882 
3883 /*----------------------------------------------------------------------------*/
3909 /*----------------------------------------------------------------------------*/
3910 cpl_error_code
3911 muse_wave_plot_column(cpl_table *aCTable, cpl_table *aRTable,
3912  const unsigned short aSlice, unsigned int aColumn,
3913  unsigned int aIter, cpl_boolean aPlotRes)
3914 {
3915 #if HAVE_POPEN && HAVE_PCLOSE
3916  cpl_ensure_code(aCTable && aRTable, CPL_ERROR_NULL_INPUT);
3917  cpl_error_code rc = muse_cpltable_check(aRTable, muse_wavedebug_def);
3918  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
3919  /* WAVECAL_TABLE doesn't has a _def structure to check, so test orders */
3920  unsigned short xorder, yorder;
3921  muse_wave_table_get_orders(aCTable, &xorder, &yorder);
3922  cpl_ensure_code(xorder > 0 && yorder > 0, CPL_ERROR_ILLEGAL_INPUT);
3923  cpl_ensure_code(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
3924  CPL_ERROR_ACCESS_OUT_OF_RANGE);
3925 
3926  FILE *gp = popen("gnuplot", "w");
3927  if (!gp) {
3928  return CPL_ERROR_ASSIGNING_STREAM;
3929  }
3930 
3931  /* select relevant rows in the table */
3932  cpl_table_unselect_all(aRTable); /* start clean */
3933 
3934  printf("Selecting data of slice %hu.\n", aSlice);
3935  const int *slice = cpl_table_get_data_int_const(aRTable, "slice");
3936  int n, nrow = cpl_table_get_nrow(aRTable);
3937  for (n = 0; n < nrow; n++) {
3938  if (slice[n] != aSlice) {
3939  cpl_table_select_row(aRTable, n);
3940  }
3941  } /* for n (table rows) */
3942  cpl_table_erase_selected(aRTable);
3943  nrow = cpl_table_get_nrow(aRTable);
3944  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
3945  cpl_table_unselect_all(aRTable);
3946 
3947  const int *iter = cpl_table_get_data_int_const(aRTable, "iteration");
3948  if (!aIter) { /* last iteration */
3949  /* table is sorted, so last iteration is that of the last row */
3950  aIter = iter[nrow - 1];
3951  }
3952  printf("Selecting data of iteration %d.\n", aIter);
3953  for (n = 0; n < nrow; n++) {
3954  if (iter[n] != (int)aIter) {
3955  cpl_table_select_row(aRTable, n);
3956  }
3957  } /* for n (table rows) */
3958  cpl_table_erase_selected(aRTable);
3959  nrow = cpl_table_get_nrow(aRTable);
3960  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
3961  cpl_table_unselect_all(aRTable);
3962 
3963  unsigned int col1 = cpl_table_get_column_min(aRTable, "x"),
3964  col2 = cpl_table_get_column_max(aRTable, "x");
3965  if (aColumn > 0) {
3966  col1 = col2 = aColumn;
3967  } else if (aColumn > (unsigned)kMuseOutputXRight) {
3968  /* approximate central column of the slice */
3969  col1 = col2 = (col1 + col2) / 2;
3970  }
3971  printf("Plotting data of columns %u..%u.\n", col1, col2);
3972 
3973  /* determine plotting limits */
3974  float ymin = cpl_table_get_column_min(aRTable, "y") - 10,
3975  ymax = cpl_table_get_column_max(aRTable, "y") + 10,
3976  lmin = cpl_table_get_column_min(aRTable, "lambda") - 10,
3977  lmax = cpl_table_get_column_max(aRTable, "lambda") + 10;
3978  double rmin = cpl_table_get_column_min(aRTable, "residual") * 1.03,
3979  rmax = cpl_table_get_column_max(aRTable, "residual") * 1.03;
3980  /* plot title with some details */
3981  fprintf(gp, "set title \"slice %hu, iteration %d, column %u..%u: polynomial "
3982  "and ", aSlice, aIter, col1, col2);
3983  printf("Setting plotting limits: ");
3984  if (aPlotRes) {
3985  fprintf(gp, "residuals (limit=%f)\"\n",
3986  cpl_table_get_double(aRTable, "rejlimit", 0, NULL));
3987  printf("[%.2f:%.2f][%.4f:%.4f]\n", lmin, lmax, rmin, rmax);
3988  fprintf(gp, "set xrange [%f:%f]\n", lmin, lmax);
3989  fprintf(gp, "set yrange [%f:%f]\n", rmin, rmax);
3990  fprintf(gp, "set xlabel \"Wavelength [Angstrom]\"\n");
3991  fprintf(gp, "set ylabel \"Residuals [Angstrom]\"\n");
3992  } else {
3993  fprintf(gp, "arc line positions\"\n");
3994  printf("[%.2f:%.2f][%.2f:%.2f]\n", ymin, ymax, lmin, lmax);
3995  fprintf(gp, "set xrange [%g:%g]\n", ymin, ymax);
3996  fprintf(gp, "set yrange [%f:%f]\n", lmin, lmax);
3997  fprintf(gp, "set xlabel \"y-position [pix]\"\n");
3998  fprintf(gp, "set ylabel \"Wavelength [Angstrom]\"\n");
3999  }
4000  fprintf(gp, "set key outside below\n");
4001  fprintf(gp, "set samples 1000\n"); /* more samples so that one can zoom in */
4002 
4003  /* create gnuplot polynomial from table coefficients */
4004  fprintf(gp, "p(x,y) = 0 ");
4005  if (!aPlotRes) { /* create full polynomial */
4006  unsigned short i;
4007  for (i = 0; i <= xorder; i++) {
4008  unsigned short j;
4009  for (j = 0; j <= yorder; j++) {
4010  char *coeff = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
4011  double cvalue = cpl_table_get(aCTable, coeff, aSlice - 1, NULL);
4012  cpl_free(coeff);
4013  fprintf(gp, " + (%g) * x**(%hu) * y**(%hu)", cvalue, i, j);
4014  } /* for j (y orders) */
4015  } /* for i (x orders) */
4016  }
4017  fprintf(gp, "\n");
4018 
4019  const int *x = cpl_table_get_data_int_const(aRTable, "x");
4020  const float *y = cpl_table_get_data_float_const(aRTable, "y"),
4021  *lambda = cpl_table_get_data_float_const(aRTable, "lambda");
4022  const double *r = cpl_table_get_data_double_const(aRTable, "residual");
4023 
4024  /* distribute columns over 256 color values */
4025  double dcol = (col2 - col1) / 255.;
4026  if (dcol == 0.) { /* take care not to produce NANs below when dividing */
4027  dcol = 1.;
4028  }
4029  /* construct the plot command as a single long line */
4030  fprintf(gp, "plot ");
4031  if (aPlotRes) {
4032  fprintf(gp, "0 t \"\", "); /* plot line to represent solution */
4033  }
4034  unsigned int ncol, npoints = 0;
4035  for (ncol = col1; ncol <= col2; ncol++) {
4036  /* plot polynomial and values for the specified CCD column */
4037  int red = (ncol - col1) / dcol, /* red */
4038  grn = (col2 - ncol) / dcol, /* green */
4039  blu = 0; /* blue */
4040  if (aPlotRes) {
4041  fprintf(gp, "\"-\" u 2:3 t \"col %u\" w p ps 0.8 lt rgb \"#%02x%02x%02x\"",
4042  ncol, red, grn, blu);
4043  } else {
4044  /* create data values from polynomial and residuals for given CCD column */
4045  fprintf(gp, "p(%u, x) t \"\" w l lw 0.7 lt rgb \"#%02x%02x%02x\", "
4046  "\"-\" u 1:(p(%u,$1)+$3) t \"col %u\" w p ps 0.8 lt rgb \"#%02x%02x%02x\"",
4047  ncol, red, grn, blu, ncol, ncol, red, grn, blu);
4048  }
4049  if (ncol == col2) {
4050  fprintf(gp, "\n"); /* end the plot command */
4051  } else {
4052  fprintf(gp, ", "); /* plot command of next column to come */
4053  }
4054  } /* for ncol (relevant columns) */
4055  for (ncol = col1; ncol <= col2; ncol++) {
4056  for (n = 0; n < nrow; n++) {
4057  if (x[n] == (int)ncol) {
4058  fprintf(gp, "%f %f %g\n", y[n], lambda[n], r[n]);
4059  npoints++;
4060  }
4061  }
4062  fprintf(gp, "EOF\n");
4063  } /* for ncol (relevant columns) */
4064  printf("Plotted %u points.\n", npoints);
4065  fflush(gp);
4066  /* request keypress, so that working with the mouse keeps *
4067  * working and gnuplot has enough time to actually draw all *
4068  * the stuff before the files get removed */
4069  printf("Press ENTER to end program and close plot\n");
4070  getchar();
4071  pclose(gp);
4072  return CPL_ERROR_NONE;
4073 #else /* no HAVE_POPEN && HAVE_PCLOSE */
4074  return CPL_ERROR_UNSUPPORTED_MODE;
4075 #endif /* HAVE_POPEN && HAVE_PCLOSE */
4076 } /* muse_wave_plot_column() */
4077 
cpl_table * muse_wave_lines_search(muse_image *aColumnImage, double aSigma, const unsigned short aSlice, const unsigned char aIFU)
Search and store emission lines in a column of an arc frame.
cpl_table * muse_wave_calib_lampwise(muse_imagelist *aImages, cpl_table *aTrace, cpl_table *aLinelist, muse_wave_params *aParams)
Find wavelength calibration solution using a list of arc images with different lamps.
cpl_table * muse_wave_calib(muse_image *aImage, cpl_table *aTrace, cpl_table *aLinelist, muse_wave_params *aParams)
Find wavelength calibration solution on an arc frame.
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
cpl_error_code muse_wave_table_get_orders(const cpl_table *aWave, unsigned short *aXOrder, unsigned short *aYOrder)
Determine the x- and y-order of the polynomial stored in a wavelength calibration table...
Structure definition for a collection of muse_images.
int muse_trace_table_get_order(const cpl_table *aTable)
determine order of tracing polynomial from table
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:85
muse_wave_weighting_type fitweighting
cpl_polynomial * muse_wave_table_get_poly_for_slice(const cpl_table *aTable, const unsigned short aSlice)
Construct polynomial from the wavelength calibration table entry for the given slice.
cpl_vector * muse_wave_lines_get_for_lamp(cpl_table *aTable, const char *aLamp, int aGoodnessLimit, double aFluxLimit)
Load wavelengths for a given lamp from a linelist table into a vector.
cpl_error_code muse_wave_lines_identify(cpl_table *aLines, cpl_vector *aLambdas, const muse_wave_params *aParams)
Identify the wavelength of arc detected lines using pattern matching.
void muse_wave_params_delete(muse_wave_params *aParams)
Deallocate memory associated to a wavelength parameters structure.
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:99
cpl_image * data
the data extension
Definition: muse_image.h:46
cpl_boolean rflag
cpl_error_code muse_wave_poly_fit(cpl_matrix *aXYPos, cpl_vector *aLambdas, cpl_vector *aDLambdas, cpl_polynomial **aPoly, double *aMSE, muse_wave_params *aParams, const unsigned short aSlice)
Compute the wavelength solution from the sample positions and the respective wavelengths.
cpl_table * muse_wave_table_create(const unsigned short aNSlices, const unsigned short aXOrder, const unsigned short aYOrder)
Create the table to save te wave wavelength calibration coefficients.
muse_image * muse_combine_median_create(muse_imagelist *aImages)
Median combine a list of input images.
Definition: muse_combine.c:316
cpl_image * stat
the statistics extension
Definition: muse_image.h:64
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
cpl_vector * muse_wave_lines_get(cpl_table *aTable, int aGoodnessLimit, double aFluxLimit)
Load usable wavelengths from a linelist table into a vector.
cpl_table * residuals
static void muse_wave_lines_add_flux_for_lsf(cpl_table *, cpl_table *)
Add the measured flux of the detected lines to a linelist.
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
int muse_pfits_get_lampnum(const cpl_propertylist *aHeaders)
query the number of lamps installed
Definition: muse_pfits.c:1225
cpl_table * muse_wave_line_handle_multiplet(muse_image *aImage, cpl_table *aLinelist, unsigned int aIdx, cpl_polynomial *aPoly, cpl_polynomial **aTrace, const muse_wave_params *aParams, const unsigned short aSlice, int aDebug)
Handle fitting of all multiplets across the columns a given slice.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
cpl_error_code muse_cpltable_check(const cpl_table *aTable, const muse_cpltable_def *aDef)
Check whether the table contains the fields of the definition.
unsigned int muse_imagelist_get_size(muse_imagelist *aList)
Return the number of stored images.
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
cpl_image * dq
the data quality extension
Definition: muse_image.h:56
cpl_table * muse_cpltable_new(const muse_cpltable_def *aDef, cpl_size aLength)
Create an empty table according to the specified definition.
const char * muse_wave_lines_get_lampname(cpl_table *aTable, const int aIdx)
Associate the ion listed in a linelist table row to a lamp name.
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
cpl_error_code muse_cplvector_erase_element(cpl_vector *aVector, int aElement)
delete the given element from the input vector
Structure containing wavelength calibration parameters.
cpl_polynomial * muse_utils_iterate_fit_polynomial(cpl_matrix *aPos, cpl_vector *aVal, cpl_vector *aErr, cpl_table *aExtra, const unsigned int aOrder, const double aRSigma, double *aMSE, double *aChiSq)
Iterate a polynomial fit.
Definition: muse_utils.c:2178
unsigned short xorder
cpl_error_code muse_wave_table_add_poly(cpl_table *aTable, cpl_polynomial *aPoly, double aMSE, unsigned short aXOrder, unsigned short aYOrder, const unsigned short aRow)
Save the given polynomials to the wavelength calibration table.
cpl_image * muse_wave_map(muse_image *aImage, const cpl_table *aWave, const cpl_table *aTrace)
Write out a wavelength map for visual checks.
const muse_cpltable_def muse_line_catalog_def[]
const muse_cpltable_def muse_wavedebug_def[]
MUSE wavelength calibration residuals table definition.
cpl_error_code muse_utils_fit_multigauss_1d(const cpl_vector *aX, const cpl_bivector *aY, cpl_vector *aCenter, double *aSigma, cpl_vector *aFlux, cpl_vector *aPoly, double *aMSE, double *aRedChisq, cpl_matrix **aCovariance)
Carry out a multi-Gaussian fit of data in a vector.
Definition: muse_utils.c:1502
char * muse_utils_header_get_lamp_names(cpl_propertylist *aHeader, char aSep)
Concatenate names of all active calibration lamps.
Definition: muse_utils.c:933
cpl_size muse_cpltable_find_sorted(const cpl_table *aTable, const char *aColumn, double aValue)
Find a row in a table.
cpl_error_code muse_wave_plot_residuals(cpl_table *aTable, const unsigned short aSlice, unsigned int aIter, cpl_boolean aPlotLambda, cpl_vector *aCuts)
Fancy plotting of wavelength calibration residuals (color coded over x/y-position) using gnuplot...
cpl_array * muse_utils_header_get_lamp_numbers(cpl_propertylist *aHeader)
List numbers of all active calibration lamps.
Definition: muse_utils.c:1000
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:399
cpl_error_code muse_wave_line_fit_iterate(cpl_table *aFitTable, double aLambda, const muse_wave_params *aParams)
Use a low-order polynomial to find and discard bad values for line centroid fits of single arc line a...
muse_imagelist * muse_imagelist_new(void)
Create a new (empty) MUSE image list.
cpl_error_code muse_wave_line_fit_single(muse_image *aImage, int aX, double aY, int aHalfWidth, double aSigma, cpl_table *aFitTable, int aRowsNeeded)
Fit a Gaussian to a single emission line in an arc frame and do simple error handling.
unsigned short yorder
cpl_size muse_cplvector_count_unique(const cpl_vector *aVector)
Count the number of unique entries in a given vector.
Definition of a cpl table structure.
cpl_boolean muse_wave_lines_check(cpl_table *aTable, cpl_propertylist *aHeader)
Check that a LINE_CATALOG has the expected format.
cpl_error_code muse_wave_plot_column(cpl_table *aCTable, cpl_table *aRTable, const unsigned short aSlice, unsigned int aColumn, unsigned int aIter, cpl_boolean aPlotRes)
Plot wavelength calibration polynomial and data or residuals using gnuplot.
muse_image * muse_image_new(void)
Allocate memory for a new muse_image object.
Definition: muse_image.c:66
cpl_error_code muse_wave_line_fit_multiple(muse_image *aImage, int aX, cpl_bivector *aPeaks, cpl_vector *aLambdas, int aHalfWidth, double aSigma, cpl_table *aFitTable, int aRowsNeeded)
Fit a multi-Gaussian to a multiplet of arc emission lines and do simple error handling.
muse_wave_params * muse_wave_params_new(void)
Allocate a wavelength parameters structure and fill it with defaults.
cpl_error_code muse_imagelist_set(muse_imagelist *aList, muse_image *aImage, unsigned int aIdx)
Set the muse_image of given list index.
const muse_cpltable_def muse_wavelines_def[]
MUSE wavelength calibration arc line fit properties table definition.
cpl_table * muse_wave_line_handle_singlet(muse_image *aImage, cpl_table *aLinelist, unsigned int aIdx, cpl_polynomial *aPoly, cpl_polynomial **aTrace, const muse_wave_params *aParams, const unsigned short aSlice, int aDebug)
Handle fitting of all single lines across the columns a given slice.
muse_ins_mode muse_pfits_get_mode(const cpl_propertylist *aHeaders)
find out the observation mode
Definition: muse_pfits.c:1097