MUSE Pipeline Reference Manual  1.0.2
muse_sky_rowbyrow.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #include <math.h>
30 #include <string.h>
31 
32 #include <cpl.h>
33 
34 #include "muse_sky.h"
35 #include "muse_instrument.h"
36 #include "muse_quality.h"
37 #include "muse_tracing.h"
38 #include "muse_utils.h"
39 
43 /*----------------------------------------------------------------------------*
44  * Debugging Macros *
45  * Set these to 1 or higher for (lots of) debugging output *
46  *----------------------------------------------------------------------------*/
47 #define SKY_ROWBYROW_DEBUG 0 /* debug output in muse_sky_subtract_rowbyrow() */
48 
49 /*----------------------------------------------------------------------------*/
72 /*----------------------------------------------------------------------------*/
73 cpl_error_code
74 muse_sky_subtract_rowbyrow_mask(muse_image *aImage, cpl_table *aTrace)
75 {
76  cpl_ensure_code(aImage, CPL_ERROR_NULL_INPUT);
77  unsigned short nslices = 0;
78  if (aTrace) {
79  nslices = cpl_table_get_nrow(aTrace);
80  } else if (!aTrace) { /* check for the slice center headers */
81  unsigned short slice = 0;
82  cpl_boolean exists = CPL_FALSE;
83  do {
84  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", ++slice);
85  exists = cpl_propertylist_has(aImage->header, keyword);
86 #if 0
87  cpl_msg_debug(__func__, "%s %s", keyword, exists ? "exists" : "doesn't exist");
88 #endif
89  cpl_free(keyword);
90  } while (exists);
91  nslices = slice - 1;
92  }
93  cpl_msg_debug(__func__, "Found %hu slices", nslices);
94  cpl_ensure_code(nslices, CPL_ERROR_ILLEGAL_INPUT);
95  const unsigned int kOrdObj = 1; /* a linear fit to reject objects */
96  const float kRSigObj = 3.; /* use 3 sigma rejection to find objects */
97 
98  /* vertical median as object mask */
99  cpl_image *objmask = cpl_image_collapse_median_create(aImage->data, 0, 0, 0);
100 #if SKY_ROWBYROW_DEBUG
101  cpl_msg_debug(__func__, "saving \"sky_removed_objmask.fits\"");
102  cpl_image_save(objmask, "sky_removed_objmask.fits", CPL_TYPE_UNSPECIFIED,
103  NULL, CPL_IO_CREATE);
104 #endif
105  /* get image properties and data */
106  int nx = cpl_image_get_size_x(aImage->data),
107  ny = cpl_image_get_size_y(aImage->data);
108  int *dq = cpl_image_get_data_int(aImage->dq);
109 
110  /* loop through all slices to fit and mask the objects */
111  unsigned short islice;
112  for (islice = 0; islice < nslices; islice++) {
113  cpl_msg_debug(__func__, "Processing slice %hu", islice + 1);
114 
115  /* find slice edge, either through trace info or *
116  * through centers plus "missing data" flags */
117  cpl_polynomial **ptrace = NULL;
118  int i1 = 0, i2 = nx, istart = 0;
119  if (aTrace) {
120  /* get the tracing polynomials for this slice */
121  ptrace = muse_trace_table_get_polys_for_slice(aTrace, islice + 1);
122  if (!ptrace) {
123  cpl_msg_warning(__func__, "slice %2d: tracing polynomials missing!",
124  islice + 1);
125  continue;
126  }
127  /* use trace of vertical center, be generous with the edges */
128  i1 = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], ny/2, NULL));
129  i2 = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], ny/2, NULL));
130  if (i1 < 1 || i2 > nx || i1 > i2) {
131  cpl_msg_warning(__func__, "slice %2d: faulty polynomial detected at "
132  "y=%d (borders: %d ... %d)", islice + 1, ny/2, i1, i2);
133  muse_trace_polys_delete(ptrace);
134  continue; /* next slice */
135  }
136  } else { /* find slice edge without tracing info */
137  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", islice + 1);
138  istart = cpl_propertylist_get_float(aImage->header, keyword);
139  cpl_free(keyword);
140  /* move outwards */
141  int ix = istart;
142  while (!(dq[(--ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
143  i1 = ix;
144  }
145  ix = istart;
146  while (!(dq[(++ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
147  i2 = ix;
148  }
149  } /* else, no aTrace */
150 
151  /* fit and apply object mask */
152  cpl_matrix *p = cpl_matrix_new(1, i2 - i1 + 1);
153  cpl_vector *v = cpl_vector_new(i2 - i1 + 1);
154  int i;
155  for (i = i1; i <= i2; i++) { /* x pos start at 1 */
156  cpl_matrix_set(p, 0, i - i1, i);
157  int err;
158  double value = cpl_image_get(objmask, i, 1, &err);
159  if (err) {
160  cpl_vector_set(v, i - i1, NAN); /* set to bad */
161  } else {
162  cpl_vector_set(v, i - i1, value);
163  }
164  } /* for i (relevant columns) */
165  /* do iterative low-order polynomial fit (originally 1st order) */
166  cpl_polynomial *p1 = muse_utils_iterate_fit_polynomial(p, v, NULL, NULL,
167  kOrdObj, kRSigObj,
168  NULL, NULL);
169  int idxold = 0;
170  for (i = i1; i <= i2; i++) { /* x pos start at 1 */
171  int idx = idxold,
172  ncol = cpl_matrix_get_ncol(p);
173  while (idx < ncol && (int)cpl_matrix_get(p, 0, idx) < i) { /* object pixels */
174  idx++;
175  }
176  /* check if we found the entry, if so skip the masking loop */
177  if (idx < ncol && (int)cpl_matrix_get(p, 0, idx) == i) {
178  idxold = idx; /* matrix is sorted, so keep track of position */
179  continue;
180  }
181  int j;
182  for (j = 0; j < ny; j++) { /* set all column pixels to "object" in DQ */
183  dq[(i-1) + j*nx] |= EURO3D_OBJECT;
184  } /* for j (vertical pixels) */
185  } /* for i (relevant columns) */
186  cpl_vector_delete(v);
187  cpl_matrix_delete(p);
188  cpl_polynomial_delete(p1);
189  muse_trace_polys_delete(ptrace); /* NULL check in there... */
190  } /* for islice */
191  cpl_image_delete(objmask);
192 
193  return CPL_ERROR_NONE;
194 } /* muse_sky_subtract_rowbyrow_mask() */
195 
196 /*----------------------------------------------------------------------------*/
222 /*----------------------------------------------------------------------------*/
223 cpl_error_code
224 muse_sky_subtract_rowbyrow(muse_image *aImage, cpl_table *aTrace, float aRSigma,
225  unsigned int aOrder)
226 {
227  cpl_ensure_code(aImage, CPL_ERROR_NULL_INPUT);
228  unsigned short nslices = 0;
229  if (aTrace) {
230  nslices = cpl_table_get_nrow(aTrace);
231  } else if (!aTrace) { /* check for the slice center headers */
232  unsigned short slice = 0;
233  cpl_boolean exists = CPL_FALSE;
234  do {
235  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", ++slice);
236  exists = cpl_propertylist_has(aImage->header, keyword);
237 #if 0
238  cpl_msg_debug(__func__, "%s %s", keyword, exists ? "exists" : "doesn't exist");
239 #endif
240  cpl_free(keyword);
241  } while (exists);
242  nslices = slice - 1;
243  }
244  cpl_msg_debug(__func__, "Found %hu slices", nslices);
245  cpl_ensure_code(nslices, CPL_ERROR_ILLEGAL_INPUT);
246 
247  /* get image properties and data */
248  int nx = cpl_image_get_size_x(aImage->data),
249  ny = cpl_image_get_size_y(aImage->data);
250  float *data = cpl_image_get_data_float(aImage->data),
251  *stat = cpl_image_get_data_float(aImage->stat);
252  int *dq = cpl_image_get_data_int(aImage->dq);
253 
254  /* loop through all slices to fit and remove the sky */
255  unsigned short islice;
256  for (islice = 0; islice < nslices; islice++) {
257  cpl_msg_debug(__func__, "Processing slice %hu", islice + 1);
258 
259  /* find slice edge, either through trace info or *
260  * through centers plus "missing data" flags */
261  cpl_polynomial **ptrace = NULL;
262  int i1 = 0, i2 = nx, istart = 0;
263  if (aTrace) {
264  /* get the tracing polynomials for this slice */
265  ptrace = muse_trace_table_get_polys_for_slice(aTrace, islice + 1);
266  if (!ptrace) {
267  cpl_msg_warning(__func__, "slice %2d: tracing polynomials missing!",
268  islice + 1);
269  continue;
270  }
271  /* use trace of vertical center, be generous with the edges */
272  i1 = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], ny/2, NULL));
273  i2 = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], ny/2, NULL));
274  if (i1 < 1 || i2 > nx || i1 > i2) {
275  cpl_msg_warning(__func__, "slice %2d: faulty polynomial detected at "
276  "y=%d (borders: %d ... %d)", islice + 1, ny/2, i1, i2);
277  muse_trace_polys_delete(ptrace);
278  continue; /* next slice */
279  }
280  } else { /* find slice edge without tracing info */
281  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", islice + 1);
282  istart = cpl_propertylist_get_float(aImage->header, keyword);
283  cpl_free(keyword);
284  /* move outwards */
285  int ix = istart;
286  while (!(dq[(--ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
287  i1 = ix;
288  }
289  ix = istart;
290  while (!(dq[(++ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
291  i2 = ix;
292  }
293  } /* else, no aTrace */
294 #if SKY_ROWBYROW_DEBUG
295  cpl_msg_debug(__func__, "1 slice %d row %d edges: %d %d", islice+1, ny/2, i1, i2);
296 #endif
297  /* loop through all image rows */
298  int j;
299  for (j = 0; j < ny; j++) {
300  /* include pixels that have more than half of them inside the slice, *
301  * use the same method to compute the cutoff as muse_pixtable_create() */
302  int ileft = istart, icenter = istart, iright = istart;
303  if (ptrace) { /* slice edges with tracing info */
304  ileft = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], j+1, NULL)),
305  icenter = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], j+1, NULL),
306  iright = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], j+1, NULL));
307  } else { /* find slice edges here without tracing info */
308  /* move outwards */
309  int ix = istart;
310  while (!(dq[(--ix-1) + j*nx] & EURO3D_MISSDATA)) {
311  ileft = ix;
312  }
313  ix = istart;
314  while (!(dq[(++ix-1) + j*nx] & EURO3D_MISSDATA)) {
315  iright = ix;
316  }
317  icenter = (iright + ileft) / 2.; /* re-"compute" center at this pos */
318  } /* else, no aTrace */
319 #if SKY_ROWBYROW_DEBUG
320  cpl_msg_debug(__func__, "2 slice %d row %d edges: %d %d", islice+1, j+1,
321  ileft, iright);
322 #endif
323 
324  /* fill data into vectors, copy the matrix */
325  cpl_matrix *pos = cpl_matrix_new(1, iright - ileft + 1);
326  cpl_vector *values = cpl_vector_new(iright - ileft + 1);
327  /* unused: *errors = cpl_vector_new(iright - ileft + 1); */
328  unsigned int nval = 0;
329  int i;
330  for (i = ileft; i <= iright; i++) { /* x pos start at 1 */
331  cpl_matrix_set(pos, 0, i - ileft, i - icenter); /* relative coordinates */
332  if (dq[(i-1) + j*nx] != EURO3D_GOODPIXEL) {
333  cpl_vector_set(values, i - ileft, NAN); /* mark as bad entry */
334  } else {
335  cpl_vector_set(values, i - ileft, data[(i-1) + j*nx]);
336  nval++;
337  }
338  /* XXX sigmas are not supported by cpl_polynomial_fit() yet *
339  * cpl_vector_set(errors, i - ileft, sqrt(data[i + j*nx])); */
340  } /* for i (relevant columns) */
341  if (nval < 1) { /* skip image rows with only NANs */
342  cpl_vector_delete(values);
343  /* unused: cpl_vector_delete(errors); */
344  cpl_matrix_delete(pos);
345  continue;
346  }
347  /* iterate the horizontal polynomial fit (originally 4th order) */
348  unsigned int order = aOrder > nval + 1 ? nval - 1 : aOrder;
349  double mse;
350  cpl_polynomial *poly = muse_utils_iterate_fit_polynomial(pos, values,
351  NULL, NULL, order,
352  aRSigma, &mse,
353  NULL);
354  int npix = cpl_vector_get_size(values); /* keep number of final pixels */
355  cpl_vector_delete(values);
356  /* unused: cpl_vector_delete(errors); */
357  cpl_matrix_delete(pos);
358 
359  /* subtract the fit, add the mean-squared error onto the variance */
360  for (i = ileft - 1; i < iright; i++) { /* x pos start at 0! */
361  /* evaluate polynomial at the same relative position within slice */
362  double sky = cpl_polynomial_eval_1d(poly, i+1 - icenter, NULL);
363 #if SKY_ROWBYROW_DEBUG > 1
364  if (islice+1 == SKY_ROWBYROW_DEBUG_SLICE &&
365  j+1 >= SKY_ROWBYROW_DEBUG_ROW1 && j+1 <= SKY_ROWBYROW_DEBUG_ROW2) {
366  printf("subtracting slice %d row %d %d %f ", islice+1, j+1,
367  i+1 - icenter, data[i + j*nx]);
368  }
369 #endif
370  data[i + j*nx] -= sky;
371  /* to correct the variance, add the mean-squared error divided *
372  * by number of pixels minus the degrees of freedom */
373  stat[i + j*nx] += mse / (npix - order-1);
374 #if SKY_ROWBYROW_DEBUG > 1
375  if (islice+1 == SKY_ROWBYROW_DEBUG_SLICE &&
376  j+1 >= SKY_ROWBYROW_DEBUG_ROW1 && j+1 <= SKY_ROWBYROW_DEBUG_ROW2) {
377  printf("%f\n", data[i + j*nx]);
378  fflush(stdout);
379  }
380 #endif
381  } /* for i (relevant columns) */
382  cpl_polynomial_delete(poly);
383  } /* for j (all image rows) */
384 
385  muse_trace_polys_delete(ptrace); /* NULL check in there... */
386  } /* for islice */
387 
388 #if SKY_ROWBYROW_DEBUG
389  cpl_msg_debug(__func__, "saving \"sky_removed_iterated_edge2.fits\"");
390  muse_image_save(aImage, "sky_removed_iterated_edge2.fits");
391 #endif
392 
393  return CPL_ERROR_NONE;
394 } /* muse_sky_subtract_rowbyrow() */
395 
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
cpl_image * data
the data extension
Definition: muse_image.h:46
cpl_error_code muse_sky_subtract_rowbyrow(muse_image *aImage, cpl_table *aTrace, float aRSigma, unsigned int aOrder)
Subtract the sky row-by-row from a CCD-based image.
cpl_image * stat
the statistics extension
Definition: muse_image.h:64
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
cpl_image * dq
the data quality extension
Definition: muse_image.h:56
cpl_polynomial * muse_utils_iterate_fit_polynomial(cpl_matrix *aPos, cpl_vector *aVal, cpl_vector *aErr, cpl_table *aExtra, const unsigned int aOrder, const double aRSigma, double *aMSE, double *aChiSq)
Iterate a polynomial fit.
Definition: muse_utils.c:2178
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:399
cpl_error_code muse_sky_subtract_rowbyrow_mask(muse_image *aImage, cpl_table *aTrace)
Prepare an (object) mask for the sky row-by-row fitting.