UVES Pipeline Reference Manual  5.4.6
uves_extract.c
1 /* *
2  * This file is part of the ESO UVES Pipeline *
3  * Copyright (C) 2004,2005 European Southern Observatory *
4  * *
5  * This library is free software; you can redistribute it and/or modify *
6  * it under the terms of the GNU General Public License as published by *
7  * the Free Software Foundation; either version 2 of the License, or *
8  * (at your option) any later version. *
9  * *
10  * This program is distributed in the hope that it will be useful, *
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13  * GNU General Public License for more details. *
14  * *
15  * You should have received a copy of the GNU General Public License *
16  * along with this program; if not, write to the Free Software *
17  * Foundation, 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA *
18  * */
19 
20 /*
21  * $Author: amodigli $
22  * $Date: 2013-08-08 13:36:46 $
23  * $Revision: 1.196 $
24  * $Name: not supported by cvs2svn $
25  *
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 # include <config.h>
30 #endif
31 
32 /*----------------------------------------------------------------------------*/
39 /*----------------------------------------------------------------------------*/
40 
41 /*-----------------------------------------------------------------------------
42  Includes
43  -----------------------------------------------------------------------------*/
44 #include <string.h>
45 #include <uves_extract.h>
46 
47 #include <uves_extract_iterate.h>
48 #include <uves_extract_profile.h>
49 #include <uves_parameters.h>
50 #include <uves_utils.h>
51 #include <uves_utils_cpl.h>
52 #include <uves_utils_wrappers.h>
53 #include <uves_dfs.h>
54 #include <uves_plot.h>
55 
56 #include <uves_dump.h>
57 #include <uves_error.h>
58 #include <uves.h>
59 
60 #include <irplib_utils.h>
61 
62 #include <cpl.h>
63 
64 #include <stdbool.h>
65 
66 /*-----------------------------------------------------------------------------
67  Defines
68  -----------------------------------------------------------------------------*/
70 #define DATA(name, pos) (name[((pos)->x-1)+((pos)->y-1)*(pos)->nx])
71 
73 #define SPECTRUM_DATA(name, pos) (name[((pos)->x-1)+((pos)->order-(pos)->minorder)*(pos)->nx])
74 
76 #define ISBAD(weights, pos) (weights[((pos)->x-1)+((pos)->y-1)*(pos)->nx] < 0)
77 
79 #define SETBAD(weights, image_bpm, pos) \
80  do { \
81  weights [((pos)->x-1)+((pos)->y-1)*(pos)->nx] = -1.0; \
82  image_bpm[((pos)->x-1)+((pos)->y-1)*(pos)->nx] = CPL_BINARY_1;\
83  } \
84  while (false)
85 
86 #define ISGOOD(bpm, pos) (bpm[((pos)->x-1)+((pos)->y-1)*(pos)->nx] == CPL_BINARY_0)
87 
88 /* Enable experimental algorithm that fits profile to all data in all orders
89  at once */
90 #define NEW_METHOD 0
91 
92 #if NEW_METHOD
93 #define CREATE_DEBUGGING_TABLE 1
94 /* else not used */
95 #endif
96 
97 /*-----------------------------------------------------------------------------
98  Functions prototypes
99  -----------------------------------------------------------------------------*/
102 static int
103 extract_order_simple(const cpl_image *image, const cpl_image *image_noise,
104  const polynomial *order_locations,
105  int order, int minorder,
106  int spectrum_row,
107  double offset,
108  double slit_length,
109  extract_method method,
110  const cpl_image *weights,
111  bool extract_partial,
112  cpl_image *spectrum,
113  cpl_image *spectrum_noise,
114  cpl_binary*spectrum_badmap,
115  cpl_table **info_tbl,
116  double *sn);
117 
118 static double area_above_line(int y, double left, double right);
119 
120 static cpl_table *opt_define_sky(const cpl_image *image, const cpl_image *weights,
121  uves_iterate_position *pos);
122 
123 static cpl_image *opt_extract_sky(const cpl_image *image, const cpl_image *image_noise,
124  const cpl_image *weights,
125  uves_iterate_position *pos,
126  cpl_image *sky_spectrum,
127  cpl_image *sky_spectrum_noise);
128 
129 static cpl_image * opt_subtract_sky(
130  const cpl_image *image, const cpl_image *image_noise,
131  const cpl_image *weights,
132  uves_iterate_position *pos,
133  const cpl_table *sky_map,
134  cpl_image *sky_spectrum,
135  cpl_image *sky_spectrum_noise);
136 
137 static cpl_table **opt_sample_spatial_profile(
138  const cpl_image *image, const cpl_image *weights,
139  uves_iterate_position *pos,
140  int chunk,
141  int sampling_factor,
142  int *nbins);
143 
144 static uves_extract_profile *opt_measure_profile(
145  const cpl_image *image, const cpl_image *image_noise,
146  const cpl_image *weights,
147  uves_iterate_position *pos,
148  int chunk, int sampling_factor,
149  int (*f) (const double x[], const double a[], double *result),
150  int (*dfda)(const double x[], const double a[], double result[]),
151  int M,
152  const cpl_image *sky_spectrum,
153  cpl_table *info_tbl,
154  cpl_table **profile_global);
155 
156 static cpl_table *opt_measure_profile_order(
157  const cpl_image *image, const cpl_image *image_noise,
158  const cpl_binary *image_bpm,
159  uves_iterate_position *pos,
160  int chunk,
161  int (*f) (const double x[], const double a[], double *result),
162  int (*dfda)(const double x[], const double a[], double result[]),
163  int M,
164  const cpl_image *sky_spectrum);
165 
166 static void
167 revise_noise(cpl_image *image_noise,
168  const cpl_binary *image_bpm,
169  const uves_propertylist *image_header,
170  uves_iterate_position *pos,
171  const cpl_image *spectrum,
172  const cpl_image *sky_spectrum,
173  const uves_extract_profile *profile,
174  enum uves_chip chip);
175 
176 static int
177 opt_extract(cpl_image *image,
178  const cpl_image *image_noise,
179  uves_iterate_position *pos,
180  const uves_extract_profile *profile,
181  bool optimal_extract_sky,
182  double kappa,
183  cpl_table *blemish_mask,
184  cpl_table *cosmic_mask,
185  int *cr_row,
186  cpl_table *profile_table,
187  int *prof_row,
188  cpl_image *spectrum,
189  cpl_image *spectrum_noise,
190  cpl_image *weights,
191  cpl_image *sky_spectrum,
192  cpl_image *sky_spectrum_noise,
193  double *sn);
194 
195 static int opt_get_order_width(const uves_iterate_position *pos);
196 static double
197 estimate_sn(const cpl_image *image, const cpl_image *image_noise,
198  uves_iterate_position *pos);
199 
200 static double opt_get_sky(const double *image_data,
201  const double *noise_data,
202  const double *weights_data,
203  uves_iterate_position *pos,
204  const cpl_table *sky_map,
205  double buffer_flux[], double buffer_noise[],
206  double *sky_background_noise);
207 
208 static double opt_get_noise_median(const double *noise_data,
209  const cpl_binary *image_bpm,
210  uves_iterate_position *pos,
211  double noise_buffer[]);
212 
213 static double opt_get_flux_sky_variance(const double *image_data,
214  const double *noise_data,
215  double *weights_data,
216  uves_iterate_position *pos,
217  const uves_extract_profile *profile,
218  bool optimal_extract_sky,
219  double median_noise,
220  double *variance,
221  double *sky_background,
222  double *sky_background_noise);
223 
224 static bool opt_reject_outlier(const double *image_data,
225  const double *noise_data,
226  cpl_binary *image_bpm,
227  double *weights_data,
228  uves_iterate_position *pos,
229  const uves_extract_profile *profile,
230  double kappa,
231  double flux,
232  double sky_background,
233  double red_chisq,
234  cpl_table *cosmic_mask, int *cr_row,
235  int *hot_pixels, int *cold_pixels);
236 
237 static double opt_get_redchisq(const uves_extract_profile *profile,
238  const uves_iterate_position *pos);
239 
240 static polynomial *repeat_orderdef(const cpl_image *image, const cpl_image *image_noise,
241  const polynomial *guess_locations,
242  int minorder, int maxorder, slit_geometry sg,
243  cpl_table *info_tbl);
244 
245 static double
246 detect_ripples(const cpl_image *spectrum, const uves_iterate_position *pos,
247  double sn);
248 
249 /*-----------------------------------------------------------------------------
250  Implementation
251  -----------------------------------------------------------------------------*/
252 
253 /*----------------------------------------------------------------------------*/
261 /*----------------------------------------------------------------------------*/
262 
263 cpl_parameterlist *
265 {
266  const char *name = "";
267  char *full_name = NULL;
268  cpl_parameter *p = NULL;
269  cpl_parameterlist *parameters = NULL;
270 
271  parameters = cpl_parameterlist_new();
272 
273  {
274  name = "method";
275  full_name = uves_sprintf("%s.%s", UVES_EXTRACT_ID, name);
276 
277  uves_parameter_new_enum(p, full_name,
278  CPL_TYPE_STRING,
279  "Extraction method. (2d/optimal not supported by uves_cal_wavecal, weighted supported only by uves_cal_wavecal, 2d not supported by uves_cal_response)",
280  UVES_EXTRACT_ID,
281  "optimal",
282  5,
283  "average",
284  "linear",
285  "2d",
286  "weighted",
287  "optimal");
288 
289  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
290  cpl_parameterlist_append(parameters, p);
291  cpl_free(full_name);
292  }
293 
294  {
295  name = "kappa";
296  full_name = uves_sprintf("%s.%s", UVES_EXTRACT_ID, name);
297 
298  uves_parameter_new_range(p, full_name,
299  CPL_TYPE_DOUBLE,
300  "In optimal extraction mode, this is the "
301  "threshold for bad (i.e. hot/cold) "
302  "pixel rejection. If a pixel deviates more than "
303  "kappa*sigma (where sigma is "
304  "the uncertainty of the pixel flux) from "
305  "the inferred spatial profile, its "
306  "weight is set to zero. Range: [-1,100]. If this parameter "
307  "is negative, no rejection is performed.",
308  UVES_EXTRACT_ID,
309  10.0,-1.,100.);
310 
311  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
312  cpl_parameterlist_append(parameters, p);
313  cpl_free(full_name);
314  }
315 
316  {
317  name = "chunk";
318  full_name = uves_sprintf("%s.%s", UVES_EXTRACT_ID, name);
319 
320  uves_parameter_new_range(p, full_name,
321  CPL_TYPE_INT,
322  "In optimal extraction mode, the chunk size (in pixels) "
323  "used for fitting the analytical profile (a fit of the "
324  "analytical profile to single bins would suffer from "
325  "low statistics).",
326  UVES_EXTRACT_ID,
327  32,
328  1, INT_MAX);
329 
330  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
331  cpl_parameterlist_append(parameters, p);
332  cpl_free(full_name);
333  }
334 
335  {
336  name = "profile";
337  full_name = uves_sprintf("%s.%s", UVES_EXTRACT_ID, name);
338 
339  uves_parameter_new_enum(p, full_name,
340  CPL_TYPE_STRING,
341  "In optimal extraction mode, the kind of profile to use. "
342  "'gauss' gives a Gaussian profile, 'moffat' gives "
343  "a Moffat profile with beta=4 and a possible linear sky "
344  "contribution. 'virtual' uses "
345  "a virtual resampling algorithm (i.e. measures and "
346  "uses the actual object profile). "
347  "'constant' assumes a constant spatial profile and "
348  "allows optimal extraction of wavelength "
349  "calibration frames. 'auto' will automatically "
350  "select the best method based on the estimated S/N of the "
351  "object. For low S/N, 'moffat' or 'gauss' are "
352  "recommended (for robustness). For high S/N, 'virtual' is "
353  "recommended (for accuracy). In the case of virtual resampling, "
354  "a precise determination of the order positions is required; "
355  "therefore the order-definition is repeated "
356  "using the (assumed non-low S/N) science frame",
357  UVES_EXTRACT_ID,
358  "auto",
359  5,
360  "constant",
361  "gauss",
362  "moffat",
363  "virtual",
364  "auto");
365 
366  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
367  cpl_parameterlist_append(parameters, p);
368  cpl_free(full_name);
369  }
370 
371  {
372  name = "skymethod";
373  full_name = uves_sprintf("%s.%s", UVES_EXTRACT_ID, name);
374 
375  uves_parameter_new_enum(p, full_name,
376  CPL_TYPE_STRING,
377  "In optimal extraction mode, the sky subtraction method "
378  "to use. 'median' estimates the sky as the median of pixels "
379  "along the slit (ignoring pixels close to the object), whereas "
380  "'optimal' does a chi square minimization along the slit "
381  "to obtain the best combined object and sky levels. The optimal "
382  "method gives the most accurate sky determination but is also "
383  "a bit slower than the median method",
384  UVES_EXTRACT_ID,
385  "optimal",
386  2,
387  "median",
388  "optimal");
389 
390  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
391  cpl_parameterlist_append(parameters, p);
392  cpl_free(full_name);
393  }
394 
395  {
396  name = "oversample";
397  full_name = uves_sprintf("%s.%s", UVES_EXTRACT_ID, name);
398 
399  uves_parameter_new_range(p, full_name,
400  CPL_TYPE_INT,
401  "The oversampling factor used for the virtual "
402  "resampling algorithm. If negative, the value 5 is "
403  "used for S/N <=200, and the value 10 is used if the estimated "
404  "S/N is > 200",
405  UVES_EXTRACT_ID,
406  -1,
407  -2, INT_MAX);
408 
409  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
410  cpl_parameterlist_append(parameters, p);
411  cpl_free(full_name);
412  }
413 
414  {
415  name = "best";
416  full_name = uves_sprintf("%s.%s", UVES_EXTRACT_ID, name);
417 
418  uves_parameter_new_value(p, full_name,
419  CPL_TYPE_BOOL,
420  "(optimal extraction only) "
421  "If false (fastest), the spectrum is extracted only once. "
422  "If true (best), the spectrum is extracted twice, the "
423  "second time using improved variance estimates "
424  "based on the first iteration. Better variance "
425  "estimates slightly improve the obtained signal to "
426  "noise but at the cost of increased execution time",
427  UVES_EXTRACT_ID,
428  true);
429 
430  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
431  cpl_parameterlist_append(parameters, p);
432  cpl_free(full_name);
433  }
434 
435  if (cpl_error_get_code() != CPL_ERROR_NONE)
436  {
437  cpl_msg_error(__func__, "Creation of extraction parameters failed: '%s'",
438  cpl_error_get_where());
439  cpl_parameterlist_delete(parameters);
440  return NULL;
441  }
442  else
443  {
444  return parameters;
445  }
446 }
447 
448 
449 
450 /*----------------------------------------------------------------------------*/
460 /*----------------------------------------------------------------------------*/
461 extract_method
462 uves_get_extract_method(const cpl_parameterlist *parameters,
463  const char *context, const char *subcontext)
464 {
465  const char *method = "";
466  extract_method result = 0;
467 
468  check( uves_get_parameter(parameters, context, subcontext, "method",
469  CPL_TYPE_STRING, &method),
470  "Could not read parameter");
471 
472  if (strcmp(method, "average" ) == 0) result = EXTRACT_AVERAGE;
473  else if (strcmp(method, "linear" ) == 0) result = EXTRACT_LINEAR;
474  else if (strcmp(method, "2d" ) == 0) result = EXTRACT_2D;
475  else if (strcmp(method, "weighted") == 0) result = EXTRACT_WEIGHTED;
476  else if (strcmp(method, "optimal" ) == 0) result = EXTRACT_OPTIMAL;
477  else
478  {
479  assure(false, CPL_ERROR_ILLEGAL_INPUT, "No such extraction method: '%s'", method);
480  }
481 
482  cleanup:
483  return result;
484 }
485 
486 /*----------------------------------------------------------------------------*/
567 /*----------------------------------------------------------------------------*/
568 cpl_image *
569 uves_extract(cpl_image *image,
570  cpl_image *image_noise,
571  const uves_propertylist *image_header,
572  const cpl_table *ordertable,
573  const polynomial *order_locations_raw,
574  double slit_length,
575  double offset,
576  const cpl_parameterlist *parameters,
577  const char *context,
578  const char *mode,
579  bool extract_partial,
580  bool debug_mode,
581  enum uves_chip chip,
582  uves_propertylist **header,
583  cpl_image **spectrum_noise,
584  cpl_image **sky_spectrum,
585  cpl_image **sky_spectrum_noise,
586  cpl_table **cosmic_mask,
587  cpl_image **cosmic_image,
588  cpl_table **profile_table,
589  cpl_image **weights,
590  cpl_table **info_tbl,
591  cpl_table **order_trace)
592 {
593  cpl_image *spectrum = NULL; /* Result */
594  cpl_mask *spectrum_bad = NULL;
595  cpl_binary*spectrum_badmap = NULL;
596  cpl_image *sky_subtracted = NULL;
597  cpl_image *temp = NULL;
598  cpl_image *reconstruct = NULL;
599  slit_geometry sg;
600 
601  /* Recipe parameters */
602  extract_method method;
603  double kappa;
604  int chunk;
605  const char *p_method;
606  int sampling_factor;
607  bool best;
608  bool optimal_extract_sky;
609  int (*prof_func) (const double x[], const double a[], double *result) = NULL;
610  int (*prof_func_der)(const double x[], const double a[], double result[]) = NULL;
611  int prof_pars = 0;
612 
613  polynomial *order_locations = NULL;/* Improved order positions (or duplicate
614  of input polynomial) */
615  int n_traces; /* The number of traces to extract
616  * within each order, only relevant
617  * for 2D extraction */
618  int iteration, trace; /* Current iteration, order, trace */
619  int n_iterations;
620  int cr_row = 0; /* Points to first unused row in cr table */
621  int prof_row = 0; /* Next unsused row of profile_table */
622  uves_extract_profile *profile = NULL;
623  uves_iterate_position *pos = NULL; /* Iterator over input image */
624  char ex_context[80];
625  cpl_table* blemish_mask=NULL;
626 
627  /* Check input */
628  assure(image != NULL, CPL_ERROR_NULL_INPUT, "Missing input image");
629  /* header may be NULL */
630  assure( spectrum_noise == NULL || image_noise != NULL, CPL_ERROR_DATA_NOT_FOUND,
631  "Need image noise in order to calculate spectrum errors");
632  assure( ordertable != NULL, CPL_ERROR_NULL_INPUT, "Missing order table");
633  assure( order_locations_raw != NULL, CPL_ERROR_NULL_INPUT, "Missing order polynomial");
634  assure( parameters != NULL, CPL_ERROR_NULL_INPUT, "Null parameter list");
635  assure( context != NULL, CPL_ERROR_NULL_INPUT, "Missing context string!");
636  assure( cpl_table_has_column(ordertable, "Order"),
637  CPL_ERROR_DATA_NOT_FOUND, "No 'Order' column in order table!");
638  passure( uves_polynomial_get_dimension(order_locations_raw) == 2, "%d",
639  uves_polynomial_get_dimension(order_locations));
640  assure( slit_length > 0, CPL_ERROR_ILLEGAL_INPUT,
641  "Slit length must a be positive number! It is %e", slit_length);
642  /* sky_spectrum may be NULL */
643  assure( (sky_spectrum == NULL) == (sky_spectrum_noise == NULL), CPL_ERROR_INCOMPATIBLE_INPUT,
644  "Need 0 or 2 of sky spectrum + sky noise spectrum");
645 
646  /* info_tbl may be NULL */
647 
648  sg.length = slit_length;
649  sg.offset = offset;
650 
651 
652  if(strcmp(mode,".efficiency")==0) {
653  sprintf(ex_context,"uves_cal_response%s.reduce",mode);
654  } else {
655  sprintf(ex_context,"%s",context);
656  }
657 
658 
659 
660  /* Get recipe parameters */
661  check( uves_get_parameter(parameters, context, UVES_EXTRACT_ID,
662  "kappa" , CPL_TYPE_DOUBLE, &kappa) ,
663  "Could not read parameter");
664  check( uves_get_parameter(parameters, context, UVES_EXTRACT_ID,
665  "chunk" , CPL_TYPE_INT, &chunk) ,
666  "Could not read parameter");
667 
668  check_nomsg( method = uves_get_extract_method(parameters, ex_context, UVES_EXTRACT_ID) );
669 
670  {
671  char *s_method;
672 
673  check( uves_get_parameter(parameters, context, UVES_EXTRACT_ID,
674  "skymethod", CPL_TYPE_STRING, &s_method),
675  "Could not read parameter");
676  if (strcmp(s_method, "median" ) == 0) optimal_extract_sky = false;
677  else if (strcmp(s_method, "optimal") == 0) optimal_extract_sky = true;
678  else
679  {
680  assure( false, CPL_ERROR_ILLEGAL_INPUT,
681  "Unrecognized sky extraction method: '%s'", s_method);
682  }
683 
684  }
685 
686  {
687  int minorder, maxorder;
688  check(( minorder = cpl_table_get_column_min(ordertable, "Order"),
689  maxorder = cpl_table_get_column_max(ordertable, "Order")),
690  "Error getting order range");
691 
692  pos = uves_iterate_new(cpl_image_get_size_x(image),
693  cpl_image_get_size_y(image),
694  order_locations_raw,
695  minorder, maxorder, sg);
696  /* needed for estimate_sn */
697  }
698  if (method == EXTRACT_OPTIMAL)
699  {
700  assure( image_noise != NULL, CPL_ERROR_ILLEGAL_INPUT,
701  "Extraction method is optimal, but no noise image is provided");
702 
703  assure( weights != NULL, CPL_ERROR_ILLEGAL_INPUT,
704  "Extraction method is optimal, but no weight image is provided");
705 
706  assure( cosmic_mask != NULL, CPL_ERROR_ILLEGAL_INPUT,
707  "Extraction method is optimal, but no cosmic ray mask table is provided");
708 
709  assure( cosmic_image != NULL, CPL_ERROR_ILLEGAL_INPUT,
710  "Extraction method is optimal, but no cosmic ray mask image is provided");
711 
712  assure( order_trace != NULL, CPL_ERROR_ILLEGAL_INPUT,
713  "Extraction method is optimal, but no order trace table is provided");
714 
715  assure( *weights == NULL, CPL_ERROR_ILLEGAL_INPUT,
716  "Weight image already exists");
717 
718  check( uves_get_parameter(parameters, context, UVES_EXTRACT_ID, "oversample",
719  CPL_TYPE_INT, &sampling_factor),
720  "Could not read parameter");
721 
722  check( uves_get_parameter(parameters, context, UVES_EXTRACT_ID, "best",
723  CPL_TYPE_BOOL, &best),
724  "Could not read parameter");
725 
726  check( uves_get_parameter(parameters, context, UVES_EXTRACT_ID, "profile",
727  CPL_TYPE_STRING, &p_method),
728  "Could not read parameter");
729 
730  assure( strcmp(p_method, "constant") == 0 ||
731  sky_spectrum != NULL, CPL_ERROR_ILLEGAL_INPUT,
732  "Extraction method is optimal, but no sky spectrum is provided");
733 
734  if (strcmp(p_method, "auto" ) == 0)
735  {
736  /* Auto-select profile measuring method.
737  At low S/N a model with fewer free
738  parameters is needed */
739 
740  double sn_estimate;
741 
742  check( sn_estimate = estimate_sn(image, image_noise,
743  pos),
744  "Could not estimate image S/N");
745 
746  if (sn_estimate < 10)
747  {
748  p_method = "gauss";
749  }
750  else
751  {
752  p_method = "virtual";
753  }
754 
755  uves_msg("Estimated S/N is %.2f, "
756  "auto-selecting profile measuring method '%s'", sn_estimate,
757  p_method);
758  }
759 
760  if (strcmp(p_method, "gauss" ) == 0)
761  {prof_func = uves_gauss ; prof_func_der = uves_gauss_derivative ; prof_pars = 4;}
762  else if (strcmp(p_method, "moffat" ) == 0)
763  {prof_func = uves_moffat; prof_func_der = uves_moffat_derivative; prof_pars = 5;}
764  else if (strcmp(p_method, "virtual") == 0)
765  {prof_func = NULL ; prof_func_der = NULL ; prof_pars = 0;}
766  else if (strcmp(p_method, "constant") != 0)
767  {
768  assure( false, CPL_ERROR_ILLEGAL_INPUT,
769  "Unrecognized profile method: '%s'", p_method);
770  }
771 
772  assure( sampling_factor != 0, CPL_ERROR_ILLEGAL_INPUT,
773  "Illegal oversampling factor = %d", sampling_factor);
774 
775  if (strcmp(p_method, "virtual") == 0 && sampling_factor < 0)
776  /* Auto-select value */
777  {
778  double sn_estimate;
779 
780  check( sn_estimate = estimate_sn(image, image_noise,
781  pos),
782  "Could not estimate image S/N");
783 
784  if (sn_estimate <= 200)
785  {
786  sampling_factor = 5;
787  }
788  else
789  {
790  sampling_factor = 10;
791  }
792 
793  uves_msg("Estimated S/N is %.2f, "
794  "auto-selecting oversampling factor = %d", sn_estimate,
795  sampling_factor);
796  }
797  }
798 
799  assure( method != EXTRACT_WEIGHTED || weights != NULL, CPL_ERROR_ILLEGAL_INPUT,
800  "Extraction method is weighted, but no weight image is provided");
801 
802  if (method == EXTRACT_2D)
803  {
804  /* 1 trace is just 1 pixel */
805  n_traces = uves_round_double(slit_length);
806 
807  assure( n_traces % 2 == 0, CPL_ERROR_ILLEGAL_INPUT,
808  "For 2d extraction slit length (%d) must be an even number", n_traces);
809  }
810  else
811  {
812  n_traces = 1;
813  }
814 
815  if (method == EXTRACT_2D)
816  {
817  uves_msg_low("Slit length = %.1f pixels", slit_length);
818  }
819  else
820  {
821  uves_msg_low("Slit length = %.1f pixels; offset = %.1f pixel(s)",
822  sg.length, sg.offset);
823  }
824 
825  /* Initialize result images */
826  check(( spectrum = cpl_image_new(pos->nx,
827  n_traces*(pos->maxorder - pos->minorder + 1),
828  CPL_TYPE_DOUBLE),
829  spectrum_bad = cpl_image_get_bpm(spectrum),
830  spectrum_badmap = cpl_mask_get_data(spectrum_bad)),
831  "Error creating spectrum image");
832 
833 
834  if (spectrum_noise != NULL)
835  {
836  check( *spectrum_noise = cpl_image_new(cpl_image_get_size_x(spectrum),
837  cpl_image_get_size_y(spectrum),
838  CPL_TYPE_DOUBLE),
839  "Could not create image");
840  }
841 
842  if (info_tbl != NULL &&
843  (method == EXTRACT_LINEAR || method == EXTRACT_AVERAGE ||
844  method == EXTRACT_OPTIMAL)
845  )
846  {
847  *info_tbl = cpl_table_new(pos->maxorder-pos->minorder+1);
848  cpl_table_new_column(*info_tbl, "Order", CPL_TYPE_INT);
849  cpl_table_new_column(*info_tbl, "ObjSnBlzCentre", CPL_TYPE_DOUBLE);
850  cpl_table_new_column(*info_tbl, "Ripple", CPL_TYPE_DOUBLE);
851  cpl_table_set_column_unit(*info_tbl,"Order", " ");
852  cpl_table_set_column_unit(*info_tbl,"ObjSnBlzCentre", " ");
853  cpl_table_set_column_unit(*info_tbl,"Ripple", " ");
854 
855  /* Pos+FWHM columns are calculated differently,
856  based on optimal extraction method,
857  and simple extraction */
858 
859  cpl_table_new_column(*info_tbl, "ObjPosOnSlit", CPL_TYPE_DOUBLE); /* From bottom of slit */
860  cpl_table_new_column(*info_tbl, "ObjFwhmAvg", CPL_TYPE_DOUBLE);
861  cpl_table_set_column_unit(*info_tbl,"ObjPosOnSlit", "pix");
862  cpl_table_set_column_unit(*info_tbl,"ObjFwhmAvg", "pix");
863  }
864 
865  /* Extra input validation + initialization for optimal extraction */
866  if (method == EXTRACT_OPTIMAL)
867  {
868  /* Initialize weights to zero (good pixels) */
869  check( *weights = cpl_image_new(pos->nx, pos->ny, CPL_TYPE_DOUBLE),
870  "Could not allocate weight image");
871 
872  /* Initialize cr and profile tables */
873  check(( *cosmic_mask = cpl_table_new(1),
874  cpl_table_new_column(*cosmic_mask, "Order", CPL_TYPE_INT),
875  cpl_table_new_column(*cosmic_mask, "X" , CPL_TYPE_INT),
876  cpl_table_new_column(*cosmic_mask, "Y" , CPL_TYPE_INT),
877  cpl_table_new_column(*cosmic_mask, "Flux" , CPL_TYPE_DOUBLE),
878  cr_row = 0),
879  "Error creating cosmic ray table");
880  cpl_table_set_column_unit(*cosmic_mask,"Order", "");
881  cpl_table_set_column_unit(*cosmic_mask,"X", "pix");
882  cpl_table_set_column_unit(*cosmic_mask,"Y", "pix");
883  cpl_table_set_column_unit(*cosmic_mask,"Flux", "ADU");
884 
885  /* We need to flag detector detector blemishes if present */
886  if(*cosmic_image!=NULL) {
887  int sx=0;
888  int sy=0;
889  int nblemish=0;
890  int i=0;
891  int j=0;
892  int row=0;
893 
894  double flux=0;
895  int* px=NULL;
896  int* py=NULL;
897 
898  double* pcmask=NULL;
899  double blemish_frac=0;
900 
901  /* we count how many blemishes we got */
902  flux=cpl_image_get_flux(*cosmic_image);
903  sx=cpl_image_get_size_x(*cosmic_image);
904  sy=cpl_image_get_size_y(*cosmic_image);
905  nblemish=sx*sy-(int)flux;
906  blemish_frac=(sx*sy-flux)/(sx*sy);
907  uves_msg("nblemish=%d frac=%g",nblemish,blemish_frac);
908 
909  if(blemish_frac< 0.02) {
910 
911  /* we copy blemishes in a table, for efficiency */
912  blemish_mask=cpl_table_new(nblemish);
913  cpl_table_new_column(blemish_mask,"X",CPL_TYPE_INT);
914  cpl_table_new_column(blemish_mask,"Y",CPL_TYPE_INT);
915  cpl_table_fill_column_window_int(blemish_mask,"X",
916  0,nblemish,0);
917  cpl_table_fill_column_window_int(blemish_mask,"Y",
918  0,nblemish,0);
919 
920  pcmask=cpl_image_get_data_double(*cosmic_image);
921  px=cpl_table_get_data_int(blemish_mask,"X");
922  py=cpl_table_get_data_int(blemish_mask,"Y");
923 
924  for(j=0;j<sy;j++) {
925  for(i=0;i<sx;i++) {
926  if(pcmask[j*sx+i]==0) {
927  px[row]=i;
928  py[row]=j;
929  row++;
930  }
931  }
932  }
933  /*
934  check_nomsg(cpl_table_save(blemish_mask,NULL,NULL,
935  "blemish_mask.fits",CPL_IO_DEFAULT));
936  */
937  cr_row=nblemish;
938  } else {
939  uves_msg_warning("%d pixels affected by detector blemishes %g (>0.02) of total. Not flag them in optimal extraction",nblemish,blemish_frac);
940 
941  }
942  } /* end special case for detector blemishes */
943 
944 
945  if (profile_table != NULL)
946  {
947  check( (*profile_table = cpl_table_new((pos->maxorder - pos->minorder + 1) *
948  pos->nx *
949  (3+uves_round_double(sg.length))),
950  cpl_table_new_column(*profile_table, "Order" , CPL_TYPE_INT),
951  cpl_table_new_column(*profile_table, "X" , CPL_TYPE_INT),
952  cpl_table_new_column(*profile_table, "DY" , CPL_TYPE_DOUBLE),
953  cpl_table_new_column(*profile_table, "Profile_raw", CPL_TYPE_DOUBLE),
954  cpl_table_new_column(*profile_table, "Profile_int", CPL_TYPE_DOUBLE)),
955  "Error creating profile table");
956  prof_row = 0;
957  cpl_table_set_column_unit(*profile_table,"Order", " ");
958  cpl_table_set_column_unit(*profile_table,"X", "pix");
959  cpl_table_set_column_unit(*profile_table,"DY", "pix");
960  cpl_table_set_column_unit(*profile_table,"Profile_raw", "ADU");
961  cpl_table_set_column_unit(*profile_table,"Profile_int", "ADU");
962  }
963 
964  if (strcmp(p_method, "constant") != 0) {
965  check( *sky_spectrum = cpl_image_new(
966  pos->nx, pos->maxorder - pos->minorder + 1, CPL_TYPE_DOUBLE),
967  "Could not allocate sky spectrum");
968  check( *sky_spectrum_noise = cpl_image_new(
969  pos->nx, pos->maxorder - pos->minorder + 1, CPL_TYPE_DOUBLE),
970  "Could not allocate sky spectrum noise");
971  }
972  }
973 
974  if (method == EXTRACT_OPTIMAL &&
975  strcmp(p_method, "constant") != 0 && prof_func == NULL)
976  {
977  /* Virtual method needs accurate order definition.
978  * Some calibration order tables are inaccurate because
979  * the poly-degree used (2,3) is too low.
980  *
981  * Besides, the (science) spectrum might be shifted compared
982  * to the order-flat-narrow frame.
983  */
984 
985  uves_msg("Refining order definition using the object frame");
986 
987  check( order_locations = repeat_orderdef(image, image_noise, order_locations_raw,
988  pos->minorder, pos->maxorder,
989  pos->sg,
990  *info_tbl),
991  "Could not refine order definition");
992  }
993  else
994  {
995  order_locations = uves_polynomial_duplicate(order_locations_raw);
996  }
997 
998  pos->order_locations = order_locations;
999 
1000  /* Input checking + output initialization done. */
1001 
1002 
1003  /* Do the processing, pseudocode for optimal extraction:
1004 
1005  extract+subtract sky (median method)
1006  globally measure profile
1007 
1008  two times
1009  for each order
1010  extract object+sky, reject hot/cold pixels
1011  revise variances
1012  */
1013  if (method == EXTRACT_OPTIMAL)
1014  {
1015  if (strcmp(p_method, "constant") == 0) {
1016 
1017  uves_msg("Assuming constant spatial profile");
1018 
1019  profile = uves_extract_profile_new_constant(sg.length);
1020 
1021  /* Pretend that we subtracted the sky here */
1022  sky_subtracted = cpl_image_duplicate(image);
1023  optimal_extract_sky = false;
1024 
1025  }
1026  else {
1027  check( sky_subtracted = opt_extract_sky(
1028  image, image_noise, *weights,
1029  pos,
1030  *sky_spectrum,
1031  *sky_spectrum_noise),
1032  "Could not extract sky");
1033  if (prof_func != NULL)
1034  {
1035  uves_msg("Measuring spatial profile "
1036  "(method = %s, chunk = %d bins)",
1037  p_method, chunk);
1038  }
1039  else
1040  {
1041  uves_msg("Measuring spatial profile "
1042  "(method = %s, oversampling = %d)",
1043  p_method, sampling_factor);
1044  }
1045 
1046  uves_extract_profile_delete(&profile);
1047  /* the new profile measuring method should use this one
1048  check( profile = opt_measure_profile(image, image_noise, *weights, */
1049  check( profile = opt_measure_profile(sky_subtracted, image_noise, *weights,
1050  pos,
1051  chunk, sampling_factor,
1052  prof_func, prof_func_der, prof_pars,
1053  *sky_spectrum,
1054  *info_tbl,
1055  order_trace),
1056  "Could not measure profile");
1057 
1058  /* In previous versions, the sky was subtracted (again) at this point
1059  using the knowledge of the analytical profile.
1060  But this is not needed anymore, now that the sky is
1061  extracted simultaneously with the flux (which is equivalent
1062  but much faster).
1063  */
1064  }
1065  }
1066 
1067  /* The loop over traces is trivial, unless method = 2d. */
1068  passure( method == EXTRACT_2D || n_traces == 1, "%d", n_traces);
1069 
1070  n_iterations = (method == EXTRACT_OPTIMAL &&
1071  best &&
1072  strcmp(p_method, "constant") != 0) ? 2 : 1;
1073  //cpl_table_dump(*cosmic_mask,0,cr_row,stdout);
1074  //uves_msg("cr_row=%d table size=%d",cr_row,cpl_table_get_nrow(*cosmic_mask));
1075  int cr_row_max=0;
1076  /* in case of blemishes cr_row> 0 */
1077  //cr_row_max=(cr_row>cr_row_max) ? cr_row: cr_row_max;
1078 
1079  //cpl_table_dump(*cosmic_mask,1,2,stdout);
1080 
1081  for (iteration = 1;
1082  iteration <= n_iterations;
1083  iteration++)
1084  {
1085  uves_msg("Extracting object %s(method = %s)",
1086  (method == EXTRACT_OPTIMAL && optimal_extract_sky)
1087  ? "and sky " : "",
1088  (method == EXTRACT_OPTIMAL) ? "optimal" :
1089  (method == EXTRACT_AVERAGE) ? "average" :
1090  (method == EXTRACT_LINEAR ) ? "linear" :
1091  (method == EXTRACT_2D ) ? "2d" :
1092  (method == EXTRACT_WEIGHTED) ? "weighted" : "???");
1093 
1094  /* Clear cosmic ray + profile table + S/N table */
1095  //uves_msg("cr_row=%d table size=%d",cr_row,cpl_table_get_nrow(*cosmic_mask));
1096  cr_row = cr_row_max;
1097  //uves_msg("cr_row=%d table size=%d",cr_row,cpl_table_get_nrow(*cosmic_mask));
1098  prof_row = 0;
1099  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++) {
1100  for (trace = 1; trace <= n_traces; trace++) {
1101  int spectrum_row; /* Spectrum image row to write to */
1102  int bins_extracted;
1103 
1104  double sn = 0;
1105 
1106  spectrum_row = (pos->order - pos->minorder)*n_traces + trace;
1107  /* Always count from order=1 in the extracted spectrum */
1108 
1109  if (method == EXTRACT_OPTIMAL)
1110  {
1111  /*
1112  * We already know the spatial profile.
1113  * Extract object+sky
1114  */
1115 
1116  check( bins_extracted = opt_extract(
1117  optimal_extract_sky ?
1118  image : sky_subtracted,
1119  image_noise,
1120  pos,
1121  profile,
1122  optimal_extract_sky,
1123  kappa,
1124  blemish_mask,
1125  *cosmic_mask,
1126  &cr_row,
1127  (profile_table != NULL) ?
1128  *profile_table : NULL,
1129  &prof_row,
1130  spectrum,
1131  (spectrum_noise != NULL) ?
1132  *spectrum_noise : NULL,
1133  *weights,
1134  optimal_extract_sky ? *sky_spectrum : NULL,
1135  optimal_extract_sky ? *sky_spectrum_noise : NULL,
1136  &sn),
1137  "Error extracting order #%d", pos->order);
1138  cr_row_max=(cr_row>cr_row_max) ? cr_row:cr_row_max;
1139  }
1140  else
1141  {
1142  /* Average, linear, 2d, weighted */
1143 
1144  /* A 2d extraction is implemented
1145  * as a repeated linear extraction
1146  * with slit_length = 1.
1147  *
1148  * For 2d mode, map
1149  * trace = 1, 2, ..., n_traces
1150  * to something that is symmetric around 0
1151  * (notice that n_traces is an even number)
1152  * offset = -n_traces/2 + 1/2, ..., n_traces/2 - 1/2
1153  */
1154 
1155  double offset_2d = trace - (n_traces+1)/2.0;
1156  double slit_2d = 1;
1157 
1158  check( bins_extracted = extract_order_simple(
1159  image, image_noise,
1160  order_locations,
1161  pos->order, pos->minorder,
1162  spectrum_row,
1163  (method == EXTRACT_2D) ? offset_2d : sg.offset,
1164  (method == EXTRACT_2D) ? slit_2d : sg.length,
1165  (method == EXTRACT_2D) ? EXTRACT_LINEAR : method,
1166  (weights != NULL) ? *weights : NULL,
1167  extract_partial,
1168  spectrum,
1169  (spectrum_noise != NULL) ? *spectrum_noise : NULL,
1170  spectrum_badmap,
1171  info_tbl,
1172  &sn),
1173  "Could not extract order #%d ; trace #%d",
1174  pos->order, trace);
1175  }
1176 
1177 
1178  if (info_tbl != NULL &&
1179  (method == EXTRACT_LINEAR || method == EXTRACT_AVERAGE ||
1180  method == EXTRACT_OPTIMAL)
1181  )
1182  {
1183  /* Do post extraction measurements of any ripples */
1184  double ripple_index = detect_ripples(spectrum, pos, sn);
1185  uves_msg("Order #%d: S/N = %.2f",
1186  pos->order, sn);
1187  uves_msg_debug("Ripple index = %.2f (should be less than 2)",
1188  ripple_index);
1189 
1190  if (false && ripple_index > 3) {
1191  /* Disabled. This would also produce warnings about arc
1192  lamp frames which have short period ripples (a.k.a ThAr emmision
1193  lines), which is just silly.
1194  */
1195  uves_msg_warning("Short period ripples detected (index = %f). "
1196  "It might help to use average or linear extraction "
1197  "or optimal/virtual extraction with larger "
1198  "oversampling factor", ripple_index);
1199  }
1200 
1201  cpl_table_set_int (*info_tbl, "Order",
1202  pos->order - pos->minorder, pos->order);
1203  cpl_table_set_double(*info_tbl, "ObjSnBlzCentre" ,
1204  pos->order - pos->minorder, sn);
1205  cpl_table_set_double(*info_tbl, "Ripple",
1206  pos->order - pos->minorder,
1207  (ripple_index > -0.5) ? ripple_index : -1);
1208  }
1209 
1211  "Order #%d; trace #%d: %d of %d bins extracted",
1212  pos->order, trace, bins_extracted, pos->nx);
1213 
1214  }/* for trace ... */
1215 
1216  }/* for order ... */
1217 
1218 
1219  if (method == EXTRACT_OPTIMAL)
1220  {
1221  if (spectrum_noise != NULL)
1222  {
1223  uves_free_image(&temp);
1224  temp = cpl_image_divide_create(spectrum, *spectrum_noise);
1225  uves_msg("Average S/N = %.3f", cpl_image_get_median(temp));
1226  }
1227 
1228  if (iteration == 1 && n_iterations >= 2)
1229  {
1230  /* If optimal extraction, repeat with more accurate error bars */
1231  uves_msg_low("Recomputing pixel variances");
1232 
1233  check( revise_noise(image_noise,
1234  cpl_mask_get_data(
1235  cpl_image_get_bpm(sky_subtracted)),
1236  image_header, pos,
1237  spectrum, *sky_spectrum, profile,
1238  chip),
1239  "Error refining input image variances");
1240  }
1241  }
1242  // AMO noise computation: put back noise bias & dark contributes
1243 
1244  }/* for iteration */
1245 
1246  /* Set cosmic mask + profile table size, and weights to non-negative */
1247  if (method == EXTRACT_OPTIMAL)
1248  {
1249  int i;
1250  /* AMO: change CRH mask start raw to include all detected CRHs */
1251  check( cpl_table_set_size(*cosmic_mask, cr_row_max),
1252  "Error setting cosmic ray table size to %d", cr_row_max);
1253  if(*cosmic_image==NULL) {
1254  *cosmic_image = cpl_image_new(pos->nx, pos->ny, CPL_TYPE_DOUBLE);
1255  }
1256  assure_mem(*cosmic_image);
1257 
1258  for (i = 0; i < cpl_table_get_nrow(*cosmic_mask); i++)
1259  {
1260  cpl_image_set(*cosmic_image,
1261  cpl_table_get_int(*cosmic_mask, "X", i, NULL),
1262  cpl_table_get_int(*cosmic_mask, "Y", i, NULL),
1263  cpl_table_get_double(*cosmic_mask, "Flux", i, NULL));
1264  }
1265 
1266  if (profile_table != NULL)
1267  {
1268  check( cpl_table_set_size(*profile_table, prof_row),
1269  "Error setting profile table size to %d", prof_row);
1270  }
1271 
1272  /* There are still pixels outside the extraction bins
1273  which have not been touched after creating
1274  the weights image. They are negative; set to zero. */
1275 
1276  check( cpl_image_threshold(*weights,
1277  0, DBL_MAX,
1278  0, DBL_MAX),
1279  "Error thresholding weight image");
1280 
1281  /* Normalize weights (to 1) to get a
1282  * more informative weight image
1283  * This is not needed for the algorithm
1284  * but is computationally cheap
1285  */
1286 
1287  {
1288  double *weights_data = cpl_image_get_data_double(*weights);
1289 
1290  for (uves_iterate_set_first(pos,
1291  1, pos->nx,
1292  pos->minorder, pos->maxorder,
1293  NULL, false);
1294  !uves_iterate_finished(pos);
1296  {
1297  double sum_weights = 0.0;
1298 
1299  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
1300  {
1301  double weight = DATA(weights_data, pos);
1302  sum_weights += weight;
1303  }
1304 
1305  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
1306  {
1307  if (sum_weights > 0)
1308  {
1309  DATA(weights_data, pos) /= sum_weights;
1310  }
1311  }
1312  }
1313  }
1314  } /* if optimal */
1315 
1316  /* Copy bad pixel map from spectrum to error bar spectrum */
1317  uves_msg_debug("Rejecting %" CPL_SIZE_FORMAT " bins", cpl_mask_count(spectrum_bad));
1318 
1319  if (spectrum_noise != NULL)
1320  {
1321  check( cpl_image_reject_from_mask(*spectrum_noise, spectrum_bad),
1322  "Error setting bad pixels");
1323  }
1324 
1325  /* Create spectrum header */
1326  if (header != NULL)
1327  {
1328  /* (pixel, pixel) or (pixel, order) space */
1330  "PIXEL", (method == EXTRACT_2D) ? "PIXEL" : "ORDER",
1331  "PIXEL", (method == EXTRACT_2D) ? "PIXEL" : "ORDER",
1332  "ADU",0,
1333  1.0, pos->minorder, /* CRVAL */
1334  1.0, 1.0, /* CRPIX */
1335  1.0, 1.0), /* CDELT (this should really be the x-binning) */
1336  "Error initializing spectrum header");
1337  }
1338 
1339  if (debug_mode && header != NULL) {
1340  if (profile == NULL) {
1341  /* If profile was not measured (i.e. linear/average etc.),
1342  set to constant */
1343  profile = uves_extract_profile_new_constant(sg.length);
1344  }
1345 
1346  check_nomsg( reconstruct =
1347  uves_create_image(pos, chip,
1348  spectrum,
1349  sky_spectrum != NULL ? *sky_spectrum : NULL,
1350  cosmic_image != NULL ? *cosmic_image : NULL,
1351  profile,
1352  NULL, NULL)); /* error bars, header */
1353 
1354  /*
1355  check(uves_propertylist_copy_property_regexp(*header, image_header, "^ESO ", 0),
1356  "Error copying hieararch keys");
1357  */
1358  check( uves_save_image_local("Reconstructed image", "simulate",
1359  reconstruct, chip, -1, -1, *header, true),
1360  "Error saving image");
1361 
1362  }
1363 
1364  if (spectrum_noise != NULL)
1365  {
1366  cpl_size x, y;
1367 
1368  /* Assert that produced noise spectrum is
1369  always positive.
1370 
1371  For efficiency, cpl_image_get_minpos
1372  is called only in case of error (using
1373  a comma expression)
1374  */
1375 
1376  /* ... then this assertion should not fail */
1377  assure( cpl_image_get_min(*spectrum_noise) > 0, CPL_ERROR_ILLEGAL_OUTPUT,
1378  "Non-positive noise: %e at (%" CPL_SIZE_FORMAT ", %" CPL_SIZE_FORMAT ")",
1379  cpl_image_get_min(*spectrum_noise),
1380  (cpl_image_get_minpos(*spectrum_noise, &x, &y), x),
1381  (cpl_image_get_minpos(*spectrum_noise, &x, &y), y));
1382 
1383  /* For debugging: this code dumps S/N statistics (and leaks memory)
1384  cpl_stats_dump(cpl_stats_new_from_image(
1385  cpl_image_divide_create(spectrum, *spectrum_noise),
1386  CPL_STATS_ALL), CPL_STATS_ALL, stdout);
1387  */
1388  }
1389 
1390 
1391  cleanup:
1392  uves_free_image(&reconstruct);
1393  uves_free_image(&sky_subtracted);
1394  uves_extract_profile_delete(&profile);
1395  uves_polynomial_delete(&order_locations);
1396  uves_iterate_delete(&pos);
1397  uves_free_image(&temp);
1398  uves_free_table(&blemish_mask);
1399 
1400  if (cpl_error_get_code() != CPL_ERROR_NONE)
1401  {
1402  uves_free_image(&spectrum);
1403  uves_free_image(spectrum_noise);
1404  uves_free_table(profile_table);
1405  }
1406 
1407  return spectrum;
1408 }
1409 
1410 /*----------------------------------------------------------------------------*/
1420 /*----------------------------------------------------------------------------*/
1421 static double
1422 detect_ripples(const cpl_image *spectrum, const uves_iterate_position *pos,
1423  double sn)
1424 {
1425  double ratio = -1; /* result */
1426  int n_traces = 1; /* Not 2d extraction */
1427  int trace = 1;
1428  int nx = cpl_image_get_size_x(spectrum);
1429  cpl_image *spectrum_order = NULL;
1430  cpl_vector *tempx = NULL;
1431  cpl_vector *tempy = NULL;
1432  double *auto_corr = NULL;
1433 
1434  int spectrum_row = (pos->order - pos->minorder)*n_traces + trace;
1435  int n_rejected;
1436 
1437  uves_free_image(&spectrum_order);
1438 
1439  check( spectrum_order = cpl_image_extract(spectrum,
1440  1, spectrum_row,
1441  nx, spectrum_row),
1442  "Error extracting order %d from spectrum", pos->order);
1443 
1444  n_rejected = cpl_image_count_rejected(spectrum_order);
1445  uves_msg_debug("Order %d: %d/%d invalid values", pos->order,
1446  n_rejected,
1447  nx);
1448 
1449  if (n_rejected == 0) /* Skip partial orders */
1450  /* Compute auto-correlation function */
1451  {
1452  double order_slope = /* dy/dx at x = nx/2 */
1453  uves_polynomial_derivative_2d(pos->order_locations, nx/2, pos->order, 1);
1454 
1455  int expected_period = uves_round_double(1.0/order_slope);
1456  int max_period = 2*expected_period;
1457  int shift; /* in pixels */
1458 
1459  uves_msg_debug("Estimated ripple period = %d pixels", expected_period);
1460 
1461  auto_corr = cpl_calloc(sizeof(double), 1+max_period);
1462 
1463  for (shift = 0; shift <= max_period; shift += 1) {
1464  int N = 0;
1465  int x;
1466 
1467  auto_corr[shift] = 0;
1468 
1469  for (x = 1; x <= nx - max_period; x++) {
1470  int rejected1, rejected2;
1471  double val1, val2;
1472 
1473  val1 = cpl_image_get(spectrum_order, x, 1, &rejected1);
1474  val2 = cpl_image_get(spectrum_order, x+shift, 1, &rejected2);
1475 
1476  if (!rejected1 && !rejected2)
1477  {
1478  auto_corr[shift] += val1*val2;
1479  N++;
1480  }
1481  }
1482 
1483  if (N != 0)
1484  {
1485  auto_corr[shift] /= N;
1486  }
1487  else
1488  {
1489  auto_corr[shift] = 0;
1490  }
1491 
1492  if (shift > 0 && auto_corr[0] > 0)
1493  {
1494  auto_corr[shift] /= auto_corr[0];
1495  }
1496 
1497  uves_msg_debug("Auto-correlation (%d pixels, %d samples) = %f",
1498  shift, N, (shift == 0) ? 1 : auto_corr[shift]);
1499  }
1500  auto_corr[0] = 1;
1501  /* Done compute auto correlation function for this order */
1502 
1503  {
1504  /* Get amplitude of normalized auto correlation function */
1505  double auto_amplitude;
1506  int imax = expected_period;
1507  int imin1 = expected_period/2;
1508  int imin2 = (expected_period*3)/2;
1509 
1510  /* Measuring the ACF maxima + minima would be non-robust to
1511  the case where there is no peak. Therefore use simply
1512  the predicted positions: */
1513 
1514  auto_amplitude = auto_corr[imax] -
1515  (auto_corr[imin1] + auto_corr[imin2])/2.0;
1516 
1517  /* The autocorrelation function is used to estimate the ripple amplitude.
1518  * Not caring too much about numerical factors and the specific
1519  * analytical form of the oscillations, the following relation holds:
1520  *
1521  * autocorrelation function relative amplitude =
1522  * (ripple relative amplitude)^2
1523  *
1524  * To convert from this amplitude to a stdev we can assume a
1525  * sine curve i.e. divide the amplitude by 2 to get the stdev
1526  * (or alternatively multiply the spectrum error bars by 2)
1527  */
1528 
1529  if (auto_amplitude > 0 && sn > 0)
1530  {
1531  double rel_ripple = sqrt(auto_amplitude);
1532  uves_msg_debug("Order %d: Relative ripple amplitude = %f, "
1533  "relative error bars = %f",
1534  pos->order, rel_ripple, 2.0*1/sn);
1535 
1536  ratio = rel_ripple * sn/2.0;
1537  }
1538  }
1539  } /* Done measuring auto correlation function */
1540 
1541  cleanup:
1542  uves_free_double(&auto_corr);
1543  uves_free_vector(&tempx);
1544  uves_unwrap_vector(&tempy);
1545  uves_free_image(&spectrum_order);
1546 
1547 
1548  return ratio;
1549 }
1550 
1551 /*----------------------------------------------------------------------------*/
1563 /*----------------------------------------------------------------------------*/
1564 static double
1565 estimate_sn(const cpl_image *image, const cpl_image *image_noise,
1566  uves_iterate_position *pos)
1567 {
1568  double sn = -1;
1569  int range = 5; /* Use central (2*range+1) bins in each order */
1570  cpl_table *sn_temp = NULL;
1571  cpl_table *sky_temp = NULL;
1572  int sn_row, sky_row;
1573  int sky_size = 2 + 2*uves_round_double(pos->sg.length); /* allocate enough rows
1574  to store all values
1575  across the slit */
1576 
1577  passure( image_noise != NULL, " ");
1578 
1579  assure( pos->nx >= 2*(range+1), CPL_ERROR_ILLEGAL_INPUT,
1580  "Input image is too small. Width = %d", pos->nx);
1581 
1582  sn_temp = cpl_table_new((pos->maxorder - pos->minorder + 1) * (2*range + 1));
1583  cpl_table_new_column(sn_temp, "SN", CPL_TYPE_DOUBLE);
1584  sn_row = 0;
1585 
1586  sky_temp = cpl_table_new(sky_size);
1587  cpl_table_new_column(sky_temp, "Sky", CPL_TYPE_DOUBLE);
1588 
1589  for (uves_iterate_set_first(pos,
1590  pos->nx/2 - range, pos->nx/2 + range,
1591  pos->minorder, pos->maxorder,
1592  NULL, false);
1593  !uves_iterate_finished(pos);
1595  {
1596  double flux = 0;
1597  double error = 0;
1598  int N = 0;
1599 
1600  sky_row = 0;
1601 
1602  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
1603  {
1604  int pis_rejected1, pis_rejected2;
1605  double pixel = cpl_image_get(image,
1606  pos->x, pos->y, &pis_rejected1);
1607  double pixel_noise = cpl_image_get(image_noise,
1608  pos->x, pos->y, &pis_rejected2);
1609 
1610  if (!pis_rejected1 && !pis_rejected2)
1611  {
1612  flux += pixel;
1613  error += pixel_noise*pixel_noise;
1614  N++;
1615 
1616  cpl_table_set_double(sky_temp, "Sky",
1617  sky_row, pixel);
1618  sky_row++;
1619  }
1620  }
1621 
1622  if (N > 0)
1623  {
1624  double sky; /* Sky level of one pixel, not full slit */
1625 
1626  while(sky_row < sky_size)
1627  /* Mark remaining values as bad before getting median */
1628  {
1629  cpl_table_set_invalid(sky_temp, "Sky",
1630  sky_row);
1631 
1632  sky_row++;
1633  }
1634 
1635  sky = cpl_table_get_column_median(sky_temp, "Sky");
1636 
1637  flux = flux - N*sky;
1638  error = sqrt(error); /* Don't propagate the (small) error
1639  from the sky subtraction */
1640 
1641  if (error > 0)
1642  {
1643  uves_msg_debug("Order %d: S/N estimate = %f",
1644  pos->order, flux/error);
1645 
1646  cpl_table_set_double(sn_temp, "SN",
1647  sn_row, flux/error);
1648  sn_row++;
1649  }
1650  }
1651  }
1652 
1653  assure(sn_row > 0, CPL_ERROR_DATA_NOT_FOUND,
1654  "Extraction of central bins failed!");
1655 
1656  cpl_table_set_size(sn_temp, sn_row);
1657 
1658  sn = cpl_table_get_column_median(sn_temp, "SN");
1659 
1660  cleanup:
1661  uves_free_table(&sn_temp);
1662  uves_free_table(&sky_temp);
1663  return sn;
1664 }
1665 
1666 /*----------------------------------------------------------------------------*/
1698 /*----------------------------------------------------------------------------*/
1699 
1700 static int
1701 extract_order_simple(const cpl_image *image,
1702  const cpl_image *image_noise,
1703  const polynomial *order_locations,
1704  int order,
1705  int minorder,
1706  int spectrum_row,
1707  double offset,
1708  double slit_length,
1709  extract_method method,
1710  const cpl_image *weights,
1711  bool extract_partial,
1712  cpl_image *spectrum,
1713  cpl_image *spectrum_noise,
1714  cpl_binary*spectrum_badmap,
1715  cpl_table **info_tbl,
1716  double *sn)
1717 {
1718  int bins_extracted = 0;
1719  double *spectrum_data;
1720  int x, nx, ny;
1721  double flux_y, flux_yy, flux_tot;
1722  int sn_row = 0; /* Number of rows in 'signal_to_noise'
1723  actually used */
1724  cpl_table *signal_to_noise = NULL;
1725 
1726  passure( method == EXTRACT_AVERAGE ||
1727  method == EXTRACT_LINEAR ||
1728  method == EXTRACT_WEIGHTED, "%d", method);
1729 
1730  /* It's probably a bug if there's a weight image and method = linear/average */
1731  passure( (method == EXTRACT_WEIGHTED) == (weights != NULL), "%d", method);
1732 
1733  nx = cpl_image_get_size_x(image);
1734  ny = cpl_image_get_size_y(image);
1735 
1736  check( (signal_to_noise = cpl_table_new(nx),
1737  cpl_table_new_column(signal_to_noise, "SN", CPL_TYPE_DOUBLE)),
1738  "Error allocating S/N table");
1739 
1740  spectrum_data = cpl_image_get_data_double(spectrum);
1741 
1742  flux_y = 0;
1743  flux_yy = 0;
1744  flux_tot = 0;
1745  /* Extract the entire image width */
1746  for (x = 1 ; x <= nx; x++) {
1747  double slope, ycenter; /* Running slope, bin center */
1748  int ylo, yhi; /* Lowest, highest pixel to look at */
1749  double flux = 0;
1750  double flux_variance = 0;
1751  double sum = 0; /* (Fractional) number of pixels extracted so far */
1752  int y;
1753 
1754  /* Get local order slope */
1755  check(( slope = (uves_polynomial_evaluate_2d(order_locations, x+1, order) -
1756  uves_polynomial_evaluate_2d(order_locations, x-1, order) ) / 2,
1757  /* Center of order */
1758  ycenter = uves_polynomial_evaluate_2d(order_locations, x, order) + offset),
1759  "Error evaluating polynomial");
1760 
1761  assure( 0 < slope && slope < 1, CPL_ERROR_ILLEGAL_INPUT,
1762  "At (x, order)=(%d, %d) slope is %f. Must be positive", x, order, slope);
1763 
1764  /* Lowest and highest pixels partially inside the slit */
1765  ylo = uves_round_double(ycenter - slit_length/2 - 0.5*slope);
1766  yhi = uves_round_double(ycenter + slit_length/2 + 0.5*slope);
1767 
1768  /* If part of the bin is outside the image... */
1769  if (ylo < 1 || ny < yhi)
1770  {
1771  if (extract_partial)
1772  {
1773  ylo = uves_max_int(ylo, 1);
1774  yhi = uves_min_int(yhi, ny);
1775  }
1776  else
1777  {
1778  /* Don't extract the bin if 'extract_partial' is false */
1779  ylo = yhi + 1;
1780  }
1781  }
1782 
1783  /* Extract */
1784  for (y = ylo; y <= yhi; y++) {
1785  /* Calculate area of pixel inside order */
1786  int pis_rejected;
1787  double pixelval;
1788  double pixelvariance;
1789  double weight;
1790 
1791  /* Read pixel flux */
1792  pixelval = cpl_image_get(image, x, y, &pis_rejected);
1793 
1794  /* Uncomment to disallow negative fluxes
1795  assure( MIDAS || pis_rejected || pixelval >= 0, CPL_ERROR_ILLEGAL_INPUT,
1796  "Negative flux: %e at (x, y) = (%d, %d)", pixelval, x, y);
1797  */
1798 
1799  /* Read pixel noise */
1800  if (spectrum_noise != NULL && !pis_rejected)
1801  {
1802  pixelvariance = cpl_image_get(image_noise, x, y, &pis_rejected);
1803  pixelvariance *= pixelvariance;
1804  }
1805  else
1806  {
1807  pixelvariance = 1;
1808  }
1809 
1810  if (!pis_rejected) {
1811  /* Get weight */
1812  if (method == EXTRACT_WEIGHTED)
1813  {
1814  /* Use already defined weight
1815  (from previous optimal extraction) */
1816 
1817  weight = cpl_image_get(weights, x, y, &pis_rejected);
1818 
1819  assure( weight >= 0, CPL_ERROR_ILLEGAL_INPUT,
1820  "Illegal weight: %e at (x, y) = (%d, %d)",
1821  weight, x, y);
1822 
1823  if (weight == 0)
1824  {
1825  /* To avoid ~100 MB log file this is commented out:
1826  uves_msg_debug("Ignoring bad pixel at (order, x, y) "
1827  "= (%d, %d, %d)", order, x, y);
1828  */
1829  }
1830  }
1831  else if (method == EXTRACT_ARCLAMP) {
1832  weight = 1.0 / pixelvariance;
1833  }
1834  else {
1835  /* Linear / average extraction */
1836  double area_outside_order_top;
1837  double area_outside_order_bottom;
1838  double left = ycenter + slit_length/2 - 0.5*slope;
1839  double right = ycenter + slit_length/2 + 0.5*slope;
1840 
1841  check( area_outside_order_top =
1842  area_above_line(y, left, right),
1843  "Error calculating area");
1844 
1845  left = ycenter - slit_length/2 - 0.5*slope;
1846  right = ycenter - slit_length/2 + 0.5*slope;
1847 
1848  check( area_outside_order_bottom =
1849  1 - area_above_line(y, left, right),
1850  "Error calculationg area");
1851 
1852  weight = 1 - (area_outside_order_top + area_outside_order_bottom);
1853 
1854  if (1 < y && y < ny && weight < 1)
1855  {
1856  /* Interpolate the flux profile at edge of slit */
1857 
1858  /* Use a piecewise linear profile like this
1859  *
1860  * C
1861  * intrp.profile => / \
1862  * ---/---\-- <= measured pixelval
1863  * | / \|
1864  * |/ B
1865  * A |________ <= measured (integrated) profile
1866  * /|
1867  * __________|
1868  *
1869  * The flux levels A and B are midway between the
1870  * current pixel flux and its neighbours' levels.
1871  * C is chosen so that the integrated over the
1872  * current pixel is consistent with the measured flux.
1873  *
1874  * This guess profile is continous as well as flux conserving
1875  */
1876 
1877  int pis_rejected_prev, pis_rejected_next;
1878 
1879  /* Define flux at pixel borders (A and B) as
1880  mean value of this and neighbouring pixel */
1881  double flux_minus = (pixelval + cpl_image_get(
1882  image, x, y - 1, &pis_rejected_prev)) / 2.0;
1883  double flux_plus = (pixelval + cpl_image_get(
1884  image, x, y + 1, &pis_rejected_next)) / 2.0;
1885  if (!pis_rejected_prev && !pis_rejected_next)
1886  {
1887  /* Define flux at pixel center, fluxc, so that the average
1888  * flux is equal to the measured value 'pixelval':
1889  *
1890  * ((flux- + fluxc)/2 + (flux+ + fluxc)/2) / 2 = pixelval
1891  * => flux- + flux+ + 2fluxc = 4pixelval
1892  * => fluxc = ...
1893  */
1894 
1895  double flux_center =
1896  2*pixelval - (flux_minus + flux_plus) / 2.0;
1897 
1898  /* Line slopes */
1899  double slope_minus =
1900  (flux_center - flux_minus )/ 0.5;
1901  double slope_plus =
1902  (flux_plus - flux_center) / 0.5;
1903 
1904  /* Define interval in [-0.5 ; 0] . Pixel center is at 0.*/
1905  double lo1 =
1906  uves_min_double(0, -0.5 + area_outside_order_bottom);
1907  double hi1 =
1908  uves_min_double(0, 0.5 - area_outside_order_top );
1909  double dy1 = hi1-lo1;
1910 
1911  /* Define interval in [0 ; 0.5] */
1912  double lo2 =
1913  uves_max_double(0, -0.5 + area_outside_order_bottom);
1914  double hi2 =
1915  uves_max_double(0, 0.5 - area_outside_order_top );
1916  double dy2 = hi2-lo2;
1917 
1918  if (dy1 + dy2 > 0)
1919  {
1920  /* Get average flux over the two intervals */
1921  pixelval = (
1922  (flux_center + slope_minus * (lo1+hi1)/2.0) * dy1
1923  +
1924  (flux_center + slope_plus * (lo2+hi2)/2.0) * dy2
1925  ) / (dy1 + dy2);
1926 
1927  /* Don't update/interpolate 'pixelvariance'
1928  * correspondingly (for simplicity) .
1929  */
1930  }
1931  /* else { don't change pixelval } */
1932  }/* Neighbours are good */
1933  }/* Profile interpolation */
1934  else
1935  {
1936  /* Neighbours not available, don't change flux */
1937  }
1938  } /* Get weight */
1939 
1940  /*
1941  * Accumulate weighted sum (linear/average):
1942  *
1943  * Flux = [ sum weight_i * flux_i ]
1944  * Variance = [ sum weight_i^2 * variance_i ]
1945  *
1946  * Arclamp:
1947  *
1948  * Flux = [ sum flux_i / variance_i ] /
1949  * [ sum 1 / variance_i ]
1950  * Variance = 1 /
1951  * = [ sum 1 / variance_i ]
1952  *
1953  * For the entire order, accumulate
1954  *
1955  * Flux_y = [ sum weight_i * flux_i * (y-ymin) ]
1956  * Flux_yy = [ sum weight_i * flux_i * (y-ymin)^2 ]
1957  * Flux_tot = [ sum weight_i * flux_i ]
1958  */
1959 
1960  flux += weight*pixelval;
1961  flux_variance += weight*weight * pixelvariance;
1962  sum += weight;
1963 
1964  /* For measuring object position + FWHM */
1965 
1966  if (method != EXTRACT_ARCLAMP)
1967  {
1968  flux_y += weight * pixelval * (y-ylo);
1969  flux_yy += weight * pixelval * (y-ylo)*(y-ylo);
1970  flux_tot+= weight * pixelval;
1971  }
1972  }/* If pixel was good */
1973  }/* for y ... */
1974 
1975  /* This debugging message significantly increases the execution time
1976  * uves_msg_debug("Order %d, x=%d: %d - %d pixels = %f flux = %f",
1977  order, x, ylo, yhi, sum, flux);
1978  */
1979 
1980  /* If any pixels were extracted */
1981  if (sum > 0)
1982  {
1983  bins_extracted += 1;
1984 
1985  if (method == EXTRACT_ARCLAMP && flux_variance > 0) {
1986  flux *= 1.0 / sum;
1987  flux_variance = 1.0 / sum;
1988  }
1989  else if (method == EXTRACT_AVERAGE || method == EXTRACT_WEIGHTED)
1990  {
1991  /* Divide by sum of weights */
1992  flux *= 1.0 / sum;
1993  flux_variance *= 1.0 / (sum*sum);
1994  }
1995  else {
1996  /* Linear extraction */
1997 
1998  /* Normalize to slit length in the case of bad pixels */
1999  flux *= slit_length / sum;
2000  flux_variance *= (slit_length*slit_length) / (sum*sum);
2001  }
2002 
2003  /* Write result */
2004 
2005  /* This will make the spectrum bad map pointer invalid:
2006  check( cpl_image_set(spectrum, x, spectrum_row, flux),
2007  "Could not write extracted flux at (%d, %d)", x, spectrum_row);
2008  */
2009  spectrum_data [(x-1) + (spectrum_row-1) * nx] = flux;
2010  spectrum_badmap[(x-1) + (spectrum_row-1) * nx] = CPL_BINARY_0;
2011 
2012  if (spectrum_noise != NULL)
2013  {
2014  check( cpl_image_set(
2015  spectrum_noise, x, spectrum_row, sqrt(flux_variance)),
2016  "Could not write noise at (%d, %d)", x, spectrum_row);
2017  }
2018 
2019  check_nomsg( cpl_table_set_double(
2020  signal_to_noise, "SN", sn_row, flux / sqrt(flux_variance)) );
2021  sn_row++;
2022 
2023  }/* if sum... */
2024  else
2025  {
2026  /* Nothing extracted, reject bin */
2027 
2028  /* This is slow:
2029  check( cpl_image_reject(spectrum, x, spectrum_row),
2030  "Could not reject bin at (x, row) = (%d, %d)", x, spectrum_row);
2031 
2032  if (spectrum_noise != NULL)
2033  {
2034  check( cpl_image_reject(spectrum_noise, x, spectrum_row),
2035  "Could not reject bin at (x, row) = (%d, %d)", x, spectrum_row);
2036  }
2037  */
2038 
2039  spectrum_badmap[(x-1) + (spectrum_row-1) * nx] = CPL_BINARY_1;
2040  }
2041 
2042  }/* for x... */
2043 
2044  if (info_tbl != NULL && *info_tbl != NULL && method != EXTRACT_ARCLAMP)
2045  {
2046  double objpos = 0;
2047  double fwhm =0;
2048  if(flux_tot != 0) {
2049  objpos = flux_y / flux_tot;
2050  } else {
2051  objpos = -1; //we set to a negative value, which won't affect
2052  //the median of positive values
2053  }
2054  if (flux_yy/flux_tot - objpos*objpos >= 0)
2055  {
2056  fwhm = sqrt(flux_yy/flux_tot - objpos*objpos) * TWOSQRT2LN2;
2057  }
2058  else
2059  {
2060  fwhm = 0;
2061  }
2062  cpl_table_set_double(*info_tbl, "ObjPosOnSlit" , order - minorder, objpos);
2063  cpl_table_set_double(*info_tbl, "ObjFwhmAvg" , order - minorder, fwhm);
2064  }
2065 
2066  /* Get S/N */
2067  check_nomsg( cpl_table_set_size(signal_to_noise, sn_row) );
2068 
2069  if (sn_row > 0)
2070  {
2071  check_nomsg( *sn = cpl_table_get_column_median(signal_to_noise, "SN"));
2072  }
2073  else
2074  {
2075  *sn = 0;
2076  }
2077 
2078  cleanup:
2079  uves_free_table(&signal_to_noise);
2080  return bins_extracted;
2081 }
2082 
2083 /*----------------------------------------------------------------------------*/
2097 /*----------------------------------------------------------------------------*/
2098 static double
2099 area_above_line(int y, double left, double right)
2100 {
2101  double area = -1; /* Result */
2102  double pixeltop = y + .5; /* Top and bottom edges of pixel */
2103  double pixelbot = y - .5;
2104  double slope = right - left;
2105 
2106  assure( 0 <= slope && slope <= 1, CPL_ERROR_ILLEGAL_INPUT, "Slope is %f", slope);
2107 
2108 /* There are 5 cases to consider
2109 
2110  Case 1:
2111  (line below pixel)
2112  ___
2113  | |
2114  | |
2115  |___|/
2116  /
2117  /
2118  /
2119 
2120  Case 2:
2121  ___
2122  | |
2123  | _|/
2124  |_/_|
2125  /
2126  Case 3:
2127  ___
2128  | _|/
2129  |_/ |
2130  /|___|
2131 
2132  Case 4:
2133  ___
2134  | / |
2135  |/ |
2136  |___|
2137 
2138  Case 5:
2139  (line above pixel)
2140  /
2141  / ___
2142  | |
2143  | |
2144  |___|
2145 
2146 */
2147 
2148  if (pixelbot > right)
2149  { /* 1 */
2150  area = 1;
2151  }
2152  else if (pixelbot > left)
2153  { /* 2. Area of triangle is height^2/(2*line_slope) */
2154  area = 1 -
2155  (right - pixelbot) *
2156  (right - pixelbot) / (2*slope);
2157  }
2158  else if (pixeltop > right)
2159  { /* 3 */
2160  area = pixeltop - (left + right)/2;
2161  }
2162  else if (pixeltop > left)
2163  { /* 4. See 2 */
2164  area =
2165  (pixeltop - left) *
2166  (pixeltop - left) / (2*slope);
2167  }
2168  else
2169  {
2170  /* 5 */
2171  area = 0;
2172  }
2173 
2174  cleanup:
2175  return area;
2176 }
2177 
2178 
2179 /*----------------------------------------------------------------------------*/
2195 /*----------------------------------------------------------------------------*/
2196 
2197 static void
2198 revise_noise(cpl_image *image_noise,
2199  const cpl_binary *image_bpm,
2200  const uves_propertylist *image_header,
2201  uves_iterate_position *pos,
2202  const cpl_image *spectrum,
2203  const cpl_image *sky_spectrum,
2204  const uves_extract_profile *profile,
2205  enum uves_chip chip)
2206 {
2207  cpl_image *revised = NULL;
2208  cpl_image *simulated = NULL;
2209  const cpl_binary *spectrum_bpm =
2210  cpl_mask_get_data_const(cpl_image_get_bpm_const(spectrum));
2211  double *simul_data;
2212  const double *spectrum_data;
2213  const double *sky_data;
2214 
2215  simulated = cpl_image_new(pos->nx, pos->ny,
2216  CPL_TYPE_DOUBLE);
2217  assure_mem( simulated );
2218 
2219  simul_data = cpl_image_get_data_double(simulated);
2220  spectrum_data = cpl_image_get_data_double_const(spectrum);
2221  sky_data = cpl_image_get_data_double_const(sky_spectrum);
2222 
2223  for (uves_iterate_set_first(pos,
2224  1, pos->nx,
2225  pos->minorder, pos->maxorder,
2226  NULL, false);
2227  !uves_iterate_finished(pos);
2229  {
2230  if (SPECTRUM_DATA(spectrum_bpm, pos) == CPL_BINARY_0)
2231  {
2232  /* Need this before calling uves_extract_profile_evaluate() */
2233  uves_extract_profile_set(profile, pos, NULL);
2234 
2235  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
2236  if (ISGOOD(image_bpm, pos))
2237  {
2238  /* Set pixel(x,y) = sky(x) + profile(x,y)*flux(x) */
2239  DATA(simul_data, pos) =
2240  SPECTRUM_DATA(sky_data, pos)/pos->sg.length +
2241  SPECTRUM_DATA(spectrum_data, pos) *
2242  uves_extract_profile_evaluate(profile, pos);
2243  }
2244  }
2245  }
2246 
2247  /* For debugging:
2248  cpl_image_save(simulated, "/tmp/simul.fits", CPL_BPP_IEEE_FLOAT, NULL, CPL_IO_DEFAULT);
2249  */
2250 
2251  {
2252  int ncom = 1; /* no median stacking is involved */
2253 
2254  /* Note! Assumes de-biased, non-flatfielded data */
2255  check( revised = uves_define_noise(simulated,
2256  image_header,
2257  ncom, chip),
2258  "Error computing noise image");
2259  }
2260 
2261  /* Copy relevant parts to the input noise image */
2262  {
2263  double *revised_data = cpl_image_get_data_double(revised);
2264  double *input_data = cpl_image_get_data_double(image_noise);
2265 
2266  for (uves_iterate_set_first(pos,
2267  1, pos->nx,
2268  pos->minorder, pos->maxorder,
2269  image_bpm, true);
2270  !uves_iterate_finished(pos);
2272  {
2273  DATA(input_data, pos) = DATA(revised_data, pos);
2274  }
2275  }
2276 
2277  cleanup:
2278  uves_free_image(&simulated);
2279  uves_free_image(&revised);
2280 
2281  return;
2282 }
2283 
2284 /*----------------------------------------------------------------------------*/
2301 /*----------------------------------------------------------------------------*/
2302 static cpl_image *
2303 opt_extract_sky(const cpl_image *image, const cpl_image *image_noise,
2304  const cpl_image *weights,
2305  uves_iterate_position *pos,
2306  cpl_image *sky_spectrum,
2307  cpl_image *sky_spectrum_noise)
2308 {
2309  cpl_image *sky_subtracted = NULL; /* Result */
2310  cpl_table *sky_map = NULL; /* Bitmap of sky/object (true/false)
2311  pixels */
2312  uves_msg("Defining sky region");
2313 
2314  check( sky_map = opt_define_sky(image, weights,
2315  pos),
2316  "Error determining sky window");
2317 
2318  uves_msg_low("%" CPL_SIZE_FORMAT "/%" CPL_SIZE_FORMAT " sky pixels",
2319  cpl_table_count_selected(sky_map),
2320  cpl_table_get_nrow(sky_map));
2321 
2322  /* Extract the sky */
2323  uves_msg("Subtracting sky (method = median of sky channels)");
2324 
2325  check( sky_subtracted = opt_subtract_sky(image, image_noise, weights,
2326  pos,
2327  sky_map,
2328  sky_spectrum,
2329  sky_spectrum_noise),
2330  "Could not subtract sky");
2331 
2332  cleanup:
2333  uves_free_table(&sky_map);
2334 
2335  return sky_subtracted;
2336 }
2337 
2338 /*----------------------------------------------------------------------------*/
2350 /*----------------------------------------------------------------------------*/
2351 static cpl_table *
2352 opt_define_sky(const cpl_image *image, const cpl_image *weights,
2353  uves_iterate_position *pos)
2354 
2355 {
2356  cpl_table *sky_map = NULL; /* Result */
2357 
2358  cpl_table **resampled = NULL;
2359  int nbins = 0;
2360  int i;
2361 
2362  /* Measure at all orders, resolution = 1 pixel */
2363  check( resampled = opt_sample_spatial_profile(image, weights,
2364  pos,
2365  50, /* stepx */
2366  1, /* sampling resolution */
2367  &nbins),
2368  "Error measuring spatial profile");
2369 
2370  sky_map = cpl_table_new(nbins);
2371  cpl_table_new_column(sky_map, "DY" , CPL_TYPE_INT); /* Bin id */
2372  cpl_table_new_column(sky_map, "Prof", CPL_TYPE_DOUBLE); /* Average profile */
2373 
2374  for (i = 0; i < nbins; i++)
2375  {
2376  cpl_table_set_int(sky_map, "DY" , i, i - nbins/2);
2377  if (cpl_table_has_valid(resampled[i], "Prof"))
2378  {
2379  /* Use 90 percentile. If the median is used, we
2380  will miss the object when the order definition
2381  is not good.
2382 
2383  (The average wouldn't work as we need to reject
2384  cosmic rays.)
2385  */
2386  int row = (cpl_table_get_nrow(resampled[i]) * 9) / 10;
2387 
2388  uves_sort_table_1(resampled[i], "Prof", false);
2389 
2390  cpl_table_set_double(sky_map, "Prof", i,
2391  cpl_table_get_double(resampled[i], "Prof", row, NULL));
2392  }
2393  else
2394  {
2395  cpl_table_set_invalid(sky_map, "Prof", i);
2396  }
2397  }
2398 
2399  /* Fail cleanly in the unlikely case that input image had
2400  too few good pixels */
2401  assure( cpl_table_has_valid(sky_map, "Prof"), CPL_ERROR_DATA_NOT_FOUND,
2402  "Too many (%" CPL_SIZE_FORMAT "/%d ) bad pixels. Could not measure sky profile",
2403  cpl_image_count_rejected(image),
2404  pos->nx * pos->ny);
2405 
2406 
2407  /* Select sky channels = bins where profile < min + 2*(median-min)
2408  * but less than (min+max)/2
2409  */
2410  {
2411  double prof_min = cpl_table_get_column_min(sky_map, "Prof");
2412  double prof_max = cpl_table_get_column_max(sky_map, "Prof");
2413  double prof_med = cpl_table_get_column_median(sky_map, "Prof");
2414  double sky_threshold = prof_min + 2*(prof_med - prof_min);
2415 
2416  sky_threshold = uves_min_double(sky_threshold, (prof_min + prof_max)/2);
2417 
2418  check( uves_plot_table(sky_map, "DY", "Prof",
2419  "Globally averaged spatial profile (sky threshold = %.5f)",
2420  sky_threshold),
2421  "Plotting failed");
2422 
2423  uves_select_table_rows(sky_map, "Prof", CPL_NOT_GREATER_THAN, sky_threshold);
2424  }
2425 
2426  cleanup:
2427  if (resampled != NULL)
2428  {
2429  for (i = 0; i < nbins; i++)
2430  {
2431  uves_free_table(&(resampled[i]));
2432  }
2433  cpl_free(resampled);
2434  }
2435 
2436  return sky_map;
2437 }
2438 
2439 /*----------------------------------------------------------------------------*/
2457 /*----------------------------------------------------------------------------*/
2458 static cpl_table **
2459 opt_sample_spatial_profile(const cpl_image *image, const cpl_image *weights,
2460  uves_iterate_position *pos,
2461  int stepx,
2462  int sampling_factor,
2463  int *nbins)
2464 
2465 {
2466  cpl_table **resampled = NULL; /* Array of tables,
2467  one table per y-bin.
2468  Contains the spatial profile
2469  for each y */
2470  int *resampled_row = NULL; /* First unused row of above */
2471 
2472  const double *image_data;
2473  const double *weights_data;
2474 
2475  assure( stepx >= 1, CPL_ERROR_ILLEGAL_INPUT, "Step size = %d", stepx);
2476  assure( sampling_factor >= 1, CPL_ERROR_ILLEGAL_INPUT,
2477  "Sampling factor = %d", sampling_factor);
2478 
2479  image_data = cpl_image_get_data_double_const(image);
2480  weights_data = cpl_image_get_data_double_const(weights);
2481 
2482  *nbins = uves_extract_profile_get_nbins(pos->sg.length, sampling_factor);
2483 
2484  resampled = cpl_calloc(*nbins, sizeof(cpl_table *));
2485  resampled_row = cpl_calloc(*nbins, sizeof(int));
2486 
2487  assure_mem(resampled );
2488  assure_mem(resampled_row);
2489 
2490  {
2491  int i;
2492  for (i = 0; i < *nbins; i++)
2493  {
2494  resampled[i] = cpl_table_new((pos->nx/stepx+1)*
2495  (pos->maxorder-pos->minorder+1));
2496 
2497  resampled_row[i] = 0;
2498  assure_mem( resampled[i] );
2499 
2500  cpl_table_new_column(resampled[i], "X" , CPL_TYPE_INT);
2501  cpl_table_new_column(resampled[i], "Order", CPL_TYPE_INT);
2502  cpl_table_new_column(resampled[i], "Prof" , CPL_TYPE_DOUBLE);
2503  /* Don't store order number */
2504  }
2505  }
2506 
2507  for (uves_iterate_set_first(pos,
2508  1, pos->nx,
2509  pos->minorder, pos->maxorder,
2510  NULL, false);
2511  !uves_iterate_finished(pos);
2512  uves_iterate_increment(pos)) {
2513  if ((pos->x - 1) % stepx == 0)
2514  /* Look only at bins divisible by stepx */
2515  {
2516  /* Linear extract bin */
2517  double flux = 0;
2518 
2519  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++) {
2520  if (!ISBAD(weights_data, pos)) {
2521  flux += DATA(image_data, pos);
2522  }
2523  }
2524 
2525  if (flux != 0) {
2526  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++) {
2527  if (!ISBAD(weights_data, pos)) {
2528  double f = DATA(image_data, pos);
2529 
2530  /* Nearest bin */
2531  int bin = uves_round_double(
2532  uves_extract_profile_get_bin(pos, sampling_factor));
2533 
2534  passure( bin < *nbins, "%d %d", bin, *nbins);
2535 
2536  /* Here the 'virtual resampling' consists
2537  of simply rounding to the nearest bin
2538  (nearest-neighbour interpolation)
2539  */
2540  cpl_table_set_int (resampled[bin], "X" ,
2541  resampled_row[bin], pos->x);
2542  cpl_table_set_int (resampled[bin], "Order",
2543  resampled_row[bin], pos->order);
2544  cpl_table_set_double(resampled[bin], "Prof" ,
2545  resampled_row[bin], f/flux);
2546 
2547  resampled_row[bin]++;
2548  }
2549  }
2550  }
2551  }
2552  }
2553 
2554  {
2555  int i;
2556  for (i = 0; i < *nbins; i++)
2557  {
2558  cpl_table_set_size(resampled[i], resampled_row[i]);
2559  }
2560  }
2561 
2562  /* This is what we return */
2563  passure( cpl_table_get_ncol(resampled[0]) == 3, "%" CPL_SIZE_FORMAT "",
2564  cpl_table_get_ncol(resampled[0]));
2565  passure( cpl_table_has_column(resampled[0], "X"), " ");
2566  passure( cpl_table_has_column(resampled[0], "Order"), " ");
2567  passure( cpl_table_has_column(resampled[0], "Prof"), " ");
2568 
2569  cleanup:
2570  cpl_free(resampled_row);
2571 
2572  return resampled;
2573 }
2574 
2575 
2576 
2577 /*----------------------------------------------------------------------------*/
2599 /*----------------------------------------------------------------------------*/
2600 static cpl_image *
2601 opt_subtract_sky(const cpl_image *image, const cpl_image *image_noise,
2602  const cpl_image *weights,
2603  uves_iterate_position *pos,
2604  const cpl_table *sky_map,
2605  cpl_image *sky_spectrum,
2606  cpl_image *sky_spectrum_noise)
2607 {
2608  cpl_image *sky_subtracted = cpl_image_duplicate(image); /* Result, bad pixels
2609  are inherited */
2610  double *sky_subtracted_data;
2611  const double *image_data;
2612  const double *noise_data;
2613  const double *weights_data;
2614  double *buffer_flux = NULL; /* These buffers exist for efficiency reasons, to */
2615  double *buffer_noise = NULL; /* avoid malloc/free for every bin */
2616 
2617  /* Needed because cpl_image_set() is slow */
2618  double *sky_spectrum_data = NULL;
2619  double *sky_noise_data = NULL;
2620  cpl_binary *sky_spectrum_bpm = NULL;
2621  cpl_binary *sky_noise_bpm = NULL;
2622  cpl_mask *temp = NULL;
2623 
2624  assure_mem( sky_subtracted );
2625 
2626  image_data = cpl_image_get_data_double_const(image);
2627  noise_data = cpl_image_get_data_double_const(image_noise);
2628  weights_data = cpl_image_get_data_double_const(weights);
2629  sky_subtracted_data = cpl_image_get_data(sky_subtracted);
2630 
2631  buffer_flux = cpl_malloc(uves_round_double(pos->sg.length + 5)*sizeof(double));
2632  buffer_noise = cpl_malloc(uves_round_double(pos->sg.length + 5)*sizeof(double));
2633 
2634 
2635  if (sky_spectrum != NULL)
2636  {
2637  sky_spectrum_data = cpl_image_get_data_double(sky_spectrum);
2638  sky_noise_data = cpl_image_get_data_double(sky_spectrum_noise);
2639 
2640  /* Reject all bins in the extracted sky spectrum,
2641  then mark pixels as good if/when they are calculated later */
2642 
2643  temp = cpl_mask_new(cpl_image_get_size_x(sky_spectrum),
2644  cpl_image_get_size_y(sky_spectrum));
2645  cpl_mask_not(temp); /* Set all pixels to CPL_BINARY_1 */
2646 
2647  cpl_image_reject_from_mask(sky_spectrum , temp);
2648  cpl_image_reject_from_mask(sky_spectrum_noise, temp);
2649 
2650  sky_spectrum_bpm = cpl_mask_get_data(cpl_image_get_bpm(sky_spectrum));
2651  sky_noise_bpm = cpl_mask_get_data(cpl_image_get_bpm(sky_spectrum_noise));
2652  }
2653 
2654  UVES_TIME_START("Subtract sky");
2655 
2656  for (uves_iterate_set_first(pos,
2657  1, pos->nx,
2658  pos->minorder, pos->maxorder,
2659  NULL, false);
2660  !uves_iterate_finished(pos);
2662  {
2663  double sky_background, sky_background_noise;
2664 
2665  /* Get sky */
2666  sky_background = opt_get_sky(image_data, noise_data,
2667  weights_data,
2668  pos,
2669  sky_map,
2670  buffer_flux, buffer_noise,
2671  &sky_background_noise);
2672 
2673  /* Save sky */
2674  if (sky_spectrum != NULL)
2675  {
2676  /* Change normalization of sky from 1 pixel to full slit,
2677  (i.e. same normalization as the extracted object)
2678 
2679  Error propagation is trivial (just multiply
2680  by same factor) because the
2681  uncertainty of 'slit_length' is negligible.
2682  */
2683 
2684  /*
2685  cpl_image_set(sky_spectrum , x, spectrum_row,
2686  slit_length * sky_background);
2687  cpl_image_set(sky_spectrum_noise, x, spectrum_row,
2688  slit_length * sky_background_noise);
2689  */
2690  SPECTRUM_DATA(sky_spectrum_data, pos) =
2691  pos->sg.length * sky_background;
2692  SPECTRUM_DATA(sky_noise_data, pos) =
2693  pos->sg.length * sky_background_noise;
2694 
2695  SPECTRUM_DATA(sky_spectrum_bpm, pos) = CPL_BINARY_0;
2696  SPECTRUM_DATA(sky_noise_bpm , pos) = CPL_BINARY_0;
2697  }
2698 
2699  /* Subtract sky */
2700  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
2701  {
2702  DATA(sky_subtracted_data, pos) =
2703  DATA(image_data, pos) - sky_background;
2704  /* Don't update noise image. Error
2705  on sky determination is small. */
2706 
2707  /* BPM is duplicate of input image */
2708  }
2709  }
2710 
2711  UVES_TIME_END;
2712 
2713  cleanup:
2714  uves_free_mask(&temp);
2715  cpl_free(buffer_flux);
2716  cpl_free(buffer_noise);
2717 
2718  return sky_subtracted;
2719 }
2720 
2721 
2722 /*----------------------------------------------------------------------------*/
2757 /*----------------------------------------------------------------------------*/
2758 
2759 static uves_extract_profile *
2760 opt_measure_profile(const cpl_image *image, const cpl_image *image_noise,
2761  const cpl_image *weights,
2762  uves_iterate_position *pos,
2763  int chunk, int sampling_factor,
2764  int (*f) (const double x[], const double a[], double *result),
2765  int (*dfda)(const double x[], const double a[], double result[]),
2766  int M,
2767  const cpl_image *sky_spectrum,
2768  cpl_table *info_tbl,
2769  cpl_table **profile_global)
2770 {
2771  uves_extract_profile *profile = NULL; /* Result */
2772  int *stepx = NULL; /* per order or per spatial bin */
2773  int *good_bins = NULL; /* per order or per spatial bin */
2774  cpl_table **profile_data = NULL; /* per order or per spatial bin */
2775  bool cont; /* continue? */
2776 
2777  cpl_mask *image_bad = NULL;
2778  cpl_binary*image_bpm = NULL;
2779 
2780  cpl_vector *plot0x = NULL;
2781  cpl_vector *plot0y = NULL;
2782  cpl_vector *plot1x = NULL;
2783  cpl_vector *plot1y = NULL;
2784  cpl_bivector *plot[] = {NULL, NULL};
2785  char *plot_titles[] = {NULL, NULL};
2786 
2787  int sample_bins = 100; /* Is this used?? */
2788 
2789  /* Needed for virtual method */
2790  int spatial_bins = uves_extract_profile_get_nbins(pos->sg.length, sampling_factor);
2791 
2792  /* Convert weights image to bpm needed for 1d_fit.
2793  * The virtual resampling measurement will use the weights image
2794  */
2795  if (f != NULL)
2796  {
2797  image_bad = cpl_mask_new(pos->nx, pos->ny);
2798  assure_mem(image_bad);
2799  image_bpm = cpl_mask_get_data(image_bad);
2800  {
2801  const double *weights_data = cpl_image_get_data_double_const(weights);
2802 
2803  for (pos->y = 1; pos->y <= pos->ny; pos->y++)
2804  {
2805  for (pos->x = 1; pos->x <= pos->nx; pos->x++)
2806  {
2807  if (ISBAD(weights_data, pos))
2808  {
2809  DATA(image_bpm, pos) = CPL_BINARY_1;
2810  }
2811  }
2812  }
2813  }
2814  }
2815 
2816  if (f != NULL)
2817  {
2818  stepx = cpl_malloc((pos->maxorder-pos->minorder+1) * sizeof(int));
2819  good_bins = cpl_malloc((pos->maxorder-pos->minorder+1) * sizeof(int));
2820  profile_data = cpl_calloc( pos->maxorder-pos->minorder+1, sizeof(cpl_table *));
2821 
2822  assure_mem(stepx);
2823  assure_mem(good_bins);
2824  assure_mem(profile_data);
2825 
2826  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++)
2827  {
2828  /*
2829  * Get width of order inside image,
2830  * and set stepx according to the
2831  * total number of sample bins
2832  */
2833  int order_width;
2834 
2835  check( order_width = opt_get_order_width(pos),
2836  "Error estimating width of order #%d", pos->order);
2837 
2838  /* If no bins were rejected, the
2839  step size to use would be
2840  order_width/sample_bins
2841  Add 1 to make stepx always positive
2842  */
2843 
2844  stepx [pos->order-pos->minorder] = order_width / sample_bins + 1;
2845  good_bins[pos->order-pos->minorder] = (2*sample_bins)/3;
2846  }
2847  }
2848  else
2849  {
2850  int i;
2851 
2852  passure( f == NULL, " ");
2853 
2854  stepx = cpl_malloc(sizeof(int) * spatial_bins);
2855  good_bins = cpl_malloc(sizeof(int) * spatial_bins);
2856  /* No, they are currently allocated by opt_sample_spatial_profile:
2857  profile_data = cpl_calloc(spatial_bins, sizeof(cpl_table *));
2858  */
2859  profile_data = NULL;
2860 
2861  assure_mem(stepx);
2862  assure_mem(good_bins);
2863 
2864  for (i = 0; i < spatial_bins; i++)
2865  {
2866  /* Across the full chip we have
2867  nx * norders * sg.ength / stepx
2868  measure positions.
2869  We want (only):
2870  sample_bins * spatial_bins * norders
2871  so stepx = ...
2872  */
2873 /* stepx [i] = uves_round_double(
2874  (pos->nx)*(pos->maxorder-pos->minorder+1)*pos->sg.length)/
2875  (sample_bins*spatial_bins)
2876  ) + 1;
2877 */
2878  stepx [i] = uves_round_double(
2879  (pos->nx*pos->sg.length)/(sample_bins*spatial_bins)
2880  ) + 1;
2881 
2882  good_bins[i] = sample_bins - 1;
2883  }
2884  }
2885 
2886  /* Initialization done */
2887 
2888  /* Measure the object profile.
2889  * Iterate until we have at least 'sample_bins' good
2890  * measure points in each order,
2891  * or until the step size has decreased to 1
2892  *
2893  * For gauss/moffat methods, the profile is measured
2894  * in chunks of fixed size (using all the information
2895  * inside each chunk), and there are no iterations.
2896  *
2897  * For virtual method, the iteration is currently
2898  * not implemented (i.e. also no iterations here)
2899  *
2900  * do
2901  * update stepx
2902  * measure using stepx
2903  * until (for every order (and every spatial bin): good_bins >= sample_bins)
2904  *
2905  * fit global polynomials to profile parameters
2906  */
2907 
2908  do {
2909  /* Update stepx */
2910  int i;
2911 
2912  for (i = 0; i < ((f == NULL) ? spatial_bins : pos->maxorder-pos->minorder+1); i++)
2913  {
2914  if (f == NULL || profile_data[i] == NULL)
2915  /* If we need to measure this order/spatial-bin (again) */
2916  /* fixme: currently no iterations for virtual resampling */
2917  {
2918  passure(good_bins[i] < sample_bins,
2919  "%d %d", good_bins[i], sample_bins);
2920 
2921  stepx[i] = (int) (stepx[i]*(good_bins[i]*0.8/sample_bins));
2922  if (stepx[i] == 0)
2923  {
2924  stepx[i] = 1;
2925  }
2926  /* Example of above formula:
2927  If we need sample_bins=200,
2928  but have only good_bins=150,
2929  then decrease stepsize to 150/200 = 75%
2930  and then by another factor 0.8 (so we are
2931  more likely to end up with a few more
2932  bins than needed, rather than a few less
2933  bins than needed).
2934 
2935  Also note that stepx always decreases, so
2936  the loop terminates.
2937  */
2938  }
2939  }
2940 
2941  cont = false;
2942 
2943  /* Measure */
2944  if (f != NULL) {
2945 #if NEW_METHOD
2946  for (pos->order = pos->minorder; pos->order <= pos->minorder; pos->order++) {
2947 #else
2948  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++) {
2949 #endif
2950  /* Zero resampling */
2951  if (profile_data[pos->order-pos->minorder] == NULL) {
2952  int bins;
2953 
2954  check( profile_data[pos->order-pos->minorder] =
2955  opt_measure_profile_order(image, image_noise, image_bpm,
2956  pos,
2957  chunk,
2958  f, dfda, M,
2959  sky_spectrum),
2960  "Error measuring profile of order #%d using chunk size = %d",
2961  pos->order, chunk);
2962 
2963  bins = cpl_table_get_nrow(profile_data[pos->order-pos->minorder]);
2964 
2965  uves_msg("Order %-2d: Chi^2/N = %.2f; FWHM = %.2f pix; Offset = %.2f pix",
2966  pos->order,
2967  (bins > 0) ? cpl_table_get_column_median(
2968  profile_data[pos->order-pos->minorder],
2969  "Reduced_chisq") : 0,
2970  /* Gaussian: fwhm = 2.35 sigma */
2971  (bins > 0) ? cpl_table_get_column_median(
2972  profile_data[pos->order-pos->minorder],
2973  "Sigma") * TWOSQRT2LN2 : 0,
2974  (bins > 0) ? cpl_table_get_column_median(
2975  profile_data[pos->order-pos->minorder],
2976  "Y0") : 0);
2977 
2978  /* Old way of doing things:
2979  good_bins[pos->order-minorder] = bins;
2980 
2981  Continue if there are not enough good bins for this order
2982  if (good_bins[pos->order-minorder] < sample_bins &&
2983  stepx[pos->order-minorder] >= 2)
2984  {
2985  cont = true;
2986  uves_free_table(&(profile_data[pos->order-minorder]));
2987  }
2988  */
2989 
2990  /* New method */
2991  cont = false;
2992 
2993  } /* if we needed to measure this order again */
2994  }
2995  }
2996  else
2997  /* Virtual method */
2998  {
2999  int nbins = 0;
3000 
3001  int step = 0; /* average of stepx */
3002  for (i = 0; i < spatial_bins; i++)
3003  {
3004  step += stepx[i];
3005  }
3006  step /= spatial_bins;
3007 
3008  *profile_global = cpl_table_new(0);
3009  assure_mem( *profile_global );
3010  cpl_table_new_column(*profile_global, "Dummy" , CPL_TYPE_DOUBLE);
3011 
3012  check( profile_data = opt_sample_spatial_profile(image, weights,
3013  pos,
3014  step,
3015  sampling_factor,
3016  &nbins),
3017  "Error measuring profile (virtual method)");
3018 
3019  passure( nbins == spatial_bins, "%d %d", nbins, spatial_bins);
3020 
3021  for (i = 0; i < spatial_bins; i++)
3022  {
3023  good_bins[i] = cpl_table_get_nrow(profile_data[i]);
3024 
3025  uves_msg_debug("Bin %d (%-3d samples): Prof = %f %d",
3026  i,
3027  good_bins[i],
3028  (good_bins[i] > 0) ?
3029  cpl_table_get_column_median(profile_data[i], "Prof") : 0,
3030  stepx[i]);
3031 
3032  /* Continue if there are not enough measure points for this spatial bin */
3033  //fixme: disabled for now, need to cleanup and only measure
3034  //bins when necessary
3035  //if (false && good_bins[i] < sample_bins && stepx[i] >= 2)
3036  // {
3037  // cont = true;
3038  // uves_free_table(&(profile_data[i]));
3039  // }
3040  }
3041  }
3042 
3043  } while(cont);
3044 
3045 
3046  /* Fit a global polynomial to each profile parameter */
3047  if (f == NULL)
3048  {
3049  int max_degree = 8;
3050  double kappa = 3.0;
3051  int i;
3052 
3053  uves_msg_low("Fitting global polynomials to "
3054  "resampled profile (%d spatial bins)",
3055  spatial_bins);
3056 
3057  uves_extract_profile_delete(&profile);
3058  profile = uves_extract_profile_new(NULL,
3059  NULL,
3060  0,
3061  pos->sg.length,
3062  sampling_factor);
3063 
3064  for (i = 0; i < spatial_bins; i++)
3065  {
3066  /* Do not make the code simpler by:
3067  * int n = cpl_table_get_nrow(profile_data[i]);
3068  * because the table size is generally non-constant
3069  */
3070 
3071  bool enough_points = (
3072  cpl_table_get_nrow(profile_data[i]) >= (max_degree + 1)*(max_degree + 1));
3073 
3074  if (enough_points)
3075  {
3076  uves_msg_debug("Fitting 2d polynomial to spatial bin %d", i);
3077 
3078  if (true) {
3079  /* Clever but slow: */
3080 
3081  double min_reject = -0.01; /* negative value means disabled.
3082  This optimization made the
3083  unit test fail. That should be
3084  investigated before enabling this
3085  optimization (is the unit test too strict?
3086  or does the quality actually decrease?).
3087  A good value is probably ~0.01
3088  */
3089  profile->dy_poly[i] = uves_polynomial_regression_2d_autodegree(
3090  profile_data[i],
3091  "X", "Order", "Prof", NULL,
3092  "Proffit", NULL, NULL, /* new columns */
3093  NULL, NULL, NULL, /* mse, red_chisq, variance */
3094  kappa,
3095  max_degree, max_degree, -1, min_reject,
3096  false, /* verbose? */
3097  NULL, NULL, 0, NULL);
3098  } else {
3099  /* For testing only. Don't do like this. */
3100  /* This is no good at high S/N where a
3101  precise profile measurement is crucial */
3102 
3103  profile->dy_poly[i] =
3104  uves_polynomial_regression_2d(profile_data[i],
3105  "X", "Order", "Prof", NULL,
3106  0, 0,
3107  "Proffit", NULL, NULL, /* new columns */
3108  NULL, NULL, NULL, kappa, -1);
3109  }
3110 
3111  if (cpl_error_get_code() == CPL_ERROR_SINGULAR_MATRIX)
3112  {
3113  uves_error_reset();
3114  uves_msg_debug("Fitting bin %d failed", i);
3115 
3116  uves_polynomial_delete(&(profile->dy_poly[i]));
3117  enough_points = false;
3118  }
3119 
3120  assure( cpl_error_get_code() == CPL_ERROR_NONE,
3121  cpl_error_get_code(),
3122  "Could not fit polynomial to bin %d", i);
3123 
3124  }/* if enough points */
3125 
3126  if (!enough_points)
3127  {
3128  /* Not enough points for fit (usually at edges of slit) */
3129 
3130  profile->dy_poly[i] = uves_polynomial_new_zero(2);
3131 
3132  cpl_table_new_column(profile_data[i], "Proffit", CPL_TYPE_DOUBLE);
3133  if (cpl_table_get_nrow(profile_data[i]) > 0)
3134  {
3135  cpl_table_fill_column_window_double(
3136  profile_data[i], "Proffit",
3137  0, cpl_table_get_nrow(profile_data[i]),
3138  0);
3139  }
3140  }
3141 
3142  /* Optimization:
3143  If zero degree, do quick evaluations later
3144  */
3145  profile->is_zero_degree[i] = (uves_polynomial_get_degree(profile->dy_poly[i]) == 0);
3146  if (profile->is_zero_degree[i])
3147  {
3148  profile->dy_double[i] = uves_polynomial_evaluate_2d(profile->dy_poly[i], 0, 0);
3149  }
3150  } /* for each spatial bin */
3151  }
3152  else
3153  /* Analytical profile */
3154  {
3155  int max_degree;
3156  double min_rms = 0.1; /* pixels, stop if this precision is achieved */
3157  double kappa = 3.0; /* The fits to individual chunks can be noisy (due
3158  to low statistics), so use a rather low kappa */
3159 
3160  bool enough_points; /* True iff the data allows fitting a polynomial */
3161 
3162  /* Merge individual order tables to global table before fitting */
3163  uves_free_table(profile_global);
3164 
3165 #if NEW_METHOD
3166  for (pos->order = pos->minorder; order <= pos->minorder; pos->order++)
3167 #else
3168  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++)
3169 #endif
3170  {
3171  if (pos->order == pos->minorder)
3172  {
3173  *profile_global = cpl_table_duplicate(profile_data[0]);
3174  }
3175  else
3176  {
3177  /* Insert at top */
3178  cpl_table_insert(*profile_global,
3179  profile_data[pos->order-pos->minorder], 0);
3180  }
3181  }
3182 
3183  uves_extract_profile_delete(&profile);
3184  profile = uves_extract_profile_new(f, dfda, M, 0, 0);
3185 
3186  /*
3187  For robustness against
3188  too small (i.e. wrong) uncertainties (which would cause
3189  single points to have extremely high weight 1/sigma^2),
3190  raise uncertainties to median before fitting.
3191  */
3192 
3193  max_degree = 5;
3194 
3195 #if ORDER_PER_ORDER
3196  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++)
3197  {
3198  int degree = 4;
3199 #else
3200 #endif
3201 
3202  enough_points =
3203 #if ORDER_PER_ORDER
3204  (cpl_table_get_nrow(profile_data[pos->order-pos->minorder])
3205  >= (degree + 1));
3206 #else
3207  (cpl_table_get_nrow(*profile_global) >= (max_degree + 1)*(max_degree + 1));
3208 #endif
3209  if (enough_points)
3210  {
3211  double mse;
3212  /* Make sure the fit has sensible values at the following positions */
3213  double min_val = -pos->sg.length/2;
3214  double max_val = pos->sg.length/2;
3215  double minmax_pos[4][2];
3216  minmax_pos[0][0] = 1 ; minmax_pos[0][1] = pos->minorder;
3217  minmax_pos[1][0] = 1 ; minmax_pos[1][1] = pos->maxorder;
3218  minmax_pos[2][0] = pos->nx; minmax_pos[2][1] = pos->minorder;
3219  minmax_pos[3][0] = pos->nx; minmax_pos[3][1] = pos->maxorder;
3220 
3221  uves_msg_low("Fitting profile centroid = polynomial(x, order)");
3222 
3223 #if ORDER_PER_ORDER
3224  check_nomsg( uves_raise_to_median_frac(
3225  profile_data[pos->order-pos->minorder], "dY0", 1.0) );
3226 
3227  profile->y0[pos->order - pos->minorder] =
3229  profile_data[pos->order-pos->minorder],
3230  "X", "Y0", "dY0", degree,
3231  "Y0fit", NULL,
3232  &mse, kappa);
3233 #else
3234  check_nomsg( uves_raise_to_median_frac(*profile_global, "dY0", 1.0) );
3235 
3236  profile->y0 =
3238  *profile_global,
3239  "X", "Order", "Y0", "dY0",
3240  "Y0fit", NULL, NULL,
3241  &mse, NULL, NULL,
3242  kappa,
3243  max_degree, max_degree, min_rms, -1,
3244  true,
3245  &min_val, &max_val, 4, minmax_pos);
3246 #endif
3247  if (cpl_error_get_code() == CPL_ERROR_SINGULAR_MATRIX)
3248  {
3249  uves_error_reset();
3250 #if ORDER_PER_ORDER
3251  uves_polynomial_delete(&(profile->y0[pos->order - pos->minorder]));
3252 #else
3253  uves_polynomial_delete(&(profile->y0));
3254 #endif
3255 
3256  enough_points = false;
3257  }
3258  else
3259  {
3260  assure( cpl_error_get_code() == CPL_ERROR_NONE,
3261  cpl_error_get_code(),
3262  "Error fitting object position");
3263 
3264  /* Fit succeeded */
3265 #if ORDER_PER_ORDER
3266 #else
3267  uves_msg_low("Object offset at chip center = %.2f pixels",
3269  profile->y0,
3270  pos->nx/2,
3271  (pos->minorder+pos->maxorder)/2));
3272 #endif
3273 
3274  if (sqrt(mse) > 0.5) /* Pixels */
3275  {
3276  uves_msg_warning("Problem localizing object "
3277  "(usually RMS ~= 0.1 pixels)");
3278  }
3279  }
3280  }
3281 
3282  if (!enough_points)
3283  {
3284 #if ORDER_PER_ORDER
3285  uves_msg_warning("Too few points (%d) to fit global polynomial to "
3286  "object centroid. Setting offset to zero",
3287  cpl_table_get_nrow(profile_data[pos->order - pos->minorder]));
3288 #else
3289  uves_msg_warning("Too few points (%" CPL_SIZE_FORMAT ") to fit global polynomial to "
3290  "object centroid. Setting offset to zero",
3291  cpl_table_get_nrow(*profile_global));
3292 #endif
3293 
3294  /* Set y0(x, m) := 0 */
3295 #if ORDER_PER_ORDER
3296  profile->y0[pos->order - pos->minorder] = uves_polynomial_new_zero(1);
3297 
3298  cpl_table_new_column(profile_data[pos->order-pos->minorder], "Y0fit", CPL_TYPE_DOUBLE);
3299  if (cpl_table_get_nrow(profile_data[pos->order-pos->minorder]) > 0)
3300  {
3301  cpl_table_fill_column_window_double(
3302  profile_data[pos->order-pos->minorder], "Y0fit",
3303  0, cpl_table_get_nrow(profile_data[pos->order-pos->minorder]),
3304  0);
3305  }
3306 #else
3307  profile->y0 = uves_polynomial_new_zero(2);
3308 
3309  cpl_table_new_column(*profile_global, "Y0fit", CPL_TYPE_DOUBLE);
3310  if (cpl_table_get_nrow(*profile_global) > 0)
3311  {
3312  cpl_table_fill_column_window_double(
3313  *profile_global, "Y0fit",
3314  0, cpl_table_get_nrow(*profile_global),
3315  0);
3316  }
3317 #endif
3318  }
3319 #if ORDER_PER_ORDER
3320  } /* for order */
3321 #else
3322 #endif
3323  max_degree = 3;
3324 
3325 #if ORDER_PER_ORDER
3326  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++)
3327  {
3328  int degree = 4;
3329 #else
3330 #endif
3331  enough_points =
3332 #if ORDER_PER_ORDER
3333  (cpl_table_get_nrow(profile_data[pos->order-pos->minorder])
3334  >= (degree + 1));
3335 #else
3336  (cpl_table_get_nrow(*profile_global) >= (max_degree + 1)*(max_degree + 1));
3337 #endif
3338  if (enough_points)
3339  {
3340  double min_val = 0.1;
3341  double max_val = pos->sg.length;
3342  double minmax_pos[4][2];
3343  minmax_pos[0][0] = 1 ; minmax_pos[0][1] = pos->minorder;
3344  minmax_pos[1][0] = 1 ; minmax_pos[1][1] = pos->maxorder;
3345  minmax_pos[2][0] = pos->nx; minmax_pos[2][1] = pos->minorder;
3346  minmax_pos[3][0] = pos->nx; minmax_pos[3][1] = pos->maxorder;
3347 
3348  uves_msg_low("Fitting profile width = polynomial(x, order)");
3349 
3350 #if ORDER_PER_ORDER
3351  check_nomsg( uves_raise_to_median_frac(
3352  profile_data[pos->order-pos->minorder], "dSigma", 1.0) );
3353 
3354 
3355  profile->sigma[pos->order - pos->minorder] =
3357  profile_data[pos->order-pos->minorder],
3358  "X", "Sigma", "dSigma", degree,
3359  "Sigmafit", NULL,
3360  NULL, kappa);
3361 #else
3362  check_nomsg( uves_raise_to_median_frac(*profile_global, "dSigma", 1.0) );
3363 
3364  profile->sigma =
3366  *profile_global,
3367  "X", "Order", "Sigma", "dSigma",
3368  "Sigmafit", NULL, NULL,
3369  NULL, NULL, NULL,
3370  kappa,
3371  max_degree, max_degree, min_rms, -1,
3372  true,
3373  &min_val, &max_val, 4, minmax_pos);
3374 #endif
3375 
3376  if (cpl_error_get_code() == CPL_ERROR_SINGULAR_MATRIX)
3377  {
3378  uves_error_reset();
3379 #if ORDER_PER_ORDER
3380  uves_polynomial_delete(&(profile->sigma[pos->order - pos->minorder]));
3381 #else
3382  uves_polynomial_delete(&(profile->sigma));
3383 #endif
3384 
3385  enough_points = false;
3386  }
3387  else
3388  {
3389  assure( cpl_error_get_code() == CPL_ERROR_NONE,
3390  cpl_error_get_code(),
3391  "Error fitting profile width");
3392 
3393 #if ORDER_PER_ORDER
3394 #else
3395  uves_msg_low("Profile FWHM at chip center = %.2f pixels",
3396  TWOSQRT2LN2 * uves_polynomial_evaluate_2d(
3397  profile->sigma,
3398  pos->nx/2,
3399  (pos->minorder+pos->maxorder)/2));
3400 #endif
3401  }
3402  }
3403 
3404  if (!enough_points)
3405  {
3406 #if ORDER_PER_ORDER
3407  uves_msg_warning("Too few points (%d) to fit global polynomial to "
3408  "object width. Setting std.dev. to 1 pixel",
3409  cpl_table_get_nrow(profile_data[pos->order - pos->minorder]));
3410 #else
3411  uves_msg_warning("Too few points (%" CPL_SIZE_FORMAT ") to fit global polynomial to "
3412  "object width. Setting std.dev. to 1 pixel",
3413  cpl_table_get_nrow(*profile_global));
3414 #endif
3415 
3416  /* Set sigma(x, m) := 1 */
3417 #if ORDER_PER_ORDER
3418  profile->sigma[pos->order - pos->minorder] = uves_polynomial_new_zero(1);
3419  uves_polynomial_shift(profile->sigma[pos->order - pos->minorder], 0, 1.0);
3420 
3421  cpl_table_new_column(profile_data[pos->order-pos->minorder], "Sigmafit", CPL_TYPE_DOUBLE);
3422  if (cpl_table_get_nrow(profile_data[pos->order-pos->minorder]) > 0)
3423  {
3424  cpl_table_fill_column_window_double(
3425  profile_data[pos->order-pos->minorder], "Sigmafit",
3426  0, cpl_table_get_nrow(profile_data[pos->order-pos->minorder]),
3427  1.0);
3428  }
3429 #else
3430  profile->sigma = uves_polynomial_new_zero(2);
3431  uves_polynomial_shift(profile->sigma, 0, 1.0);
3432 
3433  cpl_table_new_column(*profile_global, "Sigmafit", CPL_TYPE_DOUBLE);
3434  if (cpl_table_get_nrow(*profile_global) > 0)
3435  {
3436  cpl_table_fill_column_window_double(
3437  *profile_global, "Sigmafit",
3438  0, cpl_table_get_nrow(*profile_global),
3439  1.0);
3440  }
3441 #endif
3442 
3443  }
3444 
3445  /* Don't fit a 2d polynomial to chi^2/N. Just use a robust average
3446  (i.e. a (0,0) degree polynomial) */
3447 
3448 #if ORDER_PER_ORDER
3449  profile->red_chisq[pos->order - pos->minorder] = uves_polynomial_new_zero(1);
3450  uves_polynomial_shift(profile->red_chisq[pos->order - pos->minorder], 0,
3451  cpl_table_get_nrow(profile_data[pos->order - pos->minorder]) > 0 ?
3452  cpl_table_get_column_median(profile_data[pos->order - pos->minorder],
3453  "Reduced_chisq") : 1.0);
3454 #else
3455  profile->red_chisq = uves_polynomial_new_zero(2);
3456  uves_polynomial_shift(profile->red_chisq, 0,
3457  cpl_table_get_nrow(*profile_global) > 0 ?
3458  cpl_table_get_column_median(*profile_global,
3459  "Reduced_chisq") : 1.0);
3460 #endif
3461 
3462  /*
3463  if (cpl_table_get_nrow(*profile_global) >= (max_degree + 1)*(max_degree + 1))
3464  {
3465  uves_msg_low("Fitting chi^2/N = polynomial(x, order)");
3466 
3467  check( profile->red_chisq =
3468  uves_polynomial_regression_2d_autodegree(
3469  *profile_global,
3470  "X", "Order", "Reduced_chisq", NULL,
3471  NULL, NULL, NULL,
3472  NULL, NULL, NULL,
3473  kappa,
3474  max_degree, max_degree, -1, true),
3475  "Error fitting chi^2/N");
3476  }
3477  else
3478  {
3479  uves_msg_warning("Too few points (%d) to fit global polynomial to "
3480  "chi^2/N. Setting chi^2/N to 1",
3481  cpl_table_get_nrow(*profile_global));
3482 
3483  profile->red_chisq = uves_polynomial_new_zero(2);
3484  uves_polynomial_shift(profile->red_chisq, 0, 1.0);
3485  }
3486  */
3487 #if ORDER_PER_ORDER
3488  } /* for order */
3489 
3490  /* Make sure the global table is consistent */
3491  uves_free_table(profile_global);
3492  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++)
3493  {
3494  if (pos->order == pos->minorder)
3495  {
3496  *profile_global = cpl_table_duplicate(profile_data[0]);
3497  }
3498  else
3499  {
3500  /* Insert at top */
3501  cpl_table_insert(*profile_global,
3502  profile_data[pos->order-pos->minorder], 0);
3503  }
3504  }
3505 #else
3506 #endif
3507 
3508  } /* if f != NULL */
3509 
3510  /* Done fitting */
3511 
3512  /* Plot inferred profile at center of chip */
3513  {
3514  int xmin = uves_max_int(1 , pos->nx/2-100);
3515  int xmax = uves_min_int(pos->nx, pos->nx/2+100);
3516  int order = (pos->minorder + pos->maxorder)/2;
3517  int indx;
3518 
3519  plot0x = cpl_vector_new(uves_round_double(pos->sg.length+5)*(xmax-xmin+1));
3520  plot0y = cpl_vector_new(uves_round_double(pos->sg.length+5)*(xmax-xmin+1));
3521  plot1x = cpl_vector_new(uves_round_double(pos->sg.length+5)*(xmax-xmin+1));
3522  plot1y = cpl_vector_new(uves_round_double(pos->sg.length+5)*(xmax-xmin+1));
3523  indx = 0;
3524  assure_mem( plot0x );
3525  assure_mem( plot0y );
3526  assure_mem( plot1x );
3527  assure_mem( plot1y );
3528 
3529  for (uves_iterate_set_first(pos,
3530  xmin, xmax,
3531  order, order,
3532  NULL, false);
3533  !uves_iterate_finished(pos);
3535 
3536  {
3537  /* Linear extract (to enable plotting raw profile) */
3538  double flux = 0;
3539  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
3540  {
3541  int pis_rejected;
3542  double pixelval = cpl_image_get(image, pos->x, pos->y, &pis_rejected);
3543  if (!pis_rejected)
3544  {
3545  flux += pixelval;
3546  }
3547  }
3548 
3549  uves_extract_profile_set(profile, pos, NULL);
3550 
3551  /* Get empirical and model profile */
3552  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
3553  {
3554  double dy = pos->y - pos->ycenter;
3555  int pis_rejected;
3556  double pixelval = cpl_image_get(
3557  image, pos->x, uves_round_double(pos->y), &pis_rejected);
3558 
3559  if (!pis_rejected && flux != 0)
3560  {
3561  pixelval /= flux;
3562  }
3563  else
3564  {
3565  pixelval = 0; /* Plot something anyway, if pixel is bad */
3566  }
3567 
3568  cpl_vector_set(plot0x, indx, dy);
3569  cpl_vector_set(plot0y, indx, uves_extract_profile_evaluate(profile, pos));
3570 
3571  cpl_vector_set(plot1x, indx, dy);
3572  cpl_vector_set(plot1y, indx, pixelval);
3573 
3574  indx++;
3575  }
3576  }
3577 
3578  if (indx > 0)
3579  {
3580  cpl_vector_set_size(plot0x, indx);
3581  cpl_vector_set_size(plot0y, indx);
3582  cpl_vector_set_size(plot1x, indx);
3583  cpl_vector_set_size(plot1y, indx);
3584 
3585  plot[0] = cpl_bivector_wrap_vectors(plot0x, plot0y);
3586  plot[1] = cpl_bivector_wrap_vectors(plot1x, plot1y);
3587 
3588  plot_titles[0] = uves_sprintf(
3589  "Model spatial profile at (order, x) = (%d, %d)", order, pos->nx/2);
3590  plot_titles[1] = uves_sprintf(
3591  "Empirical spatial profile at (order, x) = (%d, %d)", order, pos->nx/2);
3592 
3593  check( uves_plot_bivectors(plot, plot_titles, 2, "DY", "Profile"), "Plotting failed");
3594  }
3595  else
3596  {
3597  uves_msg_warning("No points to plot. This may happen if the order "
3598  "polynomial is ill-formed");
3599  }
3600  } /* end plotting */
3601 
3602  if (f != NULL)
3603  {
3604  /*
3605  * Create column 'y0fit_world' (fitted value in absolute coordinate),
3606  * add order location center to y0fit
3607  */
3608  int i;
3609 
3610  for (i = 0; i < cpl_table_get_nrow(*profile_global); i++)
3611  {
3612  double y0fit = cpl_table_get_double(*profile_global, "Y0fit", i, NULL);
3613  int order = cpl_table_get_int (*profile_global, "Order", i, NULL);
3614  int x = cpl_table_get_int (*profile_global, "X" , i, NULL);
3615 
3616  /* This will calculate ycenter */
3618  x, x,
3619  order, order,
3620  NULL,
3621  false);
3622 
3623  cpl_table_set_double(*profile_global, "Y0fit_world", i, y0fit + pos->ycenter);
3624  }
3625 
3626  /* Warn about bad detection */
3627 #if NEW_METHOD
3628  for (pos->order = pos->minorder; pos->order <= pos->minorder; pos->order++)
3629 #else
3630  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++)
3631 #endif
3632  {
3633  if (good_bins[pos->order-pos->minorder] == 0)
3634  {
3635  uves_msg_warning("Order %d: Failed to detect object!", pos->order);
3636  }
3637  }
3638 
3639  /* Store parameters for QC
3640  (in virtual mode these are calculated elsewhere) */
3641  for (pos->order = pos->minorder; pos->order <= pos->maxorder; pos->order++)
3642  {
3643 #if ORDER_PER_ORDER
3644  double objpos=0;
3645  check_nomsg(
3646  objpos =
3647  uves_polynomial_evaluate_1d(profile->y0[pos->order-pos->minorder],
3648  pos->nx/2)
3649  - ( - pos->sg.length/2 ));
3650  double fwhm =0;
3651  check_nomsg(fwhm=uves_polynomial_evaluate_1d(profile->sigma[pos->order-pos->minorder],
3652  pos->nx/2) * TWOSQRT2LN2);
3653 
3654 
3655  check_nomsg(cpl_table_set_double(info_tbl, "ObjPosOnSlit" , pos->order - pos->minorder, objpos));
3656  check_nomsg(cpl_table_set_double(info_tbl, "ObjFwhmAvg" , pos->order - pos->minorder, fwhm));
3657 #else
3658  double objpos = 0;
3659  check_nomsg(objpos=uves_polynomial_evaluate_2d(profile->y0,
3660  pos->nx/2, pos->order)
3661  - ( - pos->sg.length/2 ));
3662  double fwhm = 0;
3663  check_nomsg(fwhm=uves_polynomial_evaluate_2d(profile->sigma ,
3664  pos->nx/2, pos->order)*
3665  TWOSQRT2LN2);
3666 
3667  check_nomsg(cpl_table_set_double(info_tbl, "ObjPosOnSlit" , pos->order - pos->minorder, objpos));
3668  check_nomsg(cpl_table_set_double(info_tbl, "ObjFwhmAvg" , pos->order - pos->minorder, fwhm));
3669 #endif
3670  }
3671 
3672  /* Quality check on assumed profile (good fit: red.chisq ~= 1) */
3673  if (cpl_table_get_nrow(*profile_global) > 0)
3674  {
3675  double med_chisq = cpl_table_get_column_median(
3676  *profile_global, "Reduced_chisq");
3677  double limit = 5.0;
3678 
3679  if (med_chisq > limit || med_chisq < 1/limit)
3680  {
3681  /* The factor 5 is somewhat arbitrary.
3682  * As an empirical fact, red_chisq ~= 1 for
3683  * virtually resampled profiles (high and low
3684  * S/N). This indicates that 1) the noise
3685  * model and 2) the inferred profile are
3686  * both correct. (If one or both of them
3687  * were wrong it would a strange coincidence
3688  * that we get red_chisq ~= 1.)
3689  */
3690  uves_msg_warning("Assumed spatial profile might not be a "
3691  "good fit to the data: median(Chi^2/N) = %f",
3692  med_chisq);
3693 
3694  if (f != NULL && med_chisq > limit)
3695  {
3696  uves_msg_warning("Recommended profile "
3697  "measuring method: virtual");
3698  }
3699  }
3700  else
3701  {
3702  uves_msg("Median(reduced Chi^2) is %f", med_chisq);
3703  }
3704  }
3705  }
3706  else
3707  {
3708  /* fixme: calculate and report chi^2 (requires passing noise image
3709  to the profile sampling function) */
3710  }
3711 
3712  cleanup:
3713  uves_free_mask(&image_bad);
3714  cpl_free(stepx);
3715  cpl_free(good_bins);
3716  if (profile_data != NULL)
3717  {
3718  int i;
3719  for (i = 0; i < ((f == NULL) ? spatial_bins : pos->maxorder-pos->minorder+1); i++)
3720  {
3721  if (profile_data[i] != NULL)
3722  {
3723  uves_free_table(&(profile_data[i]));
3724  }
3725  }
3726  cpl_free(profile_data);
3727  }
3728  cpl_bivector_unwrap_vectors(plot[0]);
3729  cpl_bivector_unwrap_vectors(plot[1]);
3730  cpl_free(plot_titles[0]);
3731  cpl_free(plot_titles[1]);
3732  uves_free_vector(&plot0x);
3733  uves_free_vector(&plot0y);
3734  uves_free_vector(&plot1x);
3735  uves_free_vector(&plot1y);
3736 
3737  return profile;
3738 }
3739 
3740 #if NEW_METHOD
3741 struct
3742 {
3743  double *flux; /* Array [0..nx][minorder..maxorder] x = 0 is not used */
3744  double *sky; /* As above */
3745  int minorder, nx; /* Needed for indexing of arrays above */
3746 
3747  int (*f) (const double x[], const double a[], double *result);
3748  int (*dfda)(const double x[], const double a[], double result[]);
3749 
3750  int deg_y0_x;
3751  int deg_y0_m;
3752  int deg_sigma_x;
3753  int deg_sigma_m;
3754 } profile_params;
3755 
3756 /*
3757  Evaluate 2d polynomial
3758  degrees must be zero or more
3759 */
3760 static double
3761 eval_pol(const double *coeffs,
3762  int degree1, int degree2,
3763  double x1, double x2)
3764 {
3765  double result = 0;
3766  double x2j; /* x2^j */
3767  int j;
3768 
3769  for (j = 0, x2j = 1;
3770  j <= degree2;
3771  j++, x2j *= x2)
3772  {
3773  /* Use Horner's scheme to sum the coefficients
3774  involving x2^j */
3775 
3776  int i = degree1;
3777  double r = coeffs[i + (degree1+1)*j];
3778 
3779  while(i > 0)
3780  {
3781  r *= x1;
3782  i -= 1;
3783  r += coeffs[i + (degree1+1)*j];
3784  }
3785 
3786  /* Finished using Horner. Add to grand result */
3787  result += x2j*r;
3788  }
3789 
3790  return result;
3791 }
3792 
3793 /*
3794  @brief evaluate 2d profile
3795  @param x length 3 array of (xi, yi, mi)
3796  @param a all polynomial coefficients
3797  @param result (output) result
3798  @return zero iff success
3799 
3800  This function evaluates
3801 
3802  P(xi, yi ; a) = S_xi + F_xi * (normalized profile)
3803 
3804  using the data in 'profile_params' which must have been
3805  already initialized
3806 */
3807 static int
3808 profile_f(const double x[], const double a[], double *result)
3809 {
3810  int xi = uves_round_double(x[0]);
3811  double yi = x[1];
3812  int mi = uves_round_double(x[2]);
3813  int idx;
3814 
3815  double y_0 = eval_pol(a,
3816  profile_params.deg_y0_x,
3817  profile_params.deg_y0_m,
3818  xi, mi);
3819  double sigma = eval_pol(a + (1 + profile_params.deg_y0_x)*(1 + profile_params.deg_y0_m),
3820  profile_params.deg_sigma_x,
3821  profile_params.deg_sigma_m,
3822  xi, mi);
3823 
3824  /* Now evaluate normalized profile */
3825  double norm_prof;
3826 
3827  double xf[1]; /* Point of evaluation */
3828 
3829  double af[5]; /* Parameters */
3830  af[0] = y_0; /* centroid */
3831  af[1] = sigma; /* stdev */
3832  af[2] = 1; /* norm */
3833  af[3] = 0; /* offset */
3834  af[4] = 0; /* non-linear sky */
3835 
3836  xf[0] = yi;
3837 
3838  if (profile_params.f(xf, af, &norm_prof) != 0)
3839  {
3840  return 1;
3841  }
3842 
3843  idx = xi + (mi - profile_params.minorder)*(profile_params.nx + 1);
3844 
3845  *result = profile_params.sky[idx] + profile_params.flux[idx] * norm_prof;
3846 
3847  return 0;
3848 }
3849 
3850 /*
3851  @brief evaluate 2d profile partial derivatives
3852  @param x length 3 array of (xk, yk, mk)
3853  @param a all polynomial coefficients
3854  @param result (output) result
3855  @return zero iff success
3856 
3857  This function evaluates the partial derivatives
3858  (with respect to the polynomial coefficients) of the function above
3859 
3860  (1) dP/da_ij(xk, yk ; a) = F_xk * d(normalized profile)/dy0 * xk^i mk^j
3861  (2) dP/da_ij(xk, yk ; a) = F_xk * d(normalized profile)/dsigma * xk^ii mk^jj
3862 
3863  (using the chain rule on the 1d profile function)
3864 
3865  Here (1) is used for the coefficients that y0 depend on, i.e.
3866  for (i + (deg_y0_x+1)*j) < (deg_y0_x+1)(deg_y0_m+1)
3867 
3868  and (2) is used for the remaining coefficients which sigma depend on
3869  (ii and jj are appropriate functions of i and j)
3870 
3871 */
3872 static int
3873 profile_dfda(const double x[], const double a[], double result[])
3874 {
3875  int xi = uves_round_double(x[0]);
3876  double yi = x[1];
3877  int mi = uves_round_double(x[2]);
3878 
3879  double y_0 = eval_pol(a,
3880  profile_params.deg_y0_x,
3881  profile_params.deg_y0_m,
3882  xi, mi);
3883  double sigma = eval_pol(a + (1 + profile_params.deg_y0_x)*(1 + profile_params.deg_y0_m),
3884  profile_params.deg_sigma_x,
3885  profile_params.deg_sigma_m,
3886  xi, mi);
3887 
3888  double norm_prof_derivatives[5];
3889 
3890  double xf[1]; /* Point of evaluation */
3891 
3892  double af[5]; /* Parameters */
3893  af[0] = y_0; /* centroid */
3894  af[1] = sigma; /* stdev */
3895  af[2] = 1; /* norm */
3896  af[3] = 0; /* offset */
3897  af[4] = 0; /* non-linear sky */
3898 
3899  xf[0] = yi;
3900 
3901  if (profile_params.dfda(xf, af, norm_prof_derivatives) != 0)
3902  {
3903  return 1;
3904  }
3905 
3906  {
3907  int idx = xi + (mi - profile_params.minorder)*(profile_params.nx + 1);
3908 
3909  /* Need only these two */
3910  double norm_prof_dy0 = norm_prof_derivatives[0];
3911  double norm_prof_dsigma = norm_prof_derivatives[1];
3912  int i, j;
3913 
3914  /* Compute all the derivatives
3915  flux(xk)*df/dy0 * x^i m^j
3916 
3917  It is only the product (x^i m^j) that changes, so use
3918  recurrence to caluculate the coefficients, in
3919  this order (starting from (i,j) = (0,0))):
3920 
3921  (0,0) -> (1,0) -> (2,0) -> ...
3922  V
3923  (0,1) -> (1,1) -> (2,1) -> ...
3924  V
3925  (0,2) -> (1,2) -> (2,2) -> ...
3926  V
3927  :
3928  */
3929  i = 0;
3930  j = 0;
3931  result[i + (profile_params.deg_y0_x + 1) * j] = profile_params.flux[idx] * norm_prof_dy0;
3932  for (j = 0; j <= profile_params.deg_y0_m; j++) {
3933  if (j >= 1)
3934  {
3935  i = 0;
3936  result[i + (profile_params.deg_y0_x + 1) * j] =
3937  result[i + (profile_params.deg_y0_x + 1) * (j-1)] * mi;
3938  }
3939  for (i = 1; i <= profile_params.deg_y0_x; i++) {
3940  result[i + (profile_params.deg_y0_x + 1) * j] =
3941  result[i-1 + (profile_params.deg_y0_x + 1) * j] * xi;
3942  }
3943  }
3944 
3945 
3946  /* Calculate the derivatives flux(xk)*df/dsigma * x^i m^j,
3947  like above (but substituting y0->sigma where relevant).
3948  Insert the derivatives in the result
3949  array starting after the derivatives related to y0,
3950  i.e. at index (deg_y0_x+1)(deg_y0_m+1).
3951  */
3952 
3953  result += (profile_params.deg_y0_x + 1) * (profile_params.deg_y0_m + 1);
3954  /* Pointer arithmetics which skips
3955  the first part of the array */
3956 
3957  i = 0;
3958  j = 0;
3959  result[i + (profile_params.deg_sigma_x + 1) * j] =
3960  profile_params.flux[idx] * norm_prof_dsigma;
3961  for (j = 0; j <= profile_params.deg_sigma_m; j++) {
3962  if (j >= 1)
3963  {
3964  i = 0;
3965  result[i + (profile_params.deg_sigma_x + 1) * j] =
3966  result[i + (profile_params.deg_sigma_x + 1) * (j-1)] * mi;
3967  }
3968  for (i = 1; i <= profile_params.deg_sigma_x; i++) {
3969  result[i + (profile_params.deg_sigma_x + 1) * j] =
3970  result[i-1 + (profile_params.deg_sigma_x + 1) * j] * xi;
3971  }
3972  }
3973  }
3974 
3975  return 0;
3976 }
3977 #endif /* NEW_METHOD */
3978 /*----------------------------------------------------------------------------*/
3998 /*----------------------------------------------------------------------------*/
3999 static cpl_table *
4000 opt_measure_profile_order(const cpl_image *image, const cpl_image *image_noise,
4001  const cpl_binary *image_bpm,
4002  uves_iterate_position *pos,
4003  int chunk,
4004  int (*f) (const double x[], const double a[], double *result),
4005  int (*dfda)(const double x[], const double a[], double result[]),
4006  int M,
4007  const cpl_image *sky_spectrum)
4008 {
4009  cpl_table *profile_data = NULL; /* Result */
4010  int profile_row;
4011  cpl_matrix *covariance = NULL;
4012 
4013 #if NEW_METHOD
4014  cpl_matrix *eval_points = NULL;
4015  cpl_vector *eval_data = NULL;
4016  cpl_vector *eval_err = NULL;
4017  cpl_vector *coeffs = NULL;
4018 #if CREATE_DEBUGGING_TABLE
4019  cpl_table *temp = NULL;
4020 #endif
4021  double *fluxes = NULL;
4022  double *skys = NULL;
4023  int *ia = NULL;
4024  /* For initial estimates of y0,sigma: */
4025  cpl_table *estimate = NULL;
4026  cpl_table *estimate_dup = NULL;
4027  polynomial *y0_estim_pol = NULL;
4028  polynomial *sigma_estim_pol = NULL;
4029 #endif
4030 
4031 
4032  cpl_vector *dy = NULL; /* spatial position */
4033  cpl_vector *prof = NULL; /* normalized profile */
4034  cpl_vector *prof2= NULL; /* kill me */
4035  cpl_vector *dprof = NULL; /* uncertainty of 'prof' */
4036  cpl_vector **data = NULL; /* array of vectors */
4037  int *size = NULL; /* array of vector sizes */
4038  double *hicut = NULL; /* array of vector sizes */
4039  double *locut = NULL; /* array of vector sizes */
4040  int nbins = 0;
4041 
4042  const double *image_data;
4043  const double *noise_data;
4044 
4045  int x;
4046 
4047 #if NEW_METHOD
4048  int norders = pos->maxorder-pos->minorder+1;
4049 #else
4050  /* eliminate warning */
4051  sky_spectrum = sky_spectrum;
4052 #endif
4053 
4054  passure( f != NULL, " ");
4055 
4056  image_data = cpl_image_get_data_double_const(image);
4057  noise_data = cpl_image_get_data_double_const(image_noise);
4058 
4059 #if NEW_METHOD
4060  profile_data = cpl_table_new((nx/chunk + 3) * norders);
4061 #else
4062  profile_data = cpl_table_new(pos->nx);
4063 #endif
4064  assure_mem( profile_data );
4065 
4066  check( (cpl_table_new_column(profile_data, "Order", CPL_TYPE_INT),
4067  cpl_table_new_column(profile_data, "X", CPL_TYPE_INT),
4068  cpl_table_new_column(profile_data, "Y0", CPL_TYPE_DOUBLE),
4069  cpl_table_new_column(profile_data, "Sigma", CPL_TYPE_DOUBLE),
4070  cpl_table_new_column(profile_data, "Norm", CPL_TYPE_DOUBLE),
4071  cpl_table_new_column(profile_data, "dY0", CPL_TYPE_DOUBLE),
4072  cpl_table_new_column(profile_data, "dSigma", CPL_TYPE_DOUBLE),
4073  cpl_table_new_column(profile_data, "dNorm", CPL_TYPE_DOUBLE),
4074  cpl_table_new_column(profile_data, "Y0_world", CPL_TYPE_DOUBLE),
4075  cpl_table_new_column(profile_data, "Y0fit_world", CPL_TYPE_DOUBLE),
4076  cpl_table_new_column(profile_data, "Reduced_chisq", CPL_TYPE_DOUBLE)),
4077  "Error initializing order trace table for order #%d", pos->order);
4078 
4079  /* For msg-output purposes, only */
4080  cpl_table_set_column_unit(profile_data, "X" , "pixels");
4081  cpl_table_set_column_unit(profile_data, "Y0", "pixels");
4082  cpl_table_set_column_unit(profile_data, "Sigma", "pixels");
4083  cpl_table_set_column_unit(profile_data, "dY0", "pixels");
4084  cpl_table_set_column_unit(profile_data, "dSigma", "pixels");
4085 
4086  profile_row = 0;
4087 
4088  UVES_TIME_START("Measure loop");
4089 
4090  nbins = uves_round_double(pos->sg.length + 5); /* more than enough */
4091  data = cpl_calloc(nbins, sizeof(cpl_vector *));
4092  size = cpl_calloc(nbins, sizeof(int));
4093  locut = cpl_calloc(nbins, sizeof(double));
4094  hicut = cpl_calloc(nbins, sizeof(double));
4095  {
4096  int i;
4097  for (i = 0; i < nbins; i++)
4098  {
4099  data[i] = cpl_vector_new(1);
4100  }
4101  }
4102 
4103 
4104 #if NEW_METHOD
4105  /* new method:
4106 
4107  for each order
4108  for each chunk
4109  bin data in spatial bins parallel to order trace
4110  define hicut/locut for each bin
4111  get the data points within locut/hicut
4112 
4113  fit model to all orders
4114  */
4115  {
4116  /* 4 degrees are needed for the model
4117  y0 = pol(x, m)
4118  sigma = pol(x, m)
4119  */
4120  int deg_y0_x = 0;
4121  int deg_y0_m = 0;
4122  int deg_sigma_x = 0;
4123  int deg_sigma_m = 0;
4124 
4125  int ncoeffs =
4126  (deg_y0_x +1)*(deg_y0_m +1) +
4127  (deg_sigma_x+1)*(deg_sigma_m+1);
4128 
4129  double red_chisq;
4130  int n = 0; /* Number of points (matrix rows) */
4131  int nbad = 0; /* Number of hot/cold pixels (full chip) */
4132 
4133 #if CREATE_DEBUGGING_TABLE
4134  temp = cpl_table_new(norders*nx*uves_round_double(pos->sg.length+3));
4135  cpl_table_new_column(temp, "x", CPL_TYPE_DOUBLE);
4136  cpl_table_new_column(temp, "y", CPL_TYPE_DOUBLE);
4137  cpl_table_new_column(temp, "order", CPL_TYPE_DOUBLE);
4138  cpl_table_new_column(temp, "dat", CPL_TYPE_DOUBLE);
4139  cpl_table_new_column(temp, "err", CPL_TYPE_DOUBLE);
4140 
4141 #endif
4142 
4143  /*
4144  uves_msg_error("Saving 'sky_subtracted.fits'");
4145  cpl_image_save(image, "sky_subtracted.fits", CPL_BPP_IEEE_FLOAT, NULL,
4146  CPL_IO_DEFAULT);
4147  */
4148 
4149 
4150 
4151 
4152 
4153 
4154 
4155  /* Allocate max. number of storage needed (and resize/shorten later when we
4156  know how much was needed).
4157 
4158  One might get the idea to allocate storage for (nx*ny) points, but this
4159  is only a maximum if the orders are non-overlapping (which cannot a priori
4160  be assumed)
4161  */
4162  eval_points = cpl_matrix_new(norders*nx*uves_round_double(pos->sg.length+3), 3);
4163  eval_data = cpl_vector_new(norders*nx*uves_round_double(pos->sg.length+3));
4164  eval_err = cpl_vector_new(norders*nx*uves_round_double(pos->sg.length+3));
4165 
4166  fluxes = cpl_calloc((nx+1)*norders, sizeof(double));
4167  skys = cpl_calloc((nx+1)*norders, sizeof(double));
4168  /* orders (m) are index'ed starting from 0,
4169  columns (x) are index'ed starting from 1 (zero'th index is not used) */
4170 
4171  estimate = cpl_table_new(norders);
4172  cpl_table_new_column(estimate, "Order", CPL_TYPE_INT);
4173  cpl_table_new_column(estimate, "Y0" , CPL_TYPE_DOUBLE);
4174  cpl_table_new_column(estimate, "Sigma", CPL_TYPE_DOUBLE);
4175 
4176  coeffs = cpl_vector_new(ncoeffs); /* Polynomial coefficients */
4177  ia = cpl_calloc(ncoeffs, sizeof(int));
4178  {
4179  int i;
4180  for (i = 0; i < ncoeffs; i++)
4181  {
4182  cpl_vector_set(coeffs, i, 0); /* First guess */
4183 
4184  ia[i] = 1; /* Yes, fit this parameter */
4185  }
4186  }
4187 
4188 // for (order = minorder; order <= maxorder; order++) {
4189  for (order = 17; order <= 17; order++) {
4190  /* For estimates of y0, sigma for
4191  this order (pixel data values are
4192  used as weights)
4193  */
4194  double sumw = 0; /* sum data */
4195  double sumwy = 0; /* sum data*y */
4196  double sumwyy = 0; /* sum data*y*y */
4197 
4198  for (x = chunk/2; x <= nx - chunk/2; x += chunk) {
4199 // for (x = 900; x <= 1100; x += chunk)
4200  /* Find cosmic rays */
4201  int i;
4202  for (i = 0; i < nbins; i++)
4203  {
4204  /* Each wavel.bin contributes with one data point
4205  to each spatial bin. Therefore each spatial
4206  bin must be able to hold (chunk+1) points. But
4207  to be *completely* safe against weird rounding
4208  (depending on the architecture), make the vectors
4209  a bit longer. */
4210  cpl_vector_set_size(data[i], 2*(chunk + 1));
4211  size[i] = 0;
4212  }
4213 
4214  /* Bin data in this chunk */
4215  for (uves_iterate_set_first(pos,
4216  x - chunk/2 + 1, x + chunk/2,
4217  order, order,
4218  image_bpm, true);
4219  !uves_iterate_finished(pos);
4221  {
4222  int bin = pos->y - pos->ylow;
4223 
4224  check_nomsg(cpl_vector_set(data[bin], size[bin],
4225  DATA(image_data, pos)));
4226  size[bin]++;
4227  }
4228 
4229  /* Get threshold values for each spatial bin in this chunk */
4230  for (i = 0; i < nbins; i++)
4231  {
4232  if (size[i] == 0)
4233  {
4234  /* locut[i] hicut[i] are not used */
4235  }
4236  else if (size[i] <= chunk/2)
4237  {
4238  /* Not enough statistics to verify that the
4239  points are not outliers. Mark them as bad.*/
4240  locut[i] = cpl_vector_get_max(data[i]) + 1;
4241  hicut[i] = cpl_vector_get_min(data[i]) - 1;
4242  }
4243  else
4244  {
4245  /* Iteratively do kappa-sigma clipping to
4246  find the threshold for the current bin */
4247  double median, stdev;
4248  double kappa = 3.0;
4249  double *data_data;
4250  int k;
4251 
4252  k = size[i];
4253 
4254  do {
4255  cpl_vector_set_size(data[i], k);
4256  size[i] = k;
4257  data_data = cpl_vector_get_data(data[i]);
4258 
4259  median = cpl_vector_get_median_const(data[i]);
4260  stdev = cpl_vector_get_stdev(data[i]);
4261  locut[i] = median - kappa*stdev;
4262  hicut[i] = median + kappa*stdev;
4263 
4264  /* Copy good points to beginning of vector */
4265  k = 0;
4266  {
4267  int j;
4268  for (j = 0; j < size[i]; j++)
4269  {
4270  if (locut[i] <= data_data[j] &&
4271  data_data[j] <= hicut[i])
4272  {
4273  data_data[k] = data_data[j];
4274  k++;
4275  }
4276  }
4277  }
4278  }
4279  while (k < size[i] && k > 1);
4280  /* while more points rejected */
4281  }
4282  }
4283 
4284  /* Collect data points in this chunk.
4285  * At the same time compute estimates of
4286  * y0, sigma for this order
4287  */
4288 
4289  for (uves_iterate_set_first(pos,
4290  x - chunk/2 + 1, x + chunk/2,
4291  order, order,
4292  NULL, false)
4293  !uves_iterate_finished(pos);
4295  {
4296  int pis_rejected;
4297  double flux = 0; /* Linear extract bin */
4298  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
4299  {
4300  int bin = pos->y - pos->ylow;
4301 
4302  if (ISGOOD(image_bpm, pos) &&
4303  (locut[bin] <= DATA(image_data, pos) &&
4304  DATA(image_data, pos) <= hicut[bin])
4305  )
4306  {
4307  double pix = DATA(image_data, pos);
4308  double dy = pos->y - pos->ycenter;
4309  flux += pix;
4310 
4311  cpl_matrix_set(eval_points, n, 0, pos->x);
4312  cpl_matrix_set(eval_points, n, 1, dy);
4313  cpl_matrix_set(eval_points, n, 2, order);
4314  cpl_vector_set(eval_data, n, pix);
4315  cpl_vector_set(eval_err , n,
4316  DATA(noise_data, pos));
4317 
4318  sumw += pix;
4319  sumwy += pix * dy;
4320  sumwyy += pix * dy * dy;
4321 #if CREATE_DEBUGGING_TABLE
4322  cpl_table_set_double(temp, "x", n, pos->x);
4323  cpl_table_set_double(temp, "y", n, dy);
4324  cpl_table_set_double(temp, "order", n, order);
4325  cpl_table_set_double(temp, "dat", n, pix);
4326  cpl_table_set_double(temp, "err", n,
4327  DATA(noise_data, pos));
4328 
4329 #endif
4330  n++;
4331  }
4332  else
4333  {
4334  nbad += 1;
4335  /* uves_msg_error("bad pixel at (%d, %d)", i, pos->y);*/
4336  }
4337  }
4338  fluxes[pos->x + (order-pos->minorder)*(pos->nx+1)] = flux;
4339  skys [pos->x + (order-pos->minorder)*(pos->nx+1)] =
4340  cpl_image_get(sky_spectrum,
4341  pos->x, order-pos->minorder+1, &pis_rejected);
4342 
4343  /* Buffer widths are nx+1, not nx */
4344  skys [pos->x + (order-pos->minorder)*(pos->nx+1)] = 0;
4345  /* need non-sky-subtracted as input image */
4346 
4347  } /* collect data */
4348  } /* for each chunk */
4349 
4350  /* Estimate fit parameters */
4351  {
4352  double y0_estim;
4353  double sigma_estim;
4354  bool y0_is_good; /* Is the estimate valid, or should it be ignored? */
4355  bool sigma_is_good;
4356 
4357  if (sumw != 0)
4358  {
4359  y0_is_good = true;
4360  y0_estim = sumwy/sumw;
4361 
4362  sigma_estim = sumwyy/sumw - (sumwy/sumw)*(sumwy/sumw);
4363  if (sigma_estim > 0)
4364  {
4365  sigma_estim = sqrt(sigma_estim);
4366  sigma_is_good = true;
4367  }
4368  else
4369  {
4370  sigma_is_good = false;
4371  }
4372  }
4373  else
4374  {
4375 
4376  y0_is_good = false;
4377  sigma_is_good = false;
4378  }
4379 
4380  cpl_table_set_int (estimate, "Order", order - pos->minorder, order);
4381 
4382  if (y0_is_good)
4383  {
4384  cpl_table_set_double(estimate, "Y0" , order - pos->minorder, y0_estim);
4385  }
4386  else
4387  {
4388  cpl_table_set_invalid(estimate, "Y0", order - pos->minorder);
4389  }
4390 
4391  if (sigma_is_good)
4392  {
4393  cpl_table_set_double(estimate, "Sigma",
4394  order - pos->minorder, sigma_estim);
4395  }
4396  else
4397  {
4398  cpl_table_set_invalid(estimate, "Sigma", order - pos->minorder);
4399  }
4400 
4401 
4402  /* There's probably a nicer way of printing this... */
4403  if (y0_is_good && sigma_is_good) {
4404  uves_msg_error("Order #%d: Offset = %.2f pix; FWHM = %.2f pix",
4405  order, y0_estim, sigma_estim*TWOSQRT2LN2);
4406  }
4407  else if (y0_is_good && !sigma_is_good) {
4408  uves_msg_error("Order #%d: Offset = %.2f pix; FWHM = -- pix",
4409  order, y0_estim);
4410  }
4411  else if (!y0_is_good && sigma_is_good) {
4412  uves_msg_error("Order #%d: Offset = -- pix; FWHM = %.2f pix",
4413  order, sigma_estim);
4414  }
4415  else {
4416  uves_msg_error("Order #%d: Offset = -- pix; FWHM = -- pix",
4417  order);
4418  }
4419  } /* end estimating */
4420 
4421  } /* for each order */
4422 
4423  cpl_matrix_set_size(eval_points, n, 3);
4424  cpl_vector_set_size(eval_data, n);
4425  cpl_vector_set_size(eval_err , n);
4426 
4427 #if CREATE_DEBUGGING_TABLE
4428  cpl_table_set_size(temp, n);
4429 #endif
4430 
4431  /* Get estimates of constant + linear coefficients
4432  (as function of order (m), not x) */
4433  {
4434  double kappa = 3.0;
4435  int degree;
4436 
4437  cpl_table_dump(estimate, 0, cpl_table_get_nrow(estimate), stdout);
4438 
4439  /* Remove rows with invalid y0, but keep rows with
4440  valid sigma (therefore we need a copy) */
4441  estimate_dup = cpl_table_duplicate(estimate);
4442  assure_mem( estimate_dup );
4443  uves_erase_invalid_table_rows(estimate_dup, "Y0");
4444 
4445  /* Linear fit, or zero'th if only one position to fit */
4446  degree = (cpl_table_get_nrow(estimate_dup) > 1) ? 1 : 0;
4447 
4448  y0_estim_pol = uves_polynomial_regression_1d(
4449  estimate_dup, "Order", "Y0", NULL,
4450  degree,
4451  NULL, NULL, /* New columns */
4452  NULL, /* mse */
4453  kappa);
4454 
4455  uves_polynomial_dump(y0_estim_pol, stdout); fflush(stdout);
4456 
4457  if (cpl_error_get_code() != CPL_ERROR_NONE)
4458  {
4459  uves_msg_warning("Could not estimate object centroid (%s). "
4460  "Setting initial offset to zero",
4461  cpl_error_get_message());
4462 
4463  uves_error_reset();
4464 
4465  /* Set y0(m) := 0 */
4466  uves_polynomial_delete(&y0_estim_pol);
4467  y0_estim_pol = uves_polynomial_new_zero(1); /* dimension = 1 */
4468  }
4469 
4470  uves_free_table(&estimate_dup);
4471  estimate_dup = cpl_table_duplicate(estimate);
4472  assure_mem( estimate_dup );
4473  uves_erase_invalid_table_rows(estimate_dup, "Sigma");
4474 
4475  degree = (cpl_table_get_nrow(estimate_dup) > 1) ? 1 : 0;
4476 
4477  sigma_estim_pol = uves_polynomial_regression_1d(
4478  estimate_dup, "Order", "Sigma", NULL,
4479  degree,
4480  NULL, NULL, /* New columns */
4481  NULL, /* mse */
4482  kappa);
4483 
4484  if (cpl_error_get_code() != CPL_ERROR_NONE)
4485  {
4486  uves_msg_warning("Could not estimate object width (%s). "
4487  "Setting initial sigma to 1 pixel",
4488  cpl_error_get_message());
4489 
4490  uves_error_reset();
4491 
4492  /* Set sigma(m) := 1 */
4493  uves_polynomial_delete(&sigma_estim_pol);
4494  sigma_estim_pol = uves_polynomial_new_zero(1);
4495  uves_polynomial_shift(sigma_estim_pol, 0, 1.0);
4496  }
4497  } /* end estimating */
4498 
4499  /* Copy estimate to 'coeffs' vector */
4500 
4501  /* Centroid, constant term x^0 m^0 */
4502  cpl_vector_set(coeffs, 0,
4503  uves_polynomial_get_coeff_1d(y0_estim_pol, 0));
4504  /* Centroid, linear term x^0 m^1 */
4505  if (deg_y0_m >= 1)
4506  {
4507  cpl_vector_set(coeffs, 0 + (deg_y0_x+1)*1,
4508  uves_polynomial_get_coeff_1d(y0_estim_pol, 1));
4509 
4510  uves_msg_error("Estimate: y0 ~= %g + %g * m",
4511  cpl_vector_get(coeffs, 0),
4512  cpl_vector_get(coeffs, 0 + (deg_y0_x+1)*1));
4513  }
4514  else
4515  {
4516  uves_msg_error("Estimate: y0 ~= %g",
4517  cpl_vector_get(coeffs, 0));
4518  }
4519 
4520 
4521  /* Sigma, constant term x^0 m^0 */
4522  cpl_vector_set(coeffs, (deg_y0_x+1)*(deg_y0_m+1),
4523  uves_polynomial_get_coeff_1d(sigma_estim_pol, 0));
4524  /* Sigma, linear term x^0 m^1 */
4525  if (deg_sigma_m >= 1)
4526  {
4527  cpl_vector_set(coeffs, (deg_y0_x+1)*(deg_y0_m+1) +
4528  0 + (deg_sigma_x+1)*1,
4529  uves_polynomial_get_coeff_1d(sigma_estim_pol, 1));
4530 
4531  uves_msg_error("Estimate: sigma ~= %g + %g * m",
4532  cpl_vector_get(coeffs, (deg_y0_x+1)*(deg_y0_m+1) +
4533  0),
4534  cpl_vector_get(coeffs, (deg_y0_x+1)*(deg_y0_m+1) +
4535  0 + (deg_y0_x+1)*1));
4536  }
4537  else
4538  {
4539  uves_msg_error("Estimate: sigma ~= %g",
4540  cpl_vector_get(coeffs, (deg_y0_x+1)*(deg_y0_m+1) +
4541  0));
4542 
4543  }
4544  /* Remaining coeff.s were set to 0 */
4545 
4546  /* Fill struct used for fitting */
4547  profile_params.flux = fluxes;
4548  profile_params.sky = skys;
4549  profile_params.minorder = pos->minorder;
4550  profile_params.nx = nx;
4551 
4552  profile_params.f = f;
4553  profile_params.dfda = dfda;
4554 
4555  profile_params.deg_y0_x = deg_y0_x;
4556  profile_params.deg_y0_m = deg_y0_m;
4557  profile_params.deg_sigma_x = deg_sigma_x;
4558  profile_params.deg_sigma_m = deg_sigma_m;
4559 
4560 // cpl_msg_set_level(CPL_MSG_DEBUG_MODE);
4561 
4562  /* Unweighted fit: */
4563  cpl_vector_fill(eval_err,
4564  cpl_vector_get_median_const(eval_err));
4565 
4566  uves_msg_error("Fitting model to %d positions; %d bad pixels found",
4567  n, nbad);
4568 
4569  uves_fit(eval_points, NULL,
4570  eval_data, eval_err,
4571  coeffs, ia,
4572  profile_f,
4573  profile_dfda,
4574  NULL, /* mse, red_chisq, covariance */
4575  &red_chisq,
4576  &covariance);
4577 // cpl_msg_set_level(CPL_MSG_INFO);
4578 
4579  if (cpl_error_get_code() == CPL_ERROR_SINGULAR_MATRIX ||
4580  cpl_error_get_code() == CPL_ERROR_CONTINUE)
4581  {
4582  uves_msg_warning("Fitting global model failed (%s)", cpl_error_get_message());
4583  uves_error_reset();
4584 #if CREATE_DEBUGGING_TABLE
4585  cpl_table_save(temp, NULL, NULL, "tab.fits", CPL_IO_DEFAULT);
4586 #endif
4587  }
4588  else
4589  {
4590  assure( cpl_error_get_code() == CPL_ERROR_NONE,
4591  cpl_error_get_code(), "Fitting global model failed");
4592 
4593  cpl_matrix_dump(covariance, stdout); fflush(stdout);
4594 
4595  uves_msg_error("Solution: y0 ~= %g", eval_pol(cpl_vector_get_data(coeffs),
4596  deg_y0_x, deg_y0_m,
4597  pos->nx/2,
4598  (pos->minorder+pos->maxorder)/2));
4599  uves_msg_error("Solution: sigma ~= %g", eval_pol(cpl_vector_get_data(coeffs)+
4600  (deg_y0_x+1)*(deg_y0_m+1),
4601  deg_y0_x, deg_y0_m,
4602  pos->nx/2,
4603  (pos->minorder+pos->maxorder)/2));
4604 
4605  /* Fill table with solution */
4606  for (order = pos->minorder; order <= pos->maxorder; order++) {
4607  for (x = chunk/2; x <= nx - chunk/2; x += chunk)
4608  {
4609  double y_0 = eval_pol(cpl_vector_get_data(coeffs),
4610  deg_y0_x, deg_y0_m, x, order);
4611  double sigma = fabs(eval_pol(cpl_vector_get_data(coeffs)+
4612  (deg_y0_x+1)*(deg_y0_m+1),
4613  deg_sigma_x, deg_sigma_m, x, order));
4614 
4615  /* Use error propagation formula to get variance of polynomials:
4616 
4617  We have p(x,m) = sum_ij a_ij x^i m^j,
4618 
4619  and thus a quadruple sum for the variance,
4620 
4621  V(x,m) = sum_i1j1i2j2 Cov(a_i1j1, a_i2j2), x^(i1+i2) m^(j1+j2)
4622 
4623  (for both y0(x,m) and sigma(x,m))
4624  */
4625  double dy0 = 0;
4626  double dsigma = 0;
4627  int i1, i2, j_1, j2; /* because POSIX 1003.1-2001 defines 'j1' */
4628 
4629  for (i1 = 0; i1 < (deg_y0_x+1); i1++)
4630  for (j_1 = 0; j_1 < (deg_y0_m+1); j_1++)
4631  for (i2 = 0; i2 < (deg_y0_x+1); i2++)
4632  for (j2 = 0; j2 < (deg_y0_m+1); j2++)
4633  {
4634  dy0 += cpl_matrix_get(covariance,
4635  i1+(deg_y0_x+1)*j_1,
4636  i2+(deg_y0_x+1)*j2) *
4637  uves_pow_int(x, i1+i2) *
4638  uves_pow_int(order, j_1+j2);
4639  }
4640  if (dy0 > 0)
4641  {
4642  dy0 = sqrt(dy0);
4643  }
4644  else
4645  /* Should not happen */
4646  {
4647  dy0 = 1.0;
4648  }
4649 
4650  for (i1 = 0; i1 < (deg_sigma_x+1); i1++)
4651  for (j_1 = 0; j_1 < (deg_sigma_m+1); j_1++)
4652  for (i2 = 0; i2 < (deg_sigma_x+1); i2++)
4653  for (j2 = 0; j2 < (deg_sigma_m+1); j2++)
4654  {
4655  /* Ignore the upper left part of the covariance
4656  matrix (the covariances related to y0)
4657  */
4658  dsigma += cpl_matrix_get(
4659  covariance,
4660  (deg_y0_x+1)*(deg_y0_m+1) + i1+(deg_sigma_x+1)*j_1,
4661  (deg_y0_x+1)*(deg_y0_m+1) + i2+(deg_sigma_x+1)*j2) *
4662  uves_pow_int(x, i1+i1) *
4663  uves_pow_int(order, j_1+j2);
4664  }
4665  if (dsigma > 0)
4666  {
4667  dsigma = sqrt(dsigma);
4668  }
4669  else
4670  /* Should not happen */
4671  {
4672  dsigma = 1.0;
4673  }
4674 
4675  check((cpl_table_set_int (profile_data, "Order", profile_row, order),
4676  cpl_table_set_int (profile_data, "X" , profile_row, x),
4677  cpl_table_set_double(profile_data, "Y0" , profile_row, y_0),
4678  cpl_table_set_double(profile_data, "Sigma", profile_row, sigma),
4679  cpl_table_set_double(profile_data, "Norm" , profile_row, 1),
4680  cpl_table_set_double(profile_data, "dY0" , profile_row, dy0),
4681  cpl_table_set_double(profile_data, "dSigma", profile_row, dsigma),
4682  cpl_table_set_double(profile_data, "dNorm", profile_row, 1),
4683  cpl_table_set_double(profile_data, "Y0_world", profile_row, -1),
4684  cpl_table_set_double(profile_data, "Reduced_chisq", profile_row,
4685  red_chisq)),
4686  "Error writing table row %d", profile_row+1);
4687  profile_row += 1;
4688  } /* For each chunk */
4689  } /* For each order */
4690 #if CREATE_DEBUGGING_TABLE
4691  cpl_table_new_column(temp, "pemp", CPL_TYPE_DOUBLE); /* empirical profile */
4692  cpl_table_new_column(temp, "fit", CPL_TYPE_DOUBLE); /* fitted profile */
4693  cpl_table_new_column(temp, "pfit", CPL_TYPE_DOUBLE); /* fitted profile, normalized */
4694  {int i;
4695  for (i = 0; i < cpl_table_get_nrow(temp); i++)
4696  {
4697  double y = cpl_table_get_double(temp, "y", i, NULL);
4698  int xi = uves_round_double(cpl_table_get_double(temp, "x", i, NULL));
4699  int mi = uves_round_double(cpl_table_get_double(temp, "order", i, NULL));
4700  double dat = cpl_table_get_double(temp, "dat", i, NULL);
4701  int idx = xi + (mi - profile_params.minorder)*(profile_params.nx + 1);
4702  double flux_fit;
4703  double xar[3];
4704  xar[0] = xi;
4705  xar[1] = y;
4706  xar[2] = mi;
4707 
4708  profile_f(xar,
4709  cpl_vector_get_data(coeffs), &flux_fit);
4710 
4711  cpl_table_set(temp, "pemp", i,
4712  (dat - profile_params.sky[idx])/profile_params.flux[idx]);
4713 
4714  cpl_table_set(temp, "fit", i, flux_fit);
4715 
4716  cpl_table_set(temp, "pfit", i,
4717  (flux_fit - profile_params.sky[idx])/profile_params.flux[idx]);
4718  }
4719  }
4720  check_nomsg(
4721  cpl_table_save(temp, NULL, NULL, "tab.fits", CPL_IO_DEFAULT));
4722 #endif
4723  }
4724  }
4725 
4726 #else /* if NEW_METHOD */
4727  dy = cpl_vector_new((chunk+1) * ((int)(pos->sg.length + 3)));
4728  prof = cpl_vector_new((chunk+1) * ((int)(pos->sg.length + 3)));
4729  prof2 = cpl_vector_new((chunk+1) * ((int)(pos->sg.length + 3)));
4730  dprof = cpl_vector_new((chunk+1) * ((int)(pos->sg.length + 3)));
4731 
4732  for (x = 1 + chunk/2; x + chunk/2 <= pos->nx; x += chunk) {
4733  /* Collapse chunk [x-chunk/2 ; x+chunk/2],
4734  then fit profile (this is to have better
4735  statistics than if fitting individual bins). */
4736  const int points_needed_for_fit = 6;
4737  int n = 0;
4738  int nbad = 0;
4739  int i;
4740 
4741  /* Use realloc rather than malloc (for each chunk) */
4742  cpl_vector_set_size(dy, (chunk+1) * ((int)(pos->sg.length + 3)));
4743  cpl_vector_set_size(prof, (chunk+1) * ((int)(pos->sg.length + 3)));
4744  cpl_vector_set_size(prof2, (chunk+1) * ((int)(pos->sg.length + 3)));
4745  cpl_vector_set_size(dprof, (chunk+1) * ((int)(pos->sg.length + 3)));
4746  n = 0; /* Number of points inserted in dy, prof, dprof */
4747 
4748  for (i = 0; i < nbins; i++)
4749  {
4750  /* Each wavel.bin contributes with one data point
4751  to each spatial bin. Therefore each spatial
4752  bin must be able to hold (chunk+1) points. But
4753  to be *completely* safe against weird rounding
4754  (depending on the architecture), make the vectors
4755  a bit longer. */
4756  cpl_vector_set_size(data[i], 2*(chunk + 1));
4757  size[i] = 0;
4758  }
4759 
4760 
4761  /* Bin data in this chunk */
4762  for (uves_iterate_set_first(pos,
4763  x - chunk/2 + 1,
4764  x + chunk/2,
4765  pos->order, pos->order,
4766  image_bpm, true);
4767  !uves_iterate_finished(pos);
4769  {
4770  int bin = pos->y - pos->ylow;
4771 
4772  /* Group into spatial bins */
4773  check_nomsg(cpl_vector_set(data[bin], size[bin],
4774  DATA(image_data, pos)));
4775  size[bin]++;
4776  }
4777 
4778  /* Get threshold values for each spatial bin in this chunk */
4779  for (i = 0; i < nbins; i++)
4780  {
4781  if (size[i] == 0)
4782  {
4783  /* locut[i] hicut[i] are not used */
4784  }
4785  else if (size[i] <= chunk/2)
4786  {
4787  /* Not enough statistics to verify that the
4788  points are not outliers. Mark them as bad.*/
4789  locut[i] = cpl_vector_get_max(data[i]) + 1;
4790  hicut[i] = cpl_vector_get_min(data[i]) - 1;
4791  }
4792  else
4793  {
4794  /* Iteratively do kappa-sigma clipping to
4795  find the threshold for the current bin */
4796  double median, stdev;
4797  double kappa = 3.0;
4798  double *data_data;
4799  int k;
4800 
4801  k = size[i];
4802 
4803  do {
4804  cpl_vector_set_size(data[i], k);
4805  size[i] = k;
4806  data_data = cpl_vector_get_data(data[i]);
4807 
4808  median = cpl_vector_get_median_const(data[i]);
4809  stdev = cpl_vector_get_stdev(data[i]);
4810  locut[i] = median - kappa*stdev;
4811  hicut[i] = median + kappa*stdev;
4812 
4813  /* Copy good points to beginning of vector */
4814  k = 0;
4815  {
4816  int j;
4817  for (j = 0; j < size[i]; j++)
4818  {
4819  if (locut[i] <= data_data[j] &&
4820  data_data[j] <= hicut[i])
4821  {
4822  data_data[k] = data_data[j];
4823  k++;
4824  }
4825  }
4826  }
4827  }
4828  while (k < size[i] && k > 1);
4829  /* while still more points rejected */
4830  }
4831  } /* for each bin */
4832 
4833  /* Collect good data in this chunk */
4834  for (uves_iterate_set_first(pos,
4835  x - chunk/2 + 1,
4836  x + chunk/2,
4837  pos->order, pos->order,
4838  NULL, false);
4839  !uves_iterate_finished(pos);
4841  {
4842  double flux = 0;
4843  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
4844  {
4845  int bin = pos->y - pos->ylow;
4846 
4847  if (ISGOOD(image_bpm, pos) &&
4848  (locut[bin] <= DATA(image_data, pos) &&
4849  DATA(image_data, pos) <= hicut[bin])
4850  )
4851  {
4852  flux += DATA(image_data, pos);
4853  }
4854  }
4855 
4856  if (flux != 0)
4857  {
4858  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
4859  {
4860  int bin = pos->y - pos->ylow;
4861 
4862  if (ISGOOD(image_bpm, pos) &&
4863  (locut[bin] <= DATA(image_data, pos) &&
4864  DATA(image_data, pos) <= hicut[bin])
4865  )
4866  {
4867  double pix = DATA(image_data, pos);
4868 
4869  cpl_vector_set(dy , n, pos->y - pos->ycenter);
4870  cpl_vector_set(prof , n, pix/flux);
4871  cpl_vector_set(dprof, n, (flux > 0) ?
4872  DATA(noise_data, pos)/flux :
4873  -DATA(noise_data, pos)/flux);
4874  n++;
4875  }
4876  else
4877  {
4878  nbad += 1;
4879  /* uves_msg_debug("Bad pixel at (%d, %d)",
4880  pos->x, pos->y); */
4881  }
4882  }
4883  }
4884  } /* collect data */
4885 
4886  if (n >= points_needed_for_fit) {
4887  double y_0, norm, background, slope, sigma, red_chisq;
4888 
4889  cpl_vector_set_size(dy, n);
4890  cpl_vector_set_size(prof, n);
4891  cpl_vector_set_size(prof2, n);
4892  cpl_vector_set_size(dprof, n);
4893 
4894  /* Fit */
4895  uves_msg_debug("Fitting chunk (%d, %d)",
4896  x-chunk/2, x+chunk/2);
4897 
4898 // cpl_vector_dump(dy, stdout);
4899 // cpl_vector_dump(prof, stdout);
4900 
4901  uves_free_matrix(&covariance);
4902 
4903  background = 0; /* The sky was already subtracted */
4904  norm = 1.0; /* We are fitting the normalized profile.
4905  Reducing the number of free parameters
4906  gives a better fit.
4907  */
4908 
4909  /* Use constant uncertainty */
4910 if (0) {
4911  /* This gives a better fit (narrower profile at low S/N)
4912  but overestimates chi^2
4913  */
4914  double median = cpl_vector_get_median_const(dprof);
4915 
4916  cpl_vector_fill(dprof, median);
4917  }
4918  uves_fit_1d(dy, NULL,
4919 #if 1
4920  prof, dprof,
4921 #else
4922  prof, NULL,
4923 #endif
4924  CPL_FIT_CENTROID |
4925  CPL_FIT_STDEV,
4926  false,
4927  &y_0, &sigma, &norm, &background, &slope,
4928 #if 1
4929  NULL, &red_chisq, /* mse, red_chisq */
4930  &covariance,
4931 #else
4932  NULL, NULL,
4933  NULL,
4934 #endif
4935  f, dfda, M);
4936 #if 1
4937 #else
4938  covariance = cpl_matrix_new(4,4);
4939  cpl_matrix_set(covariance, 0, 0, 1);
4940  cpl_matrix_set(covariance, 1, 1, 1);
4941  cpl_matrix_set(covariance, 2, 2, 1);
4942  cpl_matrix_set(covariance, 3, 3, 1);
4943  red_chisq = 1;
4944 #endif
4945  if (false) /* && 800-chunk/2 <= x && x <= 800+chunk/2 && order == 17) */
4946  {
4947 /* uves_msg_error("dumping chunk at x,order = %d, %d", x, order);
4948  uves_msg_error("dy = ");
4949  cpl_vector_dump(dy, stderr);
4950  uves_msg_error("prof = ");
4951  cpl_vector_dump(prof, stderr);
4952 */
4953 
4954 /*
4955  cpl_bivector *b = cpl_bivector_wrap_vectors(dy, prof);
4956  cpl_plot_bivector("set grid;set yrange[-1:1];set xlabel 'Wavelength [m]';",
4957  "t 'Spatial profile' w points",
4958  "",b);
4959  cpl_bivector_unwrap_vectors(b);
4960 */
4961 
4962  cpl_vector *pl[] = {NULL, NULL, NULL};
4963 
4964  cpl_vector *fit = cpl_vector_new(cpl_vector_get_size(dy));
4965  {
4966  for (i = 0; i < cpl_vector_get_size(dy); i++)
4967  {
4968  double yy = cpl_vector_get(dy, i);
4969  cpl_vector_set(fit, i,
4970  exp(-(yy-y_0)*(yy-y_0)/(2*sigma*sigma))
4971  /(sigma*sqrt(2*M_PI)));
4972  }
4973  }
4974 
4975  /* uves_msg_error("result is %f, %f, %f, %f %d %f",
4976  y_0, sigma, norm, background, cpl_error_get_code(), sigma*TWOSQRT2LN2);
4977  */
4978 
4979  pl[0] = prof2;
4980  pl[1] = dprof;
4981  pl[2] = dprof;
4982 // pl[0] = dy;
4983 // pl[1] = prof;
4984 // pl[2] = fit;
4985  uves_error_reset();
4986  cpl_plot_vectors("set grid;set yrange[0:0.5];set xlabel 'dy';",
4987  "t 'Spatial profile' w points",
4988  "",
4989  (const cpl_vector **)pl, 3);
4990 
4991 
4992  pl[0] = prof;
4993  pl[1] = dprof;
4994  pl[2] = dprof;
4995 
4996  cpl_plot_vectors("set grid;set xrange[-2:2];"
4997  "set yrange[0:0.5];set xlabel 'dy';",
4998  "t 'Spatial profile' w points",
4999  "",
5000  (const cpl_vector **)pl, 3);
5001 
5002  uves_free_vector(&fit);
5003 
5004  }
5005 
5006  /* Convert to global coordinate (at middle of chunk) */
5008  x, x,
5009  pos->order, pos->order,
5010  NULL,
5011  false);
5012  y_0 += pos->ycenter;
5013 
5014  /* Recover from a failed fit.
5015  *
5016  * The gaussian fitting routine itself guarantees
5017  * that, on success, sigma < slit_length.
5018  * Tighten this constraint by requiring that also 4sigma < slit_length (see below).
5019  * This is to avoid detecting
5020  * sky-on-top-of-interorder
5021  * rather than
5022  * object-on-top-of-sky
5023  * (observed to happen in low-S/N cases when
5024  * the sky flux dominates the object flux )
5025  *
5026  * object
5027  * /\
5028  * |-sky-/ \--sky-|
5029  * | |
5030  * | |
5031  * -----| s l i t |---interorder--
5032  *
5033  *
5034  * Also avoid fits with sigma < 0.2 which are probably CRs
5035  *
5036  */
5037  if (cpl_error_get_code() == CPL_ERROR_CONTINUE ||
5038  cpl_error_get_code()== CPL_ERROR_SINGULAR_MATRIX ||
5039  4.0*sigma >= pos->sg.length || sigma < 0.2) {
5040 
5041  uves_msg_debug("Profile fitting failed at (order, x) = (%d, %d) "
5042  "(%s), ignoring chunk",
5043  pos->order, x, cpl_error_get_message());
5044 
5045  uves_error_reset();
5046  }
5047  else {
5048  assure( cpl_error_get_code() == CPL_ERROR_NONE, cpl_error_get_code(),
5049  "Gaussian fitting failed");
5050 
5051  check(
5052  (cpl_table_set_int (profile_data, "Order", profile_row, pos->order),
5053  cpl_table_set_int (profile_data, "X" , profile_row, x),
5054  cpl_table_set_double(profile_data, "Y0" , profile_row, y_0 - pos->ycenter),
5055  cpl_table_set_double(profile_data, "Sigma", profile_row, sigma),
5056  cpl_table_set_double(profile_data, "Norm" , profile_row, norm),
5057  cpl_table_set_double(profile_data, "dY0" , profile_row,
5058  sqrt(cpl_matrix_get(covariance, 0, 0))),
5059  cpl_table_set_double(profile_data, "dSigma", profile_row,
5060  sqrt(cpl_matrix_get(covariance, 1, 1))),
5061  cpl_table_set_double(profile_data, "dNorm", profile_row,
5062  sqrt(cpl_matrix_get(covariance, 2, 2))),
5063  cpl_table_set_double(profile_data, "Y0_world", profile_row, y_0),
5064  cpl_table_set_double(profile_data, "Reduced_chisq", profile_row,
5065  red_chisq)),
5066  "Error writing table");
5067 
5068  profile_row += 1;
5069  /* uves_msg_debug("y0 = %f sigma = %f norm = %f "
5070  "background = %f", y_0, sigma, norm, background); */
5071  }
5072  }
5073  else
5074  {
5075  uves_msg_debug("Order #%d: Too few (%d) points available in "
5076  "at x = %d - %d, ignoring chunk",
5077  pos->order, n,
5078  x - chunk/2, x + chunk/2);
5079  }
5080  } /* for each chunk */
5081 
5082 #endif /* old method */
5083 
5084  cpl_table_set_size(profile_data, profile_row);
5085 
5086  UVES_TIME_END;
5087 
5088 
5089 cleanup:
5090 #if NEW_METHOD
5091  uves_free_matrix(&eval_points);
5092  uves_free_vector(&eval_data);
5093  uves_free_vector(&eval_err);
5094  uves_free_vector(&coeffs);
5095  cpl_free(fluxes);
5096  cpl_free(skys);
5097  cpl_free(ia);
5098 #if CREATE_DEBUGGING_TABLE
5099  uves_free_table(&temp);
5100 #endif
5101  uves_free_table(&estimate);
5102  uves_free_table(&estimate_dup);
5103  uves_polynomial_delete(&y0_estim_pol);
5104  uves_polynomial_delete(&sigma_estim_pol);
5105 #endif
5106 
5107  uves_free_matrix(&covariance);
5108  uves_free_vector(&dy);
5109  uves_free_vector(&prof);
5110  uves_free_vector(&prof2);
5111  uves_free_vector(&dprof);
5112  {
5113  int i;
5114  for (i = 0; i < nbins; i++)
5115  {
5116  uves_free_vector(&(data[i]));
5117  }
5118  }
5119  cpl_free(data);
5120  cpl_free(size);
5121  cpl_free(locut);
5122  cpl_free(hicut);
5123 
5124  if (cpl_error_get_code() != CPL_ERROR_NONE)
5125  {
5126  uves_free_table(&profile_data);
5127  }
5128 
5129  return profile_data;
5130 }
5131 
5132 
5133 /*----------------------------------------------------------------------------*/
5142 /*----------------------------------------------------------------------------*/
5143 static int
5144 opt_get_order_width(const uves_iterate_position *pos)
5145 {
5146  int result = -1;
5147 
5148  double x1 = 1;
5149  double x2 = pos->nx;
5150  double y_1 = uves_polynomial_evaluate_2d(pos->order_locations, x1, pos->order);
5151  double y2 = uves_polynomial_evaluate_2d(pos->order_locations, x2, pos->order);
5152  double slope = (y2 - y_1)/(x2 - x1);
5153 
5154  if (slope != 0)
5155  {
5156  /* Solve
5157  slope * x + y1 = 1 and
5158  slope * x + y1 = ny
5159  for x
5160 
5161  ... then get exact solution
5162  */
5163  double x_yeq1 = ( 1 - y_1)/slope;
5164  double x_yeqny = (pos->ny - y_1)/slope;
5165 
5166  if (1 <= x_yeq1 && x_yeq1 <= pos->nx) /* If order is partially below image */
5167  {
5168  double guess = x_yeq1;
5169 
5170  uves_msg_debug("Guess value (y = 1) x = %f", guess);
5171  /* Get exact value of x_yeq1 */
5172  x_yeq1 = uves_polynomial_solve_2d(pos->order_locations,
5173  1, /* Solve p = 1 */
5174  guess, /* guess value */
5175  1, /* multiplicity */
5176  2, /* fix this
5177  variable number */
5178  pos->order);/* ... to this value */
5179 
5180  if (cpl_error_get_code() != CPL_ERROR_NONE)
5181  {
5182  uves_error_reset();
5183  uves_msg_warning("Could not solve order polynomial = 1 at order #%d. "
5184  "Order polynomial may be ill-formed", pos->order);
5185  x_yeq1 = guess;
5186  }
5187  else
5188  {
5189  uves_msg_debug("Exact value (y = 1) x = %f", x_yeq1);
5190  }
5191  }
5192 
5193  if (1 <= x_yeqny && x_yeqny <= pos->nx) /* If order is partially above image */
5194  {
5195  double guess = x_yeqny;
5196 
5197  uves_msg_debug("Guess value (y = %d) = %f", pos->ny, guess);
5198  /* Get exact value of x_yeqny */
5199  x_yeqny = uves_polynomial_solve_2d(pos->order_locations,
5200  pos->ny, /* Solve p = ny */
5201  guess, /* guess value */
5202  1, /* multiplicity */
5203  2, /* fix this
5204  variable number */
5205  pos->order);/* ... to this value */
5206 
5207  if (cpl_error_get_code() != CPL_ERROR_NONE)
5208  {
5209  uves_error_reset();
5210  uves_msg_warning("Could not solve order polynomial = %d at order #%d. "
5211  "Order polynomial may be ill-formed",
5212  pos->ny, pos->order);
5213  x_yeqny = guess;
5214  }
5215  else
5216  {
5217  uves_msg_debug("Exact value (y = %d) x = %f", pos->ny, x_yeqny);
5218  }
5219  }
5220 
5221  if (slope > 0)
5222  {
5223  result = uves_round_double(
5224  uves_max_double(1,
5225  uves_min_double(pos->nx, x_yeqny) -
5226  uves_max_double(1, x_yeq1) + 1));
5227  }
5228  else
5229  {
5230  passure( slope < 0, "%f", slope);
5231  result = uves_round_double(
5232  uves_max_double(1,
5233  uves_min_double(pos->nx, x_yeq1 ) -
5234  uves_max_double(1, x_yeqny) + 1));
5235  }
5236  }
5237  else
5238  {
5239  result = pos->nx;
5240  }
5241 
5242  uves_msg_debug("Order width = %d pixels", result);
5243 
5244  cleanup:
5245 
5246  return result;
5247 }
5248 
5249 
5250 /*----------------------------------------------------------------------------*/
5289 /*----------------------------------------------------------------------------*/
5290 static int
5291 opt_extract(cpl_image *image,
5292  const cpl_image *image_noise,
5293  uves_iterate_position *pos,
5294  const uves_extract_profile *profile,
5295  bool optimal_extract_sky,
5296  double kappa,
5297  cpl_table *blemish_mask,
5298  cpl_table *cosmic_mask,
5299  int *cr_row,
5300  cpl_table *profile_table,
5301  int *prof_row,
5302  cpl_image *spectrum,
5303  cpl_image *spectrum_noise,
5304  cpl_image *weights,
5305  cpl_image *sky_spectrum,
5306  cpl_image *sky_spectrum_noise,
5307  double *sn)
5308 {
5309  cpl_table *signal_to_noise = NULL; /* S/N values of bins in this order
5310  * (table used as a variable length array)
5311  */
5312  int sn_row = 0; /* Number of rows in 'signal_to_noise'
5313  actually used */
5314 
5315  int bins_extracted = 0;
5316  int cold_pixels = 0; /* Number of hot/cold pixels in this order */
5317  int hot_pixels = 0;
5318  int warnings = 0; /* Warnings printed so far */
5319 
5320  const double *image_data;
5321  const double *noise_data;
5322  double *weights_data;
5323  cpl_mask *image_bad = NULL;
5324  cpl_binary*image_bpm = NULL;
5325  double *noise_buffer = NULL; /* For efficiency. To avoid allocating/deallocating
5326  space for each bin */
5327  int order_width;
5328  int spectrum_row = pos->order - pos->minorder + 1;
5329 
5330  int* px=0;
5331  int* py=0;
5332  int row=0;
5333 
5334  /* For efficiency, use direct pointer to pixel buffer,
5335  assume type double, support bad pixels */
5336 
5337  assure( cpl_image_get_type(image) == CPL_TYPE_DOUBLE &&
5338  cpl_image_get_type(image_noise) == CPL_TYPE_DOUBLE, CPL_ERROR_UNSUPPORTED_MODE,
5339  "Input image+noise must have type double. Types are %s + %s",
5340  uves_tostring_cpl_type(cpl_image_get_type(image)),
5341  uves_tostring_cpl_type(cpl_image_get_type(image_noise)));
5342 
5343  image_data = cpl_image_get_data_double_const(image);
5344  noise_data = cpl_image_get_data_double_const(image_noise);
5345  weights_data = cpl_image_get_data_double(weights);
5346 
5347  image_bad = cpl_image_get_bpm(image);
5348 
5349  /* flag blemishes as bad pixels */
5350  if(blemish_mask!=NULL) {
5351  check_nomsg(px=cpl_table_get_data_int(blemish_mask,"X"));
5352  check_nomsg(py=cpl_table_get_data_int(blemish_mask,"Y"));
5353 
5354  for(row=0;row<cpl_table_get_nrow(blemish_mask);row++) {
5355  check_nomsg(cpl_mask_set(image_bad,px[row]+1,py[row]+1,CPL_BINARY_1));
5356  }
5357  }
5358  /* end flag blemishes as bad pixels */
5359 
5360  image_bpm = cpl_mask_get_data(image_bad);
5361 
5362 
5363 
5364  noise_buffer = cpl_malloc(uves_round_double(pos->sg.length + 5)*sizeof(double));
5365 
5366  check( (signal_to_noise = cpl_table_new(pos->nx),
5367  cpl_table_new_column(signal_to_noise, "SN", CPL_TYPE_DOUBLE)),
5368  "Error allocating S/N table");
5369 
5370  check( order_width = opt_get_order_width(pos),
5371  "Error estimating width of order #%d", pos->order);
5372 
5373 
5374  /* First set all pixels in the extracted spectrum as bad,
5375  then mark them as good if/when the flux is calculated */
5376  {
5377  int x;
5378  for (x = 1; x <= pos->nx; x++)
5379  {
5380  cpl_image_reject(spectrum, x, spectrum_row);
5381  /* cpl_image_reject preserves the internal bad pixel map */
5382 
5383  if (spectrum_noise != NULL)
5384  {
5385  cpl_image_reject(spectrum_noise, x, spectrum_row);
5386  }
5387  if (optimal_extract_sky && sky_spectrum != NULL)
5388  {
5389  cpl_image_reject(sky_spectrum , x, spectrum_row);
5390  cpl_image_reject(sky_spectrum_noise, x, spectrum_row);
5391  }
5392  }
5393  }
5394 
5395  for (uves_iterate_set_first(pos,
5396  1, pos->nx,
5397  pos->order, pos->order,
5398  NULL, false);
5399  !uves_iterate_finished(pos);
5400  uves_iterate_increment(pos))
5401  {
5402  double flux = 0, variance = 0; /* Flux and variance of this bin */
5403  double sky_background = 0, sky_background_noise = 0;
5404 
5405  /*
5406  * Determine 'flux' and 'variance' of this bin.
5407  */
5408  int iteration;
5409 
5410  bool found_bad_pixel;
5411  double median_noise;
5412 
5413  double redchisq = 0;
5414 
5415  /* If rejection is asked for, get correction factor for this bin */
5416  if (kappa > 0)
5417  {
5418  redchisq = opt_get_redchisq(profile, pos);
5419  }
5420 
5421  /* Prepare for calls of uves_extract_profile_evaluate() */
5422  uves_extract_profile_set(profile, pos, &warnings);
5423 
5424  /* Pseudocode for optimal extraction of this bin:
5425  *
5426  * reset weights
5427  *
5428  * do
5429  * flux,variance := extract optimal
5430  * (only good pixels w. weight > 0)
5431  * (in first iteration, noise = max(noise, median(noise_i))
5432  *
5433  * reject the worst outlier by setting its weight to -1
5434  *
5435  * until there were no more outliers
5436  *
5437  *
5438  * Note that the first iteration increases the noise level
5439  * of each pixel to the median noise level. Otherwise, outlier
5440  * cold pixels would
5441  * would destroy the first flux estimate because of their very low
5442  * 'photonic' noise (i.e. they would have very large weight when their
5443  * uncertainties are taken into account). With the scheme above,
5444  * such a dead pixel will be rejected in the first iteration, and it is
5445  * safe to continue with optimal extractions until convergence.
5446  *
5447  */
5448 
5449  /*
5450  * Clear previously detected cosmic rays.
5451  */
5452  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
5453  {
5454  if (DATA(image_bpm, pos) == CPL_BINARY_1)
5455  {
5456  DATA(weights_data, pos) = -1.0;
5457  }
5458  else
5459  {
5460  DATA(weights_data, pos) = 0.0;
5461  }
5462  }
5463 
5464  /* Get median noise level (of all object + sky bins) */
5465  median_noise = opt_get_noise_median(noise_data, image_bpm,
5466  pos, noise_buffer);
5467 
5468  /* Extract optimally,
5469  reject outliers ... while found_bad_pixel (but at least twice) */
5470  found_bad_pixel = false;
5471 
5472  for (iteration = 0; iteration < 2 || found_bad_pixel; iteration++)
5473  {
5474  /* Get (flux,variance). In first iteration
5475  raise every noise value to median.
5476  */
5477  flux = opt_get_flux_sky_variance(image_data, noise_data,
5478  weights_data,
5479  pos,
5480  profile,
5481  optimal_extract_sky,
5482  (iteration == 0) ?
5483  median_noise : -1,
5484  &variance,
5485  &sky_background,
5486  &sky_background_noise);
5487 
5488  /* If requested, find max outlier among remaining good pixels */
5489  if (kappa > 0)
5490  {
5491  check( found_bad_pixel =
5492  opt_reject_outlier(image_data,
5493  noise_data,
5494  image_bpm,
5495  weights_data,
5496  pos,
5497  profile,
5498  kappa,
5499  flux,
5500  optimal_extract_sky ? sky_background : 0,
5501  redchisq,
5502  cosmic_mask,
5503  cr_row,
5504  &hot_pixels,
5505  &cold_pixels),
5506  "Error rejecting outlier pixel");
5507 
5508  }
5509  else
5510  {
5511  found_bad_pixel = false;
5512  }
5513 
5514  } /* while there was an outlier or iteration < 2 */
5515  //uves_msg("AMO crh tab size=%d",cpl_table_get_nrow(cosmic_mask));
5516  /* Update profile table */
5517  if (profile_table != NULL) {
5518  double lin_flux = 0; /* Linearly extracted flux */
5519  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++) {
5520  /* If pixel is not rejected */
5521  if (DATA(weights_data, pos) > 0)
5522  {
5523  double pixelval = DATA(image_data, pos);
5524  lin_flux += pixelval;
5525  }
5526  }
5527 
5528  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++) {
5529  /* If pixel is not rejected */
5530  if (DATA(weights_data, pos) > 0)
5531  {
5532  double dy = pos->y - pos->ycenter;
5533  double pixelval = DATA(image_data, pos);
5534 
5535  check_nomsg(
5536  (cpl_table_set_int (profile_table, "Order" ,
5537  *prof_row, pos->order),
5538  cpl_table_set_int (profile_table, "X" ,
5539  *prof_row, pos->x),
5540  cpl_table_set_double(profile_table, "DY" ,
5541  *prof_row, dy),
5542  cpl_table_set_double(profile_table, "Profile_raw",
5543  *prof_row, pixelval/lin_flux),
5544  cpl_table_set_double(profile_table, "Profile_int",
5545  *prof_row,
5546  uves_extract_profile_evaluate(profile, pos))));
5547  (*prof_row)++;
5548  }
5549  }
5550  }
5551 
5552  bins_extracted += 1;
5553 
5554  /* Don't do the following!! It changes the internal bpm with a low probability.
5555  That's bad because we already got a pointer to that so next time
5556  we follow that pointer the object might not exist. This is true
5557  for CPL3.0, it should be really be fixed in later versions.
5558 
5559  cpl_image_set(spectrum, pos->x, spectrum_row, flux);
5560 
5561  We don't have a pointer 'spectrum_noise', so calling cpl_image_set
5562  on that one is safe.
5563  */
5564  SPECTRUM_DATA(cpl_image_get_data_double(spectrum), pos) = flux;
5565  SPECTRUM_DATA(cpl_mask_get_data(cpl_image_get_bpm(spectrum)), pos)
5566  = CPL_BINARY_0;
5567  /* The overhead of these function calls is negligible */
5568 
5569  if (spectrum_noise != NULL)
5570  {
5571  cpl_image_set(spectrum_noise, pos->x, spectrum_row, sqrt(variance));
5572  }
5573 
5574 
5575  /* Save sky (if extracted again) */
5576  if (optimal_extract_sky)
5577  {
5578  /* Change normalization of sky from 1 pixel to full slit,
5579  (i.e. same normalization as the extracted object)
5580 
5581  Error propagation is trivial (just multiply
5582  by same factor) because the
5583  uncertainty of 'slit_length' is negligible.
5584  */
5585 
5586  cpl_image_set(sky_spectrum , pos->x, spectrum_row,
5587  pos->sg.length * sky_background);
5588  cpl_image_set(sky_spectrum_noise, pos->x, spectrum_row,
5589  pos->sg.length * sky_background_noise);
5590  }
5591 
5592  /* Update S/N. Use only central 10% (max of blaze function)
5593  * to calculate S/N.
5594  * If order is partially without image, use all bins in order.
5595  */
5596  if (order_width < pos->nx ||
5597  (0.45*pos->nx <= pos->x && pos->x <= 0.55*pos->nx)
5598  )
5599  {
5600  cpl_table_set_double(
5601  signal_to_noise, "SN", sn_row, flux / sqrt(variance));
5602  sn_row++;
5603  }
5604 
5605  } /* for each x... */
5606  uves_msg_debug("%d/%d hot/cold pixels rejected", hot_pixels, cold_pixels);
5607 
5608  /* Return S/N */
5609  check_nomsg( cpl_table_set_size(signal_to_noise, sn_row) );
5610  if (sn_row > 0)
5611  {
5612  check_nomsg( *sn = cpl_table_get_column_median(signal_to_noise, "SN"));
5613  }
5614  else
5615  {
5616  *sn = 0;
5617  }
5618 
5619  cleanup:
5620  uves_free_table(&signal_to_noise);
5621  cpl_free(noise_buffer);
5622 
5623  return bins_extracted;
5624 }
5625 
5626 /*----------------------------------------------------------------------------*/
5649 /*----------------------------------------------------------------------------*/
5650 static double
5651 opt_get_sky(const double *image_data,
5652  const double *noise_data,
5653  const double *weights_data,
5654  uves_iterate_position *pos,
5655  const cpl_table *sky_map,
5656  double buffer_flux[], double buffer_noise[],
5657  double *sky_background_noise)
5658 {
5659  double sky_background;
5660  bool found_good = false; /* Any good pixels in current bin? */
5661  double flux_max = 0; /* Of all pixels in current bin */
5662  double flux_min = 0;
5663  int ngood = 0; /* Number of elements in arrays (good sky pixels) */
5664 
5665  /* Get image data (sky pixels that are also good pixels) */
5666  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
5667  {
5668  int row = pos->y - pos->ylow;
5669 
5670  if (!ISBAD(weights_data, pos))
5671  {
5672  double fflux = DATA(image_data, pos);
5673  double noise = DATA(noise_data, pos);
5674 
5675  if (!found_good)
5676  {
5677  found_good = true;
5678  flux_max = fflux;
5679  flux_min = fflux;
5680  }
5681  else
5682  {
5683  flux_max = uves_max_double(flux_max, fflux);
5684  flux_min = uves_min_double(flux_min, fflux);
5685  }
5686 
5687  /*if (pos->order == 1 && pos->x == 2825)
5688  {
5689  uves_msg_error("%d: %f +- %f%s", pos->y, fflux, noise,
5690  cpl_table_is_selected(sky_map, row) ? " *" : "");
5691  }
5692  */
5693 
5694  if (cpl_table_is_selected(sky_map, row))
5695  {
5696  buffer_flux [ngood] = fflux;
5697  buffer_noise[ngood] = noise;
5698  ngood++;
5699  }
5700  }
5701  }
5702 
5703  /* Get median of valid rows */
5704  if (ngood > 0)
5705  {
5706  /* Get noise of one sky pixel (assumed constant for all sky pixels) */
5707  double avg_noise = uves_tools_get_median(buffer_noise, ngood);
5708 
5709  sky_background = uves_tools_get_median(buffer_flux, ngood);
5710 
5711  /* If only 1 valid sky pixel */
5712  if (ngood == 1)
5713  {
5714  *sky_background_noise = avg_noise;
5715  }
5716  else
5717  {
5718  /* 2 or more sky pixels.
5719  *
5720  * Uncertainty of median is (approximately)
5721  *
5722  * sigma_median = sigma / sqrt(N * 2/pi) ; N >= 2
5723  *
5724  * where sigma is the (constant) noise of each pixel
5725  */
5726  *sky_background_noise = avg_noise / sqrt(ngood * 2 / M_PI);
5727  }
5728  }
5729  else
5730  /* No sky pixels, set noise as max - min */
5731  {
5732  if (found_good)
5733  {
5734  sky_background = flux_min;
5735  *sky_background_noise = flux_max - flux_min;
5736 
5737  /* In the rare case where max==min, set noise to
5738  something that's not zero */
5739  if (*sky_background_noise <= 0) *sky_background_noise = 1;
5740  }
5741  else
5742  /* No good pixels in bin */
5743  {
5744  sky_background = 0;
5745  *sky_background_noise = 1;
5746  }
5747  }
5748 
5749  /* if (pos->order == 1 && pos->x == 2825) uves_msg_error("sky = %f", sky_background); */
5750  return sky_background;
5751 
5752 }
5753 
5754 
5755 /*----------------------------------------------------------------------------*/
5765 /*----------------------------------------------------------------------------*/
5766 static double
5767 opt_get_noise_median(const double *noise_data, const cpl_binary *image_bpm,
5768  uves_iterate_position *pos, double noise_buffer[])
5769 {
5770  double median_noise; /* Result */
5771  int ngood; /* Number of good pixels */
5772 
5773  ngood = 0;
5774  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
5775  {
5776  if (ISGOOD(image_bpm, pos))
5777  {
5778  noise_buffer[ngood] = DATA(noise_data, pos);
5779  ngood++;
5780  }
5781  }
5782 
5783  if (ngood >= 1)
5784  {
5785  median_noise = uves_tools_get_median(noise_buffer, ngood);
5786  }
5787  else
5788  {
5789  median_noise = 1;
5790  }
5791 
5792  return median_noise;
5793 }
5794 
5795 /*----------------------------------------------------------------------------*/
5868 /*----------------------------------------------------------------------------*/
5869 
5870 static double
5871 opt_get_flux_sky_variance(const double *image_data, const double *noise_data,
5872  double *weights_data,
5873  uves_iterate_position *pos,
5874  const uves_extract_profile *profile,
5875  bool optimal_extract_sky,
5876  double median_noise,
5877  double *variance,
5878  double *sky_background,
5879  double *sky_background_noise)
5880 {
5881  double flux; /* Result */
5882  double sumpfv = 0; /* Sum of profile*flux / variance */
5883  double sumppv = 0; /* Sum of profile^2/variance */
5884  double sum1v = 0; /* Sum of 1 / variance */
5885  double sumpv = 0; /* Sum of profile / variance */
5886  double sumfv = 0; /* Sum of flux / variance */
5887 
5888  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
5889  {
5890  /* If pixel is not rejected, set weight and accumulate */
5891  if (!ISBAD(weights_data, pos))
5892  {
5893  double pixel_variance, pixelval, weight;
5894  double prof = uves_extract_profile_evaluate(profile, pos); /* is positive */
5895 
5896  pixelval = DATA(image_data, pos);
5897  pixel_variance = DATA(noise_data, pos);
5898  pixel_variance *= pixel_variance;
5899 
5900  if (median_noise >= 0 && pixel_variance < median_noise*median_noise)
5901  {
5902  /* Increase noise to median (otherwise, 'dead' pixels
5903  that aren't yet rejected will get too much weight) */
5904  pixel_variance = median_noise*median_noise;
5905  }
5906 
5907  weight = prof / pixel_variance;
5908  DATA(weights_data, pos) = weight;
5909  /* Assuming Horne's traditional formula
5910  which is a good approximation
5911  */
5912 
5913  sumpfv += pixelval * weight;
5914  sumppv += prof * weight;
5915  if (optimal_extract_sky)
5916  /* Optimization. Don't calculate if not needed. */
5917  {
5918  sumpv += weight;
5919  sum1v += 1 / pixel_variance;
5920  sumfv += pixelval / pixel_variance;
5921  }
5922  }
5923 
5924  /*
5925  if (pos->order == 1 && pos->x == 2825){
5926  if (ISBAD(weights_data, pos))
5927  uves_msg_error("%d: *", pos->y);
5928  else
5929  uves_msg_error("%d: %f +- %f", pos->y, DATA(image_data, pos), DATA(noise_data, pos));
5930  }
5931  */
5932 
5933  }
5934 
5935  if (!optimal_extract_sky)
5936  {
5937  /* Horne's traditional formulas */
5938  if (sumppv > 0 && !irplib_isnan(sumppv) && !irplib_isinf(sumppv))
5939  {
5940  flux = sumpfv / sumppv;
5941  *variance = 1 / sumppv;
5942  }
5943  else
5944  {
5945  flux = 0;
5946  *variance = 1;
5947  }
5948  }
5949  else
5950  {
5951  /* Generalization of Horne explained above */
5952  long double denominator = (long double)sum1v*sumppv - (long double)sumpv*sumpv;
5953 /* to fix a problem on 64 bit due to the fact denominator can be very small, we cast iit to long double and then compare it abs value with a small number, like DBL_MIN */
5954  if (fabsl(denominator) > DBL_MIN)
5955  {
5956  flux = ((long double)sum1v * sumpfv - (long double)sumpv * sumfv) / denominator;
5957 /*
5958  if(flux > 1.e40 || flux < -1.e40) {
5959  uves_msg_warning("Very large optimally extracted flux=%g sum1v=%g sumpfv=%g sumpv=%g sumfv=%g denominator=%lg",flux,sum1v,sumpfv,sumpv,sumfv,denominator);
5960  }
5961 */
5962  /* Traditional formula, underestimates the error bars
5963  and results in a (false) higher S/N
5964  *variance = 1 / sumppv;
5965  */
5966 
5967  /* Formula which takes into account the uncertainty
5968  of the sky subtraction: */
5969  *variance = (long double)sum1v / denominator;
5970 
5971  *sky_background = (sumppv*sumfv - sumpv*sumpfv) / denominator;
5972  *sky_background_noise = sqrt(sumppv / denominator);
5973  }
5974  else
5975  {
5976  flux = 0;
5977  *variance = 1;
5978 
5979  *sky_background = 0;
5980  *sky_background_noise = 1;
5981  }
5982  }
5983 
5984  /*
5985  if (pos->order == 1 && pos->x == 2825)
5986  {if (sky_background)
5987  uves_msg_error("sky = %f", *sky_background);
5988  }
5989  */
5990 
5991  return flux;
5992 }
5993 
5994 
5995 /*---------------------------------------------------------------------------*/
6020 /*---------------------------------------------------------------------------*/
6021 static bool
6022 opt_reject_outlier(const double *image_data,
6023  const double *noise_data,
6024  cpl_binary *image_bpm,
6025  double *weights_data,
6026  uves_iterate_position *pos,
6027  const uves_extract_profile *profile,
6028  double kappa,
6029  double flux,
6030  double sky_background,
6031  double red_chisq,
6032  cpl_table *cosmic_mask,
6033  int *cr_row,
6034  int *hot_pixels,
6035  int *cold_pixels)
6036 {
6037  bool found_outlier = false; /* Result */
6038 
6039  int y_outlier = -1; /* Position of worst outlier */
6040  double max_residual_sq = 0; /* Residual^2/sigma^2 of
6041  worst outlier */
6042  bool outlier_is_hot = false; /* true iff residual is positive */
6043  int new_crh_tab_size=0;
6044  int crh_tab_size=0;
6045 
6046  /* Find worst outlier */
6047  for (pos->y = pos->ylow; pos->y <= pos->yhigh; pos->y++)
6048  {
6049  double prof = uves_extract_profile_evaluate(profile, pos);
6050  double pixel_variance, pixelval;
6051  double best_fit;
6052 
6053  pixel_variance = DATA(noise_data, pos);
6054  pixel_variance *= pixel_variance;
6055 
6056  pixelval = DATA(image_data, pos);
6057 
6058  best_fit = flux * prof + sky_background;/* This part used to be a stupid
6059  bug: the sky contribution was
6060  forgotten
6061  -> most pixels were outliers
6062  This bug was in the MIDAS
6063  version and independently
6064  reimplemented in
6065  first CPL versions(!)
6066  */
6067 
6068  if (!ISBAD(weights_data, pos) &&
6069  /* for efficiency, don't:
6070  fabs(pixelval - flux * prof) / sigma >= sqrt(max_residual_sq)
6071  */
6072  (pixelval - best_fit)*(pixelval - best_fit) / pixel_variance
6073  >= max_residual_sq)
6074  {
6075  max_residual_sq =
6076  (pixelval - best_fit) *
6077  (pixelval - best_fit) / pixel_variance;
6078 
6079  y_outlier = pos->y;
6080 
6081  outlier_is_hot = (pixelval > best_fit);
6082  }
6083  }
6084 
6085  /* Reject outlier
6086  if residual is larger than kappa sigma sqrt(red_chisq), i.e.
6087  if res^2/sigma^2 > kappa^2 * chi^2/N
6088  */
6089  if (max_residual_sq > kappa*kappa * red_chisq)
6090  {
6091  uves_msg_debug("Order #%d: Bad pixel at (x, y) = (%d, %d) residual^2 = %.2f sigma^2",
6092  pos->order, pos->x, y_outlier, max_residual_sq);
6093 
6094  pos->y = y_outlier;
6095  SETBAD(weights_data, image_bpm, pos);
6096 
6097  found_outlier = true;
6098  if (outlier_is_hot)
6099  {
6100  *hot_pixels += 1;
6101 
6102  /* Update cosmic ray table. If it is too short, double the size */
6103  crh_tab_size=cpl_table_get_nrow(cosmic_mask);
6104  while (*cr_row >= crh_tab_size )
6105  {
6106  new_crh_tab_size=( *cr_row > 2*crh_tab_size) ? (*cr_row)+10: 2*crh_tab_size;
6107  cpl_table_set_size(cosmic_mask,new_crh_tab_size );
6108  crh_tab_size=cpl_table_get_nrow(cosmic_mask);
6109  }
6110 
6111  check(( cpl_table_set_int (cosmic_mask, "Order", *cr_row, pos->order),
6112  cpl_table_set_int (cosmic_mask, "X" , *cr_row, pos->x),
6113  cpl_table_set_int (cosmic_mask, "Y" , *cr_row, y_outlier),
6114  cpl_table_set_double(cosmic_mask, "Flux" , *cr_row,
6115  DATA(image_data, pos)),
6116  (*cr_row)++),
6117  "Error updating cosmic ray table");
6118  }
6119  else
6120  {
6121  *cold_pixels += 1;
6122  }
6123  }
6124 
6125 
6126  cleanup:
6127  return found_outlier;
6128 }
6129 
6130 /*----------------------------------------------------------------------------*/
6140 /*----------------------------------------------------------------------------*/
6141 static double
6142 opt_get_redchisq(const uves_extract_profile *profile,
6143  const uves_iterate_position *pos)
6144 {
6145  if (profile->constant) {
6146  return 1.0;
6147  }
6148  if (profile->f != NULL)
6149  {
6150  return uves_max_double(1,
6151 #if ORDER_PER_ORDER
6153  profile->red_chisq[pos->order-pos->minorder], pos->x));
6154 #else
6156  profile->red_chisq, pos->x, pos->order));
6157 #endif
6158  }
6159  else
6160  {
6161  /* Virtual resampling, don't adjust kappa */
6162  return 1.0;
6163  }
6164 }
6165 
6166 /*----------------------------------------------------------------------------*/
6186 /*----------------------------------------------------------------------------*/
6187 static polynomial *
6188 repeat_orderdef(const cpl_image *image, const cpl_image *image_noise,
6189  const polynomial *guess_locations,
6190  int minorder, int maxorder, slit_geometry sg,
6191  cpl_table *info_tbl)
6192 {
6193  polynomial *order_locations = NULL;
6194  int nx = cpl_image_get_size_x(image);
6195  int ny = cpl_image_get_size_y(image);
6196  double max_shift = sg.length/2; /* pixels in y-direction */
6197  int stepx = 10;
6198  int x, order;
6199  int ordertab_row; /* First unused row of ordertab */
6200  cpl_table *ordertab = NULL;
6201  cpl_table *temp = NULL;
6202 
6203  ordertab = cpl_table_new((maxorder - minorder + 1)*nx);
6204  ordertab_row = 0;
6205  cpl_table_new_column(ordertab, "X" , CPL_TYPE_INT);
6206  cpl_table_new_column(ordertab, "Order", CPL_TYPE_INT);
6207  cpl_table_new_column(ordertab, "Y" , CPL_TYPE_DOUBLE);
6208  cpl_table_new_column(ordertab, "Yold" , CPL_TYPE_DOUBLE);
6209  cpl_table_new_column(ordertab, "Sigma", CPL_TYPE_DOUBLE);
6210  cpl_table_set_column_unit(ordertab, "Y", "pixels");
6211 
6212  /* Measure */
6213  for (order = minorder; order <= maxorder; order++) {
6214  for (x = 1 + stepx/2; x <= nx; x += stepx) {
6215  double ycenter;
6216  int yhigh, ylow;
6217 
6218  double y_0, sigma, norm, background;
6219  check( ycenter = uves_polynomial_evaluate_2d(guess_locations, x, order),
6220  "Error evaluating polynomial");
6221 
6222  ylow = uves_round_double(ycenter - max_shift);
6223  yhigh = uves_round_double(ycenter + max_shift);
6224 
6225  if (1 <= ylow && yhigh <= ny) {
6226  uves_fit_1d_image(image, image_noise, NULL,
6227  false, /* Horizontal? */
6228  false, false, /* Fix/fit background? */
6229  ylow, yhigh, x, /* yrange, x */
6230  &y_0, &sigma, &norm, &background, NULL,
6231  NULL, NULL, NULL, /* mse, chi^2/N, covariance */
6233 
6234  if (cpl_error_get_code() == CPL_ERROR_CONTINUE) {
6235  uves_error_reset();
6236  uves_msg_debug("Profile fitting failed "
6237  "at (x,y) = (%d, %e), ignoring bin",
6238  x, ycenter);
6239  }
6240  else {
6241  assure(cpl_error_get_code() == CPL_ERROR_NONE,
6242  cpl_error_get_code(), "Gaussian fitting failed");
6243 
6244  cpl_table_set_int (ordertab, "X" , ordertab_row, x);
6245  cpl_table_set_int (ordertab, "Order" , ordertab_row, order);
6246  cpl_table_set_double(ordertab, "Y" , ordertab_row, y_0);
6247  cpl_table_set_double(ordertab, "Yold" , ordertab_row, ycenter);
6248  cpl_table_set_double(ordertab, "Sigma" , ordertab_row, sigma);
6249  ordertab_row += 1;
6250  }
6251  }
6252  }
6253  }
6254 
6255  cpl_table_set_size(ordertab, ordertab_row);
6256 
6257  /* Fit */
6258  if (ordertab_row < 300)
6259  {
6260  uves_msg_warning("Too few points (%d) to reliably fit order polynomial. "
6261  "Using calibration solution", ordertab_row);
6262 
6263  uves_polynomial_delete(&order_locations);
6264  order_locations = uves_polynomial_duplicate(guess_locations);
6265 
6266  cpl_table_duplicate_column(ordertab, "Yfit", ordertab, "Yold");
6267  }
6268  else
6269  {
6270  int max_degree = 10;
6271  double kappa = 4.0;
6272  double min_rms = 0.05; /* Pixels (stop at this point, for efficiency) */
6273 
6274  order_locations =
6276  "X", "Order", "Y", NULL,
6277  "Yfit", NULL, NULL,
6278  NULL, NULL, NULL,
6279  kappa,
6280  max_degree, max_degree, min_rms, -1,
6281  true,
6282  NULL, NULL, -1, NULL);
6283 
6284  if (cpl_error_get_code() == CPL_ERROR_SINGULAR_MATRIX)
6285  {
6286  uves_error_reset();
6287  uves_msg_warning("Could not fit new order polynomial. "
6288  "Using calibration solution");
6289 
6290  uves_polynomial_delete(&order_locations);
6291  order_locations = uves_polynomial_duplicate(guess_locations);
6292 
6293  cpl_table_duplicate_column(ordertab, "Yfit", ordertab, "Yold");
6294 
6295  /* Compute shift, also in this case */
6296  }
6297  else
6298  {
6299  assure( cpl_error_get_code() == CPL_ERROR_NONE,
6300  cpl_error_get_code(),
6301  "Error fitting orders polynomial");
6302  }
6303  }
6304 
6305  /* Yshift := Yfit - Yold */
6306  cpl_table_duplicate_column(ordertab, "Yshift", ordertab, "Yfit"); /* Yshift := Yfit */
6307  cpl_table_subtract_columns(ordertab, "Yshift", "Yold"); /* Yshift := Yshift - Yold */
6308 
6309  {
6310  double mean = cpl_table_get_column_mean(ordertab, "Yshift");
6311  double stdev = cpl_table_get_column_mean(ordertab, "Yshift");
6312  double rms = sqrt(mean*mean + stdev*stdev);
6313 
6314  uves_msg("Average shift with respect to calibration solution is %.2f pixels", rms);
6315  }
6316 
6317  /* Compute object postion+FWHM wrt old solution (for QC) */
6318  for (order = minorder; order <= maxorder; order++)
6319  {
6320  double pos =
6321  uves_polynomial_evaluate_2d(order_locations, nx/2, order)-
6322  uves_polynomial_evaluate_2d(guess_locations, nx/2, order);
6323 
6324  double fwhm;
6325 
6326 
6327  /* Extract rows with "Order" equal to current order,
6328  but avoid == comparison of floating point values */
6329  uves_free_table(&temp);
6330  temp = uves_extract_table_rows(ordertab, "Order",
6331  CPL_EQUAL_TO,
6332  order); /* Last argument is double, will
6333  be rounded to nearest integer */
6334 
6335  if (cpl_table_get_nrow(temp) < 1)
6336  {
6337  uves_msg_warning("Problem tracing object in order %d. "
6338  "Setting QC FHWM parameter to zero",
6339  order);
6340  fwhm = 0;
6341  }
6342  else
6343  {
6344  fwhm = cpl_table_get_column_median(temp, "Sigma") * TWOSQRT2LN2;
6345  }
6346 
6347 
6348  cpl_table_set_int (info_tbl, "Order", order - minorder, order);
6349  cpl_table_set_double(info_tbl, "ObjPosOnSlit" , order - minorder,
6350  pos - (-sg.length/2 + sg.offset));
6351  cpl_table_set_double(info_tbl, "ObjFwhmAvg" , order - minorder, fwhm);
6352  }
6353 
6354  cleanup:
6355  uves_free_table(&ordertab);
6356  uves_free_table(&temp);
6357 
6358  return order_locations;
6359 }
6360 
int uves_polynomial_get_dimension(const polynomial *p)
Get the dimension of a polynomial.
static int opt_get_order_width(const uves_iterate_position *pos)
Get width of order.
double uves_polynomial_solve_2d(const polynomial *p, double value, double guess, int multiplicity, int varno, double x_value)
Solve p(x1, x2) = value.
#define uves_msg_error(...)
Print an error message.
Definition: uves_msg.h:64
void uves_polynomial_delete(polynomial **p)
Delete a polynomial.
#define uves_msg_warning(...)
Print an warning message.
Definition: uves_msg.h:87
static uves_extract_profile * opt_measure_profile(const cpl_image *image, const cpl_image *image_noise, const cpl_image *weights, uves_iterate_position *pos, int chunk, int sampling_factor, int(*f)(const double x[], const double a[], double *result), int(*dfda)(const double x[], const double a[], double result[]), int M, const cpl_image *sky_spectrum, cpl_table *info_tbl, cpl_table **profile_global)
Measure spatial profile (all orders)
bool uves_iterate_finished(const uves_iterate_position *p)
Finished iterating?
static cpl_table ** opt_sample_spatial_profile(const cpl_image *image, const cpl_image *weights, uves_iterate_position *pos, int chunk, int sampling_factor, int *nbins)
Sample spatial profile.
void uves_iterate_delete(uves_iterate_position **p)
Deallocate iterator and set pointer to NULL.
static int opt_extract(cpl_image *image, const cpl_image *image_noise, uves_iterate_position *pos, const uves_extract_profile *profile, bool optimal_extract_sky, double kappa, cpl_table *blemish_mask, cpl_table *cosmic_mask, int *cr_row, cpl_table *profile_table, int *prof_row, cpl_image *spectrum, cpl_image *spectrum_noise, cpl_image *weights, cpl_image *sky_spectrum, cpl_image *sky_spectrum_noise, double *sn)
Optimally extract order using the given the profile.
cpl_image * uves_define_noise(const cpl_image *image, const uves_propertylist *image_header, int ncom, enum uves_chip chip)
Create noise image.
Definition: uves_utils.c:2226
#define check_nomsg(CMD)
Definition: uves_error.h:204
uves_iterate_position * uves_iterate_new(int nx, int ny, const polynomial *order_locations, int minorder, int maxorder, slit_geometry sg)
Allocate iterator.
double uves_pow_int(double x, int y)
Calculate x to the y'th.
Definition: uves_utils.c:1593
#define passure(BOOL,...)
Definition: uves_error.h:207
void uves_iterate_set_first(uves_iterate_position *p, int xmin, int xmax, int ordermin, int ordermax, const cpl_binary *bpm, bool loop_y)
Initialize iteration.
int uves_gauss_derivative(const double x[], const double a[], double result[])
Evaluate the derivatives of a gaussian.
Definition: uves_utils.c:4346
static double opt_get_redchisq(const uves_extract_profile *profile, const uves_iterate_position *pos)
Get reduced chi^2 for current bin.
static bool opt_reject_outlier(const double *image_data, const double *noise_data, cpl_binary *image_bpm, double *weights_data, uves_iterate_position *pos, const uves_extract_profile *profile, double kappa, double flux, double sky_background, double red_chisq, cpl_table *cosmic_mask, int *cr_row, int *hot_pixels, int *cold_pixels)
Find and reject outlier pixel.
static double detect_ripples(const cpl_image *spectrum, const uves_iterate_position *pos, double sn)
Try to detect and warn about any optimal extraction ripples (happening if oversampling factor is too ...
double uves_polynomial_derivative_2d(const polynomial *p, double x1, double x2, int varno)
Evaluate the partial derivative of a 2d polynomial.
uves_propertylist * uves_initialize_image_header(const char *ctype1, const char *ctype2, const char *cunit1, const char *cunit2, const char *bunit, const double bscale, double crval1, double crval2, double crpix1, double crpix2, double cdelt1, double cdelt2)
Initialize image header.
Definition: uves_utils.c:2174
static polynomial * repeat_orderdef(const cpl_image *image, const cpl_image *image_noise, const polynomial *guess_locations, int minorder, int maxorder, slit_geometry sg, cpl_table *info_tbl)
Refine order definition using the science frame.
int uves_polynomial_get_degree(const polynomial *p)
Get degree.
#define uves_msg(...)
Print a message on 'info' or 'debug' level.
Definition: uves_msg.h:119
int uves_gauss(const double x[], const double a[], double *result)
Evaluate a gaussian.
Definition: uves_utils.c:4291
int uves_moffat(const double x[], const double a[], double *result)
Evaluate a Moffat.
Definition: uves_utils.c:4240
polynomial * uves_polynomial_duplicate(const polynomial *p)
Copy a polynomial.
polynomial * uves_polynomial_regression_2d(cpl_table *t, const char *X1, const char *X2, const char *Y, const char *sigmaY, int degree1, int degree2, const char *polynomial_fit, const char *residual_square, const char *variance_fit, double *mse, double *red_chisq, polynomial **variance, double kappa, double min_reject)
Fit a 2d polynomial to three table columns.
Definition: uves_utils.c:2869
cpl_image * uves_extract(cpl_image *image, cpl_image *image_noise, const uves_propertylist *image_header, const cpl_table *ordertable, const polynomial *order_locations_raw, double slit_length, double offset, const cpl_parameterlist *parameters, const char *context, const char *mode, bool extract_partial, bool debug_mode, enum uves_chip chip, uves_propertylist **header, cpl_image **spectrum_noise, cpl_image **sky_spectrum, cpl_image **sky_spectrum_noise, cpl_table **cosmic_mask, cpl_image **cosmic_image, cpl_table **profile_table, cpl_image **weights, cpl_table **info_tbl, cpl_table **order_trace)
Extract a spectrum.
Definition: uves_extract.c:569
polynomial * uves_polynomial_regression_2d_autodegree(cpl_table *t, const char *X1, const char *X2, const char *Y, const char *sigmaY, const char *polynomial_fit, const char *residual_square, const char *variance_fit, double *mean_squared_error, double *red_chisq, polynomial **variance, double kappa, int maxdeg1, int maxdeg2, double min_rms, double min_reject, bool verbose, const double *min_val, const double *max_val, int npos, double positions[][2])
Fit a 2d polynomial to three table columns.
Definition: uves_utils.c:3305
cpl_parameterlist * uves_extract_define_parameters(void)
Define recipe parameters used for extraction.
Definition: uves_extract.c:264
extract_method uves_get_extract_method(const cpl_parameterlist *parameters, const char *context, const char *subcontext)
Read extraction method from parameter list.
Definition: uves_extract.c:462
#define assure_mem(PTR)
Definition: uves_error.h:181
cpl_image * uves_create_image(uves_iterate_position *pos, enum uves_chip chip, const cpl_image *spectrum, const cpl_image *sky, const cpl_image *cosmic_image, const uves_extract_profile *profile, cpl_image **image_noise, uves_propertylist **image_header)
Reconstruct echelle image from spectrum.
Definition: uves_utils.c:4534
static double estimate_sn(const cpl_image *image, const cpl_image *image_noise, uves_iterate_position *pos)
Estimate the S/N of the input frame.
double uves_polynomial_evaluate_2d(const polynomial *p, double x1, double x2)
Evaluate a 2d polynomial.
double uves_polynomial_evaluate_1d(const polynomial *p, double x)
Evaluate a 1d polynomial.
static cpl_table * opt_measure_profile_order(const cpl_image *image, const cpl_image *image_noise, const cpl_binary *image_bpm, uves_iterate_position *pos, int chunk, int(*f)(const double x[], const double a[], double *result), int(*dfda)(const double x[], const double a[], double result[]), int M, const cpl_image *sky_spectrum)
Measure spatial profile (analytical)
double uves_polynomial_get_coeff_1d(const polynomial *p, int degree)
Get a coefficient of a 1D polynomial.
double uves_tools_get_median(double *a, int n)
returns median (not CPL median) of an array
int uves_moffat_derivative(const double x[], const double a[], double result[])
Evaluate Moffat derivative.
Definition: uves_utils.c:4259
static cpl_image * opt_subtract_sky(const cpl_image *image, const cpl_image *image_noise, const cpl_image *weights, uves_iterate_position *pos, const cpl_table *sky_map, cpl_image *sky_spectrum, cpl_image *sky_spectrum_noise)
Measure and subtract sky.
polynomial * uves_polynomial_regression_1d(cpl_table *t, const char *X, const char *Y, const char *sigmaY, int degree, const char *polynomial_fit, const char *residual_square, double *mean_squared_error, double kappa)
Fit a 1d polynomial to two table columns.
Definition: uves_utils.c:2590
void uves_polynomial_dump(const polynomial *p, FILE *stream)
Print a polynomial.
const char * uves_tostring_cpl_type(cpl_type t)
Convert a CPL type to a string.
Definition: uves_dump.c:378
static double opt_get_sky(const double *image_data, const double *noise_data, const double *weights_data, uves_iterate_position *pos, const cpl_table *sky_map, double buffer_flux[], double buffer_noise[], double *sky_background_noise)
Measure sky level (median)
#define uves_error_reset()
Definition: uves_error.h:215
#define uves_msg_low(...)
Print a message on a lower message level.
Definition: uves_msg.h:105
cpl_error_code uves_polynomial_shift(polynomial *p, int varno, double shift)
Shift a polynomial.
#define uves_msg_debug(...)
Print a debug message.
Definition: uves_msg.h:97
static double opt_get_noise_median(const double *noise_data, const cpl_binary *image_bpm, uves_iterate_position *pos, double noise_buffer[])
Measure median noise level of extraction bin.
static double area_above_line(int y, double left, double right)
Calculate the area of a pixel that is above a line.
static int extract_order_simple(const cpl_image *image, const cpl_image *image_noise, const polynomial *order_locations, int order, int minorder, int spectrum_row, double offset, double slit_length, extract_method method, const cpl_image *weights, bool extract_partial, cpl_image *spectrum, cpl_image *spectrum_noise, cpl_binary *spectrum_badmap, cpl_table **info_tbl, double *sn)
Extract one order using linear, average or weighted extraction.
static double opt_get_flux_sky_variance(const double *image_data, const double *noise_data, double *weights_data, uves_iterate_position *pos, const uves_extract_profile *profile, bool optimal_extract_sky, double median_noise, double *variance, double *sky_background, double *sky_background_noise)
Get flux, sky and variances of current bin.
static cpl_table * opt_define_sky(const cpl_image *image, const cpl_image *weights, uves_iterate_position *pos)
Define sky/object rows.
#define check(CMD,...)
Definition: uves_error.h:198
polynomial * uves_polynomial_new_zero(int dim)
Create a zero polynomial.
static void revise_noise(cpl_image *image_noise, const cpl_binary *image_bpm, const uves_propertylist *image_header, uves_iterate_position *pos, const cpl_image *spectrum, const cpl_image *sky_spectrum, const uves_extract_profile *profile, enum uves_chip chip)
Refine error bars.
static cpl_image * opt_extract_sky(const cpl_image *image, const cpl_image *image_noise, const cpl_image *weights, uves_iterate_position *pos, cpl_image *sky_spectrum, cpl_image *sky_spectrum_noise)
Extract and subtract sky.
void uves_iterate_increment(uves_iterate_position *p)
Get next position.