MUSE Pipeline Reference Manual  1.0.2
muse_processing.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #include <stdio.h>
30 #include <float.h>
31 #include <math.h>
32 #include <string.h>
33 #include <cpl.h>
34 
35 #include "muse_processing.h"
36 #include "muse_instrument.h"
37 
38 #include "muse_dfs.h"
39 #include "muse_pfits.h"
40 #include "muse_lsf.h"
41 #include "muse_datacube.h"
42 #include "muse_utils.h"
43 
44 /*----------------------------------------------------------------------------*
45  * Debugging Macros *
46  * Set these to 1 or higher for (lots of) debugging output *
47  *----------------------------------------------------------------------------*/
48 #define DEBUG_EXPOSURES_SORT 0 /* debug muse_processing_sort_exposures() */
49 
50 /*----------------------------------------------------------------------------*/
57 /*----------------------------------------------------------------------------*/
58 
61 /*----------------------------------------------------------------------------*
62  * Functions *
63  *----------------------------------------------------------------------------*/
64 
65 /*---------------------------------------------------------------------------*/
75 /*---------------------------------------------------------------------------*/
76 static cpl_array *
77 muse_processing_get_rawtags(cpl_recipe *aRecipe)
78 {
79  /* start with zero length array, that is returned on error */
80  cpl_array *rawtags = cpl_array_new(0, CPL_TYPE_STRING);
81  /* we need a recipe config for this to work */
82  cpl_recipeconfig *recipeconfig = muse_processing_get_recipeconfig(aRecipe);
83  if (!recipeconfig) {
84  cpl_msg_error(__func__, "No recipeconfig found!");
85  return rawtags;
86  }
87 
88  /* loop through all frames of the recipe (which at this point will be *
89  * just the input frames) and check if they are inputs, i.e. raw frames */
90  cpl_size iframe, nframes = cpl_frameset_get_size(aRecipe->frames);
91  for (iframe = 0; iframe < nframes; iframe++) {
92  cpl_frame *frame = cpl_frameset_get_position(aRecipe->frames, iframe);
93  const char *tag = cpl_frame_get_tag(frame); /* fiducial raw tag to check */
94  /* see if it's already in our list */
95  cpl_size itag, ntags = cpl_array_get_size(rawtags);
96  for (itag = 0; itag < ntags; itag++) {
97  if (!strcmp(cpl_array_get_string(rawtags, itag), tag)) {
98  tag = NULL; /* set to NULL to make cpl_recipeconfig_get_inputs() fail */
99  break;
100  } /* if match */
101  } /* for itag */
102  cpl_errorstate state = cpl_errorstate_get();
103  char **tags = cpl_recipeconfig_get_inputs(recipeconfig, tag);
104  if (!tags) { /* not a raw tag! */
105  cpl_errorstate_set(state); /* CPL will have set an error, swallow it */
106  continue;
107  } /* if */
108  /* we have a raw tag! record this tag in the output array of raw tags */
109  cpl_array_set_size(rawtags, ntags + 1);
110  cpl_array_set_string(rawtags, ntags, tag);
111  /* clean up the elements which we didn't actually need */
112  int i;
113  for (i = 0; tags[i]; i++) {
114  cpl_free(tags[i]);
115  }
116  cpl_free(tags);
117  } /* for iframe (all recipe frames) */
118  if (!cpl_array_get_size(rawtags)) {
119  cpl_msg_error(__func__, "No valid raw tag(s) found!");
120  }
121  return rawtags;
122 } /* muse_processing_get_rawtags() */
123 
124 /*---------------------------------------------------------------------------*/
131 /*---------------------------------------------------------------------------*/
133 muse_processing_new(const char *aName, cpl_recipe *aRecipe)
134 {
135  muse_processing *processing = cpl_malloc(sizeof(muse_processing));
136  processing->name = aName;
137  processing->recipe = aRecipe;
138  processing->inframes = cpl_frameset_duplicate(aRecipe->frames);
139  processing->usedframes = cpl_frameset_new();
140  processing->outframes = cpl_frameset_new();
141  #pragma omp critical(cpl_parameters)
142  processing->parameters = muse_cplparameterlist_duplicate(aRecipe->parameters);
143  processing->intags = muse_processing_get_rawtags(aRecipe);
144  processing->counter = cpl_malloc(sizeof(muse_processing_framecounter));
145  processing->counter[0].tag = NULL;
146  return processing;
147 } /* muse_processing_new() */
148 
149 /*---------------------------------------------------------------------------*/
154 /*---------------------------------------------------------------------------*/
155 void
157 {
158  if (!aProcessing) {
159  return;
160  }
161  cpl_array_delete(aProcessing->intags);
162  cpl_frameset_delete(aProcessing->inframes);
163  cpl_frameset_delete(aProcessing->usedframes);
164  cpl_frameset_delete(aProcessing->outframes);
165  cpl_parameterlist_delete(aProcessing->parameters);
166  int i;
167  for (i = 0; aProcessing->counter[i].tag != NULL; i++) {
168  cpl_free((char *)aProcessing->counter[i].tag);
169  }
170  cpl_free(aProcessing->counter);
171  cpl_free(aProcessing);
172 } /* muse_processing_delete() */
173 
174 /*---------------------------------------------------------------------------*/
190 /*---------------------------------------------------------------------------*/
191 void
193  cpl_frame *aFrame, cpl_frame_group aGroup,
194  int aDuplicate)
195 {
196  if (!aProcessing) {
197  cpl_msg_error(__func__, "NULL processing struct!");
198  return;
199  }
200 #if 0
201  if (aGroup == CPL_FRAME_GROUP_CALIB) {
202  cpl_msg_debug(__func__, "using %s %s frame %s", "calibration",
203  cpl_frame_get_tag(aFrame), cpl_frame_get_filename(aFrame));
204  } else if (aGroup == CPL_FRAME_GROUP_RAW) {
205  cpl_msg_debug(__func__, "using %s %s frame %s", "raw",
206  cpl_frame_get_tag(aFrame), cpl_frame_get_filename(aFrame));
207  }
208 #endif
209  /* check, if this frame is already listed */
210  const char *fn = cpl_frame_get_filename(aFrame),
211  *tag = cpl_frame_get_tag(aFrame);
212  cpl_size iframe, nframes = cpl_frameset_get_size(aProcessing->usedframes);
213  for (iframe = 0;
214  (iframe < nframes) && fn && tag; /* only check with valid info */
215  iframe++) {
216  cpl_frame *frame = cpl_frameset_get_position(aProcessing->usedframes, iframe);
217  const char *fn2 = cpl_frame_get_filename(frame),
218  *tag2 = cpl_frame_get_tag(frame);
219  if (fn2 && !strncmp(fn, fn2, strlen(fn) + 1) &&
220  tag2 && !strncmp(tag, tag2, strlen(tag) + 1)) {
221  if (!aDuplicate) { /* destroy frame, assuming it was duplicated elsewhere */
222  cpl_frame_delete(aFrame);
223  }
224  return;
225  }
226  } /* for iframe (all usedframes) */
227  cpl_frame_set_group(aFrame, aGroup);
228  if (aDuplicate) {
229  cpl_frameset_insert(aProcessing->usedframes, cpl_frame_duplicate(aFrame));
230  } else {
231  cpl_frameset_insert(aProcessing->usedframes, aFrame);
232  }
233 }
234 
235 /*---------------------------------------------------------------------------*/
248 /*---------------------------------------------------------------------------*/
249 static int
250 muse_processing_get_framecounter(muse_processing *aProcessing,
251  const char *aTag, int aIFU)
252 {
253  if (!aProcessing) {
254  cpl_msg_error(__func__, "NULL processing struct!");
255  return 0;
256  }
257  int i;
258  for (i = 0; aProcessing->counter[i].tag != NULL; i++) {
259  if (strcmp(aProcessing->counter[i].tag, aTag) == 0 &&
260  aProcessing->counter[i].ifu == aIFU) {
261  return ++aProcessing->counter[i].counter;
262  }
263  }
264  aProcessing->counter = cpl_realloc(aProcessing->counter,
265  (i+2)*sizeof(muse_processing_framecounter));
266  aProcessing->counter[i].tag = cpl_strdup(aTag);
267  aProcessing->counter[i].ifu = aIFU;
268  aProcessing->counter[i].counter = 1;
269  aProcessing->counter[i+1].tag = NULL;
270  return 1;
271 }
272 
273 /*---------------------------------------------------------------------------*/
286 /*---------------------------------------------------------------------------*/
287 static void
288 muse_processing_setup_header(muse_processing *aProcessing,
289  cpl_propertylist *aHeader, cpl_frame *aFrame,
290  int aIndex, const char *aDateObs,
291  cpl_boolean aSequence)
292 {
293  if (!aProcessing) {
294  cpl_msg_error(__func__, "NULL processing struct!");
295  return;
296  }
297  if (!aProcessing->inframes ||
298  cpl_frameset_is_empty(aProcessing->inframes)) {
299  cpl_msg_warning(__func__, "No raw input files, no DFS product header added");
300  return;
301  }
302  /* save some headers which are be over overwritten by cpl_dfs, *
303  * including OBJECT and ESO.DRS.MUSE */
304  cpl_propertylist *hkeep = cpl_propertylist_new();
305  cpl_propertylist_copy_property_regexp(hkeep, aHeader,
306  MUSE_HEADERS_KEEP_REGEXP, 0);
307  /* don't want any DFS-generated ESO.PRO keywords before calling cpl_dfs *
308  * to create them, so delete all ESO.PRO stuff and COMMENTS */
309  cpl_propertylist_erase_regexp(aHeader, "^ESO PRO|^COMMENT", 0);
310  /* remove temporary stuff that is not meant to be saved to disk */
311  cpl_propertylist_erase_regexp(aHeader, MUSE_HDR_TMP_REGEXP, 0);
312 
313  /* make sure that the raw frame(s) are first in the list of used frames */
314  cpl_frameset *fset = muse_frameset_sort_raw_other(aProcessing->usedframes,
315  aIndex, aDateObs, aSequence);
316 
317  /* XXX workaround for the EQUINOX warning, replace with double-type keyword */
318  if (cpl_propertylist_has(aHeader, "EQUINOX") &&
319  cpl_propertylist_get_type(aHeader, "EQUINOX") < CPL_TYPE_FLOAT) {
320  cpl_property *equinox = cpl_propertylist_get_property(aHeader, "EQUINOX");
321  double equvalue = cpl_property_get_long_long(equinox);
322  const char *equcomment = cpl_property_get_comment(equinox);
323  cpl_property_set_name(equinox, "EQUIBRK");
324  cpl_propertylist_insert_after_double(aHeader, "EQUIBRK", "EQUINOX",
325  equvalue);
326  cpl_propertylist_set_comment(aHeader, "EQUINOX", equcomment);
327  cpl_propertylist_erase(aHeader, "EQUIBRK");
328  } /* if non-float EQUINOX */
329 
330  char *pkgstring = cpl_sprintf("%s/%s", PACKAGE, PACKAGE_VERSION);
331  if (cpl_dfs_setup_product_header(aHeader, aFrame, fset,
332  aProcessing->parameters, aProcessing->name,
333  pkgstring, MUSE_PRO_DID, NULL)
334  != CPL_ERROR_NONE) {
335  cpl_msg_error(__func__, "Could not add DFS product header: %s",
336  cpl_error_get_message());
337  }
338  cpl_free(pkgstring);
339  cpl_frameset_delete(fset);
340 
341  /* reset overwritten headers (if they were there before) */
342  int i, n = cpl_propertylist_get_size(hkeep);
343  for (i = 0; i < n; i++) {
344  const cpl_property *p = cpl_propertylist_get_const(hkeep, i);
345  cpl_propertylist_erase(aHeader, cpl_property_get_name(p));
346  cpl_propertylist_append_property(aHeader, p);
347  } /* for i (all properties in hkeep) */
348  cpl_propertylist_delete(hkeep);
349 
350  /* MUSE data is always taken with an IFU, so override PRO.TECH */
351  cpl_propertylist_update_string(aHeader, "ESO PRO TECH", "IFU");
352  /* add PRO.SCIENCE header for science recipes */
353  if (strstr(aProcessing->name, "muse_sci") ||
354  !strcmp(aProcessing->name, "muse_exp_combine")) {
355  cpl_propertylist_update_bool(aHeader, "ESO PRO SCIENCE", 1);
356  }
357 } /* muse_processing_setup_header() */
358 
359 /*---------------------------------------------------------------------------*/
382 /*---------------------------------------------------------------------------*/
383 cpl_frame *
385  cpl_propertylist *aHeader, const char *aTag,
386  cpl_frame_type aType)
387 {
388  cpl_ensure(aProcessing, CPL_ERROR_NULL_INPUT, NULL);
389  muse_processing_prepare_header(aProcessing->recipe, aTag, aHeader);
390  const char *prefix = cpl_propertylist_has(aHeader, "MUSE PRIVATE FILE PREFIX")
391  ? cpl_propertylist_get_string(aHeader,
392  "MUSE PRIVATE FILE PREFIX")
393  : aTag;
394 
395  cpl_frame *frame = cpl_frame_new();
396  /* some error from before might disturbe further error handling */
397  cpl_errorstate prestate = cpl_errorstate_get();
398 
399  /* see if we need to take all input frames, a specific frame based on its *
400  * number in the frameset, and/or a specific frame based on observing date */
401  int fmode = muse_processing_get_frame_mode(aProcessing->recipe, aTag),
402  framecounter = 0;
403  if (fmode != MUSE_FRAME_MODE_MASTER) {
404  framecounter = muse_processing_get_framecounter(aProcessing, prefix, aIFU);
405  }
406  const char *dateobs = NULL;
407  if (fmode == MUSE_FRAME_MODE_DATEOBS) {
408  dateobs = muse_pfits_get_dateobs(aHeader);
409  }
410  cpl_boolean sequence = fmode == MUSE_FRAME_MODE_SEQUENCE;
411 
412  /* Create product frame */
413  char filename[FILENAME_MAX];
414  if (aIFU >= 0) {
415  if (framecounter == 0) {
416  snprintf(filename, FILENAME_MAX, "%s-%02d.fits", prefix, aIFU);
417  } else {
418  snprintf(filename, FILENAME_MAX, "%s_%04d-%02d.fits", prefix,
419  framecounter, aIFU);
420  }
421  } else {
422  if (framecounter == 0) {
423  snprintf(filename, FILENAME_MAX, "%s.fits", prefix);
424  } else {
425  snprintf(filename, FILENAME_MAX, "%s_%04d.fits", prefix, framecounter);
426  }
427  }
428  cpl_frame_set_filename(frame, filename);
429  cpl_frame_set_tag(frame, aTag);
430  cpl_frame_set_type(frame, aType);
431  cpl_frame_set_group(frame, CPL_FRAME_GROUP_PRODUCT);
432  cpl_frame_set_level(frame,
434  aTag));
435  if (cpl_errorstate_get() != prestate) {
436  cpl_msg_error(__func__, "Error while initialising the product frame: %s",
437  cpl_error_get_message());
438  cpl_frame_delete(frame);
439  return NULL;
440  }
441  cpl_propertylist_erase_regexp(aHeader, "MUSE PRIVATE.*", 0);
442  /* select specific frame index to use */
443  int idx = dateobs || fmode == MUSE_FRAME_MODE_SUBSET ? -1 : framecounter - 1;
444  muse_processing_setup_header(aProcessing, aHeader, frame, idx, dateobs,
445  sequence);
446  return frame;
447 } /* muse_processing_new_frame() */
448 
449 /*---------------------------------------------------------------------------*/
465 /*---------------------------------------------------------------------------*/
466 int
468  muse_image *aImage, const char *aTag)
469 {
470  cpl_ensure_code(aProcessing && aImage && aTag, CPL_ERROR_NULL_INPUT);
471 
472  cpl_frame *frame = muse_processing_new_frame(aProcessing, aIFU,
473  aImage->header, aTag,
474  CPL_FRAME_TYPE_IMAGE);
475  cpl_msg_info(__func__, "Saving image as %s", cpl_frame_get_filename(frame));
476  int r = muse_image_save(aImage, cpl_frame_get_filename(frame));
477  if (r == CPL_ERROR_NONE) {
478  cpl_frameset_insert(aProcessing->outframes, frame);
479  } else {
480  cpl_frame_delete(frame);
481  }
482  return r;
483 } /* muse_processing_save_image() */
484 
485 /*---------------------------------------------------------------------------*/
504 /*---------------------------------------------------------------------------*/
505 /* XXX removed MUSE_CUBE_TYPE_LSF from doxygen */
506 cpl_error_code
508  void *aCube, const char *aTag, muse_cube_type aType)
509 {
510  cpl_ensure_code(aProcessing && aCube && aTag, CPL_ERROR_NULL_INPUT);
511  cpl_ensure_code(aType == MUSE_CUBE_TYPE_EURO3D || aType == MUSE_CUBE_TYPE_FITS/* XXX ||
512  aType == MUSE_CUBE_TYPE_LSF */, CPL_ERROR_ILLEGAL_INPUT);
513 
514  /* both cube structures contain the header component as *
515  * first element, so it's safe to cast it to one of them */
516  cpl_frame *frame = muse_processing_new_frame(aProcessing, aIFU,
517  ((muse_datacube *)aCube)->header,
518  aTag, CPL_FRAME_TYPE_IMAGE);
519  cpl_msg_info(__func__, "Saving %s cube as \"%s\"",
520  aType == MUSE_CUBE_TYPE_EURO3D ? "Euro3D" : "FITS",
521  cpl_frame_get_filename(frame));
522  cpl_error_code rc = CPL_ERROR_NONE;
523  if (aType == MUSE_CUBE_TYPE_EURO3D) {
525  cpl_frame_get_filename(frame));
526  } else if (aType == MUSE_CUBE_TYPE_FITS) {
527  rc = muse_datacube_save((muse_datacube *)aCube,
528  cpl_frame_get_filename(frame));
529  } else {
530  //XXX rc = muse_lsf_cube_save((muse_lsf_cube *)aCube,
531  //XXX cpl_frame_get_filename(frame));
532  }
533  if (rc == CPL_ERROR_NONE) {
534  cpl_frameset_insert(aProcessing->outframes, frame);
535  } else {
536  cpl_frame_delete(frame);
537  }
538  return rc;
539 } /* muse_processing_save_cube() */
540 
541 /*---------------------------------------------------------------------------*/
562 /*---------------------------------------------------------------------------*/
563 cpl_error_code
565  void *aTable, cpl_propertylist *aHeader,
566  const char *aTag, muse_table_type aType)
567 {
568  cpl_ensure_code(aProcessing && aTable && aTag, CPL_ERROR_NULL_INPUT);
569  cpl_ensure_code(aType == MUSE_TABLE_TYPE_CPL || aType == MUSE_TABLE_TYPE_PIXTABLE,
570  CPL_ERROR_ILLEGAL_INPUT);
571 
572  /* for plain tables, the separate header is needed */
573  cpl_propertylist *header;
574  if (aType == MUSE_TABLE_TYPE_CPL) {
575  cpl_ensure_code(aHeader, CPL_ERROR_NULL_INPUT);
576  header = aHeader;
577  } else {
578  header = ((muse_pixtable *)aTable)->header;
579  }
580  cpl_frame *frame = muse_processing_new_frame(aProcessing, aIFU, header,
581  aTag, CPL_FRAME_TYPE_TABLE);
582  cpl_msg_info(__func__, "Saving %stable as \"%s\"",
583  aType == MUSE_TABLE_TYPE_PIXTABLE ? "pixel " : "",
584  cpl_frame_get_filename(frame));
585  cpl_error_code rc = CPL_ERROR_NONE;
586  if (aType == MUSE_TABLE_TYPE_CPL) {
587  rc = cpl_table_save((cpl_table *)aTable, aHeader, NULL,
588  cpl_frame_get_filename(frame), CPL_IO_CREATE);
589  } else {
590  rc = muse_pixtable_save((muse_pixtable *)aTable,
591  cpl_frame_get_filename(frame));
592  }
593  if (rc == CPL_ERROR_NONE) {
594  cpl_frameset_insert(aProcessing->outframes, frame);
595  } else {
596  cpl_msg_error(__func__, "Saving %stable failed: %s",
597  aType == MUSE_TABLE_TYPE_PIXTABLE ? "pixel " : "",
598  cpl_error_get_message());
599  cpl_frame_delete(frame);
600  }
601  return rc;
602 } /* muse_processing_save_table() */
603 
604 /*---------------------------------------------------------------------------*/
621 /*---------------------------------------------------------------------------*/
622 int
624  cpl_image *aImage, cpl_propertylist *aHeader,
625  const char *aTag)
626 {
627  cpl_ensure_code(aProcessing && aImage && aHeader && aTag, CPL_ERROR_NULL_INPUT);
628 
629  cpl_frame *frame = muse_processing_new_frame(aProcessing, aIFU,
630  aHeader, aTag,
631  CPL_FRAME_TYPE_IMAGE);
632  cpl_msg_info(__func__, "Saving image as %s", cpl_frame_get_filename(frame));
633  int r = cpl_image_save(aImage, cpl_frame_get_filename(frame),
634  CPL_TYPE_UNSPECIFIED, aHeader, CPL_IO_CREATE);
635  if (r == CPL_ERROR_NONE) {
636  cpl_frameset_insert(aProcessing->outframes, frame);
637  } else {
638  cpl_msg_error(__func__, "Saving image failed: %s", cpl_error_get_message());
639  cpl_frame_delete(frame);
640  }
641  return r;
642 }
643 
644 /*---------------------------------------------------------------------------*/
660 /*---------------------------------------------------------------------------*/
661 cpl_error_code
663  cpl_propertylist *aHeader, const char *aTag)
664 {
665  cpl_ensure_code(aProcessing && aHeader && aTag, CPL_ERROR_NULL_INPUT);
666 
667  cpl_frame *frame = muse_processing_new_frame(aProcessing, aIFU,
668  aHeader, aTag,
669  CPL_FRAME_TYPE_IMAGE);
670  cpl_msg_info(__func__, "Saving header as %s", cpl_frame_get_filename(frame));
671  cpl_error_code rc = cpl_propertylist_save(aHeader,
672  cpl_frame_get_filename(frame),
673  CPL_IO_CREATE);
674  if (rc == CPL_ERROR_NONE) {
675  cpl_frameset_insert(aProcessing->outframes, frame);
676  } else {
677  cpl_msg_error(__func__, "Saving header failed: %s", cpl_error_get_message());
678  cpl_frame_delete(frame);
679  }
680  return rc;
681 }
682 
683 /*---------------------------------------------------------------------------*/
699 /*---------------------------------------------------------------------------*/
700 int
702  muse_mask *aMask, const char *aTag)
703 {
704  cpl_ensure_code(aProcessing && aMask && aTag, CPL_ERROR_NULL_INPUT);
705 
706  cpl_frame *frame = muse_processing_new_frame(aProcessing, aIFU,
707  aMask->header, aTag,
708  CPL_FRAME_TYPE_IMAGE);
709  cpl_msg_info(__func__, "Saving mask as %s", cpl_frame_get_filename(frame));
710  int r = muse_mask_save(aMask, cpl_frame_get_filename(frame));
711  if (r == CPL_ERROR_NONE) {
712  cpl_frameset_insert(aProcessing->outframes, frame);
713  } else {
714  cpl_frame_delete(frame);
715  }
716  return r;
717 } /* muse_processing_save_mask() */
718 
719 /*---------------------------------------------------------------------------*/
732 /*---------------------------------------------------------------------------*/
733 cpl_boolean
734 muse_processing_check_intags(muse_processing *aProcessing, const char *aTag,
735  int aNChars)
736 {
737  cpl_ensure(aProcessing && aTag, CPL_ERROR_NULL_INPUT, CPL_FALSE);
738 
739  cpl_boolean hastag = CPL_FALSE;
740  /* loop through all tags, looking for a matching one */
741  cpl_size itag, ntags = cpl_array_get_size(aProcessing->intags);
742  for (itag = 0; itag < ntags; itag++) {
743  const char *tag = cpl_array_get_string(aProcessing->intags, itag);
744  if (tag && !strncmp(tag, aTag, aNChars)) {
745  hastag = CPL_TRUE;
746  break; /* found one match, that's enough */
747  } /* if found */
748  } /* for itag */
749  return hastag;
750 } /* muse_processing_check_intags() */
751 
752 /*---------------------------------------------------------------------------*/
768 /*---------------------------------------------------------------------------*/
769 cpl_error_code
770 muse_processing_check_input(muse_processing *aProcessing, unsigned char aIFU)
771 {
772  if (!aProcessing) {
773  cpl_msg_error(__func__, "NULL processing struct");
774  return CPL_ERROR_NULL_INPUT;
775  }
776  cpl_recipeconfig *recipeconfig
778  if (!recipeconfig) {
779  cpl_msg_error(__func__, "No recipeconfig found!");
780  return CPL_ERROR_ILLEGAL_INPUT;
781  }
782 
783  cpl_size itag, ntags = cpl_array_get_size(aProcessing->intags);
784  unsigned int errors = 0;
785  for (itag = 0; itag < ntags; itag++) {
786  const char *intag = cpl_array_get_string(aProcessing->intags, itag);
787  cpl_frameset *frames = muse_frameset_find(aProcessing->inframes,
788  intag, aIFU, CPL_FALSE);
789  int nframes = cpl_frameset_count_tags(frames, intag),
790  nmin = cpl_recipeconfig_get_min_count(recipeconfig, intag, intag),
791  nmax = cpl_recipeconfig_get_max_count(recipeconfig, intag, intag);
792  cpl_frameset_delete(frames);
793  if (nmin >= 0 && nframes < nmin) {
794  cpl_msg_error(__func__, "Required %d, found %d input frames with tag "
795  "\"%s\" with IFU %hhu", nmin, nframes, intag, aIFU);
796  errors++;
797  }
798  if (nmax >= 0 && nframes > nmax) {
799  cpl_msg_error(__func__, "At most %d allowed, found %d input frames with "
800  "tag \"%s\" with IFU %hhu", nmax, nframes, intag, aIFU);
801  errors++;
802  }
803  char **tags = cpl_recipeconfig_get_inputs(recipeconfig, intag);
804  if (!tags) {
805  cpl_msg_error(__func__, "Input frames with tag \"%s\" cannot be used with"
806  " this recipe", intag);
807  errors++;
808  continue;
809  }
810  /* check all other tags */
811  int i;
812  for (i = 0; tags[i]; i++) {
813  frames = muse_frameset_find(aProcessing->inframes, tags[i], aIFU,
814  CPL_FALSE);
815  nframes = cpl_frameset_count_tags(frames, tags[i]);
816  cpl_frameset_delete(frames);
817 
818  nmin = cpl_recipeconfig_get_min_count(recipeconfig, intag, tags[i]);
819  nmax = cpl_recipeconfig_get_max_count(recipeconfig, intag, tags[i]);
820  if (nmin >= 0 && nframes < nmin) {
821  cpl_msg_error(__func__, "Required %d, found %d frames with tag \"%s\" "
822  "with IFU %hhu", nmin, nframes, tags[i], aIFU);
823  errors++;
824  }
825  if (nframes == 0 && nmin <= 0) {
826  cpl_msg_debug(__func__, "Optional frame with tag \"%s\" not given",
827  tags[i]);
828  }
829  if (nmax >= 0 && nframes > nmax) {
830  cpl_msg_error(__func__, "At most %d allowed, found %d frames with tag "
831  "\"%s\" with IFU %hhu", nmax, nframes, tags[i], aIFU);
832  errors++;
833  }
834  cpl_free(tags[i]);
835  } /* for i (all tags) */
836  cpl_free(tags);
837  } /* for itag */
838 
839  if (errors) {
840  cpl_msg_error(__func__, "Found %u error(s)", errors);
841  return CPL_ERROR_DATA_NOT_FOUND;
842  }
843  return CPL_ERROR_NONE;
844 } /* muse_processing_check_input() */
845 
846 /*----------------------------------------------------------------------------*/
869 /*----------------------------------------------------------------------------*/
870 cpl_table *
872 {
873  cpl_ensure(aProcessing, CPL_ERROR_NULL_INPUT, NULL);
874  cpl_size nframes = cpl_frameset_get_size(aProcessing->inframes);
875  cpl_ensure(nframes, CPL_ERROR_DATA_NOT_FOUND, NULL);
876 
877  /* now that we have at least one frame, we can assume that the frameset is *
878  * valid and we have at least one exposure, so we can create the empty table */
879  cpl_table *exptable = cpl_table_new(0);
880  /* create columns for DATE-OBS and all IFUs */
881  cpl_table_new_column(exptable, "DATE-OBS", CPL_TYPE_STRING);
882  char colname[3]; /* further used in this function, define outside loop */
883  int i;
884  for (i = 0; i <= kMuseNumIFUs; i++) {
885  snprintf(colname, 3, "%02d", i);
886  cpl_table_new_column(exptable, colname, CPL_TYPE_STRING);
887  }
888 
889  /* loop over all input frames */
890  cpl_size iframe;
891  for (iframe = 0; iframe < nframes; iframe++) {
892  cpl_frame *frame = cpl_frameset_get_position(aProcessing->inframes,
893  iframe);
894  const char *tag = cpl_frame_get_tag(frame);
895  if (muse_processing_check_intags(aProcessing, tag, strlen(tag))) {
896  const char *filename = cpl_frame_get_filename(frame);
897 
898  /* load the primary header and get DATE-OBS and the IFU number */
899  cpl_propertylist *header = cpl_propertylist_load(filename, 0);
900  const char *date = muse_pfits_get_dateobs(header);
901  if (!date) {
902  cpl_msg_warning(__func__, "\"%s\" does not contain the DATE-OBS "
903  "keyword, it will be ignored!", filename);
904  cpl_propertylist_delete(header);
905  continue; /* skip on to the next frame */
906  }
907  int ifu = muse_utils_get_ifu(header);
908  if (!ifu) {
909  cpl_msg_debug(__func__, "\"%s\" seems to contain merged data (no "
910  "EXTNAME=CHANnn)", filename);
911  }
912 #if DEBUG_EXPOSURES_SORT
913  cpl_msg_debug(__func__, "\"%s\": IFU %2d, DATE-OBS=\"%s\"", filename, ifu, date);
914 #endif
915 
916  /* loop through all exposures (=rows) in the table and compare the date */
917  int irow = -1;
918  for (i = 0; i < cpl_table_get_nrow(exptable); i++) {
919 #if DEBUG_EXPOSURES_SORT
920  cpl_msg_debug(__func__, "i=%d, DATE-OBS=\"%s\"", i,
921  cpl_table_get_string(exptable, "DATE-OBS", i));
922 #endif
923  if (!strcmp(date, cpl_table_get_string(exptable, "DATE-OBS", i))) {
924  irow = i;
925  }
926  }
927 
928  /* didn't find a matching entry, add a new exposure (=row) */
929  if (irow < 0) {
930  cpl_table_set_size(exptable, cpl_table_get_nrow(exptable)+1);
931  irow = cpl_table_get_nrow(exptable) - 1;
932  cpl_table_set_string(exptable, "DATE-OBS", irow, date);
933  }
934 
935  /* check, if table cell for this exposure is still empty and, *
936  * if so, add the filename */
937  snprintf(colname, 3, "%02d", ifu);
938  if (cpl_table_is_valid(exptable, colname, irow)) {
939  cpl_msg_warning(__func__, "we already have IFU %d of exposure %d (\"%s\")!"
940  " Ignoring \"%s\"", ifu, irow+1,
941  cpl_table_get_string(exptable, colname, irow),
942  filename);
943  cpl_propertylist_delete(header);
944  continue; /* skip on to the next frame */
945  }
946  cpl_table_set_string(exptable, colname, irow, filename);
947  /* now we can assume that this is valid and will be used; it is not a *
948  * RAW file, but it's the main input, so the RAW group will satisfy the *
949  * DFS-related functions in CPL */
950  muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_RAW, 1);
951 
952  cpl_propertylist_delete(header);
953  } /* if correct tag */
954 #if DEBUG_EXPOSURES_SORT
955  else {
956  cpl_msg_debug(__func__, "wrong tag \"%s\" for \"%s\"", tag,
957  cpl_frame_get_filename(frame));
958  }
959 #endif
960  } /* for iframe (all input frames) */
961 
962  if (cpl_table_get_nrow(exptable) <= 0) {
963  /* no exposures found */
964  cpl_table_delete(exptable);
965  cpl_error_set(__func__, CPL_ERROR_DATA_NOT_FOUND);
966  return NULL;
967  }
968  /* loop through all exposures and print the number of valid IFUs found */
969  for (i = 0; i < cpl_table_get_nrow(exptable); i++) {
970  int nmerged = 0;
971  if (cpl_table_is_valid(exptable, "00", i)) {
972  nmerged++;
973  }
974  int j, nvalid = 0;
975  for (j = 1; j <= kMuseNumIFUs; j++) {
976  snprintf(colname, 3, "%02d", j);
977  if (cpl_table_is_valid(exptable, colname, i)) {
978  nvalid++;
979  }
980  } /* for i (table columns) */
981  cpl_msg_debug(__func__, "Data from exposure %2d is contained in %2d "
982  "IFU%s/%d merged file%s", i+1, nvalid, nvalid == 1 ? "" : "s",
983  nmerged, nmerged == 1 ? "" : "s");
984  } /* for i (table rows) */
985 
986  /* sort table by increasing DATE-OBS, so that we process the exposures *
987  * in chronological order */
988  cpl_propertylist *sorting = cpl_propertylist_new();
989  cpl_propertylist_append_bool(sorting, "DATE-OBS",
990  CPL_FALSE); /* sort ascending */
991  cpl_table_sort(exptable, sorting);
992  cpl_propertylist_delete(sorting);
993 
994  return exptable;
995 } /* muse_processing_sort_exposures() */
996 
997 /*---------------------------------------------------------------------------*/
1011 /*---------------------------------------------------------------------------*/
1012 muse_mask *
1013 muse_processing_mask_load(muse_processing *aProcessing, const char *aTag)
1014 {
1015  cpl_frameset *frames = muse_frameset_find(aProcessing->inframes,
1016  aTag, 0, CPL_FALSE);
1017  if (frames == NULL || cpl_frameset_get_size(frames) <= 0) {
1018  cpl_frameset_delete(frames);
1019  return NULL;
1020  }
1021  cpl_frame *frame = cpl_frameset_get_position(frames, 0);
1022  muse_mask *mask = muse_mask_load(cpl_frame_get_filename(frame));
1023  if (!mask) {
1024  cpl_msg_warning(__func__, "loading mask \"%s\" failed!",
1025  cpl_frame_get_filename(frame));
1026  cpl_frameset_delete(frames);
1027  return NULL;
1028  }
1029  cpl_msg_info(__func__, "using mask \"%s\" (%"CPL_SIZE_FORMAT" pixels)",
1030  cpl_frame_get_filename(frame), cpl_mask_count(mask->mask));
1031  muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_CALIB, 1);
1032  cpl_frameset_delete(frames);
1033  return mask;
1034 }
1035 
int muse_processing_save_cimage(muse_processing *aProcessing, int aIFU, cpl_image *aImage, cpl_propertylist *aHeader, const char *aTag)
Save a computed FITS image to disk.
Structure definition of a MUSE datacube.
Definition: muse_datacube.h:47
int muse_processing_get_frame_mode(const cpl_recipe *, const char *)
Get the mode for a product frame with a certain tag.
void muse_processing_delete(muse_processing *aProcessing)
Free the muse_processing structure.
int muse_processing_save_mask(muse_processing *aProcessing, int aIFU, muse_mask *aMask, const char *aTag)
Save a computed MUSE mask to disk.
cpl_error_code muse_euro3dcube_save(muse_euro3dcube *aEuro3D, const char *aFilename)
Save a Euro3D cube object to a file.
const char * name
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:99
muse_processing_framecounter * counter
muse_table_type
const char * muse_pfits_get_dateobs(const cpl_propertylist *aHeaders)
find out the date of observations
Definition: muse_pfits.c:329
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
muse_mask * muse_processing_mask_load(muse_processing *aProcessing, const char *aTag)
Load a mask file and its FITS header.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
cpl_boolean muse_processing_check_intags(muse_processing *aProcessing, const char *aTag, int aNChars)
Check that a tag is part of the input tags of a processing structure.
muse_mask * muse_mask_load(const char *aFilename)
Load a mask file and its FITS header.
Definition: muse_mask.c:90
cpl_error_code muse_datacube_save(muse_datacube *aCube, const char *aFilename)
Save the three cube extensions and the FITS headers of a MUSE datacube to a file. ...
cpl_frameset * usedframes
muse_processing * muse_processing_new(const char *aName, cpl_recipe *aRecipe)
Create a new processing structure.
cpl_error_code muse_processing_prepare_header(const cpl_recipe *, const char *, cpl_propertylist *)
Prepare and check a FITS header for a certain frame tag.
cpl_error_code muse_processing_save_header(muse_processing *aProcessing, int aIFU, cpl_propertylist *aHeader, const char *aTag)
Save a FITS header to disk.
Structure definition of MUSE pixel table.
cpl_frame * muse_processing_new_frame(muse_processing *aProcessing, int aIFU, cpl_propertylist *aHeader, const char *aTag, cpl_frame_type aType)
Create a new frame for a result file.
cpl_error_code muse_mask_save(muse_mask *aMask, const char *aFilename)
Save the data and the FITS headers of a MUSE mask to a file.
Definition: muse_mask.c:131
cpl_frameset * outframes
cpl_error_code muse_processing_save_cube(muse_processing *aProcessing, int aIFU, void *aCube, const char *aTag, muse_cube_type aType)
Save a MUSE datacube to disk.
cpl_frame_level muse_processing_get_frame_level(const cpl_recipe *, const char *)
Get the level for a product frame with a certain tag.
cpl_recipe * recipe
cpl_frameset * muse_frameset_sort_raw_other(const cpl_frameset *aFrames, int aIndex, const char *aDateObs, cpl_boolean aSequence)
Create a new frameset containing all relevant raw frames first then all other frames.
Definition: muse_utils.c:399
muse_cube_type
cpl_recipeconfig * muse_processing_get_recipeconfig(cpl_recipe *)
Get the recipe (frame) configuration.
Structure definition of a Euro3D datacube.
Definition: muse_datacube.h:96
void muse_processing_append_used(muse_processing *aProcessing, cpl_frame *aFrame, cpl_frame_group aGroup, int aDuplicate)
Add a frame to the set of used frames.
cpl_error_code muse_pixtable_save(muse_pixtable *aPixtable, const char *aFilename)
Save a MUSE pixel table to a file on disk.
int muse_processing_save_image(muse_processing *aProcessing, int aIFU, muse_image *aImage, const char *aTag)
Save a computed MUSE image to disk.
cpl_parameterlist * muse_cplparameterlist_duplicate(const cpl_parameterlist *aPList)
Duplicate a CPL parameter list.
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:399
Handling of "mask" files.
Definition: muse_mask.h:42
cpl_array * intags
cpl_frameset * inframes
cpl_error_code muse_processing_save_table(muse_processing *aProcessing, int aIFU, void *aTable, cpl_propertylist *aHeader, const char *aTag, muse_table_type aType)
Save a computed table to disk.
cpl_error_code muse_processing_check_input(muse_processing *aProcessing, unsigned char aIFU)
Check the input files for completeness.
cpl_propertylist * header
the FITS header
Definition: muse_mask.h:55
cpl_frameset * muse_frameset_find(const cpl_frameset *aFrames, const char *aTag, unsigned char aIFU, cpl_boolean aInvert)
return frameset containing data from an IFU/channel with a certain tag
Definition: muse_utils.c:158
cpl_table * muse_processing_sort_exposures(muse_processing *aProcessing)
Sort input frames (containing lists of pixel table filenames) into different exposures.
cpl_mask * mask
The mask data.
Definition: muse_mask.h:48
cpl_parameterlist * parameters