VIRCAM Pipeline  1.3.4
vircam/vircam_imdither.c
1 /* $Id: vircam_imdither.c,v 1.19 2012-01-27 12:25:10 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2012-01-27 12:25:10 $
24  * $Revision: 1.19 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <math.h>
35 #include <cpl.h>
36 #include "vircam_mods.h"
37 #include "vircam_utils.h"
38 #include "vircam_fits.h"
39 #include "vircam_stats.h"
40 #include "vircam_pfits.h"
41 
42 typedef struct {
43  vir_fits *fname;
44  vir_fits *conf;
45  float xoff;
46  float yoff;
47  int ixoff;
48  int iyoff;
49  int nx;
50  int ny;
51  float sky;
52  float skydiff;
53  float noise;
54  float expscale;
55  float weight;
56  float *data;
57  int *cdata;
58  int ndata;
59 } dstrct;
60 
61 typedef struct {
62  float *values;
63  float *confs;
64  float *weights;
65  short int *iff;
66  int n;
67  long outindex;
68  unsigned char clipped;
69 } keeptabs;
70 
71 #define NROWSBUF 512
72 
73 static dstrct *fileptrs = NULL;
74 static float *odata = NULL;
75 static float *owdata = NULL;
76 static keeptabs *clipmon = NULL;
77 
78 static float lsig;
79 static float hsig;
80 static float sumweight;
81 static int nxo;
82 static int nyo;
83 
84 static void average(keeptabs *, float *, float *, float, float, float);
85 static keeptabs *clip_open( int);
86 static void clip_close(keeptabs **);
87 static void fileptrs_close(void);
88 static void skyest(float *, int *, long, float, float *, float *);
89 static void tidy(void);
90 
93 /*---------------------------------------------------------------------------*/
154 /*---------------------------------------------------------------------------*/
155 
156 extern int vircam_imdither(vir_fits **inf, vir_fits **inconf, int nimages,
157  int nconfs, float lthr, float hthr,
158  cpl_propertylist **p, cpl_image **out,
159  cpl_image **outc, int *status) {
160 
161  int i,itx,iy,ccur,clast,ix,n,iline,icol,jy,jx,*ocdata,j;
162  long npts,ielm,iloc,index_y,index;
163  dstrct *dd;
164  keeptabs *c;
165  float minxoff,minyoff,expref,sky,skynoise,clip1,clip2,outdata;
166  float outconf,avlev,avvar,renorm,exposure;
167  double crpix1,crpix2;
168  cpl_propertylist *ehu,*p2;
169  const char *fctid = "vircam_imdither";
170  char timestamp[25];
171 
172  /* Inherited status */
173 
174  *out = NULL;
175  *outc = NULL;
176  *p = NULL;
177  if (*status != VIR_OK)
178  return(*status);
179 
180  /* Is there any point in being here? */
181 
182  if (nimages == 0) {
183  cpl_msg_error(fctid,"No input files to combine");
184  tidy();
185  FATAL_ERROR
186  }
187 
188  /* Initialise some global variables */
189 
190  lsig = lthr;
191  hsig = hthr;
192 
193  /* Allocate file struct array and fill in some values */
194 
195  fileptrs = cpl_malloc(nimages*sizeof(dstrct));
196  (void)vircam_pfits_get_exptime(vircam_fits_get_phu(inf[0]),&exposure);
197  expref = max(0.5,exposure);
198  minxoff = 1.0e10;
199  minyoff = 1.0e10;
200  for (i = 0; i < nimages; i++) {
201  dd = fileptrs + i;
202  dd->fname = inf[i];
203  dd->data = cpl_image_get_data_float(vircam_fits_get_image(inf[i]));
204  if (nconfs == 0) {
205  dd->conf = NULL;
206  } else if (nconfs == 1) {
207  dd->conf = inconf[0];
208  dd->cdata = cpl_image_get_data_int(vircam_fits_get_image(inconf[0]));
209  } else {
210  dd->conf = inconf[i];
211  dd->cdata = cpl_image_get_data_int(vircam_fits_get_image(inconf[i]));
212  }
213  ehu = vircam_fits_get_ehu(dd->fname);
214  (void)vircam_pfits_get_jxoff(ehu,&(dd->xoff));
215  (void)vircam_pfits_get_jyoff(ehu,&(dd->yoff));
216  minxoff = min(dd->xoff,minxoff);
217  minyoff = min(dd->yoff,minyoff);
218  (void)vircam_pfits_get_exptime(vircam_fits_get_phu(dd->fname),&exposure);
219  dd->expscale = exposure/expref;
220 
221  /* Now work out a background and background noise estimate */
222 
223  dd->nx = (int)cpl_image_get_size_x(vircam_fits_get_image(dd->fname));
224  dd->ny = (int)cpl_image_get_size_y(vircam_fits_get_image(dd->fname));
225  npts = dd->nx*dd->ny;
226  skyest(dd->data,dd->cdata,npts,3.0,&sky,&skynoise);
227  dd->sky = sky;
228  dd->noise = skynoise;
229 
230  /* Double check to make sure the confidence maps and images have the
231  same dimensions */
232 
233  if ((int)cpl_image_get_size_x(vircam_fits_get_image(dd->conf)) != dd->nx ||
234  (int)cpl_image_get_size_y(vircam_fits_get_image(dd->conf)) != dd->ny) {
235  cpl_msg_error(fctid,"VIRCAM_IMDITHER: Image %s and Confidence map %s don't match",
236  vircam_fits_get_fullname(dd->fname),
237  vircam_fits_get_fullname(dd->conf));
238  tidy();
239  FATAL_ERROR
240  }
241  }
242 
243  /* Redo the offsets so that they are all positive. */
244 
245  for (i = 0; i < nimages; i++) {
246  dd = fileptrs + i;
247  dd->xoff -= minxoff;
248  dd->yoff -= minyoff;
249  dd->ixoff = (int)(dd->xoff + 0.5);
250  dd->iyoff = (int)(dd->yoff + 0.5);
251  }
252 
253  /* Redo the zero point offsets so that they are all relative to
254  the first image in the list. Make sure to divide by the relative
255  exposure time first! Set up weights*/
256 
257  fileptrs->sky /= fileptrs->expscale;
258  fileptrs->skydiff = 0.0;
259  fileptrs->weight = 1.0;
260  sumweight = 1.0;
261  for (i = 1; i < nimages; i++) {
262  dd = fileptrs + i;
263  dd->sky /= dd->expscale;
264  dd->skydiff = fileptrs->sky - dd->sky;
265  dd->noise /= (float)sqrt((double)dd->expscale);
266  dd->weight = (float)(pow((double)fileptrs->noise,2.0)/pow((double)dd->noise,2.0));
267  sumweight += dd->weight;
268  }
269 
270  /* Scale data (don't do image 0 since that has 1 scale and 0 offset) */
271 
272  for (i = 1; i < nimages; i++) {
273  dd = fileptrs + i;
274  npts = dd->nx*dd->ny;
275  for (j = 0; j < npts; j++)
276  dd->data[j] = dd->data[j]/dd->expscale + dd->skydiff;
277  }
278 
279  /* Set up clipping levels */
280 
281  clip1 = fileptrs->sky - lthr*fileptrs->noise;
282  clip2 = fileptrs->sky + hthr*fileptrs->noise;
283 
284  /* Open the output file. First of all work out how big the output map
285  needs to be. Then create it based on the first image in the list */
286 
287  nxo = 0;
288  nyo = 0;
289  for (i = 0; i < nimages; i++) {
290  dd = fileptrs + i;
291  itx = dd->nx + dd->ixoff;
292  nxo = max(nxo,itx);
293  itx = dd->ny + dd->iyoff;
294  nyo = max(nyo,itx);
295  }
296 
297  /* Create the output image */
298 
299  *out = cpl_image_new((cpl_size)nxo,(cpl_size)nyo,CPL_TYPE_FLOAT);
300 
301  /* If an output confidence map has been specified, then create it now. */
302 
303  if (nconfs != 0)
304  *outc = cpl_image_new((cpl_size)nxo,(cpl_size)nyo,CPL_TYPE_INT);
305  else
306  *outc = NULL;
307 
308  /* Get the data arrays for the output images */
309 
310  npts = nxo*nyo;
311  odata = cpl_image_get_data_float(*out);
312  if (*outc != NULL)
313  owdata = cpl_malloc(npts*sizeof(float));
314  clipmon = clip_open(nimages);
315 
316  /* Right, now try and do the work. Start by deciding whether for a given
317  output line an input line is able to contribute */
318 
319  for (iy = 0; iy < nyo; iy++) {
320  ccur = (iy % 2)*nxo;
321  clast = nxo - ccur;
322  for (ix = 0; ix < nxo; ix++) {
323  c = clipmon + ccur + ix;
324  c->n = 0;
325  c->clipped = 0;
326  n = 0;
327  for (i = 0; i < nimages; i++) {
328  dd = fileptrs + i;
329  iline = iy - dd->iyoff;
330  if (iline < 0 || iline >= dd->ny)
331  continue;
332  icol = ix - dd->ixoff;
333  if (icol < 0 || icol >= dd->nx)
334  continue;
335 
336  /* Load up any input data for this pixel from the current
337  image */
338 
339  ielm = dd->nx*iline + icol;
340  c->values[n] = dd->data[ielm];
341  c->confs[n] = dd->cdata[ielm];
342  c->weights[n] = dd->weight;
343  c->iff[n] = (short int)i;
344  n++;
345  }
346  c->outindex = nxo*iy + ix;
347  c->n = n;
348  average(c,&outdata,&outconf,clip1,clip2,0.0);
349  odata[c->outindex] = outdata;
350  if (owdata != NULL)
351  owdata[c->outindex] = outconf;
352  }
353 
354  /* If we're away from the edges, have a look and see which pixels in
355  the previous row had clipping. Evaluate whether that clipping was
356  really justified or not */
357 
358  if (iy < 2)
359  continue;
360  for (ix = 1; ix < nxo-1; ix++) {
361  c = clipmon + clast + ix;
362  if (! c->clipped)
363  continue;
364 
365  /* If it was clipped, then evaluate the amount of 'noise' there
366  is spatially */
367 
368  iloc = c->outindex;
369  avlev = 0.0;
370  for (jy = -1; jy <= 1; jy++) {
371  index_y = iloc + jy*nxo;
372  for (jx = -1; jx <= 1; jx++) {
373  index = index_y + jx;
374  avlev += odata[index];
375  }
376  }
377  avlev /= 9.0;
378  avvar = 0.0;
379  for (jy = -1; jy <= 1; jy++) {
380  index_y = iloc + jy*nxo;
381  for (jx = -1; jx <= 1; jx++) {
382  index = index_y + jx;
383  avvar += fabs(odata[index] - avlev);
384  }
385  }
386  avvar /= 9.0;
387 
388  /* If the average level in this cell is below the upper clip level
389  or the mean absolute deviation is smaller than the poisson
390  noise in the cell, then the clip was probably justified. */
391 
392  if (avlev < clip2 || avvar <= 2.0*fileptrs->noise)
393  continue;
394 
395  /* Otherwise, create new clip levels and redo the average */
396 
397  average(c,&outdata,&outconf,clip1,clip2,3.0*avvar);
398  odata[c->outindex] = outdata;
399  if (owdata != NULL)
400  owdata[c->outindex] = outconf;
401  }
402  }
403 
404  /* Normalise the output confidence map */
405 
406  if (owdata != NULL) {
407  skyest(owdata,NULL,npts,3.0,&sky,&skynoise);
408  renorm = 100.0/sky;
409  ocdata = cpl_image_get_data_int(*outc);
410  for (i = 0; i < npts; i++)
411  ocdata[i] = max(0,min(1000,((int)(owdata[i]*renorm + 0.5))));
412  }
413 
414  /* Create the output propertylist with some provenance info */
415 
416  *p = cpl_propertylist_duplicate(vircam_fits_get_ehu(inf[0]));
417  vircam_prov(*p,inf,nimages);
418 
419  /* Add a timestamp to the propertylist */
420 
421  vircam_timestamp(timestamp,25);
422  p2 = vircam_fits_get_phu(inf[0]);
423  cpl_propertylist_update_string(p2,"VIR_TIME",timestamp);
424  cpl_propertylist_set_comment(p2,"VIR_TIME",
425  "Timestamp for matching to conf map");
426 
427  /* Update the WCS in the header to reflect the new offset */
428 
429  (void)vircam_pfits_get_crpix1(*p,&crpix1);
430  (void)vircam_pfits_get_crpix2(*p,&crpix2);
431  crpix1 += fileptrs->xoff;
432  crpix2 += fileptrs->yoff;
433  cpl_propertylist_update_double(*p,"CRPIX1",crpix1);
434  cpl_propertylist_update_double(*p,"CRPIX2",crpix2);
435 
436  /* Get out of here */
437 
438  tidy();
439  GOOD_STATUS
440 }
441 
442 static void average(keeptabs *c, float *outdata, float *outconf, float cliplow,
443  float cliphigh, float extra) {
444  int i,imin,imax;
445  float valuemax,valuemin,cwmin,cwmax,sum,cnumb,numb,cw,cv,reflev,noise;
446  float sky,clipval;
447 
448  /* If there aren't any pixels defined for this (kind of a funny state
449  to be in, but never mind), give it some nominal value, which is the
450  sky background of the first input image. Flag it with zero confidence */
451 
452  if (c->n <= 0) {
453  *outdata = fileptrs->sky;
454  *outconf = 0.0;
455  return;
456  }
457 
458  /* Initialise a few things (avoid boring compiler errors about
459  uninitialised variables */
460 
461  valuemin = 1.0e10;
462  valuemax = -1.0e10;
463  cwmin = 1.0e10;
464  cwmax = -1.0e10;
465  imin = 0;
466  imax = 0;
467  sum = 0.0;
468  cnumb = 0.0;
469  numb = 0.0;
470 
471  /* Now loop through all the data for this point, keeping track of the
472  min and max */
473 
474  for (i = 0; i < c->n; i++) {
475  cw = c->weights[i]*c->confs[i];
476  cv = c->values[i];
477  sum += cv*cw;
478  cnumb +=cw;
479  numb += c->confs[i];
480  if (cv < valuemin) {
481  valuemin = cv;
482  cwmin = cw;
483  imin = i;
484  }
485  if (cv > valuemax) {
486  valuemax = cv;
487  cwmax = cw;
488  imax = i;
489  }
490  }
491  if (cnumb > 0.0)
492  *outdata = sum/cnumb;
493  else
494  *outdata = fileptrs->sky;
495 
496  /* See if we need to clip. Look at bright one first */
497 
498  if (valuemax > cliphigh && numb > 150.0 && cnumb > 150.0) {
499  reflev = (sum - valuemax*cwmax)/(cnumb - cwmax);
500  noise = (fileptrs+c->iff[imax])->noise;
501  sky = (fileptrs+c->iff[imax])->sky;
502  clipval = reflev + hsig*noise*max(1.0,reflev)/max(1.0,sky) + extra;
503  if (valuemax > clipval) {
504  sum -= valuemax*cwmax;
505  cnumb -= cwmax;
506  *outdata = reflev;
507  c->clipped = 1;
508  }
509  }
510 
511  /* Now look at the lowest value */
512 
513  if (valuemin < cliplow && numb > 150.0 && cnumb > 150.0) {
514  reflev = (sum - valuemin*cwmin)/(cnumb - cwmin);
515  noise = (fileptrs+c->iff[imin])->noise;
516  clipval = reflev - lsig*noise;
517  if (valuemin < clipval) {
518  cnumb -= cwmin;
519  *outdata = reflev;
520  }
521  }
522 
523  /* Do the output confidence */
524 
525  *outconf = cnumb/sumweight;
526 }
527 
528 
529 static keeptabs *clip_open(int nimages) {
530  keeptabs *c;
531  int i;
532  short int *iff;
533  float *dptr;
534 
535  c = cpl_malloc(2*nxo*sizeof(keeptabs));
536  for (i = 0; i < 2*nxo; i++) {
537  dptr = cpl_malloc(3*nimages*sizeof(*dptr));
538  iff = cpl_malloc(nimages*sizeof(*iff));
539  (c+i)->values = dptr;
540  (c+i)->confs = dptr + nimages;
541  (c+i)->weights = dptr + 2*nimages;
542  (c+i)->iff = iff;
543  (c+i)->n = 0;
544  (c+i)->outindex = -1;
545  (c+i)->clipped = 0;
546  }
547  return(c);
548 }
549 
550 static void clip_close(keeptabs **c) {
551  int i;
552 
553  for (i = 0; i < 2*nxo; i++) {
554  freespace((*c+i)->values);
555  freespace((*c+i)->iff);
556  }
557  freespace(*c);
558 }
559 
560 static void fileptrs_close(void) {
561 
562 
563  freespace(fileptrs);
564 }
565 
566 
567 static void skyest(float *data, int *cdata, long npts, float thresh,
568  float *skymed, float *skynoise) {
569  unsigned char *bpm;
570  int i;
571 
572  /* Get a dummy bad pixel mask */
573 
574  bpm = cpl_calloc(npts,sizeof(*bpm));
575  if (cdata != NULL) {
576  for (i = 0; i < npts; i++)
577  bpm[i] = (cdata[i] == 0);
578  }
579 
580  /* Get the stats */
581 
582  vircam_qmedsig(data,bpm,npts,thresh,2,-1000.0,65535.0,skymed,skynoise);
583 
584  /* Clean up */
585 
586  freespace(bpm);
587 }
588 
589 static void tidy(void) {
590 
591  freespace(owdata);
592  clip_close(&clipmon);
593  fileptrs_close();
594 }
595 
598 /*
599 
600 $Log: not supported by cvs2svn $
601 Revision 1.18 2012/01/15 17:40:09 jim
602 Minor modifications to take into accout the changes in cpl API for v6
603 
604 Revision 1.17 2009/11/17 10:04:14 jim
605 Removed maximum limitation of 110 on output confidence
606 
607 Revision 1.16 2008/11/25 18:54:08 jim
608 Background estimate now takes confidence map into account
609 
610 Revision 1.15 2008/08/28 09:03:31 jim
611 Fixed bug that stopped the scaling being done properly
612 
613 Revision 1.14 2007/10/25 17:34:00 jim
614 Modified to remove lint warnings
615 
616 Revision 1.13 2007/03/29 12:19:39 jim
617 Little changes to improve documentation
618 
619 Revision 1.12 2007/03/01 12:42:41 jim
620 Modified slightly after code checking
621 
622 Revision 1.11 2006/11/27 12:05:49 jim
623 Changed call from cpl_propertylist_append to cpl_propertylist_update
624 
625 Revision 1.10 2006/10/02 13:47:33 jim
626 Added missing .h file to include list
627 
628 Revision 1.9 2006/06/13 21:25:41 jim
629 Fixed bug in normalising the offsets
630 
631 Revision 1.8 2006/06/09 22:25:06 jim
632 tidied up a few bugs
633 
634 Revision 1.7 2006/06/09 11:26:26 jim
635 Small changes to keep lint happy
636 
637 Revision 1.6 2006/06/08 14:53:18 jim
638 Modified a few things to keep lint happy
639 
640 Revision 1.5 2006/06/06 13:05:52 jim
641 Adds VIR_TIME to primary header
642 
643 Revision 1.4 2006/05/26 15:03:32 jim
644 Fixed bug where status variable address was being changed
645 
646 Revision 1.3 2006/05/18 09:48:16 jim
647 fixed docs
648 
649 Revision 1.2 2006/05/17 12:07:12 jim
650 Fixed error condition returns
651 
652 Revision 1.1 2006/05/15 13:14:45 jim
653 new routine
654 
655 
656 */
void vircam_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: vircam_stats.c:542
int vircam_pfits_get_jxoff(const cpl_propertylist *plist, float *xoff)
Get the value of the X jitter offset.
Definition: vircam_pfits.c:371
int vircam_pfits_get_crpix2(const cpl_propertylist *plist, double *crpix2)
Get the value of crpix2.
Definition: vircam_pfits.c:119
int vircam_pfits_get_exptime(const cpl_propertylist *plist, float *exptime)
Get the value of exposure time.
Definition: vircam_pfits.c:245
int vircam_pfits_get_crpix1(const cpl_propertylist *plist, double *crpix1)
Get the value of crpix1.
Definition: vircam_pfits.c:85
char * vircam_fits_get_fullname(vir_fits *p)
Definition: vircam_fits.c:560
cpl_image * vircam_fits_get_image(vir_fits *p)
Definition: vircam_fits.c:349
int vircam_pfits_get_jyoff(const cpl_propertylist *plist, float *yoff)
Get the value of the Y jitter offset.
Definition: vircam_pfits.c:388
int vircam_imdither(vir_fits **inf, vir_fits **inconf, int nimages, int nconfs, float lthr, float hthr, cpl_propertylist **p, cpl_image **out, cpl_image **outc, int *status)
Dither a set of jittered observations.
void vircam_timestamp(char *out, int n)
cpl_propertylist * vircam_fits_get_phu(vir_fits *p)
Definition: vircam_fits.c:416
cpl_propertylist * vircam_fits_get_ehu(vir_fits *p)
Definition: vircam_fits.c:457
void vircam_prov(cpl_propertylist *p, vir_fits **inlist, int n)