VIRCAM Pipeline  1.3.4
imcore_background.c
1 /* $Id: imcore_background.c,v 1.15 2010-12-10 12:27:21 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2010-12-10 12:27:21 $
24  * $Revision: 1.15 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <math.h>
32 
33 #include <cpl.h>
34 
35 #include "floatmath.h"
36 #include "util.h"
37 #include "imcore.h"
38 
39 static int **hist = NULL;
40 static int *nnp = NULL;
41 static int npvx;
42 static int npvy;
43 static void tidy(void);
44 static void sortit (float [], int);
45 
48 /*---------------------------------------------------------------------------*/
79 /*---------------------------------------------------------------------------*/
80 
81 extern int imcore_background(ap_t *ap, int nbsize, float nullval) {
82  float fracx,fracy,skymed,sigma,skymedc,sigmac,avsky,fnbsize,dely,delx;
83  float t1,t2,dsky,*map,**bvals,*work;
84  int ifracx,ifracy,nbsizx,nbsizy,nbx,nby,npixstripe,l,i,ll;
85  int isquare,ilev,j,iclip,mcpix,iloop,irej,nbsizo2,kk,k,iby,ibyp1,ibx,ibxp1;
86  int *shist,*conf;
87  unsigned char *mflag;
88  long nx,ny;
89 
90  /* Set up some variables */
91 
92  map = ap->indata;
93  conf = ap->confdata;
94  mflag = ap->mflag;
95  nx = ap->lsiz;
96  ny = ap->csiz;
97 
98  /* check to see if nbsize is close to exact divisor */
99 
100  fracx = ((float)nx)/((float)nbsize);
101  fracy = ((float)ny)/((float)nbsize);
102  ifracx = (int)(fracx + 0.1);
103  ifracy = (int)(fracy + 0.1);
104  nbsizx = nx/ifracx;
105  nbsizy = ny/ifracy;
106  nbsize = MAX(NINT(0.9*nbsize), MIN(nbsize, MIN(nbsizx,nbsizy)));
107  nbsize = MIN(nx,MIN(ny,nbsize)); /* trap for small maps */
108 
109  /* Divide the map into partitions */
110 
111  nbx = nx/nbsize;
112  nby = ny/nbsize;
113  npixstripe = nbsize*nx;
114  npvx = nbx;
115  npvy = nby;
116 
117  /* Get histogram workspace if you can */
118 
119  hist = cpl_malloc(nbx*sizeof(int *));
120  for (l = 0; l < nbx; l++)
121  hist[l] = cpl_malloc(MAXHIST*sizeof(int));
122 
123  /* Same for background values array */
124 
125  bvals = cpl_malloc(nby*sizeof(float *));
126  for (l = 0; l < nby; l++)
127  bvals[l] = cpl_malloc(nbx*sizeof(float));
128 
129  /* Store some of this away for use later */
130 
131  ap->backmap.nbx = nbx;
132  ap->backmap.nby = nby;
133  ap->backmap.nbsize = nbsize;
134  ap->backmap.bvals = bvals;
135 
136  /* Finally a counter array */
137 
138  nnp = cpl_malloc(nbx*sizeof(int));
139 
140  /* Loop for each row of background squares. Start by initialising
141  the accumulators and histograms */
142 
143  for (l = 0; l < nby; l++) {
144  memset((char *)nnp,0,nbx*sizeof(*nnp));
145  for (i = 0; i < nbx; i++)
146  memset((char *)hist[i],0,MAXHIST*sizeof(int));
147 
148  /* Skim through the data in this stripe. Find out which square each
149  belongs to and add it it to the relevant histogram */
150 
151  ll = l*npixstripe;
152  for (i = 0; i < npixstripe; i++) {
153  if (map[ll+i] != nullval && mflag[ll+i] != MF_ZEROCONF &&
154  mflag[ll+i] != MF_STUPID_VALUE) {
155  isquare = (int)((float)(i % nx)/(float)nbsize);
156  isquare = MIN(nbx-1,MAX(0,isquare));
157  ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(map[i+ll])));
158  hist[isquare][ilev-MINHISTVAL] += 1;
159  nnp[isquare] += 1;
160  }
161  }
162 
163  /* but only do background estimation if enough pixels ----------- */
164 
165  for (j = 0; j < nbx; j++) {
166  if (nnp[j] > 0.25*nbsize*nbsize){
167  shist = hist[j];
168  imcore_medsig(shist,MAXHIST,MINHISTVAL-1,nnp[j],&skymed,&sigma);
169 
170  /* do an iterative 3-sigma upper clip to give a more robust
171  estimator */
172 
173  iclip = MAXHISTVAL;
174  mcpix = nnp[j];
175  skymedc = skymed;
176  sigmac = sigma;
177  for (iloop = 0; iloop < 3; iloop++) {
178  irej = 0;
179  for(i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
180  irej += shist[i-MINHISTVAL];
181  if (irej == 0)
182  break;
183  iclip = NINT(skymedc+3.0*sigmac) - 1;
184  mcpix = mcpix - irej;
185  imcore_medsig(shist,MAXHIST,MINHISTVAL-1,mcpix,&skymedc,
186  &sigmac);
187  }
188  bvals[l][j] = skymedc;
189  } else {
190  bvals[l][j] = -1000.0;
191  }
192  }
193  }
194 
195  /* filter raw background values */
196 
197  bfilt(bvals,nbx,nby);
198 
199  /* compute average sky level */
200 
201  work = cpl_malloc(nbx*nby*sizeof(*work));
202  k = 0;
203  for(l = 0; l < nby; l++)
204  for(j = 0; j < nbx; j++)
205  work[k++] = bvals[l][j];
206  sortit(work,nbx*nby);
207  avsky = work[(nbx*nby)/2];
208  freespace(work);
209 
210  /* ok now correct map for background variations and put avsky back on */
211 
212  nbsizo2 = nbsize/2;
213  fnbsize = 1.0/((float)nbsize);
214  for (k = 0; k < ny; k++) {
215  kk = k*nx;
216 
217  /* Nearest background pixel vertically */
218 
219  iby = (k + 1 + nbsizo2)/nbsize;
220  ibyp1 = iby + 1;
221  iby = MIN(nby,MAX(1,iby));
222  ibyp1 = MIN(nby,ibyp1);
223  dely = (k + 1 - nbsize*iby + nbsizo2)*fnbsize;
224 
225  for (j = 0; j < nx; j++) {
226  if (map[kk+j] == nullval)
227  continue;
228 
229  /* nearest background pixel across */
230 
231  ibx = (j + 1 + nbsizo2)/nbsize;
232  ibxp1 = ibx + 1;
233  ibx = MIN(nbx,MAX(1,ibx));
234  ibxp1 = MIN(nbx,ibxp1);
235  delx = (j + 1 - nbsize*ibx + nbsizo2)*fnbsize;
236 
237  /* bilinear interpolation to find background */
238 
239  t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
240  t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
241  dsky = avsky - (1.0 - delx)*t1 - delx*t2;
242  map[kk+j] += dsky;
243  }
244  }
245 
246  /* Free some workspace */
247 
248  tidy();
249  return(VIR_OK);
250 }
251 
252 /*---------------------------------------------------------------------------*/
289 /*---------------------------------------------------------------------------*/
290 
291 extern int imcore_backstats(ap_t *ap, float nullval, int satonly,
292  float *skymed, float *skysig, float *sat) {
293  int ilev,iclip,iloop,i,*ihist,isat,iter;
294  long mpix,npix,k,mcpix,irej,lpix,nx,ny;
295  float skymedc,sigmac,*map,sata,fac,skyref;
296  unsigned char *mflag;
297 
298  /* Get some info from the ap structure */
299 
300  map = ap->indata;
301  nx = ap->lsiz;
302  ny = ap->csiz;
303  mflag = ap->mflag;
304 
305  /* Check to make sure there are some non-zero values here */
306 
307  ilev = 1;
308  for (i = 0; i < nx*ny; i++) {
309  if (map[i] != nullval && mflag[i] != MF_ZEROCONF &&
310  mflag[i] != MF_STUPID_VALUE) {
311  ilev = 0;
312  break;
313  }
314  }
315  if (ilev == 1) {
316  *skymed = 0.0;
317  *skysig = 0.0;
318  *sat = 0.0;
319  return(VIR_WARN);
320  }
321 
322  /* First, get some workspace for the background histogram */
323 
324  ihist = cpl_calloc(MAXHIST,sizeof(*ihist));
325 
326  /* Loop for up to 10 iterations. For each iteration we multiply the
327  input data by a successively higher power of 2 in order to
328  try and deal with data that has very small noise estimates */
329 
330  fac = 0.5;
331  skyref = 0.0;
332  for (iter = 0; iter <= 9; iter++) {
333  fac *= 2.0;
334  if (iter == 1)
335  skyref = skymedc;
336  for (k = 0; k < MAXHIST; k++)
337  ihist[k] = 0;
338 
339  /* Now form the histogram of all pixel intensities */
340 
341  mpix = 0;
342  isat = 0;
343  npix = nx*ny;
344  for (k = 0; k < npix; k++) {
345  if (map[k] != nullval && mflag[k] != MF_ZEROCONF &&
346  mflag[k] != MF_STUPID_VALUE) {
347  ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(fac*(map[k]-skyref))));
348  ihist[ilev - MINHISTVAL] += 1;
349  isat = MAX(isat,ilev);
350  mpix++;
351  }
352  }
353  sata = MIN(MAXHISTVAL,MAX(MINSATURATE,0.9*((float)isat))/fac);
354  lpix = ihist[isat - MINHISTVAL];
355  while (lpix < mpix/1000 && isat > MINHISTVAL) {
356  isat--;
357  lpix += ihist[isat - MINHISTVAL];
358  }
359  *sat = ((float)isat)/fac + skyref;
360  *sat = MIN(MAXHISTVAL,MAX(MINSATURATE,MAX(0.95*(*sat),sata)));
361 
362  /* If all you want is the saturation level, then get out of here...*/
363 
364  if (satonly) {
365  freespace(ihist);
366  return(VIR_OK);
367  }
368 
369  /* Now find the median and sigma */
370 
371  imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mpix,skymed,skysig);
372 
373  /* Do an iterative 3-sigma upper clip to give a more robust
374  estimator */
375 
376  iclip = MAXHISTVAL;
377  mcpix = mpix;
378  skymedc = *skymed;
379  sigmac = *skysig;
380  for (iloop = 0; iloop < 3; iloop++) {
381  irej = 0;
382  for (i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
383  irej += ihist[i - MINHISTVAL];
384  if (irej == 0)
385  break;
386  iclip = NINT(skymedc+3.0*sigmac)-1;
387  mcpix = mcpix-irej;
388  imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mcpix,&skymedc,&sigmac);
389  }
390  if (sigmac > 2.5)
391  break;
392  }
393 
394  /* Set the final answer */
395 
396  *skymed = skymedc/fac + skyref;
397  *skysig = sigmac/fac;
398  freespace(ihist);
399  return(VIR_OK);
400 }
401 
402 /*---------------------------------------------------------------------------*/
436 /*---------------------------------------------------------------------------*/
437 
438 extern void imcore_backest(ap_t *ap, float x, float y, float *skylev,
439  float *skyrms) {
440  int i,j,nbx,nby,nbsize,nbsizo2,iby,ibyp1,ibx,ibxp1;
441  float **bvals,fnbsize,dely,delx,t1,t2;
442 
443  /* Define some local variables */
444 
445  nbx = ap->backmap.nbx;
446  nby = ap->backmap.nby;
447  nbsize = ap->backmap.nbsize;
448  bvals = ap->backmap.bvals;
449 
450  /* Get closest pixel to the input location */
451 
452  i = NINT(x);
453  j = NINT(y);
454 
455  /* Now, work out where in the map to do the interpolation */
456 
457  nbsizo2 = nbsize/2;
458  fnbsize = 1.0/((float)nbsize);
459  iby = (j + nbsizo2)/nbsize;
460  ibyp1 = iby + 1;
461  iby = MIN(nby,MAX(1,iby));
462  ibyp1 = MIN(nby,ibyp1);
463  dely = (j - nbsize*iby + nbsizo2)*fnbsize;
464  ibx = (i + nbsizo2)/nbsize;
465  ibxp1 = ibx + 1;
466  ibx = MIN(nbx,MAX(1,ibx));
467  ibxp1 = MIN(nbx,ibxp1);
468  delx = (i - nbsize*ibx + nbsizo2)*fnbsize;
469 
470  /* Now do a linear interpolation to find the background. Calculate MAD of
471  the four adjacent background cells as an estimate of the RMS */
472 
473  t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
474  t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
475  *skylev = (1.0 - delx)*t1 + delx*t2;
476  *skyrms = 0.25*(fabsf(bvals[iby-1][ibx-1] - *skylev) +
477  fabsf(bvals[ibyp1-1][ibx-1] - *skylev) +
478  fabsf(bvals[iby-1][ibxp1-1] - *skylev) +
479  fabsf(bvals[ibyp1-1][ibxp1-1] - *skylev));
480 }
481 
482 /*---------------------------------------------------------------------------*/
519 /*---------------------------------------------------------------------------*/
520 
521 extern void imcore_medsig(int *shist, int nh, int ist, int itarg, float *med,
522  float *sig) {
523  int isum, medata;
524  float ffrac,sigmed;
525 
526  /* median */
527 
528  isum = 0;
529  medata = ist;
530  while (isum <= (itarg+1)/2 && (medata-MINHISTVAL) < nh) {
531  medata++;
532  isum += shist[medata-MINHISTVAL];
533  }
534  if (shist[medata-MINHISTVAL] == 0) {
535  ffrac = 0.0;
536  } else {
537  ffrac = (float)(isum - (itarg+1)/2)/(float)shist[medata-MINHISTVAL];
538  }
539  *med = (float)medata - ffrac + 0.5;
540 
541  /* sigma */
542 
543  isum = 0;
544  medata = ist;
545  while (isum <= (itarg+3)/4 && (medata-MINHISTVAL) < nh) {
546  medata++;
547  isum += shist[medata-MINHISTVAL];
548  }
549  if (shist[medata-MINHISTVAL] == 0) {
550  ffrac = 0.0;
551  } else {
552  ffrac = (float)(isum - (itarg+3)/4)/(float)shist[medata-MINHISTVAL];
553  }
554  sigmed = (float)medata - ffrac + 0.5;
555  *sig = 1.48*(*med - sigmed);
556  *sig = MAX(0.5,*sig);
557 }
558 
559 /*---------------------------------------------------------------------------*/
578 /*---------------------------------------------------------------------------*/
579 
580 static void sortit (float ia[], int n) {
581  int i, j, ii, jj, ifin;
582  float it;
583 
584  jj = 4;
585  while (jj < n)
586  jj = 2 * jj;
587  jj = MIN(n,(3 * jj)/4 - 1);
588  while (jj > 1) {
589  jj = jj/2;
590  ifin = n - jj;
591  for (ii = 0; ii < ifin; ii++) {
592  i = ii;
593  j = i + jj;
594  if (ia[i] <= ia[j])
595  continue;
596  it = ia[j];
597  do {
598  ia[j] = ia[i];
599  j = i;
600  i = i - jj;
601  if (i < 0)
602  break;
603  } while (ia[i] > it);
604  ia[j] = it;
605  }
606  }
607  return;
608 }
609 
610 
611 static void tidy(void) {
612  int i;
613 
614  freespace(nnp);
615  if (hist != NULL) {
616  for (i = 0; i < npvx; i++)
617  freespace(hist[i]);
618  }
619  freespace(hist);
620  return;
621 }
622 
625 /*
626 
627 $Log: not supported by cvs2svn $
628 Revision 1.14 2010/09/09 12:09:57 jim
629 Added docs
630 
631 Revision 1.13 2009/01/30 08:25:17 jim
632 fixed a bug imcore_backstats
633 
634 Revision 1.12 2009/01/23 12:24:33 jim
635 Fixed bugs in pixel flagging
636 
637 Revision 1.11 2009/01/19 14:34:15 jim
638 fixed doc
639 
640 Revision 1.10 2008/10/28 14:22:50 jim
641 Fixed bug in imcore_background involving flagging bad pixels
642 
643 Revision 1.9 2008/10/13 08:10:35 jim
644 fixed pixel masking scheme
645 
646 Revision 1.8 2007/12/19 13:16:50 jim
647 Modified the interation step size
648 
649 Revision 1.7 2007/04/23 12:51:27 jim
650 Traps for 0 in confidence map
651 
652 Revision 1.6 2007/03/01 12:38:26 jim
653 Small modifications after a bit of code checking
654 
655 Revision 1.5 2006/08/01 11:27:54 jim
656 Modifications to imcore background estimation and to add ability to
657 specify the smoothing kernel width
658 
659 Revision 1.4 2006/07/11 14:50:27 jim
660 Modified to do arithmetic in double precision for stats
661 
662 Revision 1.3 2006/06/30 21:31:09 jim
663 MOdifications to background routines and smoothing kernel
664 
665 Revision 1.2 2006/04/20 11:15:19 jim
666 A few minor bugs fixed
667 
668 Revision 1.1 2005/09/13 13:25:28 jim
669 Initial entry after modifications to make cpl compliant
670 
671 
672 */
673 
void imcore_backest(ap_t *ap, float x, float y, float *skylev, float *skyrms)
Work out estimated sky for a pixel position.
int imcore_backstats(ap_t *ap, float nullval, int satonly, float *skymed, float *skysig, float *sat)
Work out robust background estimate over a whole input image.
void bfilt(float **xbuf, int nx, int ny)
Do bilinear median and linear filtering on background values.
Definition: imcore_filter.c:70
void imcore_medsig(int *shist, int nh, int ist, int itarg, float *med, float *sig)
Analyse histogram to work out median and sigma.
int imcore_background(ap_t *ap, int nbsize, float nullval)
Model and create background map.