VIRCAM Pipeline  1.3.4
vircam_genlincur.c
1 /* $Id: vircam_genlincur.c,v 1.34 2012-01-15 17:40:09 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2012-01-15 17:40:09 $
24  * $Revision: 1.34 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <cpl.h>
35 
36 #include <math.h>
37 
38 #include "vircam_mods.h"
39 #include "vircam_utils.h"
40 #include "vircam_stats.h"
41 #include "vircam_pfits.h"
42 #include "vircam_channel.h"
43 
44 
45 #define SZCOLNAME 16
46 
47 static double nom_val = 10000;
48 
49 
50 static double linval(double inval, double *kfacs, double tolerance,
51  int niter, double *b, int norder);
52 static double getkfac(long index, long ncpts, float reset_time,
53  float read_time, float delay_time, float exptime);
54 static void getco(double *a, int nord, int m);
55 
58 /*---------------------------------------------------------------------------*/
115 /*---------------------------------------------------------------------------*/
116 
117 extern int vircam_genlincur(double **fdata, int nimages, double *exps,
118  double mindit, vir_tfits *chantab,
119  int norder, cpl_table **lchantab,
120  double **lindata, int *status) {
121 
122  const char *fctid = "vircam_genlincur";
123  int retval,i,j,nbad,oldnorder,k,ii,nullval;
124  long np;
125  double *meds,sigfit,**aco,c0,lin_nom,*temp,*polyco,pt,*work,kfac;
126  double sum,t10000,*kfacs;
127  parquet *p,*pp;
128  cpl_table *ctab,*lc;
129  cpl_array *exparray,*medsarray,*polyfitco,*workarray;
130  char colname[SZCOLNAME];
131 
132  /* Inherited status */
133 
134  *lchantab = NULL;
135  if (*status != VIR_OK)
136  return(*status);
137 
138  /* Check that you have enough images in the list */
139 
140  if (nimages < norder+1) {
141  cpl_msg_error(fctid,
142  "Not enought images (%" CPL_SIZE_FORMAT ") for fit order (%" CPL_SIZE_FORMAT ")",
143  (cpl_size)nimages,(cpl_size)norder);
144  FATAL_ERROR
145  }
146 
147  /* Open the parquet structure for the channel table */
148 
149  ctab = vircam_tfits_get_table(chantab);
150  retval = vircam_chan_fill(ctab,&pp,&np);
151  if (retval != VIR_OK) {
152  *status = retval;
153  return(retval);
154  }
155 
156  /* Create an output channel table. Copy the input channel table and then
157  massage the linearity part */
158 
159  lc = cpl_table_duplicate(ctab);
160  oldnorder = cpl_table_get_int(lc,"norder",0,&nullval);
161  if (oldnorder > norder) {
162  for (i = norder+1; i <= oldnorder; i++) {
163  snprintf(colname,SZCOLNAME,"coeff_%d",i);
164  cpl_table_erase_column(lc,colname);
165  }
166  } else if (oldnorder < norder) {
167  for (i = oldnorder+1; i <= norder; i++) {
168  snprintf(colname,SZCOLNAME,"coeff_%d",i);
169  if (cpl_table_has_column(lc,colname))
170  continue;
171  cpl_table_new_column(lc,colname,CPL_TYPE_DOUBLE);
172  }
173  }
174 
175  /* Get some memory for the fitting arrays */
176 
177  exparray = cpl_array_wrap_double(exps,nimages);
178  medsarray = cpl_array_new((cpl_size)nimages,CPL_TYPE_DOUBLE);
179  meds = cpl_array_get_data_double(medsarray);
180  aco = cpl_malloc(norder*sizeof(double *));
181  for (i = 0; i < norder; i++)
182  aco[i] = cpl_malloc(norder*sizeof(double));
183  temp = cpl_malloc(norder*sizeof(double));
184  kfacs = cpl_malloc(norder*sizeof(double));
185 
186  /* Get memory for output array of linearised stats */
187 
188  *lindata = cpl_malloc(nimages*np*sizeof(double));
189 
190  /* Loop for each channel */
191 
192  nbad = 0;
193  for (i = 0; i < np; i++) {
194  p = pp + i;
195 
196  /* Load the data up for this channel */
197 
198  for (j = 0; j < nimages; j++)
199  meds[j] = fdata[j][i];
200 
201  /* Do the initial fit */
202 
203  if (vircam_polyfit(exparray,medsarray,norder,1,2,2.0,2.0,&polyfitco,
204  &sigfit) != VIR_OK) {
205  nbad++;
206  cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
207  cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
208  for (k = 1; k < norder; k++) {
209  snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
210  cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
211  }
212  freearray(polyfitco);
213  continue;
214  }
215  polyco = cpl_array_get_data_double(polyfitco);
216 
217  /* Work out linearity */
218 
219  for (j = 0; j < nimages; j++)
220  if (meds[j] > nom_val)
221  break;
222  t10000 = exps[j-1] + (nom_val - meds[j-1])/(meds[j] - meds[j-1]);
223  sum = 0.0;
224  for (j = 0; j < norder; j++)
225  sum += (double)(j+1)*polyco[j]*pow(t10000,(double)j);
226  lin_nom = 100.0*fabs(sum - polyco[0])/polyco[0];
227 
228  /* Get intermediate coefficients for matrix */
229 
230  for (j = 0; j < norder; j++) {
231  getco(temp,norder,j+1);
232  for (k = 0; k < norder; k++) {
233  pt = pow(mindit,(double)(j-k));
234  aco[j][k] = pt*temp[k];
235  }
236  }
237 
238  /* Solve matrix equation to do the back substitution */
239 
240  if (vircam_solve_gauss(aco,polyco,norder) != VIR_OK) {
241  nbad++;
242  cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
243  cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
244  for (k = 1; k < norder; k++) {
245  snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
246  cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
247  }
248  freearray(polyfitco);
249  continue;
250  }
251 
252  /* Now normalise to unit slope and write the result to the table*/
253 
254  c0 = polyco[0];
255  for (j = 0; j < norder; j++) {
256  polyco[j] /= pow(c0,(double)(j+1));
257  snprintf(colname,SZCOLNAME,"coeff_%d",j+1);
258  cpl_table_set_double(lc,colname,(cpl_size)i,polyco[j]);
259  }
260  cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
261 
262  /* Work out how well the solution creates a linear system. Loop
263  for each input image and work out a 'linearised' median. Then
264  do a linear fit to the linearsed median vs exposure time. */
265 
266  workarray = cpl_array_new((cpl_size)nimages,CPL_TYPE_DOUBLE);
267  work = cpl_array_get_data_double(workarray);
268  for (j = 0; j < nimages; j++) {
269  kfac = mindit/exps[j];
270  kfacs[0] = 1.0;
271  for (ii = 1; ii < norder; ii++)
272  kfacs[ii] = pow((kfac+1.0),(double)(ii+1)) -
273  pow(kfac,(double)(ii+1));
274  work[j] = linval(meds[j],kfacs,0.5,10,polyco,norder);
275  (*lindata)[j*np+i] = work[j];
276  }
277  freearray(polyfitco);
278  (void)vircam_polyfit(exparray,workarray,2,0,2,2.0,2.0,&polyfitco,
279  &sigfit);
280  polyco = cpl_array_get_data_double(polyfitco);
281  sigfit *= 100.0/nom_val;
282  freearray(workarray);
283  freearray(polyfitco);
284 
285  /* Put the nominal linearity and the fit quality into the header */
286 
287  cpl_table_set_double(lc,"lin_10000_err",(cpl_size)i,sigfit);
288  cpl_table_set_double(lc,"lin_10000",(cpl_size)i,lin_nom);
289  }
290 
291  /* Tidy and get out of here */
292 
293  *lchantab = cpl_table_duplicate(lc);
294  cpl_array_unwrap(exparray);
295  freearray(medsarray);
296  freespace2(aco,norder);
297  freespace(temp);
298  freetable(lc);
299  vircam_chan_free(np,&pp);
300  freespace(kfacs);
301  if (nbad != 0) {
302  cpl_msg_warning(fctid,
303  "%" CPL_SIZE_FORMAT "channels have a failed solution",
304  (cpl_size)nbad);
305  WARN_RETURN
306  }
307  GOOD_STATUS
308 }
309 
310 
311 /*---------------------------------------------------------------------------*/
352 /*--------------------------------------------------------------------------*/
353 
354 extern int vircam_lincor(vir_fits *infile, vir_tfits *lchantab, int kconst,
355  int ndit, int *status) {
356  int retval,i,norder,ii;
357  long naxis[2],j,rind,aind,ncpts,np;
358  float *data,texp,reset_time,read_time,delay_time;
359  double kfac_nom,lkfac,inval,outval,*lbb,dnd,*kfacs;
360  const char *fctid = "vircam_lincor";
361  parquet *pp;
362  cpl_propertylist *plist;
363  cpl_table *lctab;
364  parquet *p;
365 
366  /* Inherited status */
367 
368  if (*status != VIR_OK)
369  return(*status);
370 
371  /* Do we even need to be here? */
372 
373  if (cpl_propertylist_has(vircam_fits_get_ehu(infile),"ESO DRS LINCOR"))
374  return(*status);
375 
376  /* Open the parquet structure for the channel table */
377 
378  lctab = vircam_tfits_get_table(lchantab);
379  retval = vircam_chan_fill(lctab,&p,&np);
380  if (retval != VIR_OK)
381  return(retval);
382 
383  /* Get the data array for the image */
384 
385  data = cpl_image_get_data(vircam_fits_get_image(infile));
386  if (data == NULL) {
387  vircam_chan_free(np,&p);
388  cpl_msg_error(fctid,"Error mapping data in input image");
389  FATAL_ERROR
390  }
391  naxis[0] = (long)cpl_image_get_size_x(vircam_fits_get_image(infile));
392  naxis[1] = (long)cpl_image_get_size_y(vircam_fits_get_image(infile));
393 
394  /* Get the required parameters from the header */
395 
396  plist = vircam_fits_get_ehu(infile);
397  if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
398  vircam_chan_free(np,&p);
399  cpl_msg_error(fctid,"No exposure time in %s",
400  vircam_fits_get_fullname(infile));
401  FATAL_ERROR
402  }
403  if (vircam_pfits_get_mindit(plist,&reset_time) != VIR_OK) {
404  vircam_chan_free(np,&p);
405  cpl_msg_error(fctid,"No mindit time in %s",
406  vircam_fits_get_fullname(infile));
407  FATAL_ERROR
408  }
409  read_time = reset_time;
410  if (vircam_pfits_get_ditdelay(plist,&delay_time) != VIR_OK) {
411  vircam_chan_free(np,&p);
412  cpl_msg_error(fctid,"No dit delay time in %s",
413  vircam_fits_get_fullname(infile));
414  FATAL_ERROR
415  }
416 
417  /* If there is a constant k factor, then calculate it now */
418 
419  kfac_nom = (double)(read_time/texp);
420 
421  /* Factor to take the number of DITs into account */
422 
423  dnd = (double)ndit;
424 
425  /* Loop for each channel now */
426 
427  for (i = 0; i < np; i++) {
428  pp = p + i;
429  ncpts = (pp->delta_i)*(pp->delta_j);
430 
431  /* Load up the fit coefficients. If there is only one coefficient
432  this is by definition 1 and therefore we can skip this channel */
433 
434  norder = pp->norder;
435  if (norder == 1)
436  continue;
437  lbb = pp->bb;
438 
439  /* Get workspace for K array and fill it in for situation of constant
440  k factor */
441 
442  kfacs = cpl_malloc(norder*sizeof(double));
443  if (kconst) {
444  kfacs[0] = 1.0;
445  for (ii = 1; ii < norder; ii++)
446  kfacs[ii] = pow((kfac_nom+1.0),(double)(ii+1)) -
447  pow(kfac_nom,(double)(ii+1));
448  }
449 
450  /* Now for each pixel */
451 
452  for (j = 0; j < ncpts; j++) {
453 
454  /* Get the 'non-constant' k-factor and fill in the K array. */
455 
456  rind = vircam_chan_d2r(pp,j);
457  aind = vircam_chan_r2a(pp,naxis,rind);
458  if (! kconst) {
459  lkfac = getkfac(j,ncpts,reset_time,read_time,delay_time,texp);
460  kfacs[0] = 1.0;
461  for (ii = 1; ii < norder; ii++)
462  kfacs[ii] = pow((lkfac+1.0),(double)(ii+1)) -
463  pow(lkfac,(double)(ii+1));
464  }
465 
466  /* Calculate the linearised value now */
467 
468  inval = ((double)data[aind])/dnd;
469  outval = linval(inval,kfacs,0.5,10,lbb,norder);
470  data[aind] = (float)(dnd*outval);
471  }
472  freespace(kfacs);
473  }
474 
475  /* Add the linearity table to the DRS header */
476 
477  cpl_propertylist_update_string(vircam_fits_get_ehu(infile),
478  "ESO DRS LINCOR",
479  vircam_tfits_get_filename(lchantab));
480 
481  /* Right, get out of here */
482 
483  vircam_chan_free(np,&p);
484  GOOD_STATUS
485 }
486 
487 /*---------------------------------------------------------------------------*/
516 /*---------------------------------------------------------------------------*/
517 
518 static double linval(double inval, double *kfacs, double tolerance,
519  int niter, double *b, int norder) {
520  int jj,iter;
521  double val_old,val,tol,sum;
522 
523  val = inval;
524  iter = 0;
525  tol = tolerance + 1.0;
526  while (iter < niter && tol > tolerance) {
527  val_old = val;
528  iter++;
529  sum = 0.0;
530  for (jj = norder - 1; jj >= 1; jj--)
531  sum = (sum + b[jj]*kfacs[jj])*val;
532  sum *= val;
533  val = inval - sum;
534  tol = fabs(val - val_old);
535  if (val > 65535.0) {
536  val = 65535.0;
537  break;
538  } else if (val < -1000.0) {
539  val = -1000.0;
540  break;
541  }
542  }
543  return(val);
544 }
545 
546 
547 /*---------------------------------------------------------------------------*/
575 /*---------------------------------------------------------------------------*/
576 
577 static double getkfac(long index, long npts, float reset_time,
578  float read_time, float delay_time, float exptime) {
579  double tkfac,dt1,dt2,dt3,dt4,df;
580 
581  df = ((double)index/(double)npts);
582  dt1 = (double)exptime;
583  dt2 = (double)read_time;
584  dt3 = (double)reset_time;
585  dt4 = (double)delay_time;
586  tkfac = (dt3 + dt4 + (dt2 - dt3)*df)/dt1;
587  return(tkfac);
588 }
589 
590 /*---------------------------------------------------------------------------*/
612 /*---------------------------------------------------------------------------*/
613 
614 static void getco(double *a, int nord, int m) {
615  int i,j,start;
616 
617  for (i = 0; i < nord; i++)
618  a[i] = 0.0;
619  start = m-1;
620  a[start] = 1.0;
621  j = 1;
622  for (i = start-1; i >= 0; i--) {
623  j++;
624  a[i] = a[i+1]*(double)(m - j + 2)/(double)(j-1);
625  }
626 }
627 
628 
632 /*
633 
634 $Log: not supported by cvs2svn $
635 Revision 1.33 2010/06/03 12:15:31 jim
636 A few mods to get rid of compiler warnings
637 
638 Revision 1.32 2009/12/11 06:53:35 jim
639 Minor changes to documentation
640 
641 Revision 1.31 2009/09/09 09:45:36 jim
642 Modified to speed things up
643 
644 Revision 1.30 2009/06/08 08:08:15 jim
645 Fixed memory leak and changed clipping parameters to vircam_polyfit
646 
647 Revision 1.29 2009/05/20 12:18:42 jim
648 Modified so that if the operation is already done, then it just returns
649 
650 Revision 1.28 2009/02/20 10:50:55 jim
651 Removed superfluous declarations
652 
653 Revision 1.27 2008/10/21 08:38:48 jim
654 Final estiamte on linearity error done with vircam_polyfit now
655 
656 Revision 1.26 2008/09/29 11:26:18 jim
657 Modified linval to provide a lower limit in case the fit goes crazy
658 
659 Revision 1.25 2008/08/28 09:05:37 jim
660 Fixed bug where QC was being duplicated from master BPM on rare occasions.
661 Fixed bug where ARCFILE wasn't being written to the paf file for illum_cor
662 tables. Sky combine is done with medians
663 
664 Revision 1.24 2008/01/22 19:45:24 jim
665 New version of genlincur to take into account the equality of readout and
666 reset time
667 
668 Revision 1.23 2007/11/26 09:57:14 jim
669 Linearity correction routines now take account of ndit
670 
671 Revision 1.22 2007/11/22 12:36:15 jim
672 Modified to return linearised values in an array
673 
674 Revision 1.21 2007/11/20 09:38:57 jim
675 changed definition of fit quality to percentage error at 10000 counts
676 
677 Revision 1.20 2007/11/14 14:47:32 jim
678 Modified the qualfit definition to be back in line with DRLD
679 
680 Revision 1.19 2007/11/14 12:34:21 jim
681 Fixed header comments
682 
683 Revision 1.18 2007/11/14 10:48:29 jim
684 Major rewrite to incorporate simpler and more robust algorithm
685 
686 Revision 1.17 2007/03/29 12:19:39 jim
687 Little changes to improve documentation
688 
689 Revision 1.16 2007/03/01 12:42:41 jim
690 Modified slightly after code checking
691 
692 Revision 1.15 2006/11/27 12:08:18 jim
693 Modified lincor to get a better answer. Also modified the way the fit quality
694 is calculated
695 
696 Revision 1.14 2006/09/29 11:19:31 jim
697 changed aliases on parameter names
698 
699 Revision 1.13 2006/07/04 09:19:05 jim
700 replaced all sprintf statements with snprintf
701 
702 Revision 1.12 2006/06/09 11:26:26 jim
703 Small changes to keep lint happy
704 
705 Revision 1.11 2006/04/20 11:23:15 jim
706 Now medians the data before accumulation in the event that k is constant.
707 
708 Revision 1.10 2006/03/23 21:18:47 jim
709 Minor changes mainly to comment headers
710 
711 Revision 1.9 2006/03/22 13:36:50 jim
712 cosmetic changes to keep lint happy
713 
714 Revision 1.8 2006/03/15 10:43:41 jim
715 Fixed a few things
716 
717 Revision 1.7 2006/03/08 14:32:21 jim
718 Lots of little modifications
719 
720 Revision 1.6 2006/03/03 14:29:46 jim
721 Modified definition of vir_fits and channel table
722 
723 Revision 1.5 2006/03/01 10:31:28 jim
724 Now uses new vir_fits objects
725 
726 Revision 1.4 2006/02/18 11:45:59 jim
727 Fixed a couple of memory bugs
728 
729 Revision 1.3 2006/01/23 22:58:14 jim
730 Added vircam_lincor module
731 
732 Revision 1.2 2006/01/23 16:06:03 jim
733 Added in header comments and section to evaluate the goodness of fit
734 
735 Revision 1.1 2006/01/23 10:31:56 jim
736 New file
737 
738 
739 */
cpl_table * vircam_tfits_get_table(vir_tfits *p)
Definition: vircam_tfits.c:364
int vircam_solve_gauss(double **a, double *b, int m)
Definition: vircam_utils.c:644
int vircam_pfits_get_ditdelay(const cpl_propertylist *plist, float *ditdelay)
Get the value of dit delay time.
Definition: vircam_pfits.c:279
long vircam_chan_r2a(parquet *p, long naxis[2], long k)
int vircam_pfits_get_exptime(const cpl_propertylist *plist, float *exptime)
Get the value of exposure time.
Definition: vircam_pfits.c:245
int vircam_genlincur(double **fdata, int nimages, double *exps, double mindit, vir_tfits *chantab, int norder, cpl_table **lchantab, double **lindata, int *status)
Generate a linearity curve for each readout channel in a list of images.
int vircam_lincor(vir_fits *infile, vir_tfits *lchantab, int kconst, int ndit, int *status)
Apply linearity curves to data.
char * vircam_fits_get_fullname(vir_fits *p)
Definition: vircam_fits.c:560
cpl_image * vircam_fits_get_image(vir_fits *p)
Definition: vircam_fits.c:349
int vircam_pfits_get_mindit(const cpl_propertylist *plist, float *mindit)
Get the value of mindit time.
Definition: vircam_pfits.c:262
int vircam_polyfit(const cpl_array *xarray, const cpl_array *yarray, int ncoefs, int ilim, int niter, float lclip, float hclip, cpl_array **polycf, double *sigfit)
Definition: vircam_utils.c:759
cpl_propertylist * vircam_fits_get_ehu(vir_fits *p)
Definition: vircam_fits.c:457
int vircam_chan_fill(cpl_table *tab, parquet **p, long *np)
char * vircam_tfits_get_filename(vir_tfits *p)
Definition: vircam_tfits.c:510
long vircam_chan_d2r(parquet *p, long l)
void vircam_chan_free(int np, parquet **p)