VIRCAM Pipeline  1.3.3
vircam_linearity_analyse.c
1 /* $Id: vircam_linearity_analyse.c,v 1.65 2012-01-16 12:32:18 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2012-01-16 12:32:18 $
24  * $Revision: 1.65 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <stdio.h>
35 #include <cpl.h>
36 #include <math.h>
37 
38 #include "vircam_utils.h"
39 #include "vircam_pfits.h"
40 #include "vircam_dfs.h"
41 #include "vircam_mods.h"
42 #include "vircam_channel.h"
43 #include "vircam_stats.h"
44 #include "vircam_paf.h"
45 
46 /* Function prototypes */
47 
48 static int vircam_linearity_analyse_create(cpl_plugin *) ;
49 static int vircam_linearity_analyse_exec(cpl_plugin *) ;
50 static int vircam_linearity_analyse_destroy(cpl_plugin *) ;
51 static int vircam_linearity_analyse(cpl_parameterlist *, cpl_frameset *) ;
52 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
53  cpl_parameterlist *parlist);
54 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
55  cpl_parameterlist *parlist);
56 static int vircam_linearity_analyse_domedark_groups(void);
57 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
58  parquet *p, int np);
59 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
60  int nchan, double *facrng,
61  double *maxdiff);
62 static void vircam_mjdsort(double **fdata, double *mjd, int n);
63 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows);
64 static void vircam_linearity_analyse_init(void);
65 static void vircam_linearity_analyse_tidy(int level);
66 
67 /* Static global variables */
68 
69 static struct {
70 
71  /* Input */
72 
73  int norder;
74  float lthr;
75  float hthr;
76  int maxbpmfr;
77  int adjust;
78  int diagnostic;
79  int extenum;
80 
81  /* Output */
82 
83  float linearity;
84  float linerror;
85  float bad_pixel_stat;
86  int bad_pixel_num;
87  float facrng;
88  float maxdiff;
89 
90 } vircam_linearity_analyse_config;
91 
92 typedef struct {
93  cpl_frameset *darks;
94  cpl_frameset *domes;
95  int ndarks;
96  int ndomes;
97  float exptime;
98  unsigned char flag;
99  vir_fits **proc;
100 } ddgrp;
101 
102 #define OK_FLAG 0
103 #define SATURATE_FLAG 1
104 
105 #define SUBSET 128
106 #define SUBSET2 (SUBSET/2)
107 
108 static struct {
109  cpl_size *labels;
110  cpl_frameset *domelist;
111  int ndomes;
112  cpl_frameset *darklist;
113  int ndarks;
114  cpl_frameset *domecheck;
115  int ndomecheck;
116  cpl_frameset *darkcheck;
117  int ndarkcheck;
118  cpl_frame *inherit;
119  cpl_frame *chanfrm;
120  vir_tfits *chantab;
121  cpl_table *lchantab;
122  cpl_array *bpm_array;
123  ddgrp *ddg;
124  int nddg;
125  vir_fits **flatlist;
126  int nflatlist;
127  cpl_propertylist *plist;
128  cpl_propertylist *elist;
129  int nx;
130  int ny;
131  cpl_propertylist *phupaf;
132  cpl_table *diag1;
133  cpl_table *diag2;
134  int nfdata;
135  int nuse;
136 } ps;
137 
138 static int isfirst;
139 static int dummy;
140 static float sat;
141 static cpl_frame *product_frame_chantab = NULL;
142 static cpl_frame *product_frame_bpm = NULL;
143 static cpl_frame *product_frame_diag1 = NULL;
144 static cpl_frame *product_frame_diag2 = NULL;
145 
146 static char vircam_linearity_analyse_description[] =
147 "vircam_linearity_analyse -- VIRCAM linearity mapping recipe.\n\n"
148 "Form master dark images from the input raw frames and use these to\n"
149 "dark correct a series of dome flat exposures Using the dark\n"
150 "corrected dome flat series, work out linearity coefficients for\n"
151 "each data channel. The program expects the following files in the SOF\n"
152 " Tag Description\n"
153 " -----------------------------------------------------------------------\n"
154 " %-21s A list of raw dome flat images\n"
155 " %-21s A list of raw dark images\n"
156 " %-21s The channel table\n"
157 " %-21s A list of raw dome flat images at the monitor exposure time\n"
158 " %-21s A list of raw dark images at the monitor exposure time\n"
159 "The first three of these are required. The last two are only required if"
160 "the light source monitoring algorithm is to be used"
161 "\n";
162 
284 /* Function code */
285 
286 /*---------------------------------------------------------------------------*/
294 /*---------------------------------------------------------------------------*/
295 
296 int cpl_plugin_get_info(cpl_pluginlist *list) {
297  cpl_recipe *recipe = cpl_calloc(1,sizeof(*recipe));
298  cpl_plugin *plugin = &recipe->interface;
299  char alldesc[SZ_ALLDESC];
300  (void)snprintf(alldesc,SZ_ALLDESC,vircam_linearity_analyse_description,
301  VIRCAM_LIN_DOME_RAW,VIRCAM_LIN_DARK_RAW,
302  VIRCAM_CAL_CHANTAB_INIT,VIRCAM_LIN_DOME_CHECK,
303  VIRCAM_LIN_DARK_CHECK);
304 
305  cpl_plugin_init(plugin,
306  CPL_PLUGIN_API,
307  VIRCAM_BINARY_VERSION,
308  CPL_PLUGIN_TYPE_RECIPE,
309  "vircam_linearity_analyse",
310  "VIRCAM linearity analysis recipe",
311  alldesc,
312  "Jim Lewis",
313  "jrl@ast.cam.ac.uk",
315  vircam_linearity_analyse_create,
316  vircam_linearity_analyse_exec,
317  vircam_linearity_analyse_destroy);
318 
319  cpl_pluginlist_append(list,plugin);
320 
321  return(0);
322 }
323 
324 /*---------------------------------------------------------------------------*/
333 /*---------------------------------------------------------------------------*/
334 
335 static int vircam_linearity_analyse_create(cpl_plugin *plugin) {
336  cpl_recipe *recipe;
337  cpl_parameter *p;
338 
339  /* Get the recipe out of the plugin */
340 
341  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
342  recipe = (cpl_recipe *)plugin;
343  else
344  return(-1);
345 
346  /* Create the parameters list in the cpl_recipe object */
347 
348  recipe->parameters = cpl_parameterlist_new();
349 
350  /* Fill in the parameters. First the polynomial order */
351 
352  p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.norder",
353  CPL_TYPE_INT,
354  "Order of polynomial fit",
355  "vircam.vircam_linearity_analyse",
356  4,1,6);
357  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nord");
358  cpl_parameterlist_append(recipe->parameters,p);
359 
360  /* The lower threshold */
361 
362  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.lthr",
363  CPL_TYPE_DOUBLE,
364  "Lower bad pixel threshold",
365  "vircam.vircam_linearity_analyse",8.0);
366  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"lthr");
367  cpl_parameterlist_append(recipe->parameters,p);
368 
369  /* The upper threshold */
370 
371  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.hthr",
372  CPL_TYPE_DOUBLE,
373  "Upper bad pixel threshold",
374  "vircam.vircam_linearity_analyse",8.0);
375  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"hthr");
376  cpl_parameterlist_append(recipe->parameters,p);
377 
378  /* The maximum number of frames to be used in forming the bad pixel mask */
379 
380  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.maxbpmfr",
381  CPL_TYPE_INT,
382  "Maximum # frames used in bpm analysis",
383  "vircam.vircam_linearity_analyse",10);
384  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"maxbpmfr");
385  cpl_parameterlist_append(recipe->parameters,p);
386 
387  /* The flag to allow statistics of the dome flat frames to be adjusted
388  using a set of monitor exposures */
389 
390  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.adjust",
391  CPL_TYPE_BOOL,
392  "Adjust stats with monitor set",
393  "vircam.vircam_linearity_analyse",1);
394  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"adjust");
395  cpl_parameterlist_append(recipe->parameters,p);
396 
397  /* The flag to allow diagnostic curves to be written out */
398 
399  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.diagnostic",
400  CPL_TYPE_BOOL,
401  "Write out diagnostic tables",
402  "vircam.vircam_linearity_analyse",0);
403  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"diagnostic");
404  cpl_parameterlist_append(recipe->parameters,p);
405 
406  /* Extension number of input frames to use */
407 
408  p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.extenum",
409  CPL_TYPE_INT,
410  "Extension number to be done, 0 == all",
411  "vircam.vircam_linearity_analyse",
412  1,0,16);
413  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
414  cpl_parameterlist_append(recipe->parameters,p);
415 
416  /* Get out of here */
417 
418  return(0);
419 }
420 
421 /*---------------------------------------------------------------------------*/
427 /*---------------------------------------------------------------------------*/
428 
429 static int vircam_linearity_analyse_exec(cpl_plugin *plugin) {
430  cpl_recipe *recipe;
431 
432  /* Get the recipe out of the plugin */
433 
434  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
435  recipe = (cpl_recipe *)plugin;
436  else
437  return(-1);
438 
439  return(vircam_linearity_analyse(recipe->parameters,recipe->frames));
440 }
441 
442 /*---------------------------------------------------------------------------*/
448 /*---------------------------------------------------------------------------*/
449 
450 static int vircam_linearity_analyse_destroy(cpl_plugin *plugin) {
451  cpl_recipe *recipe ;
452 
453  /* Get the recipe out of the plugin */
454 
455  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
456  recipe = (cpl_recipe *)plugin;
457  else
458  return(-1);
459 
460  cpl_parameterlist_delete(recipe->parameters);
461  return(0);
462 }
463 
464 /*---------------------------------------------------------------------------*/
471 /*---------------------------------------------------------------------------*/
472 
473 static int vircam_linearity_analyse(cpl_parameterlist *parlist,
474  cpl_frameset *framelist) {
475  const char *fctid="vircam_linearity_analyse";
476  char colname[16];
477  int i,jst,jfn,j,status,k,nbad,ngood,krem,n;
478  int ndarks,ndomes,kk,live,ngood_flats,*bpm,nalloc,adjust,ndit;
479  cpl_size nlab;
480  long np;
481  float med,mindit,*exps,badfrac,expt;
482  unsigned char *rejmask,*rejplus;
483  double *dexps,**fdata,*d,*mjds,*mjdcheck,mjd,**cdata,*cf,fac;
484  double facrng,maxdiff,*lindata;
485  vir_fits **darks,**domes,*test,*outdark,*fframe;
486  parquet *pp;
487  cpl_parameter *p;
488  cpl_propertylist *drs,*plist;
489  cpl_image *outimage;
490  cpl_frame *frame;
491 
492  /* Check validity of input frameset */
493 
494  if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
495  cpl_msg_error(fctid,"Input framelist NULL or has no input data");
496  return(-1);
497  }
498 
499  /* Check the files in the frameset */
500 
501  if (vircam_frameset_fexists(framelist) != VIR_OK) {
502  cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
503  return(-1);
504  }
505 
506  /* Initialise a few things */
507 
508  vircam_linearity_analyse_init();
509 
510  /* Get the parameters */
511 
512  p = cpl_parameterlist_find(parlist,
513  "vircam.vircam_linearity_analyse.norder");
514  vircam_linearity_analyse_config.norder = cpl_parameter_get_int(p);
515  p = cpl_parameterlist_find(parlist,
516  "vircam.vircam_linearity_analyse.lthr");
517  vircam_linearity_analyse_config.lthr = (float)cpl_parameter_get_double(p);
518  p = cpl_parameterlist_find(parlist,
519  "vircam.vircam_linearity_analyse.hthr");
520  vircam_linearity_analyse_config.hthr = (float)cpl_parameter_get_double(p);
521  p = cpl_parameterlist_find(parlist,
522  "vircam.vircam_linearity_analyse.maxbpmfr");
523  vircam_linearity_analyse_config.maxbpmfr = cpl_parameter_get_int(p);
524  p = cpl_parameterlist_find(parlist,
525  "vircam.vircam_linearity_analyse.adjust");
526  vircam_linearity_analyse_config.adjust = cpl_parameter_get_bool(p);
527  p = cpl_parameterlist_find(parlist,
528  "vircam.vircam_linearity_analyse.diagnostic");
529  vircam_linearity_analyse_config.diagnostic = cpl_parameter_get_bool(p);
530  p = cpl_parameterlist_find(parlist,
531  "vircam.vircam_linearity_analyse.extenum");
532  vircam_linearity_analyse_config.extenum = cpl_parameter_get_int(p);
533 
534  /* Sort out raw from calib frames */
535 
536  if (vircam_dfs_set_groups(framelist) != VIR_OK) {
537  cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
538  vircam_linearity_analyse_tidy(2);
539  return(-1);
540  }
541 
542  /* Get framelist labels */
543 
544  if ((ps.labels = cpl_frameset_labelise(framelist,vircam_compare_tags,
545  &nlab)) == NULL) {
546  cpl_msg_error(fctid,"Cannot labelise the input frames");
547  vircam_linearity_analyse_tidy(2);
548  return(-1);
549  }
550 
551  /* Get the dome flat frames */
552 
553  if ((ps.domelist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
554  VIRCAM_LIN_DOME_RAW)) == NULL) {
555  cpl_msg_error(fctid,"Cannot find dome flat frames in input frameset");
556  vircam_linearity_analyse_tidy(2);
557  return(-1);
558  }
559  ps.ndomes = cpl_frameset_get_size(ps.domelist);
560  ps.inherit = cpl_frameset_get_first(ps.domelist);
561 
562  /* Check to make sure that NDIT == 1 */
563 
564  plist = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_frame(ps.domelist,0)),0);
565  (void)vircam_pfits_get_ndit(plist,&ndit);
566  freepropertylist(plist);
567  if (ndit != 1) {
568  cpl_msg_error(fctid,
569  "NDIT=%" CPL_SIZE_FORMAT ". Recipe requires that ndit == 1",
570  (cpl_size)ndit);
571  vircam_linearity_analyse_tidy(2);
572  return(-1);
573  }
574 
575  /* Get the dark frames */
576 
577  if ((ps.darklist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
578  VIRCAM_LIN_DARK_RAW)) == NULL) {
579  cpl_msg_error(fctid,"Cannot find dark frames in input frameset");
580  vircam_linearity_analyse_tidy(2);
581  return(-1);
582  }
583  ps.ndarks = cpl_frameset_get_size(ps.darklist);
584 
585  /* If you are planning to adjust the stats for the dome frames by using
586  a monitor exposure set, then read that frameset in now */
587 
588  if (vircam_linearity_analyse_config.adjust) {
589  if ((ps.domecheck = vircam_frameset_subgroup(framelist,ps.labels,nlab,
590  VIRCAM_LIN_DOME_CHECK)) == NULL) {
591  cpl_msg_info(fctid,"No monitor frames found in sof. No adjustments made to stats");
592  vircam_linearity_analyse_config.adjust = 0;
593  ps.ndomecheck = 0;
594  } else {
595  ps.ndomecheck = cpl_frameset_get_size(ps.domecheck);
596  if ((ps.darkcheck = vircam_frameset_subgroup(framelist,ps.labels,
597  nlab,VIRCAM_LIN_DARK_CHECK)) == NULL) {
598  cpl_msg_info(fctid,"No darks for monitor frames found in sof. No adjustments made to stats");
599  vircam_linearity_analyse_config.adjust = 0;
600  ps.ndomecheck = 0;
601  freeframeset(ps.domecheck);
602  ps.ndarkcheck = 0;
603  } else {
604  ps.ndarkcheck = cpl_frameset_get_size(ps.darkcheck);
605  }
606  }
607  }
608 
609  /* Check to see if there is a channel table. If so, then read it */
610 
611  if ((ps.chanfrm = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
612  VIRCAM_CAL_CHANTAB_INIT)) == NULL) {
613  cpl_msg_error(fctid,"No initial channel table found");
614  vircam_linearity_analyse_tidy(2);
615  return(-1);
616  }
617 
618  /* Group the domes and darks by exposure time */
619 
620  if (vircam_linearity_analyse_domedark_groups() != 0) {
621  vircam_linearity_analyse_tidy(2);
622  return(-1);
623  }
624 
625  /* See if the number of exposure times is too small for the order of the
626  polynomial you want to fit. If it is, the adjust the order */
627 
628  if (ps.nddg < vircam_linearity_analyse_config.norder+1) {
629  cpl_msg_warning(fctid,
630  "Number of exposure times is too small: %" CPL_SIZE_FORMAT ", order: %" CPL_SIZE_FORMAT "\nTaking fit down to order %" CPL_SIZE_FORMAT,
631  (cpl_size)(ps.nddg),
632  (cpl_size)(vircam_linearity_analyse_config.norder),
633  (cpl_size)(ps.nddg-1));
634  vircam_linearity_analyse_config.norder = ps.nddg - 1;
635  }
636 
637  /* Now, how many image extensions do we want to do? If the extension
638  number is zero, then we loop for all possible extensions. If it
639  isn't then we just do the extension specified */
640 
641  vircam_exten_range(vircam_linearity_analyse_config.extenum,
642  (const cpl_frame *)cpl_frameset_get_frame(ps.ddg[0].darks,0),
643  &jst,&jfn);
644  if (jst == -1 || jfn == -1) {
645  cpl_msg_error(fctid,"Unable to continue");
646  vircam_linearity_analyse_tidy(2);
647  return(-1);
648  }
649 
650  /* Now loop for all the extensions and do the BPM analysis... */
651 
652  for (j = jst; j <= jfn; j++) {
653  cpl_msg_info(fctid,"Beginning BPM work on extension %" CPL_SIZE_FORMAT,
654  (cpl_size)j);
655  isfirst = (j == jst);
656  dummy = 0;
657  vircam_linearity_analyse_config.bad_pixel_stat = 0.0;
658  vircam_linearity_analyse_config.bad_pixel_num = 0;
659  vircam_linearity_analyse_config.linerror = 0.0;
660  vircam_linearity_analyse_config.linearity = 0.0;
661  vircam_linearity_analyse_config.facrng = 0.0;
662  vircam_linearity_analyse_config.maxdiff = 0.0;
663 
664  /* Get some standard info in case we need it for dummy products */
665 
666  test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[0].domes),
667  CPL_TYPE_FLOAT,j);
668  ps.plist = cpl_propertylist_duplicate(vircam_fits_get_phu(test));
669  ps.elist = cpl_propertylist_duplicate(vircam_fits_get_ehu(test));
670  ps.nx = (int)cpl_image_get_size_x(vircam_fits_get_image(test));
671  ps.ny = (int)cpl_image_get_size_y(vircam_fits_get_image(test));
672  vircam_fits_delete(test);
673 
674  /* Load up the channel table for this detector and verify it. Get
675  saturation level for this detector from FITS header */
676 
677  ps.chantab = vircam_tfits_load(ps.chanfrm,j);
678  if (ps.chantab == NULL) {
679  cpl_msg_error(fctid,
680  "Channel table extension %" CPL_SIZE_FORMAT " failed to load",
681  (cpl_size)j);
682  dummy = 1;
683  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
684  return(-1);
685  continue;
686  } else if (vircam_chantab_verify(vircam_tfits_get_table(ps.chantab))
687  != VIR_OK) {
688  cpl_msg_error(fctid,
689  "Channel table extension %" CPL_SIZE_FORMAT " has errors",
690  (cpl_size)j);
691  dummy = 1;
692  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
693  return(-1);
694  continue;
695  }
697  &sat) != VIR_OK) {
698  cpl_msg_error(fctid,
699  "Channel table extension header %" CPL_SIZE_FORMAT " missing saturation info",
700  (cpl_size)j);
701  dummy = 1;
702  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
703  return(-1);
704  vircam_linearity_analyse_tidy(1);
705  continue;
706  }
707 
708  /* Get the channel structure */
709 
710  vircam_chan_fill(vircam_tfits_get_table(ps.chantab),&pp,&np);
711 
712  /* If doing diagnostics, then create the tables now */
713 
714  if (vircam_linearity_analyse_config.diagnostic) {
715  ps.diag1 = vircam_linearity_analyse_diagtab_init(np,ps.ndomes);
716  if (vircam_linearity_analyse_config.adjust)
717  ps.diag2 = vircam_linearity_analyse_diagtab_init(np,ps.ndomecheck);
718  }
719 
720  /* Check DETLIVE for this extension */
721 
722  if (vircam_pfits_get_detlive((const cpl_propertylist *)ps.elist,&live)
723  != VIR_OK) {
724  cpl_msg_error(fctid,"No DET LIVE keyword in this extension");
725  dummy = 1;
726  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
727  return(-1);
728  vircam_linearity_analyse_tidy(1);
729  continue;
730  }
731  if (! live) {
732  cpl_msg_info(fctid,"Detector flagged dead");
733  dummy = 1;
734  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
735  return(-1);
736  vircam_linearity_analyse_tidy(1);
737  continue;
738  }
739 
740  /* Get the value of MINDIT so that you can work out the rough level
741  of the flats before the reset frame was subtracted off */
742 
743  if (vircam_pfits_get_mindit((const cpl_propertylist *)ps.elist,
744  &mindit) != VIR_OK) {
745  cpl_msg_error(fctid,
746  "No value of MINDIT found in extension %" CPL_SIZE_FORMAT,
747  (cpl_size)j);
748  dummy = 1;
749  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
750  return(-1);
751  continue;
752  }
753 
754  /* Get a representative from each exposure group and check if it
755  is saturated. If it is, then reject the group from further
756  analysis */
757 
758  ngood = 0;
759  exps = cpl_malloc(ps.nddg*sizeof(float));
760  ngood_flats = 0;
761  for (i = 0; i < ps.nddg; i++) {
762  test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[i].domes),
763  CPL_TYPE_FLOAT,j);
764  med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
765  med *= (1.0 + mindit/ps.ddg[i].exptime);
766  if (med > sat) {
767  ps.ddg[i].flag = SATURATE_FLAG;
768  } else {
769  ngood++;
770  exps[ngood-1] = ps.ddg[i].exptime;
771  ngood_flats += ps.ddg[i].ndomes;
772  ps.ddg[i].flag = OK_FLAG;
773  }
774  vircam_fits_delete(test);
775  }
776  exps = cpl_realloc(exps,ngood*sizeof(float));
777 
778  /* Are there enough non-saturated exposures for linearity fit? */
779 
780  if (ngood < vircam_linearity_analyse_config.norder+1) {
781  cpl_msg_info(fctid,
782  "Too few unsaturated flats for linearity fit for extension %" CPL_SIZE_FORMAT,
783  (cpl_size)j);
784  dummy = 1;
785  cpl_free(exps);
786  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
787  return(-1);
788  continue;
789  }
790 
791  /* Sort the exposure array */
792 
793  vircam_sort(&exps,ngood,1);
794 
795  /* Loop for each exposure time. When you have enough flats, then
796  you can quit this loop */
797 
798  ps.nuse = min(vircam_linearity_analyse_config.maxbpmfr,ngood_flats);
799  ps.nflatlist = 0;
800  ps.flatlist = cpl_malloc(ps.nuse*sizeof(vir_fits *));
801  for (i = ngood-1; i >= 0; i--) {
802  krem = -1;
803  for (k = 0; k <= ps.nddg; k++) {
804  if (ps.ddg[k].exptime == exps[i]) {
805  krem = k;
806  break;
807  }
808  }
809 
810  /* Load the dark frames from this exposure time */
811 
812  darks = vircam_fits_load_list(ps.ddg[krem].darks,CPL_TYPE_FLOAT,j);
813  ndarks = ps.ddg[krem].ndarks;
814  if (darks == NULL) {
815  cpl_msg_error(fctid,
816  "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
817  (cpl_size)j,ps.ddg[krem].exptime);
818  continue;
819  }
820 
821  /* Form a mean dark for this exposure time. If there is only one
822  then don't bother, just load up that one frame. */
823 
824  if (ndarks == 1) {
825  outdark = vircam_fits_duplicate(darks[0]);
826  } else {
827  status = VIR_OK;
828  (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,&outimage,
829  &rejmask,&rejplus,&drs,&status);
830  freespace(rejmask);
831  freespace(rejplus);
832  freepropertylist(drs);
833  if (status != VIR_OK) {
834  cpl_msg_error(fctid,
835  "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
836  (cpl_size)j,ps.ddg[krem].exptime);
837  freefitslist(darks,ndarks);
838  continue;
839  }
840  outdark = vircam_fits_wrap(outimage,darks[0],NULL,NULL);
841  }
842  freefitslist(darks,ndarks);
843 
844  /* Load the flats for this group */
845 
846  domes = vircam_fits_load_list(ps.ddg[krem].domes,CPL_TYPE_FLOAT,j);
847  ndomes = ps.ddg[krem].ndomes;
848  if (domes == NULL) {
849  cpl_msg_error(fctid,
850  "Error loading domes extension %" CPL_SIZE_FORMAT ", exptime %g",
851  (cpl_size)j,ps.ddg[i].exptime);
852  freefits(outdark);
853  continue;
854  }
855 
856  /* Now loop for each flat in this group or until you have
857  filled the flats buffer */
858 
859  for (kk = 0; kk < ndomes; kk++) {
860  status = VIR_OK;
861  vircam_darkcor(domes[kk],outdark,1.0,&status);
862  ps.flatlist[ps.nflatlist] = vircam_fits_duplicate(domes[kk]);
863  ps.ddg[krem].proc[kk] = ps.flatlist[ps.nflatlist];
864  ps.nflatlist++;
865  if (ps.nflatlist == ps.nuse)
866  break;
867  }
868 
869  /* Tidy up a bit */
870 
871  freefitslist(domes,ndomes);
872  freefits(outdark);
873 
874  /* Do we have enough yet? */
875 
876  if (ps.nflatlist == ps.nuse)
877  break;
878  }
879  freespace(exps);
880  ps.flatlist = cpl_realloc(ps.flatlist,
881  (ps.nflatlist)*sizeof(vir_fits *));
882 
883  /* Generate a bad pixel mask now */
884 
885  status = VIR_OK;
886  (void)vircam_genbpm(ps.flatlist,ps.nflatlist,
887  vircam_linearity_analyse_config.lthr,
888  vircam_linearity_analyse_config.hthr,
889  &(ps.bpm_array),&nbad,&badfrac,&status);
890  bpm = cpl_array_get_data_int(ps.bpm_array);
891 
892  /* Store away some useful info */
893 
894  vircam_linearity_analyse_config.bad_pixel_num = nbad;
895  vircam_linearity_analyse_config.bad_pixel_stat = badfrac;
896 
897  /* Right. Free the pointer for flatlist, but don't delete the
898  vir_fits structures in the list because they are copied into
899  the ddg structure */
900 
901  freespace(ps.flatlist);
902  ps.nflatlist = 0;
903 
904  /* Get an initial allocation of space to hold the stats */
905 
906  nalloc = 16;
907  fdata = cpl_malloc(nalloc*sizeof(double *));
908  dexps = cpl_malloc(nalloc*sizeof(double));
909  mjds = cpl_malloc(nalloc*sizeof(double));
910 
911  /* Loop through the ddg structure, missing out any that have
912  overexposed flats. Work out the stats for the images that
913  remain. */
914 
915  cpl_msg_info(fctid,
916  "Beginning linearity work on extension %" CPL_SIZE_FORMAT,
917  (cpl_size)j);
918  ps.nfdata = 0;
919  outdark = NULL;
920  for (i = 0; i < ps.nddg; i++) {
921  if (ps.ddg[i].flag == SATURATE_FLAG)
922  continue;
923  for (k = 0; k < ps.ddg[i].ndomes; k++) {
924 
925  /* If this particular frame wasn't processed, then you need
926  to form a dark for this group. */
927 
928  if (ps.ddg[i].proc[k] == NULL) {
929  if (outdark == NULL) {
930 
931  /* Load the dark frames from this exposure time */
932 
933  darks = vircam_fits_load_list(ps.ddg[i].darks,
934  CPL_TYPE_FLOAT,j);
935  ndarks = ps.ddg[i].ndarks;
936  if (darks == NULL) {
937  cpl_msg_error(fctid,
938  "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
939  (cpl_size)j,ps.ddg[i].exptime);
940  continue;
941  }
942 
943  /* Form a mean dark for this exposure time. If there
944  is only one then don't bother, just load up that
945  one frame. */
946 
947  if (ps.ddg[i].ndarks == 1) {
948  outdark = vircam_fits_duplicate(darks[0]);
949  } else {
950  status = VIR_OK;
951  (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
952  &outimage,&rejmask,&rejplus,
953  &drs,&status);
954  freespace(rejmask);
955  freespace(rejplus);
956  freepropertylist(drs);
957  if (status != VIR_OK) {
958  cpl_msg_error(fctid,
959  "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
960  (cpl_size)j,ps.ddg[i].exptime);
961  freefitslist(darks,ndarks);
962  continue;
963  }
964  outdark = vircam_fits_wrap(outimage,darks[0],
965  NULL,NULL);
966  }
967  freefitslist(darks,ndarks);
968  }
969 
970  /* Load the flat and dark correct it */
971 
972  frame = cpl_frameset_get_frame(ps.ddg[i].domes,k);
973  fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
974  vircam_darkcor(fframe,outdark,1.0,&status);
975 
976  /* If this frame has already been corrected, then use it */
977 
978  } else {
979  fframe = ps.ddg[i].proc[k];
980  }
981 
982  /* Generate the stats for this frame and store it */
983 
984  d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
985  if (ps.nfdata >= nalloc) {
986  nalloc += 16;
987  fdata = cpl_realloc(fdata,nalloc*sizeof(double *));
988  dexps = cpl_realloc(dexps,nalloc*sizeof(double));
989  mjds = cpl_realloc(mjds,nalloc*sizeof(double));
990  }
991  (void)vircam_pfits_get_mjd(vircam_fits_get_phu(fframe),&mjd);
992  mjds[ps.nfdata] = mjd;
993  dexps[ps.nfdata] = (double)(ps.ddg[i].exptime);
994  fdata[ps.nfdata] = d;
995 
996  /* If doing diagnostic curves, then add the relevant
997  information to the first table now */
998 
999  if (ps.diag1 != NULL) {
1000  cpl_table_set_string(ps.diag1,"filename",
1001  (cpl_size)(ps.nfdata),
1002  vircam_fits_get_filename(fframe));
1003  cpl_table_set_double(ps.diag1,"exptime",
1004  (cpl_size)ps.nfdata,
1005  dexps[ps.nfdata]);
1006  cpl_table_set_double(ps.diag1,"mjd",(cpl_size)(ps.nfdata),
1007  mjd);
1008  for (n = 1; n <= np; n++) {
1009  snprintf(colname,16,"rawflux_%02d",n);
1010  cpl_table_set_double(ps.diag1,colname,
1011  (cpl_size)(ps.nfdata),d[n-1]);
1012  }
1013  }
1014  if (ps.ddg[i].proc[k] != NULL) {
1015  freefits(ps.ddg[i].proc[k]);
1016  } else {
1017  freefits(fframe);
1018  }
1019  ps.nfdata++;
1020  }
1021  freefits(outdark);
1022  }
1023  freefits(outdark);
1024  if (ps.diag1 != NULL)
1025  cpl_table_set_size(ps.diag1,(cpl_size)(ps.nfdata));
1026 
1027  /* Now, if we are going to tweak the stats using the monitor exposure
1028  then we should do that now */
1029 
1030  if (vircam_linearity_analyse_config.adjust) {
1031 
1032  /* Get some workspace for the data array */
1033 
1034  cdata = cpl_malloc(ps.ndomecheck*sizeof(double *));
1035 
1036  /* Get the exposure time for the monitor set and make sure
1037  that the set isn't saturated */
1038 
1039  test = vircam_fits_load(cpl_frameset_get_first(ps.domecheck),
1040  CPL_TYPE_FLOAT,j);
1041  (void)vircam_pfits_get_exptime(vircam_fits_get_phu(test),&expt);
1042  med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
1043  med *= (1.0 + mindit/expt);
1044  adjust = 1;
1045  if (med > sat) {
1046  cpl_msg_info(fctid,"Monitor exposures saturated. No drift adjustment made");
1047  adjust = 0;
1048  }
1049  vircam_fits_delete(test);
1050 
1051  /* Ok assuming all that's going well, then form a mean dark */
1052 
1053  if (adjust) {
1054 
1055  darks = vircam_fits_load_list(ps.darkcheck,CPL_TYPE_FLOAT,j);
1056  ndarks = ps.ndarkcheck;
1057  if (darks == NULL) {
1058  cpl_msg_error(fctid,
1059  "Error loading check darks extension %" CPL_SIZE_FORMAT,
1060  (cpl_size)j);
1061  continue;
1062  }
1063 
1064  /* Form a mean dark for this exposure time. If there
1065  is only one then don't bother, just load up that
1066  one frame. */
1067 
1068  if (ndarks == 1) {
1069  outdark = vircam_fits_duplicate(darks[0]);
1070  } else {
1071  status = VIR_OK;
1072  (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
1073  &outimage,&rejmask,&rejplus,
1074  &drs,&status);
1075  freespace(rejmask);
1076  freespace(rejplus);
1077  freepropertylist(drs);
1078  if (status != VIR_OK) {
1079  cpl_msg_error(fctid,
1080  "Combine failure extension %" CPL_SIZE_FORMAT " monitor",
1081  (cpl_size)j);
1082  freefitslist(darks,ndarks);
1083  continue;
1084  }
1085  outdark = vircam_fits_wrap(outimage,darks[0],
1086  NULL,NULL);
1087  }
1088  freefitslist(darks,ndarks);
1089 
1090  /* Now, loop through the monitor domes, dark correct and then
1091  do the stats */
1092 
1093  mjdcheck = cpl_malloc(ps.ndomecheck*sizeof(double));
1094  for (i = 0; i < ps.ndomecheck; i++) {
1095  frame = cpl_frameset_get_frame(ps.domecheck,i);
1096  fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
1097  vircam_darkcor(fframe,outdark,1.0,&status);
1098  d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
1099  cdata[i] = d;
1102  mjdcheck[i] = mjd;
1103 
1104  /* If doing diagnostics then fill in the table now */
1105 
1106  if (ps.diag2 != NULL) {
1107  cpl_table_set_string(ps.diag2,"filename",(cpl_size)i,
1108  vircam_fits_get_filename(fframe));
1109  cpl_table_set_double(ps.diag2,"exptime",
1110  (cpl_size)i,(double)expt);
1111  cpl_table_set_double(ps.diag2,"mjd",
1112  (cpl_size)i,mjd);
1113  for (n = 1; n <= np; n++) {
1114  snprintf(colname,16,"rawflux_%02d",n);
1115  cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
1116  d[n-1]);
1117  snprintf(colname,16,"linflux_%02d",n);
1118  cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
1119  d[n-1]);
1120  }
1121  }
1122  freefits(fframe);
1123  }
1124  freefits(outdark);
1125 
1126  /* Generate the correction factors now */
1127 
1128  cf = vircam_linearity_tweakfac(cdata,mjdcheck,ps.ndomecheck,
1129  np,&facrng,&maxdiff);
1130  vircam_linearity_analyse_config.facrng = 100.0*(float)facrng;
1131  vircam_linearity_analyse_config.maxdiff = 100.0*(float)maxdiff;
1132  if (ps.diag2 != NULL) {
1133  for (i = 0; i < ps.ndomecheck; i++)
1134  cpl_table_set_double(ps.diag2,"adjust_fac",
1135  (cpl_size)i,cf[i]);
1136  }
1137 
1138  /* Ok, now do the correction for each of the linearity
1139  sequence frames */
1140 
1141  for (i = 0; i < ps.nfdata; i++) {
1142  mjd = mjds[i];
1143  krem = -1;
1144  for (k = 0; k < ps.ndomecheck; k++) {
1145  if (mjd < mjdcheck[k]) {
1146  krem = k;
1147  break;
1148  }
1149  }
1150  if (krem == -1) {
1151  fac = cf[ps.ndomecheck-1];
1152  } else if (krem == 0) {
1153  fac = cf[0];
1154  } else {
1155  fac = 0.5*(cf[krem -1] + cf[krem]);
1156  }
1157  for (k = 0; k < np; k++)
1158  fdata[i][k] /= fac;
1159  if (ps.diag1 != NULL)
1160  cpl_table_set_double(ps.diag1,"adjust_fac",
1161  (cpl_size)i,fac);
1162  }
1163 
1164  /* Get rid of some stuff now */
1165 
1166  freespace2(cdata,ps.ndomecheck);
1167  freespace(cf);
1168  freespace(mjdcheck);
1169  }
1170  }
1171 
1172  /* Do some intermediate tidying */
1173 
1174  freespace(mjds);
1175  vircam_chan_free(np,&pp);
1176 
1177  /* Right, there should be no images left in memory now. Do the
1178  linearity analysis now. */
1179 
1180  (void)vircam_genlincur(fdata,ps.nfdata,dexps,(double)mindit,ps.chantab,
1181  vircam_linearity_analyse_config.norder,
1182  &(ps.lchantab),&lindata,&status);
1183  if (ps.diag1 != NULL) {
1184  for (i = 0; i < ps.nfdata; i++) {
1185  for (n = 0; n < np; n++) {
1186  snprintf(colname,16,"linflux_%02d",n+1);
1187  cpl_table_set_double(ps.diag1,colname,
1188  (cpl_size)i,lindata[i*np+n]);
1189  }
1190  }
1191  }
1192  freespace2(fdata,ps.nfdata);
1193  freespace(dexps);
1194  freespace(lindata);
1195  if (status != VIR_OK) {
1196  cpl_msg_error(fctid,
1197  "Linearity curve fit failed extension %" CPL_SIZE_FORMAT,
1198  (cpl_size)j);
1199  dummy = 1;
1200  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
1201  return(-1);
1202  vircam_linearity_analyse_tidy(1);
1203  continue;
1204  }
1205 
1206  /* Get some QC1 parameters by finding the average fit error and
1207  the average percentage non-linearity */
1208 
1209  vircam_linearity_analyse_config.linearity =
1210  (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000");
1211  vircam_linearity_analyse_config.linerror =
1212  (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000_err");
1213 
1214  /* Save new linearity info */
1215 
1216  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
1217  return(-1);
1218  }
1219 
1220  /* Get out of here */
1221 
1222  vircam_linearity_analyse_tidy(2);
1223  return(0);
1224 }
1225 
1226 
1227 /*---------------------------------------------------------------------------*/
1234 /*---------------------------------------------------------------------------*/
1235 
1236 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
1237  cpl_parameterlist *parlist) {
1238  cpl_propertylist *plist,*elist,*pafprop;
1239  cpl_image *outimg;
1240  const char *outtab = "lchantab.fits";
1241  const char *outbpm = "bpm.fits";
1242  const char *outtabpaf = "lchantab";
1243  const char *outbpmpaf = "bpm";
1244  const char *outdiag1 = "ldiag1.fits";
1245  const char *outdiag2 = "ldiag2.fits";
1246  const char *fctid = "vircam_linearity_analyse_save";
1247  const char *recipeid = "vircam_linearity_analyse";
1248  int nx,ny,nord,*bpm,i;
1249 
1250  /* Do some stuff for the first extension to set up the frame */
1251 
1252  nord = vircam_linearity_analyse_config.norder;
1253  if (isfirst) {
1254 
1255  /* Set up the output frame */
1256 
1257  product_frame_chantab = cpl_frame_new();
1258  cpl_frame_set_filename(product_frame_chantab,outtab);
1259  cpl_frame_set_tag(product_frame_chantab,VIRCAM_PRO_CHANTAB);
1260  cpl_frame_set_type(product_frame_chantab,CPL_FRAME_TYPE_TABLE);
1261  cpl_frame_set_group(product_frame_chantab,CPL_FRAME_GROUP_PRODUCT);
1262  cpl_frame_set_level(product_frame_chantab,CPL_FRAME_LEVEL_FINAL);
1263 
1264  /* Set up the PRO keywords for primary */
1265 
1266  ps.phupaf = vircam_paf_phu_items(ps.plist);
1267  plist = cpl_propertylist_duplicate(ps.plist);
1268  vircam_dfs_set_product_primary_header(plist,product_frame_chantab,
1269  framelist,parlist,
1270  (char *)recipeid,
1271  "PRO-1.15",ps.inherit,0);
1272 
1273  /* Now define the table propertylist and give it an extension name and
1274  PRO keywords */
1275 
1276  elist = cpl_propertylist_duplicate(ps.elist);
1277  cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
1278  vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
1279  framelist,parlist,(char *)recipeid,
1280  "PRO-1.15",ps.inherit);
1281 
1282  /* Add QC1 info */
1283 
1284  cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
1285  vircam_linearity_analyse_config.linearity);
1286  cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
1287  "% non-linearity at 10000 ADU");
1288  cpl_propertylist_update_float(elist,"ESO QC LINERROR",
1289  vircam_linearity_analyse_config.linerror);
1290  cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
1291  "% error non-linearity at 10000 ADU");
1292  cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
1293  vircam_linearity_analyse_config.facrng);
1294  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
1295  "total % range in screen variation");
1296  cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
1297  vircam_linearity_analyse_config.maxdiff);
1298  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
1299  "maximum % step in screen variation");
1300  cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
1301 
1302  /* Set up a dummy table if necessary */
1303 
1304  if (dummy == 1) {
1305  vircam_dummy_property(elist);
1306  if (ps.lchantab == NULL)
1307  ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
1308  }
1309 
1310  /* And finally save the table */
1311 
1312  if (cpl_table_save(ps.lchantab,plist,elist,outtab,CPL_IO_DEFAULT)
1313  != CPL_ERROR_NONE) {
1314  cpl_msg_error(fctid,"Cannot save product table extension");
1315  freepropertylist(plist);
1316  freepropertylist(elist);
1317  return(-1);
1318  }
1319  cpl_frameset_insert(framelist,product_frame_chantab);
1320 
1321  /* Write PAF */
1322 
1323  pafprop = vircam_paf_req_items(elist);
1324  vircam_merge_propertylists(pafprop,ps.phupaf);
1325  vircam_paf_append(pafprop,plist,"ESO INS FILT1 NAME");
1326  vircam_paf_append(pafprop,elist,"ESO PRO CATG");
1327  vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
1328  if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
1329  "QC file",pafprop) != VIR_OK)
1330  cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
1331  cpl_propertylist_delete(pafprop);
1332 
1333  /* Quick tidy */
1334 
1335  freepropertylist(plist);
1336  freepropertylist(elist);
1337 
1338  /* Set up the output bad pixel mask primary */
1339 
1340  product_frame_bpm = cpl_frame_new();
1341  cpl_frame_set_filename(product_frame_bpm,outbpm);
1342  cpl_frame_set_tag(product_frame_bpm,VIRCAM_PRO_BPM);
1343  cpl_frame_set_type(product_frame_bpm,CPL_FRAME_TYPE_IMAGE);
1344  cpl_frame_set_group(product_frame_bpm,CPL_FRAME_GROUP_PRODUCT);
1345  cpl_frame_set_level(product_frame_bpm,CPL_FRAME_LEVEL_FINAL);
1346 
1347  /* Set up the PRO keywords for primary header in the bpm */
1348 
1349  plist = ps.plist;
1350  vircam_dfs_set_product_primary_header(plist,product_frame_bpm,
1351  framelist,parlist,
1352  (char *)recipeid,"PRO-1.15",
1353  ps.inherit,0);
1354 
1355  /* Now save the PHU 'image' */
1356 
1357  if (cpl_image_save(NULL,outbpm,CPL_TYPE_UCHAR,plist,
1358  CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
1359  cpl_msg_error(fctid,"Cannot save product PHU");
1360  cpl_frame_delete(product_frame_bpm);
1361  return(-1);
1362  }
1363  cpl_frameset_insert(framelist,product_frame_bpm);
1364 
1365  /* Section for saving diagnostic tables. First the linearity
1366  sequence diagnostics */
1367 
1368  if (ps.diag1 != NULL) {
1369 
1370  /* Set up the output frame */
1371 
1372  product_frame_diag1 = cpl_frame_new();
1373  cpl_frame_set_filename(product_frame_diag1,outdiag1);
1374  cpl_frame_set_tag(product_frame_diag1,VIRCAM_PRO_LIN_DIAG1);
1375  cpl_frame_set_type(product_frame_diag1,CPL_FRAME_TYPE_TABLE);
1376  cpl_frame_set_group(product_frame_diag1,CPL_FRAME_GROUP_PRODUCT);
1377  cpl_frame_set_level(product_frame_diag1,CPL_FRAME_LEVEL_FINAL);
1378 
1379  /* Set up the PRO keywords for primary */
1380 
1381  plist = cpl_propertylist_duplicate(ps.plist);
1382  vircam_dfs_set_product_primary_header(plist,product_frame_diag1,
1383  framelist,parlist,
1384  (char *)recipeid,
1385  "PRO-1.15",ps.inherit,0);
1386 
1387  /* Now define the table propertylist and give it an extension name and
1388  PRO keywords */
1389 
1390  elist = cpl_propertylist_duplicate(ps.elist);
1391  vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
1392  framelist,parlist,
1393  (char *)recipeid,"PRO-1.15",
1394  ps.inherit);
1395 
1396  /* Set up a dummy property if necessary */
1397 
1398  if (dummy == 1)
1399  vircam_dummy_property(elist);
1400 
1401  /* And finally save the table */
1402 
1403  if (cpl_table_save(ps.diag1,plist,elist,outdiag1,CPL_IO_DEFAULT)
1404  != CPL_ERROR_NONE) {
1405  cpl_msg_error(fctid,"Cannot save product table extension");
1406  freepropertylist(plist);
1407  freepropertylist(elist);
1408  return(-1);
1409  }
1410  cpl_frameset_insert(framelist,product_frame_diag1);
1411  freepropertylist(plist);
1412  freepropertylist(elist);
1413  }
1414 
1415  /* Now the monitor sequence diagnostics */
1416 
1417  if (ps.diag2 != NULL) {
1418 
1419  /* Set up the output frame */
1420 
1421  product_frame_diag2 = cpl_frame_new();
1422  cpl_frame_set_filename(product_frame_diag2,outdiag2);
1423  cpl_frame_set_tag(product_frame_diag2,VIRCAM_PRO_LIN_DIAG2);
1424  cpl_frame_set_type(product_frame_diag2,CPL_FRAME_TYPE_TABLE);
1425  cpl_frame_set_group(product_frame_diag2,CPL_FRAME_GROUP_PRODUCT);
1426  cpl_frame_set_level(product_frame_diag2,CPL_FRAME_LEVEL_FINAL);
1427 
1428  /* Set up the PRO keywords for primary */
1429 
1430  plist = cpl_propertylist_duplicate(ps.plist);
1431  vircam_dfs_set_product_primary_header(plist,product_frame_diag2,
1432  framelist,parlist,
1433  (char *)recipeid,
1434  "PRO-1.15",ps.inherit,0);
1435 
1436  /* Now define the table propertylist and give it an extension name
1437  and PRO keywords */
1438 
1439  elist = cpl_propertylist_duplicate(ps.elist);
1440  vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
1441  framelist,parlist,
1442  (char *)recipeid,"PRO-1.15",
1443  ps.inherit);
1444 
1445  /* Set up a dummy property if necessary */
1446 
1447  if (dummy == 1)
1448  vircam_dummy_property(elist);
1449 
1450  /* And finally save the table */
1451 
1452  if (cpl_table_save(ps.diag2,plist,elist,outdiag2,CPL_IO_DEFAULT)
1453  != CPL_ERROR_NONE) {
1454  cpl_msg_error(fctid,"Cannot save product table extension");
1455  freepropertylist(plist);
1456  freepropertylist(elist);
1457  return(-1);
1458  }
1459  cpl_frameset_insert(framelist,product_frame_diag2);
1460  freepropertylist(plist);
1461  freepropertylist(elist);
1462  }
1463 
1464  /* Section for all other extensions */
1465 
1466  } else {
1467 
1468  /* Do the table extension PRO keywords */
1469 
1470  elist = cpl_propertylist_duplicate(ps.elist);
1471  cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
1472  vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
1473  framelist,parlist,
1474  (char *)recipeid,
1475  "PRO-1.15",ps.inherit);
1476 
1477  /* Add QC1 info */
1478 
1479  cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
1480  vircam_linearity_analyse_config.linearity);
1481  cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
1482  "% non-linearity at 10000 ADU");
1483  cpl_propertylist_update_float(elist,"ESO QC LINERROR",
1484  vircam_linearity_analyse_config.linerror);
1485  cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
1486  "% error non-linearity at 10000 ADU");
1487  cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
1488  vircam_linearity_analyse_config.facrng);
1489  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
1490  "total % range in screen variation");
1491  cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
1492  vircam_linearity_analyse_config.maxdiff);
1493  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
1494  "maximum % step in screen variation");
1495  cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
1496 
1497  /* Set up a dummy table if necessary */
1498 
1499  if (dummy == 1) {
1500  vircam_dummy_property(elist);
1501  if (ps.lchantab == NULL)
1502  ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
1503  }
1504 
1505  /* And finally save the table */
1506 
1507  if (cpl_table_save(ps.lchantab,NULL,elist,outtab,CPL_IO_EXTEND)
1508  != CPL_ERROR_NONE) {
1509  cpl_msg_error(fctid,"Cannot save product table extension");
1510  freepropertylist(elist);
1511  return(-1);
1512  }
1513 
1514  /* Write PAF */
1515 
1516  pafprop = vircam_paf_req_items(elist);
1517  vircam_merge_propertylists(pafprop,ps.phupaf);
1518  vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
1519  vircam_paf_append(pafprop,elist,"ESO PRO CATG");
1520  vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
1521  if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
1522  "QC file",pafprop) != VIR_OK)
1523  cpl_msg_warning(fctid,"Unable to save PAF for BPM");
1524  cpl_propertylist_delete(pafprop);
1525 
1526  /* Quick tidy */
1527 
1528  freepropertylist(elist);
1529 
1530  /* Now the diagnostic tables */
1531 
1532  if (ps.diag1 != NULL) {
1533  elist = cpl_propertylist_duplicate(ps.elist);
1534  vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
1535  framelist,parlist,
1536  (char *)recipeid,"PRO-1.15",
1537  ps.inherit);
1538 
1539  /* Set up a dummy property if necessary */
1540 
1541  if (dummy == 1)
1542  vircam_dummy_property(elist);
1543 
1544  /* And finally save the table */
1545 
1546  if (cpl_table_save(ps.diag1,NULL,elist,outdiag1,CPL_IO_EXTEND)
1547  != CPL_ERROR_NONE) {
1548  cpl_msg_error(fctid,"Cannot save product table extension");
1549  freepropertylist(elist);
1550  return(-1);
1551  }
1552  freepropertylist(elist);
1553  }
1554  if (ps.diag2 != NULL) {
1555  elist = cpl_propertylist_duplicate(ps.elist);
1556  vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
1557  framelist,parlist,
1558  (char *)recipeid,"PRO-1.15",
1559  ps.inherit);
1560 
1561  /* Set up a dummy property if necessary */
1562 
1563  if (dummy == 1)
1564  vircam_dummy_property(elist);
1565 
1566  /* And finally save the table */
1567 
1568  if (cpl_table_save(ps.diag2,NULL,elist,outdiag2,CPL_IO_EXTEND)
1569  != CPL_ERROR_NONE) {
1570  cpl_msg_error(fctid,"Cannot save product table extension");
1571  freepropertylist(elist);
1572  return(-1);
1573  }
1574  freepropertylist(elist);
1575  }
1576 
1577  }
1578 
1579  /* Save the bpm extension now */
1580 
1581  plist = ps.elist;
1582  nx = ps.nx;
1583  ny = ps.ny;
1584  if (dummy && ps.bpm_array == NULL) {
1585  ps.bpm_array = cpl_array_new((cpl_size)(nx*ny),CPL_TYPE_INT);
1586  bpm = cpl_array_get_data_int(ps.bpm_array);
1587  for (i = 0; i < nx*ny; i++)
1588  bpm[i] = 0;
1589  }
1590  bpm = cpl_array_get_data_int(ps.bpm_array);
1591  vircam_dfs_set_product_exten_header(plist,product_frame_bpm,
1592  framelist,parlist,(char *)recipeid,
1593  "PRO-1.15",ps.inherit);
1594  cpl_propertylist_update_float(plist,"ESO QC BAD_PIXEL_STAT",
1595  vircam_linearity_analyse_config.bad_pixel_stat);
1596  cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_STAT",
1597  "Fraction of pixels that are bad");
1598  cpl_propertylist_update_int(plist,"ESO QC BAD_PIXEL_NUM",
1599  vircam_linearity_analyse_config.bad_pixel_num);
1600  cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_NUM",
1601  "Number of pixels that are bad");
1602  cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps.nuse);
1603  if (dummy)
1604  vircam_dummy_property(plist);
1605  outimg = cpl_image_wrap_int((cpl_size)nx,(cpl_size)ny,bpm);
1606  if (cpl_image_save(outimg,outbpm,CPL_TYPE_UCHAR,plist,
1607  CPL_IO_EXTEND) != CPL_ERROR_NONE) {
1608  cpl_msg_error(fctid,"Cannot save product image extension");
1609  return(-1);
1610  }
1611 
1612  /* Write PAF */
1613 
1614  pafprop = vircam_paf_req_items(plist);
1615  vircam_merge_propertylists(pafprop,ps.phupaf);
1616  vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
1617  vircam_paf_append(pafprop,ps.plist,"ESO PRO CATG");
1618  vircam_paf_append(pafprop,plist,"ESO PRO DATANCOM");
1619  if (vircam_paf_print((char *)outbpmpaf,"VIRCAM/vircam_linearity_analyse",
1620  "QC file",pafprop) != VIR_OK)
1621  cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
1622  cpl_propertylist_delete(pafprop);
1623 
1624  /* Quick tidy */
1625 
1626  cpl_image_unwrap(outimg);
1627 
1628  return(0);
1629 }
1630 
1631 /*---------------------------------------------------------------------------*/
1639 /*---------------------------------------------------------------------------*/
1640 
1641 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
1642  cpl_parameterlist *parlist) {
1643  int retval;
1644  const char *fctid = "vircam_linearity_analyse_lastbit";
1645 
1646  /* Save the new channel table and bad pixel map */
1647 
1648  cpl_msg_info(fctid,
1649  "Saving linearity table and bpm for extension %" CPL_SIZE_FORMAT,
1650  (cpl_size)jext);
1651  retval = vircam_linearity_analyse_save(framelist,parlist);
1652  if (retval != 0) {
1653  vircam_linearity_analyse_tidy(2);
1654  return(-1);
1655  }
1656 
1657  /* Do some intermediate tidying */
1658 
1659  vircam_linearity_analyse_tidy(1);
1660  return(0);
1661 }
1662 
1663 /*---------------------------------------------------------------------------*/
1667 /*---------------------------------------------------------------------------*/
1668 
1669 static int vircam_linearity_analyse_domedark_groups(void) {
1670  int i,j,found;
1671  float texp;
1672  cpl_frame *frame;
1673  cpl_propertylist *plist;
1674  const char *fctid = "vircam_linearity_analyse_domedark_groups";
1675 
1676  /* Start by getting the memory for the domedark groups */
1677 
1678  ps.ddg = cpl_calloc(ps.ndomes,sizeof(ddgrp));
1679  ps.nddg = 0;
1680 
1681  /* Loop for each of the dome frames and get its exposure time. If this
1682  doesn't exist, then signal an error and go on */
1683 
1684  for (i = 0; i < ps.ndomes; i++) {
1685  frame = cpl_frameset_get_frame(ps.domelist,i);
1686  plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
1687  if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
1688  cpl_msg_warning(fctid,"No exposure time found in %s",
1689  cpl_frame_get_filename(frame));
1690  cpl_propertylist_delete(plist);
1691  continue;
1692  }
1693  cpl_propertylist_delete(plist);
1694 
1695  /* Search the domedark groups to see if this exposure time has already
1696  been used. If not, then create a new group. If it has then just add
1697  this frame to the correct group */
1698 
1699  found = 0;
1700  for (j = 0; j < ps.nddg; j++) {
1701  if (ps.ddg[j].exptime == texp) {
1702  found = 1;
1703  break;
1704  }
1705  }
1706  if (found) {
1707  cpl_frameset_insert(ps.ddg[j].domes,cpl_frame_duplicate(frame));
1708  ps.ddg[j].ndomes += 1;
1709  } else {
1710  ps.ddg[ps.nddg].exptime = texp;
1711  ps.ddg[ps.nddg].darks = cpl_frameset_new();
1712  ps.ddg[ps.nddg].domes = cpl_frameset_new();
1713  ps.ddg[ps.nddg].ndarks = 0;
1714  ps.ddg[ps.nddg].ndomes = 1;
1715  ps.ddg[ps.nddg].flag = OK_FLAG;
1716  cpl_frameset_insert(ps.ddg[ps.nddg].domes,
1717  cpl_frame_duplicate(frame));
1718  ps.nddg += 1;
1719  }
1720  }
1721 
1722  /* Right, now loop through all the darks and get their exposure times */
1723 
1724  for (i = 0; i < ps.ndarks; i++) {
1725  frame = cpl_frameset_get_frame(ps.darklist,i);
1726  plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
1727  if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
1728  cpl_msg_warning(fctid,"No exposure time found in %s",
1729  cpl_frame_get_filename(frame));
1730  cpl_propertylist_delete(plist);
1731  continue;
1732  }
1733  cpl_propertylist_delete(plist);
1734 
1735  /* Search the domedark groups to see if this dark fits into one of
1736  the defined groups. If not, then ignore it. If it does, then
1737  add it into the dark frameset */
1738 
1739  found = 0;
1740  for (j = 0; j < ps.nddg; j++) {
1741  if (ps.ddg[j].exptime == texp) {
1742  found = 1;
1743  break;
1744  }
1745  }
1746  if (found) {
1747  cpl_frameset_insert(ps.ddg[j].darks,cpl_frame_duplicate(frame));
1748  ps.ddg[j].ndarks += 1;
1749  }
1750  }
1751 
1752  /* Now go through the domedark groups and ditch any that don't have any
1753  dark frames */
1754 
1755  i = 0;
1756  while (i < ps.nddg) {
1757  if (ps.ddg[i].ndarks == 0) {
1758  cpl_msg_warning(fctid,
1759  "No dark frames exist for exposure %g\nThrowing these away",
1760  ps.ddg[i].exptime);
1761  freeframeset(ps.ddg[i].darks);
1762  freeframeset(ps.ddg[i].domes);
1763  for (j = i+1; j < ps.nddg; j++)
1764  ps.ddg[j-1] = ps.ddg[j];
1765  ps.nddg -= 1;
1766  } else
1767  i++;
1768  }
1769 
1770  /* Allocate some space for vir_fits arrays for processed domes */
1771 
1772  for (i = 0; i < ps.nddg; i++) {
1773  ps.ddg[i].proc = cpl_malloc(ps.ddg[i].ndomes*sizeof(vir_fits *));
1774  for (j = 0; j < ps.ddg[i].ndomes; j++)
1775  ps.ddg[i].proc[j] = NULL;
1776  }
1777 
1778  /* Resize the output array and return so long as there is anything
1779  left. If there isn't then signal a major error */
1780 
1781  if (ps.nddg > 0) {
1782  ps.ddg = cpl_realloc(ps.ddg,ps.nddg*sizeof(ddgrp));
1783  return(0);
1784  } else {
1785  cpl_msg_error(fctid,"There are no darks defined for input domes");
1786  return(-1);
1787  }
1788 }
1789 
1790 /*---------------------------------------------------------------------------*/
1799 /*---------------------------------------------------------------------------*/
1800 
1801 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
1802  parquet *p, int np) {
1803  int i,ist,ifn,jst,jfn,n,jind2,iind2,jj,nx,ii;
1804  parquet *pp;
1805  double *d;
1806  float *tmp,*data;
1807 
1808  /* Get the workspace for the output result */
1809 
1810  d = cpl_malloc(np*sizeof(double));
1811 
1812  /* Get the input data array */
1813 
1814  nx = (int)cpl_image_get_size_x(vircam_fits_get_image(fframe));
1815  data = cpl_image_get_data_float(vircam_fits_get_image(fframe));
1816 
1817  /* Get some workspace for doing the median calculations */
1818 
1819  tmp = cpl_malloc(SUBSET*SUBSET*sizeof(float));
1820 
1821  /* Loop for each channel in the parquet structure */
1822 
1823  for (i = 0; i < np; i++) {
1824  pp = p + i;
1825 
1826  /* Take the central part of the channel */
1827 
1828  ist = ((pp->delta_i)/2 - SUBSET2);
1829  ifn = ist + SUBSET - 1;
1830  jst = ((pp->delta_j)/2 - SUBSET2);
1831  jfn = jst + SUBSET - 1;
1832 
1833  /* Put the data into the workspace and do a median */
1834 
1835  n = 0;
1836  for (jj = jst; jj <= jfn; jj++) {
1837  jind2 = (jj + pp->iymin - 1)*nx;
1838  for (ii = ist; ii <= ifn; ii++) {
1839  iind2 = jind2 + ii + pp->ixmin - 1;
1840  if (bpm[iind2] == 0)
1841  tmp[n++] = data[iind2];
1842  }
1843  }
1844  d[i] = (double)vircam_med(tmp,NULL,(long)n);
1845  }
1846 
1847  /* Tidy and get out of here */
1848 
1849  freespace(tmp);
1850  return(d);
1851 }
1852 
1853 /*---------------------------------------------------------------------------*/
1865 /*---------------------------------------------------------------------------*/
1866 
1867 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
1868  int nchan, double *facrng,
1869  double *maxdiff) {
1870  int i,ist,ifn,j;
1871  double *factors,sum,midval,minfac,maxfac;
1872 
1873  /* Get some memory for the output array */
1874 
1875  factors = cpl_malloc(nim*sizeof(double));
1876 
1877  /* First sort the data into order of mjd */
1878 
1879  vircam_mjdsort(fdata,mjd,nim);
1880 
1881  /* Which index is the midpoint? */
1882 
1883  if (nim % 2 == 0) {
1884  ist = nim/2 - 1;
1885  ifn = ist + 1;
1886  } else {
1887  ist = nim/2;
1888  ifn = ist;
1889  }
1890 
1891  /* Loop for each channel */
1892 
1893  for (i = 0; i < nchan; i++) {
1894 
1895  /* Get midpoint value */
1896 
1897  midval = 0.5*(fdata[ist][i] + fdata[ifn][i]);
1898 
1899  /* Now normalise all the ith channels by this value */
1900 
1901  for (j = 0; j < nim; j++)
1902  fdata[j][i] /= midval;
1903  }
1904 
1905  /* Now loop for each image and average the values for all the channels in
1906  in image */
1907 
1908  *maxdiff = 0.0;
1909  maxfac = 0.0;
1910  minfac = 0.0;
1911  for (j = 0; j < nim; j++) {
1912  sum = 0.0;
1913  for (i = 0; i < nchan; i++)
1914  sum += fdata[j][i];
1915  factors[j] = sum/(double)nchan;
1916  if (j == 0) {
1917  maxfac = factors[j];
1918  minfac = factors[j];
1919  } else {
1920  minfac = min(minfac,factors[j]);
1921  maxfac = max(maxfac,factors[j]);
1922  *maxdiff = max(*maxdiff,fabs(factors[j]-factors[j-1]));
1923  }
1924  }
1925  *facrng = maxfac - minfac;
1926 
1927  /* Get out of here */
1928 
1929  return(factors);
1930 }
1931 
1932 /*---------------------------------------------------------------------------*/
1940 /*---------------------------------------------------------------------------*/
1941 
1942 static void vircam_mjdsort(double **fdata, double *mjd, int n) {
1943  int iii,ii,i,ifin,j;
1944  double tmpmjd,*tmpdata;
1945 
1946 
1947  iii = 2;
1948  while (iii < n)
1949  iii *= 2;
1950  iii = min(n,(3*iii)/4 - 1);
1951 
1952  while (iii > 1) {
1953  iii /= 2;
1954  ifin = n - iii;
1955  for (ii = 0; ii < ifin; ii++) {
1956  i = ii;
1957  j = i + iii;
1958  if (mjd[i] > mjd[j]) {
1959  tmpmjd = mjd[j];
1960  tmpdata = fdata[j];
1961  while (1) {
1962  mjd[j] = mjd[i];
1963  fdata[j] = fdata[i];
1964  j = i;
1965  i = i - iii;
1966  if (i < 0 || mjd[0] <= tmpmjd)
1967  break;
1968  }
1969  mjd[j] = tmpmjd;
1970  fdata[j] = tmpdata;
1971  }
1972  }
1973  }
1974 }
1975 
1976 /*---------------------------------------------------------------------------*/
1983 /*---------------------------------------------------------------------------*/
1984 
1985 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows) {
1986  int i;
1987  char colname[16];
1988  cpl_table *t;
1989 
1990  /* Create a new table */
1991 
1992  t = cpl_table_new(nrows);
1993 
1994  /* Add the first few columns */
1995 
1996  cpl_table_new_column(t,"filename",CPL_TYPE_STRING);
1997  cpl_table_new_column(t,"exptime",CPL_TYPE_DOUBLE);
1998  cpl_table_set_column_unit(t,"exptime","seconds");
1999  cpl_table_new_column(t,"mjd",CPL_TYPE_DOUBLE);
2000  cpl_table_set_column_unit(t,"mjd","days");
2001 
2002  /* Add columns for each of the channels' raw median flux and linearised
2003  median flux */
2004 
2005  for (i = 1; i <= np; i++) {
2006  (void)snprintf(colname,16,"rawflux_%02d",i);
2007  cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
2008  cpl_table_set_column_unit(t,colname,"ADU");
2009  (void)snprintf(colname,16,"linflux_%02d",i);
2010  cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
2011  cpl_table_set_column_unit(t,colname,"ADU");
2012  }
2013 
2014  /* Finally add the correction factors that were used */
2015 
2016  cpl_table_new_column(t,"adjust_fac",CPL_TYPE_DOUBLE);
2017 
2018  /* Right, get out of here */
2019 
2020  return(t);
2021 }
2022 
2023 /*---------------------------------------------------------------------------*/
2027 /*---------------------------------------------------------------------------*/
2028 
2029 static void vircam_linearity_analyse_init(void) {
2030  ps.labels = NULL;
2031  ps.domelist = NULL;
2032  ps.darklist = NULL;
2033  ps.domecheck = NULL;
2034  ps.darkcheck = NULL;
2035  ps.ndomes = 0;
2036  ps.ndarks = 0;
2037  ps.ndomecheck = 0;
2038  ps.ndarkcheck = 0;
2039  ps.chanfrm = NULL;
2040  ps.chantab = NULL;
2041  ps.lchantab = NULL;
2042  ps.flatlist = NULL;
2043  ps.bpm_array = NULL;
2044  ps.ddg = NULL;
2045  ps.plist = NULL;
2046  ps.elist = NULL;
2047  ps.phupaf = NULL;
2048  ps.diag1 = NULL;
2049  ps.diag2 = NULL;
2050  ps.inherit = NULL;
2051 }
2052 
2053 /*---------------------------------------------------------------------------*/
2057 /*---------------------------------------------------------------------------*/
2058 
2059 static void vircam_linearity_analyse_tidy(int level) {
2060  int i;
2061 
2062  freetfits(ps.chantab);
2063  freearray(ps.bpm_array);
2064  freefitslist(ps.flatlist,ps.nflatlist);
2065  freetable(ps.lchantab);
2066  freepropertylist(ps.plist);
2067  freepropertylist(ps.elist);
2068  freetable(ps.diag1);
2069  freetable(ps.diag2);
2070  if (level == 1)
2071  return;
2072 
2073  freespace(ps.labels);
2074  freeframeset(ps.domelist);
2075  freeframeset(ps.darklist);
2076  freeframeset(ps.domecheck);
2077  freeframeset(ps.darkcheck);
2078  freeframe(ps.chanfrm);
2079  if (ps.ddg != NULL) {
2080  for (i = 0; i < ps.nddg; i++) {
2081  freeframeset(ps.ddg[i].darks);
2082  freeframeset(ps.ddg[i].domes);
2083  freefitslist(ps.ddg[i].proc,ps.ddg[i].ndomes);
2084  }
2085  freespace(ps.ddg);
2086  }
2087  freepropertylist(ps.phupaf);
2088 }
2089 
2092 /*
2093 
2094 $Log: not supported by cvs2svn $
2095 Revision 1.64 2012/01/15 17:40:09 jim
2096 Minor modifications to take into accout the changes in cpl API for v6
2097 
2098 Revision 1.63 2010/12/09 13:20:26 jim
2099 Default polynomial order is now 4
2100 
2101 Revision 1.62 2010/07/02 07:17:35 jim
2102 Fixed typo
2103 
2104 Revision 1.61 2010/06/30 12:42:00 jim
2105 A few fixes to stop compiler compaints
2106 
2107 Revision 1.60 2010/03/09 14:29:55 jim
2108 Now modified ESO PRO DATANCOM to reflect the number of images used in
2109 the analysis
2110 
2111 Revision 1.59 2009/09/09 09:50:21 jim
2112 Modified to try and get headers right
2113 
2114 Revision 1.58 2009/07/13 08:17:57 jim
2115 Fixed bug that meant saturation level wasn't being propogated
2116 
2117 Revision 1.57 2009/06/23 05:22:26 jim
2118 Adds saturation back into the channel table header
2119 
2120 Revision 1.56 2008/12/08 06:32:42 jim
2121 Changed 'missing dark frame' error to warning
2122 
2123 Revision 1.55 2008/12/05 13:28:32 jim
2124 Fixed save routine so that the correct version of PRO CATG is written to the
2125 paf file
2126 
2127 Revision 1.54 2008/10/01 04:59:13 jim
2128 Added call to vircam_frameset_fexists to check input frameset
2129 
2130 Revision 1.53 2008/09/30 11:33:00 jim
2131 Fixed bug where saturation flag wasn't being set to OK.
2132 
2133 Revision 1.52 2008/09/29 11:23:00 jim
2134 Minor fix to docs
2135 
2136 Revision 1.51 2008/01/22 19:47:56 jim
2137 New version to implement new algorithm
2138 
2139 Revision 1.50 2007/11/26 09:58:49 jim
2140 Now fails if given observation files done with NDIT != 1
2141 
2142 Revision 1.49 2007/11/23 18:34:28 jim
2143 fixed memory allocation bug
2144 
2145 Revision 1.48 2007/11/22 12:36:55 jim
2146 Modified to create diagnostic tables
2147 
2148 Revision 1.47 2007/11/20 09:41:13 jim
2149 Added ability to alter dome sequence stats by using the monitoring exposures
2150 
2151 Revision 1.46 2007/11/14 10:42:25 jim
2152 Substantial changes to incorporate new linearity analysis algorithm and to
2153 restrict the amount of memory required to do the analysis (especially
2154 the BPM work)
2155 
2156 Revision 1.45 2007/09/07 13:32:12 jim
2157 uses a sorted framelist to ensure that the correct information is given
2158 to the output product header
2159 
2160 Revision 1.44 2007/09/06 21:37:53 jim
2161 fixed call to vircam_dfs_setup_product_ routines to use the full input
2162 frameset
2163 
2164 Revision 1.43 2007/08/29 09:20:33 jim
2165 Primary header is now derived from the same header that forms the PAF rather
2166 than starting off empty and allowing CPL to copy everything it thinks you
2167 want...
2168 
2169 Revision 1.42 2007/08/23 09:02:03 jim
2170 Modified to check domes for DETLIVE before checking darks
2171 
2172 Revision 1.41 2007/07/09 13:21:55 jim
2173 Modified to use new version of vircam_exten_range
2174 
2175 Revision 1.40 2007/06/13 08:11:27 jim
2176 Modified docs to reflect changes in DFS tags
2177 
2178 Revision 1.39 2007/04/30 09:40:17 jim
2179 Added more stuff to paf files
2180 
2181 Revision 1.38 2007/04/04 10:36:07 jim
2182 Fixed typo preventing output of main PAF. Also modified to use dfs tags
2183 
2184 Revision 1.37 2007/03/29 12:19:38 jim
2185 Little changes to improve documentation
2186 
2187 Revision 1.36 2007/03/01 12:41:49 jim
2188 Modified slightly after code checking
2189 
2190 Revision 1.35 2007/02/19 21:13:04 jim
2191 added bad pixel number QC parameter
2192 
2193 Revision 1.34 2007/02/15 11:54:09 jim
2194 Modified to make a distinction between initial channel table and one that
2195 has the proper linearity information
2196 
2197 Revision 1.33 2007/02/15 06:59:38 jim
2198 Added ability to write QC paf files
2199 
2200 Revision 1.32 2007/02/07 10:12:40 jim
2201 Removed calls to vircam_ndit_correct as this is now no longer necessary
2202 
2203 Revision 1.31 2007/02/06 13:11:12 jim
2204 Fixed entry for PRO dictionary in cpl_dfs_set_product_header
2205 
2206 Revision 1.30 2006/12/13 11:45:36 jim
2207 Fixed scaling of sigma error
2208 
2209 Revision 1.29 2006/12/11 22:47:12 jim
2210 Fixed subtle bug in the way that stats were being done.
2211 
2212 Revision 1.28 2006/11/27 12:15:08 jim
2213 changed calls to cpl_propertylist_append to cpl_propertylist_update
2214 
2215 Revision 1.27 2006/11/10 09:23:46 jim
2216 Fixed save routine so to use a new version of vircam_chantab_new
2217 
2218 Revision 1.26 2006/10/31 10:27:27 jim
2219 Fixed a few bugs and modified to make sure than an extension name appear
2220 in each fits extension
2221 
2222 Revision 1.25 2006/09/09 16:49:40 jim
2223 Header comment update
2224 
2225 Revision 1.24 2006/09/08 09:20:22 jim
2226 major upgrade to main processing routine: to deal with bad input better; to
2227 write out dummy results in the case of failure; to combine raw darks on the
2228 fly for use in dark correction, rather than using master darks;
2229 
2230 Revision 1.23 2006/08/03 13:26:44 jim
2231 fixed another typo
2232 
2233 Revision 1.22 2006/08/03 10:36:32 jim
2234 Fixed typo
2235 
2236 Revision 1.21 2006/06/20 19:06:38 jim
2237 Added correction for ndit. Now adjusts the value of norder if the number
2238 of frames given is too small for the order of polynomial requested
2239 
2240 Revision 1.20 2006/06/15 09:58:58 jim
2241 Minor changes to docs
2242 
2243 Revision 1.19 2006/06/06 13:03:42 jim
2244 Fixed scaling that was causing funny stats
2245 
2246 Revision 1.18 2006/05/27 21:40:06 jim
2247 Bad pixels are now defined by a number of sigma above or below the mean
2248 
2249 Revision 1.17 2006/05/09 09:30:47 jim
2250 Fixed _save routine so that bad pixel mask is saved with unsigned char
2251 data type
2252 
2253 Revision 1.16 2006/05/08 12:32:12 jim
2254 Changed default calling parameters for vircam_imcombine
2255 
2256 Revision 1.15 2006/05/04 11:53:15 jim
2257 Fixed the way the _save routine works to be more consistent with the
2258 standard CPL way of doing things
2259 
2260 Revision 1.14 2006/05/03 12:55:17 jim
2261 Fixed some memory leaks
2262 
2263 Revision 1.13 2006/05/02 13:26:32 jim
2264 fixed bug where the wrong amount of memory was being allocated for the dark
2265 exposure times
2266 
2267 Revision 1.12 2006/05/02 11:36:29 jim
2268 fixed illegal propertylist_delete calls
2269 
2270 Revision 1.11 2006/04/27 09:46:01 jim
2271 Modified DFS frame types to conform to new dictionary
2272 
2273 Revision 1.10 2006/04/25 13:45:57 jim
2274 Fixed to adhere to new calling sequence for vircam_dfs routines
2275 
2276 Revision 1.9 2006/04/24 12:12:59 jim
2277 Fixed --help documentation and sorted out filename extension problem
2278 (.fit -> .fits)
2279 
2280 Revision 1.8 2006/04/20 11:31:34 jim
2281 Added bad pixel masking
2282 
2283 Revision 1.7 2006/03/23 21:18:45 jim
2284 Minor changes mainly to comment headers
2285 
2286 Revision 1.6 2006/03/22 14:02:51 jim
2287 cosmetic changes to keep lint happy
2288 
2289 Revision 1.5 2006/03/22 12:13:51 jim
2290 Modified to use new vircam_mask capability
2291 
2292 Revision 1.4 2006/03/15 10:43:40 jim
2293 Fixed a few things
2294 
2295 Revision 1.3 2006/03/03 14:29:06 jim
2296 Now calls routines with vir_fits.
2297 
2298 Revision 1.2 2006/02/22 10:01:22 jim
2299 Added full documentation
2300 
2301 Revision 1.1 2006/02/18 11:49:58 jim
2302 new file
2303 
2304 
2305 */