DETMON Pipeline Reference Manual  1.3.0
detmon/detmon_pernoise.c
1 /* $Id: detmon.c,v 1.11 2013-07-19 12:00:24 jtaylor Exp $
2  *
3  * This file is part of the irplib package
4  * Copyright (C) 2002, 2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jtaylor $
23  * $Date: 2013-07-19 12:00:24 $
24  * $Revision: 1.11 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 #include <complex.h>
33 
34 /*---------------------------------------------------------------------------
35  Includes
36  ---------------------------------------------------------------------------*/
37 
38 #include <math.h>
39 #include <string.h>
40 #include <assert.h>
41 #include <float.h>
42 
43 #include <cpl.h>
44 #include "detmon.h"
45 #include "detmon_pernoise.h"
46 
47 #include "irplib_ksigma_clip.h"
48 #include "irplib_hist.h"
49 #include "irplib_utils.h"
50 
51 
52 /* Computes the square of an euclidean distance bet. 2 points */
53 #define pdist(x1,y1,x2,y2) (((x1-x2)*(x1-x2))+((y1-y2)*(y1-y2)))
54 
55 #define cpl_drand() ((double)rand()/(double)RAND_MAX)
56 
57 /*--------------------------------------------------------------------------*/
58 
59 /*
60  * @defgroup detmon Detector monitoring functions
61  */
62 
63 /*--------------------------------------------------------------------------*/
64 
65 /*---------------------------------------------------------------------------
66  Defines
67  ---------------------------------------------------------------------------*/
68 
69 
70 #define HIST_FACT 2.354820045
71 
72 enum pixeltypes
73 {
74  HOT = 0,
75  DEAD = 1,
76  NOISY = 2
77 };
78 
79 enum stackingtypes
80 {
81  MINMAX = 0,
82  MEAN = 1,
83  MEDIAN = 2,
84  KSIGMA = 3
85 };
86 
87 enum readouts
88 {
89  HORIZONTAL = 1,
90  VERTICAL = 2
91 };
92 
93 
94 
95 static struct
96 {
97  int mode;
98  cpl_boolean direction;
99  double speed;
100  int llx;
101  int lly;
102  int urx;
103  int ury;
104  int kappa;
105  int exts;
106  int nb_extensions;
107 } detmon_pernoise_config;
108 
109 
110 #define NIR TRUE
111 #define OPT FALSE
112 
113 /*---------------------------------------------------------------------------
114  Private function prototypes
115  ---------------------------------------------------------------------------*/
116 
117 /* Functions for Periodic Noise Characterisation, detmon_pernoise() */
118 
119 int
120 detmon_pernoise_dfs_set_groups(cpl_frameset *,
121  const char *);
122 
123 static cpl_error_code
124 detmon_pernoise_retrieve_parlist(const char *,
125  const char *,
126  const cpl_parameterlist *);
127 
128 static cpl_error_code
129 detmon_pernoise_qc(cpl_propertylist *,
130  cpl_table *,
131  int);
132 
133 static cpl_error_code
134 detmon_pernoise_save(const cpl_parameterlist *,
135  cpl_frameset *,
136  const char *,
137  const char *,
138  const char *,
139  const char *,
140  cpl_table **,
141  cpl_propertylist **,
142  const int,
143  const int,
144  const cpl_frameset *);
145 
146 cpl_error_code
147 detmon_pernoise_rm_bg(cpl_image *,
148  int,
149  int);
150 
151 
152 static cpl_error_code
153 detmon_dark_save(const cpl_parameterlist *,
154  cpl_frameset *,
155  const char *,
156  const char *,
157  const char *,
158  const char *,
159  const char *,
160  const char *,
161  cpl_imagelist **,
162  cpl_table **,
163  cpl_imagelist **,
164  cpl_propertylist **,
165  const int,
166  const int,
167  const cpl_frameset *);
168 
169 
170 
171 static cpl_error_code
172 detmon_retrieve_dark_params(const char *,
173  const char *,
174  const cpl_parameterlist *);
175 
176 
177 
178 /*--------------------------------------------------------------------------*/
179 
180 /*
181  * @brief Reduce periodic noise
182  * @param image Input image
183  *
184  * @return 1-d image, representing the distribution of noise. NULL if failed.
185  */
186 
187 /*--------------------------------------------------------------------------*/
188 
189 
190 #define HORIZONTAL TRUE
191 
192 cpl_table *
193 detmon_pernoise_reduce(cpl_image * image)
194 {
195  int nsamples, nffts;
196  int i, j;
197 
198  int status;
199  float * hanning = 0;
200  float * power = 0;
201  cpl_image * power_im = 0;
202  cpl_image * output = 0;
203  cpl_image * pos_spec = 0;
204  cpl_table * table = 0;
205  cpl_image* fourier_im = 0;
206  double freq;
207  cpl_error_code error = CPL_ERROR_NONE;
208  cpl_image * sub_image = 0;
209  int nffts_old;
210 
211 
212  if(detmon_pernoise_config.direction == HORIZONTAL) {
213  error = cpl_image_flip(image, 1);
214  cpl_ensure(!error, error, NULL);
215  }
216 
217  nsamples = cpl_image_get_size_x(image);
218  nffts = cpl_image_get_size_y(image);
219 
220 
221  /* Rewrite the previous lines with a better style (: ...) */
222 
223  /*
224  * 1. preprocessing task:
225  * Estimate the background fitting the image to a
226  * 2-D polynomial and substract it from the image.
227  */
228 
229  error = detmon_pernoise_rm_bg(image, nsamples, nffts);
230  cpl_ensure(!error, error, NULL);
231 
232  sub_image = cpl_image_extract(image, nsamples/8 + 1, nffts/8+1,
233  nsamples*7/8, nffts*7/8);
234  nffts_old = nffts;
235  nsamples = cpl_image_get_size_x(sub_image);
236  nffts = cpl_image_get_size_y(sub_image);
237 
238  /*
239  * 2. preprocessing task:
240  * Remove the effect of hot and dark pixels, replacing their values by
241  * the average value of the neighbouring pixels.
242  */
243 
244  /*
245  * 3. preprocessing task:
246  * Apply a Hanning vector
247  */
248 
249  hanning = cpl_malloc(sizeof(float) * nsamples);
250 
251  for(i = 0; i < nsamples; i++) {
252  *(hanning + i) = 0.5 - 0.5 * cos(2 * CPL_MATH_PI * (float) i / nsamples);
253  for(j = 0; j < nffts; j++) {
254  double value =
255  cpl_image_get(sub_image, i + 1, j + 1, &status);
256  error = cpl_image_set(sub_image, i + 1, j + 1, (*(hanning + i)) * value);
257  }
258  }
259 
260  cpl_free(hanning);
261  if (error != CPL_ERROR_NONE)
262  {
263  goto cleanup;
264  }
265 
266  power = (float *) cpl_calloc(sizeof(float), nsamples * nffts);
267 
268 
269  fourier_im = cpl_image_new(nsamples,nffts, CPL_TYPE_FLOAT_COMPLEX);
270  error = cpl_fft_image(fourier_im, sub_image, CPL_FFT_FORWARD);
271 
272  for(i = 1; i <= nffts; i++) {
273  for(j = 1; j <= nsamples; j++) {
274  int rej = 0;
275  double complex cvalue = cpl_image_get_complex(fourier_im,j, i, &rej );
276  double value = cabs(cvalue);
277  /*
278  *(power + j + i * nsamples) = *((float complex *)fourier + j + i * nsamples) *
279  conjf(*((float complex *)fourier +j + i * nsamples));
280  */
281  cpl_image_set(power_im, j, i, value);
282  }
283  /* Is it necessary to divide here by 2 * pi? */
284  }
285  cpl_image_delete(fourier_im);
286 
287  output = cpl_image_collapse_create(power_im, 0);
288  pos_spec = cpl_image_extract(output, 1, 1, nsamples/2, 1);
289 
290  cpl_image_delete(power_im);
291  cpl_free(power);
292 
293  cpl_image_delete(output);
294 
295  table = cpl_table_new(nsamples/2);
296  cpl_table_new_column(table, "FREQ", CPL_TYPE_DOUBLE);
297  cpl_table_new_column(table, "POW", CPL_TYPE_DOUBLE);
298 
299  freq = detmon_pernoise_config.speed*1000/nffts_old;
300 
301  for(i = 0; i < nsamples/2; i++) {
302  error = cpl_table_set(table, "FREQ", i, freq/(nsamples/2)*i);
303  error = cpl_table_set(table, "POW", i, cpl_image_get(pos_spec, i+1, 1, &status));
304  }
305 
306  for(i= 0; i < 5; i++) {
307  error = cpl_table_set(table, "POW", i, 0.0);
308  }
309 
310 
311 cleanup:
312  cpl_image_delete(pos_spec);
313 
314  cpl_image_delete(sub_image);
315  if (error != CPL_ERROR_NONE)
316  {
317  cpl_table_delete(table);
318  table = 0;
319  }
320  return table;
321 }
322 #undef HORIZONTAL
323 
324 
325 
326 /*---------------------------------------------------------------------------*/
327 
328 /*
329  * @brief Measure persistence noise
330  * @param frameset Input set of frames
331  * @param parlist Input parameterlist
332  * @param tag Input tag
333  * @param recipe_name input recipe name
334  * @param pipeline_name input pipeline id
335  * @param ptrocatg_tbl input table with pro catgs
336  * @param package package id
337  * @param comapre comparison function
338  * @return CPL_ERROR_NONE on success.
339  */
340 
341 /*---------------------------------------------------------------------------*/
342 cpl_error_code
343 detmon_pernoise(cpl_frameset * frameset,
344  const cpl_parameterlist * parlist,
345  const char * tag,
346  const char * recipe_name,
347  const char * pipeline_name,
348  const char * procatg_tbl,
349  const char * package,
350  int (*compare)(const cpl_frame *,
351  const cpl_frame *))
352 {
353  cpl_size nsets;
354 
355  int i;
356  cpl_error_code error;
357 
358  if(detmon_pernoise_dfs_set_groups(frameset, tag)) {
359  cpl_msg_error(cpl_func, "Cannot identify RAW and CALIB frames");
360  }
361 
362  /*
363  * This function reads all inputs parameters from parlist
364  * and stores them in a global variable detmon_ronbias_config.
365  * Similar to detmon_lg_retrieve_parlist(). See detmon.c
366  */
367  error = detmon_pernoise_retrieve_parlist(pipeline_name,
368  recipe_name, parlist);
369  cpl_ensure_code(!error, error);
370 
371  /* Labelise all input frames */
372  if(compare == NULL)
373  nsets = 1;
374  else {
375  cpl_msg_info(cpl_func, "Identify the different settings");
376  cpl_size* selection = cpl_frameset_labelise(frameset, compare, &nsets);
377  if(selection == NULL)
378  cpl_msg_error(cpl_func, "Cannot labelise input frames");
379  }
380 
381  detmon_pernoise_config.nb_extensions = 1;
382  if(detmon_pernoise_config.exts < 0) {
383  const cpl_frame *cur_frame =
384  cpl_frameset_get_first_const(frameset);
385  /* Get the nb of extensions */
386  detmon_pernoise_config.nb_extensions =
387  cpl_frame_get_nextensions(cur_frame);
388  }
389 
390  /* Extract settings and reduce each of them */
391  for(i = 0; i < nsets; i++)
392  {
393  int j;
394  cpl_table ** freq_table;
395  cpl_propertylist ** qclist =
396  (cpl_propertylist **)
397  cpl_malloc(detmon_pernoise_config.nb_extensions *
398  sizeof(cpl_propertylist *));
399 
400  cpl_imagelist ** raws = (cpl_imagelist **) cpl_malloc(detmon_pernoise_config.nb_extensions * sizeof(cpl_imagelist *));
401  cpl_image ** input = (cpl_image **) cpl_malloc(detmon_pernoise_config.nb_extensions * sizeof(cpl_image *));
402 
403  /* Initialise memory for products */
404  if(detmon_pernoise_config.mode == 1)
405  {
406  freq_table =
407  (cpl_table **) cpl_malloc(detmon_pernoise_config.nb_extensions *
408  4 * sizeof(cpl_table *));
409  } else
410  {
411  freq_table =
412  (cpl_table **) cpl_malloc(detmon_pernoise_config.nb_extensions *
413  sizeof(cpl_table *));
414  }
415 
416  if(detmon_pernoise_config.exts >= 0)
417  {
418  *raws =
419  cpl_imagelist_load_frameset(frameset, CPL_TYPE_FLOAT, 1,
420  detmon_pernoise_config.exts);
421  *input = cpl_image_subtract_create(cpl_imagelist_get(*raws,0),
422  cpl_imagelist_get(*raws,1));
423  } else
424  {
425  cpl_imagelist *raws_all_exts =
426  cpl_imagelist_load_frameset(frameset, CPL_TYPE_FLOAT, 1,
427  -1);
428  for(j = 0; j < detmon_pernoise_config.nb_extensions; j++)
429  {
430  int nframes = cpl_frameset_get_size(frameset);
431  int k;
432  for(k = 0; k < nframes; k++)
433  {
434  cpl_image *image =
435  cpl_imagelist_unset(raws_all_exts,
436  (detmon_pernoise_config.
437  nb_extensions - 1 - j) * k);
438  cpl_imagelist_set(raws[j], image, k);
439  }
440  input[j] =
441  cpl_image_subtract_create(cpl_imagelist_get(raws[j],0),
442  cpl_imagelist_get(raws[j],1));
443  }
444  }
445 
446  for(j = 0; j < detmon_pernoise_config.nb_extensions; j++) {
447  cpl_msg_info(cpl_func, "Starting reduction");
448  qclist[j] = cpl_propertylist_new();
449  if(detmon_pernoise_config.mode == 1)
450  {
451  int nx = cpl_image_get_size_x(input[j]);
452  int ny = cpl_image_get_size_y(input[j]);
453  int k = 0;
454  cpl_image* quad[4];
455 
456  quad[0] = cpl_image_extract(input[j], 1, 1, nx/2, ny/2);
457  quad[1] = cpl_image_extract(input[j], 1, ny/2+1, nx/2, ny);
458  quad[2] = cpl_image_extract(input[j], nx/2+1, 1, nx, ny/2);
459  quad[3] = cpl_image_extract(input[j], nx/2+1, ny/2+1, nx, ny);
460 
461  for (k = 0; k < 4; k++)
462  {
463  freq_table[j * 4 + k] = detmon_pernoise_reduce(quad[k]);
464  }
465  for(k = 0; k < 4; k++)
466  {
467  error = detmon_pernoise_qc(qclist[j], freq_table[j + k], k+1);
468  if (error != CPL_ERROR_NONE)
469  break;
470  }
471  for (k = 0; k < 4; k++)
472  {
473  cpl_image_delete(quad[k]);
474  }
475  } else
476  {
477  freq_table[j] = detmon_pernoise_reduce(input[j]);
478  if(freq_table[j] != NULL)
479  {
480  error = detmon_pernoise_qc(qclist[j], freq_table[j], 0);
481  }
482  }
483  if (error != CPL_ERROR_NONE)
484  {
485  break;
486  }
487  }
488  if (error == CPL_ERROR_NONE)
489  {
490  error = detmon_pernoise_save(parlist, frameset, recipe_name,
491  pipeline_name, procatg_tbl,
492  package, freq_table, qclist, 0,
493  0, frameset);
494  }
495 
496  for(j = 0; j < detmon_pernoise_config.nb_extensions; j++)
497  {
498  cpl_propertylist_delete(qclist[j]);
499  cpl_imagelist_delete(raws[j]);
500  cpl_image_delete(input[j]);
501  }
502  cpl_free(qclist);
503  cpl_free(raws);
504  cpl_free(input);
505  if(detmon_pernoise_config.mode == 1)
506  {
507  for(j= 0; j < detmon_pernoise_config.nb_extensions * 4; j++) {
508  cpl_table_delete(freq_table[j]);
509  }
510  } else {
511  for(j= 0; j < detmon_pernoise_config.nb_extensions; j++) {
512  cpl_table_delete(freq_table[j]);
513  }
514  }
515  cpl_free(freq_table);
516  if (error != CPL_ERROR_NONE)
517  {
518  break;
519  }
520  }
521 
522  return cpl_error_get_code();
523 }
524 
525 /*---------------------------------------------------------------------------*/
526 
527 /*
528  * @brief classifies input frames matching input tag
529  * @param set input frameset
530  * @param tag input tag
531  * @return 0 on success, else -1
532  */
533 
534 /*---------------------------------------------------------------------------*/
535 int
536 detmon_pernoise_dfs_set_groups(cpl_frameset * set, const char *tag)
537 {
538 
539 
540  int nframes;
541  int i;
542 
543  /* Check entries */
544  if(set == NULL)
545  return -1;
546 
547  /* Initialize */
548  nframes = cpl_frameset_get_size(set);
549 
550  /* Loop on frames */
551  for(i = 0; i < nframes; i++) {
552  cpl_frame* cur_frame = cpl_frameset_get_frame(set, i);
553  const char* cur_tag = cpl_frame_get_tag(cur_frame);
554 
555  /* RAW frames */
556  if(!strcmp(cur_tag, tag))
557  cpl_frame_set_group(cur_frame, CPL_FRAME_GROUP_RAW);
558  /* CALIB frames */
559 
560  /* else if (!strcmp(tag, IIINSTRUMENT_CALIB_FLAT))
561  cpl_frame_set_group(cur_frame, CPL_FRAME_GROUP_CALIB) ;
562  */
563  }
564  return 0;
565 }
566 
567 /*---------------------------------------------------------------------------*/
568 
569 /*
570  * @brief Initialize pernoise recipe parameters structure
571  * @param parlist input parameter list
572  * @param recipe_name Input recipe name
573  * @param pipeline_name Input pipeline name
574  * @param mode mode to indicate if pix2pix,opt/nir,etc
575  * @param direction read-out direction
576  * @param speed read-out speed
577  * @param llx lower-left X coordinate
578  * @param lly lower-left Y coordinate
579  * @param urx upper-right X coordinate
580  * @param ury upper-right Y coordinate
581  * @param kappa kappa value in kappa-sigma clip
582  * @param exts extension id
583 
584  * @return CPL_ERROR_NONE on success.
585  */
586 
587 /*---------------------------------------------------------------------------*/
588  cpl_error_code
589 detmon_fill_pernoise_params(cpl_parameterlist * parlist,
590  const char *recipe_name,
591  const char *pipeline_name,
592  int mode,
593  const char * direction,
594  double speed,
595  int llx,
596  int lly,
597  int urx,
598  int ury,
599  double kappa,
600  int exts)
601 {
602  detmon_fill_parlist(parlist, recipe_name, pipeline_name, 9,
603 
604  "mode",
605  "Mode",
606  "CPL_TYPE_INT", mode,
607 
608  "direction",
609  "Readout direction",
610  "CPL_TYPE_BOOL", direction,
611 
612  "speed",
613  "Readout speed",
614  "CPL_TYPE_DOUBLE", speed,
615 
616  "llx",
617  "(yet unsupported) x coordinate of the lower-left "
618  "point of the region of interest. If not modified, default value will be 1.",
619  "CPL_TYPE_INT", llx,
620  "lly",
621  "(yet unsupported) y coordinate of the lower-left "
622  "point of the region of interest. If not modified, default value will be 1.",
623  "CPL_TYPE_INT", lly,
624  "urx",
625  "(yet unsupported) x coordinate of the upper-right "
626  "point of the region of interest. If not modified, default value will be X dimension of the input image.",
627  "CPL_TYPE_INT", urx,
628  "ury",
629  "(yet unsupported) y coordinate of the upper-right "
630  "point of the region of interest. If not modified, default value will be Y dimension of the input image.",
631  "CPL_TYPE_INT", ury,
632 
633  "kappa",
634  "Kappa used for determining threshold of bad (hot, cold) pixels",
635  "CPL_TYPE_DOUBLE", kappa,
636 
637  "exts",
638  "Activate the multi-exts option",
639  "CPL_TYPE_INT", exts);
640 
641  return 0;
642 }
643 
644 /*---------------------------------------------------------------------------*/
645 /*
646  * @brief initialise pernoise parameter structure
647  * @param parlist Shift to apply on the x-axis
648  * @param recipe_name Input recipe id
649  * @param pipeline_name Input pipeline id
650  * @return 0 .
651  */
652 
653 /*---------------------------------------------------------------------------*/
654 int
655 detmon_fill_pernoise_params_default(cpl_parameterlist * parlist,
656  const char *recipe_name,
657  const char *pipeline_name)
658 {
659  detmon_fill_pernoise_params(parlist, recipe_name, pipeline_name,
660  1, /* --mode */
661  "CPL_TRUE", /* --direction */
662  84.5, /* --speed */
663  -1, /* --llx */
664  -1, /* --lly */
665  -1, /* --urx */
666  -1, /* --ury */
667  100, /* --kappa */
668  0); /* --exts */
669 
670  return 0;
671 
672 }
673 
674 /*---------------------------------------------------------------------------*/
675 /*
676  * @brief Retrieve pernoise parameter structure values
677  * @param pipeline_name Input pipeline id
678  * @param recipe_name Input recipe id
679  * @param parlist Shift to apply on the x-axis
680  * @return 0 .
681  */
682 /*---------------------------------------------------------------------------*/
683 
684 static cpl_error_code
685 detmon_pernoise_retrieve_parlist(const char *pipeline_name,
686  const char *recipe_name,
687  const cpl_parameterlist * parlist)
688 {
689  char *par_name;
690  cpl_parameter *par;
691 
692  /* --mode */
693  detmon_pernoise_config.mode =
694  detmon_retrieve_par_int("mode", pipeline_name, recipe_name,
695  parlist);
696 
697  /* --direction */
698  par_name = cpl_sprintf("%s.%s.direction", pipeline_name, recipe_name);
699  assert(par_name != NULL);
700  par = cpl_parameterlist_find((cpl_parameterlist *) parlist, par_name);
701  detmon_pernoise_config.direction = cpl_parameter_get_bool(par);
702  cpl_free(par_name);
703 
704  /* --speed */
705  par_name = cpl_sprintf("%s.%s.speed", pipeline_name, recipe_name);
706  assert(par_name != NULL);
707  par = cpl_parameterlist_find((cpl_parameterlist *) parlist, par_name);
708  detmon_pernoise_config.speed = cpl_parameter_get_double(par);
709  cpl_free(par_name);
710 
711  /* --llx */
712  detmon_pernoise_config.llx =
713  detmon_retrieve_par_int("llx", pipeline_name, recipe_name,
714  parlist);
715 
716  /* --lly */
717  detmon_pernoise_config.lly =
718  detmon_retrieve_par_int("lly", pipeline_name, recipe_name,
719  parlist);
720  /* --urx */
721  detmon_pernoise_config.urx =
722  detmon_retrieve_par_int("urx", pipeline_name, recipe_name,
723  parlist);
724  /* --ury */
725  detmon_pernoise_config.ury =
726  detmon_retrieve_par_int("ury", pipeline_name, recipe_name,
727  parlist);
728  /* --kappa */
729  detmon_pernoise_config.kappa =
730  detmon_retrieve_par_double("kappa", pipeline_name, recipe_name,
731  parlist);
732 
733  /* --exts */
734  detmon_pernoise_config.exts =
735  detmon_retrieve_par_int("exts", pipeline_name, recipe_name,
736  parlist);
737 
738  if(cpl_error_get_code()) {
739  cpl_msg_error(cpl_func, "Failed to retrieve the input parameters");
740  cpl_ensure_code(0, CPL_ERROR_DATA_NOT_FOUND);
741  }
742 
743  return cpl_error_get_code();
744 }
745 
746 /*---------------------------------------------------------------------------*/
747 /*
748  * @brief Save results of pernoise recipe
749  * @param parlist input parameter list
750  * @param frameset input frameset
751  * @param recipe_name input recipe name
752  * @param pipeline_name input pipeline name
753  * @param procatg_tbl input procategory table
754  * @param package input package id
755  * @param freq_table computed frequency table
756  * @param qclist computed QC parameters
757  * @param flag_sets switch
758  * @param which_sets specifier
759  * @param usedframes used frames
760 
761  * @return CPL_ERROR_NONE on success.
762  */
763 /*---------------------------------------------------------------------------*/
764 static cpl_error_code
765 detmon_pernoise_save(const cpl_parameterlist * parlist,
766  cpl_frameset * frameset,
767  const char *recipe_name,
768  const char *pipeline_name,
769  const char *procatg_tbl,
770  const char *package,
771  cpl_table ** freq_table,
772  cpl_propertylist ** qclist,
773  const int flag_sets,
774  const int which_set,
775  const cpl_frameset * usedframes)
776 {
777 
778  cpl_frame *ref_frame;
779  cpl_propertylist *plist;
780  char *name_o = NULL; /* Avoid (false) uninit warning */
781  int i;
782  cpl_propertylist *paflist;
783  cpl_error_code error;
784 
785  cpl_propertylist * pro_tbl = cpl_propertylist_new();
786 
787  cpl_propertylist_append_string(pro_tbl,
788  CPL_DFS_PRO_CATG, procatg_tbl);
789 
790  cpl_propertylist_append(pro_tbl, qclist[0]);
791 
792  /*******************************/
793  /* Write the FREQ TABLE */
794 
795  /*******************************/
796 
797  if(detmon_pernoise_config.mode != 1) {
798  /* Set the file name for the table */
799  if(!flag_sets) {
800  name_o = cpl_sprintf("%s_freq_table.fits", recipe_name);
801  assert(name_o != NULL);
802  } else {
803  name_o =
804  cpl_sprintf("%s_freq_table_set%02d.fits", recipe_name,
805  which_set);
806  assert(name_o != NULL);
807  }
808 
809  /* Save the table */
810  if(cpl_dfs_save_table(frameset, NULL, parlist, usedframes, NULL, freq_table[0],
811  NULL, recipe_name, pro_tbl, NULL,
812  package, name_o)) {
813  cpl_msg_error(cpl_func, "Cannot save the product: %s", name_o);
814  cpl_free(name_o);
815  cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
816  }
817 
818  if(detmon_pernoise_config.exts < 0) {
819 
820  for(i = 1; i < detmon_pernoise_config.nb_extensions; i++) {
821  error =
822  cpl_table_save(freq_table[i], NULL, qclist[i], name_o,
823  CPL_IO_EXTEND);
824  cpl_ensure_code(!error, error);
825  }
826  }
827 
828  /* Free */
829  cpl_free(name_o);
830 
831  } else {
832  for (int j = 1; j <= 4; j++) {
833  /* Set the file name for the table */
834  if(!flag_sets) {
835  name_o = cpl_sprintf("%s_freq_table_quad%02d.fits",
836  recipe_name, j);
837  assert(name_o != NULL);
838  } else {
839  name_o =
840  cpl_sprintf("%s_freq_table_quad%02d_set%02d.fits",
841  recipe_name, j, which_set);
842  assert(name_o != NULL);
843  }
844 
845  /* Save the table */
846  if(cpl_dfs_save_table(frameset, NULL, parlist, usedframes, NULL,
847  freq_table[j - 1],
848  NULL, recipe_name, pro_tbl, NULL,
849  package, name_o)) {
850  cpl_msg_error(cpl_func, "Cannot save the product: %s", name_o);
851  cpl_free(name_o);
852  cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
853  }
854 
855  if(detmon_pernoise_config.exts < 0) {
856  for(i = 1; i < detmon_pernoise_config.nb_extensions; i++) {
857  error = cpl_table_save(freq_table[(j-1) + 4 * i],
858  NULL, qclist[i], name_o,
859  CPL_IO_EXTEND);
860  cpl_ensure_code(!error, error);
861  }
862  }
863 
864  /* Free */
865  cpl_free(name_o);
866  }
867 
868  }
869  /*******************************/
870  /* Write the PAF file(s) */
871  /*******************************/
872 
873  /* Get FITS header from reference file */
874  ref_frame = cpl_frameset_get_first(frameset);
875  if((plist = cpl_propertylist_load(cpl_frame_get_filename(ref_frame),
876  0)) == NULL) {
877  cpl_msg_error(cpl_func, "getting header from reference frame");
878  cpl_ensure_code(0, cpl_error_get_code());
879  }
880 
881  /* Get the keywords for the paf file */
882  paflist = cpl_propertylist_new();
883  cpl_propertylist_copy_property_regexp(paflist, plist,
884  "^(ARCFILE|MJD-OBS|ESO TPL ID|"
885  "DATE-OBS|ESO DET DIT|ESO DET NDIT|"
886  "ESO DET NCORRS|"
887  "ESO DET MODE NAME)$", 0);
888 
889  for(i = 0; i < detmon_pernoise_config.nb_extensions; i++) {
890  cpl_propertylist * c_paflist = cpl_propertylist_duplicate(paflist);
891  error = cpl_propertylist_append(c_paflist, qclist[i]);
892  cpl_ensure_code(!error, error);
893 
894  /* Set the file name for the bpm */
895  if(detmon_pernoise_config.exts >= 0) {
896  if(!flag_sets) {
897  name_o = cpl_sprintf("%s.paf", recipe_name);
898  assert(name_o != NULL);
899  } else {
900  name_o = cpl_sprintf("%s_set%02d.paf", recipe_name, which_set);
901  assert(name_o != NULL);
902  }
903  } else {
904  if(!flag_sets) {
905  name_o = cpl_sprintf("%s_ext%02d.paf", recipe_name, i+1);
906  assert(name_o != NULL);
907  } else {
908  name_o = cpl_sprintf("%s_set%02d_ext%02d.paf", recipe_name, which_set, i+1);
909  assert(name_o != NULL);
910  }
911  }
912  /* Save the PAF */
913  if(cpl_dfs_save_paf(pipeline_name, recipe_name, c_paflist, name_o)) {
914  cpl_msg_error(cpl_func, "Cannot save the product: %s", name_o);
915  cpl_free(name_o);
916  cpl_propertylist_delete(paflist);
917  cpl_propertylist_delete(plist);
918  cpl_free(name_o);
919  cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
920  }
921  cpl_propertylist_delete(c_paflist);
922  cpl_free(name_o);
923  }
924 
925  cpl_propertylist_delete(plist);
926  cpl_propertylist_delete(paflist);
927  cpl_propertylist_delete(pro_tbl);
928  return cpl_error_get_code();
929 }
930 /*---------------------------------------------------------------------------*/
931 /*
932  * @brief Computes QC for pernoise recipe
933 
934  * @param qclist computed QC parameters
935  * @param table input table
936  * @param iquad input frame quadrant id
937 
938  * @return CPL_ERROR_NONE on success.
939  */
940 /*---------------------------------------------------------------------------*/
941 
942 static cpl_error_code
943 detmon_pernoise_qc(cpl_propertylist * qclist,
944  cpl_table * table,
945  int iquad)
946 {
947  cpl_error_code error;
948  char * propname;
949 
950  double freqs[3] = {0, 0, 0};
951  double pows[3] = {0, 0, 0};
952 
953  /* error = cpl_propertylist_append_bool(reflist, "POW", TRUE);
954  cpl_ensure_code(!error, error);
955 
956  error = cpl_table_sort(table, reflist);
957  cpl_ensure_code(!error, error);
958  */
959 
960  int nrows = cpl_table_get_nrow(table);
961  int i;
962 
963  double * all_freqs = cpl_table_get_data_double(table, "FREQ");
964  double * all_pows = cpl_table_get_data_double(table, "POW");
965 
966  for ( i= 1; i< nrows-1; i++){
967  if (all_pows[i] > pows[0]) {
968  if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
969  pows[2]=pows[1];
970  pows[1]=pows[0];
971  pows[0]=all_pows[i];
972 
973  freqs[2]=freqs[1];
974  freqs[1]=freqs[0];
975  freqs[0]=all_freqs[i];
976  }
977  } else if (all_pows[i] > pows[1]) {
978  if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
979  pows[2]=pows[1];
980  pows[1]=all_pows[i];
981 
982  freqs[2]=freqs[1];
983  freqs[1]=all_freqs[i];
984  }
985 
986  } else if(all_pows[i] > pows[2]) {
987  if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
988  pows[2]=all_pows[i];
989 
990  freqs[2]=all_freqs[i];
991  }
992 
993  }
994  }
995 
996  if (detmon_pernoise_config.mode == 1) {
997  propname = cpl_sprintf("ESO QC FREQ1 %d", iquad);
998  assert(propname != NULL);
999  } else {
1000  propname = cpl_sprintf("ESO QC FREQ1");
1001  }
1002 
1003  error = cpl_propertylist_append_double(qclist, propname, freqs[0]);
1004  error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
1005  cpl_ensure_code(!error, error);
1006 
1007  cpl_free(propname);
1008 
1009  if (detmon_pernoise_config.mode == 1) {
1010  propname = cpl_sprintf("ESO QC FREQ2 %d", iquad);
1011  assert(propname != NULL);
1012  } else {
1013  propname = cpl_sprintf("ESO QC FREQ2");
1014  }
1015 
1016  error = cpl_propertylist_append_double(qclist, propname, freqs[1]);
1017  error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
1018  cpl_ensure_code(!error, error);
1019 
1020  cpl_free(propname);
1021 
1022  if (detmon_pernoise_config.mode == 1) {
1023  propname = cpl_sprintf("ESO QC FREQ3 %d", iquad);
1024  assert(propname != NULL);
1025  } else {
1026  propname = cpl_sprintf("ESO QC FREQ3");
1027  }
1028 
1029  error = cpl_propertylist_append_double(qclist, propname, freqs[2]);
1030  error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
1031  cpl_ensure_code(!error, error);
1032 
1033  cpl_free(propname);
1034 
1035  if (detmon_pernoise_config.mode == 1) {
1036  propname = cpl_sprintf("ESO QC POW1 %d", iquad);
1037  assert(propname != NULL);
1038  } else {
1039  propname = cpl_sprintf("ESO QC POW1");
1040  }
1041 
1042  error = cpl_propertylist_append_double(qclist, propname, pows[0]);
1043  error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1044  cpl_ensure_code(!error, error);
1045 
1046  cpl_free(propname);
1047 
1048  if (detmon_pernoise_config.mode == 1) {
1049  propname = cpl_sprintf("ESO QC POW2 %d", iquad);
1050  assert(propname != NULL);
1051  } else {
1052  propname = cpl_sprintf("ESO QC POW2");
1053  }
1054 
1055  error = cpl_propertylist_append_double(qclist, propname, pows[1]);
1056  error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1057  cpl_ensure_code(!error, error);
1058 
1059  cpl_free(propname);
1060 
1061  if (detmon_pernoise_config.mode == 1) {
1062  propname = cpl_sprintf("ESO QC POW3 %d", iquad);
1063  assert(propname != NULL);
1064  } else {
1065  propname = cpl_sprintf("ESO QC POW3");
1066  }
1067 
1068  error = cpl_propertylist_append_double(qclist, propname, pows[2]);
1069  error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1070  cpl_ensure_code(!error, error);
1071 
1072 
1073  cpl_free(propname);
1074 
1075  return cpl_error_get_code();
1076 }
1077 
1078 /*---------------------------------------------------------------------------*/
1079 
1080 /*
1081  * @brief Remove background
1082  * @param image Input image
1083  * @param nsamples Input number of (random)samples
1084  * @param nffts input number of FFTs
1085  * @return CPL_ERROR_NONE on success.
1086  */
1087 
1088 /*---------------------------------------------------------------------------*/
1089 cpl_error_code
1090 detmon_pernoise_rm_bg(cpl_image * image, int nsamples, int nffts)
1091 {
1092  cpl_vector *values = cpl_vector_new(nsamples * nffts);
1093 
1094  int rejected;
1095  int i, j;
1096  cpl_vector *xy_pos = cpl_vector_new(nsamples * nffts * 2);
1097  cpl_polynomial * poly_2d = 0;
1098  cpl_image * poly_ima = 0;
1099  cpl_size degree = 3;
1100  cpl_error_code error = CPL_ERROR_NONE;
1101  cpl_matrix * samppos = 0;
1102 
1103  for(i = 1; i <= nffts; i++) {
1104  for(j = 1; j <= nsamples; j++) {
1105  cpl_vector_set(xy_pos, (i - 1) * nsamples + (j - 1), j);
1106  cpl_vector_set(xy_pos, (i - 1) * nsamples + (j - 1) + nsamples * nffts, i);
1107  cpl_vector_set(values, (i - 1) * nsamples + (j - 1),
1108  cpl_image_get(image, j, i, &rejected));
1109  error = cpl_error_get_code();
1110  if (error != CPL_ERROR_NONE)
1111  {
1112  break;
1113  }
1114  }
1115  if (error != CPL_ERROR_NONE)
1116  {
1117  break;
1118  }
1119  }
1120  if (error != CPL_ERROR_NONE)
1121  {
1122  goto cleanup;
1123  }
1124 
1125  poly_2d = cpl_polynomial_new(2);
1126  samppos =
1127  cpl_matrix_wrap(2, nsamples * nffts, cpl_vector_get_data(xy_pos));
1128 
1129  cpl_polynomial_fit(poly_2d, samppos, NULL, values, NULL,
1130  CPL_FALSE, NULL, &degree);
1131 
1132  cpl_matrix_unwrap(samppos);
1133 
1134  poly_ima = cpl_image_new(nsamples, nffts, CPL_TYPE_FLOAT);
1135 
1136  cpl_image_fill_polynomial(poly_ima, poly_2d, 1, 1, 1, 1);
1137 
1138  cpl_image_subtract(image, poly_ima);
1139 
1140 cleanup:
1141  cpl_polynomial_delete(poly_2d);
1142  cpl_image_delete(poly_ima);
1143  cpl_vector_delete(xy_pos);
1144  cpl_vector_delete(values);
1145 
1146  return cpl_error_get_code();
1147 }
1148