45 #include "detmon_pernoise.h"
47 #include "irplib_ksigma_clip.h"
48 #include "irplib_hist.h"
49 #include "irplib_utils.h"
53 #define pdist(x1,y1,x2,y2) (((x1-x2)*(x1-x2))+((y1-y2)*(y1-y2)))
55 #define cpl_drand() ((double)rand()/(double)RAND_MAX)
70 #define HIST_FACT 2.354820045
98 cpl_boolean direction;
107 } detmon_pernoise_config;
120 detmon_pernoise_dfs_set_groups(cpl_frameset *,
123 static cpl_error_code
124 detmon_pernoise_retrieve_parlist(
const char *,
126 const cpl_parameterlist *);
128 static cpl_error_code
129 detmon_pernoise_qc(cpl_propertylist *,
133 static cpl_error_code
134 detmon_pernoise_save(
const cpl_parameterlist *,
144 const cpl_frameset *);
147 detmon_pernoise_rm_bg(cpl_image *,
152 static cpl_error_code
153 detmon_dark_save(
const cpl_parameterlist *,
167 const cpl_frameset *);
171 static cpl_error_code
172 detmon_retrieve_dark_params(
const char *,
174 const cpl_parameterlist *);
190 #define HORIZONTAL TRUE
193 detmon_pernoise_reduce(cpl_image * image)
201 cpl_image * power_im = 0;
202 cpl_image * output = 0;
203 cpl_image * pos_spec = 0;
204 cpl_table * table = 0;
205 cpl_image* fourier_im = 0;
207 cpl_error_code error = CPL_ERROR_NONE;
208 cpl_image * sub_image = 0;
212 if(detmon_pernoise_config.direction == HORIZONTAL) {
213 error = cpl_image_flip(image, 1);
214 cpl_ensure(!error, error, NULL);
217 nsamples = cpl_image_get_size_x(image);
218 nffts = cpl_image_get_size_y(image);
229 error = detmon_pernoise_rm_bg(image, nsamples, nffts);
230 cpl_ensure(!error, error, NULL);
232 sub_image = cpl_image_extract(image, nsamples/8 + 1, nffts/8+1,
233 nsamples*7/8, nffts*7/8);
235 nsamples = cpl_image_get_size_x(sub_image);
236 nffts = cpl_image_get_size_y(sub_image);
249 hanning = cpl_malloc(
sizeof(
float) * nsamples);
251 for(i = 0; i < nsamples; i++) {
252 *(hanning + i) = 0.5 - 0.5 * cos(2 * CPL_MATH_PI * (
float) i / nsamples);
253 for(j = 0; j < nffts; j++) {
255 cpl_image_get(sub_image, i + 1, j + 1, &status);
256 error = cpl_image_set(sub_image, i + 1, j + 1, (*(hanning + i)) * value);
261 if (error != CPL_ERROR_NONE)
266 power = (
float *) cpl_calloc(
sizeof(
float), nsamples * nffts);
269 fourier_im = cpl_image_new(nsamples,nffts, CPL_TYPE_FLOAT_COMPLEX);
270 error = cpl_fft_image(fourier_im, sub_image, CPL_FFT_FORWARD);
272 for(i = 1; i <= nffts; i++) {
273 for(j = 1; j <= nsamples; j++) {
275 double complex cvalue = cpl_image_get_complex(fourier_im,j, i, &rej );
276 double value = cabs(cvalue);
281 cpl_image_set(power_im, j, i, value);
285 cpl_image_delete(fourier_im);
287 output = cpl_image_collapse_create(power_im, 0);
288 pos_spec = cpl_image_extract(output, 1, 1, nsamples/2, 1);
290 cpl_image_delete(power_im);
293 cpl_image_delete(output);
295 table = cpl_table_new(nsamples/2);
296 cpl_table_new_column(table,
"FREQ", CPL_TYPE_DOUBLE);
297 cpl_table_new_column(table,
"POW", CPL_TYPE_DOUBLE);
299 freq = detmon_pernoise_config.speed*1000/nffts_old;
301 for(i = 0; i < nsamples/2; i++) {
302 error = cpl_table_set(table,
"FREQ", i, freq/(nsamples/2)*i);
303 error = cpl_table_set(table,
"POW", i, cpl_image_get(pos_spec, i+1, 1, &status));
306 for(i= 0; i < 5; i++) {
307 error = cpl_table_set(table,
"POW", i, 0.0);
312 cpl_image_delete(pos_spec);
314 cpl_image_delete(sub_image);
315 if (error != CPL_ERROR_NONE)
317 cpl_table_delete(table);
343 detmon_pernoise(cpl_frameset * frameset,
344 const cpl_parameterlist * parlist,
346 const char * recipe_name,
347 const char * pipeline_name,
348 const char * procatg_tbl,
349 const char * package,
350 int (*compare)(
const cpl_frame *,
356 cpl_error_code error;
358 if(detmon_pernoise_dfs_set_groups(frameset, tag)) {
359 cpl_msg_error(cpl_func,
"Cannot identify RAW and CALIB frames");
367 error = detmon_pernoise_retrieve_parlist(pipeline_name,
368 recipe_name, parlist);
369 cpl_ensure_code(!error, error);
375 cpl_msg_info(cpl_func,
"Identify the different settings");
376 cpl_size* selection = cpl_frameset_labelise(frameset, compare, &nsets);
377 if(selection == NULL)
378 cpl_msg_error(cpl_func,
"Cannot labelise input frames");
381 detmon_pernoise_config.nb_extensions = 1;
382 if(detmon_pernoise_config.exts < 0) {
383 const cpl_frame *cur_frame =
384 cpl_frameset_get_first_const(frameset);
386 detmon_pernoise_config.nb_extensions =
387 cpl_frame_get_nextensions(cur_frame);
391 for(i = 0; i < nsets; i++)
394 cpl_table ** freq_table;
395 cpl_propertylist ** qclist =
396 (cpl_propertylist **)
397 cpl_malloc(detmon_pernoise_config.nb_extensions *
398 sizeof(cpl_propertylist *));
400 cpl_imagelist ** raws = (cpl_imagelist **) cpl_malloc(detmon_pernoise_config.nb_extensions *
sizeof(cpl_imagelist *));
401 cpl_image ** input = (cpl_image **) cpl_malloc(detmon_pernoise_config.nb_extensions *
sizeof(cpl_image *));
404 if(detmon_pernoise_config.mode == 1)
407 (cpl_table **) cpl_malloc(detmon_pernoise_config.nb_extensions *
408 4 *
sizeof(cpl_table *));
412 (cpl_table **) cpl_malloc(detmon_pernoise_config.nb_extensions *
413 sizeof(cpl_table *));
416 if(detmon_pernoise_config.exts >= 0)
419 cpl_imagelist_load_frameset(frameset, CPL_TYPE_FLOAT, 1,
420 detmon_pernoise_config.exts);
421 *input = cpl_image_subtract_create(cpl_imagelist_get(*raws,0),
422 cpl_imagelist_get(*raws,1));
425 cpl_imagelist *raws_all_exts =
426 cpl_imagelist_load_frameset(frameset, CPL_TYPE_FLOAT, 1,
428 for(j = 0; j < detmon_pernoise_config.nb_extensions; j++)
430 int nframes = cpl_frameset_get_size(frameset);
432 for(k = 0; k < nframes; k++)
435 cpl_imagelist_unset(raws_all_exts,
436 (detmon_pernoise_config.
437 nb_extensions - 1 - j) * k);
438 cpl_imagelist_set(raws[j], image, k);
441 cpl_image_subtract_create(cpl_imagelist_get(raws[j],0),
442 cpl_imagelist_get(raws[j],1));
446 for(j = 0; j < detmon_pernoise_config.nb_extensions; j++) {
447 cpl_msg_info(cpl_func,
"Starting reduction");
448 qclist[j] = cpl_propertylist_new();
449 if(detmon_pernoise_config.mode == 1)
451 int nx = cpl_image_get_size_x(input[j]);
452 int ny = cpl_image_get_size_y(input[j]);
456 quad[0] = cpl_image_extract(input[j], 1, 1, nx/2, ny/2);
457 quad[1] = cpl_image_extract(input[j], 1, ny/2+1, nx/2, ny);
458 quad[2] = cpl_image_extract(input[j], nx/2+1, 1, nx, ny/2);
459 quad[3] = cpl_image_extract(input[j], nx/2+1, ny/2+1, nx, ny);
461 for (k = 0; k < 4; k++)
463 freq_table[j * 4 + k] = detmon_pernoise_reduce(quad[k]);
465 for(k = 0; k < 4; k++)
467 error = detmon_pernoise_qc(qclist[j], freq_table[j + k], k+1);
468 if (error != CPL_ERROR_NONE)
471 for (k = 0; k < 4; k++)
473 cpl_image_delete(quad[k]);
477 freq_table[j] = detmon_pernoise_reduce(input[j]);
478 if(freq_table[j] != NULL)
480 error = detmon_pernoise_qc(qclist[j], freq_table[j], 0);
483 if (error != CPL_ERROR_NONE)
488 if (error == CPL_ERROR_NONE)
490 error = detmon_pernoise_save(parlist, frameset, recipe_name,
491 pipeline_name, procatg_tbl,
492 package, freq_table, qclist, 0,
496 for(j = 0; j < detmon_pernoise_config.nb_extensions; j++)
498 cpl_propertylist_delete(qclist[j]);
499 cpl_imagelist_delete(raws[j]);
500 cpl_image_delete(input[j]);
505 if(detmon_pernoise_config.mode == 1)
507 for(j= 0; j < detmon_pernoise_config.nb_extensions * 4; j++) {
508 cpl_table_delete(freq_table[j]);
511 for(j= 0; j < detmon_pernoise_config.nb_extensions; j++) {
512 cpl_table_delete(freq_table[j]);
515 cpl_free(freq_table);
516 if (error != CPL_ERROR_NONE)
522 return cpl_error_get_code();
536 detmon_pernoise_dfs_set_groups(cpl_frameset * set,
const char *tag)
548 nframes = cpl_frameset_get_size(set);
551 for(i = 0; i < nframes; i++) {
552 cpl_frame* cur_frame = cpl_frameset_get_frame(set, i);
553 const char* cur_tag = cpl_frame_get_tag(cur_frame);
556 if(!strcmp(cur_tag, tag))
557 cpl_frame_set_group(cur_frame, CPL_FRAME_GROUP_RAW);
589 detmon_fill_pernoise_params(cpl_parameterlist * parlist,
590 const char *recipe_name,
591 const char *pipeline_name,
593 const char * direction,
602 detmon_fill_parlist(parlist, recipe_name, pipeline_name, 9,
606 "CPL_TYPE_INT", mode,
610 "CPL_TYPE_BOOL", direction,
614 "CPL_TYPE_DOUBLE", speed,
617 "(yet unsupported) x coordinate of the lower-left "
618 "point of the region of interest. If not modified, default value will be 1.",
621 "(yet unsupported) y coordinate of the lower-left "
622 "point of the region of interest. If not modified, default value will be 1.",
625 "(yet unsupported) x coordinate of the upper-right "
626 "point of the region of interest. If not modified, default value will be X dimension of the input image.",
629 "(yet unsupported) y coordinate of the upper-right "
630 "point of the region of interest. If not modified, default value will be Y dimension of the input image.",
634 "Kappa used for determining threshold of bad (hot, cold) pixels",
635 "CPL_TYPE_DOUBLE", kappa,
638 "Activate the multi-exts option",
639 "CPL_TYPE_INT", exts);
655 detmon_fill_pernoise_params_default(cpl_parameterlist * parlist,
656 const char *recipe_name,
657 const char *pipeline_name)
659 detmon_fill_pernoise_params(parlist, recipe_name, pipeline_name,
684 static cpl_error_code
685 detmon_pernoise_retrieve_parlist(
const char *pipeline_name,
686 const char *recipe_name,
687 const cpl_parameterlist * parlist)
693 detmon_pernoise_config.mode =
694 detmon_retrieve_par_int(
"mode", pipeline_name, recipe_name,
698 par_name = cpl_sprintf(
"%s.%s.direction", pipeline_name, recipe_name);
699 assert(par_name != NULL);
700 par = cpl_parameterlist_find((cpl_parameterlist *) parlist, par_name);
701 detmon_pernoise_config.direction = cpl_parameter_get_bool(par);
705 par_name = cpl_sprintf(
"%s.%s.speed", pipeline_name, recipe_name);
706 assert(par_name != NULL);
707 par = cpl_parameterlist_find((cpl_parameterlist *) parlist, par_name);
708 detmon_pernoise_config.speed = cpl_parameter_get_double(par);
712 detmon_pernoise_config.llx =
713 detmon_retrieve_par_int(
"llx", pipeline_name, recipe_name,
717 detmon_pernoise_config.lly =
718 detmon_retrieve_par_int(
"lly", pipeline_name, recipe_name,
721 detmon_pernoise_config.urx =
722 detmon_retrieve_par_int(
"urx", pipeline_name, recipe_name,
725 detmon_pernoise_config.ury =
726 detmon_retrieve_par_int(
"ury", pipeline_name, recipe_name,
729 detmon_pernoise_config.kappa =
730 detmon_retrieve_par_double(
"kappa", pipeline_name, recipe_name,
734 detmon_pernoise_config.exts =
735 detmon_retrieve_par_int(
"exts", pipeline_name, recipe_name,
738 if(cpl_error_get_code()) {
739 cpl_msg_error(cpl_func,
"Failed to retrieve the input parameters");
740 cpl_ensure_code(0, CPL_ERROR_DATA_NOT_FOUND);
743 return cpl_error_get_code();
764 static cpl_error_code
765 detmon_pernoise_save(
const cpl_parameterlist * parlist,
766 cpl_frameset * frameset,
767 const char *recipe_name,
768 const char *pipeline_name,
769 const char *procatg_tbl,
771 cpl_table ** freq_table,
772 cpl_propertylist ** qclist,
775 const cpl_frameset * usedframes)
778 cpl_frame *ref_frame;
779 cpl_propertylist *plist;
782 cpl_propertylist *paflist;
783 cpl_error_code error;
785 cpl_propertylist * pro_tbl = cpl_propertylist_new();
787 cpl_propertylist_append_string(pro_tbl,
788 CPL_DFS_PRO_CATG, procatg_tbl);
790 cpl_propertylist_append(pro_tbl, qclist[0]);
797 if(detmon_pernoise_config.mode != 1) {
800 name_o = cpl_sprintf(
"%s_freq_table.fits", recipe_name);
801 assert(name_o != NULL);
804 cpl_sprintf(
"%s_freq_table_set%02d.fits", recipe_name,
806 assert(name_o != NULL);
810 if(cpl_dfs_save_table(frameset, NULL, parlist, usedframes, NULL, freq_table[0],
811 NULL, recipe_name, pro_tbl, NULL,
813 cpl_msg_error(cpl_func,
"Cannot save the product: %s", name_o);
815 cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
818 if(detmon_pernoise_config.exts < 0) {
820 for(i = 1; i < detmon_pernoise_config.nb_extensions; i++) {
822 cpl_table_save(freq_table[i], NULL, qclist[i], name_o,
824 cpl_ensure_code(!error, error);
832 for (
int j = 1; j <= 4; j++) {
835 name_o = cpl_sprintf(
"%s_freq_table_quad%02d.fits",
837 assert(name_o != NULL);
840 cpl_sprintf(
"%s_freq_table_quad%02d_set%02d.fits",
841 recipe_name, j, which_set);
842 assert(name_o != NULL);
846 if(cpl_dfs_save_table(frameset, NULL, parlist, usedframes, NULL,
848 NULL, recipe_name, pro_tbl, NULL,
850 cpl_msg_error(cpl_func,
"Cannot save the product: %s", name_o);
852 cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
855 if(detmon_pernoise_config.exts < 0) {
856 for(i = 1; i < detmon_pernoise_config.nb_extensions; i++) {
857 error = cpl_table_save(freq_table[(j-1) + 4 * i],
858 NULL, qclist[i], name_o,
860 cpl_ensure_code(!error, error);
874 ref_frame = cpl_frameset_get_first(frameset);
875 if((plist = cpl_propertylist_load(cpl_frame_get_filename(ref_frame),
877 cpl_msg_error(cpl_func,
"getting header from reference frame");
878 cpl_ensure_code(0, cpl_error_get_code());
882 paflist = cpl_propertylist_new();
883 cpl_propertylist_copy_property_regexp(paflist, plist,
884 "^(ARCFILE|MJD-OBS|ESO TPL ID|"
885 "DATE-OBS|ESO DET DIT|ESO DET NDIT|"
887 "ESO DET MODE NAME)$", 0);
889 for(i = 0; i < detmon_pernoise_config.nb_extensions; i++) {
890 cpl_propertylist * c_paflist = cpl_propertylist_duplicate(paflist);
891 error = cpl_propertylist_append(c_paflist, qclist[i]);
892 cpl_ensure_code(!error, error);
895 if(detmon_pernoise_config.exts >= 0) {
897 name_o = cpl_sprintf(
"%s.paf", recipe_name);
898 assert(name_o != NULL);
900 name_o = cpl_sprintf(
"%s_set%02d.paf", recipe_name, which_set);
901 assert(name_o != NULL);
905 name_o = cpl_sprintf(
"%s_ext%02d.paf", recipe_name, i+1);
906 assert(name_o != NULL);
908 name_o = cpl_sprintf(
"%s_set%02d_ext%02d.paf", recipe_name, which_set, i+1);
909 assert(name_o != NULL);
913 if(cpl_dfs_save_paf(pipeline_name, recipe_name, c_paflist, name_o)) {
914 cpl_msg_error(cpl_func,
"Cannot save the product: %s", name_o);
916 cpl_propertylist_delete(paflist);
917 cpl_propertylist_delete(plist);
919 cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
921 cpl_propertylist_delete(c_paflist);
925 cpl_propertylist_delete(plist);
926 cpl_propertylist_delete(paflist);
927 cpl_propertylist_delete(pro_tbl);
928 return cpl_error_get_code();
942 static cpl_error_code
943 detmon_pernoise_qc(cpl_propertylist * qclist,
947 cpl_error_code error;
950 double freqs[3] = {0, 0, 0};
951 double pows[3] = {0, 0, 0};
960 int nrows = cpl_table_get_nrow(table);
963 double * all_freqs = cpl_table_get_data_double(table,
"FREQ");
964 double * all_pows = cpl_table_get_data_double(table,
"POW");
966 for ( i= 1; i< nrows-1; i++){
967 if (all_pows[i] > pows[0]) {
968 if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
975 freqs[0]=all_freqs[i];
977 }
else if (all_pows[i] > pows[1]) {
978 if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
983 freqs[1]=all_freqs[i];
986 }
else if(all_pows[i] > pows[2]) {
987 if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
990 freqs[2]=all_freqs[i];
996 if (detmon_pernoise_config.mode == 1) {
997 propname = cpl_sprintf(
"ESO QC FREQ1 %d", iquad);
998 assert(propname != NULL);
1000 propname = cpl_sprintf(
"ESO QC FREQ1");
1003 error = cpl_propertylist_append_double(qclist, propname, freqs[0]);
1004 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
1005 cpl_ensure_code(!error, error);
1009 if (detmon_pernoise_config.mode == 1) {
1010 propname = cpl_sprintf(
"ESO QC FREQ2 %d", iquad);
1011 assert(propname != NULL);
1013 propname = cpl_sprintf(
"ESO QC FREQ2");
1016 error = cpl_propertylist_append_double(qclist, propname, freqs[1]);
1017 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
1018 cpl_ensure_code(!error, error);
1022 if (detmon_pernoise_config.mode == 1) {
1023 propname = cpl_sprintf(
"ESO QC FREQ3 %d", iquad);
1024 assert(propname != NULL);
1026 propname = cpl_sprintf(
"ESO QC FREQ3");
1029 error = cpl_propertylist_append_double(qclist, propname, freqs[2]);
1030 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
1031 cpl_ensure_code(!error, error);
1035 if (detmon_pernoise_config.mode == 1) {
1036 propname = cpl_sprintf(
"ESO QC POW1 %d", iquad);
1037 assert(propname != NULL);
1039 propname = cpl_sprintf(
"ESO QC POW1");
1042 error = cpl_propertylist_append_double(qclist, propname, pows[0]);
1043 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1044 cpl_ensure_code(!error, error);
1048 if (detmon_pernoise_config.mode == 1) {
1049 propname = cpl_sprintf(
"ESO QC POW2 %d", iquad);
1050 assert(propname != NULL);
1052 propname = cpl_sprintf(
"ESO QC POW2");
1055 error = cpl_propertylist_append_double(qclist, propname, pows[1]);
1056 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1057 cpl_ensure_code(!error, error);
1061 if (detmon_pernoise_config.mode == 1) {
1062 propname = cpl_sprintf(
"ESO QC POW3 %d", iquad);
1063 assert(propname != NULL);
1065 propname = cpl_sprintf(
"ESO QC POW3");
1068 error = cpl_propertylist_append_double(qclist, propname, pows[2]);
1069 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1070 cpl_ensure_code(!error, error);
1075 return cpl_error_get_code();
1090 detmon_pernoise_rm_bg(cpl_image * image,
int nsamples,
int nffts)
1092 cpl_vector *values = cpl_vector_new(nsamples * nffts);
1096 cpl_vector *xy_pos = cpl_vector_new(nsamples * nffts * 2);
1097 cpl_polynomial * poly_2d = 0;
1098 cpl_image * poly_ima = 0;
1099 cpl_size degree = 3;
1100 cpl_error_code error = CPL_ERROR_NONE;
1101 cpl_matrix * samppos = 0;
1103 for(i = 1; i <= nffts; i++) {
1104 for(j = 1; j <= nsamples; j++) {
1105 cpl_vector_set(xy_pos, (i - 1) * nsamples + (j - 1), j);
1106 cpl_vector_set(xy_pos, (i - 1) * nsamples + (j - 1) + nsamples * nffts, i);
1107 cpl_vector_set(values, (i - 1) * nsamples + (j - 1),
1108 cpl_image_get(image, j, i, &rejected));
1109 error = cpl_error_get_code();
1110 if (error != CPL_ERROR_NONE)
1115 if (error != CPL_ERROR_NONE)
1120 if (error != CPL_ERROR_NONE)
1125 poly_2d = cpl_polynomial_new(2);
1127 cpl_matrix_wrap(2, nsamples * nffts, cpl_vector_get_data(xy_pos));
1129 cpl_polynomial_fit(poly_2d, samppos, NULL, values, NULL,
1130 CPL_FALSE, NULL, °ree);
1132 cpl_matrix_unwrap(samppos);
1134 poly_ima = cpl_image_new(nsamples, nffts, CPL_TYPE_FLOAT);
1136 cpl_image_fill_polynomial(poly_ima, poly_2d, 1, 1, 1, 1);
1138 cpl_image_subtract(image, poly_ima);
1141 cpl_polynomial_delete(poly_2d);
1142 cpl_image_delete(poly_ima);
1143 cpl_vector_delete(xy_pos);
1144 cpl_vector_delete(values);
1146 return cpl_error_get_code();