00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027 #ifdef HAVE_CONFIG_H
00028 #include <config.h>
00029 #endif
00030
00031
00037
00040
00041
00042
00043 #include <math.h>
00044
00045 #include <xsh_drl.h>
00046 #include <xsh_data_pre.h>
00047 #include <xsh_dfs.h>
00048 #include <xsh_pfits.h>
00049 #include <xsh_error.h>
00050 #include <xsh_msg.h>
00051 #include <xsh_badpixelmap.h>
00052
00053 #include <cpl.h>
00054
00055
00056
00057
00058
00059
00060
00061
00062
00063
00064
00065
00066
00067 static void add_noisy_pixel_to_ascii_file( int x, int y )
00068 {
00069 static FILE *fout ;
00070 const char * fname = "noisy_pixels.dat" ;
00071
00072 if ( fout == NULL )
00073 fout = fopen( fname, "w" ) ;
00074 fprintf( fout, "%d %d\n", x, y ) ;
00075 }
00076
00089 static int
00090 flag_noisy_pixels (cpl_imagelist * raws,
00091 cpl_image * bpmap,
00092 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00093 cpl_image * noisymap,
00094 #endif
00095 int nx, int ny,
00096 xsh_clipping_param * noise_clipping)
00097 {
00098 int ix, iy, i;
00099 cpl_image **pimg = NULL;
00100 int nimg, nbad = 0, npix = 0;
00101 double medStackv = 0.,
00102 aveStackv = 0.,
00103 sumStackv = 0., errorStackv = 0.;
00104 double *Stackv = NULL,
00106 *pStackv = NULL;
00107 double *stack = NULL;
00108 double *bisStackv = NULL,
00109 *pbis = NULL;
00110 double diff = 0.0, diff2 = 0.0;
00111 double min = 999999., max = -9999999.;
00112 double maxdelta = 0.0;
00113 cpl_binary *bpmap_mask = NULL ;
00114 int idpix = 0 ;
00115 float ** pixdata = NULL ;
00116 cpl_binary **pixmask = NULL ;
00117 int nhot = 0 ;
00118 int totpix = nx*ny ;
00119 nimg = cpl_imagelist_get_size (raws);
00120
00121 XSH_MALLOC(pimg, cpl_image*, nimg);
00122
00123
00124 for (i = 0; i < nimg; i++) {
00125 *(pimg + i) = cpl_imagelist_get (raws, i);
00126 }
00127
00128 XSH_MALLOC(Stackv, double, nx * ny);
00129
00130 pStackv = Stackv;
00131
00132 xsh_msg_dbg_low( "Rejected in bpmap: %d",
00133 cpl_image_count_rejected( bpmap ) ) ;
00134
00135
00136 XSH_MALLOC(stack, double, nimg);
00137
00138
00139 bpmap_mask = cpl_mask_get_data( cpl_image_get_bpm(bpmap) ) ;
00140 assure( bpmap_mask != NULL, cpl_error_get_code(),
00141 "Cant get bpmap Mask" ) ;
00142
00143 XSH_MALLOC(bisStackv, double, totpix);
00144 pbis = bisStackv ;
00145
00146
00147 {
00148 int j ;
00149 XSH_MALLOC(pixdata,float *,nimg);
00150 XSH_MALLOC(pixmask,cpl_binary *,nimg);
00151
00152 for ( j = 0 ; j<nimg ; j++ ) {
00153 *(pixdata+j) = cpl_image_get_data_float( *(pimg+j));
00154 *(pixmask+j) = cpl_mask_get_data( cpl_image_get_bpm(*(pimg+j)) ) ;
00155 }
00156 }
00157 idpix = 0 ;
00158
00159 for ( iy = 1; iy <= ny; iy++) {
00160 for (ix = 1; ix <= nx; ix++) {
00161 int j, count = 0;
00162 double sum = 0.;
00163
00164
00165 if ( *(bpmap_mask+idpix) != 0 ) {
00166 nbad++;
00167 xsh_msg_dbg_medium("Pixel %d,%d globally bad", ix, iy);
00168 *pStackv = 0. ;
00169 }
00170 else {
00171 for (j = 0; j < nimg; j++) {
00172 int rej;
00173 double fval = *(*(pixdata+j)+idpix) ;
00174 rej = *(*(pixmask+j)+idpix ) ;
00175
00176 if (rej != 0) {
00177 xsh_msg_dbg_medium("Dont use pixel %d,%d [%d]", ix, iy, j);
00178 continue;
00179 }
00180 sum += fval;
00181
00182 *(stack + count) = fval;
00183 count++;
00184 }
00185 }
00186
00187 if ( count > 1 ) {
00188
00189 double avg = sum / (double) count;
00190 double dif, dif2 = 0., sigma = 0. ;
00191 int k;
00192
00193 xsh_msg_dbg_medium( " [%d,%d] Count=%d - sum = %lf, avg = %lf",
00194 ix, iy, count, sum, avg ) ;
00195 for (k = 0; k < count; k++) {
00196 xsh_msg_dbg_medium( " stack[%d] = %lf", k, *(stack+k) ) ;
00197 dif = *(stack + k) - avg;
00198 dif2 += dif * dif;
00199 }
00200 sigma = sqrt (dif2 / (double) (count - 1));
00201 *pStackv = sigma;
00202 *pbis++ = sigma ;
00203 sumStackv += sigma;
00204 xsh_msg_dbg_medium( "++ dif2 = %lf, count= %d, sigma=%lf, Stackv=%lf, sum=%lf",
00205 dif2, count, sigma, *pStackv, sumStackv ) ;
00206 if ( (npix % 500) == 0 )
00207 xsh_msg_dbg_medium( "+++++ (%d,%d) sumStackv = %lf",
00208 ix, iy, sumStackv ) ;
00209 npix++;
00210 }
00211 else {
00212 xsh_msg_dbg_medium( "Not enough good pixels (%d)", count ) ;
00213 *pStackv = 0. ;
00214 }
00215 pStackv++;
00216 idpix++ ;
00217 }
00218 }
00219 xsh_msg_dbg_medium( "+++++ sumStackv = %lf", sumStackv ) ;
00220
00221
00222
00223
00224 medStackv = xsh_tools_get_median_double( bisStackv, npix ) ;
00225
00226
00227 aveStackv = sumStackv / (double) npix;
00228 xsh_msg_dbg_low( "--- Npix=%d - sumStackv=%lf - aveStackv=%lf",
00229 npix, sumStackv, aveStackv ) ;
00230
00231
00232
00233 for( pStackv = bisStackv, i = 0 ; i<npix ; i++, pStackv++ ) {
00234 diff = *pStackv - aveStackv;
00235 if (diff < min)
00236 min = diff;
00237 if (diff > max)
00238 max = diff;
00239 diff2 += diff * diff;
00240 }
00241 cpl_free (bisStackv);
00242 bisStackv = NULL ;
00243
00244 xsh_msg_dbg_low("Diff Min: %lf, Max: %lf", min, max);
00245 errorStackv = sqrt (diff2 / (double) (npix - 1));
00246 xsh_msg_dbg_low( "errorStackv=%lf", errorStackv ) ;
00247
00248
00249 maxdelta = noise_clipping->sigma * errorStackv;
00250
00251 xsh_msg_dbg_low("Npix:%d/%d, sum: %lf, med:%lf, avg:%lf, err:%lf, delta: %lf",
00252 npix, nx * ny, sumStackv, medStackv, aveStackv, errorStackv, maxdelta);
00253
00254 pStackv = Stackv;
00255 idpix = 0 ;
00256
00257 for (iy = 1; iy <= ny; iy++) {
00258 for (ix = 1; ix <= nx; ix++) {
00259 if ( *(bpmap_mask+idpix) == 0 ) {
00260 double delta = *pStackv - medStackv;
00261 if (fabs (delta) > maxdelta) {
00262 nhot++ ;
00263
00264 xsh_bpmap_set_bad_pixel (bpmap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
00265 if ( xsh_debug_level_get() >= XSH_DEBUG_LEVEL_LOW )
00266 add_noisy_pixel_to_ascii_file( ix, iy ) ;
00267 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00268
00269 xsh_bpmap_set_bad_pixel (noisymap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
00270 #endif
00271
00272 }
00273
00274 }
00275 pStackv++;
00276 idpix++ ;
00277 }
00278 }
00279 xsh_msg( "Found %d Electronic Pickup Noise Hot Pixels", nhot ) ;
00280
00281 cleanup:
00282 XSH_FREE(stack);
00283 XSH_FREE(Stackv);
00284 XSH_FREE(bisStackv);
00285 XSH_FREE(pimg);
00286 XSH_FREE(pixdata);
00287 XSH_FREE(pixmask);
00288 return nhot;
00289 }
00290
00291 static void
00292 set_pickup_noise_pixels_qc (cpl_propertylist * header, int nbad,
00293 xsh_instrument * instrument)
00294 {
00295 xsh_pfits_set_qc( header, (void *)&nbad,
00296 XSH_QC_BP_MAP_PICKUP_NOISE_PIX, instrument ) ;
00297 }
00298
00309 cpl_frame *
00310 xsh_compute_noise_map (cpl_imagelist * dataList,
00311 cpl_frame * medFrame,
00312 xsh_clipping_param * noise_clipping,
00313 xsh_instrument* instr
00314 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00315 , cpl_frame ** noisyFrame
00316 #endif
00317 )
00318 {
00319 int nx, ny, nframes = 0;
00320 cpl_image *resBpMap = NULL;
00321 cpl_propertylist *bpmapHeader = NULL;
00322 cpl_frame *resFrame = NULL;
00323 int prevnbad;
00324 const char *bpmapFile = cpl_frame_get_filename (medFrame);
00325 xsh_pre *medPre = NULL;
00326 int iter = 0,curnbad = 0;
00327 char result_name[256] ;
00328 const char* tag = XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr);
00329 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00330 char noisy_name[256];
00331 cpl_image * noisyMap = NULL ;
00332 cpl_propertylist * noisymapHeader = NULL ;
00333 #endif
00334 int binx=0;
00335 int biny=0;
00336 cpl_image* ima_aux=NULL;
00337
00338
00339
00340
00341
00342
00343
00344 cpl_msg_indent_more ();
00345 xsh_msg ("*** Removing Noisy Pixels (%s)",
00346 xsh_instrument_arm_tostring(instr));
00347
00348
00349 xsh_msg ("Bpmap file = \"%s\"", bpmapFile);
00350
00351
00352 medPre = xsh_pre_load (medFrame,instr);
00353 XSH_ASSURE_NOT_NULL(medPre);
00354
00355 resBpMap = xsh_pre_get_qual (medPre);
00356 bpmapHeader = medPre->qual_header;
00357
00358 nx = medPre->nx;
00359 ny = medPre->ny;
00360
00361 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00362 check( noisyMap = cpl_image_new( nx, ny, CPL_TYPE_INT ) ) ;
00363 check( noisymapHeader = cpl_propertylist_duplicate( bpmapHeader ) ) ;
00364 #endif
00365 xsh_msg( " Image size: %d,%d", nx, ny ) ;
00366
00367 xsh_set_image_cpl_bpmap (resBpMap, resBpMap, nx, ny);
00368 prevnbad = cpl_image_count_rejected (resBpMap);
00369
00370 nframes = cpl_imagelist_get_size (dataList);
00371 xsh_msg ("%d dark images in image list", nframes);
00372
00373
00374 for (iter = 0; iter < noise_clipping->niter; iter++) {
00375 int nhot ;
00376
00377 xsh_msg (">>>> Iteration Nb %d/%d", iter+1, noise_clipping->niter);
00378 cpl_msg_indent_more ();
00379
00380
00381 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00382 nhot = flag_noisy_pixels (dataList, resBpMap, noisyMap,
00383 nx, ny, noise_clipping);
00384 #else
00385 nhot = flag_noisy_pixels (dataList, resBpMap, nx, ny, noise_clipping);
00386 #endif
00387 cpl_msg_indent_less ();
00388 if ( nhot == 0 ) break ;
00389 }
00390
00391
00392 curnbad = cpl_image_count_rejected (resBpMap);
00393 xsh_msg ("End of noisy pixels, total bad pixels: %d", curnbad);
00394
00395 xsh_msg (" Nb of noisy pixels: %d", curnbad - prevnbad);
00396
00397
00398
00399
00400
00401
00402 strcpy(result_name,bpmapFile) ;
00403 xsh_msg ("save frame %s\n", result_name);
00404
00405
00406
00407
00408
00409
00410 check(set_pickup_noise_pixels_qc (medPre->data_header, curnbad, instr ));
00411
00412 check(xsh_pfits_set_qc_nhpix( medPre->data_header, curnbad - prevnbad )) ;
00413 if ( xsh_instrument_get_arm(instr) == XSH_ARM_NIR ) {
00414 sprintf(noisy_name,"%s.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr));
00415 } else {
00416 check(binx=xsh_pfits_get_binx(medPre->data_header));
00417 check(biny=xsh_pfits_get_biny(medPre->data_header));
00418 sprintf(noisy_name,"%s_%dx%d.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,
00419 instr),
00420 binx,biny);
00421 }
00422 ima_aux=cpl_image_cast(medPre->data,CPL_TYPE_FLOAT);
00423 check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
00424 medPre->data_header, CPL_IO_DEFAULT));
00425 xsh_free_image(&ima_aux);
00426
00427 xsh_pfits_set_extname (medPre->errs_header, "ERRS");
00428 ima_aux=cpl_image_cast(medPre->errs,CPL_TYPE_FLOAT);
00429 check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
00430 medPre->errs_header, CPL_IO_EXTEND));
00431 xsh_free_image(&ima_aux);
00432
00433
00434 xsh_pfits_set_extname (bpmapHeader, "QUAL");
00435 check(cpl_image_save (resBpMap, result_name, XSH_PRE_DATA_BPP, bpmapHeader,
00436 CPL_IO_EXTEND));
00437
00438
00439 check(resFrame=xsh_frame_product(result_name,tag,
00440 CPL_FRAME_TYPE_IMAGE,
00441 CPL_FRAME_GROUP_PRODUCT,
00442 CPL_FRAME_LEVEL_FINAL));
00443
00444 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00445
00446
00447 set_pickup_noise_pixels_qc (noisymapHeader, curnbad, instr );
00448 xsh_pfits_set_qc_nhpix( noisymapHeader, (double)(curnbad - prevnbad) ) ;
00449 cpl_image_save (noisyMap, noisy_name, XSH_PRE_DATA_BPP,
00450 noisymapHeader,
00451 CPL_IO_DEFAULT );
00452
00453 check(*noisyFrame=xsh_frame_product(noisy_name,tag,
00454 CPL_FRAME_TYPE_IMAGE,
00455 CPL_FRAME_GROUP_PRODUCT,
00456 CPL_FRAME_LEVEL_TEMPORARY));
00457 xsh_add_temporary_file(noisy_name);
00458 #endif
00459
00460 cleanup:
00461
00462 xsh_free_propertylist(&noisymapHeader);
00463 xsh_free_image(&noisyMap);
00464 xsh_free_image(&ima_aux);
00465 xsh_pre_free( &medPre ) ;
00466
00467 return resFrame ;
00468 }
00469
00470
00471