#!/bin/bash
#*******************************************************************************
# JMMC project ( http://www.jmmc.fr ) - Copyright (C) CNRS.
#*******************************************************************************

# Note on pm correction: use 'pmRa/cos(DE)' not directly 'pmRa' as pmRa(catalog) is expressed as pmRa*cos(DE)!

echo "processing @ date: `date -u`"

# Use large heap + CMS GC:
STILTS_JAVA_OPTIONS="-Xms4g -Xmx4g"
# use /home/... as /tmp is full:
STILTS_JAVA_OPTIONS="$STILTS_JAVA_OPTIONS -Djava.io.tmpdir=/home/bourgesl/tmp/"
# other options:   -verbose -debug
STILTS_JAVA_OPTIONS="$STILTS_JAVA_OPTIONS -disk -verbose"


# debug bash commands:
#set -eux;


# define catalogs:
CAT_ASCC="ASCC_full.fits"
CAT_SIMBAD_ORIG="SIMBAD_full.vot"
CAT_MDFC="mdfc-v10.fits"


# 0. Download full ASCC (vizier):
if [ ! -e $CAT_ASCC ]; then
    # wget -O $CAT_ASCC_ORIG "http://cdsxmatch.u-strasbg.fr/QueryCat/QueryCat?catName=I%2F280B%2Fascc&mode=allsky&format=votable"
    # stilts ${STILTS_JAVA_OPTIONS} tcopy in=$CAT_ASCC_ORIG out=$CAT_ASCC
    
    wget -O $CAT_ASCC "http://vizier.u-strasbg.fr/viz-bin/asu-binfits?-source=I%2F280B%2Fascc&-out.max=unlimited&-oc.form=dec&-sort=&-c.eq=J2000&-out.add=_RAJ2000&-out.add=_DEJ2000&-out=pmRA&-out=pmDE&-out=Vmag&-out=TYC1&-out=TYC2&-out=TYC3&"

    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC > $CAT_ASCC.stats.log
fi
echo "- STATS on $CAT_ASCC :"
cat $CAT_ASCC.stats.log
#
# Total Rows: 2501313
# +----------+------------+------------+--------------+-----------+---------+
# | column   | mean       | stdDev     | min          | max       | good    |
# +----------+------------+------------+--------------+-----------+---------+
# | _RAJ2000 | 188.87096  | 100.47709  | 3.4094515E-4 | 359.99988 | 2501313 |
# | _DEJ2000 | -3.317228  | 41.249565  | -89.88966    | 89.911354 | 2501313 |
# | pmRA     | -1.7540642 | 32.2306    | -4417.35     | 6773.35   | 2501313 |
# | pmDE     | -6.0875754 | 30.390753  | -5813.7      | 10308.08  | 2501313 |
# | Vmag     | 11.069625  | 1.1319602  | -1.121       | 16.746    | 2500626 |
# | TYC1     | 4933.0464  | 2840.6257  | 0            | 9537      | 2501313 |
# | TYC2     | 1085.0278  | 858.9906   | 0            | 12121     | 2501313 |
# | TYC3     | 0.9864971  | 0.15440936 | 0            | 4         | 2501313 |
# +----------+------------+------------+--------------+-----------+---------+


#    Download full SIMBAD (xmatch):
if [ ! -e $CAT_SIMBAD_ORIG ]; then
    wget -O $CAT_SIMBAD_ORIG "http://cdsxmatch.u-strasbg.fr/QueryCat/QueryCat?catName=SIMBAD&mode=allsky&format=votable"
fi

#    Download MDFC (OCA):
if [ ! -e $CAT_MDFC ]; then
    CAT_MDFC_ZIP="mdfc-v10.zip"
    if [ ! -e $CAT_MDFC_ZIP ]; then
        wget -O $CAT_MDFC_ZIP "https://matisse.oca.eu/foswiki/pub/Main/TheMid-infraredStellarDiametersAndFluxesCompilationCatalogue%28MDFC%29/mdfc-v10.zip"
    fi
    unzip $CAT_MDFC_ZIP $CAT_MDFC
    if [ ! -e $CAT_MDFC ]; then
        echo "missing MDFC catalog"
        exit 1
    fi
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_MDFC > $CAT_MDFC.stats.log
fi
echo "- STATS on $CAT_MDFC :"
cat $CAT_MDFC.stats.log
#
# Total Rows: 465857
# +--------------+-------------+-------------+--------------+-----------+--------+
# | column       | mean        | stdDev      | min          | max       | good   |
# +--------------+-------------+-------------+--------------+-----------+--------+
# | Name         |             |             |              |           | 465857 |
# | SpType       |             |             |              |           | 465857 |
# | RAJ2000      |             |             |              |           | 465857 |
# | DEJ2000      |             |             |              |           | 465857 |
# | distance     | 567.73395   | 660.71625   | 1.3011867    | 18404.096 | 460564 |
# | teff_midi    | 4384.0073   | 1514.4413   | 3327         | 25645     | 403    |
# | teff_gaia    | 6099.031    | 1522.948    | 2241.0       | 64977.0   | 463781 |
# | Comp         |             |             |              |           | 4946   |
# | mean_sep     | 27.04654    | 96.56517    | -1.0         | 999.9     | 28003  |
# | mag1         | 9.140074    | 1.6275178   | -1.47        | 16.2      | 28003  |
# | mag2         | 11.752113   | 2.1467986   | 0.18         | 24.6      | 27611  |
# | diam_midi    | 3.2239852   | 1.9918951   | 0.92         | 20.398    | 402    |
# | e_diam_midi  | 0.016890548 | 0.011792271 | 0.004        | 0.087     | 402    |
# | diam_cohen   | 2.7435546   | 1.189701    | 1.59         | 10.03     | 422    |
# | e_diam_cohen | 0.036658768 | 0.016024187 | 0.018        | 0.121     | 422    |
# | diam_gaia    | 0.17881162  | 0.3026772   | 9.893455E-4  | 43.475067 | 429824 |
# | LDD_meas     | 2.67697     | 3.2892106   | 0.225        | 39.759    | 567    |
# | e_diam_meas  | 0.34178808  | 4.2889524   | 0.001        | 120.0     | 788    |
# | UDD_meas     | 4.4887686   | 16.708101   | 0.05         | 420.0     | 700    |
# | band_meas    |             |             |              |           | 781    |
# | LDD_est      | 0.18713047  | 0.33603582  | 0.002        | 44.85     | 465605 |
# | e_diam_est   | 0.007890246 | 0.029105874 | 0.0          | 2.885     | 465605 |
# | UDDL_est     | 0.1849376   | 0.33243763  | 0.002        | 44.459    | 465605 |
# | UDDM_est     | 0.18540157  | 0.3329401   | 0.002        | 44.476    | 465605 |
# | UDDN_est     | 0.18608275  | 0.33407295  | 0.002        | 44.603    | 465605 |
# | Jmag         | 8.347455    | 1.5624622   | -2.989       | 15.447    | 465188 |
# | Hmag         | 8.036082    | 1.7315052   | -4.007       | 15.439    | 465857 |
# | Kmag         | 7.9406104   | 1.7788626   | -4.378       | 15.638    | 465857 |
# | W4mag        | 7.4334354   | 1.5221424   | -6.859       | 10.149    | 465113 |
# | CalFlag      | 0.09299205  | 0.55026287  | 0            | 7         | 465857 |
# | IRflag       | 1.0827936   | 1.554538    | 0            | 7         | 465857 |
# | nb_Lflux     | 1.9706455   | 0.34433657  | 0            | 5         | 465857 |
# | med_Lflux    | 1.5254774   | 42.61173    | 1.6299028E-4 | 15720.791 | 465755 |
# | disp_Lflux   | 0.30919802  | 15.989581   | 2.0396099E-8 | 7742.2144 | 432072 |
# | nb_Mflux     | 1.9757823   | 0.36098757  | 0            | 6         | 465857 |
# | med_Mflux    | 0.8892675   | 19.973076   | 8.886453E-5  | 7141.7    | 465757 |
# | disp_Mflux   | 0.22529167  | 9.646683    | 5.681251E-10 | 4292.073  | 432130 |
# | nb_Nflux     | 2.4884224   | 0.94565606  | 0            | 9         | 465857 |
# | med_Nflux    | 0.33924013  | 18.71475    | 9.8775825E-5 | 9278.1    | 465750 |
# | disp_Nflux   | 0.0845671   | 4.924083    | 6.325757E-10 | 2386.5    | 433969 |
# | Lcorflux_30  | 1.2305617   | 10.585987   | 1.6299028E-4 | 1681.6018 | 465521 |
# | Lcorflux_100 | 1.0548543   | 5.1469135   | 1.6299027E-4 | 956.6236  | 465521 |
# | Lcorflux_130 | 0.9943006   | 4.3013268   | 1.6299025E-4 | 872.3458  | 465521 |
# | Mcorflux_30  | 0.75047773  | 5.9194183   | 8.886453E-5  | 1237.3656 | 465507 |
# | Mcorflux_100 | 0.6851532   | 3.415813    | 8.886452E-5  | 521.01306 | 465507 |
# | Mcorflux_130 | 0.65762204  | 2.9565108   | 8.886452E-5  | 496.79926 | 465507 |
# | Ncorflux_30  | 0.23722996  | 5.3514657   | 9.877582E-5  | 2300.2861 | 465501 |
# | Ncorflux_100 | 0.22025737  | 3.0437691   | 9.877579E-5  | 994.1644  | 465501 |
# | Ncorflux_130 | 0.21549404  | 2.8604133   | 9.877577E-5  | 994.1628  | 465501 |
# +--------------+-------------+-------------+--------------+-----------+--------+


# 1. Convert catalogs to FITS:
#    Convert SIMBAD to FITS:
CAT_SIMBAD="$CAT_SIMBAD_ORIG.fits"
if [ $CAT_SIMBAD_ORIG -nt $CAT_SIMBAD ]; then
    stilts ${STILTS_JAVA_OPTIONS} tcopy ifmt=votable in=$CAT_SIMBAD_ORIG out=$CAT_SIMBAD
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_SIMBAD > $CAT_SIMBAD.stats.log
fi
echo "- STATS on $CAT_SIMBAD :"
cat $CAT_SIMBAD.stats.log
#
# Total Rows: 20046872
# +---------------+-------------+------------+------------+--------------+----------+
# | column        | mean        | stdDev     | min        | max          | good     |
# +---------------+-------------+------------+------------+--------------+----------+
# | main_id       |             |            |            |              | 20046872 |
# | ra            | 192.32353   | 97.65777   | 0.0        | 359.99997    | 20046872 |
# | dec           | -4.727993   | 37.815365  | -89.90739  | 90.0         | 20046872 |
# | coo_err_maj   | 9.08302     | 1168.314   | 0.0        | 636847.25    | 14878256 |
# | coo_err_min   | 8.777306    | 1168.2705  | 0.0        | 636847.25    | 14878254 |
# | coo_err_angle | 107.97242   | 41.307148  | 0.0        | 179.0        | 20046872 |
# | otype         |             |            |            |              | 20046872 |
# | nbref         | 2.2484744   | 12.740412  | 0          | 18018        | 20046872 |
# | ra_sexa       |             |            |            |              | 20046872 |
# | dec_sexa      |             |            |            |              | 20046872 |
# | coo_qual      |             |            |            |              | 20046872 |
# | coo_bibcode   |             |            |            |              | 19897303 |
# | main_type     |             |            |            |              | 20024597 |
# | other_types   |             |            |            |              | 19976140 |
# | radvel        | 54186.414   | 82484.08   | -84723.35  | 299297.47    | 8049089  |
# | radvel_err    | 1288.86     | 138735.61  | -1798754.8 | 2.62345392E8 | 5808668  |
# | redshift      | 0.36689687  | 0.7390348  | -9.99898   | 33.79        | 8049328  |
# | redshift_err  | 0.004299173 | 0.4627722  | -6.0       | 875.09       | 5808668  |
# | sp_type       |             |            |            |              | 1153941  |
# | morph_type    |             |            |            |              | 148872   |
# | plx           | 1.5877287   | 3.368635   | -12.4      | 768.0665     | 9797234  |
# | plx_err       | 0.1499272   | 0.26179945 | 0.0        | 72.3         | 9795668  |
# | pmra          | -1.9796356  | 27.553507  | -8123.7    | 6765.995     | 11597742 |
# | pmdec         | -5.207331   | 26.206423  | -5817.8    | 10362.394    | 11597742 |
# | pm_err_maj    | 0.22045195  | 1.6695876  | 0.0        | 3852.0       | 11595166 |
# | pm_err_min    | 0.19469301  | 4.8746777  | 0.0        | 16383.0      | 11595089 |
# | pm_err_pa     | 767.24896   | 4655.5195  | 0          | 32767        | 11840323 |
# | size_maj      | 5.0777264   | 398.8873   | -0.017833  | 429050.0     | 1780493  |
# | size_min      | 4.69365     | 353.3584   | -0.017833  | 429050.0     | 1780472  |
# | size_angle    | 97.324356   | 1044.089   | -267       | 32767        | 1688353  |
# | B             | 15.000807   | 4.38233    | -1.46      | 45.38        | 5262636  |
# | V             | 14.69841    | 4.5203314  | -1.46      | 38.76        | 5708932  |
# | R             | 16.167086   | 4.350359   | -1.46      | 50.0         | 2304083  |
# | J             | 12.363684   | 3.878805   | -3.0       | 35.847       | 7845341  |
# | H             | 11.803009   | 4.137609   | -3.73      | 34.92        | 7871322  |
# | K             | 11.502464   | 3.9855666  | -4.05      | 33.274       | 7884112  |
# | u             | 21.63187    | 2.216926   | 0.003      | 37.003       | 3057365  |
# | g             | 18.634373   | 3.4144177  | 0.008      | 34.298       | 5120861  |
# | r             | 17.75938    | 3.3297157  | 0.003      | 35.667       | 5169208  |
# | i             | 17.601536   | 3.6581275  | 0.004      | 35.288       | 4518979  |
# | z             | 19.039694   | 2.1839924  | 0.004      | 36.46        | 3330716  |
# +---------------+-------------+------------+------------+--------------+----------+


# ASCC: filter columns: (ra/dec J2000 corrected with pm) - from epoch 1991.25
CAT_ASCC_MIN="1_min_$CAT_ASCC"
if [ $CAT_ASCC -nt $CAT_ASCC_MIN ]; then
    # convert ASCC coords J2000 to epoch 1991.25 (as in original table):
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC cmd='progress ; keepcols "_RAJ2000 _DEJ2000 pmRA pmDE Vmag" ; addcol ra_a_91 "_RAJ2000 + (1991.25 - 2000.0) * (0.001 * pmRA / 3600.0) / cos(degreesToRadians(_DEJ2000))" ; addcol de_a_91 "_DEJ2000 + (1991.25 - 2000.0) * (0.001 * pmDE / 3600.0)" ' out=$CAT_ASCC_MIN
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_MIN > $CAT_ASCC_MIN.stats.log
fi
echo "- STATS on $CAT_ASCC_MIN :"
cat $CAT_ASCC_MIN.stats.log
#
# Total Rows: 2501313
# +----------+------------+-----------+--------------+-----------+---------+
# | column   | mean       | stdDev    | min          | max       | good    |
# +----------+------------+-----------+--------------+-----------+---------+
# | _RAJ2000 | 188.87096  | 100.47709 | 3.4094515E-4 | 359.99988 | 2501313 |
# | _DEJ2000 | -3.317228  | 41.249565 | -89.88966    | 89.911354 | 2501313 |
# | pmRA     | -1.7540642 | 32.2306   | -4417.35     | 6773.35   | 2501313 |
# | pmDE     | -6.0875754 | 30.390753 | -5813.7      | 10308.08  | 2501313 |
# | Vmag     | 11.069625  | 1.1319602 | -1.121       | 16.746    | 2500626 |
# | ra_a_91  | 188.87096  | 100.4771  | 2.8266E-4    | 359.9999  | 2501313 |
# | de_a_91  | -3.3172133 | 41.24957  | -89.88964    | 89.91136  | 2501313 |
# +----------+------------+-----------+--------------+-----------+---------+


# ASCC: internal match within 3 as to identify inner groups and keep all (V defined)
CAT_ASCC_MIN_GRP="1_min_Grp_$CAT_ASCC"
if [ $CAT_ASCC_MIN -nt $CAT_ASCC_MIN_GRP ]; then
    stilts ${STILTS_JAVA_OPTIONS} tmatch1 matcher='sky' params='3.0' values='ra_a_91 de_a_91' action='identify' in=$CAT_ASCC_MIN out=$CAT_ASCC_MIN_GRP
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC_MIN_GRP cmd='progress ; colmeta -name ASCC_GROUP_SIZE GroupSize ; select !NULL_Vmag||ASCC_GROUP_SIZE>0 ; delcols "GroupID" ' out=$CAT_ASCC_MIN_GRP
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_MIN_GRP > $CAT_ASCC_MIN_GRP.stats.log
fi
echo "- STATS on $CAT_ASCC_MIN_GRP :"
cat $CAT_ASCC_MIN_GRP.stats.log
#
# Total Rows: 2500764
# +-----------------+------------+------------+--------------+-----------+---------+
# | column          | mean       | stdDev     | min          | max       | good    |
# +-----------------+------------+------------+--------------+-----------+---------+
# | _RAJ2000        | 188.87296  | 100.47335  | 3.4094515E-4 | 359.99988 | 2500764 |
# | _DEJ2000        | -3.3170354 | 41.25193   | -89.88966    | 89.911354 | 2500764 |
# | pmRA            | -1.754506  | 32.229416  | -4417.35     | 6773.35   | 2500764 |
# | pmDE            | -6.086718  | 30.385551  | -5813.7      | 10308.08  | 2500764 |
# | Vmag            | 11.069625  | 1.1319602  | -1.121       | 16.746    | 2500626 |
# | ra_a_91         | 188.87297  | 100.47336  | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91         | -3.3170204 | 41.251934  | -89.88964    | 89.91136  | 2500764 |
# | ASCC_GROUP_SIZE | 2.0283375  | 0.17042844 | 2            | 4         | 31760   |
# +-----------------+------------+------------+--------------+-----------+---------+


# Filter SIMBAD objects:
CAT_SIMBAD_OBJ="1_obj_$CAT_SIMBAD"
CAT_SIMBAD_OBJ_STAR="1_obj_star_$CAT_SIMBAD"
CAT_SIMBAD_OBJ_Vgt16="1_obj_star_Vgt16_$CAT_SIMBAD"

if [ $CAT_SIMBAD -nt $CAT_SIMBAD_OBJ ]; then
    # filter SIMBAD main_type (stars only) to exclude Planet, Galaxy ...
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_SIMBAD cmd='progress; select !NULL_sp_type||NULL_morph_type&&!NULL_main_type&&(main_type.equals(\"Star\")||!(main_type.contains(\"Galaxy\")||main_type.contains(\"EmG\")||main_type.contains(\"Seyfert\")||main_type.contains(\"GroupG\")||main_type.contains(\"QSO\")||main_type.contains(\"Radio\")||main_type.contains(\"AGN\")||main_type.contains(\"Cl\")||main_type.contains(\"in\")||main_type.contains(\"Inexistent\")||main_type.contains(\"Planet\")||main_type.contains(\"EmObj\")||main_type.contains(\"ISM\")||main_type.contains(\"HII\")||main_type.contains(\"SFR\")||main_type.contains(\"SNR\"))) ' out=$CAT_SIMBAD_OBJ_STAR
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_SIMBAD_OBJ_STAR > $CAT_SIMBAD_OBJ_STAR.stats.log

    # filter on V > 16 (not in TYCHO2)
    MAG_FILTER="NULL_V||V<=16.0"
    
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_SIMBAD_OBJ_STAR cmd="progress; select !(${MAG_FILTER}) " out=$CAT_SIMBAD_OBJ_Vgt16
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_SIMBAD_OBJ_Vgt16 > $CAT_SIMBAD_OBJ_Vgt16.stats.log

    # filter on V <= 16 (max in TYCHO2)
    # convert SIMBAD coords to epoch 1991.25:
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_SIMBAD_OBJ_STAR cmd="progress; select ${MAG_FILTER} ; addcol ra_s_91 'ra  + ((NULL_pmra ) ? 0.0 : (1991.25 - 2000.0) * (0.001 * pmra / 3600.0) / cos(degreesToRadians(dec)))' ; addcol de_s_91 'dec + ((NULL_pmdec) ? 0.0 : (1991.25 - 2000.0) * (0.001 * pmdec / 3600.0))' " out=$CAT_SIMBAD_OBJ
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_SIMBAD_OBJ > $CAT_SIMBAD_OBJ.stats.log
fi
echo "- STATS on $CAT_SIMBAD_OBJ_STAR :"
cat $CAT_SIMBAD_OBJ_STAR.stats.log
#
# Total Rows: 10940381
# +---------------+--------------+------------+-----------+-------------+----------+
# | column        | mean         | stdDev     | min       | max         | good     |
# +---------------+--------------+------------+-----------+-------------+----------+
# | main_id       |              |            |           |             | 10940381 |
# | ra            | 198.72646    | 97.29648   | 0.0       | 359.9999    | 10940381 |
# | dec           | -9.219172    | 39.5739    | -89.88967 | 90.0        | 10940381 |
# | coo_err_maj   | 13.29239     | 1443.1635  | 0.0       | 636847.25   | 9747164  |
# | coo_err_min   | 12.90619     | 1443.13    | 0.0       | 636847.25   | 9747162  |
# | coo_err_angle | 97.04841     | 29.490423  | 0.0       | 179.0       | 10940381 |
# | otype         |              |            |           |             | 10940381 |
# | nbref         | 2.2413998    | 10.516274  | 0         | 15045       | 10940381 |
# | ra_sexa       |              |            |           |             | 10940381 |
# | dec_sexa      |              |            |           |             | 10940381 |
# | coo_qual      |              |            |           |             | 10940381 |
# | coo_bibcode   |              |            |           |             | 10867367 |
# | main_type     |              |            |           |             | 10940378 |
# | other_types   |              |            |           |             | 10889670 |
# | radvel        | 5910.5576    | 27590.592  | -76618.44 | 299297.47   | 4627730  |
# | radvel_err    | 386.1714     | 27752.762  | -9.99     | 3.5740656E7 | 4356141  |
# | redshift      | 0.035487704  | 0.2659467  | -9.99898  | 33.79       | 4627773  |
# | redshift_err  | 0.0012881273 | 0.09257325 | -3.33E-5  | 119.217995  | 4356141  |
# | sp_type       |              |            |           |             | 1153941  |
# | morph_type    |              |            |           |             | 22       |
# | plx           | 1.8982488    | 3.6020474  | -12.4     | 768.0665    | 7523146  |
# | plx_err       | 0.11790899   | 0.23809984 | 0.0       | 72.3        | 7521689  |
# | pmra          | -1.9694955   | 30.564892  | -8123.7   | 6765.995    | 8681176  |
# | pmdec         | -6.137317    | 29.10485   | -5817.8   | 10362.394   | 8681176  |
# | pm_err_maj    | 0.1815509    | 1.895925   | 0.0       | 3852.0      | 8678684  |
# | pm_err_min    | 0.16262384   | 5.6300926  | 0.0       | 16383.0     | 8678681  |
# | pm_err_pa     | 255.63455    | 2321.1572  | 0         | 32767       | 8722814  |
# | size_maj      | 30.022696    | 942.19714  | 0.0       | 429050.0    | 246728   |
# | size_min      | 29.510498    | 941.9144   | 0.0       | 429050.0    | 246726   |
# | size_angle    | 197.23134    | 1931.8157  | -173      | 32767       | 240660   |
# | B             | 13.31502     | 2.6770668  | -1.46     | 45.38       | 4109959  |
# | V             | 13.027824    | 3.067928   | -1.46     | 38.76       | 4495380  |
# | R             | 13.854828    | 2.5778325  | -1.46     | 50.0        | 1440623  |
# | J             | 11.295947    | 2.42195    | -3.0      | 31.108      | 6696852  |
# | H             | 10.613803    | 2.4435267  | -3.73     | 31.13       | 6691953  |
# | K             | 10.377926    | 2.4639437  | -4.05     | 29.58       | 6702826  |
# | u             | 20.802761    | 2.6609378  | 0.003     | 32.0        | 559943   |
# | g             | 15.889571    | 3.255791   | 0.008     | 33.7        | 1986494  |
# | r             | 15.028681    | 3.122581   | 0.003     | 32.7        | 2005976  |
# | i             | 14.419087    | 3.4102943  | 0.004     | 32.83       | 1690098  |
# | z             | 17.80761     | 2.413315   | 0.004     | 31.866      | 622982   |
# +---------------+--------------+------------+-----------+-------------+----------+

echo "- STATS on $CAT_SIMBAD_OBJ_Vgt16 :"
cat $CAT_SIMBAD_OBJ_Vgt16.stats.log
#
# Total Rows: 796552
# +---------------+--------------+------------+------------+-----------+--------+
# | column        | mean         | stdDev     | min        | max       | good   |
# +---------------+--------------+------------+------------+-----------+--------+
# | main_id       |              |            |            |           | 796552 |
# | ra            | 144.60864    | 109.68682  | 0.0        | 359.9994  | 796552 |
# | dec           | -31.606236   | 39.3168    | -87.763954 | 89.08628  | 796552 |
# | coo_err_maj   | 0.10717148   | 3.430733   | 0.0        | 1080.0    | 564158 |
# | coo_err_min   | 0.10231079   | 3.4264762  | 0.0        | 1080.0    | 564158 |
# | coo_err_angle | 111.26492    | 42.345695  | 0.0        | 179.0     | 796552 |
# | otype         |              |            |            |           | 796552 |
# | nbref         | 2.8144364    | 9.368843   | 0          | 3105      | 796552 |
# | ra_sexa       |              |            |            |           | 796552 |
# | dec_sexa      |              |            |            |           | 796552 |
# | coo_qual      |              |            |            |           | 796552 |
# | coo_bibcode   |              |            |            |           | 784118 |
# | main_type     |              |            |            |           | 796552 |
# | other_types   |              |            |            |           | 752584 |
# | radvel        | 16433.625    | 54792.18   | -1309.945  | 293619.6  | 85014  |
# | radvel_err    | 790.714      | 26620.78   | -9.99      | 2997918.5 | 68557  |
# | redshift      | 0.13541621   | 0.57964    | -0.00436   | 8.8047    | 85042  |
# | redshift_err  | 0.0026375367 | 0.08879737 | -3.33E-5   | 9.99998   | 68557  |
# | sp_type       |              |            |            |           | 32304  |
# | morph_type    |              |            |            |           | 10     |
# | plx           | 0.7102169    | 2.6725035  | 0.0        | 496.0     | 343695 |
# | plx_err       | 0.13833894   | 0.19485395 | 0.0027     | 37.0      | 343695 |
# | pmra          | -0.09212074  | 26.079765  | -2759.0    | 3981.977  | 472297 |
# | pmdec         | -3.8415132   | 24.284582  | -2466.832  | 1680.502  | 472297 |
# | pm_err_maj    | 0.20427741   | 0.53861004 | 0.0        | 75.0      | 472234 |
# | pm_err_min    | 0.21059032   | 23.845345  | 0.0        | 16383.0   | 472234 |
# | pm_err_pa     | 238.58136    | 2199.443   | 0          | 32767     | 474393 |
# | size_maj      | 0.1689309    | 0.47150233 | 8.0E-6     | 16.78     | 2836   |
# | size_min      | 0.1363979    | 0.3466439  | 0.0        | 8.8       | 2835   |
# | size_angle    | 101.51163    | 1394.6456  | -89        | 32767     | 2752   |
# | B             | 19.484404    | 2.4215317  | 8.09       | 45.38     | 285678 |
# | V             | 18.517584    | 2.1595938  | 16.001     | 38.76     | 796552 |
# | R             | 19.15771     | 3.812743   | 2.383      | 30.24     | 81507  |
# | J             | 14.814598    | 2.984647   | 1.715      | 30.012    | 372305 |
# | H             | 14.025344    | 3.5139523  | 0.075      | 30.394    | 348900 |
# | K             | 13.783096    | 3.396141   | -0.935     | 28.074    | 353419 |
# | u             | 22.153986    | 3.8076148  | 14.7       | 30.22     | 26892  |
# | g             | 18.078993    | 1.7257268  | 13.595     | 29.39     | 51023  |
# | r             | 17.53991     | 1.7453885  | 12.383     | 29.39     | 52243  |
# | i             | 19.393364    | 4.0448337  | 9.146      | 31.338    | 60353  |
# | z             | 20.030676    | 3.8631122  | 11.749     | 30.431    | 45193  |
# +---------------+--------------+------------+------------+-----------+--------+

echo "- STATS on $CAT_SIMBAD_OBJ :"
cat $CAT_SIMBAD_OBJ.stats.log
#
# Total Rows: 10143829
# +---------------+--------------+------------+------------+-------------+----------+
# | column        | mean         | stdDev     | min        | max         | good     |
# +---------------+--------------+------------+------------+-------------+----------+
# | main_id       |              |            |            |             | 10143829 |
# | ra            | 202.9761     | 94.95882   | 0.0        | 359.9999    | 10143829 |
# | dec           | -7.4612103   | 39.054325  | -89.88967  | 90.0        | 10143829 |
# | coo_err_maj   | 14.102424    | 1486.8291  | 0.0        | 636847.25   | 9183006  |
# | coo_err_min   | 13.692797    | 1486.7948  | 0.0        | 636847.25   | 9183004  |
# | coo_err_angle | 95.932045    | 27.929405  | 0.0        | 179.0       | 10143829 |
# | otype         |              |            |            |             | 10143829 |
# | nbref         | 2.1964016    | 10.599808  | 0          | 15045       | 10143829 |
# | ra_sexa       |              |            |            |             | 10143829 |
# | dec_sexa      |              |            |            |             | 10143829 |
# | coo_qual      |              |            |            |             | 10143829 |
# | coo_bibcode   |              |            |            |             | 10083249 |
# | main_type     |              |            |            |             | 10143826 |
# | other_types   |              |            |            |             | 10137086 |
# | radvel        | 5713.625     | 26780.44   | -76618.44  | 299297.47   | 4542716  |
# | radvel_err    | 379.70288    | 27770.44   | -9.99      | 3.5740656E7 | 4287584  |
# | redshift      | 0.033616997  | 0.25606933 | -9.99898   | 33.79       | 4542731  |
# | redshift_err  | 0.0012665507 | 0.09263221 | -3.33E-5   | 119.217995  | 4287584  |
# | sp_type       |              |            |            |             | 1121637  |
# | morph_type    |              |            |            |             | 12       |
# | plx           | 1.9551224    | 3.6308615  | -12.4      | 768.0665    | 7179451  |
# | plx_err       | 0.11693077   | 0.23993146 | 0.0        | 72.3        | 7177994  |
# | pmra          | -2.07751     | 30.7996    | -8123.7    | 6765.995    | 8208879  |
# | pmdec         | -6.269406    | 29.352655  | -5817.8    | 10362.394   | 8208879  |
# | pm_err_maj    | 0.18024312   | 1.945418   | 0.0        | 3852.0      | 8206450  |
# | pm_err_min    | 0.15986364   | 0.8956201  | 0.0        | 787.0       | 8206447  |
# | pm_err_pa     | 256.61533    | 2327.96    | 0          | 32767       | 8248421  |
# | size_maj      | 30.369839    | 947.65375  | 0.0        | 429050.0    | 243892   |
# | size_min      | 29.851944    | 947.3677   | 0.0        | 429050.0    | 243891   |
# | size_angle    | 198.33858    | 1937.1305  | -173       | 32767       | 237908   |
# | B             | 12.854159    | 2.0514386  | -1.46      | 36.65       | 3824281  |
# | V             | 11.845591    | 1.5958984  | -1.46      | 16.0        | 3698828  |
# | R             | 13.536812    | 2.0938969  | -1.46      | 50.0        | 1359116  |
# | J             | 11.088817    | 2.2169862  | -3.0       | 31.108      | 6324547  |
# | H             | 10.42615     | 2.2236588  | -3.73      | 31.13       | 6343053  |
# | K             | 10.188389    | 2.255132   | -4.05      | 29.58       | 6349407  |
# | u             | 20.734592    | 2.5709207  | 0.003      | 32.0        | 533051   |
# | g             | 15.831853    | 3.2667122  | 0.008      | 33.7        | 1935471  |
# | r             | 14.961531    | 3.1235623  | 0.003      | 32.7        | 1953733  |
# | i             | 14.2348795   | 3.2410936  | 0.004      | 32.83       | 1629745  |
# | z             | 17.633728    | 2.1669247  | 0.004      | 31.866      | 577789   |
# | ra_s_91       | 202.9761     | 94.958824  | 0.0        | 359.99994   | 10143829 |
# | de_s_91       | -7.4611983   | 39.05433   | -89.889656 | 90.0        | 10143829 |
# +---------------+--------------+------------+------------+-------------+----------+

#TODO: below stats
#exit 1


# SIMBAD: internal match within 3 as to identify inner groups and filter columns:
# (main_id, ra/dec 1991) only to identify groups
CAT_SIMBAD_MIN="1_min_$CAT_SIMBAD_OBJ"
if [ $CAT_SIMBAD_OBJ -nt $CAT_SIMBAD_MIN ]; then
    CAT_SIMBAD_MIN_0="1_min_0_$CAT_SIMBAD_OBJ"

    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_SIMBAD_OBJ cmd='progress ; keepcols "ra_s_91 de_s_91 main_id" ' out=$CAT_SIMBAD_MIN_0
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_SIMBAD_MIN_0 > $CAT_SIMBAD_MIN_0.stats.log

# Total Rows: 10161185
# +---------+------------+-----------+------------+-----------+----------+
# | column  | mean       | stdDev    | min        | max       | good     |
# +---------+------------+-----------+------------+-----------+----------+
# | ra_s_91 | 204.87096  | 96.242096 | 0.0        | 359.99994 | 10161185 |
# | de_s_91 | -6.5451155 | 39.15329  | -89.889656 | 90.0      | 10161185 |
# | main_id |            |           |            |           | 10161185 |
# +---------+------------+-----------+------------+-----------+----------+
# 

    # identify inner groups in SIMBAD:
    stilts ${STILTS_JAVA_OPTIONS} tmatch1 matcher='sky' params='3.0' values='ra_s_91 de_s_91' action='identify' in=$CAT_SIMBAD_MIN_0 out=$CAT_SIMBAD_MIN
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_SIMBAD_MIN cmd='progress ; colmeta -name SIMBAD_GROUP_SIZE GroupSize ; delcols "GroupID" ' out=$CAT_SIMBAD_MIN
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_SIMBAD_MIN > $CAT_SIMBAD_MIN.stats.log

# Total Rows: 10161185
# +-------------------+------------+-----------+------------+-----------+----------+
# | column            | mean       | stdDev    | min        | max       | good     |
# +-------------------+------------+-----------+------------+-----------+----------+
# | ra_s_91           | 204.87096  | 96.242096 | 0.0        | 359.99994 | 10161185 |
# | de_s_91           | -6.5451155 | 39.15329  | -89.889656 | 90.0      | 10161185 |
# | main_id           |            |           |            |           | 10161185 |
# | SIMBAD_GROUP_SIZE | 64.669334  | 415.8102  | 2          | 3537      | 288715   |
# +-------------------+------------+-----------+------------+-----------+----------+
# 
fi


# SIMBAD: column subset for candidates
CAT_SIMBAD_SUB="1_sub_$CAT_SIMBAD_OBJ"
if [ $CAT_SIMBAD_OBJ -nt $CAT_SIMBAD_SUB ]; then
    # sort V first to ensure brightest source in groups ...
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_SIMBAD_OBJ cmd='progress ; colmeta -name ra_s ra ; colmeta -name de_s dec ; keepcols "ra_s de_s ra_s_91 de_s_91 main_id sp_type main_type other_types V"; sort V ' out=$CAT_SIMBAD_SUB
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_SIMBAD_SUB > $CAT_SIMBAD_SUB.stats.log

# Total Rows: 10161185
# +-------------+------------+-----------+------------+-----------+----------+
# | column      | mean       | stdDev    | min        | max       | good     |
# +-------------+------------+-----------+------------+-----------+----------+
# | ra_s        | 204.87094  | 96.242096 | 0.0        | 359.9999  | 10161185 |
# | de_s        | -6.5451274 | 39.15329  | -89.88967  | 90.0      | 10161185 |
# | ra_s_91     | 204.87096  | 96.242096 | 0.0        | 359.99994 | 10161185 |
# | de_s_91     | -6.5451155 | 39.15329  | -89.889656 | 90.0      | 10161185 |
# | main_id     |            |           |            |           | 10161185 |
# | sp_type     |            |           |            |           | 989288   |
# | main_type   |            |           |            |           | 10161185 |
# | other_types |            |           |            |           | 10143119 |
# | V           | 11.897528  | 1.6359329 | -1.46      | 16.0      | 3761900  |
# +-------------+------------+-----------+------------+-----------+----------+
# 
fi


# MDFC: filter columns for candidates
CAT_MDFC_MIN="1_min_$CAT_MDFC"
if [ $CAT_MDFC -nt $CAT_MDFC_MIN ]; then
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_MDFC cmd='progress ; keepcols "Name RAJ2000 DEJ2000 IRflag med_Lflux disp_Lflux med_Mflux disp_Mflux med_Nflux disp_Nflux" ; colmeta -ucd IR_FLAG IRflag ; colmeta -name Lflux_med -ucd PHOT_FLUX_L med_Lflux ; addcol -after Lflux_med -ucd PHOT_FLUX_L_ERROR e_Lflux_med 1.4826*disp_Lflux ; colmeta -name Mflux_med -ucd PHOT_FLUX_M med_Mflux ; addcol -after Mflux_med -ucd PHOT_FLUX_M_ERROR e_Mflux_med 1.4826*disp_Mflux ; colmeta -name Nflux_med -ucd PHOT_FLUX_N med_Nflux ; addcol -after Nflux_med -ucd PHOT_FLUX_N_ERROR e_Nflux_med 1.4826*disp_Nflux; delcols "disp_Lflux disp_Mflux disp_Nflux" ' out=$CAT_MDFC_MIN
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_MDFC_MIN > $CAT_MDFC_MIN.stats.log

# Total Rows: 465857
# +-------------+------------+-----------+--------------+-----------+--------+
# | column      | mean       | stdDev    | min          | max       | good   |
# +-------------+------------+-----------+--------------+-----------+--------+
# | Name        |            |           |              |           | 465857 |
# | RAJ2000     |            |           |              |           | 465857 |
# | DEJ2000     |            |           |              |           | 465857 |
# | IRflag      | 1.0827936  | 1.554538  | 0            | 7         | 465857 |
# | Lflux_med   | 1.5254774  | 42.61173  | 1.6299028E-4 | 15720.791 | 465755 |
# | e_Lflux_med | 0.458417   | 23.706154 | 3.0239256E-8 | 11478.607 | 432072 |
# | Mflux_med   | 0.8892675  | 19.973076 | 8.886453E-5  | 7141.7    | 465757 |
# | e_Mflux_med | 0.33401743 | 14.302171 | 8.423023E-10 | 6363.4277 | 432130 |
# | Nflux_med   | 0.33924013 | 18.71475  | 9.8775825E-5 | 9278.1    | 465750 |
# | e_Nflux_med | 0.12537919 | 7.3004456 | 9.378568E-10 | 3538.2249 | 433969 |
# +-------------+------------+-----------+--------------+-----------+--------+
fi


# 2. Crossmatch ASCC / SIMBAD within 3 as to identify both groups
CAT_ASCC_SIMBAD_GRP="2_Grp_xmatch_ASCC_SIMBAD_full.fits"
if [ $CAT_ASCC_MIN_GRP -nt $CAT_ASCC_SIMBAD_GRP ] || [ $CAT_SIMBAD_MIN -nt $CAT_ASCC_SIMBAD_GRP ]; then
    CAT_ASCC_SIMBAD_GRP_0="2_Grp_xmatch_ASCC_SIMBAD_full_0.fits"

    # use best1 to get groups in SIMBAD for every ASCC source at epoch 1991.25
    stilts ${STILTS_JAVA_OPTIONS} tmatch2 matcher='sky' params='3.0' values1='ra_a_91 de_a_91' values2='ra_s_91 de_s_91' find='best1' join='all1' in1=$CAT_ASCC_MIN_GRP in2=$CAT_SIMBAD_MIN out=$CAT_ASCC_SIMBAD_GRP_0
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_SIMBAD_GRP_0 > ${CAT_ASCC_SIMBAD_GRP_0}.stats.log

# Total Rows: 2500764
# +-------------------+------------+------------+--------------+-----------+---------+
# | column            | mean       | stdDev     | min          | max       | good    |
# +-------------------+------------+------------+--------------+-----------+---------+
# | _RAJ2000          | 188.87296  | 100.47335  | 3.4094515E-4 | 359.99988 | 2500764 |
# | _DEJ2000          | -3.3170354 | 41.25193   | -89.88966    | 89.911354 | 2500764 |
# | pmRA              | -1.754506  | 32.229416  | -4417.35     | 6773.35   | 2500764 |
# | pmDE              | -6.086718  | 30.385551  | -5813.7      | 10308.08  | 2500764 |
# | Vmag              | 11.069625  | 1.1319602  | -1.121       | 16.746    | 2500626 |
# | ra_a_91           | 188.87297  | 100.47336  | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91           | -3.3170204 | 41.251934  | -89.88964    | 89.91136  | 2500764 |
# | ASCC_GROUP_SIZE   | 2.0283375  | 0.17042844 | 2            | 4         | 31760   |
# | ra_s_91           | 188.4954   | 100.38276  | 2.7943638E-4 | 359.99994 | 2473644 |
# | de_s_91           | -3.4155102 | 41.383537  | -89.889656   | 89.83232  | 2473644 |
# | main_id           |            |            |              |           | 2473644 |
# | SIMBAD_GROUP_SIZE | 2.7460039  | 3.0601673  | 2            | 647       | 55178   |
# | GroupID           | 2985.8335  | 1723.312   | 1            | 5970      | 11961   |
# | GroupSize         | 2.0052671  | 0.07238353 | 2            | 3         | 11961   |
# | Separation        | 0.10310553 | 0.14512001 | 1.8650917E-7 | 2.9982488 | 2473644 |
# +-------------------+------------+------------+--------------+-----------+---------+
# 

    # Only consider groups between ASCC x SIMBAD within 3as:
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC_SIMBAD_GRP_0 cmd='progress ; delcols "ra_s_91 de_s_91 main_id SIMBAD_GROUP_SIZE GroupID Separation" ; colmeta -name SIMBAD_GROUP_SIZE GroupSize ' out=$CAT_ASCC_SIMBAD_GRP
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_SIMBAD_GRP > $CAT_ASCC_SIMBAD_GRP.stats.log

# Total Rows: 2500764
# +-------------------+------------+------------+--------------+-----------+---------+
# | column            | mean       | stdDev     | min          | max       | good    |
# +-------------------+------------+------------+--------------+-----------+---------+
# | _RAJ2000          | 188.87296  | 100.47335  | 3.4094515E-4 | 359.99988 | 2500764 |
# | _DEJ2000          | -3.3170354 | 41.25193   | -89.88966    | 89.911354 | 2500764 |
# | pmRA              | -1.754506  | 32.229416  | -4417.35     | 6773.35   | 2500764 |
# | pmDE              | -6.086718  | 30.385551  | -5813.7      | 10308.08  | 2500764 |
# | Vmag              | 11.069625  | 1.1319602  | -1.121       | 16.746    | 2500626 |
# | ra_a_91           | 188.87297  | 100.47336  | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91           | -3.3170204 | 41.251934  | -89.88964    | 89.91136  | 2500764 |
# | ASCC_GROUP_SIZE   | 2.0283375  | 0.17042844 | 2            | 4         | 31760   |
# | SIMBAD_GROUP_SIZE | 2.0052671  | 0.07238353 | 2            | 3         | 11961   |
# +-------------------+------------+------------+--------------+-----------+---------+
# 
fi


# 2. Crossmatch ASCC / SIMBAD within 1.5 as to get both DATA
CAT_ASCC_SIMBAD_XMATCH="2_xmatch_ASCC_SIMBAD_full.fits"
if [ $CAT_ASCC_SIMBAD_GRP -nt $CAT_ASCC_SIMBAD_XMATCH ] || [ $CAT_SIMBAD_SUB -nt $CAT_ASCC_SIMBAD_XMATCH ]; then
    # use best to have the best ASCC x SIMBAD object: 
    # can not rely on SIMBAD V that may be missing or wrong ... (could help if missing value could be considered too)
    
    # xmatch on COO(1991.25):
    # note: simbad may have multiple entries (children) for X, X A, X B so main_id & sp_type may differ.
    stilts ${STILTS_JAVA_OPTIONS} tmatch2 matcher='sky' params='1.5' values1='ra_a_91 de_a_91' values2='ra_s_91 de_s_91' find='best' join='all1' in1=$CAT_ASCC_SIMBAD_GRP in2=$CAT_SIMBAD_SUB out=$CAT_ASCC_SIMBAD_XMATCH 
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC_SIMBAD_XMATCH cmd='progress ; addcol XM_SIMBAD_SEP_2000 "skyDistanceDegrees(_RAJ2000,_DEJ2000,ra_s,de_s) * 3600.0" ; addcol XM_SIMBAD_SEP_91 "skyDistanceDegrees(ra_a_91,de_a_91,ra_s_91,de_s_91) * 3600.0" ; delcols "ra_s de_s ra_s_91 de_s_91" ; colmeta -name XM_SIMBAD_SEP -ucd XM_SIMBAD_SEP Separation ' out=$CAT_ASCC_SIMBAD_XMATCH
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_SIMBAD_XMATCH > $CAT_ASCC_SIMBAD_XMATCH.stats.log

# Total Rows: 2500764
# +--------------------+------------+-------------+--------------+-----------+---------+
# | column             | mean       | stdDev      | min          | max       | good    |
# +--------------------+------------+-------------+--------------+-----------+---------+
# | _RAJ2000           | 188.87296  | 100.47335   | 3.4094515E-4 | 359.99988 | 2500764 |
# | _DEJ2000           | -3.3170354 | 41.25193    | -89.88966    | 89.911354 | 2500764 |
# | pmRA               | -1.754506  | 32.229416   | -4417.35     | 6773.35   | 2500764 |
# | pmDE               | -6.086718  | 30.385551   | -5813.7      | 10308.08  | 2500764 |
# | Vmag               | 11.069625  | 1.1319602   | -1.121       | 16.746    | 2500626 |
# | ra_a_91            | 188.87297  | 100.47336   | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91            | -3.3170204 | 41.251934   | -89.88964    | 89.91136  | 2500764 |
# | ASCC_GROUP_SIZE    | 2.0283375  | 0.17042844  | 2            | 4         | 31760   |
# | SIMBAD_GROUP_SIZE  | 2.0052671  | 0.07238353  | 2            | 3         | 11961   |
# | main_id            |            |             |              |           | 2462762 |
# | sp_type            |            |             |              |           | 508464  |
# | main_type          |            |             |              |           | 2462762 |
# | other_types        |            |             |              |           | 2462736 |
# | V                  | 11.067058  | 1.1076136   | -1.46        | 15.79     | 2434987 |
# | XM_SIMBAD_SEP      | 0.09782921 | 0.107170224 | 1.8650917E-7 | 1.499898  | 2462762 |
# | XM_SIMBAD_SEP_2000 | 0.11608709 | 0.1416425   | 2.2025993E-6 | 21.158558 | 2462762 |
# | XM_SIMBAD_SEP_91   | 0.09782921 | 0.107170224 | 1.8622927E-7 | 1.499898  | 2462762 |
# +--------------------+------------+-------------+--------------+-----------+---------+
# 
fi

# ~2500 rows where XM_SIMBAD_SEP_2000 > 1.5 as (mostly because of pmRa/De difference or ASCC mistake = wrong association)

# Compare crossmatch on 1991 vs 2000:
CAT_ASCC_SIMBAD_XMATCH_J2K="2_xmatch_ASCC_SIMBAD_full_J2000.fits"
if [ $CAT_ASCC_SIMBAD_XMATCH -nt $CAT_ASCC_SIMBAD_XMATCH_J2K ] || [ $CAT_SIMBAD_SUB -nt $CAT_ASCC_SIMBAD_XMATCH_J2K ]; then
    # use best to have the best ASCC x SIMBAD object: 
    # can not rely on SIMBAD V that may be missing or wrong ... (could help if missing value could be considered too)
    
    # xmatch on COO(J2000):
    stilts ${STILTS_JAVA_OPTIONS} tmatch2 matcher='sky' params='1.5' values1='_RAJ2000 _DEJ2000' values2='ra_s de_s' find='best' join='all1' in1=$CAT_ASCC_SIMBAD_XMATCH in2=$CAT_SIMBAD_SUB out=$CAT_ASCC_SIMBAD_XMATCH_J2K 
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC_SIMBAD_XMATCH_J2K cmd='progress ; addcol XM_SIMBAD_FLAG "!main_Id_1.equals(main_Id_2)?1:NULL" ; delcols "XM_SIMBAD_SEP_91 XM_SIMBAD_SEP_2000 ra_s de_s ra_s_91 de_s_91 main_id_2 sp_type_2 main_type_2 other_types_2 Separation" ; colmeta -name main_id main_id_1 ; colmeta -name sp_type sp_type_1 ; colmeta -name main_type main_type_1 ; colmeta -name other_types other_types_1 ; colmeta -name V V_1 ; colmeta -name V_SIMBAD V_2' out=$CAT_ASCC_SIMBAD_XMATCH_J2K
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_SIMBAD_XMATCH_J2K > $CAT_ASCC_SIMBAD_XMATCH_J2K.stats.log

# Total Rows: 2500764
# +-------------------+------------+-------------+--------------+-----------+---------+
# | column            | mean       | stdDev      | min          | max       | good    |
# +-------------------+------------+-------------+--------------+-----------+---------+
# | _RAJ2000          | 188.87296  | 100.47335   | 3.4094515E-4 | 359.99988 | 2500764 |
# | _DEJ2000          | -3.3170354 | 41.25193    | -89.88966    | 89.911354 | 2500764 |
# | pmRA              | -1.754506  | 32.229416   | -4417.35     | 6773.35   | 2500764 |
# | pmDE              | -6.086718  | 30.385551   | -5813.7      | 10308.08  | 2500764 |
# | Vmag              | 11.069625  | 1.1319602   | -1.121       | 16.746    | 2500626 |
# | ra_a_91           | 188.87297  | 100.47336   | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91           | -3.3170204 | 41.251934   | -89.88964    | 89.91136  | 2500764 |
# | ASCC_GROUP_SIZE   | 2.0283375  | 0.17042844  | 2            | 4         | 31760   |
# | SIMBAD_GROUP_SIZE | 2.0052671  | 0.07238353  | 2            | 3         | 11961   |
# | main_id           |            |             |              |           | 2462762 |
# | sp_type           |            |             |              |           | 508464  |
# | main_type         |            |             |              |           | 2462762 |
# | other_types       |            |             |              |           | 2462736 |
# | V                 | 11.067058  | 1.1076136   | -1.46        | 15.79     | 2434987 |
# | XM_SIMBAD_SEP     | 0.09782921 | 0.107170224 | 1.8650917E-7 | 1.499898  | 2462762 |
# | V_SIMBAD          | 11.065956  | 1.108435    | -1.46        | 15.88     | 2432423 |
# | XM_SIMBAD_FLAG    | 1.0        | 0.0         | 1            | 1         | 7207    |
# +-------------------+------------+-------------+--------------+-----------+---------+
# 
fi


# 2. Crossmatch ASCC / SIMBAD / MDFC (SIMBAD main_id):
CAT_ASCC_SIMBAD_MDFC_XMATCH="2_xmatch_ASCC_SIMBAD_MDFC_full.fits"
if [ $CAT_ASCC_SIMBAD_XMATCH_J2K -nt $CAT_ASCC_SIMBAD_MDFC_XMATCH ] || [ $CAT_MDFC_MIN -nt $CAT_ASCC_SIMBAD_MDFC_XMATCH ]; then
    stilts ${STILTS_JAVA_OPTIONS} tmatch2 matcher='exact' values1='main_id' values2='Name' join='all1' in1=$CAT_ASCC_SIMBAD_XMATCH_J2K in2=$CAT_MDFC_MIN out=$CAT_ASCC_SIMBAD_MDFC_XMATCH
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC_SIMBAD_MDFC_XMATCH cmd='progress ; delcols "RAJ2000 DEJ2000 Name" ' out=$CAT_ASCC_SIMBAD_MDFC_XMATCH
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_SIMBAD_MDFC_XMATCH > $CAT_ASCC_SIMBAD_MDFC_XMATCH.stats.log

# Total Rows: 2500764
# +-------------------+------------+-------------+--------------+-----------+---------+
# | column            | mean       | stdDev      | min          | max       | good    |
# +-------------------+------------+-------------+--------------+-----------+---------+
# | _RAJ2000          | 188.87296  | 100.47335   | 3.4094515E-4 | 359.99988 | 2500764 |
# | _DEJ2000          | -3.3170354 | 41.25193    | -89.88966    | 89.911354 | 2500764 |
# | pmRA              | -1.754506  | 32.229416   | -4417.35     | 6773.35   | 2500764 |
# | pmDE              | -6.086718  | 30.385551   | -5813.7      | 10308.08  | 2500764 |
# | Vmag              | 11.069625  | 1.1319602   | -1.121       | 16.746    | 2500626 |
# | ra_a_91           | 188.87297  | 100.47336   | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91           | -3.3170204 | 41.251934   | -89.88964    | 89.91136  | 2500764 |
# | ASCC_GROUP_SIZE   | 2.0283375  | 0.17042844  | 2            | 4         | 31760   |
# | SIMBAD_GROUP_SIZE | 2.0052671  | 0.07238353  | 2            | 3         | 11961   |
# | main_id           |            |             |              |           | 2462762 |
# | sp_type           |            |             |              |           | 508464  |
# | main_type         |            |             |              |           | 2462762 |
# | other_types       |            |             |              |           | 2462736 |
# | V                 | 11.067058  | 1.1076136   | -1.46        | 15.79     | 2434987 |
# | XM_SIMBAD_SEP     | 0.09782921 | 0.107170224 | 1.8650917E-7 | 1.499898  | 2462762 |
# | V_SIMBAD          | 11.065956  | 1.108435    | -1.46        | 15.88     | 2432423 |
# | XM_SIMBAD_FLAG    | 1.0        | 0.0         | 1            | 1         | 7207    |
# | IRflag            | 1.0827082  | 1.5544553   | 0            | 7         | 465818  |
# | Lflux_med         | 1.5206355  | 42.534527   | 1.6299028E-4 | 15720.791 | 465717  |
# | e_Lflux_med       | 0.45798847 | 23.706205   | 3.0239256E-8 | 11478.607 | 432047  |
# | Mflux_med         | 0.8862267  | 19.91887    | 8.886453E-5  | 7141.7    | 465720  |
# | e_Mflux_med       | 0.3335035  | 14.301642   | 8.423023E-10 | 6363.4277 | 432102  |
# | Nflux_med         | 0.33530098 | 18.591125   | 9.8775825E-5 | 9278.1    | 465712  |
# | e_Nflux_med       | 0.12085117 | 6.6976385   | 9.378568E-10 | 3538.2249 | 433938  |
# +-------------------+------------+-------------+--------------+-----------+---------+
# 
fi

# xmatch on RA/DE (J2000) within 1.5as (as mdfc coords are different):

CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST="2_xmatch_ASCC_SIMBAD_MDFC_full_dist.fits"
if [ $CAT_ASCC_SIMBAD_XMATCH_J2K -nt $CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST ] || [ $CAT_MDFC_MIN -nt $CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST ]; then
    stilts ${STILTS_JAVA_OPTIONS} tmatch2 matcher='sky' params='1.5' values1='_RAJ2000 _DEJ2000' values2='hmsToDegrees(RAJ2000) dmsToDegrees(DEJ2000)' find='best' join='all1' in1=$CAT_ASCC_SIMBAD_XMATCH_J2K in2=$CAT_MDFC_MIN out=$CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST cmd='progress ; delcols "RAJ2000 DEJ2000 Name Separation" ' out=$CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST > $CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST.stats.log

# Total Rows: 2500764
# +-------------------+------------+-------------+--------------+-----------+---------+
# | column            | mean       | stdDev      | min          | max       | good    |
# +-------------------+------------+-------------+--------------+-----------+---------+
# | _RAJ2000          | 188.87296  | 100.47335   | 3.4094515E-4 | 359.99988 | 2500764 |
# | _DEJ2000          | -3.3170354 | 41.25193    | -89.88966    | 89.911354 | 2500764 |
# | pmRA              | -1.754506  | 32.229416   | -4417.35     | 6773.35   | 2500764 |
# | pmDE              | -6.086718  | 30.385551   | -5813.7      | 10308.08  | 2500764 |
# | Vmag              | 11.069625  | 1.1319602   | -1.121       | 16.746    | 2500626 |
# | ra_a_91           | 188.87297  | 100.47336   | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91           | -3.3170204 | 41.251934   | -89.88964    | 89.91136  | 2500764 |
# | ASCC_GROUP_SIZE   | 2.0283375  | 0.17042844  | 2            | 4         | 31760   |
# | SIMBAD_GROUP_SIZE | 2.0052671  | 0.07238353  | 2            | 3         | 11961   |
# | main_id           |            |             |              |           | 2462762 |
# | sp_type           |            |             |              |           | 508464  |
# | main_type         |            |             |              |           | 2462762 |
# | other_types       |            |             |              |           | 2462736 |
# | V                 | 11.067058  | 1.1076136   | -1.46        | 15.79     | 2434987 |
# | XM_SIMBAD_SEP     | 0.09782921 | 0.107170224 | 1.8650917E-7 | 1.499898  | 2462762 |
# | V_SIMBAD          | 11.065956  | 1.108435    | -1.46        | 15.88     | 2432423 |
# | XM_SIMBAD_FLAG    | 1.0        | 0.0         | 1            | 1         | 7207    |
# | IRflag            | 1.0827082  | 1.5544553   | 0            | 7         | 465818  |
# | Lflux_med         | 1.5206355  | 42.534527   | 1.6299028E-4 | 15720.791 | 465717  |
# | e_Lflux_med       | 0.45798847 | 23.706205   | 3.0239256E-8 | 11478.607 | 432047  |
# | Mflux_med         | 0.8862267  | 19.91887    | 8.886453E-5  | 7141.7    | 465720  |
# | e_Mflux_med       | 0.3335035  | 14.301642   | 8.423023E-10 | 6363.4277 | 432102  |
# | Nflux_med         | 0.33530098 | 18.591125   | 9.8775825E-5 | 9278.1    | 465712  |
# | e_Nflux_med       | 0.12085117 | 6.6976385   | 9.378568E-10 | 3538.2249 | 433938  |
# +-------------------+------------+-------------+--------------+-----------+---------+
# 
fi

# Use xmatch(COO) for MDFC:
CAT_ASCC_SIMBAD_MDFC_XMATCH=$CAT_ASCC_SIMBAD_MDFC_XMATCH_DIST


# 3. Prepare candidates: columns, sort, then filters:
FULL_CANDIDATES="3_xmatch_ASCC_SIMBAD_MDFC_full_cols.fits"
if [ $CAT_ASCC_SIMBAD_MDFC_XMATCH -nt $FULL_CANDIDATES ]; then
    # note: merge (main_type,other_types) as other_types can be NULL:
    # convert XM_SIMBAD_FLAG (bad xm on 91 vs 2000?) into GROUP_SIZE_3:
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CAT_ASCC_SIMBAD_MDFC_XMATCH cmd='progress ; addcol -after sp_type -ucd OBJ_TYPES OTYPES NULL_main_type?\"\":replaceAll(concat(\",\",main_type,\",\",replaceAll(other_types,\"[|]\",\",\"),\",\"),\"\ \",\"\") ; delcols "main_type other_types Vmag V" ; addcol -before _RAJ2000 -ucd POS_EQ_RA_MAIN RAJ2000 degreesToHms(_RAJ2000,6) ; addcol -after RAJ2000 -ucd POS_EQ_DEC_MAIN DEJ2000 degreesToDms(_DEJ2000,6) ; colmeta -ucd POS_EQ_PMRA pmRA ; colmeta -ucd POS_EQ_PMDEC pmDE ; colmeta -name MAIN_ID -ucd ID_MAIN main_id ; colmeta -name SP_TYPE -ucd SPECT_TYPE_MK sp_type ; addcol -after XM_SIMBAD_SEP -ucd GROUP_SIZE GROUP_SIZE_3 max((!NULL_ASCC_GROUP_SIZE?ASCC_GROUP_SIZE:0),(!NULL_SIMBAD_GROUP_SIZE?SIMBAD_GROUP_SIZE:0))+(!NULL_XM_SIMBAD_FLAG?1:0) ; sort _DEJ2000 ; delcols "_RAJ2000 _DEJ2000 ASCC_GROUP_SIZE SIMBAD_GROUP_SIZE XM_SIMBAD_FLAG" ' out=$FULL_CANDIDATES
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$FULL_CANDIDATES > $FULL_CANDIDATES.stats.log

# Total Rows: 2500764
# +---------------+-------------+------------+--------------+-----------+---------+
# | column        | mean        | stdDev     | min          | max       | good    |
# +---------------+-------------+------------+--------------+-----------+---------+
# | RAJ2000       |             |            |              |           | 2500764 |
# | DEJ2000       |             |            |              |           | 2500764 |
# | pmRA          | -1.754506   | 32.229416  | -4417.35     | 6773.35   | 2500764 |
# | pmDE          | -6.086718   | 30.385551  | -5813.7      | 10308.08  | 2500764 |
# | ra_a_91       | 188.87297   | 100.47336  | 2.8266E-4    | 359.9999  | 2500764 |
# | de_a_91       | -3.3170204  | 41.251934  | -89.88964    | 89.91136  | 2500764 |
# | MAIN_ID       |             |            |              |           | 2428771 |
# | SP_TYPE       |             |            |              |           | 497178  |
# | OTYPES        |             |            |              |           | 2428771 |
# | XM_SIMBAD_SEP | 0.108560905 | 0.12304193 | 8.73148E-6   | 1.382638  | 2428771 |
# | GROUP_SIZE_3  | 0.026821403 | 0.2305579  | 0            | 4         | 2500764 |
# | IRflag        | 1.0827082   | 1.5544553  | 0            | 7         | 465818  |
# | Lflux_med     | 1.5206355   | 42.534527  | 1.6299028E-4 | 15720.791 | 465717  |
# | e_Lflux_med   | 0.45798847  | 23.706205  | 3.0239256E-8 | 11478.607 | 432047  |
# | Mflux_med     | 0.8862267   | 19.91887   | 8.886453E-5  | 7141.7    | 465720  |
# | e_Mflux_med   | 0.3335035   | 14.301642  | 8.423023E-10 | 6363.4277 | 432102  |
# | Nflux_med     | 0.33530098  | 18.591125  | 9.8775825E-5 | 9278.1    | 465712  |
# | e_Nflux_med   | 0.12085117  | 6.6976385  | 9.378568E-10 | 3538.2249 | 433938  |
# +---------------+-------------+------------+--------------+-----------+---------+
fi


# Prepare candidate catalogs using the SearchCal cfg format:

# Convert to VOTables:
# - bright: select stars WITH SP_TYPE + neighbours
CATALOG_BRIGHT="3_xmatch_ASCC_SIMBAD_MDFC_full_sptype.vot"
if [ $FULL_CANDIDATES -nt $CATALOG_BRIGHT ]; then
    CATALOG_BRIGHT_0="3_xmatch_ASCC_SIMBAD_MDFC_full_sptype_0.fits"

    # Get all objects with SP_TYPE:
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$FULL_CANDIDATES cmd='progress; select !NULL_SP_TYPE ' out=$CATALOG_BRIGHT_0
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CATALOG_BRIGHT_0 > ${CATALOG_BRIGHT_0}.stats.log

# Total Rows: 508464
# +---------------+-------------+-------------+--------------+-----------+--------+
# | column        | mean        | stdDev      | min          | max       | good   |
# +---------------+-------------+-------------+--------------+-----------+--------+
# | RAJ2000       |             |             |              |           | 508464 |
# | DEJ2000       |             |             |              |           | 508464 |
# | pmRA          | -1.6801732  | 53.189293   | -4417.35     | 6773.35   | 508464 |
# | pmDE          | -8.926331   | 50.0674     | -5813.06     | 10308.08  | 508464 |
# | ra_a_91       | 191.13023   | 100.668076  | 2.8266E-4    | 359.99915 | 508464 |
# | de_a_91       | -0.35351607 | 40.475056   | -89.831245   | 89.77407  | 508464 |
# | MAIN_ID       |             |             |              |           | 508464 |
# | SP_TYPE       |             |             |              |           | 508464 |
# | OTYPES        |             |             |              |           | 508464 |
# | XM_SIMBAD_SEP | 0.043346085 | 0.066441275 | 1.8650917E-7 | 1.497625  | 508464 |
# | GROUP_SIZE_3  | 0.046644405 | 0.31546915  | 0            | 5         | 508464 |
# | V_SIMBAD      | 9.719448    | 1.2888868   | -1.46        | 15.88     | 499906 |
# | IRflag        | 1.0780823   | 1.5560942   | 0            | 7         | 449423 |
# | Lflux_med     | 1.5545044   | 43.29874    | 1.6299028E-4 | 15720.791 | 449325 |
# | e_Lflux_med   | 0.46855867  | 24.129604   | 3.0239256E-8 | 11478.607 | 416933 |
# | Mflux_med     | 0.9054484   | 20.274939   | 8.886453E-5  | 7141.7    | 449329 |
# | e_Mflux_med   | 0.34119678  | 14.555591   | 8.423023E-10 | 6363.4277 | 416987 |
# | Nflux_med     | 0.34295824  | 18.921856   | 9.8775825E-5 | 9278.1    | 449321 |
# | e_Nflux_med   | 0.12372045  | 6.8172946   | 9.378568E-10 | 3538.2249 | 418763 |
# +---------------+-------------+-------------+--------------+-----------+--------+
# 

    # Get all neightbours within 3 as:
    # use all to get all objects within 3 as in ASCC FULL for every ASCC BRIGHT source at epoch 1991.25
    stilts ${STILTS_JAVA_OPTIONS} tmatch2 matcher='sky' params='3.0' values1='ra_a_91 de_a_91' values2='ra_a_91 de_a_91' find='all' join='all1' fixcols='all' suffix2='' in1=$CATALOG_BRIGHT_0 in2=$FULL_CANDIDATES out=$CATALOG_BRIGHT
    # remove duplicates (on de_a_91)
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$CATALOG_BRIGHT cmd='progress ; sort de_a_91 ; uniq de_a_91 ; delcols "*_1 ra_a_91 de_a_91 GroupID GroupSize Separation" ' out=$CATALOG_BRIGHT
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CATALOG_BRIGHT > ${CATALOG_BRIGHT}.stats.log

# Total Rows: 518624
# +---------------+------------+------------+--------------+-----------+--------+
# | column        | mean       | stdDev     | min          | max       | good   |
# +---------------+------------+------------+--------------+-----------+--------+
# | RAJ2000       |            |            |              |           | 518624 |
# | DEJ2000       |            |            |              |           | 518624 |
# | pmRA          | -1.671666  | 53.731403  | -4417.35     | 6773.35   | 518624 |
# | pmDE          | -9.020842  | 50.7888    | -5813.06     | 10308.08  | 518624 |
# | MAIN_ID       |            |            |              |           | 513388 |
# | SP_TYPE       |            |            |              |           | 508464 |
# | OTYPES        |            |            |              |           | 513388 |
# | XM_SIMBAD_SEP | 0.04336556 | 0.06696961 | 1.8650917E-7 | 1.497625  | 513388 |
# | GROUP_SIZE_3  | 0.08555331 | 0.417141   | 0            | 5         | 518624 |
# | V_SIMBAD      | 9.720669   | 1.2878852  | -1.46        | 15.88     | 503965 |
# | IRflag        | 1.0780928  | 1.5560931  | 0            | 7         | 449427 |
# | Lflux_med     | 1.5546576  | 43.298584  | 1.6299028E-4 | 15720.791 | 449329 |
# | e_Lflux_med   | 0.4686577  | 24.129526  | 3.0239256E-8 | 11478.607 | 416937 |
# | Mflux_med     | 0.9055667  | 20.274897  | 8.886453E-5  | 7141.7    | 449333 |
# | e_Mflux_med   | 0.3412708  | 14.555553  | 8.423023E-10 | 6363.4277 | 416991 |
# | Nflux_med     | 0.3429848  | 18.921776  | 9.8775825E-5 | 9278.1    | 449325 |
# | e_Nflux_med   | 0.12372719 | 6.8172626  | 9.378568E-10 | 3538.2249 | 418767 |
# +---------------+------------+------------+--------------+-----------+--------+
# 
fi


# - faint: select stars WITHOUT SP_TYPE nor neighbours (not in CATALOG_BRIGHT)
CATALOG_FAINT="3_xmatch_ASCC_SIMBAD_MDFC_full_NO_sptype.vot"
if [ $FULL_CANDIDATES -nt $CATALOG_FAINT ]; then
    CATALOG_FAINT_0="3_xmatch_ASCC_SIMBAD_MDFC_full_NO_sptype_0.fits"

    # delete any MDFC columns as it should correspond to JSDC BRIGHT
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$FULL_CANDIDATES cmd='progress; delcols "ra_a_91 de_a_91 SP_TYPE IRflag Lflux_med e_Lflux_med Mflux_med e_Mflux_med Nflux_med e_Nflux_med" ' out=$CATALOG_FAINT_0
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CATALOG_FAINT_0 > ${CATALOG_FAINT_0}.stats.log

# Total Rows: 2500764
# +---------------+-------------+-------------+--------------+----------+---------+
# | column        | mean        | stdDev      | min          | max      | good    |
# +---------------+-------------+-------------+--------------+----------+---------+
# | RAJ2000       |             |             |              |          | 2500764 |
# | DEJ2000       |             |             |              |          | 2500764 |
# | pmRA          | -1.754506   | 32.229416   | -4417.35     | 6773.35  | 2500764 |
# | pmDE          | -6.086718   | 30.385551   | -5813.7      | 10308.08 | 2500764 |
# | MAIN_ID       |             |             |              |          | 2462762 |
# | OTYPES        |             |             |              |          | 2462762 |
# | XM_SIMBAD_SEP | 0.09782921  | 0.107170224 | 1.8650917E-7 | 1.499898 | 2462762 |
# | GROUP_SIZE_3  | 0.028648445 | 0.24030682  | 0            | 5        | 2500764 |
# | V_SIMBAD      | 11.065956   | 1.108435    | -1.46        | 15.88    | 2432423 |
# +---------------+-------------+-------------+--------------+----------+---------+
# 
    # use DEJ2000 (unique)
    stilts ${STILTS_JAVA_OPTIONS} tmatch2 matcher='exact' values1='DEJ2000' values2='DEJ2000' join='1not2' in1=$CATALOG_FAINT_0 in2=$CATALOG_BRIGHT out=$CATALOG_FAINT
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CATALOG_FAINT > $CATALOG_FAINT.stats.log

# Total Rows: 1982140
# +---------------+-------------+-------------+--------------+----------+---------+
# | column        | mean        | stdDev      | min          | max      | good    |
# +---------------+-------------+-------------+--------------+----------+---------+
# | RAJ2000       |             |             |              |          | 1982140 |
# | DEJ2000       |             |             |              |          | 1982140 |
# | pmRA          | -1.7761809  | 23.560997   | -3681.47     | 4147.35  | 1982140 |
# | pmDE          | -5.3190317  | 22.070175   | -5813.7      | 3203.63  | 1982140 |
# | MAIN_ID       |             |             |              |          | 1949374 |
# | OTYPES        |             |             |              |          | 1949374 |
# | XM_SIMBAD_SEP | 0.11217282  | 0.111095354 | 1.6588735E-6 | 1.499898 | 1949374 |
# | GROUP_SIZE_3  | 0.013759371 | 0.16204742  | 0            | 4        | 1982140 |
# | V_SIMBAD      | 11.41752    | 0.7209013   | 2.22         | 15.79    | 1928458 |
# +---------------+-------------+-------------+--------------+----------+---------+
# 
fi


# - test: select only stars with neighbours (GROUP_SIZE_3 > 0)
CATALOG_TEST="3_xmatch_ASCC_SIMBAD_MDFC_full_test.vot"
if [ $FULL_CANDIDATES -nt $CATALOG_TEST ]; then
    stilts ${STILTS_JAVA_OPTIONS} tpipe in=$FULL_CANDIDATES cmd='progress; select GROUP_SIZE_3>0 ; delcols "ra_a_91 de_a_91 IRflag Lflux_med e_Lflux_med Mflux_med e_Mflux_med Nflux_med e_Nflux_med" ' out=$CATALOG_TEST
    stilts ${STILTS_JAVA_OPTIONS} tcat omode="stats" in=$CATALOG_TEST > $CATALOG_TEST.stats.log

# Total Rows: 37101
# +---------------+-------------+------------+--------------+----------+-------+
# | column        | mean        | stdDev     | min          | max      | good  |
# +---------------+-------------+------------+--------------+----------+-------+
# | RAJ2000       |             |            |              |          | 37101 |
# | DEJ2000       |             |            |              |          | 37101 |
# | pmRA          | -1.05295    | 87.421394  | -3678.16     | 2183.43  | 37101 |
# | pmDE          | -11.266943  | 81.37067   | -3567.01     | 1806.86  | 37101 |
# | MAIN_ID       |             |            |              |          | 30740 |
# | SP_TYPE       |             |            |              |          | 11386 |
# | OTYPES        |             |            |              |          | 30740 |
# | XM_SIMBAD_SEP | 0.103412405 | 0.25830042 | 1.8650917E-7 | 1.499898 | 30740 |
# | GROUP_SIZE_3  | 1.9310261   | 0.4678412  | 1            | 5        | 37101 |
# | V_SIMBAD      | 9.780439    | 1.5308161  | -0.1         | 15.88    | 23697 |
# +---------------+-------------+------------+--------------+----------+-------+
# 
fi


# 4. Generate SearchCal config files:
CONFIG="vobsascc_simbad_sptype.cfg"
if [ $CATALOG_BRIGHT -nt $CONFIG ]; then
    xsltproc -o $CONFIG sclguiVOTableToTSV.xsl $CATALOG_BRIGHT
fi

CONFIG="vobsascc_simbad_no_sptype.cfg"
if [ $CATALOG_FAINT -nt $CONFIG ]; then
    xsltproc -o $CONFIG sclguiVOTableToTSV.xsl $CATALOG_FAINT
fi

CONFIG="vobsascc_simbad_test.cfg"
if [ $CATALOG_TEST -nt $CONFIG ]; then
    xsltproc -o $CONFIG sclguiVOTableToTSV.xsl $CATALOG_TEST
fi


echo "That's all folks ! @ date: `date -u`"


