#!/bin/bash set -eu # exit on most (not all!) error codes, and error on unset variables export LC_ALL=C # set locale for computation with awk ###################################################################### # Author : Frédéric Hourdin (LMDZ team) # 2019/02/21 # Running automatically a series of SCM experiments with LMDZ # Launched from HighTune/bench.sh # Modifications: # 17/07/2025 - N. Villefranque - source env_LMDZ.profile ###################################################################### # Set number of used cores for parallel simulations: defaults to max available physical cores, but can be overriden by exporting $HTUNE_SERIE_NPROCS nproc=$(( $(lscpu -e=core | sort | uniq | wc -l) - 1 )) set +u if [[ $HTUNE_SERIE_NPROCS -gt 0 ]]; then if [[ $HTUNE_SERIE_NPROCS -ge $nproc ]]; then echo "Requested number of procs $HTUNE_SERIE_NPROCS clamped to max available procs N=$nproc" HTUNE_SERIE_NPROCS=$nproc fi nproc=$HTUNE_SERIE_NPROCS fi echo "Using $nproc procs for $0" set -u local=$(pwd) echo $local install_dir=$local/../../../install_shared source $install_dir/base_env.profile source $install_dir/env_LMDZ.profile WDIR=_$(basename $local) LMAX=79 phys="NPv6.2" CTRL="$local/CTRL" ecrad_dir=$local/../../models/ECRAD/ nam="namelist_ecrad-default" #ecrad namlist name stored in models/ECRAD/runECRAD path_data="data" #path for ecrad data from the directory where LMDZ is running ########### ## THIS IS NO LONGER USEFUL BECAUSE THESE VARIABLES ARE NOW DEFINED IN # env_LMDZ.profile which is sourced here above # ## LMDZ path, modified by setup_LMDZ.sh #LMDZdir=/home/hourdin/ITUNE/R480/htexplo/../LMDZ20240722.trunk ## rad, modified by setup_LMDZ.sh #rad="oldrad" ###### ###################################################################### # Reading arguments ###################################################################### if [[ $# -lt 1 ]] ; then echo "run on $local" bash serie_LMDZ.sh --help exit 1 fi debug=0 opt_LMDZ_read="" while (( $# > 0 )) ; do case $1 in -debug) debug=1; shift ;; # useful for ecRad runs -nproc) nproc=$2 ; opt_LMDZ_read="$opt_LMDZ_read -nproc $nproc" ; shift ; shift ;; -rad) rad=$2 ; opt_LMDZ_read="$opt_LMDZ_read -rad $rad" ; shift ; shift ;; -LMAX) LMAX=$2 ; opt_LMDZ_read="$opt_LMDZ_read -LMAX $LMAX" ; shift ; shift ;; -phys) phys=$2 ; opt_LMDZ_read="$opt_LMDZ_read -phys $phys" ; shift ; shift ;; -h|-help|--help) echo Usage: $0 "[-nproc N] [-debug] [-rad rad] [-LMAX LMAX] [-phys phys] [- CASE1/SUBCASE1 [CASE2/SUBCASE2 ...] wave_number" ; exit 1 ;; *) listecas=${*:1:$#-1} # all but last arguments WAVEN=${*: -1} # last arg shift $# esac done echo "listecas $listecas wave $WAVEN" EXPE="$local/WAVE$WAVEN" if [[ $debug = 1 ]] ; then set -vx ; fi ###################################################################### # Various checks ###################################################################### if [[ ! -d $LMDZdir ]]; then echo "Directory $LMDZdir does not exist. Please modify variable \$LMDZdir" exit 1 fi if [[ ! -f $LMDZdir/1D/INPUT/PHYS/physiq.def_$phys ]] ; then echo "$LMDZdir/1D/INPUT/PHYS/physiq.def_$phys does not exists" exit 1 fi ##################################################################### # I. Creating def files if they do not exist ##################################################################### if [[ ! -d $EXPE/DEF ]] ; then ################################################################## # I.1 Getting parameter generated by the latin hypercube sampling ################################################################## if [[ ! -f $EXPE/Par1D_Wave$WAVEN.asc ]] ; then echo "File $EXPE/Par1D_Wave$WAVEN.asc does not exist." echo "Please modify variable \$EXPE or run param2R.sh or param2Rwave.sh on directory $EXPE" exit 1 fi sed -e 's/\"//g' "$EXPE/Par1D_Wave$WAVEN.asc" >| param.asc set +u params=() for i in $(head -1 param.asc) ; do params=( ${params[*]} $i ) done nl=$(wc -l param.asc | awk ' { print $1 } ') il=2 ##################################################################### # I.2 Creating physiq.def_ files and namelist_ecrad if needed ##################################################################### # For safety, temporary dir, moved to DEF only if all the def files have been created TMPdef=TMPdef$$ ; if [[ -d "$TMPdef" ]] ; then echo "$TMPdef exits so clean and rerun" ; else mkdir $TMPdef ; fi #echo il nl $il $nl while [[ $il -le $nl ]] ; do vals=() for i in $(sed -n -e ${il}p param.asc) ; do vals=( ${vals[*]} $i ) done #echo ${vals[*]} #echo ${#vals[*]} sim=${vals[0]} #----------------------------------------------------------------- # Preparing SCM simulations # copying and modiying input files #----------------------------------------------------------------- set -v names=$(head -1 param.asc | sed -e 's/\"//g' -e 's/ /,/g') vals=$(sed -n -e ${il}p param.asc | sed -e 's/\"//g' -e 's/ /,/g') pwd #echo $local/phydef2phydef.sh -names $names -vals $vals -input $LMDZdir/1D/INPUT/PHYS/physiq.def_$phys -output $TMPdef/physiq.def_$sim #echo $local/phydef2phydef.sh -model CONT -names $names -vals $vals -input $local/orchidee.def -output $TMPdef/orchidee.def_$sim "$local/phydef2phydef.sh" -names $names -vals $vals -input "$LMDZdir/1D/INPUT/PHYS/physiq.def_$phys" -output $TMPdef/physiq.def_$sim "$local/phydef2phydef.sh" -model CONT -names $names -vals $vals -input "$local/orchidee.def" -output $TMPdef/orchidee.def_$sim if [[ $rad == ecrad ]] ; then #if we run LMDZ with ecrad online if [[ $names == *"RAD"* ]] ; then #and their somes radiative parameters mkdir -p "${EXPE}/NAMECRAD" cp -f "$ecrad_dir/runECRAD/"*sh "$local" cp -f "$ecrad_dir/runECRAD/${nam}" "$local" "$local/nam2nam.sh" -names $names -vals $vals -input "$local/${nam}" -output "${EXPE}/NAMECRAD/namelist_ecrad_${sim}" sed_i -e "s#directory_name.*.#directory_name=\"$path_data\"#" "${EXPE}/NAMECRAD/namelist_ecrad_${sim}" fi fi case $rad in oldrad) NSW=2 ; iflag_rrtm=0 ;; rrtm) NSW=6 ; iflag_rrtm=1 ;; ecrad) NSW=6 ; iflag_rrtm=2 ;; *) NSW=6 ; iflag_rrtm=1 ;; esac sed_i -e 's/^NSW=.*$/NSW='$NSW'/' -e 's/iflag_rrtm=.*$/iflag_rrtm='$iflag_rrtm'/' $TMPdef/physiq.def_$sim set +v (( il = $il + 1 )) done mv "TMPdef$$" "$EXPE/DEF" fi # Stopping here if no case to be done if [[ "$listecas" = "" ]] ; then echo "No case to be run" ; exit 0 ; fi ###################################################################### # II. forcing recompilation of lmdz1d.e ###################################################################### dimfile="$LMDZdir/modipsl/modeles/LMDZ/libf/grid/dimensions.h" if [[ -f $dimfile ]] ; then echo "$dimfile exists meaning you may be compiling LMDZ somewhere else" echo "rm $dimfile manually and rerun." exit 1 fi cd "$LMDZdir/1D/bin/" #### # Amaury Barral 12/2024: Verrue nécessaire depuis le passage au nouvel install_lmdz if [[ ! -f $LMDZdir/modipsl/modeles/LMDZ/arch/arch-local.path ]]; then cd $LMDZdir/modipsl/modeles/LMDZ/arch/ cp ../arch.env arch-local.env cp ../arch.path arch-local.path cp ../arch.fcm arch-local.fcm cd - fi sed_i -e "s:^fcm=0:fcm=1:" compile #### echo "Compiling lmdz1d, output in $(pwd)/compile$$.out" ./compile -L "$LMAX" -rad "$rad" 2> "$(pwd)/compile$$.err" if [[ ! -f $LMDZdir/1D/bin/lmdz1d_${rad}_L$LMAX.e ]] ; then echo "Compilation failed in $0" ; exit 1 fi echo '####################################################################' echo '####################################################################' echo 'III. Loop on cases' echo '####################################################################' echo '####################################################################' for cas in $listecas ; do freq='hourly' optvar='' nday="" optvar_poches='wake_deltat wake_deltaq wake_omg' case $cas in ARMCU/REF) lmdzcas=arm_cu ;; RICO/REF) lmdzcas=rico ;; FIRE/REF) lmdzcas=FIRE/REF ;; BOMEX/REF) lmdzcas=bomex ;; AYOTTE/24SC) lmdzcas=ayotte_24SC ;; AYOTTE/05WC) lmdzcas=ayotte_05WC ;; IHOP/REF) lmdzcas=ihop ;; SANDU/SLOW) lmdzcas=sanduslow ;; SANDU/REF) lmdzcas=sanduref ;; AMMA/REF) lmdzcas=amma ; optvar="$optvar_poches" ;; SANDU/FAST) lmdzcas=sandufast ;; RCEOCE/REF) lmdzcas=rce_oce_les ; freq='daily' ; optvar="$optvar_poches" ; nday=44 ;; GABLS1/REF) lmdzcas=GABLS1/REF ; optvar=tke ;; GABLS4/STAGE3) lmdzcas=GABLS4/STAGE3 ;; GABLS4/STAGE3SHORT) lmdzcas=GABLS4/STAGE3-SHORT ;; CINDY/REF) lmdzcas=cindy3b ; freq='daily' ; nday=59 if [ ! -d $LMDZdir/1D/OLDCASES/$lmdzcas ] ; then # Creating cindy3b from cindynamo cp -r $LMDZdir/1D/OLDCASES/cindynamo $LMDZdir/1D/OLDCASES/$lmdzcas nday=48 sed_i -e 's/^calend=.*.$/calend=earth_365d/' \ -e 's/^dayref=.*.$/dayref=275/' \ $LMDZdir/1D/OLDCASES/$lmdzcas/run.def #cp -f $LMDZdir/1D/OLDCASES/cindynamo/setup/cas_nsa_3b.nc $LMDZdir/1D/OLDCASES/$lmdzcas/setup/cas.nc # On tourne pour le moment avec le cas.nc par defaut. cas_nsa_3b.nc ne semble pas fonctionner ... cd $LMDZdir/1D/OLDCASES/$lmdzcas/setup/ ; cp -f cas_nsa_3b.nc cas.nc ; cd - fi ;; *) echo "Case $cas cannot be found" && exit 1 ;; esac if [ "$(echo "$lmdzcas" | grep '/')" = "" ] ; then std_fmt=0 ; else std_fmt=1 ; fi echo '#######################################################################' echo ' III.1 Runing the original case' echo '#######################################################################' # Note that this should be consistent with EXECDIR defined in run.sh # here, $phys is definition e.g. NPv... whereas in run.sh $phys is physics # folder e.g. "lmd" and NPv.. is stored in $DEF EXECdir=$LMDZdir/1D/EXEC$WDIR/${phys}L${LMAX}${rad} CASEdir=$EXECdir/$lmdzcas if [ -d $CASEdir ] ; then # remove case if already there cd "$EXECdir" && mkdir -p POUB && mv "$lmdzcas" "POUB/$lmdzcas$$" fi cd $LMDZdir/1D # adapt run.sh and copy to run$lmdzcas.sh run_script="run$(echo $lmdzcas | sed -e 's:/::').sh" sed -e 's/L=79/L='"$LMAX"'/' -e 's/LLM=.*.$/LLM='"$LMAX"'/' -e 's/listedef=.*.$/listedef=\"'"$phys"'\"/' -e 's:listecas=.*.$:listecas="'"$lmdzcas"'":' -e 's/.*RUN THE DIAGNOSTICS.*$/exit/' -e 's/config=${DEF}L${L}.*phys.*dtsuf/config=${DEF}L${L}${phys_}$dtsuf$rad/' run.sh > "$run_script" chmod +x "./$run_script" # case was previously removed to POUB so it should be ran again echo "Case $lmdzcas does not exist on $EXECdir. Will be rerun automatically" ; sleep 3 cmd="./$run_script -rad $rad -wdir $WDIR" echo "running $cmd to setup the $cas case" #$cmd 2> `pwd`/run$$.err | tee `pwd`/run$$.out | grep "Do you wish to continue" $cmd 2> "$(pwd)/run$$.err" | tee "$(pwd)/run$$.out" if [[ ! -f $CASEdir/hourly.nc ]] ; then echo "Simulations run$cas was not successfull." echo "You should try to rerun \"$cmd\" on $LMDZdir/1D" echo "to understand the problem, and ask for help." fi cd - echo '#######################################################################' echo ' III.2 Modifying some of the .def files' echo '#######################################################################' cd "$CASEdir" # Controling frequencey of outputs echo "$cas" cp -f "$LMDZdir/1D/INPUT/DEF/config.def" "$LMDZdir/1D/INPUT/DEF/config_orig.def" sed \ -e 's/phys_out_filetimesteps=.*.$/phys_out_filetimesteps= '"$( if [[ "$freq" = "hourly" ]] ; then echo "1hr" ; else echo 1day ; fi )"' 1ts 1day 1hr 6hr 1ts/' \ -e 's/phys_out_filenames=.*.$/phys_out_filenames= hourly histhf day histins histLES filehf/' \ -e 's/phys_out_filekeys=.*.$/phys_out_filekeys= y n n n n n/' \ -e 's/phys_out_filelevels=.*.$/phys_out_filelevels= 5 1 10 10 10 0/' \ -e 's/phys_out_filetypes=.*.$/phys_out_filetypes= ave(X) inst(X) ave(X) inst(X) inst(X) inst(X)/' \ $LMDZdir/1D/INPUT/DEF/config.def > "$CASEdir/config.def" if [[ $std_fmt = 0 ]] ; then def_file="run.def" ; else def_file="cas.def" ; fi if [[ $nday = "" ]] ; then nday=$( grep '^nday/*=' "$CASEdir/$def_file" | cut -d= -f2 ) else sed_i -e 's/^nday.*.$/nday='$nday'/' "$CASEdir/$def_file" fi # The number of day of a single simulation. Needed for time management. echo "std_fmt $std_fmt" if [[ $nday -lt 0 ]] ; then nday=1 ; fi rm -rf SCM* ulimit -s unlimited DATA="$EXPE/$cas" ##################################################################### # III.3 Function run_SCM : run SCM and post-process toward standard format ##################################################################### function run_SCM { set -vx name=$1 sampling=0 # 1) Running SCM ------------------------------------------------------------ optvar_='' for var_ in $optvar ; do echo "flag_$var_= 0 , 10, 10, 10, 10, 10 " >> config.def optvar_="$optvar_$var_," echo "OPTVAR $optvar_" done time ./lmdz1d.e # -- one more chance if the model crashes for bad reasons (macOS) ------------ for _ in 1 ; do if [[ ! -f restartphy.nc ]] ; then echo "WARNING !!! Simulation $name crashes, one more try of lmdz1d.e" rm -f restartphy.nc ./lmdz1d.e fi done # 2) Post-processing toward Dephy standard # -- $sampling=1 : more internal variables of the termal plume model -------- if [[ $sampling = 0 ]] ; then ncks -O -v "${optvar_}theta,rhum,vitu,vitv,geop,phis,ovap,f_th,w_th,rneb,lwp,pres,re,temp,lwcon,iwcon,ref_liq,ref_ice" hourly.nc tmp.nc optvar__= for var_ in $optvar ; do var__=$( echo $var_ | sed -e 's/_//g' ) if [ "$var__" != "$var_" ] ; then ncrename -O -v "$var_,$var__" tmp.nc fi optvar__="$optvar__$var__," echo "OPTVAR2 $optvar_" done ncrename -O -v ovap,qv -v lwcon,qc -v iwcon,qi -v f_th,fm -v w_th,wa -v rhum,hur \ -v vitu,u -v vitv,v -v pres,pf -v presnivs,levf -d presnivs,levf \ -v time_counter,time -d time_counter,time tmp.nc ncap2 -O -s 'zf=(geop-phis)/9.8' tmp.nc tmp2.nc ncap2 -O -s 'ref_liq=ref_liq*1e-6' tmp2.nc tmp2.nc ncap2 -O -s 'ref_ice=ref_ice*1e-6' tmp2.nc tmp2.nc #computing cloud cover as max(rneb) ncwa -v rneb -a levf -y max tmp2.nc tmp2_cc.nc ncrename -v rneb,cc tmp2_cc.nc ncatted -a "long_name,cc,o,c,Cloud cover as max(rneb)" tmp2_cc.nc ncks -C -A -v cc tmp2_cc.nc tmp2.nc #computing incloud water liquid content ncap2 -O -s "qcin=qc/rneb" tmp2.nc tmp2_qcin.nc ncap2 -O -s "where(rneb<0.00001) qcin=0.0" tmp2_qcin.nc tmp2_qcin.nc ncatted -a "long_name,qcin,o,c,incloud water liquid content" tmp2_qcin.nc ncks -C -A -v qcin tmp2_qcin.nc tmp2.nc ncks -v "${optvar__}cc,qcin,zf,pf,re,ref_liq,ref_ice,temp,theta,hur,u,v,qv,qc,qi,fm,wa,rneb,lwp" tmp2.nc "$name.nc" if [[ $debug = 0 ]] ; then rm -rf "$(pwd)" ; fi fi # -- calendar issues -------------------------------------------------------- ncatted -a "calendar,time,o,c,standard" -O "$name.nc" if [[ $cas = "IHOP/REF" ]] ; then ncatted -a "units,time,o,c,seconds since 2002-06-14 05:59:59" \ -a "time_origin,time,o,c, 2002-JUN-14 05:59:59" -O "$name.nc" fi if [[ $cas = "RICO/REF" ]] ; then ncatted -a "units,time,o,c,seconds since 2004-12-27 00:00:00" \ -a "time_origin,time,o,c, 2004-DEC-27 00:00:00" -O "$name.nc" fi echo "end RUN_SCM" } ##################################################################### # III.3 Preparing a REF directroy containing the minimum needed for one # simulation. To be duplicated for each simultion ##################################################################### save_scm=0 if [[ $save_scm = 1 ]] ; then mv REF REF$$/ mv SCM* REF$$/ else rm -rf REF "SCM-${WAVEN}"-* fi mkdir REF cd REF ln -s ../* . rm -- sta* lim* physiq.def used* ferret* *pdf hourly.nc hist*nc paramLMDZ_phy.nc day.nc || true cp ../physiq.def . cd .. mkdir -p "$DATA" echo '####################################################################' echo ' III.4 starting the loop on simulations' echo '####################################################################' nsims=$(ls $EXPE/DEF/physiq.def_* | wc -w) if [[ ! -f $CTRL/$cas/SCM.nc ]] ; then if [[ -d CTRL ]] ; then mv CTRL CTRL$$ ; fi cp -r REF CTRL # Note: log path must be outside $(pwd) since run_SCM may delete $(pwd) echo "cible $CTRL/$cas/SCM, log: $CTRL/$cas/listing" cd CTRL ; mkdir -p "$CTRL/$cas" ; run_SCM "$CTRL/$cas/SCM" &> "$CTRL/$cas/listing"; cd .. fi # loop on simulations from the physiq.def files contained in DEF mapfile -t sims < <(cd "$EXPE"/DEF ; find ./physiq.def_* | sed -e 's%./physiq.def_%%') nsims=${#sims[@]} # The loop is splitted in pieces of $nproc_ simulations on parallel machines # nproc_ is taken as the min value of $nproc and the number of simulations nsims # The "wait" command is used to synchronize at the end of a series nproc_=$nproc if [[ $nproc_ -gt $nsims ]] ; then nproc=$nsims ; fi # Runing from simulation $simi to $simf simi=1 simf=$nproc while [[ $simi -le $nsims ]] ; do # list of simulations to be run in parallel sims_loc=$(for is in $( seq $simi "$simf" ); do echo "${sims[$(( $is - 1 ))]}"; done) for sim in $sims_loc ; do # III.3.a Running one SCM simulation out="listing" cp -r REF "$sim" cd "$sim" # Computing the number of output to give the % of simulation when running deltat=$(grep 'phys_out_filetimesteps=' config.def | tail -1 | cut -d= -f2 | awk ' { print $1 } ') if [[ "$deltat" = "1hr" ]] ; then ntimeout=$(( $nday * 24 )) ; else ntimeout=$nday ; fi # Runing SCM if [[ ! -f $DATA/$sim.nc ]] ; then # Sequential mode cp -f "$EXPE/DEF/physiq.def_$sim" physiq.def if [[ -d $EXPE/NAMECRAD/ && $rad = "ecrad" ]] ; then rm namelist_ecrad cp -f "$EXPE/NAMECRAD/namelist_ecrad_$sim" namelist_ecrad fi echo "!!!!!!!!!! RUNNING $cas $sim !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" ( time run_SCM "$DATA/$sim" &> "$out"; grep crashes "$out" || true ) & else echo "WARNING WARNING WARNING $DATA/$sim.nc ALREADY EXISTS" ; exit 1 fi cd .. done # Waiting for the end of $nproc simulations wait # Waiting for processes run with & to be closed echo "$sims_loc" # computing the indices of the next $nproc simulations simi=$(( $simf + 1 )) simf=$(( $simf + $nproc )) if [[ $simf -gt $nsims ]] ; then simf=$nsims ; fi # Checking the correct ending of the last simulation if [[ ! -f $DATA/$sim.nc ]] ; then ## Printing the % of computation done #if [ -f hourly.nc ] ; then # itimeout=`ncdump -h hourly.nc | grep -i time | head -1 | cut -d\( -f2 | awk ' { print $1 } '` # echo $itimeout $ntimeout | awk ' { printf "echo Simus -> '$sim'.nc, Waiting creation '$sim'.nc %8.2f /100 %1s\r" , 100 * $1 / $2 , "%" } ' #fi echo "=========================================================" echo "Files on CASEdir=$CASEdir" echo "=========================================================" cd "$CASEdir" ; ls ; cd - echo "=========================================================" echo "Files on CASEdir=$CASEdir/$sim" echo "=========================================================" cd "$CASEdir/$sim" ; ls ; cd - echo "=========================================================" echo "Executable files" echo "=========================================================" ls -lrt "$CASEdir/$sim/lmdz1d.e" "$CASEdir/lmdz1d.e" $LMDZdir/1D/bin/* echo "=========================================================" echo "The execution of $LMDZdir/1D/run$lmdzcas.sh failed" echo "Last simulations to be checked" ls -lrt "$CASEdir/SCM"*/listing echo "=========================================================" abort_serie_LMDZ "$cas" "$CASEdir/$sim" "$WAVEN" echo "serie_LMDZ.sh crashed for case $cas at wave $WAVEN" echo "for debugging, you can rerun it interactively on $local with debugging option to save temporary files" echo "./serie_LMDZ.sh -debug $opt_LMDZ_read $cas $WAVEN" echo "and check the results on $CASEdir" fi echo "$ntimeout" | awk ' { printf "echo SIMUS -> '"$sim"'.nc, Waiting creation '"$sim"'.nc %8.2f %1s\n" , 100 , "%" } ' done # End loop on simulation ## Cleaning #debug=1 #if [ "$debug" = "0" ] ; then # rm -rf $LMDZdir/1D/EXEC/${phys}L${LMAX}${rad}/$lmdzcas/SCM* # #rm -f limit* start* hist* filehf* day.nc tmp*nc* par*nc hour*nc #fi ############################################################################# # To be sure that all simulations are finished ############################################################################# for sim in $( ( cd "$EXPE/DEF" ; ls "physiq.def_"* | sed -e 's/physiq.def_//' ) ) ; do echo "Final check for $sim" if [[ ! -f $DATA/$sim.nc ]] ; then echo "Pb with passenger $DATA/$sim.nc" ; exit 1 ; fi done ################################################################################ cd .. done #Loop on SCM cases ################################################################################