Warning, /utils/scripts/gluemnc is written in an unsupported language. File is not indexed.
view on githubraw file Latest commit f460f9a5 on 2025-03-19 20:40:18 UTC
b3563e12af Jean*0001 #! /usr/bin/env bash
0002 #
f460f9a57e Ivan*0003 # Prerequisites:
0004 # - Add utilities directory to your search path
0005 # ```
0006 # export PATH=$PATH:MITgcm/utils/scripts
0007 # ```
0008 # - Download NCO libraries to use `ncks`; see documentation for details.
0009 #
0010 # Description:
2f5a5ed175 Jean*0011 # This is a shell script to combine multiple MITgcm mnc output files from
c9520951e9 Bayl*0012 # different tiles into one global file.
0013 # All of the variables should be in one directory, where this script is run.
0014 #
0015 # To combine all state.0000000000.t*.nc files,
2f5a5ed175 Jean*0016 # gluemnc state.0000000000.*.nc
0017 # This will result in an output file state.0000000000.glob.nc
c9520951e9 Bayl*0018 # Where glob is for global.
0019 #
0020 # You can even combine all mnc files, use
0021 # gluemnc *.nc
0022 # This will result in a series of global files,
0023 # state.0000000000.glob.nc state.0000000100.glob.nc, ...
0024 # grid.0000000000.glob.nc grid.0000000100.glob.nc, ...
0025 # diag.0000000000.glob.nc diag.0000000100.glob.nc, ...
0026 #
5a7515e136 Bayl*0027 # A lot of hard drive activity is needed. If you have a fast drive
2f5a5ed175 Jean*0028 # export TMPDIR=<path of hard drive>. On some high-performance
5a7515e136 Bayl*0029 # systems, this is already done for you.
0030 #
c9520951e9 Bayl*0031 # **********WARNINGS**********
0032 # This will probably not work at all with exch2/cubed sphere.
0033 # In that case, you probably can assemble all of the tiles on a face,
0034 # but combining faces is currently not implemented.
0035 #
0036 # Be sure you have enough disk space for the copies! In this version
0037 # nothing is done to assure all of the data is copied.
0038 #
0039 # Be careful! It will be easy to exceed the 2 GB limit for the old 32-bit
0040 # version of netcdf. If you do not have large-file support or 64-bit netcdf
0041 # you will have to be clever in dividing up your tiled files,
0042 # e.g., along the time dimension before combining to global files.
5a7515e136 Bayl*0043 # The nco operator ncks is adept at shortening files to fewer snapshots.
c9520951e9 Bayl*0044 # *****************************
0045 #
015eade80b Bayl*0046 # Good luck and happy gluing,
c9520951e9 Bayl*0047 # Baylor Fox-Kemper
0048
5a7515e136 Bayl*0049 DEBUG="--dbg_lvl=0"
0050 LOGFILE="/dev/null"
0051
0052 DIRORIG=`pwd`
015eade80b Bayl*0053
5a7515e136 Bayl*0054 if [ ! ${#TMPDIR} -gt 0 ]; then
0055 TMPDIR=$DIRORIG
0056 fi
0057
0058 export DIRNAME="$TMPDIR/gluedir.$RANDOM"
6e746ed878 Bayl*0059 mkdir $DIRNAME
1ee755b0e2 Bayl*0060
5a7515e136 Bayl*0061 echo Using temporary directory $DIRNAME
0062
1ee755b0e2 Bayl*0063 if [ -f xplodemnc ]; then
0064 cp xplodemnc $DIRNAME
0065 else
0066 cp `which xplodemnc` $DIRNAME
0067 fi
0068
f5d24cf70c Mart*0069 # find an unambiguous name for a new record dimension
0070 myrecname=record`echo $DIRNAME | awk -F. '{print $NF}'`
0071
6e746ed878 Bayl*0072 cd $DIRNAME
015eade80b Bayl*0073
c9520951e9 Bayl*0074 inone=$1
0075 inone=${1:?"You must input mnc filenames to be glued"}
0076
0077 for somefile in $@
0078 do
5a7515e136 Bayl*0079 ln -s $DIRORIG/$somefile .
c9520951e9 Bayl*0080 if [ ! -s $somefile ]; then
0081 echo "Error: $somefile is missing or empty"
0082 exit 1
0083 fi
0084 done
0085
0086 prels=${@%.t???.nc}
0087
0088 for somepre in $prels
0089 do
0090 inls=0
0091 for somepres in $sprels
0092 do
0093 if [ "$somepre" = "$somepres" ]; then
0094 inls=1
0095 fi
0096 done
0097 if [ "$inls" = "0" ]; then
0098 sprels=$sprels" "$somepre
0099 fi
0100 done
0101
0102 prels=$sprels
0103
c22edc7ca6 Mart*0104 # ML: determine the coordinate variable (this is hack for the unlikely
0105 # case that we do not have X or Y as coordinate variables; this can
0106 # happen, when only U-point or V-point variables are written to a
0107 # diagnostics stream; I do not know if this always works, but it works for me)
0108 echo Determine a usable coordinate variable
0109 somefile=${prels}.t001.nc
0110 # first try X and Y
0111 Xcoord=X
0112 Ycoord=Y
0113 Xtest=$(ncdump -vX -l 10000 $somefile | grep "X = ")
0114 Ytest=$(ncdump -vY -l 10000 $somefile | grep "Y = ")
0115 if [ ${#Xtest} = 0 ]; then
0116 echo "X not found, trying Xp1"
0117 Xtest=$(ncdump -vXp1 -l 10000 $somefile | grep "Xp1 = ")
0118 Xcoord=Xp1
0119 fi
0120 if [ ${#Xtest} = 0 ]; then
0121 echo "no X-coordinate found"
0122 Xcoord=
0123 fi
0124 if [ ${#Ytest} = 0 ]; then
0125 echo "Y not found, trying Yp1"
0126 Ytest=$(ncdump -vYp1 -l 10000 $somefile | grep "Yp1 = ")
0127 Ycoord=Yp1
0128 fi
0129 if [ ${#Ytest} = 0 ]; then
0130 echo "no Y-coordinate found"
0131 Ycoord=
0132 fi
0133 if [ ${#Xcoord} = 0 ]; then
0134 echo cannot continue
0135 exit
0136 fi
0137 if [ ${#Ycoord} = 0 ]; then
0138 echo cannot continue
0139 exit
0140 fi
0141
c9520951e9 Bayl*0142 for somepre in $prels
0143 do
0144 echo Making $somepre.glob.nc...
0145 Xsls=
0146 Ysls=
0147
0148 for somefile in $@
0149 do
0150 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
0151 echo Scanning $somefile...
c22edc7ca6 Mart*0152 Xs=$(ncdump -v${Xcoord} -l 10000 $somefile | grep "${Xcoord} = ")
c9520951e9 Bayl*0153 Xs=${Xs#*;}
0154 Xs=${Xs%;*}
0155 Xs=$(echo $Xs | sed s/' '//g)
0156 Xsls=$Xsls$Xs" "
0157
c22edc7ca6 Mart*0158 Ys=$(ncdump -v${Ycoord} -l 10000 $somefile | grep "${Ycoord} = ")
c9520951e9 Bayl*0159 Ys=${Ys#*;}
0160 Ys=${Ys%;*}
0161 Ys=$(echo $Ys | sed s/' '//g)
0162 Ysls=$Ysls$Ys" "
0163 fi
0164 done
0165
0166 sYsls=
0167 sXsls=
0168
0169 # Determine all the X locations
0170 countx=0
0171 for someXs in $Xsls
2f5a5ed175 Jean*0172 do
c9520951e9 Bayl*0173 inls=0
0174 for somesXs in $sXsls
0175 do
0176 if [ "$someXs" = "$somesXs" ]; then
0177 inls=1
0178 fi
0179 done
0180 if [ "$inls" = "0" ]; then
0181 sXsls=$sXsls$someXs" "
0182 countx=$((countx))+1
0183 fi
0184 done
015eade80b Bayl*0185 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
c9520951e9 Bayl*0186 echo $((countx)) tiles found in x-direction.
015eade80b Bayl*0187 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
c9520951e9 Bayl*0188
0189 # Determine all the Y locations
0190 county=0
0191 for someYs in $Ysls
2f5a5ed175 Jean*0192 do
c9520951e9 Bayl*0193 inls=0
0194 for somesYs in $sYsls
0195 do
0196 if [ "$someYs" = "$somesYs" ]; then
0197 inls=1
0198 fi
0199 done
0200 if [ "$inls" = "0" ]; then
0201 sYsls=$sYsls$someYs" "
0202 county=$((county))+1
0203 fi
0204 done
015eade80b Bayl*0205 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
c9520951e9 Bayl*0206 echo $((county)) tiles found in y-direction.
015eade80b Bayl*0207 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
c9520951e9 Bayl*0208
0209 countyy=1000
0210 countxx=1000
0211
015eade80b Bayl*0212 cntls=
c9520951e9 Bayl*0213 for someX in $sXsls
0214 do
0215 countxx=$((countxx+1))
015eade80b Bayl*0216 cntls=$cntls$countxx" "
0217 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0218 echo Prepping X tile $((countxx-1000))
0219 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0220
c9520951e9 Bayl*0221 for somefile in $@
0222 do
0223 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
c22edc7ca6 Mart*0224 Xs=$(ncdump -v${Xcoord} -l 10000 $somefile | grep "${Xcoord} = ")
c9520951e9 Bayl*0225 Xs=${Xs#*;}
0226 Xs=${Xs%;*}
0227 Xs=$(echo $Xs | sed s/' '//g)
0228
0229 if [ "$someX" = $Xs ]; then
2f5a5ed175 Jean*0230 ./xplodemnc $somefile
79ad2ffb0a Jean*0231 if [ -f iter.$somefile ]; then
0232 mv iter.$somefile iter.${somefile%t???.nc}glob.nc
0233 fi
c9520951e9 Bayl*0234 for somesplit in $(ls *.$somefile)
0235 do
f5d24cf70c Mart*0236 # Added to account for grid files with no T dimension defined:
0237 # Search for the unlimited dimension and get its name, assuming
0238 # that its first character is a letter from the alphabet, the "tr"
0239 # removes the blank characters
0240 recname=$(ncdump -h $somesplit \
0241 | sed -n 's/\([a-z,A-Z]*\) = \(UNLIMITED\) .*/\1/p' \
0242 | tr -d ' \t' )
0243 if [[ -z "$recname" ]]; then
0244 echo "No record dimension found, adding one now: "$myrecname
0245 ncecat $DEBUG -O -u $myrecname $somesplit $somesplit > $LOGFILE
0246 recname=$myrecname
2f5a5ed175 Jean*0247 fi
c9520951e9 Bayl*0248 withY=$(ncdump -h $somesplit | grep "Y =")
0249 if [ ${#withY} -gt 1 ]; then
0250 echo Changing Y to record variable in $somesplit
f5d24cf70c Mart*0251 ncpdq $DEBUG -O -a Y,$recname $somesplit $somesplit > $LOGFILE
b3563e12af Jean*0252 mv $somesplit i$countxx.$somesplit
c9520951e9 Bayl*0253 fi
2f5a5ed175 Jean*0254
5a7515e136 Bayl*0255 if [ -f $somesplit ]; then
0256 withYp1=$(ncdump -h $somesplit | grep "Yp1 =")
0257 if [ ${#withYp1} -gt 1 ]; then
0258 Yp1len=${withYp1#*= }
0259 Yp1len=$((${Yp1len% ;}-1))
0260 # Strip off the repeated value in Yp1
0261 echo Changing Yp1 to record variable in $somesplit
f5d24cf70c Mart*0262 ncpdq $DEBUG -O -a Yp1,$recname -F -d Yp1,1,$Yp1len $somesplit $somesplit > $LOGFILE
b3563e12af Jean*0263 mv $somesplit i$countxx.$somesplit
5a7515e136 Bayl*0264 fi
c9520951e9 Bayl*0265 fi
0266 done
0267 fi
0268 fi
0269 done
015eade80b Bayl*0270 done
0271 echo Tile names $cntls
0272 for countxx in $cntls
0273 do
0274 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0275 echo Combining X tile $((countxx-1000))
0276 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0277 varls=
b3563e12af Jean*0278 cxfilels=$(ls i$countxx.*)
015eade80b Bayl*0279 oldvar=
0280 for somefile in $cxfilels
0281 do
79ad2ffb0a Jean*0282 varname=`echo $somefile | sed 's/^i[0-9][0-9][0-9][0-9]\.//; s/\..*nc//'`
015eade80b Bayl*0283 if [ "$varname" = "$oldvar" ]; then
0284 echo $varname repeated
0285 else
0286 varls=$varls$varname" "
0287 fi
0288 oldvar=$varname
0289 done
0290
0291 echo Found these variables to combine: $varls
0292
c9520951e9 Bayl*0293 for somevar in $varls
0294 do
015eade80b Bayl*0295 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
0296 echo Combining $somevar files in Y
0297 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
c9520951e9 Bayl*0298
b3563e12af Jean*0299 filelist=(`ls i$countxx.$somevar.$somepre.*.nc`)
0300 withY=$(ncdump -h ${filelist[0]} | grep "Y =")
0301 withYp1=$(ncdump -h ${filelist[0]} | grep "Yp1 =")
c9520951e9 Bayl*0302
b3563e12af Jean*0303 ncrcat $DEBUG i$countxx.$somevar.$somepre.*.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
015eade80b Bayl*0304 echo Just combined $countxx.$somevar
b3563e12af Jean*0305 rm i$countxx.$somevar.$somepre.t???.nc
c9520951e9 Bayl*0306
f5d24cf70c Mart*0307 # Recover the name of the record variable, there are two possibilities:
0308 # T (MITgcm convention) and $myrecname.
0309 # I must admit that this could be more elegant.
0310 recname=$(ncdump -h $somevar.$somepre.gloy.$countxx.nc | grep $myrecname)
0311 if [[ -z $recname ]]; then
0312 recname=T
0313 else
0314 recname=$myrecname
0315 fi
c9520951e9 Bayl*0316 if [ ${#withY} -gt 1 ]; then
f5d24cf70c Mart*0317 echo Changing $recname to record variable in $somevar.$somepre.gloy.$countxx.nc
0318 ncpdq $DEBUG -O -a $recname,Y $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
c9520951e9 Bayl*0319 fi
0320
0321 if [ ${#withYp1} -gt 1 ]; then
f5d24cf70c Mart*0322 echo Changing $recname to record variable in $somevar.$somepre.gloy.$countxx.nc
0323 ncpdq $DEBUG -O -a $recname,Yp1 $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
c9520951e9 Bayl*0324 fi
0325 done
0326 done
0327 for somevar in $varls
0328 do
015eade80b Bayl*0329 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0330 echo Combining $somevar files in X...
0331 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
c9520951e9 Bayl*0332 for somegloy in $(ls $somevar.$somepre.gloy.*.nc)
0333 do
0334 withX=$(ncdump -h $somegloy | grep "X =")
0335 withXp1=$(ncdump -h $somegloy | grep "Xp1 =")
f5d24cf70c Mart*0336 recname=$(ncdump -h $somegloy \
0337 | sed -n 's/\([a-z,A-Z]*\) = \(UNLIMITED\) .*/\1/p' \
0338 | tr -d ' \t' )
2f5a5ed175 Jean*0339
c9520951e9 Bayl*0340 if [ ${#withX} -gt 1 ]; then
0341 echo Changing X to record variable in $somegloy
f5d24cf70c Mart*0342 ncpdq $DEBUG -O -a X,$recname $somegloy $somegloy > $LOGFILE
c9520951e9 Bayl*0343 fi
0344
0345 if [ ${#withXp1} -gt 1 ]; then
0346 Xp1len=${withXp1#*= }
0347 Xp1len=$((${Xp1len% ;}-1))
0348 # Strip off the repeated value in Xp1
015eade80b Bayl*0349 echo Changing Xp1 to record variable in $somegloy
f5d24cf70c Mart*0350 echo ncpdq $DEBUG -O -a Xp1,$recname -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
0351 ncpdq $DEBUG -O -a Xp1,$recname -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
c9520951e9 Bayl*0352 fi
0353 done
0354 echo Combining $somevar.gloy files...
5a7515e136 Bayl*0355 ncrcat $DEBUG $somevar.$somepre.gloy.*.nc $somevar.$somepre.glob.nc > $LOGFILE
0356 # rm $somevar.$somepre.gloy.*.nc
c9520951e9 Bayl*0357
f5d24cf70c Mart*0358 # recname is still valid, so change back to it without testing for it again
c9520951e9 Bayl*0359 if [ ${#withX} -gt 1 ]; then
f5d24cf70c Mart*0360 echo Changing $recname to record variable in $somevar.$somepre.glob.nc
0361 ncpdq $DEBUG -O -a $recname,X $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
c9520951e9 Bayl*0362 fi
0363
0364 if [ ${#withXp1} -gt 1 ]; then
f5d24cf70c Mart*0365 echo Changing $recname to record variable in $somevar.$somepre.glob.nc
0366 ncpdq $DEBUG -O -a $recname,Xp1 $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
0367 fi
0368 if [ "$recname" = "$myrecname" ]; then
0369 # only for variables that did not have a record dimension to begin with
0370 echo "removing record dimension $recname from $somevar.$somepre.glob.nc"
0371 ncwa $DEBUG -O -a $recname $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc
c9520951e9 Bayl*0372 fi
1676642123 Mart*0373 ncks $DEBUG -A $somevar.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
79ad2ffb0a Jean*0374 # rm $somevar.$somepre.glob.nc
c9520951e9 Bayl*0375 done
79ad2ffb0a Jean*0376 if [ -f iter.$somepre.glob.nc ]; then
1676642123 Mart*0377 ncks $DEBUG -A iter.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
79ad2ffb0a Jean*0378 fi
0379 # rm iter.$somepre.glob.nc
0380
2f5a5ed175 Jean*0381 # another hack by rpa to accomodate grid.nc files
0382 # (there are several variables with just Z dimension that we want to keep)
430a231cb7 Mart*0383 # varsz=$( ncdump -h $somepre.t001.nc | sed -n 's/^\s*\(double\|float\).* \(\w*\)(Z\w*).*/\2/p' )
f5d24cf70c Mart*0384 # The OR ("\|") and "\s", "\w" only work for GNU-sed, but not for
0385 # BSD-sed or SunOS-sed, therefore we need to use some work-arounds:
fbb692b4e1 Jean*0386 varsz=$( ncdump -h $somepre.t001.nc | grep -E "double|float" \
f5d24cf70c Mart*0387 | grep -v , | sed -n 's/.* \(.*\)(Z.*).*/\1/p' )
2f5a5ed175 Jean*0388 fixed=
0389 for varz in $varsz
0390 do
430a231cb7 Mart*0391 # check to make sure the variable does not already exist in the glob file
2f5a5ed175 Jean*0392 if [[ -z $( ncdump -h $somepre.glob.nc | grep " $varz(" ) ]]
0393 then
0394 echo "Adding variable $varz to $somepre.glob.nc"
0395 ncks $DEBUG -A -v $varz $somepre.t001.nc $somepre.glob.nc > $LOGFILE
0396 fixed='yes'
0397 fi
0398 done
0399
5a7515e136 Bayl*0400 cp $somepre.glob.nc $DIRORIG
c9520951e9 Bayl*0401 done
0402
5a7515e136 Bayl*0403 cd $DIRORIG
6e746ed878 Bayl*0404 rm -rf $DIRNAME
c9520951e9 Bayl*0405
