Warning, /utils/scripts/gluemnc is written in an unsupported language. File is not indexed.
view on githubraw file Latest commit add29e06 on 2018-01-31 20:35:05 UTC
b3563e12af Jean*0001 #! /usr/bin/env bash
0002 #
2f5a5ed175 Jean*0003 # This is a shell script to combine multiple MITgcm mnc output files from
c9520951e9 Bayl*0004 # different tiles into one global file.
0005 # All of the variables should be in one directory, where this script is run.
0006 #
0007 # To combine all state.0000000000.t*.nc files,
2f5a5ed175 Jean*0008 # gluemnc state.0000000000.*.nc
0009 # This will result in an output file state.0000000000.glob.nc
c9520951e9 Bayl*0010 # Where glob is for global.
0011 #
0012 # You can even combine all mnc files, use
0013 # gluemnc *.nc
0014 # This will result in a series of global files,
0015 # state.0000000000.glob.nc state.0000000100.glob.nc, ...
0016 # grid.0000000000.glob.nc grid.0000000100.glob.nc, ...
0017 # diag.0000000000.glob.nc diag.0000000100.glob.nc, ...
0018 #
5a7515e136 Bayl*0019 # A lot of hard drive activity is needed. If you have a fast drive
2f5a5ed175 Jean*0020 # export TMPDIR=<path of hard drive>. On some high-performance
5a7515e136 Bayl*0021 # systems, this is already done for you.
0022 #
c9520951e9 Bayl*0023 # **********WARNINGS**********
0024 # This will probably not work at all with exch2/cubed sphere.
0025 # In that case, you probably can assemble all of the tiles on a face,
0026 # but combining faces is currently not implemented.
0027 #
0028 # Be sure you have enough disk space for the copies! In this version
0029 # nothing is done to assure all of the data is copied.
0030 #
0031 # Be careful! It will be easy to exceed the 2 GB limit for the old 32-bit
0032 # version of netcdf. If you do not have large-file support or 64-bit netcdf
0033 # you will have to be clever in dividing up your tiled files,
0034 # e.g., along the time dimension before combining to global files.
5a7515e136 Bayl*0035 # The nco operator ncks is adept at shortening files to fewer snapshots.
c9520951e9 Bayl*0036 # *****************************
0037 #
015eade80b Bayl*0038 # Good luck and happy gluing,
c9520951e9 Bayl*0039 # Baylor Fox-Kemper
0040
5a7515e136 Bayl*0041 DEBUG="--dbg_lvl=0"
0042 LOGFILE="/dev/null"
0043
0044 DIRORIG=`pwd`
015eade80b Bayl*0045
5a7515e136 Bayl*0046 if [ ! ${#TMPDIR} -gt 0 ]; then
0047 TMPDIR=$DIRORIG
0048 fi
0049
0050 export DIRNAME="$TMPDIR/gluedir.$RANDOM"
6e746ed878 Bayl*0051 mkdir $DIRNAME
1ee755b0e2 Bayl*0052
5a7515e136 Bayl*0053 echo Using temporary directory $DIRNAME
0054
1ee755b0e2 Bayl*0055 if [ -f xplodemnc ]; then
0056 cp xplodemnc $DIRNAME
0057 else
0058 cp `which xplodemnc` $DIRNAME
0059 fi
0060
f5d24cf70c Mart*0061 # find an unambiguous name for a new record dimension
0062 myrecname=record`echo $DIRNAME | awk -F. '{print $NF}'`
0063
6e746ed878 Bayl*0064 cd $DIRNAME
015eade80b Bayl*0065
c9520951e9 Bayl*0066 inone=$1
0067 inone=${1:?"You must input mnc filenames to be glued"}
0068
0069 for somefile in $@
0070 do
5a7515e136 Bayl*0071 ln -s $DIRORIG/$somefile .
c9520951e9 Bayl*0072 if [ ! -s $somefile ]; then
0073 echo "Error: $somefile is missing or empty"
0074 exit 1
0075 fi
0076 done
0077
0078 prels=${@%.t???.nc}
0079
0080 for somepre in $prels
0081 do
0082 inls=0
0083 for somepres in $sprels
0084 do
0085 if [ "$somepre" = "$somepres" ]; then
0086 inls=1
0087 fi
0088 done
0089 if [ "$inls" = "0" ]; then
0090 sprels=$sprels" "$somepre
0091 fi
0092 done
0093
0094 prels=$sprels
0095
c22edc7ca6 Mart*0096 # ML: determine the coordinate variable (this is hack for the unlikely
0097 # case that we do not have X or Y as coordinate variables; this can
0098 # happen, when only U-point or V-point variables are written to a
0099 # diagnostics stream; I do not know if this always works, but it works for me)
0100 echo Determine a usable coordinate variable
0101 somefile=${prels}.t001.nc
0102 # first try X and Y
0103 Xcoord=X
0104 Ycoord=Y
0105 Xtest=$(ncdump -vX -l 10000 $somefile | grep "X = ")
0106 Ytest=$(ncdump -vY -l 10000 $somefile | grep "Y = ")
0107 if [ ${#Xtest} = 0 ]; then
0108 echo "X not found, trying Xp1"
0109 Xtest=$(ncdump -vXp1 -l 10000 $somefile | grep "Xp1 = ")
0110 Xcoord=Xp1
0111 fi
0112 if [ ${#Xtest} = 0 ]; then
0113 echo "no X-coordinate found"
0114 Xcoord=
0115 fi
0116 if [ ${#Ytest} = 0 ]; then
0117 echo "Y not found, trying Yp1"
0118 Ytest=$(ncdump -vYp1 -l 10000 $somefile | grep "Yp1 = ")
0119 Ycoord=Yp1
0120 fi
0121 if [ ${#Ytest} = 0 ]; then
0122 echo "no Y-coordinate found"
0123 Ycoord=
0124 fi
0125 if [ ${#Xcoord} = 0 ]; then
0126 echo cannot continue
0127 exit
0128 fi
0129 if [ ${#Ycoord} = 0 ]; then
0130 echo cannot continue
0131 exit
0132 fi
0133
c9520951e9 Bayl*0134 for somepre in $prels
0135 do
0136 echo Making $somepre.glob.nc...
0137 Xsls=
0138 Ysls=
0139
0140 for somefile in $@
0141 do
0142 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
0143 echo Scanning $somefile...
c22edc7ca6 Mart*0144 Xs=$(ncdump -v${Xcoord} -l 10000 $somefile | grep "${Xcoord} = ")
c9520951e9 Bayl*0145 Xs=${Xs#*;}
0146 Xs=${Xs%;*}
0147 Xs=$(echo $Xs | sed s/' '//g)
0148 Xsls=$Xsls$Xs" "
0149
c22edc7ca6 Mart*0150 Ys=$(ncdump -v${Ycoord} -l 10000 $somefile | grep "${Ycoord} = ")
c9520951e9 Bayl*0151 Ys=${Ys#*;}
0152 Ys=${Ys%;*}
0153 Ys=$(echo $Ys | sed s/' '//g)
0154 Ysls=$Ysls$Ys" "
0155 fi
0156 done
0157
0158 sYsls=
0159 sXsls=
0160
0161 # Determine all the X locations
0162 countx=0
0163 for someXs in $Xsls
2f5a5ed175 Jean*0164 do
c9520951e9 Bayl*0165 inls=0
0166 for somesXs in $sXsls
0167 do
0168 if [ "$someXs" = "$somesXs" ]; then
0169 inls=1
0170 fi
0171 done
0172 if [ "$inls" = "0" ]; then
0173 sXsls=$sXsls$someXs" "
0174 countx=$((countx))+1
0175 fi
0176 done
015eade80b Bayl*0177 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
c9520951e9 Bayl*0178 echo $((countx)) tiles found in x-direction.
015eade80b Bayl*0179 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
c9520951e9 Bayl*0180
0181 # Determine all the Y locations
0182 county=0
0183 for someYs in $Ysls
2f5a5ed175 Jean*0184 do
c9520951e9 Bayl*0185 inls=0
0186 for somesYs in $sYsls
0187 do
0188 if [ "$someYs" = "$somesYs" ]; then
0189 inls=1
0190 fi
0191 done
0192 if [ "$inls" = "0" ]; then
0193 sYsls=$sYsls$someYs" "
0194 county=$((county))+1
0195 fi
0196 done
015eade80b Bayl*0197 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
c9520951e9 Bayl*0198 echo $((county)) tiles found in y-direction.
015eade80b Bayl*0199 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
c9520951e9 Bayl*0200
0201 countyy=1000
0202 countxx=1000
0203
015eade80b Bayl*0204 cntls=
c9520951e9 Bayl*0205 for someX in $sXsls
0206 do
0207 countxx=$((countxx+1))
015eade80b Bayl*0208 cntls=$cntls$countxx" "
0209 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0210 echo Prepping X tile $((countxx-1000))
0211 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0212
c9520951e9 Bayl*0213 for somefile in $@
0214 do
0215 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
c22edc7ca6 Mart*0216 Xs=$(ncdump -v${Xcoord} -l 10000 $somefile | grep "${Xcoord} = ")
c9520951e9 Bayl*0217 Xs=${Xs#*;}
0218 Xs=${Xs%;*}
0219 Xs=$(echo $Xs | sed s/' '//g)
0220
0221 if [ "$someX" = $Xs ]; then
2f5a5ed175 Jean*0222 ./xplodemnc $somefile
79ad2ffb0a Jean*0223 if [ -f iter.$somefile ]; then
0224 mv iter.$somefile iter.${somefile%t???.nc}glob.nc
0225 fi
c9520951e9 Bayl*0226 for somesplit in $(ls *.$somefile)
0227 do
f5d24cf70c Mart*0228 # Added to account for grid files with no T dimension defined:
0229 # Search for the unlimited dimension and get its name, assuming
0230 # that its first character is a letter from the alphabet, the "tr"
0231 # removes the blank characters
0232 recname=$(ncdump -h $somesplit \
0233 | sed -n 's/\([a-z,A-Z]*\) = \(UNLIMITED\) .*/\1/p' \
0234 | tr -d ' \t' )
0235 if [[ -z "$recname" ]]; then
0236 echo "No record dimension found, adding one now: "$myrecname
0237 ncecat $DEBUG -O -u $myrecname $somesplit $somesplit > $LOGFILE
0238 recname=$myrecname
2f5a5ed175 Jean*0239 fi
c9520951e9 Bayl*0240 withY=$(ncdump -h $somesplit | grep "Y =")
0241 if [ ${#withY} -gt 1 ]; then
0242 echo Changing Y to record variable in $somesplit
f5d24cf70c Mart*0243 ncpdq $DEBUG -O -a Y,$recname $somesplit $somesplit > $LOGFILE
b3563e12af Jean*0244 mv $somesplit i$countxx.$somesplit
c9520951e9 Bayl*0245 fi
2f5a5ed175 Jean*0246
5a7515e136 Bayl*0247 if [ -f $somesplit ]; then
0248 withYp1=$(ncdump -h $somesplit | grep "Yp1 =")
0249 if [ ${#withYp1} -gt 1 ]; then
0250 Yp1len=${withYp1#*= }
0251 Yp1len=$((${Yp1len% ;}-1))
0252 # Strip off the repeated value in Yp1
0253 echo Changing Yp1 to record variable in $somesplit
f5d24cf70c Mart*0254 ncpdq $DEBUG -O -a Yp1,$recname -F -d Yp1,1,$Yp1len $somesplit $somesplit > $LOGFILE
b3563e12af Jean*0255 mv $somesplit i$countxx.$somesplit
5a7515e136 Bayl*0256 fi
c9520951e9 Bayl*0257 fi
0258 done
0259 fi
0260 fi
0261 done
015eade80b Bayl*0262 done
0263 echo Tile names $cntls
0264 for countxx in $cntls
0265 do
0266 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0267 echo Combining X tile $((countxx-1000))
0268 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0269 varls=
b3563e12af Jean*0270 cxfilels=$(ls i$countxx.*)
015eade80b Bayl*0271 oldvar=
0272 for somefile in $cxfilels
0273 do
79ad2ffb0a Jean*0274 varname=`echo $somefile | sed 's/^i[0-9][0-9][0-9][0-9]\.//; s/\..*nc//'`
015eade80b Bayl*0275 if [ "$varname" = "$oldvar" ]; then
0276 echo $varname repeated
0277 else
0278 varls=$varls$varname" "
0279 fi
0280 oldvar=$varname
0281 done
0282
0283 echo Found these variables to combine: $varls
0284
c9520951e9 Bayl*0285 for somevar in $varls
0286 do
015eade80b Bayl*0287 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
0288 echo Combining $somevar files in Y
0289 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
c9520951e9 Bayl*0290
b3563e12af Jean*0291 filelist=(`ls i$countxx.$somevar.$somepre.*.nc`)
0292 withY=$(ncdump -h ${filelist[0]} | grep "Y =")
0293 withYp1=$(ncdump -h ${filelist[0]} | grep "Yp1 =")
c9520951e9 Bayl*0294
b3563e12af Jean*0295 ncrcat $DEBUG i$countxx.$somevar.$somepre.*.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
015eade80b Bayl*0296 echo Just combined $countxx.$somevar
b3563e12af Jean*0297 rm i$countxx.$somevar.$somepre.t???.nc
c9520951e9 Bayl*0298
f5d24cf70c Mart*0299 # Recover the name of the record variable, there are two possibilities:
0300 # T (MITgcm convention) and $myrecname.
0301 # I must admit that this could be more elegant.
0302 recname=$(ncdump -h $somevar.$somepre.gloy.$countxx.nc | grep $myrecname)
0303 if [[ -z $recname ]]; then
0304 recname=T
0305 else
0306 recname=$myrecname
0307 fi
c9520951e9 Bayl*0308 if [ ${#withY} -gt 1 ]; then
f5d24cf70c Mart*0309 echo Changing $recname to record variable in $somevar.$somepre.gloy.$countxx.nc
0310 ncpdq $DEBUG -O -a $recname,Y $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
c9520951e9 Bayl*0311 fi
0312
0313 if [ ${#withYp1} -gt 1 ]; then
f5d24cf70c Mart*0314 echo Changing $recname to record variable in $somevar.$somepre.gloy.$countxx.nc
0315 ncpdq $DEBUG -O -a $recname,Yp1 $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
c9520951e9 Bayl*0316 fi
0317 done
0318 done
0319 for somevar in $varls
0320 do
015eade80b Bayl*0321 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
0322 echo Combining $somevar files in X...
0323 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
c9520951e9 Bayl*0324 for somegloy in $(ls $somevar.$somepre.gloy.*.nc)
0325 do
0326 withX=$(ncdump -h $somegloy | grep "X =")
0327 withXp1=$(ncdump -h $somegloy | grep "Xp1 =")
f5d24cf70c Mart*0328 recname=$(ncdump -h $somegloy \
0329 | sed -n 's/\([a-z,A-Z]*\) = \(UNLIMITED\) .*/\1/p' \
0330 | tr -d ' \t' )
2f5a5ed175 Jean*0331
c9520951e9 Bayl*0332 if [ ${#withX} -gt 1 ]; then
0333 echo Changing X to record variable in $somegloy
f5d24cf70c Mart*0334 ncpdq $DEBUG -O -a X,$recname $somegloy $somegloy > $LOGFILE
c9520951e9 Bayl*0335 fi
0336
0337 if [ ${#withXp1} -gt 1 ]; then
0338 Xp1len=${withXp1#*= }
0339 Xp1len=$((${Xp1len% ;}-1))
0340 # Strip off the repeated value in Xp1
015eade80b Bayl*0341 echo Changing Xp1 to record variable in $somegloy
f5d24cf70c Mart*0342 echo ncpdq $DEBUG -O -a Xp1,$recname -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
0343 ncpdq $DEBUG -O -a Xp1,$recname -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
c9520951e9 Bayl*0344 fi
0345 done
0346 echo Combining $somevar.gloy files...
5a7515e136 Bayl*0347 ncrcat $DEBUG $somevar.$somepre.gloy.*.nc $somevar.$somepre.glob.nc > $LOGFILE
0348 # rm $somevar.$somepre.gloy.*.nc
c9520951e9 Bayl*0349
f5d24cf70c Mart*0350 # recname is still valid, so change back to it without testing for it again
c9520951e9 Bayl*0351 if [ ${#withX} -gt 1 ]; then
f5d24cf70c Mart*0352 echo Changing $recname to record variable in $somevar.$somepre.glob.nc
0353 ncpdq $DEBUG -O -a $recname,X $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
c9520951e9 Bayl*0354 fi
0355
0356 if [ ${#withXp1} -gt 1 ]; then
f5d24cf70c Mart*0357 echo Changing $recname to record variable in $somevar.$somepre.glob.nc
0358 ncpdq $DEBUG -O -a $recname,Xp1 $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
0359 fi
0360 if [ "$recname" = "$myrecname" ]; then
0361 # only for variables that did not have a record dimension to begin with
0362 echo "removing record dimension $recname from $somevar.$somepre.glob.nc"
0363 ncwa $DEBUG -O -a $recname $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc
c9520951e9 Bayl*0364 fi
1676642123 Mart*0365 ncks $DEBUG -A $somevar.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
79ad2ffb0a Jean*0366 # rm $somevar.$somepre.glob.nc
c9520951e9 Bayl*0367 done
79ad2ffb0a Jean*0368 if [ -f iter.$somepre.glob.nc ]; then
1676642123 Mart*0369 ncks $DEBUG -A iter.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
79ad2ffb0a Jean*0370 fi
0371 # rm iter.$somepre.glob.nc
0372
2f5a5ed175 Jean*0373 # another hack by rpa to accomodate grid.nc files
0374 # (there are several variables with just Z dimension that we want to keep)
430a231cb7 Mart*0375 # varsz=$( ncdump -h $somepre.t001.nc | sed -n 's/^\s*\(double\|float\).* \(\w*\)(Z\w*).*/\2/p' )
f5d24cf70c Mart*0376 # The OR ("\|") and "\s", "\w" only work for GNU-sed, but not for
0377 # BSD-sed or SunOS-sed, therefore we need to use some work-arounds:
0378 varsz=$( ncdump -h $somepre.t001.nc | egrep "double|float" \
0379 | grep -v , | sed -n 's/.* \(.*\)(Z.*).*/\1/p' )
2f5a5ed175 Jean*0380 fixed=
0381 for varz in $varsz
0382 do
430a231cb7 Mart*0383 # check to make sure the variable does not already exist in the glob file
2f5a5ed175 Jean*0384 if [[ -z $( ncdump -h $somepre.glob.nc | grep " $varz(" ) ]]
0385 then
0386 echo "Adding variable $varz to $somepre.glob.nc"
0387 ncks $DEBUG -A -v $varz $somepre.t001.nc $somepre.glob.nc > $LOGFILE
0388 fixed='yes'
0389 fi
0390 done
0391
5a7515e136 Bayl*0392 cp $somepre.glob.nc $DIRORIG
c9520951e9 Bayl*0393 done
0394
5a7515e136 Bayl*0395
0396 cd $DIRORIG
6e746ed878 Bayl*0397 rm -rf $DIRNAME
c9520951e9 Bayl*0398
