Warning, /utils/matlab/mnc_assembly.m is written in an unsupported language. File is not indexed.
view on githubraw file Latest commit add29e06 on 2018-01-31 20:35:05 UTC
92749caea6 Ed H*0001 function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
0002
0003 % Function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
0004 %
0005 % INPUTS
0006 % fpat string containing the file pattern
64e70a7dc2 Mart*0007 % vars structure array of variable names
92749caea6 Ed H*0008 %
0009 % fout output file pattern (DEF: "all.%05d.nc")
0010 % fsize max output file size (DEF: 2.0e+9 = +/-2GB)
0011 %
0012 % OUTPUTS
0013 % nt number of usable tiles found
0014 % nf number of output files written
0015 %
0016 % This function "assembles" MNC output. It finds all the per-tile
0017 % NetCDF files that match the input pattern, does some basic "sanity"
0018 % tests to determine whether the files have compatible sizes, and
0019 % then assembles all of the requested data (all of the variables)
0020 % into one or more "global" NetCDF files. The global files have
0021 % the following dimension conventions:
0022 %
0023 % "exch 1": all values are within a global horizontal grid
0024 % and indicies are (X,Y,Z,T)
0025 %
0026 % "exch 2": all values are within one of up to six "faces"
0027 % of a global cube with indicies (Xf,Yf,F,Z,T)
0028 %
0029 % where "X,Y.Z,T" are global space/time indicies, "Xf,Yf" are local
0030 % per-face spatial indicies, and "F" is a face index.
1bde030d1b Ed H*0031 %
0032 % An example of how to use this script is:
0033 %
0034 % vars = struct([]);
0035 % vars(1).name = 'iter';
0036 % vars(2).name = 'U';
0037 % vars(3).name = 'Unk';
0038 % vars(4).name = 'V';
0039 % vars(5).name = 'Temp';
0040 % vars(6).name = 'S';
0041 % fpat = 'exp0_20041126_0001/state.0000.%06d.nc';
0042 % [nt,nf] = mnc_assembly(fpat,vars);
0043 %
0044 % and the resutlt is written as "all.00000.nc"
92749caea6 Ed H*0045
0046 %===== Argument checking and defaults =====
0047
0048 if nargin < 2
0049 disp('Error: there must be at least 2 arguments!');
0050 return
0051 end
0052
0053 if nargin < 3
0054 fout = 'all.%05d.nc';
0055 end
0056 if nargin < 4
0057 fsize = 2.0e+9;
0058 end
0059
0060
0061 %===== Find and open all the matching files =====
0062
0063 nt = 0;
0064 nf = 0;
0065 all_ncf = struct([]);
0066
0067 % Find all of the files
0068 exch2_msg = 0;
0069 tmax = 200;
0070 frdone = 0;
0071 it = 0;
0072 while frdone == 0
0073
0074 it = it + 1;
0075 fnm = sprintf(fpat,it);
0076 % disp(fnm);
0077
0078 % Check that the file exists
0079 fid = fopen(fnm, 'r');
0080 if fid < 0
0081 if it >= tmax
0082 frdone = 1;
0083 end
0084 continue;
0085 end
0086
0087 % Open the NetCDF file
0088 fnc = netcdf(fnm, 'nowrite');
0089 if length(fnc) == 0
0090 continue;
0091 end
0092
0093 % Check for exch1/exch2 grid
0094 exch = 1;
0095 exch2_myFace = fnc.exch2_myFace(:);
0096 if length(exch2_myFace) ~= 0
0097 exch = 2;
0098 if exch2_msg == 0
0099 exch2_msg = 1;
0100 disp(' Grid type appears to be: "exch2"');
0101 end
0102 end
0103
0104 n = length(all_ncf) + 1;
0105 all_ncf(n).name = fnm;
0106 all_ncf(n).nc = {fnc};
0107 all_ncf(n).exch = exch;
0108 all_ncf(n).tile_number = fnc.tile_number(1);
0109 all_ncf(n).bi = fnc.bi(1);
0110 all_ncf(n).bj = fnc.bj(1);
0111 all_ncf(n).sNx = fnc.sNx(1);
0112 all_ncf(n).sNy = fnc.sNy(1);
072e7c3589 Mart*0113 all_ncf(n).Nx = fnc.Nx(1);
0114 all_ncf(n).Ny = fnc.Ny(1);
92749caea6 Ed H*0115 all_ncf(n).Z = fnc.Z(1);
0116
0117 if exch == 2
0118 all_ncf(n).exch2_myFace = exch2_myFace;
0119 all_ncf(n).exch2_tbasex = fnc.exch2_tbasex(1);
0120 all_ncf(n).exch2_tbasey = fnc.exch2_tbasex(1);
0121 end
0122
0123 clear fnc
0124 end
0125
0126
0127 %===== Do some basic sanity checks =====
0128
0129 % check for number of files/tiles found
0130 if length(all_ncf) == 0
0131 disp('Error: no tiles found--no need to do any assembly!');
0132 return
0133 elseif length(all_ncf) == 1
0134 disp('Error: one tile found--no need to do any assembly!');
0135 return
0136 else
0137 disp(sprintf(' Found %d files matching the pattern: "%s"', ...
0138 length(all_ncf), fpat ));
0139 end
0140
0141 % check for consistent "exch" version
0142 if prod(double([all_ncf.exch] == all_ncf(1).exch)) ~= 1
0143 disp('Error: not all the "exch" types of the files match.');
0144 return;
0145 end
0146
0147 % check for consistent sNx,sNy
0148 if (prod(double([all_ncf.sNx] == all_ncf(1).sNx)) ~= 1) ...
072e7c3589 Mart*0149 | (prod(double([all_ncf.sNy] == all_ncf(1).sNy)) ~= 1)
92749caea6 Ed H*0150 disp('Error: the "sNx,sNy" values for all the tiles are not');
0151 disp(' uniform. Future versions of this function will be');
0152 disp(' able to handle non-uniform grid sizes but this');
0153 disp(' feature is not yet implemented.');
0154 return;
0155 end
0156
0157 % check for redundant tiles and "time series" output
0158 if length(all_ncf) ~= length(unique([all_ncf.tile_number]))
0159 disp('Error: redundant tiles were found. Please check that');
0160 disp(' the file pattern does not specify output spanning');
0161 disp(' multiple model runs or even multiple time series');
0162 disp(' within a single model run. For multi-time-series');
0163 disp(' data sets, EACH "LEVEL" IN THE OUTPUT SERIES MUST');
0164 disp(' BE ASSEMBLED SEPARATERLY.');
0165 return
0166 end
0167
0168
0169 %===== Get the dims/vars associations =====
0170
0171 mydims = struct('names', {}, 'lens', {});
0172 myvars = struct([]);
0173 clear tncf;
0174 for ivar = 1:length(vars)
0175 mydim_names = {};
0176 mydim_sizes = {};
0177 myatt.names = {};
0178 myatt.types = {};
0179 myatt.data = {};
0180 myname = vars(ivar).name;
0181 disp([' Looking for variable: ' myname]);
0182
0183 itile = 1;
0184 tncf = all_ncf(itile).nc{1};
0185 ncv = tncf{myname};
0186 len = length(ncv);
0187 if length(ncv) == 0
0188 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
0189 ' so it will be ignored.'];
0190 disp(sprintf(warns,myname,all_ncf(itile).name));
0191 continue
0192 end
0193 mytype = datatype(ncv);
0194 tmpdims = dim(ncv);
0195 for inm = 1:length(tmpdims)
0196 mydim_names{inm} = name(tmpdims{inm});
0197 mydim_sizes{inm} = tmpdims{inm}(:);
0198 end
0199 for iat = 1:length(att(ncv))
0200 aaa = att(ncv);
0201 myatt.names(iat) = { name(aaa{iat}) };
0202 myatt.types(iat) = { datatype(aaa{iat}) };
0203 aab = aaa{iat};
0204 myatt.data(iat) = { aab(:) };
0205 end
0206
0207 % confirm: vars have same dim names across all files
0208 ierr = 0;
0209 for itile = 2:length(all_ncf)
0210 tncf = all_ncf(itile).nc{1};
0211 ncv = tncf{myname};
0212 len = length(ncv);
0213 if length(ncv) == 0
0214 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
0215 ' so it will be ignored.'];
0216 disp(sprintf(warns,myname,all_ncf(itile).name));
0217 continue
0218 end
0219 tmpdims = dim(ncv);
0220 for inm = 1:length(tmpdims)
0221 if mydim_names{inm} ~= name(tmpdims{inm})
0222 warns = ...
0223 [' Warning: variable "%s" is not CONSISTENTLY defined.\n' ...
0224 ' It has different dimensions in different files so\n' ...
0225 ' so it will be ignored.'];
0226 disp(sprintf(warns,myname));
0227 ierr = 1;
0228 break
0229 end
0230 mydim_sizes{inm} = max([ tmpdims{inm}(:) mydim_sizes{inm} ]);
0231 end
0232
0233 end
0234
0235 if ierr == 0
0236 % check: does the variable have a "horizontal" component
0237 has_horiz = 0;
0238 horiz_names = { 'X' 'Y' 'Xp1' 'Yp1' };
0239 for id = 1:length(mydim_names)
0240 if length([intersect(horiz_names,mydim_names{id})]) > 0
0241 has_horiz = 1;
0242 end
0243 end
0244 % disp([ ' ' myname ' ' sprintf('%d',has_horiz) ]);
0245
0246 imy = length(myvars) + 1;
0247 myvars(imy).name = myname;
0248 myvars(imy).type = mytype;
0249 myvars(imy).dim_names = mydim_names;
0250 myvars(imy).dim_sizes = mydim_sizes;
0251 myvars(imy).atts = myatt;
0252 myvars(imy).has_horiz = has_horiz;
0253
05cd833f96 Mart*0254 % this is necessary to make it work with Matlab 6.5
0255 if isempty([mydims.names])
0256 addl = mydim_names;
0257 else
0258 addl = setdiff(mydim_names,[mydims.names]);
0259 end
92749caea6 Ed H*0260 for iaddl = 1:length(addl)
0261 np1 = length(mydims) + 1;
0262 mydims(np1).names = addl(iaddl);
0263 mydims(np1).lens = mydim_sizes(find(strcmp(addl(iaddl),mydim_names)));
0264 end
0265
0266 end
0267 end
0268
0269 % For exch == 2, we need to add a "face" dimension
0270 if all_ncf(1).exch == 2
0271 np1 = length(mydims) + 1;
0272 mydims(np1).names = { 'iface' };
0273 mydims(np1).lens = { length(unique([all_ncf.exch2_myFace])) };
0274 end
0275
0276 % myvars.name
0277 % myvars.dim_names
0278 % myvars.dim_sizes
0279 % myvars(2).dim_names
0280 % myvars(2).dim_names(4)
0281
0282 % mydims
0283 % length(mydims)
0284 % [ mydims.names ]
0285 % [ mydims.lens ]
0286
0287
0288 %===== Assemble! =====
0289
0290
0291 if all_ncf(1).exch == 1
0292
0293 % exch "1":
0294
072e7c3589 Mart*0295 % $$$ bi_max = max([all_ncf.bi]);
0296 % $$$ bj_max = max([all_ncf.bj]);
0297 % $$$ Xmax = bi_max * all_ncf(1).sNx;
0298 % $$$ Ymax = bj_max * all_ncf(1).sNy;
0299 Xmax = all_ncf(1).Nx;
0300 Ymax = all_ncf(1).Ny;
0301 % at this point I have to make some assumptions about the domain
0302 % decomposition
0303 bi_max = Xmax/all_ncf(1).sNx;
0304 bj_max = Ymax/all_ncf(1).sNy;
0305 itile = 0;
0306 for bj=1:bj_max
0307 for bi=1:bi_max
0308 itile = itile+1;
0309 all_ncf(itile).bi=bi;
0310 all_ncf(itile).bj=bj;
0311 end
0312 end
0313
92749caea6 Ed H*0314 horzdim = struct('names',{},'lens',{});
0315 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
0316 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
0317 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
0318 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
0319 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
0320
0321 iseq = 0;
0322 foutnm = sprintf(fout, iseq);
0323 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
0324
0325 for idim = 1:length(mydims)
0326 dname = mydims(idim).names{1};
0327 ind = find(strcmp(dname,[horzdim.names]));
0328 if length(ind) ~= 0
0329 dlen = horzdim(ind).lens{1};
0330 else
0331 dlen = mydims(idim).lens{1};
0332 end
0333 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
0334 eval(comm);
0335 end
0336
0337 for ivar = 1:length(myvars)
0338 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
0339 id = 1;
0340 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
0341 for id = 2:length(myvars(ivar).dim_names)
0342 comm = [ comm sprintf(',''%s''',myvars(ivar).dim_names{id}) ];
0343 end
0344 comm = [ comm ' );' ];
0345 eval(comm);
0346 for iat = 1:length(myvars(ivar).atts.names)
0347 comm = sprintf( ...
0348 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
0349 myvars(ivar).name, ...
0350 myvars(ivar).atts.names{iat}, ...
0351 myvars(ivar).atts.types{iat} );
0352 eval(comm);
0353 end
0354 end
0355
0356 % for itime = 1:Tmax
0357
0358 % Here is where we need to check the output file size and start
0359 % another file in the sequence, if necessary.
0360
0361 for ivar = 1:length(myvars)
0362 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
0363 for itile = 1:length(all_ncf)
0364
072e7c3589 Mart*0365 if (myvars(ivar).has_horiz == 1) | (itile == 1)
92749caea6 Ed H*0366
0367 clear nct;
0368 nct = all_ncf(itile).nc{1};
0369 ox_off = (all_ncf(itile).bi - 1)*all_ncf(itile).sNx;
0370 oy_off = (all_ncf(itile).bj - 1)*all_ncf(itile).sNy;
0371 diml_in = '';
0372 diml_out = '';
0373 for jj = 1:length(myvars(ivar).dim_names)
0374 doff = 1;
0375 if jj > 1
0376 diml_in = sprintf('%s,',diml_in);
0377 diml_out = sprintf('%s,',diml_out);
0378 end
0379 dlen = myvars(ivar).dim_sizes{jj};
0380 diml_in = sprintf('%s%s',diml_in, ':');
0381 fchar = myvars(ivar).dim_names{jj}(1);
0382 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
0383 if strcmp(myvars(ivar).dim_names{jj}(1),'X') == 1
0384 doff = ox_off + doff;
0385 dlen = ox_off + dlen;
0386 end
0387 if strcmp(myvars(ivar).dim_names{jj}(1),'Y') == 1
0388 doff = oy_off + doff;
0389 dlen = oy_off + dlen;
0390 end
0391 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
0392 end
0393
0394 comm = sprintf( ...
0395 'fonc{''%s''}(%s) = nct{''%s''}(%s);', ...
0396 myvars(ivar).name, diml_out, myvars(ivar).name, diml_in );
0397 % disp([ ' comm: ' comm ]);
0398 eval(comm);
0399
0400 end
0401
0402 end
0403 end
0404 % end
0405
0406 fonc = close(fonc);
0407
0408 elseif all_ncf(1).exch == 2
0409
0410 % exch "2":
0411 Xmax = 0;
0412 Ymax = 0;
0413 for ii = 1:length(all_ncf)
0414 Xmax = max(Xmax, (all_ncf(ii).exch2_tbasex + all_ncf(ii).sNx));
0415 Ymax = max(Ymax, (all_ncf(ii).exch2_tbasey + all_ncf(ii).sNy));
0416 end
0417
0418 horzdim = struct('names',{},'lens',{});
0419 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
0420 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
0421 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
0422 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
0423 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
0424
0425 iseq = 0;
0426 foutnm = sprintf(fout, iseq);
0427 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
0428
0429 for idim = 1:length(mydims)
0430 dname = mydims(idim).names{1};
0431 ind = find(strcmp(dname,[horzdim.names]));
0432 if length(ind) ~= 0
0433 dlen = horzdim(ind).lens{1};
0434 else
0435 dlen = mydims(idim).lens{1};
0436 end
0437 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
0438 eval(comm);
0439 end
0440
0441 for ivar = 1:length(myvars)
0442 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
0443 id = 1;
0444 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
0445 for id = 2:length(myvars(ivar).dim_names)
0446 dname = myvars(ivar).dim_names{id};
072e7c3589 Mart*0447 if (dname(1) == 'Y') & (myvars(ivar).has_horiz == 1)
92749caea6 Ed H*0448 comm = [ comm sprintf(',''%s''','iface') ];
0449 end
0450 comm = [ comm sprintf(',''%s''',dname) ];
0451 end
0452 comm = [ comm ' );' ];
0453 eval(comm);
0454 for iat = 1:length(myvars(ivar).atts.names)
0455 comm = sprintf( ...
0456 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
0457 myvars(ivar).name, ...
0458 myvars(ivar).atts.names{iat}, ...
0459 myvars(ivar).atts.types{iat} );
0460 eval(comm);
0461 end
0462 end
0463
0464 % Here is where we need to check the output file size and start
0465 % another file in the sequence, if necessary.
0466
0467 for ivar = 1:length(myvars)
0468 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
0469 for itile = 1:length(all_ncf)
0470
072e7c3589 Mart*0471 if (myvars(ivar).has_horiz == 1) | (itile == 1)
92749caea6 Ed H*0472
0473 clear nct;
0474 nct = all_ncf(itile).nc{1};
0475 ox_off = all_ncf(itile).exch2_tbasex;
0476 oy_off = all_ncf(itile).exch2_tbasey;
0477 diml_tin = '';
0478 diml_res = '';
0479 diml_in = '';
0480 diml_out = '';
0481 if length(myvars(ivar).dim_names) < 2
0482 comm = sprintf( ...
0483 'fonc{''%s''}(%s%d) = nct{''%s''}(:);', ...
0484 myvars(ivar).name, '1:', myvars(ivar).dim_sizes{1}, ...
0485 myvars(ivar).name );
0486 % disp([ ' ' comm ]);
0487 eval(comm);
0488 else
0489 for jj = 1:length(myvars(ivar).dim_names)
0490 doff = 1;
0491 if jj > 1
0492 diml_tin = sprintf('%s,',diml_tin);
0493 diml_res = sprintf('%s,',diml_res);
0494 diml_in = sprintf('%s,',diml_in);
0495 diml_out = sprintf('%s,',diml_out);
0496 end
0497 dnam = myvars(ivar).dim_names{jj};
0498 dlen = myvars(ivar).dim_sizes{jj};
0499 dlenr = dlen;
0500 fchar = myvars(ivar).dim_names{jj}(1);
0501 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
0502 if strcmp(dnam(1),'X') == 1
0503 doff = ox_off + doff;
0504 dlen = ox_off + dlen;
0505 end
0506 if strcmp(dnam(1),'Y') == 1
0507 diml_res = sprintf('%s%s',diml_res, '[],');
0508 diml_in = sprintf('%s%s',diml_in, ':,');
0509 diml_out = sprintf('%s%d%s',diml_out,all_ncf(itile).exch2_myFace,',');
0510 doff = oy_off + doff;
0511 dlen = oy_off + dlen;
0512 end
0513 diml_tin = sprintf('%s%s',diml_tin, ':');
0514 diml_res = sprintf('%s%d',diml_res, dlenr);
0515 diml_in = sprintf('%s%s',diml_in, ':');
0516 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
0517 end
0518
0519 comm = sprintf( ...
0520 'tmp = reshape(nct{''%s''}(%s), %s); fonc{''%s''}(%s) = tmp(%s);', ...
0521 myvars(ivar).name, diml_tin, diml_res, myvars(ivar).name, ...
0522 diml_out, diml_in );
0523 % disp([ ' ' comm ]);
0524 eval(comm);
0525 end
0526
0527 end
0528
0529 end
0530 end
0531 % end
0532
0533 fonc = close(fonc);
0534
0535 end
0536
