seaice_itd/code/EXCH.h

0ba3967dec Mart*0001 C
                0002 CBOP
                0003 C     !ROUTINE: EXCH.h
                0004 C     !INTERFACE:
                0005 C     include "EXCH.h"
                0006 C
                0007 C     !DESCRIPTION:
                0008 C     *==========================================================*
                0009 C     | EXCH.h
                0010 C     *==========================================================*
                0011 C     | Support data structures for
                0012 C     | the MITgcm-UV "exchange routines" code. This data should
                0013 C     | be private to the execution environment routines.
                0014 C     *==========================================================*
                0015 CEOP
                0016
                0017 #ifndef _RL
                0018 #define _RL Real*8
                0019 #endif
                0020
                0021 #ifndef _RS
                0022 #define _RS Real*4
                0023 #endif
                0024
                0025 #ifndef _R8
                0026 #define _R8 Real*8
                0027 #endif
                0028
                0029 #ifndef _R4
                0030 #define _R4 Real*4
                0031 #endif
                0032
                0033 #ifndef _tileCommModeW
                0034 #define _tileCommModeW(a,b) tileCommModeW(a,b)
                0035 #endif
                0036
                0037 #ifndef _tileCommModeE
                0038 #define _tileCommModeE(a,b) tileCommModeE(a,b)
                0039 #endif
                0040
                0041 #ifndef _tileBiE
                0042 #define _tileBiE(a,b) tileBiE(a,b)
                0043 #endif
                0044
                0045 #ifndef _tileBiW
                0046 #define _tileBiW(a,b) tileBiW(a,b)
                0047 #endif
                0048
                0049 #ifndef _tileBjE
                0050 #define _tileBjE(a,b) tileBjE(a,b)
                0051 #endif
                0052
                0053 #ifndef _tileBjW
                0054 #define _tileBjW(a,b) tileBjW(a,b)
                0055 #endif
                0056
                0057 #ifndef _mpiPidW
                0058 #define _mpiPidW(a,b) pidW(a,b)
                0059 #endif
                0060
                0061 #ifndef _mpiIdE
                0062 #define _mpiIdE(a,b) pidE(a,b)
                0063 #endif
                0064
                0065 #ifndef _tileTagSendW
                0066 #define _tileTagSendW(a,b) tileTagSendW(a,b)
                0067 #endif
                0068
                0069 #ifndef _tileTagSendE
                0070 #define _tileTagSendE(a,b) tileTagSendE(a,b)
                0071 #endif
                0072
                0073 #ifndef _tileTagRecvW
                0074 #define _tileTagRecvW(a,b) tileTagRecvW(a,b)
                0075 #endif
                0076
                0077 #ifndef _tileTagRecvE
                0078 #define _tileTagRecvE(a,b) tileTagRecvE(a,b)
                0079 #endif
                0080
                0081 #ifndef _tileCommModeS
                0082 #define _tileCommModeS(a,b) tileCommModeS(a,b)
                0083 #endif
                0084
                0085 #ifndef _tileCommModeN
                0086 #define _tileCommModeN(a,b) tileCommModeN(a,b)
                0087 #endif
                0088
                0089 #ifndef _tileBiN
                0090 #define _tileBiN(a,b) tileBiN(a,b)
                0091 #endif
                0092
                0093 #ifndef _tileBiS
                0094 #define _tileBiS(a,b) tileBiS(a,b)
                0095 #endif
                0096
                0097 #ifndef _tileBjN
                0098 #define _tileBjN(a,b) tileBjN(a,b)
                0099 #endif
                0100
                0101 #ifndef _tileBjS
                0102 #define _tileBjS(a,b) tileBjS(a,b)
                0103 #endif
                0104
                0105 #ifndef _mpiPidS
                0106 #define _mpiPidS(a,b) pidS(a,b)
                0107 #endif
                0108
                0109 #ifndef _mpiIdN
                0110 #define _mpiIdN(a,b) pidN(a,b)
                0111 #endif
                0112
                0113 #ifndef _tileTagSendS
                0114 #define _tileTagSendS(a,b) tileTagSendS(a,b)
                0115 #endif
                0116
                0117 #ifndef _tileTagSendN
                0118 #define _tileTagSendN(a,b) tileTagSendN(a,b)
                0119 #endif
                0120
                0121 #ifndef _tileTagRecvS
                0122 #define _tileTagRecvS(a,b) tileTagRecvS(a,b)
                0123 #endif
                0124
                0125 #ifndef _tileTagRecvN
                0126 #define _tileTagRecvN(a,b) tileTagRecvN(a,b)
                0127 #endif
                0128
                0129 #ifndef _theSimulationMode
                0130 #define _theSimulationMode theSimulationMode
                0131 #endif
                0132
                0133 #ifndef _EXCH_SPIN_LIMIT
                0134 #define _EXCH_SPIN_LIMIT EXCH_SPIN_LIMIT
                0135 #endif
                0136
                0137 C      MAX_OLX_EXCH - Maximum overlap region allowed in X
                0138 C      MAX_OLY_EXCH - Maximum overlap region allowed in Y
                0139 C      MAX_NR_EXCH  - Maximum number of vertical levels allowed
                0140 C      NUMBER_OF_BUFFER_LEVELS - Number of levels of buffer allowed.
                0141 C      EXCH_SPIN_LIMIT - Error trapping threshold for deadlocked exchange
                0142        INTEGER MAX_OLX_EXCH
                0143        PARAMETER ( MAX_OLX_EXCH = MAX_OLX )
                0144        INTEGER MAX_OLY_EXCH
                0145        PARAMETER ( MAX_OLY_EXCH = MAX_OLY )
                0146        INTEGER MAX_NR_EXCH
                0147        PARAMETER ( MAX_NR_EXCH  = nR + 6 )
                0148        INTEGER NUMBER_OF_BUFFER_LEVELS
                0149        PARAMETER ( NUMBER_OF_BUFFER_LEVELS = 1 )
                0150        INTEGER EXCH_SPIN_LIMIT
                0151        PARAMETER ( EXCH_SPIN_LIMIT = 100000000 )
                0152
                0153 C
                0154 C      L_BUFFER[XY]  - Maximum size for exchange buffer in
                0155 C      L_WBUFFER    west,
                0156 C      L_EBUFFER    east,
                0157 C      L_SBUFFER   south,
                0158 C      L_NBUFFER   north.
                0159        INTEGER L_BUFFERX
                0160        PARAMETER ( L_BUFFERX =
                0161      &  (sNy+2*MAX_OLY_EXCH)*MAX_OLX_EXCH*MAX_NR_EXCH )
                0162        INTEGER L_BUFFERY
                0163        PARAMETER ( L_BUFFERY =
                0164      &  (sNx+2*MAX_OLX_EXCH)*MAX_OLY_EXCH*MAX_NR_EXCH )
                0165        INTEGER L_WBUFFER
                0166        INTEGER L_EBUFFER
                0167        INTEGER L_SBUFFER
                0168        INTEGER L_NBUFFER
                0169        PARAMETER ( L_WBUFFER = L_BUFFERX,
                0170      &             L_EBUFFER = L_BUFFERX,
                0171      &             L_SBUFFER = L_BUFFERY,
                0172      &             L_NBUFFER = L_BUFFERY )
                0173
                0174 C--    COMMON / EXCH_L / LOGICAL number common arrays for exchanges
                0175 C      exchNeedsMemSync - TRUE if memory sync. required to ensure
                0176 C                         memory consistency during exchange
                0177 C      exchUsesBarrier  - TRUE if we use a call to BAR to do sync.
                0178 C                         between processes. On some machines we wont
                0179 C                         spin on the Ack setting ( the T90 ),
                0180 C                         instead we will use s system barrier.
                0181 C                         On the T90 the system barrier is very fast and
                0182 C                         switches out the thread while it waits. On most
                0183 C                         machines the system barrier is much too slow and if
                0184 C                         we own the machine and have one thread per process
                0185 C                         preemption is not a problem.
                0186 C      exchCollectStatistics - Turns exchange statistics collecting on and off.
                0187
                0188        COMMON / EXCH_L / exchNeedsMemSync, exchUsesBarrier,
                0189      &                   exchCollectStatistics
                0190        LOGICAL exchNeedsMemSync
                0191        LOGICAL exchUsesBarrier
                0192        LOGICAL exchCollectStatistics
                0193
                0194 C--    COMMON / EXCH_R / REAL number common arrays for exchanges
                0195 C      xxxxSendBuf - Buffer used for sending data to another tile.
                0196 C      xxxxRecvBuf - Buffer used for receiving data from another tile.
                0197        COMMON / EXCH_R /
                0198      &  westSendBuf_RL, eastSendBuf_RL,
                0199      &  southSendBuf_RL, northSendBuf_RL,
                0200      &  westRecvBuf_RL, eastRecvBuf_RL,
                0201      &  southRecvBuf_RL, northRecvBuf_RL,
                0202      &  westSendBuf_RS, eastSendBuf_RS,
                0203      &  southSendBuf_RS, northSendBuf_RS,
                0204      &  westRecvBuf_RS, eastRecvBuf_RS,
                0205      &  southRecvBuf_RS, northRecvBuf_RS,
                0206      &  westSendBuf_R8, eastSendBuf_R8,
                0207      &  southSendBuf_R8, northSendBuf_R8,
                0208      &  westRecvBuf_R8, eastRecvBuf_R8,
                0209      &  southRecvBuf_R8, northRecvBuf_R8,
                0210      &  westSendBuf_R4, eastSendBuf_R4,
                0211      &  southSendBuf_R4, northSendBuf_R4,
                0212      &  westRecvBuf_R4, eastRecvBuf_R4,
                0213      &  southRecvBuf_R4, northRecvBuf_R4
                0214        _RL   westSendBuf_RL( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0215      &                       nSx, nSy )
                0216        _RL   eastSendBuf_RL( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0217      &                       nSx, nSy )
                0218        _RL  southSendBuf_RL( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0219      &                       nSx, nSy )
                0220        _RL  northSendBuf_RL( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0221      &                       nSx, nSy )
                0222        _RL   westRecvBuf_RL( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0223      &                       nSx, nSy )
                0224        _RL   eastRecvBuf_RL( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0225      &                       nSx, nSy )
                0226        _RL  southRecvBuf_RL( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0227      &                       nSx, nSy )
                0228        _RL  northRecvBuf_RL( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0229      &                       nSx, nSy )
                0230        _RS   westSendBuf_RS( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0231      &                       nSx, nSy )
                0232        _RS   eastSendBuf_RS( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0233      &                       nSx, nSy )
                0234        _RS  southSendBuf_RS( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0235      &                       nSx, nSy )
                0236        _RS  northSendBuf_RS( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0237      &                       nSx, nSy )
                0238        _RS   westRecvBuf_RS( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0239      &                       nSx, nSy )
                0240        _RS   eastRecvBuf_RS( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0241      &                       nSx, nSy )
                0242        _RS  southRecvBuf_RS( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0243      &                       nSx, nSy )
                0244        _RS  northRecvBuf_RS( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0245      &                       nSx, nSy )
                0246        _R8   westSendBuf_R8( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0247      &                       nSx, nSy )
                0248        _R8   eastSendBuf_R8( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0249      &                       nSx, nSy )
                0250        _R8  southSendBuf_R8( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0251      &                       nSx, nSy )
                0252        _R8  northSendBuf_R8( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0253      &                       nSx, nSy )
                0254        _R8   westRecvBuf_R8( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0255      &                       nSx, nSy )
                0256        _R8   eastRecvBuf_R8( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0257      &                       nSx, nSy )
                0258        _R8  southRecvBuf_R8( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0259      &                       nSx, nSy )
                0260        _R8  northRecvBuf_R8( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0261      &                       nSx, nSy )
                0262        _R4   westSendBuf_R4( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0263      &                       nSx, nSy )
                0264        _R4   eastSendBuf_R4( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0265      &                       nSx, nSy )
                0266        _R4  southSendBuf_R4( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0267      &                       nSx, nSy )
                0268        _R4  northSendBuf_R4( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0269      &                       nSx, nSy )
                0270        _R4   westRecvBuf_R4( L_WBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0271      &                       nSx, nSy )
                0272        _R4   eastRecvBuf_R4( L_EBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0273      &                       nSx, nSy )
                0274        _R4  southRecvBuf_R4( L_SBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0275      &                       nSx, nSy )
                0276        _R4  northRecvBuf_R4( L_NBUFFER, NUMBER_OF_BUFFER_LEVELS,
                0277      &                       nSx, nSy )
                0278
                0279 C--    COMMON / EXCH_I / INTEGER common arrays for exchanges
                0280 C      xxxxSendAck - Flag indicating ready to send data.
                0281 C      xxxxRecvAck - Falg indicating receive data is ready.
                0282 C      exchBufferLevel - Current cyclic buffer level.
                0283 C      exchNReqsX, exchNReqsY - Pending message counts
                0284 C      exchReqIdX, exchReqIdY -Pending message identifiers
                0285 C      *Spin* - Exchange statistics holder
                0286 C       Count - No. spins for each thread
                0287 C         Max - Maximum spins for an exchange
                0288 C         Min - Minimum spins for an exchange
                0289        COMMON / EXCH_I /
                0290      &  westSendAck, eastSendAck, southSendAck, northSendAck,
                0291      &  westRecvAck, eastRecvAck, southRecvAck, northRecvAck,
                0292      &  exchangeBufLevel,
                0293      &  exchNReqsX, exchNReqsY, exchReqIdX, exchReqIdY,
                0294      &  exchRecvXSpinCount, exchRecvXSpinMax, exchRecvXSpinMin,
                0295      &  exchRecvXExchCount,
                0296      &  exchRecvYSpinCount, exchRecvYSpinMax, exchRecvYSpinMin,
                0297      &  exchRecvYExchCount
                0298        INTEGER  westSendAck(            NUMBER_OF_BUFFER_LEVELS,
                0299      &                       nSx, nSy )
                0300        INTEGER  eastSendAck(            NUMBER_OF_BUFFER_LEVELS,
                0301      &                       nSx, nSy )
                0302        INTEGER southSendAck(            NUMBER_OF_BUFFER_LEVELS,
                0303      &                       nSx, nSy )
                0304        INTEGER northSendAck(            NUMBER_OF_BUFFER_LEVELS,
                0305      &                       nSx, nSy )
                0306        INTEGER  westRecvAck(            NUMBER_OF_BUFFER_LEVELS,
                0307      &                       nSx, nSy )
                0308        INTEGER  eastRecvAck(            NUMBER_OF_BUFFER_LEVELS,
                0309      &                       nSx, nSy )
                0310        INTEGER southRecvAck(            NUMBER_OF_BUFFER_LEVELS,
                0311      &                       nSx, nSy )
                0312        INTEGER northRecvAck(            NUMBER_OF_BUFFER_LEVELS,
                0313      &                       nSx, nSy )
                0314        INTEGER exchangeBufLevel(  cacheLineSize/4, nSx, nSy )
                0315        INTEGER exchNReqsX(cacheLineSize/4,nSx,nSy)
                0316        INTEGER exchNReqsY(cacheLineSize/4,nSx,nSy)
                0317        INTEGER exchReqIdX(2*nSx+2*nSy,cacheLineSize/4,nSx,nSy)
                0318        INTEGER exchReqIdY(2*nSx+2*nSy,cacheLineSize/4,nSx,nSy)
                0319        INTEGER exchRecvXSpinCount(cacheLineSize/4, nSx, nSy)
                0320        INTEGER exchRecvXExchCount(cacheLineSize/4, nSx, nSy)
                0321        INTEGER exchRecvXSpinMax  (cacheLineSize/4, nSx, nSy)
                0322        INTEGER exchRecvXSpinMin  (cacheLineSize/4, nSx, nSy)
                0323        INTEGER exchRecvYSpinCount(cacheLineSize/4, nSx, nSy)
                0324        INTEGER exchRecvYExchCount(cacheLineSize/4, nSx, nSy)
                0325        INTEGER exchRecvYSpinMax  (cacheLineSize/4, nSx, nSy)
                0326        INTEGER exchRecvYSpinMin  (cacheLineSize/4, nSx, nSy)
                0327