pkg/flt/exch_send_put_vec.F

aa582c8e8c Jean*0001 #include "CPP_EEOPTIONS.h"
                0002 #undef DBUG_EXCH_VEC
eacecc7041 Jean*0003
                0004 C--   Contents
48e4fc2750 Jean*0005 C--   o EXCH_SEND_PUT_VEC_X_RL
                0006 C--   o EXCH_SEND_PUT_VEC_Y_RL
eacecc7041 Jean*0007
                0008 C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
aa582c8e8c Jean*0009 CBOP 0
48e4fc2750 Jean*0010 C !ROUTINE: EXCH_SEND_PUT_VEC_X_RL
c806179eb4 Alis*0011
aa582c8e8c Jean*0012 C !INTERFACE:
48e4fc2750 Jean*0013       SUBROUTINE EXCH_SEND_PUT_VEC_X_RL(
121ecfb57a Jean*0014      I                        arrayE, arrayW,
                0015      O                        bufRecE, bufRecW,
                0016      I                        myd1, myThid )
aa582c8e8c Jean*0017 C     !DESCRIPTION:
                0018 C     *==========================================================*
48e4fc2750 Jean*0019 C     | SUBROUTINE EXCH_SEND_PUT_VEC_X_RL
aa582c8e8c Jean*0020 C     | o "Send" or "put" X edges for RL array.
                0021 C     *==========================================================*
                0022 C     | Routine that invokes actual message passing send or
bd12238ff3 Jean*0023 C     |   direct "put" of data to update buffer in X direction
                0024 C     | Note: Since only master-thread send/put, assumes input
                0025 C     |       & output arrays are shared (i.e. incommon block)
aa582c8e8c Jean*0026 C     *==========================================================*
                0027
                0028 C     !USES:
c806179eb4 Alis*0029       IMPLICIT NONE
                0030
                0031 C     == Global variables ==
                0032 #include "SIZE.h"
                0033 #include "EEPARAMS.h"
                0034 #include "EESUPPORT.h"
                0035 #include "EXCH.h"
aa582c8e8c Jean*0036
                0037 C     !INPUT/OUTPUT PARAMETERS:
                0038 C     arrayE        :: Input buffer array to send to Eastern Neighbour
                0039 C     arrayW        :: Input buffer array to send to Western Neighbour
                0040 C     bufRecE       :: buffer array to collect Eastern Neighbour values
                0041 C     bufRecW       :: buffer array to collect Western Neighbour values
                0042 C     myd1          :: size
                0043 C     myThid        :: my Thread Id. number
c806179eb4 Alis*0044       INTEGER myd1
121ecfb57a Jean*0045       _RL  arrayE(myd1,nSx,nSy),  arrayW(myd1,nSx,nSy)
                0046       _RL bufRecE(myd1,nSx,nSy), bufRecW(myd1,nSx,nSy)
c806179eb4 Alis*0047       INTEGER myThid
aa582c8e8c Jean*0048 CEOP
                0049
                0050 C     !LOCAL VARIABLES:
                0051 C     I             :: Loop counters
                0052 C     bi, bj        :: tile indices
                0053 C     biW, bjW      :: West tile indices
                0054 C     biE, bjE      :: East tile indices
b0bdd58b37 Chri*0055 C     theProc       :: Variables used in message building
                0056 C     theTag        :: Variables used in message building
                0057 C     theType       :: Variables used in message building
                0058 C     theSize       :: Variables used in message building
aa582c8e8c Jean*0059 C     westCommMode  :: variables holding type of communication
                0060 C     eastCommMode  ::  a particular tile face uses.
eacecc7041 Jean*0061       INTEGER I
c806179eb4 Alis*0062       INTEGER bi, bj, biW, bjW, biE, bjE
                0063       INTEGER westCommMode
                0064       INTEGER eastCommMode
                0065 #ifdef ALLOW_USE_MPI
                0066       INTEGER theProc, theTag, theType, theSize, mpiRc
                0067 #endif
aa582c8e8c Jean*0068 #ifdef DBUG_EXCH_VEC
                0069       INTEGER ioUnit
                0070 #endif
                0071
                0072 C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
c806179eb4 Alis*0073 C--   Write data to exchange buffer
ef53b829d7 Jean*0074 C     Various actions are possible depending on the communication mode
c806179eb4 Alis*0075 C     as follows:
                0076 C       Mode      Action
                0077 C     --------   ---------------------------
                0078 C     COMM_NONE  Do nothing
                0079 C
                0080 C     COMM_MSG   Message passing communication ( e.g. MPI )
                0081 C                Fill west send buffer from this tile.
                0082 C                Send data with tag identifying tile and direction.
                0083 C                Fill east send buffer from this tile.
                0084 C                Send data with tag identifying tile and direction.
                0085 C
                0086 C     COMM_PUT   "Put" communication ( UMP_, shmemput, etc... )
                0087 C                Fill east receive buffer of west-neighbor tile
                0088 C                Fill west receive buffer of east-neighbor tile
                0089 C                Sync. memory
bd12238ff3 Jean*0090 C                Write data-ready Ack for east edge of west-neighbor tile
                0091 C                Write data-ready Ack for west edge of east-neighbor tile
c806179eb4 Alis*0092 C                Sync. memory
aa582c8e8c Jean*0093
bd12238ff3 Jean*0094 C     Prevent anyone to access shared buffer while an other thread modifies it
                0095       _BARRIER
                0096
                0097       _BEGIN_MASTER(myThid)
                0098
aa582c8e8c Jean*0099 #ifdef DBUG_EXCH_VEC
                0100       ioUnit = errorMessageUnit
                0101       WRITE(ioUnit,'(A,2L5)')
                0102      &    'SEND_PUT_X: exchNeedsMemsync,exchUsesBarrier=',
                0103      &     exchNeedsMemsync,exchUsesBarrier
                0104 #endif
                0105
bd12238ff3 Jean*0106       DO bj=1,nSy
                0107        DO bi=1,nSx
c806179eb4 Alis*0108
                0109         westCommMode  = _tileCommModeW(bi,bj)
                0110         eastCommMode  = _tileCommModeE(bi,bj)
                0111         biE =  _tileBiE(bi,bj)
                0112         bjE =  _tileBjE(bi,bj)
                0113         biW =  _tileBiW(bi,bj)
                0114         bjW =  _tileBjW(bi,bj)
                0115
                0116 C       o Send or Put west edge
                0117         IF ( westCommMode .EQ. COMM_MSG  ) THEN
                0118 C        Send the data
                0119 #ifdef ALLOW_USE_MPI
                0120          IF ( usingMPI ) THEN
48e4fc2750 Jean*0121           theProc = tilePidW(bi,bj)
                0122           theTag  = _tileTagSendW(bi,bj)
                0123           theSize = myd1
                0124           theType = _MPI_TYPE_RL
                0125           exchNReqsX(1,bi,bj) = exchNReqsX(1,bi,bj)+1
aa582c8e8c Jean*0126 #ifdef DBUG_EXCH_VEC
                0127           write(ioUnit,'(A,7I5,I8)') 'qq1xW: ',myProcId,bi,bj,
48e4fc2750 Jean*0128      &          theProc,theTag, exchNReqsX(1,bi,bj),
                0129      &          exchReqIdX(exchNReqsX(1,bi,bj),1,bi,bj), theSize
aa582c8e8c Jean*0130 #endif
48e4fc2750 Jean*0131           CALL MPI_Isend( arrayW(1,bi,bj), theSize, theType,
                0132      &                    theProc, theTag, MPI_COMM_MODEL,
                0133      &                    exchReqIdX(exchNReqsX(1,bi,bj),1,bi,bj),
                0134      &                    mpiRc )
c806179eb4 Alis*0135          ENDIF
                0136 #endif /* ALLOW_USE_MPI */
aa582c8e8c Jean*0137          eastRecvAck(1,biW,bjW) = 1
c806179eb4 Alis*0138         ELSEIF ( westCommMode .EQ. COMM_PUT  ) THEN
121ecfb57a Jean*0139 c         write(0,*) 'SEND_PUT_VEC_X: copy E:',biW,bjW,' <- W:',bi,bj
c806179eb4 Alis*0140           DO I=1,myd1
121ecfb57a Jean*0141             bufRecE(I,biW,bjW) = arrayW(I,bi,bj)
c806179eb4 Alis*0142           ENDDO
121ecfb57a Jean*0143         ELSEIF ( westCommMode .NE. COMM_NONE ) THEN
c806179eb4 Alis*0144          STOP ' S/R EXCH: Invalid commW mode.'
                0145         ENDIF
                0146
                0147 C       o Send or Put east edge
                0148         IF ( eastCommMode .EQ. COMM_MSG  ) THEN
                0149 C        Send the data
                0150 #ifdef ALLOW_USE_MPI
                0151          IF ( usingMPI ) THEN
48e4fc2750 Jean*0152           theProc = tilePidE(bi,bj)
                0153           theTag  = _tileTagSendE(bi,bj)
                0154           theSize = myd1
                0155           theType = _MPI_TYPE_RL
                0156           exchNReqsX(1,bi,bj) = exchNReqsX(1,bi,bj)+1
aa582c8e8c Jean*0157 #ifdef DBUG_EXCH_VEC
c806179eb4 Alis*0158 c         if (theProc .eq. 2 .or. theProc .eq. 4) then
                0159 c         if (arrayE(1,bi,bj) .ne. 0.) then
aa582c8e8c Jean*0160           write(ioUnit,'(A,7I5,I8)') 'qq1xE: ',myProcId,bi,bj,
                0161      &       theProc,theTag, exchNReqsX(1,bi,bj),
                0162      &       exchReqIdX(exchNReqsX(1,bi,bj),1,bi,bj), theSize
c806179eb4 Alis*0163 c         endif
                0164 c         endif
aa582c8e8c Jean*0165 #endif
48e4fc2750 Jean*0166           CALL MPI_Isend( arrayE(1,bi,bj), theSize, theType,
                0167      &                    theProc, theTag, MPI_COMM_MODEL,
                0168      &                    exchReqIdX(exchNReqsX(1,bi,bj),1,bi,bj),
                0169      &                    mpiRc)
c806179eb4 Alis*0170          ENDIF
                0171 #endif /* ALLOW_USE_MPI */
aa582c8e8c Jean*0172          westRecvAck(1,biE,bjE) = 1
c806179eb4 Alis*0173         ELSEIF ( eastCommMode .EQ. COMM_PUT  ) THEN
121ecfb57a Jean*0174 c         write(0,*) 'SEND_PUT_VEC_X: copy W:',biE,bjE,' <- E:',bi,bj
c806179eb4 Alis*0175           DO I=1,myd1
121ecfb57a Jean*0176             bufRecW(I,biE,bjE) = arrayE(I,bi,bj)
c806179eb4 Alis*0177           ENDDO
121ecfb57a Jean*0178         ELSEIF ( eastCommMode .NE. COMM_NONE ) THEN
c806179eb4 Alis*0179          STOP ' S/R EXCH: Invalid commE mode.'
                0180         ENDIF
                0181
                0182        ENDDO
                0183       ENDDO
                0184
bd12238ff3 Jean*0185       _END_MASTER(myThid)
                0186
c806179eb4 Alis*0187 C--   Signal completetion ( making sure system-wide memory state is
                0188 C--                         consistent ).
                0189
                0190 C     ** NOTE ** We are relying on being able to produce strong-ordered
                0191 C     memory semantics here. In other words we assume that there is a
                0192 C     mechanism which can ensure that by the time the Ack is seen the
                0193 C     overlap region data that will be exchanged is up to date.
                0194       IF ( exchNeedsMemSync  ) CALL MEMSYNC
                0195
                0196       DO bj=myByLo(myThid),myByHi(myThid)
                0197        DO bi=myBxLo(myThid),myBxHi(myThid)
                0198         biE = _tileBiE(bi,bj)
                0199         bjE = _tileBjE(bi,bj)
                0200         biW = _tileBiW(bi,bj)
                0201         bjW = _tileBjW(bi,bj)
                0202         westCommMode = _tileCommModeW(bi,bj)
                0203         eastCommMode = _tileCommModeE(bi,bj)
aa582c8e8c Jean*0204         IF ( westCommMode .EQ. COMM_PUT ) eastRecvAck(1,biW,bjW) = 1
                0205         IF ( eastCommMode .EQ. COMM_PUT ) westRecvAck(1,biE,bjE) = 1
                0206         IF ( westCommMode .EQ. COMM_GET ) eastRecvAck(1,biW,bjW) = 1
                0207         IF ( eastCommMode .EQ. COMM_GET ) westRecvAck(1,biE,bjE) = 1
c806179eb4 Alis*0208        ENDDO
                0209       ENDDO
                0210
                0211 C--   Make sure "ack" setting is seen system-wide.
                0212 C     Here strong-ordering is not an issue but we want to make
                0213 C     sure that processes that might spin on the above Ack settings
                0214 C     will see the setting.
                0215 C     ** NOTE ** On some machines we wont spin on the Ack setting
                0216 C     ( particularly the T90 ), instead we will use s system barrier.
ef53b829d7 Jean*0217 C     On the T90 the system barrier is very fast and switches out the
c806179eb4 Alis*0218 C     thread while it waits. On most machines the system barrier
                0219 C     is much too slow and if we own the machine and have one thread
                0220 C     per process preemption is not a problem.
                0221       IF ( exchNeedsMemSync  ) CALL MEMSYNC
                0222
bd12238ff3 Jean*0223 C     Wait until all threads finish filling buffer <-- jmc: really needed ?
                0224       _BARRIER
                0225
ef53b829d7 Jean*0226       RETURN
c806179eb4 Alis*0227       END
                0228
eacecc7041 Jean*0229 C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
aa582c8e8c Jean*0230 CBOP 0
48e4fc2750 Jean*0231 C !ROUTINE: EXCH_SEND_PUT_VEC_Y_RL
eacecc7041 Jean*0232
aa582c8e8c Jean*0233 C !INTERFACE:
48e4fc2750 Jean*0234       SUBROUTINE EXCH_SEND_PUT_VEC_Y_RL(
121ecfb57a Jean*0235      I                        arrayN, arrayS,
                0236      O                        bufRecN, bufRecS,
                0237      I                        myd1, myThid )
aa582c8e8c Jean*0238 C     !DESCRIPTION:
                0239 C     *==========================================================*
48e4fc2750 Jean*0240 C     | SUBROUTINE EXCH_SEND_PUT_VEC_Y_RL
aa582c8e8c Jean*0241 C     | o "Send" or "put" Y edges for RL array.
                0242 C     *==========================================================*
                0243 C     | Routine that invokes actual message passing send or
bd12238ff3 Jean*0244 C     |   direct "put" of data to update buffer in X direction
                0245 C     | Note: Since only master-thread send/put, assumes input
                0246 C     |       & output arrays are shared (i.e. incommon block)
aa582c8e8c Jean*0247 C     *==========================================================*
                0248
                0249 C     !USES:
c806179eb4 Alis*0250       IMPLICIT NONE
                0251
                0252 C     == Global variables ==
                0253 #include "SIZE.h"
                0254 #include "EEPARAMS.h"
                0255 #include "EESUPPORT.h"
                0256 #include "EXCH.h"
aa582c8e8c Jean*0257
                0258 C     !INPUT/OUTPUT PARAMETERS:
                0259 C     arrayN        :: buffer array to collect Northern Neighbour values
                0260 C     arrayS        :: buffer array to collect Southern Neighbour values
                0261 C     myd1          :: size
                0262 C     myThid        :: my Thread Id. number
                0263 C     arrayN        :: Input buffer array to send to Northern Neighbour
                0264 C     arrayS        :: Input buffer array to send to Southern Neighbour
                0265 C     bufRecN       :: buffer array to collect Northern Neighbour values
                0266 C     bufRecS       :: buffer array to collect Southern Neighbour values
                0267 C     myd1          :: size
                0268 C     myThid        :: my Thread Id. number
c806179eb4 Alis*0269       INTEGER myd1
121ecfb57a Jean*0270       _RL  arrayN(myd1,nSx,nSy),  arrayS(myd1,nSx,nSy)
                0271       _RL bufRecN(myd1,nSx,nSy), bufRecS(myd1,nSx,nSy)
c806179eb4 Alis*0272       INTEGER myThid
aa582c8e8c Jean*0273 CEOP
                0274
                0275 C     !LOCAL VARIABLES:
                0276 C     I             :: Loop index
                0277 C     bi, bj        :: tile indices
                0278 C     biS, bjS      :: South tile indices
                0279 C     biN, bjN      :: North tile indices
b0bdd58b37 Chri*0280 C     theProc       :: Variables used in message building
                0281 C     theTag        :: Variables used in message building
bd12238ff3 Jean*0282 C     theType       :: Variables used in message building
b0bdd58b37 Chri*0283 C     theSize       :: Variables used in message building
aa582c8e8c Jean*0284 C     southCommMode :: variables holding type of communication
                0285 C     northCommMode ::  a particular tile face uses.
eacecc7041 Jean*0286       INTEGER I
c806179eb4 Alis*0287       INTEGER bi, bj, biS, bjS, biN, bjN
                0288       INTEGER southCommMode
                0289       INTEGER northCommMode
                0290 #ifdef ALLOW_USE_MPI
                0291       INTEGER theProc, theTag, theType, theSize, mpiRc
                0292 #endif
aa582c8e8c Jean*0293 #ifdef DBUG_EXCH_VEC
                0294       INTEGER ioUnit
                0295 #endif
                0296
                0297 C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
c806179eb4 Alis*0298 C--   Write data to exchange buffer
ef53b829d7 Jean*0299 C     Various actions are possible depending on the communication mode
c806179eb4 Alis*0300 C     as follows:
                0301 C       Mode      Action
                0302 C     --------   ---------------------------
                0303 C     COMM_NONE  Do nothing
                0304 C
                0305 C     COMM_MSG   Message passing communication ( e.g. MPI )
                0306 C                Fill west send buffer from this tile.
                0307 C                Send data with tag identifying tile and direction.
                0308 C                Fill east send buffer from this tile.
                0309 C                Send data with tag identifying tile and direction.
                0310 C
                0311 C     COMM_PUT   "Put" communication ( UMP_, shmemput, etc... )
                0312 C                Fill east receive buffer of south-neighbor tile
                0313 C                Fill west receive buffer of north-neighbor tile
                0314 C                Sync. memory
bd12238ff3 Jean*0315 C                Write data-ready Ack for east edge of south-neighbor tile
                0316 C                Write data-ready Ack for west edge of north-neighbor tile
c806179eb4 Alis*0317 C                Sync. memory
aa582c8e8c Jean*0318
bd12238ff3 Jean*0319 C     Prevent anyone to access shared buffer while an other thread modifies it
                0320       _BARRIER
                0321
                0322       _BEGIN_MASTER(myThid)
                0323
aa582c8e8c Jean*0324 #ifdef DBUG_EXCH_VEC
                0325       ioUnit = errorMessageUnit
                0326 #endif
                0327
bd12238ff3 Jean*0328       DO bj=1,nSy
                0329        DO bi=1,nSx
c806179eb4 Alis*0330
                0331         southCommMode  = _tileCommModeS(bi,bj)
                0332         northCommMode  = _tileCommModeN(bi,bj)
                0333         biN =  _tileBiN(bi,bj)
                0334         bjN =  _tileBjN(bi,bj)
                0335         biS =  _tileBiS(bi,bj)
                0336         bjS =  _tileBjS(bi,bj)
                0337
                0338 C       o Send or Put south edge
                0339         IF ( southCommMode .EQ. COMM_MSG  ) THEN
                0340 C        Send the data
                0341 #ifdef ALLOW_USE_MPI
                0342          IF ( usingMPI ) THEN
48e4fc2750 Jean*0343           theProc = tilePidS(bi,bj)
                0344           theTag  = _tileTagSendS(bi,bj)
                0345           theSize = myd1
                0346           theType = _MPI_TYPE_RL
                0347           exchNReqsY(1,bi,bj) = exchNReqsY(1,bi,bj)+1
                0348           CALL MPI_Isend( arrayS(1,bi,bj), theSize, theType,
                0349      &                    theProc, theTag, MPI_COMM_MODEL,
                0350      &                    exchReqIdY(exchNReqsY(1,bi,bj),1,bi,bj),
                0351      &                    mpiRc )
c806179eb4 Alis*0352          ENDIF
                0353 #endif /* ALLOW_USE_MPI */
aa582c8e8c Jean*0354          northRecvAck(1,biS,bjS) = 1
c806179eb4 Alis*0355         ELSEIF ( southCommMode .EQ. COMM_PUT  ) THEN
121ecfb57a Jean*0356 c         write(0,*) 'SEND_PUT_VEC_Y: copy N:',biS,bjS,' <- S:',bi,bj
c806179eb4 Alis*0357           DO I=1,myd1
121ecfb57a Jean*0358             bufRecN(I,biS,bjS) = arrayS(I,bi,bj)
c806179eb4 Alis*0359           ENDDO
121ecfb57a Jean*0360         ELSEIF ( southCommMode .NE. COMM_NONE ) THEN
c806179eb4 Alis*0361          STOP ' S/R EXCH: Invalid commS mode.'
                0362         ENDIF
                0363
                0364 C       o Send or Put north edge
                0365         IF ( northCommMode .EQ. COMM_MSG  ) THEN
                0366 C        Send the data
                0367 #ifdef ALLOW_USE_MPI
                0368          IF ( usingMPI ) THEN
48e4fc2750 Jean*0369           theProc = tilePidN(bi,bj)
                0370           theTag  = _tileTagSendN(bi,bj)
                0371           theSize = myd1
                0372           theType = _MPI_TYPE_RL
                0373           exchNReqsY(1,bi,bj) = exchNReqsY(1,bi,bj)+1
                0374           CALL MPI_Isend( arrayN(1,bi,bj), theSize, theType,
                0375      &                    theProc, theTag, MPI_COMM_MODEL,
                0376      &                    exchReqIdY(exchNReqsY(1,bi,bj),1,bi,bj),
                0377      &                    mpiRc )
c806179eb4 Alis*0378          ENDIF
                0379 #endif /* ALLOW_USE_MPI */
aa582c8e8c Jean*0380          southRecvAck(1,biN,bjN) = 1
c806179eb4 Alis*0381         ELSEIF ( northCommMode .EQ. COMM_PUT  ) THEN
121ecfb57a Jean*0382 c         write(0,*) 'SEND_PUT_VEC_Y: copy S:',biN,bjN,' <- N:',bi,bj
c806179eb4 Alis*0383           DO I=1,myd1
121ecfb57a Jean*0384             bufRecS(I,biN,bjN) = arrayN(I,bi,bj)
c806179eb4 Alis*0385           ENDDO
121ecfb57a Jean*0386         ELSEIF ( northCommMode .NE. COMM_NONE ) THEN
c806179eb4 Alis*0387          STOP ' S/R EXCH: Invalid commN mode.'
                0388         ENDIF
                0389
                0390        ENDDO
                0391       ENDDO
                0392
bd12238ff3 Jean*0393       _END_MASTER(myThid)
                0394
c806179eb4 Alis*0395 C--   Signal completetion ( making sure system-wide memory state is
                0396 C--                         consistent ).
                0397
                0398 C     ** NOTE ** We are relying on being able to produce strong-ordered
                0399 C     memory semantics here. In other words we assume that there is a
                0400 C     mechanism which can ensure that by the time the Ack is seen the
                0401 C     overlap region data that will be exchanged is up to date.
                0402       IF ( exchNeedsMemSync  ) CALL MEMSYNC
                0403
                0404       DO bj=myByLo(myThid),myByHi(myThid)
                0405        DO bi=myBxLo(myThid),myBxHi(myThid)
                0406         biN = _tileBiN(bi,bj)
                0407         bjN = _tileBjN(bi,bj)
                0408         biS = _tileBiS(bi,bj)
                0409         bjS = _tileBjS(bi,bj)
aa582c8e8c Jean*0410         southCommMode = _tileCommModeS(bi,bj)
c806179eb4 Alis*0411         northCommMode = _tileCommModeN(bi,bj)
aa582c8e8c Jean*0412         IF ( southCommMode .EQ. COMM_PUT ) northRecvAck(1,biS,bjS) = 1
                0413         IF ( northCommMode .EQ. COMM_PUT ) southRecvAck(1,biN,bjN) = 1
                0414         IF ( southCommMode .EQ. COMM_GET ) northRecvAck(1,biS,bjS) = 1
                0415         IF ( northCommMode .EQ. COMM_GET ) southRecvAck(1,biN,bjN) = 1
c806179eb4 Alis*0416        ENDDO
                0417       ENDDO
                0418
                0419 C--   Make sure "ack" setting is seen system-wide.
                0420 C     Here strong-ordering is not an issue but we want to make
                0421 C     sure that processes that might spin on the above Ack settings
                0422 C     will see the setting.
                0423 C     ** NOTE ** On some machines we wont spin on the Ack setting
                0424 C     ( particularly the T90 ), instead we will use s system barrier.
ef53b829d7 Jean*0425 C     On the T90 the system barrier is very fast and switches out the
c806179eb4 Alis*0426 C     thread while it waits. On most machines the system barrier
                0427 C     is much too slow and if we own the machine and have one thread
                0428 C     per process preemption is not a problem.
                0429       IF ( exchNeedsMemSync  ) CALL MEMSYNC
                0430
bd12238ff3 Jean*0431 C     Wait until all threads finish filling buffer <-- jmc: really needed ?
                0432       _BARRIER
                0433
ef53b829d7 Jean*0434       RETURN
c806179eb4 Alis*0435       END