File indexing completed on 2023-11-30 06:11:11 UTC
view on githubraw file Latest commit f0ff6e91 on 2023-11-29 18:40:14 UTC
5fc0cd3689 Dani*0001 #include "STREAMICE_OPTIONS.h"
0002
0003
0004
0005
07e785229e dngo*0006 SUBROUTINE STREAMICE_CG_SOLVE_PETSC(
5fc0cd3689 Dani*0007 U cg_Uin,
0008 U cg_Vin,
0009 I cg_Bu,
0010 I cg_Bv,
0011 I A_uu,
0012 I A_uv,
0013 I A_vu,
0014 I A_vv,
07e785229e dngo*0015 I tolerance,
5fc0cd3689 Dani*0016 O iters,
d2cdb9260d Dani*0017 I maxIter,
5fc0cd3689 Dani*0018 I myThid )
8527100523 Dani*0019
8a34959769 dngo*0020
07e785229e dngo*0021
5fc0cd3689 Dani*0022
8a34959769 dngo*0023
0024
0025 #ifdef ALLOW_PETSC
0026 #ifdef STREAMICE_PETSC_3_8
f0ff6e912a dngo*0027
0028
0029
0030
0031
0032
0033
0034
0035
0036
8a34959769 dngo*0037 #include "petsc/finclude/petsc.h"
0038 #include "petsc/finclude/petscvec.h"
0039 use petscvec
0040 #include "petsc/finclude/petscmat.h"
0041 use petscmat
0042 #include "petsc/finclude/petscksp.h"
0043 use petscksp
0044 #include "petsc/finclude/petscpc.h"
0045 use petscpc
f0ff6e912a dngo*0046 #include "STREAMICE_PETSC_MOD.h"
8a34959769 dngo*0047 IMPLICIT NONE
0048 # else
5fc0cd3689 Dani*0049 IMPLICIT NONE
8a34959769 dngo*0050 #include "finclude/petsc.h"
f0ff6e912a dngo*0051 #include "STREAMICE_PETSC_MOD.h"
8a34959769 dngo*0052 #endif
0053 #endif
5fc0cd3689 Dani*0054
0055 #include "SIZE.h"
0056 #include "EEPARAMS.h"
0057 #include "PARAMS.h"
0058 #include "STREAMICE.h"
0059 #include "STREAMICE_CG.h"
0060
0061
0062
0063
0064
0065
0066 INTEGER myThid
d2cdb9260d Dani*0067 INTEGER iters, maxiter
5fc0cd3689 Dani*0068 _RL tolerance
0069 _RL cg_Uin (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
0070 _RL cg_Vin (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
0071 _RL cg_Bu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
0072 _RL cg_Bv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
07e785229e dngo*0073 _RL
5fc0cd3689 Dani*0074 & A_uu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
0075 & A_vu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
0076 & A_uv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
0077 & A_vv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1)
0078
8527100523 Dani*0079 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
0080 LOGICAL create_mat, destroy_mat
0081 #endif
0082
07e785229e dngo*0083 #ifdef ALLOW_STREAMICE
0084 #ifdef ALLOW_PETSC
5fc0cd3689 Dani*0085
0086 INTEGER i, j, bi, bj, cg_halo, conv_flag
0087 INTEGER iter, is, js, ie, je, colx, coly, k
07e785229e dngo*0088 _RL dot_p1, dot_p2, resid, resid_0
5fc0cd3689 Dani*0089 _RL dot_p1_tile (nSx,nSy)
0090 _RL dot_p2_tile (nSx,nSy)
0091 CHARACTER*(MAX_LEN_MBUF) msgBuf
0092
07e785229e dngo*0093 INTEGER indices(2*(sNx*nSx*sNy*nSy))
5fc0cd3689 Dani*0094 INTEGER n_dofs_cum_sum (0:nPx*nPy-1), idx(1)
07e785229e dngo*0095 _RL rhs_values(2*(sNx*nSx*sNy*nSy))
0096 _RL solution_values(2*(sNx*nSx*sNy*nSy))
0097
5fc0cd3689 Dani*0098 _RL mat_values (18,1), mat_val_return(1)
0099 INTEGER indices_col(18)
0100 INTEGER local_dofs, global_dofs, dof_index, dof_index_col
0101 INTEGER local_offset
8a34959769 dngo*0102 #ifdef STREAMICE_PETSC_3_8
0103 INTEGER local_null
0104 #endif
07e785229e dngo*0105
0106
0107
08f6c3ab35 Dani*0108 PC subpc
8a34959769 dngo*0109 #ifdef STREAMICE_PETSC_3_8
0110 KSP subksp
0111 #else
8527100523 Dani*0112 KSP subksp(1)
8a34959769 dngo*0113 #endif
5fc0cd3689 Dani*0114 Vec rhs
0115 Vec solution
0116 PetscErrorCode ierr
0117 #ifdef ALLOW_USE_MPI
0118 integer mpiRC, mpiMyWid
0119 #endif
0120
0121 #ifdef ALLOW_USE_MPI
0122
0123 CALL MPI_COMM_RANK( MPI_COMM_WORLD, mpiMyWId, mpiRC )
0124 local_dofs = n_dofs_process (mpiMyWid)
0125 global_dofs = 0
07e785229e dngo*0126
5fc0cd3689 Dani*0127 n_dofs_cum_sum(0) = 0
0128 DO i=0,nPx*nPy-1
0129 global_dofs = global_dofs + n_dofs_process (i)
0130 if (i.ge.1) THEN
0131 n_dofs_cum_sum(i) = n_dofs_cum_sum(i-1)+
0132 & n_dofs_process(i-1)
0133 endif
0134 ENDDO
0135 local_offset = n_dofs_cum_sum(mpimywid)
0136
0137 #else
0138
0139 local_dofs = n_dofs_process (0)
0140 global_dofs = local_dofs
0141 local_offset = 0
0142
07e785229e dngo*0143 #endif
5fc0cd3689 Dani*0144
07e785229e dngo*0145
5fc0cd3689 Dani*0146
07e785229e dngo*0147
5fc0cd3689 Dani*0148
18a089944d Dani*0149 CALL TIMER_START ('STREAMICE_PETSC_SETUP',myThid)
0150
5fc0cd3689 Dani*0151 call VecCreate(PETSC_COMM_WORLD, rhs, ierr)
0152 call VecSetSizes(rhs, local_dofs, global_dofs, ierr)
0153 call VecSetType(rhs, VECMPI, ierr)
0154
0155 call VecCreate(PETSC_COMM_WORLD, solution, ierr)
07e785229e dngo*0156 call VecSetSizes(solution, local_dofs, global_dofs, ierr)
5fc0cd3689 Dani*0157 call VecSetType(solution, VECMPI, ierr)
0158
0159 do i=1,local_dofs
0160 indices(i) = i-1 + local_offset
0161 end do
0162 do i=1,2*nSx*nSy*sNx*sNy
0163 rhs_values (i) = 0. _d 0
0164 solution_values (i) = 0. _d 0
0165 enddo
0166
07e785229e dngo*0167
5fc0cd3689 Dani*0168
0169 DO bj = myByLo(myThid), myByHi(myThid)
0170 DO bi = myBxLo(myThid), myBxHi(myThid)
0171 DO j=1,sNy
0172 DO i=1,sNx
0173
0174 dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
0175 & - local_offset
0176
0177 if (dof_index.ge.0) THEN
07e785229e dngo*0178
5fc0cd3689 Dani*0179 rhs_values(dof_index+1) = cg_Bu(i,j,bi,bj)
0180 solution_values(dof_index+1) = cg_Uin(i,j,bi,bj)
0181
0182 endif
0183
07e785229e dngo*0184
5fc0cd3689 Dani*0185
0186 dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
0187 & - local_offset
0188
0189 if (dof_index.ge.0) THEN
0190
0191 rhs_values(dof_index+1) = cg_Bv(i,j,bi,bj)
0192 solution_values(dof_index+1) = cg_Vin(i,j,bi,bj)
0193
0194 endif
0195
0196 ENDDO
0197 ENDDO
0198 ENDDO
0199 ENDDO
0200
0201 call VecSetValues(rhs, local_dofs, indices, rhs_values,
0202 & INSERT_VALUES, ierr)
0203 call VecAssemblyBegin(rhs, ierr)
0204 call VecAssemblyEnd(rhs, ierr)
0205
0206 call VecSetValues(solution, local_dofs, indices,
0207 & solution_values, INSERT_VALUES, ierr)
0208 call VecAssemblyBegin(solution, ierr)
0209 call VecAssemblyEnd(solution, ierr)
0210
8527100523 Dani*0211 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
0212 #ifdef ALLOW_PETSC
0213 if (STREAMICE_need2createmat) then
0214 #endif
0215 #endif
0216
07e785229e dngo*0217
0218
0219 call MatCreateAIJ (PETSC_COMM_WORLD,
5fc0cd3689 Dani*0220 & local_dofs, local_dofs,
07e785229e dngo*0221 & global_dofs, global_dofs,
0222 & 18, PETSC_NULL_INTEGER,
5fc0cd3689 Dani*0223 & 18, PETSC_NULL_INTEGER,
0224 & matrix, ierr)
0225
07e785229e dngo*0226
5fc0cd3689 Dani*0227
0228 DO bj = myByLo(myThid), myByHi(myThid)
0229 DO bi = myBxLo(myThid), myBxHi(myThid)
0230 DO j=1,sNy
0231 DO i=1,sNx
0232
0233 dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
07e785229e dngo*0234
5fc0cd3689 Dani*0235
0236 IF (dof_index .ge. 0) THEN
0237
0238 DO k=1,18
0239 indices_col(k) = 0
0240 mat_values(k,1) = 0. _d 0
0241 ENDDO
0242 k=0
0243
0244 DO coly=-1,1
0245 DO colx=-1,1
0246
0247 dof_index_col = streamice_petsc_dofs_u(i+colx,j+coly,bi,bj)
0248
0249 if (dof_index_col.ge.0) THEN
07e785229e dngo*0250
0251
0252
5fc0cd3689 Dani*0253 k=k+1
0254 mat_values (k,1) = A_uu(i,j,bi,bj,colx,coly)
0255 indices_col (k) = dof_index_col
0256 endif
07e785229e dngo*0257
5fc0cd3689 Dani*0258 dof_index_col = streamice_petsc_dofs_v(i+colx,j+coly,bi,bj)
0259
0260 if (dof_index_col.ge.0) THEN
07e785229e dngo*0261
0262
5fc0cd3689 Dani*0263 k=k+1
0264 mat_values (k,1) = A_uv(i,j,bi,bj,colx,coly)
0265 indices_col (k) = dof_index_col
0266 endif
07e785229e dngo*0267
5fc0cd3689 Dani*0268 ENDDO
0269 ENDDO
0270
8a34959769 dngo*0271 #ifdef STREAMICE_PETSC_3_8
0272 call MatSetValues1n (matrix, 1, dof_index, k, indices_col,
0273 #else
0274 call MatSetValues (matrix, 1, dof_index, k, indices_col,
0275 #endif
5fc0cd3689 Dani*0276 & mat_values,INSERT_VALUES,ierr)
0277
0278 ENDIF
0279
07e785229e dngo*0280
5fc0cd3689 Dani*0281
0282 dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
07e785229e dngo*0283
5fc0cd3689 Dani*0284
0285 IF (dof_index .ge. 0) THEN
0286
0287 DO k=1,18
0288 indices_col(k) = 0
0289 mat_values(k,1) = 0. _d 0
0290 ENDDO
0291 k=0
0292
0293 DO coly=-1,1
0294 DO colx=-1,1
0295
0296 dof_index_col = streamice_petsc_dofs_u(i+colx,j+coly,bi,bj)
0297
0298 if (dof_index_col.ge.0) THEN
07e785229e dngo*0299
0300
5fc0cd3689 Dani*0301 k=k+1
0302 mat_values (k,1) = A_vu(i,j,bi,bj,colx,coly)
0303 indices_col (k) = dof_index_col
0304 endif
0305
0306 dof_index_col = streamice_petsc_dofs_v(i+colx,j+coly,bi,bj)
0307
0308 if (dof_index_col.ge.0) THEN
07e785229e dngo*0309
0310
5fc0cd3689 Dani*0311 k=k+1
0312 mat_values (k,1) = A_vv(i,j,bi,bj,colx,coly)
0313 indices_col (k) = dof_index_col
0314 endif
0315
0316 ENDDO
0317 ENDDO
0318
8a34959769 dngo*0319 #ifdef STREAMICE_PETSC_3_8
0320 call MatSetValues1n (matrix, 1, dof_index, k, indices_col,
0321 #else
0322 call MatSetValues (matrix, 1, dof_index, k, indices_col,
0323 #endif
5fc0cd3689 Dani*0324 & mat_values,INSERT_VALUES,ierr)
0325 ENDIF
0326
0327 ENDDO
0328 ENDDO
0329 ENDDO
0330 ENDDO
0331
0332 call MatAssemblyBegin(matrix,MAT_FINAL_ASSEMBLY,ierr)
0333 call MatAssemblyEnd(matrix,MAT_FINAL_ASSEMBLY,ierr)
0334
0335 call KSPCreate(PETSC_COMM_WORLD, ksp, ierr)
8a34959769 dngo*0336 #ifdef STREAMICE_PETSC_3_8
07e785229e dngo*0337 call KSPSetOperators(ksp, matrix, matrix,
8a34959769 dngo*0338 & ierr)
0339 #else
0340 call KSPSetOperators(ksp, matrix, matrix,
0341 & DIFFERENT_NONZERO_PATTERN,ierr)
0342 #endif
5fc0cd3689 Dani*0343
8527100523 Dani*0344 IF (PETSC_PRECOND_TYPE.eq.'MUMPS') then
0345 call KSPSetType(ksp,KSPPREONLY,ierr)
0346 ELSE
0347 SELECT CASE (PETSC_SOLVER_TYPE)
5fc0cd3689 Dani*0348 CASE ('CG')
0349 PRINT *, "PETSC SOLVER: SELECTED CG"
0350 call KSPSetType(ksp, KSPCG, ierr)
0351 CASE ('GMRES')
0352 PRINT *, "PETSC SOLVER: SELECTED GMRES"
0353 call KSPSetType(ksp, KSPGMRES, ierr)
0354 CASE ('BICG')
0355 PRINT *, "PETSC SOLVER: SELECTED BICG"
07e785229e dngo*0356 call KSPSetType(ksp, KSPBCGS, ierr)
5fc0cd3689 Dani*0357 CASE DEFAULT
0358 PRINT *, "PETSC SOLVER: SELECTED DEFAULT"
0359 call KSPSetType(ksp, KSPCG, ierr)
8527100523 Dani*0360 END SELECT
0361 ENDIF
5fc0cd3689 Dani*0362
0363 call KSPGetPC(ksp, pc, ierr)
0364 call KSPSetTolerances(ksp,tolerance,
07e785229e dngo*0365 & PETSC_DEFAULT_REAL,
0366 & PETSC_DEFAULT_REAL,
d2cdb9260d Dani*0367 & maxiter,ierr)
5fc0cd3689 Dani*0368
0369 SELECT CASE (PETSC_PRECOND_TYPE)
0370 CASE ('BLOCKJACOBI')
0371 PRINT *, "PETSC PRECOND: SELECTED BJACOBI"
0372 call PCSetType(pc, PCBJACOBI, ierr)
08f6c3ab35 Dani*0373 call kspsetup (ksp, ierr)
8a34959769 dngo*0374 #ifdef STREAMICE_PETSC_3_8
0375 local_null =0
0376 call PCBJacobiGetSubKSP1 (pc,local_null,local_null,
0377 #else
0378 call PCBJacobiGetSubKSP1 (pc,PETSC_NULL_INTEGER,
0379 & PETSC_NULL_INTEGER,
0380 #endif
08f6c3ab35 Dani*0381 & subksp,ierr);
0382 call KSPGetPC (subksp, subpc, ierr)
07e785229e dngo*0383 call PCSetType (subpc, PCICC, ierr)
0384 call PCFactorSetLevels(subpc,streamice_petsc_pcfactorlevels,
0385 & ierr)
5fc0cd3689 Dani*0386 CASE ('JACOBI')
0387 PRINT *, "PETSC PRECOND: SELECTED JACOBI"
0388 call PCSetType(pc, PCJACOBI, ierr)
0389 CASE ('ILU')
0390 PRINT *, "PETSC PRECOND: SELECTED ILU"
0391 call PCSetType(pc, PCILU, ierr)
18a089944d Dani*0392
0393 CASE ('GAMG')
0394 PRINT *, "PETSC PRECOND: SELECTED GAMG"
0395 call PCSetType(pc, PCGAMG, ierr)
0396 call PCGAMGSetCoarseEqLim(pc,10000,ierr)
96b006450c dngo*0397
18a089944d Dani*0398 call PCGAMGSetNSmooths(pc, 0,ierr)
8a34959769 dngo*0399
0400
8527100523 Dani*0401 call kspsetup (ksp, ierr)
0402
0403 CASE ('MUMPS')
0404 PRINT *, "PETSC PRECOND: SELECTED MUMPS"
0405 call PCSetType(pc,PCLU,ierr)
8a34959769 dngo*0406 call PCFactorSetMatSolverType(pc,MATSOLVERMUMPS,ierr)
0407 call PCFactorSetUpMatSolverType(pc,ierr)
8527100523 Dani*0408 call PCFactorGetMatrix(pc,mumpsFac,ierr)
0409 call MatMumpsSetIcntl(mumpsfac,24,1,ierr)
0410 call kspsetup (ksp, ierr)
18a089944d Dani*0411
5fc0cd3689 Dani*0412 CASE DEFAULT
0413 PRINT *, "PETSC PRECOND: SELECTED DEFAULT"
0414 call PCSetType(pc, PCBJACOBI, ierr)
0415 END SELECT
0416
18a089944d Dani*0417 CALL TIMER_STOP ('STREAMICE_PETSC_SETUP',myThid)
8527100523 Dani*0418 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
0419 #ifdef ALLOW_PETSC
0420 endif
0421 #endif
0422 #endif
0423
18a089944d Dani*0424 CALL TIMER_START ('STREAMICE_PETSC_SOLVE',myThid)
0425
5fc0cd3689 Dani*0426 call KSPSolve(ksp, rhs, solution, ierr)
18a089944d Dani*0427
0428 CALL TIMER_STOP ('STREAMICE_PETSC_SOLVE',myThid)
0429
5fc0cd3689 Dani*0430 call KSPGetIterationNumber(ksp,iters,ierr)
0431
0432 call VecGetValues(solution,local_dofs,indices,
0433 & solution_values,ierr)
0434
0435 DO bj = myByLo(myThid), myByHi(myThid)
0436 DO bi = myBxLo(myThid), myBxHi(myThid)
0437 DO j=1,sNy
0438 DO i=1,sNx
0439
0440 dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
0441 & - local_offset
0442 if (dof_index.ge.0) THEN
0443 cg_Uin(i,j,bi,bj) = solution_values(dof_index+1)
0444 endif
07e785229e dngo*0445
5fc0cd3689 Dani*0446 dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
0447 & - local_offset
0448 if (dof_index.ge.0) THEN
0449 cg_Vin(i,j,bi,bj) = solution_values(dof_index+1)
0450 endif
0451
0452 ENDDO
0453 ENDDO
0454 ENDDO
0455 ENDDO
0456
8527100523 Dani*0457 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
0458 if (streamice_need2destroymat) then
0459 #endif
5fc0cd3689 Dani*0460 call KSPDestroy (ksp, ierr)
8527100523 Dani*0461 call MatDestroy (matrix, ierr)
0462 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
0463 endif
0464 #endif
5fc0cd3689 Dani*0465 call VecDestroy (rhs, ierr)
0466 call VecDestroy (solution, ierr)
0467
07e785229e dngo*0468
5fc0cd3689 Dani*0469
07e785229e dngo*0470 #endif /* ALLOW_PETSC */
0471 #endif /* ALLOW_STREAMICE */
5fc0cd3689 Dani*0472 RETURN
0473 END