Back to home page

MITgcm

 
 

    


File indexing completed on 2023-11-30 06:11:11 UTC

view on githubraw file Latest commit f0ff6e91 on 2023-11-29 18:40:14 UTC
5fc0cd3689 Dani*0001 #include "STREAMICE_OPTIONS.h"
                0002 
                0003 C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|
                0004 
                0005 CBOP
07e785229e dngo*0006       SUBROUTINE STREAMICE_CG_SOLVE_PETSC(
5fc0cd3689 Dani*0007      U                               cg_Uin,     ! x-velocities
                0008      U                               cg_Vin,     ! y-velocities
                0009      I                               cg_Bu,      ! force in x dir
                0010      I                               cg_Bv,      ! force in y dir
                0011      I                               A_uu,       ! section of matrix that multiplies u and projects on u
                0012      I                               A_uv,       ! section of matrix that multiplies v and projects on u
                0013      I                               A_vu,       ! section of matrix that multiplies u and projects on v
                0014      I                               A_vv,       ! section of matrix that multiplies v and projects on v
07e785229e dngo*0015      I                               tolerance,
5fc0cd3689 Dani*0016      O                               iters,
d2cdb9260d Dani*0017      I                               maxIter,
5fc0cd3689 Dani*0018      I                               myThid )
8527100523 Dani*0019 
8a34959769 dngo*0020 C     *============================================================*
07e785229e dngo*0021 C     | SUBROUTINE                                                 |
5fc0cd3689 Dani*0022 C     | o                                                          |
8a34959769 dngo*0023 C     *============================================================*
                0024 
                0025 #ifdef ALLOW_PETSC
                0026 #ifdef STREAMICE_PETSC_3_8
f0ff6e912a dngo*0027 
                0028 C     THE FOLLOWING MODULES ARE PART OF THE PETSC
                0029 C     LIBRARY, USED BY STREAMICE WHEN AVAILABLE TO ACCELERATE
                0030 C     SOLVES OF LINEAR SYSTEMS
                0031 
                0032 C     assumes paths
                0033 C     {petsc_dir}/include/petsc/finclude/
                0034 C     {petsc_dir}/include
                0035 C     in the include statement
                0036 
8a34959769 dngo*0037 #include "petsc/finclude/petsc.h"
                0038 #include "petsc/finclude/petscvec.h"
                0039       use petscvec
                0040 #include "petsc/finclude/petscmat.h"
                0041       use petscmat
                0042 #include "petsc/finclude/petscksp.h"
                0043       use petscksp
                0044 #include "petsc/finclude/petscpc.h"
                0045       use petscpc
f0ff6e912a dngo*0046 #include "STREAMICE_PETSC_MOD.h"
8a34959769 dngo*0047       IMPLICIT NONE
                0048 # else
5fc0cd3689 Dani*0049       IMPLICIT NONE
8a34959769 dngo*0050 #include "finclude/petsc.h"
f0ff6e912a dngo*0051 #include "STREAMICE_PETSC_MOD.h"
8a34959769 dngo*0052 #endif
                0053 #endif
5fc0cd3689 Dani*0054 
                0055 #include "SIZE.h"
                0056 #include "EEPARAMS.h"
                0057 #include "PARAMS.h"
                0058 #include "STREAMICE.h"
                0059 #include "STREAMICE_CG.h"
                0060 
                0061 C     === Global variables ===
                0062 
                0063 C     !INPUT/OUTPUT ARGUMENTS
                0064 C     cg_Uin, cg_Vin - input and output velocities
                0065 C     cg_Bu, cg_Bv - driving stress
                0066       INTEGER myThid
d2cdb9260d Dani*0067       INTEGER iters, maxiter
5fc0cd3689 Dani*0068       _RL tolerance
                0069       _RL cg_Uin (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
                0070       _RL cg_Vin (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
                0071       _RL cg_Bu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
                0072       _RL cg_Bv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
07e785229e dngo*0073       _RL
5fc0cd3689 Dani*0074      & A_uu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
                0075      & A_vu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
                0076      & A_uv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
                0077      & A_vv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1)
                0078 
8527100523 Dani*0079 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
                0080        LOGICAL create_mat, destroy_mat
                0081 #endif
                0082 
07e785229e dngo*0083 #ifdef ALLOW_STREAMICE
                0084 #ifdef ALLOW_PETSC
5fc0cd3689 Dani*0085 C     LOCAL VARIABLES
                0086       INTEGER i, j, bi, bj, cg_halo, conv_flag
                0087       INTEGER iter, is, js, ie, je, colx, coly, k
07e785229e dngo*0088       _RL dot_p1, dot_p2, resid, resid_0
5fc0cd3689 Dani*0089       _RL dot_p1_tile (nSx,nSy)
                0090       _RL dot_p2_tile (nSx,nSy)
                0091       CHARACTER*(MAX_LEN_MBUF) msgBuf
                0092 
07e785229e dngo*0093       INTEGER indices(2*(sNx*nSx*sNy*nSy))
5fc0cd3689 Dani*0094       INTEGER n_dofs_cum_sum (0:nPx*nPy-1), idx(1)
07e785229e dngo*0095       _RL rhs_values(2*(sNx*nSx*sNy*nSy))
                0096       _RL solution_values(2*(sNx*nSx*sNy*nSy))
                0097 c      _RL mat_values (2*Nx*Ny,2*(sNx*nSx*sNy*nSy))
5fc0cd3689 Dani*0098       _RL mat_values (18,1), mat_val_return(1)
                0099       INTEGER indices_col(18)
                0100       INTEGER local_dofs, global_dofs, dof_index, dof_index_col
                0101       INTEGER local_offset
8a34959769 dngo*0102 #ifdef STREAMICE_PETSC_3_8
                0103       INTEGER local_null
                0104 #endif
07e785229e dngo*0105 c      Mat matrix
                0106 c      KSP ksp
                0107 c      PC  pc
08f6c3ab35 Dani*0108       PC  subpc
8a34959769 dngo*0109 #ifdef STREAMICE_PETSC_3_8
                0110       KSP subksp
                0111 #else
8527100523 Dani*0112       KSP subksp(1)
8a34959769 dngo*0113 #endif
5fc0cd3689 Dani*0114       Vec rhs
                0115       Vec solution
                0116       PetscErrorCode ierr
                0117 #ifdef ALLOW_USE_MPI
                0118       integer mpiRC, mpiMyWid
                0119 #endif
                0120 
                0121 #ifdef ALLOW_USE_MPI
                0122 
                0123       CALL MPI_COMM_RANK( MPI_COMM_WORLD, mpiMyWId, mpiRC )
                0124       local_dofs = n_dofs_process (mpiMyWid)
                0125       global_dofs = 0
07e785229e dngo*0126 
5fc0cd3689 Dani*0127       n_dofs_cum_sum(0) = 0
                0128       DO i=0,nPx*nPy-1
                0129        global_dofs = global_dofs + n_dofs_process (i)
                0130        if (i.ge.1) THEN
                0131          n_dofs_cum_sum(i) = n_dofs_cum_sum(i-1)+
                0132      &                     n_dofs_process(i-1)
                0133        endif
                0134       ENDDO
                0135       local_offset = n_dofs_cum_sum(mpimywid)
                0136 
                0137 #else
                0138 
                0139       local_dofs = n_dofs_process (0)
                0140       global_dofs = local_dofs
                0141       local_offset = 0
                0142 
07e785229e dngo*0143 #endif
5fc0cd3689 Dani*0144 
07e785229e dngo*0145 c      call petscInitialize(PETSC_NULL_CHARACTER,ierr)
5fc0cd3689 Dani*0146 
07e785229e dngo*0147 c----------------------
5fc0cd3689 Dani*0148 
18a089944d Dani*0149       CALL TIMER_START ('STREAMICE_PETSC_SETUP',myThid)
                0150 
5fc0cd3689 Dani*0151       call VecCreate(PETSC_COMM_WORLD, rhs, ierr)
                0152       call VecSetSizes(rhs, local_dofs, global_dofs, ierr)
                0153       call VecSetType(rhs, VECMPI, ierr)
                0154 
                0155       call VecCreate(PETSC_COMM_WORLD, solution, ierr)
07e785229e dngo*0156       call VecSetSizes(solution, local_dofs, global_dofs, ierr)
5fc0cd3689 Dani*0157       call VecSetType(solution, VECMPI, ierr)
                0158 
                0159       do i=1,local_dofs
                0160         indices(i) = i-1 + local_offset
                0161       end do
                0162       do i=1,2*nSx*nSy*sNx*sNy
                0163         rhs_values (i) = 0. _d 0
                0164         solution_values (i) = 0. _d 0
                0165       enddo
                0166 
07e785229e dngo*0167 c gather rhs and initial guess values to populate petsc vectors
5fc0cd3689 Dani*0168 
                0169       DO bj = myByLo(myThid), myByHi(myThid)
                0170        DO bi = myBxLo(myThid), myBxHi(myThid)
                0171         DO j=1,sNy
                0172          DO i=1,sNx
                0173 
                0174           dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
                0175      &                - local_offset
                0176 
                0177           if (dof_index.ge.0) THEN
07e785229e dngo*0178 
5fc0cd3689 Dani*0179            rhs_values(dof_index+1) = cg_Bu(i,j,bi,bj)
                0180            solution_values(dof_index+1) = cg_Uin(i,j,bi,bj)
                0181 
                0182           endif
                0183 
07e785229e dngo*0184 c---------------
5fc0cd3689 Dani*0185 
                0186           dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
                0187      &                - local_offset
                0188 
                0189           if (dof_index.ge.0) THEN
                0190 
                0191            rhs_values(dof_index+1) = cg_Bv(i,j,bi,bj)
                0192            solution_values(dof_index+1) = cg_Vin(i,j,bi,bj)
                0193 
                0194           endif
                0195 
                0196          ENDDO
                0197         ENDDO
                0198        ENDDO
                0199       ENDDO
                0200 
                0201       call VecSetValues(rhs, local_dofs, indices, rhs_values,
                0202      &                  INSERT_VALUES, ierr)
                0203       call VecAssemblyBegin(rhs, ierr)
                0204       call VecAssemblyEnd(rhs, ierr)
                0205 
                0206       call VecSetValues(solution, local_dofs, indices,
                0207      &                  solution_values, INSERT_VALUES, ierr)
                0208       call VecAssemblyBegin(solution, ierr)
                0209       call VecAssemblyEnd(solution, ierr)
                0210 
8527100523 Dani*0211 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
                0212 #ifdef ALLOW_PETSC
                0213       if (STREAMICE_need2createmat) then
                0214 #endif
                0215 #endif
                0216 
07e785229e dngo*0217 c     IF USING v3.0 THEN
                0218 c     call MatCreateMPIAIJ (PETSC_COMM_WORLD,
                0219       call MatCreateAIJ (PETSC_COMM_WORLD,
5fc0cd3689 Dani*0220      &                      local_dofs, local_dofs,
07e785229e dngo*0221      &                      global_dofs, global_dofs,
                0222      &                      18, PETSC_NULL_INTEGER,
5fc0cd3689 Dani*0223      &                      18, PETSC_NULL_INTEGER,
                0224      &                      matrix, ierr)
                0225 
07e785229e dngo*0226 c populate petsc matrix
5fc0cd3689 Dani*0227 
                0228       DO bj = myByLo(myThid), myByHi(myThid)
                0229        DO bi = myBxLo(myThid), myBxHi(myThid)
                0230         DO j=1,sNy
                0231          DO i=1,sNx
                0232 
                0233           dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
07e785229e dngo*0234 c     &                - local_offset
5fc0cd3689 Dani*0235 
                0236           IF (dof_index .ge. 0) THEN
                0237 
                0238            DO k=1,18
                0239             indices_col(k) = 0
                0240             mat_values(k,1) = 0. _d 0
                0241            ENDDO
                0242            k=0
                0243 
                0244            DO coly=-1,1
                0245             DO colx=-1,1
                0246 
                0247              dof_index_col = streamice_petsc_dofs_u(i+colx,j+coly,bi,bj)
                0248 
                0249              if (dof_index_col.ge.0) THEN
07e785229e dngo*0250 c               pscal = A_uu(i,j,bi,bj,colx,coly)
                0251 c               CALL MatSetValue (matrix,dof_index, dof_index_col,
                0252 c     &              pscal,INSERT_VALUES,ierr)
5fc0cd3689 Dani*0253                 k=k+1
                0254                 mat_values (k,1) = A_uu(i,j,bi,bj,colx,coly)
                0255                 indices_col (k) = dof_index_col
                0256              endif
07e785229e dngo*0257 
5fc0cd3689 Dani*0258              dof_index_col = streamice_petsc_dofs_v(i+colx,j+coly,bi,bj)
                0259 
                0260              if (dof_index_col.ge.0) THEN
07e785229e dngo*0261 c               CALL MatSetValue (matrix,dof_index, dof_index_col,
                0262 c     &              A_uv(i,j,bi,bj,colx,coly),INSERT_VALUES,ierr)
5fc0cd3689 Dani*0263                 k=k+1
                0264                 mat_values (k,1) = A_uv(i,j,bi,bj,colx,coly)
                0265                 indices_col (k) = dof_index_col
                0266              endif
07e785229e dngo*0267 
5fc0cd3689 Dani*0268             ENDDO
                0269            ENDDO
                0270 
8a34959769 dngo*0271 #ifdef STREAMICE_PETSC_3_8
                0272            call MatSetValues1n (matrix, 1, dof_index, k, indices_col,
                0273 #else
                0274            call MatSetValues (matrix, 1, dof_index, k, indices_col,
                0275 #endif
5fc0cd3689 Dani*0276      &                        mat_values,INSERT_VALUES,ierr)
                0277 
                0278           ENDIF
                0279 
07e785229e dngo*0280 c ----------------------------------------------
5fc0cd3689 Dani*0281 
                0282           dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
07e785229e dngo*0283 c     &                - local_offset
5fc0cd3689 Dani*0284 
                0285           IF (dof_index .ge. 0) THEN
                0286 
                0287            DO k=1,18
                0288             indices_col(k) = 0
                0289             mat_values(k,1) = 0. _d 0
                0290            ENDDO
                0291            k=0
                0292 
                0293            DO coly=-1,1
                0294             DO colx=-1,1
                0295 
                0296              dof_index_col = streamice_petsc_dofs_u(i+colx,j+coly,bi,bj)
                0297 
                0298              if (dof_index_col.ge.0) THEN
07e785229e dngo*0299 c               CALL MatSetValue (matrix,dof_index, dof_index_col,
                0300 c     &              A_vu(i,j,bi,bj,colx,coly),INSERT_VALUES,ierr)
5fc0cd3689 Dani*0301                 k=k+1
                0302                 mat_values (k,1) = A_vu(i,j,bi,bj,colx,coly)
                0303                 indices_col (k) = dof_index_col
                0304              endif
                0305 
                0306              dof_index_col = streamice_petsc_dofs_v(i+colx,j+coly,bi,bj)
                0307 
                0308              if (dof_index_col.ge.0) THEN
07e785229e dngo*0309 c               CALL MatSetValue (matrix,dof_index, dof_index_col,
                0310 c     &              A_vv(i,j,bi,bj,colx,coly),INSERT_VALUES,ierr)
5fc0cd3689 Dani*0311                 k=k+1
                0312                 mat_values (k,1) = A_vv(i,j,bi,bj,colx,coly)
                0313                 indices_col (k) = dof_index_col
                0314              endif
                0315 
                0316             ENDDO
                0317            ENDDO
                0318 
8a34959769 dngo*0319 #ifdef STREAMICE_PETSC_3_8
                0320            call MatSetValues1n (matrix, 1, dof_index, k, indices_col,
                0321 #else
                0322            call MatSetValues (matrix, 1, dof_index, k, indices_col,
                0323 #endif
5fc0cd3689 Dani*0324      &                        mat_values,INSERT_VALUES,ierr)
                0325           ENDIF
                0326 
                0327          ENDDO
                0328         ENDDO
                0329        ENDDO
                0330       ENDDO
                0331 
                0332       call MatAssemblyBegin(matrix,MAT_FINAL_ASSEMBLY,ierr)
                0333       call MatAssemblyEnd(matrix,MAT_FINAL_ASSEMBLY,ierr)
                0334 
                0335       call KSPCreate(PETSC_COMM_WORLD, ksp, ierr)
8a34959769 dngo*0336 #ifdef STREAMICE_PETSC_3_8
07e785229e dngo*0337       call KSPSetOperators(ksp, matrix, matrix,
8a34959769 dngo*0338      &                      ierr)
                0339 #else
                0340       call KSPSetOperators(ksp, matrix, matrix,
                0341      &                     DIFFERENT_NONZERO_PATTERN,ierr)
                0342 #endif
5fc0cd3689 Dani*0343 
8527100523 Dani*0344       IF (PETSC_PRECOND_TYPE.eq.'MUMPS') then
                0345        call KSPSetType(ksp,KSPPREONLY,ierr)
                0346       ELSE
                0347        SELECT CASE (PETSC_SOLVER_TYPE)
5fc0cd3689 Dani*0348        CASE ('CG')
                0349        PRINT *, "PETSC SOLVER: SELECTED CG"
                0350        call KSPSetType(ksp, KSPCG, ierr)
                0351        CASE ('GMRES')
                0352        PRINT *, "PETSC SOLVER: SELECTED GMRES"
                0353        call KSPSetType(ksp, KSPGMRES, ierr)
                0354        CASE ('BICG')
                0355        PRINT *, "PETSC SOLVER: SELECTED BICG"
07e785229e dngo*0356        call KSPSetType(ksp, KSPBCGS, ierr)
5fc0cd3689 Dani*0357        CASE DEFAULT
                0358        PRINT *, "PETSC SOLVER: SELECTED DEFAULT"
                0359        call KSPSetType(ksp, KSPCG, ierr)
8527100523 Dani*0360        END SELECT
                0361       ENDIF
5fc0cd3689 Dani*0362 
                0363       call KSPGetPC(ksp, pc, ierr)
                0364       call KSPSetTolerances(ksp,tolerance,
07e785229e dngo*0365      &     PETSC_DEFAULT_REAL,
                0366      &     PETSC_DEFAULT_REAL,
d2cdb9260d Dani*0367      &     maxiter,ierr)
5fc0cd3689 Dani*0368 
                0369       SELECT CASE (PETSC_PRECOND_TYPE)
                0370        CASE ('BLOCKJACOBI')
                0371        PRINT *, "PETSC PRECOND: SELECTED BJACOBI"
                0372        call PCSetType(pc, PCBJACOBI, ierr)
08f6c3ab35 Dani*0373        call kspsetup (ksp, ierr)
8a34959769 dngo*0374 #ifdef STREAMICE_PETSC_3_8
                0375        local_null =0 !PETSC_NULL_INTEGER
                0376        call PCBJacobiGetSubKSP1 (pc,local_null,local_null,
                0377 #else
                0378        call PCBJacobiGetSubKSP1 (pc,PETSC_NULL_INTEGER,
                0379      &                           PETSC_NULL_INTEGER,
                0380 #endif
08f6c3ab35 Dani*0381      &                             subksp,ierr);
                0382        call KSPGetPC (subksp, subpc, ierr)
07e785229e dngo*0383        call PCSetType (subpc, PCICC, ierr)
                0384        call PCFactorSetLevels(subpc,streamice_petsc_pcfactorlevels,
                0385      &  ierr)
5fc0cd3689 Dani*0386        CASE ('JACOBI')
                0387        PRINT *, "PETSC PRECOND: SELECTED JACOBI"
                0388        call PCSetType(pc, PCJACOBI, ierr)
                0389        CASE ('ILU')
                0390        PRINT *, "PETSC PRECOND: SELECTED ILU"
                0391        call PCSetType(pc, PCILU, ierr)
18a089944d Dani*0392 
                0393        CASE ('GAMG')
                0394        PRINT *, "PETSC PRECOND: SELECTED GAMG"
                0395        call PCSetType(pc, PCGAMG, ierr)
                0396        call PCGAMGSetCoarseEqLim(pc,10000,ierr)
96b006450c dngo*0397 C       call PCGAMGSetSymGraph(pc, PETSC_TRUE,ierr)
18a089944d Dani*0398        call PCGAMGSetNSmooths(pc, 0,ierr)
8a34959769 dngo*0399 C       call PCGAMGSetThreshold(pc, .001,ierr)
                0400 C       call PCGAMGSetReuseProl(pc,PETSC_FALSE,ierr)
8527100523 Dani*0401        call kspsetup (ksp, ierr)
                0402 
                0403        CASE ('MUMPS')
                0404        PRINT *, "PETSC PRECOND: SELECTED MUMPS"
                0405        call PCSetType(pc,PCLU,ierr)
8a34959769 dngo*0406        call PCFactorSetMatSolverType(pc,MATSOLVERMUMPS,ierr)
                0407        call PCFactorSetUpMatSolverType(pc,ierr)
8527100523 Dani*0408        call PCFactorGetMatrix(pc,mumpsFac,ierr)
                0409        call MatMumpsSetIcntl(mumpsfac,24,1,ierr)
                0410        call kspsetup (ksp, ierr)
18a089944d Dani*0411 
5fc0cd3689 Dani*0412        CASE DEFAULT
                0413        PRINT *, "PETSC PRECOND: SELECTED DEFAULT"
                0414        call PCSetType(pc, PCBJACOBI, ierr)
                0415       END SELECT
                0416 
18a089944d Dani*0417       CALL TIMER_STOP ('STREAMICE_PETSC_SETUP',myThid)
8527100523 Dani*0418 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
                0419 #ifdef ALLOW_PETSC
                0420       endif
                0421 #endif
                0422 #endif
                0423 
18a089944d Dani*0424       CALL TIMER_START ('STREAMICE_PETSC_SOLVE',myThid)
                0425 
5fc0cd3689 Dani*0426       call KSPSolve(ksp, rhs, solution, ierr)
18a089944d Dani*0427 
                0428       CALL TIMER_STOP ('STREAMICE_PETSC_SOLVE',myThid)
                0429 
5fc0cd3689 Dani*0430       call KSPGetIterationNumber(ksp,iters,ierr)
                0431 
                0432       call VecGetValues(solution,local_dofs,indices,
                0433      &      solution_values,ierr)
                0434 
                0435       DO bj = myByLo(myThid), myByHi(myThid)
                0436        DO bi = myBxLo(myThid), myBxHi(myThid)
                0437         DO j=1,sNy
                0438          DO i=1,sNx
                0439 
                0440           dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
                0441      &                - local_offset
                0442           if (dof_index.ge.0) THEN
                0443            cg_Uin(i,j,bi,bj) = solution_values(dof_index+1)
                0444           endif
07e785229e dngo*0445 
5fc0cd3689 Dani*0446           dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
                0447      &                - local_offset
                0448           if (dof_index.ge.0) THEN
                0449            cg_Vin(i,j,bi,bj) = solution_values(dof_index+1)
                0450           endif
                0451 
                0452          ENDDO
                0453         ENDDO
                0454        ENDDO
                0455       ENDDO
                0456 
8527100523 Dani*0457 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
                0458       if (streamice_need2destroymat) then
                0459 #endif
5fc0cd3689 Dani*0460       call KSPDestroy (ksp, ierr)
8527100523 Dani*0461       call MatDestroy (matrix, ierr)
                0462 #if (defined (ALLOW_OPENAD) && defined (ALLOW_STREAMICE_OAD_FP))
                0463       endif
                0464 #endif
5fc0cd3689 Dani*0465       call VecDestroy (rhs, ierr)
                0466       call VecDestroy (solution, ierr)
                0467 
07e785229e dngo*0468 c      call PetscFinalize(ierr)
5fc0cd3689 Dani*0469 
07e785229e dngo*0470 #endif /* ALLOW_PETSC */
                0471 #endif /* ALLOW_STREAMICE */
5fc0cd3689 Dani*0472       RETURN
                0473       END