Actual source code: ex76.c

  1: #include <petscksp.h>
  2: #include <petsc/private/petscimpl.h>

  4: static char help[] = "Solves a linear system using PCHPDDM.\n\n";

  6: int main(int argc, char **args)
  7: {
  8:   Vec             b;            /* computed solution and RHS */
  9:   Mat             A, aux, X, B; /* linear system matrix */
 10:   KSP             ksp;          /* linear solver context */
 11:   PC              pc;
 12:   IS              is, sizes;
 13:   const PetscInt *idx;
 14:   PetscMPIInt     rank, size;
 15:   PetscInt        m, N = 1;
 16:   PetscViewer     viewer;
 17:   char            dir[PETSC_MAX_PATH_LEN], name[PETSC_MAX_PATH_LEN], type[256];
 18:   PetscBool3      share = PETSC_BOOL3_UNKNOWN;
 19:   PetscBool       flg, set;

 21:   PetscFunctionBeginUser;
 22:   PetscCall(PetscInitialize(&argc, &args, NULL, help));
 23:   PetscCall(PetscLogDefaultBegin());
 24:   PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
 25:   PetscCheck(size == 4, PETSC_COMM_WORLD, PETSC_ERR_USER, "This example requires 4 processes");
 26:   PetscCall(PetscOptionsGetInt(NULL, NULL, "-rhs", &N, NULL));
 27:   PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
 28:   PetscCall(MatCreate(PETSC_COMM_WORLD, &A));
 29:   PetscCall(MatCreate(PETSC_COMM_SELF, &aux));
 30:   PetscCall(ISCreate(PETSC_COMM_SELF, &is));
 31:   PetscCall(PetscStrncpy(dir, ".", sizeof(dir)));
 32:   PetscCall(PetscOptionsGetString(NULL, NULL, "-load_dir", dir, sizeof(dir), NULL));
 33:   /* loading matrices */
 34:   PetscCall(PetscSNPrintf(name, sizeof(name), "%s/sizes_%d_%d.dat", dir, rank, size));
 35:   PetscCall(PetscViewerBinaryOpen(PETSC_COMM_SELF, name, FILE_MODE_READ, &viewer));
 36:   PetscCall(ISCreate(PETSC_COMM_SELF, &sizes));
 37:   PetscCall(ISLoad(sizes, viewer));
 38:   PetscCall(ISGetIndices(sizes, &idx));
 39:   PetscCall(MatSetSizes(A, idx[0], idx[1], idx[2], idx[3]));
 40:   PetscCall(MatSetUp(A));
 41:   PetscCall(ISRestoreIndices(sizes, &idx));
 42:   PetscCall(ISDestroy(&sizes));
 43:   PetscCall(PetscViewerDestroy(&viewer));
 44:   PetscCall(PetscSNPrintf(name, sizeof(name), "%s/A.dat", dir));
 45:   PetscCall(PetscViewerBinaryOpen(PETSC_COMM_WORLD, name, FILE_MODE_READ, &viewer));
 46:   PetscCall(MatLoad(A, viewer));
 47:   PetscCall(PetscViewerDestroy(&viewer));
 48:   PetscCall(PetscSNPrintf(name, sizeof(name), "%s/is_%d_%d.dat", dir, rank, size));
 49:   PetscCall(PetscViewerBinaryOpen(PETSC_COMM_SELF, name, FILE_MODE_READ, &viewer));
 50:   PetscCall(ISLoad(is, viewer));
 51:   PetscCall(ISSetBlockSize(is, 2));
 52:   PetscCall(PetscViewerDestroy(&viewer));
 53:   PetscCall(PetscSNPrintf(name, sizeof(name), "%s/Neumann_%d_%d.dat", dir, rank, size));
 54:   PetscCall(PetscViewerBinaryOpen(PETSC_COMM_SELF, name, FILE_MODE_READ, &viewer));
 55:   PetscCall(MatLoad(aux, viewer));
 56:   PetscCall(PetscViewerDestroy(&viewer));
 57:   flg = PETSC_FALSE;
 58:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-pc_hpddm_levels_1_st_share_sub_ksp", &flg, &set));
 59:   if (flg) { /* PETSc LU/Cholesky is struggling numerically for bs > 1          */
 60:              /* only set the proper bs for the geneo_share_* tests, 1 otherwise */
 61:     PetscCall(MatSetBlockSizesFromMats(aux, A, A));
 62:     share = PETSC_BOOL3_TRUE;
 63:   } else if (set) share = PETSC_BOOL3_FALSE;
 64:   PetscCall(MatSetOption(A, MAT_SYMMETRIC, PETSC_TRUE));
 65:   PetscCall(MatSetOption(aux, MAT_SYMMETRIC, PETSC_TRUE));
 66:   /* ready for testing */
 67:   PetscOptionsBegin(PETSC_COMM_WORLD, "", "", "");
 68:   PetscCall(PetscStrncpy(type, MATAIJ, sizeof(type)));
 69:   PetscCall(PetscOptionsFList("-mat_type", "Matrix type", "MatSetType", MatList, type, type, 256, &flg));
 70:   PetscOptionsEnd();
 71:   PetscCall(MatConvert(A, type, MAT_INPLACE_MATRIX, &A));
 72:   PetscCall(MatConvert(aux, type, MAT_INPLACE_MATRIX, &aux));
 73:   PetscCall(KSPCreate(PETSC_COMM_WORLD, &ksp));
 74:   PetscCall(KSPSetOperators(ksp, A, A));
 75:   PetscCall(KSPGetPC(ksp, &pc));
 76:   PetscCall(PCSetType(pc, PCHPDDM));
 77: #if defined(PETSC_HAVE_HPDDM) && defined(PETSC_HAVE_DYNAMIC_LIBRARIES) && defined(PETSC_USE_SHARED_LIBRARIES)
 78:   flg = PETSC_FALSE;
 79:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-reset", &flg, NULL));
 80:   if (flg) {
 81:     PetscCall(PetscOptionsSetValue(NULL, "-pc_hpddm_block_splitting", "true"));
 82:     PetscCall(PCSetFromOptions(pc));
 83:     PetscCall(PCSetUp(pc));
 84:     PetscCall(PetscOptionsClearValue(NULL, "-pc_hpddm_block_splitting"));
 85:   }
 86:   PetscCall(PCHPDDMSetAuxiliaryMat(pc, is, aux, NULL, NULL));
 87:   PetscCall(PCHPDDMHasNeumannMat(pc, PETSC_FALSE)); /* PETSC_TRUE is fine as well, just testing */
 88:   if (share == PETSC_BOOL3_UNKNOWN) PetscCall(PCHPDDMSetSTShareSubKSP(pc, PetscBool3ToBool(share)));
 89:   flg = PETSC_FALSE;
 90:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-set_rhs", &flg, NULL));
 91:   if (flg) {          /* user-provided RHS for concurrent generalized eigenvalue problems                          */
 92:     Mat      a, c, P; /* usually assembled automatically in PCHPDDM, this is solely for testing PCHPDDMSetRHSMat() */
 93:     PetscInt rstart, rend, location;

 95:     PetscCall(MatDuplicate(aux, MAT_DO_NOT_COPY_VALUES, &B)); /* duplicate so that MatStructure is SAME_NONZERO_PATTERN */
 96:     PetscCall(MatGetDiagonalBlock(A, &a));
 97:     PetscCall(MatGetOwnershipRange(A, &rstart, &rend));
 98:     PetscCall(ISGetLocalSize(is, &m));
 99:     PetscCall(MatCreateSeqAIJ(PETSC_COMM_SELF, rend - rstart, m, 1, NULL, &P));
100:     for (m = rstart; m < rend; ++m) {
101:       PetscCall(ISLocate(is, m, &location));
102:       PetscCheck(location >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "IS of the auxiliary Mat does not include all local rows of A");
103:       PetscCall(MatSetValue(P, m - rstart, location, 1.0, INSERT_VALUES));
104:     }
105:     PetscCall(MatAssemblyBegin(P, MAT_FINAL_ASSEMBLY));
106:     PetscCall(MatAssemblyEnd(P, MAT_FINAL_ASSEMBLY));
107:     PetscCall(PetscObjectTypeCompare((PetscObject)a, MATSEQAIJ, &flg));
108:     if (flg) PetscCall(MatPtAP(a, P, MAT_INITIAL_MATRIX, 1.0, &X)); // MatPtAP() is used to extend diagonal blocks with zeros on the overlap
109:     else {                                                          // workaround for MatPtAP() limitations with some types
110:       PetscCall(MatConvert(a, MATSEQAIJ, MAT_INITIAL_MATRIX, &c));
111:       PetscCall(MatPtAP(c, P, MAT_INITIAL_MATRIX, 1.0, &X));
112:       PetscCall(MatDestroy(&c));
113:     }
114:     PetscCall(MatDestroy(&P));
115:     PetscCall(MatAXPY(B, 1.0, X, SUBSET_NONZERO_PATTERN));
116:     PetscCall(MatDestroy(&X));
117:     PetscCall(MatSetOption(B, MAT_SYMMETRIC, PETSC_TRUE));
118:     PetscCall(PCHPDDMSetRHSMat(pc, B));
119:     PetscCall(MatDestroy(&B));
120:   }
121: #else
122:   (void)share;
123: #endif
124:   PetscCall(MatDestroy(&aux));
125:   PetscCall(KSPSetFromOptions(ksp));
126:   PetscCall(PetscObjectTypeCompare((PetscObject)pc, PCASM, &flg));
127:   if (flg) {
128:     flg = PETSC_FALSE;
129:     PetscCall(PetscOptionsGetBool(NULL, NULL, "-pc_hpddm_define_subdomains", &flg, NULL));
130:     if (flg) {
131:       IS rows;

133:       PetscCall(MatGetOwnershipIS(A, &rows, NULL));
134:       PetscCall(PCASMSetLocalSubdomains(pc, 1, &is, &rows));
135:       PetscCall(ISDestroy(&rows));
136:     }
137:   }
138:   PetscCall(ISDestroy(&is));
139:   PetscCall(MatCreateVecs(A, NULL, &b));
140:   PetscCall(VecSet(b, 1.0));
141:   PetscCall(KSPSolve(ksp, b, b));
142:   PetscCall(VecGetLocalSize(b, &m));
143:   PetscCall(VecDestroy(&b));
144:   if (N > 1) {
145:     KSPType type;

147:     PetscCall(PetscOptionsClearValue(NULL, "-ksp_converged_reason"));
148:     PetscCall(KSPSetFromOptions(ksp));
149:     PetscCall(MatCreateDense(PETSC_COMM_WORLD, m, PETSC_DECIDE, PETSC_DECIDE, N, NULL, &B));
150:     PetscCall(MatCreateDense(PETSC_COMM_WORLD, m, PETSC_DECIDE, PETSC_DECIDE, N, NULL, &X));
151:     PetscCall(MatSetRandom(B, NULL));
152:     /* this is algorithmically optimal in the sense that blocks of vectors are coarsened or interpolated using matrix--matrix operations */
153:     /* PCHPDDM however heavily relies on MPI[S]BAIJ format for which there is no efficient MatProduct implementation */
154:     PetscCall(KSPMatSolve(ksp, B, X));
155:     PetscCall(KSPGetType(ksp, &type));
156:     PetscCall(PetscStrcmp(type, KSPHPDDM, &flg));
157: #if defined(PETSC_HAVE_HPDDM)
158:     if (flg) {
159:       PetscReal    norm;
160:       KSPHPDDMType type;

162:       PetscCall(KSPHPDDMGetType(ksp, &type));
163:       if (type == KSP_HPDDM_TYPE_PREONLY || type == KSP_HPDDM_TYPE_CG || type == KSP_HPDDM_TYPE_GMRES || type == KSP_HPDDM_TYPE_GCRODR) {
164:         Mat C;

166:         PetscCall(MatDuplicate(X, MAT_DO_NOT_COPY_VALUES, &C));
167:         PetscCall(KSPSetMatSolveBatchSize(ksp, 1));
168:         PetscCall(KSPMatSolve(ksp, B, C));
169:         PetscCall(MatAYPX(C, -1.0, X, SAME_NONZERO_PATTERN));
170:         PetscCall(MatNorm(C, NORM_INFINITY, &norm));
171:         PetscCall(MatDestroy(&C));
172:         PetscCheck(norm <= 100 * PETSC_MACHINE_EPSILON, PetscObjectComm((PetscObject)pc), PETSC_ERR_PLIB, "KSPMatSolve() and KSPSolve() difference has nonzero norm %g with pseudo-block KSPHPDDMType %s", (double)norm, KSPHPDDMTypes[type]);
173:       }
174:     }
175: #endif
176:     PetscCall(MatDestroy(&X));
177:     PetscCall(MatDestroy(&B));
178:   }
179:   PetscCall(PetscObjectTypeCompare((PetscObject)pc, PCHPDDM, &flg));
180: #if defined(PETSC_HAVE_HPDDM) && defined(PETSC_HAVE_DYNAMIC_LIBRARIES) && defined(PETSC_USE_SHARED_LIBRARIES)
181:   if (flg) PetscCall(PCHPDDMGetSTShareSubKSP(pc, &flg));
182: #endif
183:   if (flg && PetscDefined(USE_LOG)) {
184:     PetscCall(PetscOptionsHasName(NULL, NULL, "-pc_hpddm_harmonic_overlap", &flg));
185:     if (!flg) {
186:       PetscLogEvent      event;
187:       PetscEventPerfInfo info1, info2;

189:       PetscCall(PetscLogEventRegister("MatLUFactorSym", PC_CLASSID, &event));
190:       PetscCall(PetscLogEventGetPerfInfo(PETSC_DETERMINE, event, &info1));
191:       PetscCall(PetscLogEventRegister("MatLUFactorNum", PC_CLASSID, &event));
192:       PetscCall(PetscLogEventGetPerfInfo(PETSC_DETERMINE, event, &info2));
193:       if (!info1.count && !info2.count) {
194:         PetscCall(PetscLogEventRegister("MatCholFctrSym", PC_CLASSID, &event));
195:         PetscCall(PetscLogEventGetPerfInfo(PETSC_DETERMINE, event, &info1));
196:         PetscCall(PetscLogEventRegister("MatCholFctrNum", PC_CLASSID, &event));
197:         PetscCall(PetscLogEventGetPerfInfo(PETSC_DETERMINE, event, &info2));
198:         PetscCheck(info2.count > info1.count, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Cholesky numerical factorization (%d) not called more times than Cholesky symbolic factorization (%d), broken -pc_hpddm_levels_1_st_share_sub_ksp", info2.count, info1.count);
199:       } else PetscCheck(info2.count > info1.count, PETSC_COMM_SELF, PETSC_ERR_PLIB, "LU numerical factorization (%d) not called more times than LU symbolic factorization (%d), broken -pc_hpddm_levels_1_st_share_sub_ksp", info2.count, info1.count);
200:     }
201:   }
202: #if defined(PETSC_HAVE_HPDDM) && defined(PETSC_HAVE_DYNAMIC_LIBRARIES) && defined(PETSC_USE_SHARED_LIBRARIES)
203:   if (N == 1) {
204:     flg = PETSC_FALSE;
205:     PetscCall(PetscOptionsGetBool(NULL, NULL, "-successive_solves", &flg, NULL));
206:     if (flg) {
207:       KSPConvergedReason reason[2];
208:       PetscInt           iterations[3];

210:       PetscCall(KSPGetConvergedReason(ksp, reason));
211:       PetscCall(KSPGetTotalIterations(ksp, iterations));
212:       PetscCall(PetscOptionsClearValue(NULL, "-ksp_converged_reason"));
213:       PetscCall(KSPSetFromOptions(ksp));
214:       flg = PETSC_FALSE;
215:       PetscCall(PetscOptionsGetBool(NULL, NULL, "-pc_hpddm_block_splitting", &flg, NULL));
216:       if (!flg) {
217:         PetscCall(MatCreate(PETSC_COMM_SELF, &aux));
218:         PetscCall(ISCreate(PETSC_COMM_SELF, &is));
219:         PetscCall(PetscSNPrintf(name, sizeof(name), "%s/is_%d_%d.dat", dir, rank, size));
220:         PetscCall(PetscViewerBinaryOpen(PETSC_COMM_SELF, name, FILE_MODE_READ, &viewer));
221:         PetscCall(ISLoad(is, viewer));
222:         PetscCall(ISSetBlockSize(is, 2));
223:         PetscCall(PetscViewerDestroy(&viewer));
224:         PetscCall(PetscSNPrintf(name, sizeof(name), "%s/Neumann_%d_%d.dat", dir, rank, size));
225:         PetscCall(PetscViewerBinaryOpen(PETSC_COMM_SELF, name, FILE_MODE_READ, &viewer));
226:         PetscCall(MatLoad(aux, viewer));
227:         PetscCall(PetscViewerDestroy(&viewer));
228:         PetscCall(MatSetBlockSizesFromMats(aux, A, A));
229:         PetscCall(MatSetOption(aux, MAT_SYMMETRIC, PETSC_TRUE));
230:         PetscCall(MatConvert(aux, type, MAT_INPLACE_MATRIX, &aux));
231:       }
232:       PetscCall(MatCreateVecs(A, NULL, &b));
233:       PetscCall(PetscObjectStateIncrease((PetscObject)A));
234:       if (!flg) PetscCall(PCHPDDMSetAuxiliaryMat(pc, NULL, aux, NULL, NULL));
235:       PetscCall(VecSet(b, 1.0));
236:       PetscCall(KSPSolve(ksp, b, b));
237:       PetscCall(KSPGetConvergedReason(ksp, reason + 1));
238:       PetscCall(KSPGetTotalIterations(ksp, iterations + 1));
239:       iterations[1] -= iterations[0];
240:       PetscCheck(reason[0] == reason[1] && PetscAbs(iterations[0] - iterations[1]) <= 3, PetscObjectComm((PetscObject)ksp), PETSC_ERR_PLIB, "Successive calls to KSPSolve() did not converge for the same reason (%s v. %s) or with the same number of iterations (+/- 3, %" PetscInt_FMT " v. %" PetscInt_FMT ")", KSPConvergedReasons[reason[0]], KSPConvergedReasons[reason[1]], iterations[0], iterations[1]);
241:       PetscCall(PetscObjectStateIncrease((PetscObject)A));
242:       if (!flg) PetscCall(PCHPDDMSetAuxiliaryMat(pc, is, aux, NULL, NULL));
243:       PetscCall(PCSetFromOptions(pc));
244:       PetscCall(VecSet(b, 1.0));
245:       PetscCall(KSPSolve(ksp, b, b));
246:       PetscCall(KSPGetConvergedReason(ksp, reason + 1));
247:       PetscCall(KSPGetTotalIterations(ksp, iterations + 2));
248:       iterations[2] -= iterations[0] + iterations[1];
249:       PetscCheck(reason[0] == reason[1] && PetscAbs(iterations[0] - iterations[2]) <= 3, PetscObjectComm((PetscObject)ksp), PETSC_ERR_PLIB, "Successive calls to KSPSolve() did not converge for the same reason (%s v. %s) or with the same number of iterations (+/- 3, %" PetscInt_FMT " v. %" PetscInt_FMT ")", KSPConvergedReasons[reason[0]], KSPConvergedReasons[reason[1]], iterations[0], iterations[2]);
250:       PetscCall(VecDestroy(&b));
251:       PetscCall(ISDestroy(&is));
252:       PetscCall(MatDestroy(&aux));
253:     }
254:   }
255:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewer", &flg, NULL));
256:   if (flg) {
257:     PetscCall(PetscObjectTypeCompare((PetscObject)pc, PCHPDDM, &flg));
258:     if (flg) {
259:       PetscCall(PetscStrncpy(dir, "XXXXXX", sizeof(dir)));
260:       if (rank == 0) PetscCall(PetscMkdtemp(dir));
261:       PetscCallMPI(MPI_Bcast(dir, 6, MPI_CHAR, 0, PETSC_COMM_WORLD));
262:       for (PetscInt i = 0; i < 2; ++i) {
263:         PetscCall(PetscSNPrintf(name, sizeof(name), "%s/%s", dir, i == 0 ? "A" : "A.dat"));
264:         PetscCall(PetscViewerASCIIOpen(PETSC_COMM_WORLD, name, &viewer));
265:         PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_ASCII_INFO_DETAIL));
266:         PetscCall(PCView(pc, viewer));
267:         PetscCall(PetscViewerPopFormat(viewer));
268:         PetscCall(PetscViewerDestroy(&viewer));
269:       }
270:       PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
271:       if (rank == 0) PetscCall(PetscRMTree(dir));
272:     }
273:   }
274: #endif
275:   PetscCall(KSPDestroy(&ksp));
276:   PetscCall(MatDestroy(&A));
277:   PetscCall(PetscFinalize());
278:   return 0;
279: }

281: /*TEST

283:    test:
284:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
285:       nsize: 4
286:       args: -ksp_rtol 1e-3 -ksp_converged_reason -pc_type {{bjacobi hpddm}shared output} -pc_hpddm_coarse_sub_pc_type lu -sub_pc_type lu -options_left no -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO

288:    testset:
289:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
290:       suffix: define_subdomains
291:       nsize: 4
292:       args: -ksp_rtol 1e-3 -ksp_converged_reason -pc_hpddm_define_subdomains -options_left no -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO
293:       test:
294:         args: -pc_type {{asm hpddm}shared output} -pc_hpddm_coarse_sub_pc_type lu -sub_pc_type lu -viewer
295:       test:
296:         args: -pc_type hpddm -pc_hpddm_levels_1_st_pc_type cholesky -pc_hpddm_levels_1_eps_nev 5 -pc_hpddm_coarse_sub_pc_type lu -pc_hpddm_levels_1_sub_pc_type lu -pc_hpddm_coarse_correction none

298:    testset:
299:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
300:       nsize: 4
301:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_coarse_pc_type redundant -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO
302:       test:
303:         suffix: geneo
304:         args: -pc_hpddm_coarse_p {{1 2}shared output} -pc_hpddm_levels_1_st_pc_type cholesky -pc_hpddm_levels_1_eps_nev {{5 15}separate output} -mat_type {{aij baij sbaij}shared output}
305:       test:
306:         suffix: geneo_block_splitting
307:         output_file: output/ex76_geneo_pc_hpddm_levels_1_eps_nev-15.out
308:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 1[6-9]/Linear solve converged due to CONVERGED_RTOL iterations 11/g"
309:         args: -pc_hpddm_coarse_p 2 -pc_hpddm_levels_1_eps_nev 15 -pc_hpddm_block_splitting -pc_hpddm_levels_1_st_pc_type lu -pc_hpddm_levels_1_eps_gen_non_hermitian -mat_type {{aij baij}shared output} -successive_solves
310:       test:
311:         suffix: geneo_share
312:         output_file: output/ex76_geneo_pc_hpddm_levels_1_eps_nev-5.out
313:         args: -pc_hpddm_levels_1_st_pc_type cholesky -pc_hpddm_levels_1_eps_nev 5 -pc_hpddm_levels_1_st_share_sub_ksp -reset {{false true}shared output}
314:       test:
315:         suffix: harmonic_overlap_1_define_false
316:         output_file: output/ex76_geneo_share.out
317:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 1[0-3]/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
318:         args: -pc_hpddm_harmonic_overlap 1 -pc_hpddm_levels_1_eps_nev 30 -pc_hpddm_levels_1_eps_relative_threshold 1e+1 -pc_hpddm_levels_1_st_pc_type lu -pc_hpddm_levels_1_eps_pc_type lu -pc_hpddm_define_subdomains false -pc_hpddm_levels_1_pc_type asm -pc_hpddm_levels_1_pc_asm_overlap 2 -mat_type baij
319:       test:
320:         suffix: harmonic_overlap_1
321:         output_file: output/ex76_geneo_share.out
322:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 1[0-3]/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
323:         args: -pc_hpddm_harmonic_overlap 1 -pc_hpddm_levels_1_eps_nev 30 -pc_hpddm_levels_1_eps_relative_threshold 1e+1 -pc_hpddm_levels_1_st_pc_type lu -pc_hpddm_levels_1_eps_pc_type lu -mat_type baij
324:       test:
325:         suffix: harmonic_overlap_1_share_petsc
326:         output_file: output/ex76_geneo_share.out
327:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 1[0-3]/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
328:         args: -pc_hpddm_harmonic_overlap 1 -pc_hpddm_levels_1_eps_nev 30 -pc_hpddm_levels_1_eps_relative_threshold 1e+1 -pc_hpddm_levels_1_st_pc_type lu -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_levels_1_sub_pc_factor_mat_solver_type petsc -pc_hpddm_levels_1_eps_pc_type lu -mat_type baij
329:       test:
330:         requires: mumps
331:         suffix: harmonic_overlap_1_share_mumps
332:         output_file: output/ex76_geneo_share.out
333:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 1[0-3]/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
334:         args: -pc_hpddm_harmonic_overlap 1 -pc_hpddm_levels_1_eps_nev 30 -pc_hpddm_levels_1_eps_relative_threshold 1e+1 -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_levels_1_sub_pc_factor_mat_solver_type mumps
335:       test:
336:         requires: mumps
337:         suffix: harmonic_overlap_1_share_mumps_not_set_explicitly
338:         output_file: output/ex76_geneo_share.out
339:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 1[0-3]/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
340:         args: -pc_hpddm_harmonic_overlap 1 -pc_hpddm_levels_1_eps_nev 30 -pc_hpddm_levels_1_eps_relative_threshold 1e+1 -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_levels_1_eps_mat_type baij
341:       test:
342:         requires: mkl_pardiso
343:         suffix: harmonic_overlap_1_share_mkl_pardiso
344:         output_file: output/ex76_geneo_share.out
345:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations [12][0-3]/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
346:         args: -pc_hpddm_harmonic_overlap 1 -pc_hpddm_levels_1_eps_nev 30 -pc_hpddm_levels_1_eps_relative_threshold 1e+1 -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_levels_1_eps_mat_type shell -pc_hpddm_levels_1_sub_pc_factor_mat_solver_type mkl_pardiso
347:       test:
348:         requires: mkl_pardiso !mumps
349:         suffix: harmonic_overlap_1_share_mkl_pardiso_no_set_explicitly
350:         output_file: output/ex76_geneo_share.out
351:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations [12][0-3]/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
352:         args: -pc_hpddm_harmonic_overlap 1 -pc_hpddm_levels_1_eps_nev 30 -pc_hpddm_levels_1_eps_relative_threshold 1e+1 -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_levels_1_eps_mat_type shell
353:       test:
354:         suffix: harmonic_overlap_2_relative_threshold
355:         output_file: output/ex76_geneo_share.out
356:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 9/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
357:         args: -pc_hpddm_harmonic_overlap 2 -pc_hpddm_levels_1_svd_nsv 15 -pc_hpddm_levels_1_svd_relative_threshold 1e-1 -pc_hpddm_levels_1_st_share_sub_ksp -mat_type sbaij
358:       test:
359:         suffix: harmonic_overlap_2
360:         output_file: output/ex76_geneo_share.out
361:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 9/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
362:         args: -pc_hpddm_harmonic_overlap 2 -pc_hpddm_levels_1_svd_nsv 12 -pc_hpddm_levels_1_st_share_sub_ksp -mat_type sbaij

364:    testset:
365:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
366:       nsize: 4
367:       args: -ksp_converged_reason -ksp_max_it 150 -pc_type hpddm -pc_hpddm_levels_1_eps_nev 5 -pc_hpddm_coarse_p 1 -pc_hpddm_coarse_pc_type redundant -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO -pc_hpddm_define_subdomains
368:       test:
369:         suffix: geneo_share_cholesky
370:         output_file: output/ex76_geneo_share.out
371:         # extra -pc_hpddm_levels_1_eps_gen_non_hermitian needed to avoid failures with PETSc Cholesky
372:         args: -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_st_pc_type cholesky -mat_type {{aij sbaij}shared output} -pc_hpddm_levels_1_eps_gen_non_hermitian -pc_hpddm_has_neumann -pc_hpddm_levels_1_st_share_sub_ksp {{false true}shared output} -successive_solves
373:       test:
374:         suffix: geneo_share_cholesky_matstructure
375:         output_file: output/ex76_geneo_share.out
376:         # extra -pc_hpddm_levels_1_eps_gen_non_hermitian needed to avoid failures with PETSc Cholesky
377:         filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 14/Linear solve converged due to CONVERGED_RTOL iterations 15/g"
378:         args: -pc_hpddm_levels_1_sub_pc_type cholesky -mat_type {{baij sbaij}shared output} -pc_hpddm_levels_1_eps_gen_non_hermitian -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_levels_1_st_matstructure same -set_rhs {{false true} shared output}
379:       test:
380:         requires: mumps
381:         suffix: geneo_share_lu
382:         output_file: output/ex76_geneo_share.out
383:         # extra -pc_factor_mat_solver_type mumps needed to avoid failures with PETSc LU
384:         args: -pc_hpddm_levels_1_sub_pc_type lu -pc_hpddm_levels_1_st_pc_type lu -mat_type baij -pc_hpddm_levels_1_st_pc_factor_mat_solver_type mumps -pc_hpddm_levels_1_sub_pc_factor_mat_solver_type mumps -pc_hpddm_has_neumann -pc_hpddm_levels_1_st_share_sub_ksp {{false true}shared output}
385:       test:
386:         requires: mumps
387:         suffix: geneo_share_lu_matstructure
388:         output_file: output/ex76_geneo_share.out
389:         # extra -pc_factor_mat_solver_type mumps needed to avoid failures with PETSc LU
390:         args: -pc_hpddm_levels_1_sub_pc_type lu -mat_type aij -pc_hpddm_levels_1_sub_pc_factor_mat_solver_type mumps -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_levels_1_st_matstructure {{same different}shared output} -pc_hpddm_levels_1_st_pc_type lu -pc_hpddm_levels_1_st_pc_factor_mat_solver_type mumps -successive_solves -pc_hpddm_levels_1_eps_target 1e-5
391:       test:
392:         suffix: geneo_share_not_asm
393:         output_file: output/ex76_geneo_pc_hpddm_levels_1_eps_nev-5.out
394:         # extra -pc_hpddm_levels_1_eps_gen_non_hermitian needed to avoid failures with PETSc Cholesky
395:         args: -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_st_pc_type cholesky -pc_hpddm_levels_1_eps_gen_non_hermitian -pc_hpddm_has_neumann -pc_hpddm_levels_1_st_share_sub_ksp true -pc_hpddm_levels_1_pc_type gasm -successive_solves

397:    test:
398:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
399:       suffix: fgmres_geneo_20_p_2
400:       nsize: 4
401:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type lu -pc_hpddm_levels_1_eps_nev 20 -pc_hpddm_coarse_p 2 -pc_hpddm_coarse_pc_type redundant -ksp_type fgmres -pc_hpddm_coarse_mat_type {{baij sbaij}shared output} -pc_hpddm_log_separate {{false true}shared output} -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO

403:    testset:
404:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
405:       output_file: output/ex76_fgmres_geneo_20_p_2.out
406:       nsize: 4
407:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_nev 20 -pc_hpddm_levels_2_p 2 -pc_hpddm_levels_2_mat_type {{baij sbaij}shared output} -pc_hpddm_levels_2_eps_nev {{5 20}shared output} -pc_hpddm_levels_2_sub_pc_type cholesky -pc_hpddm_levels_2_ksp_type gmres -ksp_type fgmres -pc_hpddm_coarse_mat_type {{baij sbaij}shared output} -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO
408:       test:
409:         suffix: fgmres_geneo_20_p_2_geneo
410:         args: -mat_type {{aij sbaij}shared output}
411:       test:
412:         suffix: fgmres_geneo_20_p_2_geneo_algebraic
413:         args: -pc_hpddm_levels_2_st_pc_type mat
414:    # PCHPDDM + KSPHPDDM test to exercise multilevel + multiple RHS in one go
415:    test:
416:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
417:       suffix: fgmres_geneo_20_p_2_geneo_rhs
418:       output_file: output/ex76_fgmres_geneo_20_p_2.out
419:       # for -pc_hpddm_coarse_correction additive
420:       filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 37/Linear solve converged due to CONVERGED_RTOL iterations 25/g"
421:       nsize: 4
422:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_nev 20 -pc_hpddm_levels_2_p 2 -pc_hpddm_levels_2_mat_type baij -pc_hpddm_levels_2_eps_nev 5 -pc_hpddm_levels_2_sub_pc_type cholesky -pc_hpddm_levels_2_ksp_max_it 10 -pc_hpddm_levels_2_ksp_type hpddm -pc_hpddm_levels_2_ksp_hpddm_type gmres -ksp_type hpddm -ksp_hpddm_variant flexible -pc_hpddm_coarse_mat_type baij -mat_type aij -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO -rhs 4 -pc_hpddm_coarse_correction {{additive deflated balanced}shared output}

424:    testset:
425:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES) mumps defined(PETSC_HAVE_OPENMP_SUPPORT)
426:       filter: grep -E -e "Linear solve" -e "      executing" | sed -e "s/MPI =      1/MPI =      2/g" -e "s/OMP =      1/OMP =      2/g"
427:       nsize: 4
428:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_nev 15 -pc_hpddm_levels_1_st_pc_type cholesky -pc_hpddm_coarse_p {{1 2}shared output} -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO -pc_hpddm_coarse_pc_factor_mat_solver_type mumps -pc_hpddm_coarse_mat_mumps_icntl_4 2 -pc_hpddm_coarse_mat_mumps_use_omp_threads {{1 2}shared output}
429:       test:
430:         suffix: geneo_mumps_use_omp_threads_1
431:         output_file: output/ex76_geneo_mumps_use_omp_threads.out
432:         args: -pc_hpddm_coarse_mat_type {{baij sbaij}shared output}
433:       test:
434:         suffix: geneo_mumps_use_omp_threads_2
435:         output_file: output/ex76_geneo_mumps_use_omp_threads.out
436:         args: -pc_hpddm_coarse_mat_type aij -pc_hpddm_levels_1_eps_threshold 0.4 -pc_hpddm_coarse_pc_type cholesky -pc_hpddm_coarse_mat_filter 1e-12

438:    testset: # converge really poorly because of a tiny -pc_hpddm_levels_1_eps_threshold, but needed for proper code coverage where some subdomains don't call EPSSolve()
439:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
440:       nsize: 4
441:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_threshold 0.005 -pc_hpddm_levels_1_eps_use_inertia -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO -pc_hpddm_levels_1_st_share_sub_ksp -pc_hpddm_define_subdomains -pc_hpddm_has_neumann -ksp_rtol 0.9
442:       filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 1/Linear solve converged due to CONVERGED_RTOL iterations 141/g"
443:       test:
444:         suffix: inertia_petsc
445:         output_file: output/ex76_1.out
446:         args: -pc_hpddm_levels_1_sub_pc_factor_mat_solver_type petsc
447:       test:
448:         suffix: inertia_mumps
449:         output_file: output/ex76_1.out
450:         requires: mumps

452:    test:
453:       requires: hpddm slepc datafilespath double !complex !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_DYNAMIC_LIBRARIES) defined(PETSC_USE_SHARED_LIBRARIES)
454:       suffix: reuse_symbolic
455:       output_file: output/ex77_preonly.out
456:       nsize: 4
457:       args: -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_nev 20 -rhs 4 -pc_hpddm_coarse_correction {{additive deflated balanced}shared output} -ksp_pc_side {{left right}shared output} -ksp_max_it 20 -ksp_type hpddm -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO -pc_hpddm_define_subdomains -ksp_error_if_not_converged

459: TEST*/