Actual source code: ex31.c
1: static char help[] = "Test that shared points on interface of partitions can be rebalanced.\n\n";
2: static char FILENAME[] = "ex31.c";
4: #include <petscdmplex.h>
5: #include <petscviewerhdf5.h>
6: #include <petscsf.h>
8: typedef struct {
9: PetscBool parallel; /* Use ParMetis or Metis */
10: PetscBool useInitialGuess; /* Only active when in parallel, uses RefineKway of ParMetis */
11: PetscInt entityDepth; /* depth of the entities to rebalance ( 0 => vertices) */
12: } AppCtx;
14: static PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options)
15: {
16: PetscFunctionBegin;
17: options->parallel = PETSC_FALSE;
18: options->useInitialGuess = PETSC_FALSE;
19: options->entityDepth = 0;
21: PetscOptionsBegin(comm, "", "Meshing Interpolation Test Options", "DMPLEX");
22: PetscCall(PetscOptionsBoundedInt("-entity_depth", "Depth of the entities to rebalance (0 => vertices)", FILENAME, options->entityDepth, &options->entityDepth, NULL, 0));
23: PetscCall(PetscOptionsBool("-parallel", "Use ParMetis instead of Metis", FILENAME, options->parallel, &options->parallel, NULL));
24: PetscCall(PetscOptionsBool("-use_initial_guess", "Use RefineKway function of ParMetis", FILENAME, options->useInitialGuess, &options->useInitialGuess, NULL));
25: PetscOptionsEnd();
26: PetscFunctionReturn(PETSC_SUCCESS);
27: }
29: static PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
30: {
31: PetscFunctionBegin;
32: PetscCall(DMCreate(comm, dm));
33: PetscCall(DMSetType(*dm, DMPLEX));
34: PetscCall(DMSetFromOptions(*dm));
35: PetscCall(DMViewFromOptions(*dm, NULL, "-dm_view"));
36: PetscFunctionReturn(PETSC_SUCCESS);
37: }
39: int main(int argc, char **argv)
40: {
41: MPI_Comm comm;
42: DM dm, dmdist;
43: PetscPartitioner part;
44: AppCtx user;
45: IS is = NULL;
46: PetscSection s = NULL, gsection = NULL;
47: PetscMPIInt size;
48: PetscSF sf;
49: PetscInt pStart, pEnd, p, minBefore, maxBefore, minAfter, maxAfter, gSizeBefore, gSizeAfter;
50: PetscBool success;
52: PetscFunctionBeginUser;
53: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
54: comm = PETSC_COMM_WORLD;
55: PetscCallMPI(MPI_Comm_size(comm, &size));
56: PetscCall(ProcessOptions(comm, &user));
57: PetscCall(CreateMesh(comm, &user, &dm));
59: /* partition dm using PETSCPARTITIONERPARMETIS */
60: PetscCall(DMPlexGetPartitioner(dm, &part));
61: PetscCall(PetscObjectSetOptionsPrefix((PetscObject)part, "p_"));
62: PetscCall(PetscPartitionerSetType(part, PETSCPARTITIONERPARMETIS));
63: PetscCall(PetscPartitionerSetFromOptions(part));
64: PetscCall(PetscSectionCreate(comm, &s));
65: PetscCall(PetscPartitionerDMPlexPartition(part, dm, NULL, s, &is));
67: PetscCall(DMPlexDistribute(dm, 0, NULL, &dmdist));
68: if (dmdist) {
69: PetscCall(DMDestroy(&dm));
70: dm = dmdist;
71: }
73: /* cleanup */
74: PetscCall(PetscSectionDestroy(&s));
75: PetscCall(ISDestroy(&is));
77: /* We make a PetscSection with a DOF on every mesh entity of depth
78: * user.entityDepth, then make a global section and look at its storage size.
79: * We do the same thing after the rebalancing and then assert that the size
80: * remains the same. We also make sure that the balance has improved at least
81: * a little bit compared to the initial decomposition. */
83: if (size > 1) {
84: PetscCall(PetscSectionCreate(comm, &s));
85: PetscCall(PetscSectionSetNumFields(s, 1));
86: PetscCall(PetscSectionSetFieldComponents(s, 0, 1));
87: PetscCall(DMPlexGetDepthStratum(dm, user.entityDepth, &pStart, &pEnd));
88: PetscCall(PetscSectionSetChart(s, pStart, pEnd));
89: for (p = pStart; p < pEnd; ++p) {
90: PetscCall(PetscSectionSetDof(s, p, 1));
91: PetscCall(PetscSectionSetFieldDof(s, p, 0, 1));
92: }
93: PetscCall(PetscSectionSetUp(s));
94: PetscCall(DMGetPointSF(dm, &sf));
95: PetscCall(PetscSectionCreateGlobalSection(s, sf, PETSC_FALSE, PETSC_FALSE, &gsection));
96: PetscCall(PetscSectionGetStorageSize(gsection, &gSizeBefore));
97: minBefore = gSizeBefore;
98: maxBefore = gSizeBefore;
99: PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &gSizeBefore, 1, MPIU_INT, MPI_SUM, comm));
100: PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &minBefore, 1, MPIU_INT, MPI_MIN, comm));
101: PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &maxBefore, 1, MPIU_INT, MPI_MAX, comm));
102: PetscCall(PetscSectionDestroy(&gsection));
103: }
105: PetscCall(DMPlexRebalanceSharedPoints(dm, user.entityDepth, user.useInitialGuess, user.parallel, &success));
107: if (size > 1) {
108: PetscCall(PetscSectionCreateGlobalSection(s, sf, PETSC_FALSE, PETSC_FALSE, &gsection));
109: PetscCall(PetscSectionGetStorageSize(gsection, &gSizeAfter));
110: minAfter = gSizeAfter;
111: maxAfter = gSizeAfter;
112: PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &gSizeAfter, 1, MPIU_INT, MPI_SUM, comm));
113: PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &minAfter, 1, MPIU_INT, MPI_MIN, comm));
114: PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &maxAfter, 1, MPIU_INT, MPI_MAX, comm));
115: PetscCheck(gSizeAfter == gSizeBefore, comm, PETSC_ERR_PLIB, "Global section has not the same size before and after.");
116: PetscCheck(minAfter >= minBefore && maxAfter <= maxBefore && (minAfter > minBefore || maxAfter < maxBefore), comm, PETSC_ERR_PLIB, "DMPlexRebalanceSharedPoints did not improve mesh point balance.");
117: PetscCall(PetscSectionDestroy(&gsection));
118: PetscCall(PetscSectionDestroy(&s));
119: }
121: PetscCall(DMDestroy(&dm));
122: PetscCall(PetscFinalize());
123: return 0;
124: }
126: /*TEST
128: testset:
129: args: -dm_plex_dim 3 -dm_plex_simplex 0
131: test:
132: # rebalance a mesh
133: suffix: 0
134: nsize: {{2 3 4}}
135: requires: parmetis
136: args: -dm_plex_box_faces {{2,3,4 5,4,3 7,11,5}} -entity_depth {{0 1}} -parallel {{FALSE TRUE}} -use_initial_guess FALSE
138: test:
139: # rebalance a mesh but use the initial guess (uses a random algorithm and gives different results on different machines, so just check that it runs).
140: suffix: 1
141: nsize: {{2 3 4}}
142: requires: parmetis
143: args: -dm_plex_box_faces {{2,3,4 5,4,3 7,11,5}} -entity_depth {{0 1}} -parallel TRUE -use_initial_guess TRUE
145: test:
146: # no-op in serial
147: suffix: 2
148: nsize: {{1}}
149: requires: parmetis
150: args: -dm_plex_box_faces 2,3,4 -entity_depth 0 -parallel FALSE -use_initial_guess FALSE
152: TEST*/