Actual source code: fetidp.c

petsc-3.8.4 2018-03-24
Report Typos and Errors
  1: #include <petsc/private/kspimpl.h> /*I <petscksp.h> I*/
  2:  #include <../src/ksp/pc/impls/bddc/bddc.h>
  3:  #include <../src/ksp/pc/impls/bddc/bddcprivate.h>

  5: static PetscBool  cited  = PETSC_FALSE;
  6: static PetscBool  cited2 = PETSC_FALSE;
  7: static const char citation[] =
  8: "@article{ZampiniPCBDDC,\n"
  9: "author = {Stefano Zampini},\n"
 10: "title = {{PCBDDC}: A Class of Robust Dual-Primal Methods in {PETS}c},\n"
 11: "journal = {SIAM Journal on Scientific Computing},\n"
 12: "volume = {38},\n"
 13: "number = {5},\n"
 14: "pages = {S282-S306},\n"
 15: "year = {2016},\n"
 16: "doi = {10.1137/15M1025785},\n"
 17: "URL = {http://dx.doi.org/10.1137/15M1025785},\n"
 18: "eprint = {http://dx.doi.org/10.1137/15M1025785}\n"
 19: "}\n"
 20: "@article{ZampiniDualPrimal,\n"
 21: "author = {Stefano Zampini},\n"
 22: "title = {{D}ual-{P}rimal methods for the cardiac {B}idomain model},\n"
 23: "volume = {24},\n"
 24: "number = {04},\n"
 25: "pages = {667-696},\n"
 26: "year = {2014},\n"
 27: "doi = {10.1142/S0218202513500632},\n"
 28: "URL = {http://www.worldscientific.com/doi/abs/10.1142/S0218202513500632},\n"
 29: "eprint = {http://www.worldscientific.com/doi/pdf/10.1142/S0218202513500632}\n"
 30: "}\n";
 31: static const char citation2[] =
 32: "@article{li2013nonoverlapping,\n"
 33: "title={A nonoverlapping domain decomposition method for incompressible Stokes equations with continuous pressures},\n"
 34: "author={Li, Jing and Tu, Xuemin},\n"
 35: "journal={SIAM Journal on Numerical Analysis},\n"
 36: "volume={51},\n"
 37: "number={2},\n"
 38: "pages={1235--1253},\n"
 39: "year={2013},\n"
 40: "publisher={Society for Industrial and Applied Mathematics}\n"
 41: "}\n";

 43: /*
 44:     This file implements the FETI-DP method in PETSc as part of KSP.
 45: */
 46: typedef struct {
 47:   KSP parentksp;
 48: } KSP_FETIDPMon;

 50: typedef struct {
 51:   KSP              innerksp;         /* the KSP for the Lagrange multipliers */
 52:   PC               innerbddc;        /* the inner BDDC object */
 53:   PetscBool        fully_redundant;  /* true for using a fully redundant set of multipliers */
 54:   PetscBool        userbddc;         /* true if the user provided the PCBDDC object */
 55:   PetscBool        saddlepoint;      /* support for saddle point problems */
 56:   IS               pP;               /* index set for pressure variables */
 57:   Vec              rhs_flip;         /* see KSPFETIDPSetUpOperators */
 58:   KSP_FETIDPMon    *monctx;          /* monitor context, used to pass user defined monitors
 59:                                         in the physical space */
 60:   PetscObjectState matstate;         /* these are needed just in the saddle point case */
 61:   PetscObjectState matnnzstate;      /* where we are going to use MatZeroRows on pmat */
 62:   PetscBool        statechanged;
 63:   PetscBool        check;
 64: } KSP_FETIDP;

 66: static PetscErrorCode KSPFETIDPSetPressureOperator_FETIDP(KSP ksp, Mat P)
 67: {
 68:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

 72:   if (P) fetidp->saddlepoint = PETSC_TRUE;
 73:   PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject)P);
 74:   return(0);
 75: }

 77: /*@
 78:  KSPFETIDPSetPressureOperator - Sets the operator used to setup the pressure preconditioner for saddle point FETI-DP.

 80:    Collective on KSP

 82:    Input Parameters:
 83: +  ksp - the FETI-DP Krylov solver
 84: -  P - the linear operator to be preconditioned, usually the mass matrix.

 86:    Level: advanced

 88:    Notes: The operator can be either passed in a) monolithic global ordering, b) pressure-only global ordering
 89:           or c) interface pressure ordering (if -ksp_fetidp_pressure_all false).
 90:           In cases b) and c), the pressure ordering of dofs needs to satisfy
 91:              pid_1 < pid_2  iff  gid_1 < gid_2
 92:           where pid_1 and pid_2 are two different pressure dof numbers and gid_1 and gid_2 the corresponding
 93:           id in the monolithic global ordering.

 95: .seealso: MATIS, PCBDDC, KSPFETIDPGetInnerBDDC, KSPFETIDPGetInnerKSP, KSPSetOperators
 96: @*/
 97: PetscErrorCode KSPFETIDPSetPressureOperator(KSP ksp, Mat P)
 98: {

104:   PetscTryMethod(ksp,"KSPFETIDPSetPressureOperator_C",(KSP,Mat),(ksp,P));
105:   return(0);
106: }

108: static PetscErrorCode KSPFETIDPGetInnerKSP_FETIDP(KSP ksp, KSP* innerksp)
109: {
110:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

113:   *innerksp = fetidp->innerksp;
114:   return(0);
115: }

117: /*@
118:  KSPFETIDPGetInnerKSP - Gets the KSP object for the Lagrange multipliers

120:    Input Parameters:
121: +  ksp - the FETI-DP KSP
122: -  innerksp - the KSP for the multipliers

124:    Level: advanced

126:    Notes:

128: .seealso: MATIS, PCBDDC, KSPFETIDPSetInnerBDDC, KSPFETIDPGetInnerBDDC
129: @*/
130: PetscErrorCode KSPFETIDPGetInnerKSP(KSP ksp, KSP* innerksp)
131: {

137:   PetscUseMethod(ksp,"KSPFETIDPGetInnerKSP_C",(KSP,KSP*),(ksp,innerksp));
138:   return(0);
139: }

141: static PetscErrorCode KSPFETIDPGetInnerBDDC_FETIDP(KSP ksp, PC* pc)
142: {
143:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

146:   *pc = fetidp->innerbddc;
147:   return(0);
148: }

150: /*@
151:  KSPFETIDPGetInnerBDDC - Gets the BDDC preconditioner used to setup the FETI-DP matrix for the Lagrange multipliers

153:    Input Parameters:
154: +  ksp - the FETI-DP Krylov solver
155: -  pc - the BDDC preconditioner

157:    Level: advanced

159:    Notes:

161: .seealso: MATIS, PCBDDC, KSPFETIDPSetInnerBDDC, KSPFETIDPGetInnerKSP
162: @*/
163: PetscErrorCode KSPFETIDPGetInnerBDDC(KSP ksp, PC* pc)
164: {

170:   PetscUseMethod(ksp,"KSPFETIDPGetInnerBDDC_C",(KSP,PC*),(ksp,pc));
171:   return(0);
172: }

174: static PetscErrorCode KSPFETIDPSetInnerBDDC_FETIDP(KSP ksp, PC pc)
175: {
176:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

180:   PetscObjectReference((PetscObject)pc);
181:   PCDestroy(&fetidp->innerbddc);
182:   fetidp->innerbddc = pc;
183:   fetidp->userbddc  = PETSC_TRUE;
184:   return(0);
185: }

187: /*@
188:  KSPFETIDPSetInnerBDDC - Sets the BDDC preconditioner used to setup the FETI-DP matrix for the Lagrange multipliers

190:    Collective on KSP

192:    Input Parameters:
193: +  ksp - the FETI-DP Krylov solver
194: -  pc - the BDDC preconditioner

196:    Level: advanced

198:    Notes:

200: .seealso: MATIS, PCBDDC, KSPFETIDPGetInnerBDDC, KSPFETIDPGetInnerKSP
201: @*/
202: PetscErrorCode KSPFETIDPSetInnerBDDC(KSP ksp, PC pc)
203: {
204:   PetscBool      isbddc;

210:   PetscObjectTypeCompare((PetscObject)pc,PCBDDC,&isbddc);
211:   if (!isbddc) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_ARG_WRONG,"KSPFETIDPSetInnerBDDC need a PCBDDC preconditioner");
212:   PetscTryMethod(ksp,"KSPFETIDPSetInnerBDDC_C",(KSP,PC),(ksp,pc));
213:   return(0);
214: }

216: static PetscErrorCode KSPBuildSolution_FETIDP(KSP ksp,Vec v,Vec *V)
217: {
218:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;
219:   Mat            F;
220:   Vec            Xl;

224:   KSPGetOperators(fetidp->innerksp,&F,NULL);
225:   KSPBuildSolution(fetidp->innerksp,NULL,&Xl);
226:   if (v) {
227:     PCBDDCMatFETIDPGetSolution(F,Xl,v);
228:     *V   = v;
229:   } else {
230:     PCBDDCMatFETIDPGetSolution(F,Xl,*V);
231:   }
232:   return(0);
233: }

235: static PetscErrorCode KSPMonitor_FETIDP(KSP ksp,PetscInt it,PetscReal rnorm,void* ctx)
236: {
237:   KSP_FETIDPMon  *monctx = (KSP_FETIDPMon*)ctx;

241:   KSPMonitor(monctx->parentksp,it,rnorm);
242:   return(0);
243: }

245: static PetscErrorCode KSPComputeEigenvalues_FETIDP(KSP ksp,PetscInt nmax,PetscReal *r,PetscReal *c,PetscInt *neig)
246: {
247:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

251:   KSPComputeEigenvalues(fetidp->innerksp,nmax,r,c,neig);
252:   return(0);
253: }

255: static PetscErrorCode KSPComputeExtremeSingularValues_FETIDP(KSP ksp,PetscReal *emax,PetscReal *emin)
256: {
257:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

261:   KSPComputeExtremeSingularValues(fetidp->innerksp,emax,emin);
262:   return(0);
263: }

265: static PetscErrorCode KSPFETIDPCheckOperators(KSP ksp, PetscViewer viewer)
266: {
267:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;
268:   PC_BDDC        *pcbddc = (PC_BDDC*)fetidp->innerbddc->data;
269:   PC_IS          *pcis = (PC_IS*)fetidp->innerbddc->data;
270:   Mat_IS         *matis = (Mat_IS*)fetidp->innerbddc->pmat->data;
271:   Mat            F;
272:   FETIDPMat_ctx  fetidpmat_ctx;
273:   Vec            test_vec,test_vec_p = NULL,fetidp_global;
274:   IS             dirdofs,isvert;
275:   MPI_Comm       comm = PetscObjectComm((PetscObject)ksp);
276:   PetscScalar    sval,*array;
277:   PetscReal      val,rval;
278:   const PetscInt *vertex_indices;
279:   PetscInt       i,n_vertices;
280:   PetscBool      isascii;

285:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&isascii);
286:   if (!isascii) SETERRQ(comm,PETSC_ERR_SUP,"Unsupported viewer");
287:   PetscViewerASCIIPrintf(viewer,"----------FETI-DP MAT  --------------\n");
288:   PetscViewerASCIIAddTab(viewer,2);
289:   KSPGetOperators(fetidp->innerksp,&F,NULL);
290:   PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_INFO);
291:   MatView(F,viewer);
292:   PetscViewerPopFormat(viewer);
293:   PetscViewerASCIISubtractTab(viewer,2);
294:   MatShellGetContext(F,(void**)&fetidpmat_ctx);
295:   PetscViewerASCIIPrintf(viewer,"----------FETI-DP TESTS--------------\n");
296:   PetscViewerASCIIPrintf(viewer,"All tests should return zero!\n");
297:   PetscViewerASCIIPrintf(viewer,"FETIDP MAT context in the ");
298:   if (fetidp->fully_redundant) {
299:     PetscViewerASCIIPrintf(viewer,"fully redundant case for lagrange multipliers.\n");
300:   } else {
301:     PetscViewerASCIIPrintf(viewer,"Non-fully redundant case for lagrange multiplier.\n");
302:   }
303:   PetscViewerFlush(viewer);

305:   /* Get Vertices used to define the BDDC */
306:   PCBDDCGraphGetCandidatesIS(pcbddc->mat_graph,NULL,NULL,NULL,NULL,&isvert);
307:   ISGetLocalSize(isvert,&n_vertices);
308:   ISGetIndices(isvert,&vertex_indices);

310:   /******************************************************************/
311:   /* TEST A/B: Test numbering of global fetidp dofs                 */
312:   /******************************************************************/
313:   MatCreateVecs(F,&fetidp_global,NULL);
314:   VecDuplicate(fetidpmat_ctx->lambda_local,&test_vec);
315:   VecSet(fetidp_global,1.0);
316:   VecSet(test_vec,1.);
317:   VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
318:   VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
319:   if (fetidpmat_ctx->l2g_p) {
320:     VecDuplicate(fetidpmat_ctx->vP,&test_vec_p);
321:     VecSet(test_vec_p,1.);
322:     VecScatterBegin(fetidpmat_ctx->l2g_p,fetidp_global,fetidpmat_ctx->vP,INSERT_VALUES,SCATTER_REVERSE);
323:     VecScatterEnd(fetidpmat_ctx->l2g_p,fetidp_global,fetidpmat_ctx->vP,INSERT_VALUES,SCATTER_REVERSE);
324:   }
325:   VecAXPY(test_vec,-1.0,fetidpmat_ctx->lambda_local);
326:   VecNorm(test_vec,NORM_INFINITY,&val);
327:   VecDestroy(&test_vec);
328:   MPI_Reduce(&val,&rval,1,MPIU_REAL,MPI_MAX,0,comm);
329:   PetscViewerASCIIPrintf(viewer,"A: CHECK glob to loc: % 1.14e\n",rval);

331:   if (fetidpmat_ctx->l2g_p) {
332:     VecAXPY(test_vec_p,-1.0,fetidpmat_ctx->vP);
333:     VecNorm(test_vec_p,NORM_INFINITY,&val);
334:     MPI_Reduce(&val,&rval,1,MPIU_REAL,MPI_MAX,0,comm);
335:     PetscViewerASCIIPrintf(viewer,"A: CHECK glob to loc (p): % 1.14e\n",rval);
336:   }

338:   if (fetidp->fully_redundant) {
339:     VecSet(fetidp_global,0.0);
340:     VecSet(fetidpmat_ctx->lambda_local,0.5);
341:     VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
342:     VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
343:     VecSum(fetidp_global,&sval);
344:     val  = PetscRealPart(sval)-fetidpmat_ctx->n_lambda;
345:     MPI_Reduce(&val,&rval,1,MPIU_REAL,MPI_MAX,0,comm);
346:     PetscViewerASCIIPrintf(viewer,"B: CHECK loc to glob: % 1.14e\n",rval);
347:   }

349:   if (fetidpmat_ctx->l2g_p) {
350:     VecSet(pcis->vec1_N,1.0);
351:     VecSet(pcis->vec1_global,0.0);
352:     VecScatterBegin(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);
353:     VecScatterEnd(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);

355:     VecSet(fetidp_global,0.0);
356:     VecSet(fetidpmat_ctx->vP,-1.0);
357:     VecScatterBegin(fetidpmat_ctx->l2g_p,fetidpmat_ctx->vP,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
358:     VecScatterEnd(fetidpmat_ctx->l2g_p,fetidpmat_ctx->vP,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
359:     VecScatterBegin(fetidpmat_ctx->g2g_p,fetidp_global,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);
360:     VecScatterEnd(fetidpmat_ctx->g2g_p,fetidp_global,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);
361:     VecScatterBegin(fetidpmat_ctx->g2g_p,pcis->vec1_global,fetidp_global,INSERT_VALUES,SCATTER_FORWARD);
362:     VecScatterEnd(fetidpmat_ctx->g2g_p,pcis->vec1_global,fetidp_global,INSERT_VALUES,SCATTER_FORWARD);
363:     VecSum(fetidp_global,&sval);
364:     val  = PetscRealPart(sval);
365:     MPI_Reduce(&val,&rval,1,MPIU_REAL,MPI_MAX,0,comm);
366:     PetscViewerASCIIPrintf(viewer,"B: CHECK loc to glob (p): % 1.14e\n",rval);
367:   }

369:   /******************************************************************/
370:   /* TEST C: It should hold B_delta*w=0, w\in\widehat{W}            */
371:   /* This is the meaning of the B matrix                            */
372:   /******************************************************************/

374:   VecSetRandom(pcis->vec1_N,NULL);
375:   VecSet(pcis->vec1_global,0.0);
376:   VecScatterBegin(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);
377:   VecScatterEnd(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);
378:   VecScatterBegin(matis->rctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);
379:   VecScatterEnd(matis->rctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);
380:   VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);
381:   VecScatterEnd(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);
382:   /* Action of B_delta */
383:   MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);
384:   VecSet(fetidp_global,0.0);
385:   VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
386:   VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
387:   VecNorm(fetidp_global,NORM_INFINITY,&val);
388:   PetscViewerASCIIPrintf(viewer,"C: CHECK infty norm of B_delta*w (w continuous): % 1.14e\n",val);

390:   /******************************************************************/
391:   /* TEST D: It should hold E_Dw = w - P_Dw w\in\widetilde{W}       */
392:   /* E_D = R_D^TR                                                   */
393:   /* P_D = B_{D,delta}^T B_{delta}                                  */
394:   /* eq.44 Mandel Tezaur and Dohrmann 2005                          */
395:   /******************************************************************/

397:   /* compute a random vector in \widetilde{W} */
398:   VecSetRandom(pcis->vec1_N,NULL);
399:   /* set zero at vertices and essential dofs */
400:   VecGetArray(pcis->vec1_N,&array);
401:   for (i=0;i<n_vertices;i++) array[vertex_indices[i]] = 0.0;
402:   PCBDDCGraphGetDirichletDofs(pcbddc->mat_graph,&dirdofs);
403:   if (dirdofs) {
404:     const PetscInt *idxs;
405:     PetscInt       ndir;

407:     ISGetLocalSize(dirdofs,&ndir);
408:     ISGetIndices(dirdofs,&idxs);
409:     for (i=0;i<ndir;i++) array[idxs[i]] = 0.0;
410:     ISRestoreIndices(dirdofs,&idxs);
411:   }
412:   VecRestoreArray(pcis->vec1_N,&array);
413:   /* store w for final comparison */
414:   VecDuplicate(pcis->vec1_B,&test_vec);
415:   VecScatterBegin(pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);
416:   VecScatterEnd(pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);

418:   /* Jump operator P_D : results stored in pcis->vec1_B */
419:   /* Action of B_delta */
420:   MatMult(fetidpmat_ctx->B_delta,test_vec,fetidpmat_ctx->lambda_local);
421:   VecSet(fetidp_global,0.0);
422:   VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
423:   VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
424:   /* Action of B_Ddelta^T */
425:   VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
426:   VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
427:   MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);

429:   /* Average operator E_D : results stored in pcis->vec2_B */
430:   PCBDDCScalingExtension(fetidpmat_ctx->pc,test_vec,pcis->vec1_global);
431:   VecScatterBegin(pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);
432:   VecScatterEnd(pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);

434:   /* test E_D=I-P_D */
435:   VecAXPY(pcis->vec1_B,1.0,pcis->vec2_B);
436:   VecAXPY(pcis->vec1_B,-1.0,test_vec);
437:   VecNorm(pcis->vec1_B,NORM_INFINITY,&val);
438:   VecDestroy(&test_vec);
439:   MPI_Reduce(&val,&rval,1,MPIU_REAL,MPI_MAX,0,comm);
440:   PetscViewerASCIIPrintf(viewer,"D: CHECK infty norm of E_D + P_D - I: % 1.14e\n",PetscGlobalRank,val);

442:   /******************************************************************/
443:   /* TEST E: It should hold R_D^TP_Dw=0 w\in\widetilde{W}           */
444:   /* eq.48 Mandel Tezaur and Dohrmann 2005                          */
445:   /******************************************************************/

447:   VecSetRandom(pcis->vec1_N,NULL);
448:   /* set zero at vertices and essential dofs */
449:   VecGetArray(pcis->vec1_N,&array);
450:   for (i=0;i<n_vertices;i++) array[vertex_indices[i]] = 0.0;
451:   if (dirdofs) {
452:     const PetscInt *idxs;
453:     PetscInt       ndir;

455:     ISGetLocalSize(dirdofs,&ndir);
456:     ISGetIndices(dirdofs,&idxs);
457:     for (i=0;i<ndir;i++) array[idxs[i]] = 0.0;
458:     ISRestoreIndices(dirdofs,&idxs);
459:   }
460:   VecRestoreArray(pcis->vec1_N,&array);

462:   /* Jump operator P_D : results stored in pcis->vec1_B */

464:   VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);
465:   VecScatterEnd(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);
466:   /* Action of B_delta */
467:   MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);
468:   VecSet(fetidp_global,0.0);
469:   VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
470:   VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,fetidp_global,ADD_VALUES,SCATTER_FORWARD);
471:   /* Action of B_Ddelta^T */
472:   VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
473:   VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
474:   MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);
475:   /* scaling */
476:   PCBDDCScalingExtension(fetidpmat_ctx->pc,pcis->vec1_B,pcis->vec1_global);
477:   VecNorm(pcis->vec1_global,NORM_INFINITY,&val);
478:   PetscViewerASCIIPrintf(viewer,"E: CHECK infty norm of R^T_D P_D: % 1.14e\n",val);

480:   if (!fetidp->fully_redundant) {
481:     /******************************************************************/
482:     /* TEST F: It should holds B_{delta}B^T_{D,delta}=I               */
483:     /* Corollary thm 14 Mandel Tezaur and Dohrmann 2005               */
484:     /******************************************************************/
485:     VecDuplicate(fetidp_global,&test_vec);
486:     VecSetRandom(fetidp_global,NULL);
487:     if (fetidpmat_ctx->l2g_p) {
488:       VecSet(fetidpmat_ctx->vP,0.);
489:       VecScatterBegin(fetidpmat_ctx->l2g_p,fetidpmat_ctx->vP,fetidp_global,INSERT_VALUES,SCATTER_FORWARD);
490:       VecScatterEnd(fetidpmat_ctx->l2g_p,fetidpmat_ctx->vP,fetidp_global,INSERT_VALUES,SCATTER_FORWARD);
491:     }
492:     /* Action of B_Ddelta^T */
493:     VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
494:     VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidp_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);
495:     MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);
496:     /* Action of B_delta */
497:     MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);
498:     VecSet(test_vec,0.0);
499:     VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);
500:     VecScatterEnd(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);
501:     VecAXPY(fetidp_global,-1.,test_vec);
502:     VecNorm(fetidp_global,NORM_INFINITY,&val);
503:     PetscViewerASCIIPrintf(viewer,"E: CHECK infty norm of P^T_D - I: % 1.14e\n",val);
504:     VecDestroy(&test_vec);
505:   }
506:   PetscViewerASCIIPrintf(viewer,"-------------------------------------\n");
507:   PetscViewerFlush(viewer);
508:   VecDestroy(&test_vec_p);
509:   ISDestroy(&dirdofs);
510:   VecDestroy(&fetidp_global);
511:   ISRestoreIndices(isvert,&vertex_indices);
512:   PCBDDCGraphRestoreCandidatesIS(pcbddc->mat_graph,NULL,NULL,NULL,NULL,&isvert);
513:   return(0);
514: }

516: static PetscErrorCode KSPFETIDPSetUpOperators(KSP ksp)
517: {
518:   KSP_FETIDP       *fetidp = (KSP_FETIDP*)ksp->data;
519:   PC_BDDC          *pcbddc = (PC_BDDC*)fetidp->innerbddc->data;
520:   Mat              A,Ap;
521:   PetscInt         fid = -1;
522:   PetscBool        ismatis,pisz,allp;
523:   PetscBool        flip; /* Usually, Stokes is written (B = -\int_\Omega \nabla \cdot u q)
524:                            | A B'| | v | = | f |
525:                            | B 0 | | p | = | g |
526:                             If -ksp_fetidp_saddlepoint_flip is true, the code assumes it is written as
527:                            | A B'| | v | = | f |
528:                            |-B 0 | | p | = |-g |
529:                          */
530:   PetscObjectState matstate, matnnzstate;
531:   PetscErrorCode   ierr;

534:   pisz = PETSC_TRUE;
535:   flip = PETSC_FALSE;
536:   allp = PETSC_FALSE;
537:   PetscOptionsBegin(PetscObjectComm((PetscObject)ksp),((PetscObject)ksp)->prefix,"FETI-DP options","PC");
538:   PetscOptionsInt("-ksp_fetidp_pressure_field","Field id for pressures for saddle-point problems",NULL,fid,&fid,NULL);
539:   PetscOptionsBool("-ksp_fetidp_pressure_iszero","Zero pressure block",NULL,pisz,&pisz,NULL);
540:   PetscOptionsBool("-ksp_fetidp_pressure_all","Use the whole pressure set instead of just that at the interface",NULL,allp,&allp,NULL);
541:   PetscOptionsBool("-ksp_fetidp_saddlepoint_flip","Flip the sign of the pressure-velocity (lower-left) block",NULL,flip,&flip,NULL);
542:   PetscOptionsEnd();

544:   fetidp->saddlepoint = (fid >= 0 ? PETSC_TRUE : fetidp->saddlepoint);

546:   KSPGetOperators(ksp,&A,&Ap);
547:   PetscObjectTypeCompare((PetscObject)A,MATIS,&ismatis);
548:   if (!ismatis) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_USER,"Amat should be of type MATIS");

550:   /* Quiet return if the matrix states are unchanged.
551:      Needed only for the saddle point case since it uses MatZeroRows
552:      on a matrix that may not have changed */
553:   PetscObjectStateGet((PetscObject)A,&matstate);
554:   MatGetNonzeroState(A,&matnnzstate);
555:   if (matstate == fetidp->matstate && matnnzstate == fetidp->matnnzstate) return(0);
556:   fetidp->matstate     = matstate;
557:   fetidp->matnnzstate  = matnnzstate;
558:   fetidp->statechanged = fetidp->saddlepoint;

560:   /* see if MATIS has same fields attached */
561:   if (!pcbddc->n_ISForDofsLocal && !pcbddc->n_ISForDofs) {
562:     PetscContainer c;

564:     PetscObjectQuery((PetscObject)A,"_convert_nest_lfields",(PetscObject*)&c);
565:     if (c) {
566:       MatISLocalFields lf;
567:       PetscContainerGetPointer(c,(void**)&lf);
568:       PCBDDCSetDofsSplittingLocal(fetidp->innerbddc,lf->nr,lf->rf);
569:     }
570:   }

572:   if (!fetidp->saddlepoint) {
573:     PCSetOperators(fetidp->innerbddc,A,A);
574:   } else {
575:     Mat      nA,lA;
576:     Mat      PPmat;
577:     IS       pP;
578:     PetscInt totP;

580:     MatISGetLocalMat(A,&lA);
581:     PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_lA",(PetscObject)lA);

583:     pP = fetidp->pP;
584:     if (!pP) { /* first time, need to compute pressure dofs */
585:       PC_IS                  *pcis = (PC_IS*)fetidp->innerbddc->data;
586:       Mat_IS                 *matis = (Mat_IS*)(A->data);
587:       ISLocalToGlobalMapping l2g;
588:       IS                     lP = NULL,II,pII,lPall,Pall,is1,is2;
589:       const PetscInt         *idxs;
590:       PetscInt               nl,ni,*widxs;
591:       PetscInt               i,j,n_neigh,*neigh,*n_shared,**shared,*count;
592:       PetscInt               rst,ren,n;
593:       PetscBool              ploc;

595:       MatGetLocalSize(A,&nl,NULL);
596:       MatGetOwnershipRange(A,&rst,&ren);
597:       MatGetLocalSize(lA,&n,NULL);
598:       MatGetLocalToGlobalMapping(A,&l2g,NULL);

600:       if (!pcis->is_I_local) { /* need to compute interior dofs */
601:         PetscCalloc1(n,&count);
602:         ISLocalToGlobalMappingGetInfo(l2g,&n_neigh,&neigh,&n_shared,&shared);
603:         for (i=1;i<n_neigh;i++)
604:           for (j=0;j<n_shared[i];j++)
605:             count[shared[i][j]] += 1;
606:         for (i=0,j=0;i<n;i++) if (!count[i]) count[j++] = i;
607:         ISLocalToGlobalMappingRestoreInfo(l2g,&n_neigh,&neigh,&n_shared,&shared);
608:         ISCreateGeneral(PETSC_COMM_SELF,j,count,PETSC_OWN_POINTER,&II);
609:       } else {
610:         PetscObjectReference((PetscObject)pcis->is_I_local);
611:         II   = pcis->is_I_local;
612:       }

614:       /* interior dofs in layout */
615:       MatISSetUpSF(A);
616:       PetscMemzero(matis->sf_leafdata,n*sizeof(PetscInt));
617:       PetscMemzero(matis->sf_rootdata,nl*sizeof(PetscInt));
618:       ISGetLocalSize(II,&ni);
619:       ISGetIndices(II,&idxs);
620:       for (i=0;i<ni;i++) matis->sf_leafdata[idxs[i]] = 1;
621:       ISRestoreIndices(II,&idxs);
622:       PetscSFReduceBegin(matis->sf,MPIU_INT,matis->sf_leafdata,matis->sf_rootdata,MPIU_REPLACE);
623:       PetscSFReduceEnd(matis->sf,MPIU_INT,matis->sf_leafdata,matis->sf_rootdata,MPIU_REPLACE);
624:       PetscMalloc1(PetscMax(nl,n),&widxs);
625:       for (i=0,ni=0;i<nl;i++) if (matis->sf_rootdata[i]) widxs[ni++] = i+rst;
626:       ISCreateGeneral(PetscObjectComm((PetscObject)ksp),ni,widxs,PETSC_COPY_VALUES,&pII);

628:       /* pressure dofs */
629:       Pall  = NULL;
630:       lPall = NULL;
631:       ploc  = PETSC_FALSE;
632:       if (fid >= 0) {
633:         if (pcbddc->n_ISForDofsLocal) {
634:           PetscInt np;

636:           if (fid >= pcbddc->n_ISForDofsLocal) SETERRQ2(PetscObjectComm((PetscObject)ksp),PETSC_ERR_USER,"Invalid field id for pressure %D, max %D",fid,pcbddc->n_ISForDofsLocal);
637:           /* need a sequential IS */
638:           ISGetLocalSize(pcbddc->ISForDofsLocal[fid],&np);
639:           ISGetIndices(pcbddc->ISForDofsLocal[fid],&idxs);
640:           ISCreateGeneral(PETSC_COMM_SELF,np,idxs,PETSC_COPY_VALUES,&lPall);
641:           ISRestoreIndices(pcbddc->ISForDofsLocal[fid],&idxs);
642:           ploc = PETSC_TRUE;
643:         } else if (pcbddc->n_ISForDofs) {
644:           if (fid >= pcbddc->n_ISForDofs) SETERRQ2(PetscObjectComm((PetscObject)ksp),PETSC_ERR_USER,"Invalid field id for pressure %D, max %D",fid,pcbddc->n_ISForDofs);
645:           PetscObjectReference((PetscObject)pcbddc->ISForDofs[fid]);
646:           Pall = pcbddc->ISForDofs[fid];
647:         } else SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_USER,"Missing fields! Use PCBDDCSetDofsSplitting/Local");
648:       } else { /* fallback to zero pressure block */
649:         IS list[2];

651:         MatFindZeroDiagonals(A,&list[1]);
652:         ISComplement(list[1],rst,ren,&list[0]);
653:         PCBDDCSetDofsSplitting(fetidp->innerbddc,2,list);
654:         ISDestroy(&list[0]);
655:         Pall = list[1];
656:       }
657:       /* if the user requested the entire pressure,
658:          remove the interior pressure dofs from II (or pII) */
659:       if (allp) {
660:         if (ploc) {
661:           IS nII;
662:           ISDifference(II,lPall,&nII);
663:           ISDestroy(&II);
664:           II   = nII;
665:         } else {
666:           IS nII;
667:           ISDifference(pII,Pall,&nII);
668:           ISDestroy(&pII);
669:           pII  = nII;
670:         }
671:       }
672:       if (ploc) {
673:         ISDifference(lPall,II,&lP);
674:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_lP",(PetscObject)lP);
675:       } else {
676:         ISDifference(Pall,pII,&pP);
677:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_pP",(PetscObject)pP);
678:         /* need all local pressure dofs */
679:         PetscMemzero(matis->sf_leafdata,n*sizeof(PetscInt));
680:         PetscMemzero(matis->sf_rootdata,nl*sizeof(PetscInt));
681:         ISGetLocalSize(Pall,&ni);
682:         ISGetIndices(Pall,&idxs);
683:         for (i=0;i<ni;i++) matis->sf_rootdata[idxs[i]-rst] = 1;
684:         ISRestoreIndices(Pall,&idxs);
685:         PetscSFBcastBegin(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);
686:         PetscSFBcastEnd(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);
687:         for (i=0,ni=0;i<n;i++) if (matis->sf_leafdata[i]) widxs[ni++] = i;
688:         ISCreateGeneral(PETSC_COMM_SELF,ni,widxs,PETSC_COPY_VALUES,&lPall);
689:       }

691:       if (!Pall) {
692:         PetscMemzero(matis->sf_leafdata,n*sizeof(PetscInt));
693:         PetscMemzero(matis->sf_rootdata,nl*sizeof(PetscInt));
694:         ISGetLocalSize(lPall,&ni);
695:         ISGetIndices(lPall,&idxs);
696:         for (i=0;i<ni;i++) matis->sf_leafdata[idxs[i]] = 1;
697:         ISRestoreIndices(lPall,&idxs);
698:         PetscSFReduceBegin(matis->sf,MPIU_INT,matis->sf_leafdata,matis->sf_rootdata,MPIU_REPLACE);
699:         PetscSFReduceEnd(matis->sf,MPIU_INT,matis->sf_leafdata,matis->sf_rootdata,MPIU_REPLACE);
700:         for (i=0,ni=0;i<nl;i++) if (matis->sf_rootdata[i]) widxs[ni++] = i+rst;
701:         ISCreateGeneral(PetscObjectComm((PetscObject)ksp),ni,widxs,PETSC_COPY_VALUES,&Pall);
702:       }
703:       PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_aP",(PetscObject)Pall);

705:       if (flip) {
706:         PetscInt npl;
707:         ISGetLocalSize(Pall,&npl);
708:         ISGetIndices(Pall,&idxs);
709:         MatCreateVecs(A,NULL,&fetidp->rhs_flip);
710:         VecSet(fetidp->rhs_flip,1.);
711:         VecSetOption(fetidp->rhs_flip,VEC_IGNORE_OFF_PROC_ENTRIES,PETSC_TRUE);
712:         for (i=0;i<npl;i++) {
713:           VecSetValue(fetidp->rhs_flip,idxs[i],-1.,INSERT_VALUES);
714:         }
715:         VecAssemblyBegin(fetidp->rhs_flip);
716:         VecAssemblyEnd(fetidp->rhs_flip);
717:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_flip",(PetscObject)fetidp->rhs_flip);
718:         ISRestoreIndices(Pall,&idxs);
719:       }
720:       ISDestroy(&Pall);
721:       ISDestroy(&pII);

723:       /* local selected pressures in subdomain-wise and global ordering */
724:       PetscMemzero(matis->sf_leafdata,n*sizeof(PetscInt));
725:       PetscMemzero(matis->sf_rootdata,nl*sizeof(PetscInt));
726:       if (!ploc) {
727:         PetscInt *widxs2;

729:         if (!pP) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_PLIB,"Missing parallel pressure IS");
730:         ISGetLocalSize(pP,&ni);
731:         ISGetIndices(pP,&idxs);
732:         for (i=0;i<ni;i++) matis->sf_rootdata[idxs[i]-rst] = 1;
733:         ISRestoreIndices(pP,&idxs);
734:         PetscSFBcastBegin(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);
735:         PetscSFBcastEnd(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);
736:         for (i=0,ni=0;i<n;i++) if (matis->sf_leafdata[i]) widxs[ni++] = i;
737:         PetscMalloc1(ni,&widxs2);
738:         ISLocalToGlobalMappingApply(l2g,ni,widxs,widxs2);
739:         ISCreateGeneral(PETSC_COMM_SELF,ni,widxs,PETSC_COPY_VALUES,&lP);
740:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_lP",(PetscObject)lP);
741:         ISCreateGeneral(PetscObjectComm((PetscObject)ksp),ni,widxs2,PETSC_OWN_POINTER,&is1);
742:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_gP",(PetscObject)is1);
743:         ISDestroy(&is1);
744:       } else {
745:         if (!lP) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Missing sequential pressure IS");
746:         ISGetLocalSize(lP,&ni);
747:         ISGetIndices(lP,&idxs);
748:         for (i=0;i<ni;i++)
749:           if (idxs[i] >=0 && idxs[i] < n)
750:             matis->sf_leafdata[idxs[i]] = 1;
751:         ISRestoreIndices(lP,&idxs);
752:         PetscSFReduceBegin(matis->sf,MPIU_INT,matis->sf_leafdata,matis->sf_rootdata,MPIU_REPLACE);
753:         ISLocalToGlobalMappingApply(l2g,ni,idxs,widxs);
754:         ISCreateGeneral(PetscObjectComm((PetscObject)ksp),ni,widxs,PETSC_COPY_VALUES,&is1);
755:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_gP",(PetscObject)is1);
756:         ISDestroy(&is1);
757:         PetscSFReduceEnd(matis->sf,MPIU_INT,matis->sf_leafdata,matis->sf_rootdata,MPIU_REPLACE);
758:         for (i=0,ni=0;i<nl;i++) if (matis->sf_rootdata[i]) widxs[ni++] = i+rst;
759:         ISCreateGeneral(PetscObjectComm((PetscObject)ksp),ni,widxs,PETSC_COPY_VALUES,&pP);
760:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_pP",(PetscObject)pP);
761:       }
762:       PetscFree(widxs);

764:       /* If there's any "interior pressure",
765:          we may want to use a discrete harmonic solver instead
766:          of a Stokes harmonic for the Dirichlet preconditioner
767:          Need to extract the interior velocity dofs in interior dofs ordering (iV)
768:          and interior pressure dofs in local ordering (iP) */
769:       if (!allp) {
770:         ISLocalToGlobalMapping l2g_t;

772:         ISDifference(lPall,lP,&is1);
773:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_iP",(PetscObject)is1);
774:         ISDifference(II,is1,&is2);
775:         ISDestroy(&is1);
776:         ISLocalToGlobalMappingCreateIS(II,&l2g_t);
777:         ISGlobalToLocalMappingApplyIS(l2g_t,IS_GTOLM_DROP,is2,&is1);
778:         ISGetLocalSize(is1,&i);
779:         ISGetLocalSize(is2,&j);
780:         if (i != j) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Inconsistent local sizes %D and %D for iV",i,j);
781:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_iV",(PetscObject)is1);
782:         ISLocalToGlobalMappingDestroy(&l2g_t);
783:         ISDestroy(&is1);
784:         ISDestroy(&is2);
785:       }
786:       ISDestroy(&lPall);
787:       ISDestroy(&II);

789:       /* exclude selected pressures from the inner BDDC */
790:       if (pcbddc->DirichletBoundariesLocal) {
791:         IS       list[2],plP,isout;
792:         PetscInt np;

794:         /* need a parallel IS */
795:         ISGetLocalSize(lP,&np);
796:         ISGetIndices(lP,&idxs);
797:         ISCreateGeneral(PetscObjectComm((PetscObject)ksp),np,idxs,PETSC_USE_POINTER,&plP);
798:         list[0] = plP;
799:         list[1] = pcbddc->DirichletBoundariesLocal;
800:         ISConcatenate(PetscObjectComm((PetscObject)ksp),2,list,&isout);
801:         ISSortRemoveDups(isout);
802:         ISDestroy(&plP);
803:         ISRestoreIndices(lP,&idxs);
804:         PCBDDCSetDirichletBoundariesLocal(fetidp->innerbddc,isout);
805:         ISDestroy(&isout);
806:       } else if (pcbddc->DirichletBoundaries) {
807:         IS list[2],isout;

809:         list[0] = pP;
810:         list[1] = pcbddc->DirichletBoundaries;
811:         ISConcatenate(PetscObjectComm((PetscObject)ksp),2,list,&isout);
812:         ISSortRemoveDups(isout);
813:         PCBDDCSetDirichletBoundaries(fetidp->innerbddc,isout);
814:         ISDestroy(&isout);
815:       } else {
816:         IS       plP;
817:         PetscInt np;

819:         /* need a parallel IS */
820:         ISGetLocalSize(lP,&np);
821:         ISGetIndices(lP,&idxs);
822:         ISCreateGeneral(PetscObjectComm((PetscObject)ksp),np,idxs,PETSC_COPY_VALUES,&plP);
823:         PCBDDCSetDirichletBoundariesLocal(fetidp->innerbddc,plP);
824:         ISDestroy(&plP);
825:         ISRestoreIndices(lP,&idxs);
826:       }
827:       ISDestroy(&lP);
828:       fetidp->pP = pP;
829:     }

831:     /* total number of selected pressure dofs */
832:     ISGetSize(fetidp->pP,&totP);

834:     /* Set operator for inner BDDC */
835:     if (totP || fetidp->rhs_flip) {
836:       MatDuplicate(A,MAT_COPY_VALUES,&nA);
837:     } else {
838:       PetscObjectReference((PetscObject)A);
839:       nA   = A;
840:     }
841:     if (fetidp->rhs_flip) {
842:       MatDiagonalScale(nA,fetidp->rhs_flip,NULL);
843:       if (totP) {
844:         Mat lA2;

846:         MatISGetLocalMat(nA,&lA);
847:         MatDuplicate(lA,MAT_COPY_VALUES,&lA2);
848:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_lA",(PetscObject)lA2);
849:         MatDestroy(&lA2);
850:       }
851:     }

853:     if (totP) {
854:       MatSetOption(nA,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_FALSE);
855:       MatZeroRowsColumnsIS(nA,fetidp->pP,1.,NULL,NULL);
856:     } else {
857:       PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_lA",NULL);
858:     }
859:     PCSetOperators(fetidp->innerbddc,nA,nA);
860:     MatDestroy(&nA);

862:     /* non-zero rhs on interior dofs when applying the preconditioner */
863:     if (totP) pcbddc->switch_static = PETSC_TRUE;

865:     /* if there are no pressures, set inner bddc flag for benign saddle point */
866:     if (!totP) {
867:       pcbddc->benign_saddle_point = PETSC_TRUE;
868:       pcbddc->compute_nonetflux   = PETSC_TRUE;
869:     }

871:     /* Divergence mat */
872:     if (totP) {
873:       Mat       B;
874:       IS        P;
875:       PetscBool save;

877:       PetscObjectQuery((PetscObject)fetidp->innerbddc,"__KSPFETIDP_aP",(PetscObject*)&P);
878:       MatCreateSubMatrix(A,P,NULL,MAT_INITIAL_MATRIX,&B);
879:       save = pcbddc->compute_nonetflux; /* SetDivergenceMat activates nonetflux computation */
880:       PCBDDCSetDivergenceMat(fetidp->innerbddc,B,PETSC_FALSE,NULL);
881:       pcbddc->compute_nonetflux = save;
882:       MatDestroy(&B);
883:     }

885:     /* Operators for pressure preconditioner */
886:     if (totP) {

888:       /* Extract pressure block */
889:       if (!pisz) {
890:         Mat C;

892:         MatCreateSubMatrix(A,fetidp->pP,fetidp->pP,MAT_INITIAL_MATRIX,&C);
893:         MatScale(C,-1.);
894:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_C",(PetscObject)C);
895:         MatDestroy(&C);
896:       } else if (A != Ap) { /* user has provided a different Pmat, use it to extract the pressure preconditioner */
897:         Mat C;

899:         MatCreateSubMatrix(Ap,fetidp->pP,fetidp->pP,MAT_INITIAL_MATRIX,&C);
900:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject)C);
901:         MatDestroy(&C);
902:       }
903:       PetscObjectQuery((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject*)&PPmat);

905:       /* Preconditioned operator for the pressure block */
906:       if (PPmat) {
907:         Mat       C;
908:         IS        Pall;
909:         PetscInt  AM,PAM,PAN,pam,pan,am,an,pl,pIl,pAg,pIg;
910:         PetscBool ismatis;

912:         PetscObjectTypeCompare((PetscObject)PPmat,MATIS,&ismatis);
913:         if (ismatis) {
914:           MatISGetMPIXAIJ(PPmat,MAT_INITIAL_MATRIX,&C);
915:           PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject)C);
916:           MatDestroy(&C);
917:           PetscObjectQuery((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject*)&PPmat);
918:         }
919:         PetscObjectQuery((PetscObject)fetidp->innerbddc,"__KSPFETIDP_aP",(PetscObject*)&Pall);
920:         MatGetSize(A,&AM,NULL);
921:         MatGetSize(PPmat,&PAM,&PAN);
922:         ISGetSize(Pall,&pAg);
923:         ISGetSize(fetidp->pP,&pIg);
924:         MatGetLocalSize(PPmat,&pam,&pan);
925:         MatGetLocalSize(A,&am,&an);
926:         ISGetLocalSize(Pall,&pIl);
927:         ISGetLocalSize(fetidp->pP,&pl);
928:         if (PAM != PAN) SETERRQ2(PetscObjectComm((PetscObject)ksp),PETSC_ERR_USER,"Pressure matrix must be square, unsupported %D x %D",PAM,PAN);
929:         if (pam != pan) SETERRQ2(PetscObjectComm((PetscObject)ksp),PETSC_ERR_USER,"Local sizes of pressure matrix must be equal, unsupported %D x %D",pam,pan);
930:         if (pam != am && pam != pl && pam != pIl) SETERRQ4(PETSC_COMM_SELF,PETSC_ERR_USER,"Invalid number of local rows %D for pressure matrix! Supported are %D, %D or %D",pam,am,pl,pIl);
931:         if (pan != an && pan != pl && pan != pIl) SETERRQ4(PETSC_COMM_SELF,PETSC_ERR_USER,"Invalid number of local columns %D for pressure matrix! Supported are %D, %D or %D",pan,an,pl,pIl);
932:         if (PAM == AM) { /* monolithic ordering, restrict to pressure */
933:           MatCreateSubMatrix(PPmat,fetidp->pP,fetidp->pP,MAT_INITIAL_MATRIX,&C);
934:         } else if (pAg == PAM) { /* global ordering for pressure only */
935:           if (!allp) { /* solving for interface pressure only */
936:             IS restr;

938:             ISRenumber(fetidp->pP,NULL,NULL,&restr);
939:             MatCreateSubMatrix(PPmat,restr,restr,MAT_INITIAL_MATRIX,&C);
940:             ISDestroy(&restr);
941:           } else {
942:             PetscObjectReference((PetscObject)PPmat);
943:             C     = PPmat;
944:           }
945:         } else if (pIg == PAM) { /* global ordering for selected pressure only */
946:           PetscObjectReference((PetscObject)PPmat);
947:           C     = PPmat;
948:         } else {
949:           SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_USER,"Unable to use the pressure matrix");
950:         }
951:         PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject)C);
952:         MatDestroy(&C);
953:       } else {
954:         Mat C;

956:         PetscObjectQuery((PetscObject)fetidp->innerbddc,"__KSPFETIDP_C",(PetscObject*)&C);
957:         if (C) { /* non-zero pressure block, most likely Almost Incompressible Elasticity */
958:           PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject)C);
959:         } else { /* identity (need to be scaled properly by the user using e.g. a Richardson method */
960:           PetscInt nl;

962:           ISGetLocalSize(fetidp->pP,&nl);
963:           MatCreate(PetscObjectComm((PetscObject)ksp),&PPmat);
964:           MatSetSizes(PPmat,nl,nl,totP,totP);
965:           MatSetType(PPmat,MATAIJ);
966:           MatMPIAIJSetPreallocation(PPmat,1,NULL,0,NULL);
967:           MatSeqAIJSetPreallocation(PPmat,1,NULL);
968:           MatAssemblyBegin(PPmat,MAT_FINAL_ASSEMBLY);
969:           MatAssemblyEnd(PPmat,MAT_FINAL_ASSEMBLY);
970:           MatShift(PPmat,1.);
971:           PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_PPmat",(PetscObject)PPmat);
972:           MatDestroy(&PPmat);
973:         }
974:       }
975:     } else { /* totP == 0 */
976:       PetscObjectCompose((PetscObject)fetidp->innerbddc,"__KSPFETIDP_pP",NULL);
977:     }
978:   }
979:   return(0);
980: }

982: static PetscErrorCode KSPSetUp_FETIDP(KSP ksp)
983: {
984:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;
985:   PC_BDDC        *pcbddc = (PC_BDDC*)fetidp->innerbddc->data;
986:   PetscBool      flg;

990:   KSPFETIDPSetUpOperators(ksp);
991:   /* set up BDDC */
992:   PCSetErrorIfFailure(fetidp->innerbddc,ksp->errorifnotconverged);
993:   PCSetUp(fetidp->innerbddc);
994:   /* FETI-DP as it is implemented needs an exact coarse solver */
995:   if (pcbddc->coarse_ksp) {
996:     KSPSetTolerances(pcbddc->coarse_ksp,PETSC_SMALL,PETSC_SMALL,PETSC_DEFAULT,1000);
997:     KSPSetNormType(pcbddc->coarse_ksp,KSP_NORM_DEFAULT);
998:   }
999:   /* FETI-DP as it is implemented needs exact local Neumann solvers */
1000:   KSPSetTolerances(pcbddc->ksp_R,PETSC_SMALL,PETSC_SMALL,PETSC_DEFAULT,1000);
1001:   KSPSetNormType(pcbddc->ksp_R,KSP_NORM_DEFAULT);

1003:   /* setup FETI-DP operators
1004:      If fetidp->statechanged is true, we need update the operators
1005:      that are needed in the saddle-point case. This should be replaced
1006:      by a better logic when the FETI-DP matrix and preconditioner will
1007:      have their own classes */
1008:   if (pcbddc->new_primal_space || fetidp->statechanged) {
1009:     Mat F; /* the FETI-DP matrix */
1010:     PC  D; /* the FETI-DP preconditioner */
1011:     KSPReset(fetidp->innerksp);
1012:     PCBDDCCreateFETIDPOperators(fetidp->innerbddc,fetidp->fully_redundant,((PetscObject)ksp)->prefix,&F,&D);
1013:     KSPSetOperators(fetidp->innerksp,F,F);
1014:     KSPSetTolerances(fetidp->innerksp,ksp->rtol,ksp->abstol,ksp->divtol,ksp->max_it);
1015:     KSPSetPC(fetidp->innerksp,D);
1016:     KSPSetFromOptions(fetidp->innerksp);
1017:     MatCreateVecs(F,&(fetidp->innerksp)->vec_rhs,&(fetidp->innerksp)->vec_sol);
1018:     MatDestroy(&F);
1019:     PCDestroy(&D);
1020:     if (fetidp->check) {
1021:       PetscViewer viewer;

1023:       if (!pcbddc->dbg_viewer) {
1024:         viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)ksp));
1025:       } else {
1026:         viewer = pcbddc->dbg_viewer;
1027:       }
1028:       KSPFETIDPCheckOperators(ksp,viewer);
1029:     }
1030:   }
1031:   fetidp->statechanged     = PETSC_FALSE;
1032:   pcbddc->new_primal_space = PETSC_FALSE;

1034:   /* propagate settings to the inner solve */
1035:   KSPGetComputeSingularValues(ksp,&flg);
1036:   KSPSetComputeSingularValues(fetidp->innerksp,flg);
1037:   if (ksp->res_hist) {
1038:     KSPSetResidualHistory(fetidp->innerksp,ksp->res_hist,ksp->res_hist_max,ksp->res_hist_reset);
1039:   }
1040:   KSPSetErrorIfNotConverged(fetidp->innerksp,ksp->errorifnotconverged);
1041:   KSPSetUp(fetidp->innerksp);
1042:   return(0);
1043: }

1045: static PetscErrorCode KSPSolve_FETIDP(KSP ksp)
1046: {
1048:   Mat            F,A;
1049:   MatNullSpace   nsp;
1050:   Vec            X,B,Xl,Bl;
1051:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;
1052:   PC_BDDC        *pcbddc = (PC_BDDC*)fetidp->innerbddc->data;

1055:   PetscCitationsRegister(citation,&cited);
1056:   if (fetidp->saddlepoint) {
1057:     PetscCitationsRegister(citation2,&cited2);
1058:   }
1059:   KSPGetOperators(ksp,&A,NULL);
1060:   KSPGetRhs(ksp,&B);
1061:   KSPGetSolution(ksp,&X);
1062:   KSPGetOperators(fetidp->innerksp,&F,NULL);
1063:   KSPGetRhs(fetidp->innerksp,&Bl);
1064:   KSPGetSolution(fetidp->innerksp,&Xl);
1065:   PCBDDCMatFETIDPGetRHS(F,B,Bl);
1066:   if (ksp->transpose_solve) {
1067:     KSPSolveTranspose(fetidp->innerksp,Bl,Xl);
1068:   } else {
1069:     KSPSolve(fetidp->innerksp,Bl,Xl);
1070:   }
1071:   PCBDDCMatFETIDPGetSolution(F,Xl,X);
1072:   MatGetNullSpace(A,&nsp);
1073:   if (nsp) {
1074:     MatNullSpaceRemove(nsp,X);
1075:   }
1076:   /* update ksp with stats from inner ksp */
1077:   KSPGetConvergedReason(fetidp->innerksp,&ksp->reason);
1078:   KSPGetIterationNumber(fetidp->innerksp,&ksp->its);
1079:   ksp->totalits += ksp->its;
1080:   KSPGetResidualHistory(fetidp->innerksp,NULL,&ksp->res_hist_len);
1081:   /* restore defaults for inner BDDC (Pre/PostSolve flags) */
1082:   pcbddc->temp_solution_used        = PETSC_FALSE;
1083:   pcbddc->rhs_change                = PETSC_FALSE;
1084:   pcbddc->exact_dirichlet_trick_app = PETSC_FALSE;
1085:   return(0);
1086: }

1088: static PetscErrorCode KSPReset_FETIDP(KSP ksp)
1089: {
1090:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;
1091:   PC_BDDC        *pcbddc;

1095:   ISDestroy(&fetidp->pP);
1096:   VecDestroy(&fetidp->rhs_flip);
1097:   /* avoid PCReset that does not take into account ref counting */
1098:   PCDestroy(&fetidp->innerbddc);
1099:   PCCreate(PetscObjectComm((PetscObject)ksp),&fetidp->innerbddc);
1100:   PCSetType(fetidp->innerbddc,PCBDDC);
1101:   pcbddc = (PC_BDDC*)fetidp->innerbddc->data;
1102:   pcbddc->symmetric_primal = PETSC_FALSE;
1103:   PetscLogObjectParent((PetscObject)ksp,(PetscObject)fetidp->innerbddc);
1104:   KSPDestroy(&fetidp->innerksp);
1105:   fetidp->saddlepoint  = PETSC_FALSE;
1106:   fetidp->matstate     = -1;
1107:   fetidp->matnnzstate  = -1;
1108:   fetidp->statechanged = PETSC_TRUE;
1109:   return(0);
1110: }

1112: static PetscErrorCode KSPDestroy_FETIDP(KSP ksp)
1113: {
1114:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

1118:   KSPReset_FETIDP(ksp);
1119:   PCDestroy(&fetidp->innerbddc);
1120:   KSPDestroy(&fetidp->innerksp);
1121:   PetscFree(fetidp->monctx);
1122:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPSetInnerBDDC_C",NULL);
1123:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPGetInnerBDDC_C",NULL);
1124:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPGetInnerKSP_C",NULL);
1125:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPSetPressureOperator_C",NULL);
1126:   PetscFree(ksp->data);
1127:   return(0);
1128: }

1130: static PetscErrorCode KSPView_FETIDP(KSP ksp,PetscViewer viewer)
1131: {
1132:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;
1134:   PetscBool      iascii;

1137:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
1138:   if (iascii) {
1139:     PetscViewerASCIIPrintf(viewer,"  fully redundant: %D\n",fetidp->fully_redundant);
1140:     PetscViewerASCIIPrintf(viewer,"  saddle point:    %D\n",fetidp->saddlepoint);
1141:     PetscViewerASCIIPrintf(viewer,"  inner solver details\n");
1142:     PetscViewerASCIIAddTab(viewer,2);
1143:   }
1144:   KSPView(fetidp->innerksp,viewer);
1145:   if (iascii) {
1146:     PetscViewerASCIISubtractTab(viewer,2);
1147:     PetscViewerASCIIPrintf(viewer,"  BDDC solver details\n");
1148:     PetscViewerASCIIAddTab(viewer,2);
1149:   }
1150:   PCView(fetidp->innerbddc,viewer);
1151:   if (iascii) {
1152:     PetscViewerASCIISubtractTab(viewer,2);
1153:   }
1154:   return(0);
1155: }

1157: static PetscErrorCode KSPSetFromOptions_FETIDP(PetscOptionItems *PetscOptionsObject,KSP ksp)
1158: {
1159:   KSP_FETIDP     *fetidp = (KSP_FETIDP*)ksp->data;

1163:   /* set options prefixes for the inner objects, since the parent prefix will be valid at this point */
1164:   PetscObjectSetOptionsPrefix((PetscObject)fetidp->innerksp,((PetscObject)ksp)->prefix);
1165:   PetscObjectAppendOptionsPrefix((PetscObject)fetidp->innerksp,"fetidp_");
1166:   if (!fetidp->userbddc) {
1167:     PetscObjectSetOptionsPrefix((PetscObject)fetidp->innerbddc,((PetscObject)ksp)->prefix);
1168:     PetscObjectAppendOptionsPrefix((PetscObject)fetidp->innerbddc,"fetidp_bddc_");
1169:   }
1170:   PetscOptionsHead(PetscOptionsObject,"KSP FETIDP options");
1171:   PetscOptionsBool("-ksp_fetidp_fullyredundant","Use fully redundant multipliers","none",fetidp->fully_redundant,&fetidp->fully_redundant,NULL);
1172:   PetscOptionsBool("-ksp_fetidp_saddlepoint","Activates support for saddle-point problems",NULL,fetidp->saddlepoint,&fetidp->saddlepoint,NULL);
1173:   PetscOptionsBool("-ksp_fetidp_check","Activates verbose debugging output FETI-DP operators",NULL,fetidp->check,&fetidp->check,NULL);
1174:   PetscOptionsTail();
1175:   PCSetFromOptions(fetidp->innerbddc);
1176:   return(0);
1177: }

1179: /*MC
1180:      KSPFETIDP - The FETI-DP method

1182:    This class implements the FETI-DP method [1].
1183:    The matrix for the KSP must be of type MATIS.
1184:    The FETI-DP linear system (automatically generated constructing an internal PCBDDC object) is solved using an internal KSP object.

1186:    Options Database Keys:
1187: +   -ksp_fetidp_fullyredundant <false>   : use a fully redundant set of Lagrange multipliers
1188: .   -ksp_fetidp_saddlepoint <false>      : activates support for saddle point problems, see [2]
1189: .   -ksp_fetidp_saddlepoint_flip <false> : usually, an incompressible Stokes problem is written as
1190:                                            | A B^T | | v | = | f |
1191:                                            | B 0   | | p | = | g |
1192:                                            with B representing -\int_\Omega \nabla \cdot u q.
1193:                                            If -ksp_fetidp_saddlepoint_flip is true, the code assumes that the user provides it as
1194:                                            | A B^T | | v | = | f |
1195:                                            |-B 0   | | p | = |-g |
1196: .   -ksp_fetidp_pressure_field <-1>      : activates support for saddle point problems, and identifies the pressure field id.
1197:                                            If this information is not provided, the pressure field is detected by using MatFindZeroDiagonals().
1198: .   -ksp_fetidp_pressure_iszero <true>   : if false, extracts the pressure block from the matrix (i.e. for Almost Incompressible Elasticity)
1199: -   -ksp_fetidp_pressure_all <false>     : if false, uses the interface pressures, as described in [2]. If true, uses the entire pressure field.

1201:    Level: Advanced

1203:    Notes: Options for the inner KSP and for the customization of the PCBDDC object can be specified at command line by using the prefixes -fetidp_ and -fetidp_bddc_. E.g.,
1204: .vb
1205:       -fetidp_ksp_type gmres -fetidp_bddc_pc_bddc_symmetric false
1206: .ve
1207:    will use GMRES for the solution of the linear system on the Lagrange multipliers, generated using a non-symmetric PCBDDC.

1209:    For saddle point problems with continuous pressures, the preconditioned operator for the pressure solver can be specified with KSPFETIDPSetPressureOperator().
1210:    If it's not provided, an identity matrix will be created; the user then needs to scale it through a Richardson solver.
1211:    Options for the pressure solver can be prefixed with -fetidp_fielsplit_p_, E.g.
1212: .vb
1213:       -fetidp_fielsplit_p_ksp_type preonly -fetidp_fielsplit_p_pc_type lu -fetidp_fielsplit_p_pc_factor_mat_solver_package mumps
1214: .ve
1215:    In order to use the deluxe version of FETI-DP, you must customize the inner BDDC operator with -fetidp_bddc_pc_bddc_use_deluxe_scaling -fetidp_bddc_pc_bddc_deluxe_singlemat and use
1216:    non-redundant multipliers, i.e. -ksp_fetidp_fullyredundant false. Options for the scaling solver are prefixed by -fetidp_bddelta_, E.g.
1217: .vb
1218:       -fetidp_bddelta_pc_factor_mat_solver_package mumps -my_fetidp_bddelta_pc_type lu
1219: .ve

1221:    Some of the basic options such as maximum number of iterations and tolerances are automatically passed from this KSP to the inner KSP that actually performs the iterations.

1223:    The converged reason and number of iterations computed are passed from the inner KSP to this KSP at the end of the solution.

1225:    Developer Notes: Even though this method does not directly use any norms, the user is allowed to set the KSPNormType to any value.
1226:     This is so users do not have to change KSPNormType options when they switch from other KSP methods to this one.

1228:    References:
1229: .vb
1230: .  [1] - C. Farhat, M. Lesoinne, P. LeTallec, K. Pierson, and D. Rixen, FETI-DP: a dual-primal unified FETI method. I. A faster alternative to the two-level FETI method, Internat. J. Numer. Methods Engrg., 50 (2001), pp. 1523--1544
1231: .  [2] - X. Tu, J. Li, A FETI-DP type domain decomposition algorithm for three-dimensional incompressible Stokes equations, SIAM J. Numer. Anal., 53 (2015), pp. 720-742
1232: .ve

1234: .seealso: MATIS, PCBDDC, KSPFETIDPSetInnerBDDC(), KSPFETIDPGetInnerBDDC(), KSPFETIDPGetInnerKSP()
1235: M*/
1236: PETSC_EXTERN PetscErrorCode KSPCreate_FETIDP(KSP ksp)
1237: {
1239:   KSP_FETIDP     *fetidp;
1240:   KSP_FETIDPMon  *monctx;
1241:   PC_BDDC        *pcbddc;
1242:   PC             pc;

1245:   KSPSetSupportedNorm(ksp,KSP_NORM_NONE,PC_LEFT,3);
1246:   KSPSetSupportedNorm(ksp,KSP_NORM_NONE,PC_RIGHT,2);
1247:   KSPSetSupportedNorm(ksp,KSP_NORM_PRECONDITIONED,PC_LEFT,2);
1248:   KSPSetSupportedNorm(ksp,KSP_NORM_PRECONDITIONED,PC_RIGHT,2);
1249:   KSPSetSupportedNorm(ksp,KSP_NORM_UNPRECONDITIONED,PC_LEFT,2);
1250:   KSPSetSupportedNorm(ksp,KSP_NORM_UNPRECONDITIONED,PC_RIGHT,2);
1251:   KSPSetSupportedNorm(ksp,KSP_NORM_NATURAL,PC_LEFT,2);

1253:   PetscNewLog(ksp,&fetidp);
1254:   fetidp->matstate     = -1;
1255:   fetidp->matnnzstate  = -1;
1256:   fetidp->statechanged = PETSC_TRUE;

1258:   ksp->data = (void*)fetidp;
1259:   ksp->ops->setup                        = KSPSetUp_FETIDP;
1260:   ksp->ops->solve                        = KSPSolve_FETIDP;
1261:   ksp->ops->destroy                      = KSPDestroy_FETIDP;
1262:   ksp->ops->computeeigenvalues           = KSPComputeEigenvalues_FETIDP;
1263:   ksp->ops->computeextremesingularvalues = KSPComputeExtremeSingularValues_FETIDP;
1264:   ksp->ops->view                         = KSPView_FETIDP;
1265:   ksp->ops->setfromoptions               = KSPSetFromOptions_FETIDP;
1266:   ksp->ops->buildsolution                = KSPBuildSolution_FETIDP;
1267:   ksp->ops->buildresidual                = KSPBuildResidualDefault;
1268:   KSPGetPC(ksp,&pc);
1269:   PCSetType(pc,PCNONE);
1270:   /* create the inner KSP for the Lagrange multipliers */
1271:   KSPCreate(PetscObjectComm((PetscObject)ksp),&fetidp->innerksp);
1272:   KSPGetPC(fetidp->innerksp,&pc);
1273:   PCSetType(pc,PCNONE);
1274:   PetscLogObjectParent((PetscObject)ksp,(PetscObject)fetidp->innerksp);
1275:   /* monitor */
1276:   PetscNew(&monctx);
1277:   monctx->parentksp = ksp;
1278:   fetidp->monctx = monctx;
1279:   KSPMonitorSet(fetidp->innerksp,KSPMonitor_FETIDP,fetidp->monctx,NULL);
1280:   /* create the inner BDDC */
1281:   PCCreate(PetscObjectComm((PetscObject)ksp),&fetidp->innerbddc);
1282:   PCSetType(fetidp->innerbddc,PCBDDC);
1283:   /* make sure we always obtain a consistent FETI-DP matrix
1284:      for symmetric problems, the user can always customize it through the command line */
1285:   pcbddc = (PC_BDDC*)fetidp->innerbddc->data;
1286:   pcbddc->symmetric_primal = PETSC_FALSE;
1287:   PetscLogObjectParent((PetscObject)ksp,(PetscObject)fetidp->innerbddc);
1288:   /* composed functions */
1289:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPSetInnerBDDC_C",KSPFETIDPSetInnerBDDC_FETIDP);
1290:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPGetInnerBDDC_C",KSPFETIDPGetInnerBDDC_FETIDP);
1291:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPGetInnerKSP_C",KSPFETIDPGetInnerKSP_FETIDP);
1292:   PetscObjectComposeFunction((PetscObject)ksp,"KSPFETIDPSetPressureOperator_C",KSPFETIDPSetPressureOperator_FETIDP);
1293:   /* need to call KSPSetUp_FETIDP even with KSP_SETUP_NEWMATRIX */
1294:   ksp->setupnewmatrix = PETSC_TRUE;
1295:   return(0);
1296: }