Actual source code: mpicusp.cu
petsc-3.8.4 2018-03-24
2: /*
3: This file contains routines for Parallel vector operations.
4: */
5: #define PETSC_SKIP_COMPLEX
6: #define PETSC_SKIP_SPINLOCK
8: #include <petscconf.h>
9: #include <../src/vec/vec/impls/mpi/pvecimpl.h>
10: #include <../src/vec/vec/impls/seq/seqcusp/cuspvecimpl.h>
12: PetscErrorCode VecDestroy_MPICUSP(Vec v)
13: {
17: try {
18: if (v->spptr) {
19: delete ((Vec_CUSP*)v->spptr)->GPUarray;
20: delete (Vec_CUSP*) v->spptr;
21: }
22: } catch(char * ex) {
23: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CUSP error: %s", ex);
24: }
25: VecDestroy_MPI(v);
26: return(0);
27: }
29: PetscErrorCode VecNorm_MPICUSP(Vec xin,NormType type,PetscReal *z)
30: {
31: PetscReal sum,work = 0.0;
35: if (type == NORM_2 || type == NORM_FROBENIUS) {
36: VecNorm_SeqCUSP(xin,NORM_2,&work);
37: work *= work;
38: MPIU_Allreduce(&work,&sum,1,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
39: *z = PetscSqrtReal(sum);
40: } else if (type == NORM_1) {
41: /* Find the local part */
42: VecNorm_SeqCUSP(xin,NORM_1,&work);
43: /* Find the global max */
44: MPIU_Allreduce(&work,z,1,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
45: } else if (type == NORM_INFINITY) {
46: /* Find the local max */
47: VecNorm_SeqCUSP(xin,NORM_INFINITY,&work);
48: /* Find the global max */
49: MPIU_Allreduce(&work,z,1,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)xin));
50: } else if (type == NORM_1_AND_2) {
51: PetscReal temp[2];
52: VecNorm_SeqCUSP(xin,NORM_1,temp);
53: VecNorm_SeqCUSP(xin,NORM_2,temp+1);
54: temp[1] = temp[1]*temp[1];
55: MPIU_Allreduce(temp,z,2,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
56: z[1] = PetscSqrtReal(z[1]);
57: }
58: return(0);
59: }
61: PetscErrorCode VecDot_MPICUSP(Vec xin,Vec yin,PetscScalar *z)
62: {
63: PetscScalar sum,work;
67: VecDot_SeqCUSP(xin,yin,&work);
68: MPIU_Allreduce(&work,&sum,1,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
69: *z = sum;
70: return(0);
71: }
73: PetscErrorCode VecTDot_MPICUSP(Vec xin,Vec yin,PetscScalar *z)
74: {
75: PetscScalar sum,work;
79: VecTDot_SeqCUSP(xin,yin,&work);
80: MPIU_Allreduce(&work,&sum,1,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
81: *z = sum;
82: return(0);
83: }
85: PetscErrorCode VecMDot_MPICUSP(Vec xin,PetscInt nv,const Vec y[],PetscScalar *z)
86: {
87: PetscScalar awork[128],*work = awork;
91: if (nv > 128) {
92: PetscMalloc1(nv,&work);
93: }
94: VecMDot_SeqCUSP(xin,nv,y,work);
95: MPIU_Allreduce(work,z,nv,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
96: if (nv > 128) {
97: PetscFree(work);
98: }
99: return(0);
100: }
102: /*MC
103: VECMPICUSP - VECMPICUSP = "mpicusp" - The basic parallel vector, modified to use CUSP
105: Options Database Keys:
106: . -vec_type mpicusp - sets the vector type to VECMPICUSP during a call to VecSetFromOptions()
108: Level: beginner
110: .seealso: VecCreate(), VecSetType(), VecSetFromOptions(), VecCreateMpiWithArray(), VECMPI, VecType, VecCreateMPI(), VecCreateMpi()
111: M*/
114: PetscErrorCode VecDuplicate_MPICUSP(Vec win,Vec *v)
115: {
117: Vec_MPI *vw,*w = (Vec_MPI*)win->data;
118: PetscScalar *array;
121: VecCreate(PetscObjectComm((PetscObject)win),v);
122: PetscLayoutReference(win->map,&(*v)->map);
124: VecCreate_MPI_Private(*v,PETSC_FALSE,w->nghost,0);
125: vw = (Vec_MPI*)(*v)->data;
126: PetscMemcpy((*v)->ops,win->ops,sizeof(struct _VecOps));
128: /* save local representation of the parallel vector (and scatter) if it exists */
129: if (w->localrep) {
130: VecGetArray(*v,&array);
131: VecCreateSeqWithArray(PETSC_COMM_SELF,1,win->map->n+w->nghost,array,&vw->localrep);
132: PetscMemcpy(vw->localrep->ops,w->localrep->ops,sizeof(struct _VecOps));
133: VecRestoreArray(*v,&array);
134: PetscLogObjectParent((PetscObject)*v,(PetscObject)vw->localrep);
135: vw->localupdate = w->localupdate;
136: if (vw->localupdate) {
137: PetscObjectReference((PetscObject)vw->localupdate);
138: }
139: }
141: /* New vector should inherit stashing property of parent */
142: (*v)->stash.donotstash = win->stash.donotstash;
143: (*v)->stash.ignorenegidx = win->stash.ignorenegidx;
145: /* change type_name appropriately */
146: PetscObjectChangeTypeName((PetscObject)(*v),VECMPICUSP);
148: PetscObjectListDuplicate(((PetscObject)win)->olist,&((PetscObject)(*v))->olist);
149: PetscFunctionListDuplicate(((PetscObject)win)->qlist,&((PetscObject)(*v))->qlist);
150: (*v)->map->bs = PetscAbs(win->map->bs);
151: (*v)->bstash.bs = win->bstash.bs;
152: return(0);
153: }
155: PetscErrorCode VecDotNorm2_MPICUSP(Vec s,Vec t,PetscScalar *dp,PetscScalar *nm)
156: {
158: PetscScalar work[2],sum[2];
161: VecDotNorm2_SeqCUSP(s,t,work,work+1);
162: MPIU_Allreduce(&work,&sum,2,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)s));
163: *dp = sum[0];
164: *nm = sum[1];
165: return(0);
166: }
168: PETSC_EXTERN PetscErrorCode VecCreate_MPICUSP(Vec vv)
169: {
173: VecCreate_MPI_Private(vv,PETSC_FALSE,0,0);
174: PetscObjectChangeTypeName((PetscObject)vv,VECMPICUSP);
176: vv->ops->dotnorm2 = VecDotNorm2_MPICUSP;
177: vv->ops->waxpy = VecWAXPY_SeqCUSP;
178: vv->ops->duplicate = VecDuplicate_MPICUSP;
179: vv->ops->dot = VecDot_MPICUSP;
180: vv->ops->mdot = VecMDot_MPICUSP;
181: vv->ops->tdot = VecTDot_MPICUSP;
182: vv->ops->norm = VecNorm_MPICUSP;
183: vv->ops->scale = VecScale_SeqCUSP;
184: vv->ops->copy = VecCopy_SeqCUSP;
185: vv->ops->set = VecSet_SeqCUSP;
186: vv->ops->swap = VecSwap_SeqCUSP;
187: vv->ops->axpy = VecAXPY_SeqCUSP;
188: vv->ops->axpby = VecAXPBY_SeqCUSP;
189: vv->ops->maxpy = VecMAXPY_SeqCUSP;
190: vv->ops->aypx = VecAYPX_SeqCUSP;
191: vv->ops->axpbypcz = VecAXPBYPCZ_SeqCUSP;
192: vv->ops->pointwisemult = VecPointwiseMult_SeqCUSP;
193: vv->ops->setrandom = VecSetRandom_SeqCUSP;
194: vv->ops->placearray = VecPlaceArray_SeqCUSP;
195: vv->ops->replacearray = VecReplaceArray_SeqCUSP;
196: vv->ops->resetarray = VecResetArray_SeqCUSP;
197: vv->ops->dot_local = VecDot_SeqCUSP;
198: vv->ops->tdot_local = VecTDot_SeqCUSP;
199: vv->ops->norm_local = VecNorm_SeqCUSP;
200: vv->ops->mdot_local = VecMDot_SeqCUSP;
201: vv->ops->destroy = VecDestroy_MPICUSP;
202: vv->ops->pointwisedivide = VecPointwiseDivide_SeqCUSP;
203: vv->ops->getlocalvector = VecGetLocalVector_SeqCUSP;
204: vv->ops->restorelocalvector = VecRestoreLocalVector_SeqCUSP;
205: vv->ops->getlocalvectorread = VecGetLocalVector_SeqCUSP;
206: vv->ops->restorelocalvectorread = VecRestoreLocalVector_SeqCUSP;
207: VecCUSPAllocateCheck(vv);CHKERRCUSP(ierr);
208: vv->valid_GPU_array = PETSC_CUSP_GPU;
209: VecSet(vv,0.0);
210: return(0);
211: }
213: PETSC_EXTERN PetscErrorCode VecCreate_CUSP(Vec v)
214: {
216: PetscMPIInt size;
219: MPI_Comm_size(PetscObjectComm((PetscObject)v),&size);
220: if (size == 1) {
221: VecSetType(v,VECSEQCUSP);
222: } else {
223: VecSetType(v,VECMPICUSP);
224: }
225: return(0);
226: }