46 #ifndef MUELU_PERFUTILS_DEF_HPP 47 #define MUELU_PERFUTILS_DEF_HPP 53 #include <Teuchos_CommHelpers.hpp> 56 #include <Xpetra_Export.hpp> 57 #include <Xpetra_Import.hpp> 58 #include <Xpetra_Matrix.hpp> 59 #include <Xpetra_CrsMatrixWrap.hpp> 68 void calculateStats(Type& minVal, Type& maxVal,
double& avgVal,
double& devVal,
int& minProc,
int& maxProc,
const RCP<
const Teuchos::Comm<int> >& comm,
int numActiveProcs,
const Type& v) {
70 Type sumVal, sum2Val, v2 = v*v;
71 double zero = Teuchos::ScalarTraits<double>::zero();
79 w = (minVal == v) ? comm->getRank() : -1;
81 w = (maxVal == v) ? comm->getRank() : -1;
84 avgVal = (numActiveProcs > 0 ? (as<double>(Teuchos::ScalarTraits<Type>::real(sumVal)) / numActiveProcs) : zero);
85 devVal = (numActiveProcs > 1 ? sqrt((as<double>(Teuchos::ScalarTraits<Type>::real(sum2Val - sumVal*avgVal)))/(numActiveProcs-1)) : zero);
91 double avgVal, devVal;
93 calculateStats<Type>(minVal, maxVal, avgVal, devVal, minProc, maxProc, comm, numActiveProcs, v);
95 const double zero = Teuchos::ScalarTraits<double>::zero();
96 const double one = Teuchos::ScalarTraits<double>::one();
97 std::ostringstream buf;
99 if ((avgVal != zero) && (paramList.is_null() || !paramList->isParameter(
"print abs") || paramList->get<
bool>(
"print abs") ==
false)) {
100 double relDev = (devVal/avgVal)*100;
101 double relMin = (as<double>(Teuchos::ScalarTraits<Type>::real(minVal))/avgVal-one)*100;
102 double relMax = (as<double>(Teuchos::ScalarTraits<Type>::real(maxVal))/avgVal-one)*100;
103 buf <<
"avg = " << std::scientific << std::setw(10) << std::setprecision(2) << avgVal <<
", " 104 <<
"dev = " << std::fixed << std::setw(6) << std::setprecision(1) << relDev <<
"%, " 105 <<
"min = " << std::fixed << std::setw(7) << std::setprecision(1) << std::setw(7) << relMin <<
"%" 106 <<
" (" << std::scientific << std::setw(10) << std::setprecision(2) << minVal <<
" on " << std::fixed << std::setw(4) << minProc <<
"), " 107 <<
"max = " << std::fixed << std::setw(7) << std::setprecision(1) << relMax <<
"%" 108 <<
" (" << std::scientific << std::setw(10) << std::setprecision(2) << maxVal <<
" on " << std::fixed << std::setw(4) << maxProc <<
")";
110 double relDev = (avgVal != zero ? (devVal/avgVal)*100 : zero);
111 buf <<
"avg = " << std::scientific << std::setw(10) << std::setprecision(2) << avgVal <<
", " 112 <<
"dev = " << std::fixed << std::setw(6) << std::setprecision(1) << relDev <<
"%, " 113 <<
"min = " << std::scientific << std::setw(10) << std::setprecision(2) << minVal
114 <<
" (on " << std::fixed << std::setw(4) << minProc <<
"), " 115 <<
"max = " << std::scientific << std::setw(10) << std::setprecision(2) << maxVal
116 <<
" (on " << std::fixed << std::setw(4) << maxProc <<
")";
122 bool cmp_less(
typename Map::value_type& v1,
typename Map::value_type& v2) {
123 return v1.second < v2.second;
126 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
131 typedef Xpetra::global_size_t global_size_t;
133 std::ostringstream ss;
135 ss << msgTag <<
" size = " << A.getGlobalNumRows() <<
" x " << A.getGlobalNumCols();
136 if(A.haveGlobalConstants())
137 ss <<
", nnz = " << A.getGlobalNumEntries();
140 if (params.is_null())
143 bool printLoadBalanceInfo =
false, printCommInfo =
false, printEntryStats =
false;
144 if (params->isParameter(
"printLoadBalancingInfo") && params->get<
bool>(
"printLoadBalancingInfo"))
145 printLoadBalanceInfo =
true;
146 if (params->isParameter(
"printCommInfo") && params->get<
bool>(
"printCommInfo"))
147 printCommInfo =
true;
148 if (params->isParameter(
"printEntryStats") && params->get<
bool>(
"printEntryStats"))
149 printEntryStats =
true;
151 if (!printLoadBalanceInfo && !printCommInfo && !printEntryStats)
154 RCP<const Import> importer = A.getCrsGraph()->getImporter();
155 RCP<const Export> exporter = A.getCrsGraph()->getExporter();
157 size_t numMyNnz = A.getNodeNumEntries(), numMyRows = A.getNodeNumRows();
160 RCP<const Teuchos::Comm<int> > origComm = A.getRowMap()->getComm();
161 bool activeProc =
true;
162 int numProc = origComm->getSize();
163 int numActiveProcs = 0;
165 RCP<const Teuchos::MpiComm<int> > mpiComm = rcp_dynamic_cast<
const Teuchos::MpiComm<int> >(origComm);
166 MPI_Comm rawComm = (*mpiComm->getRawMpiComm())();
168 std::vector<size_t> numRowsPerProc(numProc);
169 Teuchos::gatherAll(*origComm, 1, &numMyRows, numProc, &numRowsPerProc[0]);
172 bool rootFlag =
true;
173 for (
int i = 0; i < numProc; i++) {
174 if (numRowsPerProc[i]) {
183 if(numMyRows == 0) {activeProc =
false; numMyNnz = 0;}
197 ParameterList absList;
198 absList.set(
"print abs",
true);
200 RCP<const Matrix> rcpA = rcpFromRef(A);
201 RCP<const CrsMatrixWrap> crsWrapA = rcp_dynamic_cast<
const Xpetra::CrsMatrixWrap<Scalar, LocalOrdinal, GlobalOrdinal, Node> >(rcpA);
202 RCP<const CrsMatrix> crsA;
203 if (!crsWrapA.is_null())
204 crsA = crsWrapA->getCrsMatrix();
205 if (printEntryStats && !crsA.is_null()) {
206 typedef Teuchos::ScalarTraits<Scalar> STS;
207 typedef typename STS::magnitudeType magnitudeType;
208 typedef Teuchos::ScalarTraits<magnitudeType> MTS;
209 ArrayRCP<const size_t> rowptr_RCP;
210 ArrayRCP<const LocalOrdinal> colind_RCP;
211 ArrayRCP<const Scalar> vals_RCP;
212 ArrayRCP<size_t> offsets_RCP;
213 ArrayView<const size_t> rowptr;
214 ArrayView<const Scalar> vals;
215 ArrayView<size_t> offsets;
217 crsA->getAllValues(rowptr_RCP, colind_RCP, vals_RCP);
218 crsA->getLocalDiagOffsets(offsets_RCP);
219 rowptr = rowptr_RCP();
221 offsets = offsets_RCP();
223 Scalar val, minVal, maxVal;
224 magnitudeType absVal, minAbsVal, maxAbsVal;
226 minVal = STS::rmax();
227 maxVal = STS::rmin();
228 minAbsVal = MTS::rmax();
229 maxAbsVal = MTS::zero();
231 for (
int i = 0; i < offsets.size(); i++) {
232 val = vals[rowptr[i]+offsets[i]];
233 if (STS::real(val) < STS::real(minVal))
235 if (STS::real(val) > STS::real(maxVal))
237 absVal = STS::magnitude(val);
238 minAbsVal = std::min(minAbsVal, absVal);
239 maxAbsVal = std::max(maxAbsVal, absVal);
242 ss << msgTag <<
" diag min : " << stringStats<Scalar>(origComm, numActiveProcs, minVal) << std::endl;
243 ss << msgTag <<
" diag max : " << stringStats<Scalar>(origComm, numActiveProcs, maxVal) << std::endl;
244 ss << msgTag <<
" abs(diag) min : " << stringStats<Scalar>(origComm, numActiveProcs, minAbsVal) << std::endl;
245 ss << msgTag <<
" abs(diag) max : " << stringStats<Scalar>(origComm, numActiveProcs, maxAbsVal) << std::endl;
249 minVal = STS::rmax();
250 maxVal = STS::rmin();
251 minAbsVal = MTS::rmax();
252 maxAbsVal = MTS::zero();
254 for (
int i = 0; i < vals.size(); i++) {
256 if (STS::real(val) < STS::real(minVal))
258 if (STS::real(val) > STS::real(maxVal))
260 absVal = STS::magnitude(val);
261 minAbsVal = std::min(minAbsVal, absVal);
262 maxAbsVal = std::max(maxAbsVal, absVal);
265 ss << msgTag <<
" entry min : " << stringStats<Scalar>(origComm, numActiveProcs, minVal) << std::endl;
266 ss << msgTag <<
" entry max : " << stringStats<Scalar>(origComm, numActiveProcs, maxVal) << std::endl;
267 ss << msgTag <<
" abs(entry) min : " << stringStats<Scalar>(origComm, numActiveProcs, minAbsVal) << std::endl;
268 ss << msgTag <<
" abs(entry) max : " << stringStats<Scalar>(origComm, numActiveProcs, maxAbsVal) << std::endl;
273 if (printLoadBalanceInfo) {
274 ss << msgTag <<
" Load balancing info" << std::endl;
275 ss << msgTag <<
" # active processes: " << numActiveProcs <<
"/" << numProc << std::endl;
276 ss << msgTag <<
" # rows per proc : " << stringStats<global_size_t>(origComm, numActiveProcs, numMyRows) << std::endl;
277 ss << msgTag <<
" # nnz per proc : " << stringStats<global_size_t>(origComm, numActiveProcs, numMyNnz) << std::endl;
280 if (printCommInfo && numActiveProcs != 1) {
281 typedef std::map<int,size_t> map_type;
283 if (!importer.is_null()) {
284 ArrayView<const int> exportPIDs = importer->getExportPIDs();
285 if (exportPIDs.size())
286 for (
int i = 0; i < exportPIDs.size(); i++)
287 neighMap[exportPIDs[i]]++;
291 size_t numExportSend = 0;
292 size_t numImportSend = 0;
298 numExportSend = (!exporter.is_null() ? exporter->getNumExportIDs() : 0);
299 numImportSend = (!importer.is_null() ? importer->getNumExportIDs() : 0);
300 numMsgs = neighMap.size();
301 map_type::const_iterator it = std::min_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
302 minMsg = (it != neighMap.end() ? it->second : 0);
303 it = std::max_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
304 maxMsg = (it != neighMap.end() ? it->second : 0);
307 ss << msgTag <<
" Communication info" << std::endl;
308 ss << msgTag <<
" # num export send : " << stringStats<global_size_t>(origComm, numActiveProcs, numExportSend) << std::endl;
309 ss << msgTag <<
" # num import send : " << stringStats<global_size_t>(origComm, numActiveProcs, numImportSend) << std::endl;
310 ss << msgTag <<
" # num msgs : " << stringStats<global_size_t>(origComm, numActiveProcs, numMsgs, rcpFromRef(absList)) << std::endl;
311 ss << msgTag <<
" # min msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, minMsg) << std::endl;
312 ss << msgTag <<
" # max msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, maxMsg) << std::endl;
318 int strLength = outstr.size();
319 MPI_Bcast(&strLength, 1, MPI_INT, root, rawComm);
320 if (origComm->getRank() != root)
321 outstr.resize(strLength);
322 MPI_Bcast(&outstr[0], strLength, MPI_CHAR, root, rawComm);
328 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
331 typedef Xpetra::global_size_t global_size_t;
333 std::ostringstream ss;
336 RCP<const Teuchos::Comm<int> > origComm = importer->getSourceMap()->getComm();
337 bool activeProc =
true;
338 int numActiveProcs = origComm->getSize();
340 RCP<const Teuchos::MpiComm<int> > mpiComm = rcp_dynamic_cast<
const Teuchos::MpiComm<int> >(origComm);
341 MPI_Comm rawComm = (*mpiComm->getRawMpiComm())();
346 ParameterList absList;
347 absList.set(
"print abs",
true);
349 typedef std::map<int,size_t> map_type;
351 ArrayView<const int> exportPIDs = importer->getExportPIDs();
352 if (exportPIDs.size())
353 for (
int i = 0; i < exportPIDs.size(); i++)
354 neighMap[exportPIDs[i]]++;
357 size_t numImportSend = 0;
363 numImportSend = importer->getNumExportIDs();
364 numMsgs = neighMap.size();
365 map_type::const_iterator it = std::min_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
366 minMsg = (it != neighMap.end() ? it->second : 0);
367 it = std::max_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
368 maxMsg = (it != neighMap.end() ? it->second : 0);
371 ss << msgTag <<
" Communication info" << std::endl;
372 ss << msgTag <<
" # num import send : " << stringStats<global_size_t>(origComm, numActiveProcs, numImportSend) << std::endl;
373 ss << msgTag <<
" # num msgs : " << stringStats<global_size_t>(origComm, numActiveProcs, numMsgs, rcpFromRef(absList)) << std::endl;
374 ss << msgTag <<
" # min msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, minMsg) << std::endl;
375 ss << msgTag <<
" # max msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, maxMsg) << std::endl;
381 int strLength = outstr.size();
382 MPI_Bcast(&strLength, 1, MPI_INT, root, rawComm);
383 if (origComm->getRank() != root)
384 outstr.resize(strLength);
385 MPI_Bcast(&outstr[0], strLength, MPI_CHAR, root, rawComm);
391 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
396 std::ostringstream out;
398 RCP<const Teuchos::Comm<int> > comm = A.getRowMap()->getComm();
399 int myRank = comm->getRank();
401 out << msgTag <<
" " << myRank <<
":";
403 RCP<const Import> importer = (A.getCrsGraph() != Teuchos::null ? A.getCrsGraph()->getImporter() : Teuchos::null);
404 if (importer.is_null()) {
409 ArrayView<const int> exportPIDs = importer->getExportPIDs();
411 if (exportPIDs.size()) {
413 int neigh = exportPIDs[0];
415 for (
int i = 1; i < exportPIDs.size(); i++) {
416 if (exportPIDs[i] != exportPIDs[i-1]) {
417 out <<
" " << neigh <<
"(" << weight <<
")";
419 neigh = exportPIDs[i];
426 out <<
" " << neigh <<
"(" << weight <<
")" << std::endl;
432 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
439 bool hasCrsGraph =
true;
452 #endif // MUELU_PERFUTILS_DEF_HPP static bool CheckMatrix(const Matrix &A)
std::string stringStats(const RCP< const Teuchos::Comm< int > > &comm, int numActiveProcs, const Type &v, RCP< ParameterList > paramList=Teuchos::null)
#define MueLu_sumAll(rcpComm, in, out)
#define MueLu_maxAll(rcpComm, in, out)
Namespace for MueLu classes and methods.
#define MueLu_minAll(rcpComm, in, out)
static std::string CommPattern(const Matrix &A, const std::string &msgTag, RCP< const Teuchos::ParameterList > params=Teuchos::null)
static std::string PrintImporterInfo(RCP< const Import > importer, const std::string &msgTag)
MueLu::DefaultScalar Scalar
static std::string PrintMatrixInfo(const Matrix &A, const std::string &msgTag, RCP< const Teuchos::ParameterList > params=Teuchos::null)
void calculateStats(Type &minVal, Type &maxVal, double &avgVal, double &devVal, int &minProc, int &maxProc, const RCP< const Teuchos::Comm< int > > &comm, int numActiveProcs, const Type &v)
bool cmp_less(typename Map::value_type &v1, typename Map::value_type &v2)