USGS

Isis 3.0 Application Source Code Reference

Home

ProgramAnalyzer.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * $Revision: 1.3 $
4  * $Date: 2010/02/25 18:39:05 $
5  *
6  * Unless noted otherwise, the portions of Isis written by the USGS are
7  * public domain. See individual third-party library and package descriptions
8  * for intellectual property information, user agreements, and related
9  * information.
10  *
11  * Although Isis has been used by the USGS, no warranty, expressed or
12  * implied, is made by the USGS as to the accuracy and functioning of such
13  * software and related material nor shall the fact of distribution
14  * constitute any such warranty, and no responsibility is assumed by the
15  * USGS in connection therewith.
16  *
17  * For additional information, launch
18  * $ISISROOT/doc//documents/Disclaimers/Disclaimers.html
19  * in a browser or see the Privacy & Disclaimers page on the Isis website,
20  * http://isis.astrogeology.usgs.gov, and the USGS privacy and disclaimers on
21  * http://www.usgs.gov/privacy.html.
22  */
23 #include <memory>
24 #include <cmath>
25 #include <cstdlib>
26 #include <ctime>
27 #include <iostream>
28 #include <iomanip>
29 #include <sstream>
30 
31 #include "ProgramAnalyzer.h"
32 #include "IString.h"
33 #include "Pvl.h"
34 #include "PvlObject.h"
35 
36 
37 using namespace std;
38 
39 namespace Isis {
40 
41 /** Default constructor */
42 ProgramAnalyzer::ProgramAnalyzer() {
43  init();
44 }
45 
46 /**
47  * Analyzes the given log file on construction
48  *
49  * @param logfile A print.prt file
50  */
51 ProgramAnalyzer::ProgramAnalyzer(const QString &logfile) {
52  init();
53  add(logfile);
54 }
55 
56 /**
57  * @brief Set the list of program exclusions
58  *
59  * When provided, the QString should contain names of ISIS programs that will be
60  * excluded in the analysis. If more than one program is desired, separate
61  * them by commas.
62  *
63  * The exclusion list takes precedence over any applications added in the
64  * inclusion list. In other words, if the same program is included in both the
65  * inclusion and exclusion list, it will be excluded.
66  *
67  * Note this method can be called repeatedly to add names.
68  *
69  * @param name Comma separated list of names of programs to exclude
70  */
71 void ProgramAnalyzer::setExclude(const QString &name) {
72  foreach (QString singleName, name.split(",")) {
73  exclude(singleName);
74  }
75 }
76 
77 /**
78  * @brief Loads a list of excluded program names from a file
79  *
80  * The file is expected to contain the name of an ISIS program, one per line.
81  * These programs will be logged but not included in the analysis.
82  *
83  * The exclusion list takes precedence over any applications added in the
84  * inclusion list. In other words, if the same program is included in both the
85  * inclusion and exclusion list, it will be excluded.
86  *
87  * Note this method can be called repeatedly to add names.
88  *
89  * @param name Name of file containing program list to exclude
90  */
91 void ProgramAnalyzer::exclude(const QString &name) {
92  QString prog = name.trimmed();
93  if ( !_excludes.exists(prog) ) {
94  _excludes.add(prog, 0);
95  }
96  return;
97 }
98 
99 
100 /**
101  * @brief Set the list of program inclusions
102  *
103  * When provided, the QString should contain names of ISIS programs that will be
104  * included in the analysis. If more than one program is desired, separate
105  * them by commas.
106  *
107  * If this option is used, it will only included programs given in this list.
108  * It operates as both an inclusive and exclusive list, so there is no need to
109  * also utilize the exclude features of this class.
110  *
111  * However, if you do use the exclusion features of this clas, the exclusion
112  * list takes precedence over any applications added in the inclusion list. In
113  * other words, if the same program is included in both the inclusion and
114  * exclusion list, it will be excluded.
115  *
116  * Note this method can be called repeatedly to add names.
117  *
118  * @param name Comma separated list of names of programs to include
119  */
120 void ProgramAnalyzer::setInclude(const QString &name) {
121  foreach (QString singleName, name.split(",")) {
122  include(singleName);
123  }
124 }
125 
126 /**
127  * @brief Loads a list of included program names from a file
128  *
129  * The file is expected to contain the name of an ISIS program, one per line.
130  * These programs will be included in the analysis.
131  *
132  * If this option is used, it will only included programs given in this list.
133  * It operates as both an inclusive and exclusive list, so there is no need to
134  * also utilize the exclude features of this class.
135  *
136  * However, if you do use the exclusion feartures, the exclusion list takes
137  * precedence over any applications added in the inclusion list. In other
138  * words, if the same program is included in both the inclusion and exclusion
139  * list, it will be excluded.
140  *
141  * Note this method can be called repeatedly to add names.
142  *
143  * @param name Name of file containing program list to include
144  */
145 void ProgramAnalyzer::include(const QString &name) {
146  QString prog = name.trimmed();
147  if ( !_includes.exists(prog) ) {
148  _includes.add(prog, 0);
149  }
150  return;
151 }
152 
153 /**
154  * @brief Adds a print.prt file to the analysis
155  *
156  * The programs contained in the log file, assumed to be a print.prt file, will
157  * be added to the list of programs to be analyzed. They are subject to the
158  * exclude and include program lists.
159  *
160  * @param logfile An ISIS3 print.prt file
161  */
162 void ProgramAnalyzer::add(const QString &logfile) {
163  Pvl plog(logfile);
164  for(int i = 0; i < plog.objects(); i++) {
165  add(plog.object(i));
166  }
167 }
168 
169 /**
170  * @brief Adds a program object originating from a print.prt file
171  *
172  * The PvlObject provided is assumed to orginate from an ISIS3 print.prt log
173  * file. It contains information that will be extracted and analyzed according
174  * to the features of this class.
175  *
176  *
177  * @param program Pvl object containing the output log of an ISIS3 application
178  */
179 void ProgramAnalyzer::add(PvlObject &program) {
180  _count++;
181  QString prog(program.name());
182  if ( _excludes.exists(prog) ) {
183  _excludes.get(prog)++;
184  return;
185  }
186 
187  if ( _includes.size() > 0 ) {
188  if ( !_includes.exists(prog) ) {
189  return;
190  }
191  _includes.get(prog)++;
192  }
193 
194  ProgramData pdata;
195  pdata.name = prog;
196  pdata.runtime = getKey(program, "ExecutionDateTime");
197  pdata.from = getKey(program, "From", "UserParameters");
198  pdata.to = getKey(program, "To", "UserParameters");
199  accounting(program, pdata);
200  analyze(pdata);
201  return;
202 }
203 
204 /**
205  * @brief Reports program counters for the current state
206  *
207  * This method reports counts of programs as they were added to the object. It
208  * reports total programs, numbers for analyzed, included, excluded, unique,
209  * valid, errors, zero CPU/Connect times and incomplete or invalid (typcially
210  * negative times) for programs it evaluted.
211  *
212  * @param name Name of Pvl group to create
213  *
214  * @return PvlGroup Pvl group containing program numbers/counts
215  */
216 PvlGroup ProgramAnalyzer::review(const QString &name) const {
217  PvlGroup pvl(name);
218 
219  pvl += PvlKeyword("Programs", toString(size()));
220  pvl += PvlKeyword("Unique", toString(Programs()));
221  pvl += PvlKeyword("Included", toString(LimitTotals(_includes)));
222  pvl += PvlKeyword("Excluded", toString(LimitTotals(_excludes)));
223  pvl += PvlKeyword("Valid", toString(valid()));
224  pvl += PvlKeyword("Errors", toString(errors()));
225  pvl += PvlKeyword("ZeroTime", toString(zerotime()));
226  pvl += PvlKeyword("NoData", toString(nodata()));
227  pvl += PvlKeyword("BadData", toString(baddata()));
228  pvl += PvlKeyword("Total", toString(count()));
229  return (pvl);
230 }
231 
232 
233 /**
234  * @brief Reports cumulative runtime performance statistics for programs
235  *
236  * This method formats the contents of the program analysis in a Pvl group that
237  * provides information for all programs regardin CPU, connect and I/O times.
238  *
239  *
240  * @param name Name of Pvl group to create
241  *
242  * @return PvlGroup Pvl group containing cumulative program analysis
243  */
244 PvlGroup ProgramAnalyzer::cumulative(const QString &name) const {
245  return (toPvl(_totals, name));
246 }
247 
248 /**
249  * @brief Reports analysis for a specific program
250  *
251  * This object maintains individual statistics for each unique program. This
252  * method reports the analysis for a particular program.
253  *
254  *
255  * @param name Name of the program to analyze
256  *
257  * @return PvlGroup Pvl group containing program analysis
258  */
259 PvlGroup ProgramAnalyzer::summarize(const QString &name) const {
260  return (toPvl(_programs.get(name)));
261 }
262 
263 /**
264  * @brief Reports analysis for the nth occuring application in the list
265  *
266  * This object maintains individual statistics for each unique program. This
267  * method reports the analysis for a program that occurs in the list at the
268  * given index.
269 
270  * @param index Index of the application to summerize
271  *
272  * @return PvlGroup Pvl group containing the program analysis
273  */
274 PvlGroup ProgramAnalyzer::summarize(const int index) const {
275  return (toPvl(_programs.getNth(index)));
276 }
277 
278 /**
279  * @brief Writes a header in CVS format to a ostream
280  *
281  * @param out Output stream to write the header
282  *
283  * @return ostream& Returns the stream
284  */
285 ostream &ProgramAnalyzer::header(ostream &out) const {
286  out << "Program,From,To,ExecutionDateTime,ConnectTime,CpuTime,IOTime\n";
287  return (out);
288 }
289 
290 /**
291  * @brief Writes the analysis to the stream in CSV format
292  *
293  * This method provides the analysis in CSV format for more traditional
294  * manipulation. This format is well suited to be plotted for further analysis
295  * of the program/system performance.
296  *
297  * The columns provided are: Program name, FROM file, TO file, runtime, connect
298  * time, CPU time, and I/O time (difference in runtime and CPU time).
299  *
300  *
301  * @param out Output stream to write data to
302  *
303  * @return ostream& The output stream
304  */
305 ostream &ProgramAnalyzer::listify(ostream &out) const {
306  vector<ProgramData>::const_iterator progs = _pdata.begin();
307  while (progs != _pdata.end()) {
308  if ( progs->status == VALID ) {
309  out << format(progs->name) << ",";
310  out << format(progs->from) << ",";
311  out << format(progs->to) << ",";
312  out << format(progs->runtime) << ",";
313  out << DblToStr(progs->connectTime, 2) << ",";
314  out << DblToStr(progs->cpuTime, 2) << ",";
315  out << DblToStr(progs->connectTime-progs->cpuTime, 2) << "\n";
316  }
317  ++progs;
318  }
319  return (out);
320 }
321 
322 
323 /**
324  * @brief Initializes the class
325  *
326  * This init function is reintrant and will reset all internal parameters to the
327  * empty state.
328  */
329 void ProgramAnalyzer::init() {
330  _count = 0;
331  _excludes = LogList();
332  _includes = LogList();
333  _programs = RunList();
334  _totals = RunTimeStats("Cumulative");
335  _pdata.clear();
336 }
337 
338 
339 /**
340  * @brief Provides a count of analyzed programs
341  *
342  * Iterates through all programs included in the analysis and provides a count
343  * fo the total. It will search through for a given status and only includes
344  * those which have the indicated status. The valid status to check are those
345  * defined in the Status enum list.
346  *
347  * @param status Status of the program to count
348  *
349  * @return int
350  */
351 int ProgramAnalyzer::getCount (ProgramAnalyzer::Status status) const {
352  vector<ProgramData>::const_iterator progs = _pdata.begin();
353  int n(0);
354  while (progs != _pdata.end()) {
355  if ( progs->status == status ) n++;
356  ++progs;
357  }
358  return (n);
359 }
360 
361 
362 /**
363  * @brief Extracts a keyword value from the Pvl object
364  *
365  *
366  * @param obj Pvl object to search for the keyword
367  * @param key Name of keyword to find
368  * @param grp Optional group within the object to find the keyword
369  *
370  * @return QString Value of the keyword if the keyword exists, otherwise an empty
371  * QString is returned.
372  */
373 QString ProgramAnalyzer::getKey(PvlObject &obj, const QString &key,
374  const QString &grp) const {
375 
376  QString value("");
377  if ( !grp.isEmpty() ) {
378  PvlGroup &g = obj.findGroup(grp);
379  value = findKey(g, key);
380  }
381  else {
382  value = findKey(obj, key);
383  }
384  return (value);
385 }
386 
387 /**
388  * @brief Converts times represented in text to digital values
389  *
390  * The text QString, atime, is expected to be of the format "HH:MM:SS.sss" where
391  * "HH" is hours, "MM" is minutes and "SS.sss" is seconds.milliseconds. The
392  * units returned are in seconds.
393  *
394  * @param atime Text QString containing time to convert
395  * @param dtime Time in seconds
396  *
397  * @return ProgramAnalyzer::Status Returns BADDATA if the text QString is empty
398  * or malformed, or VALID if the conversion
399  * succeeds.
400  */
401 ProgramAnalyzer::Status ProgramAnalyzer::convertTime(const QString &atime,
402  double &dtime) const {
403  if ( atime.isEmpty() ) return (BADDATA);
404  QStringList t = atime.split(":");
405  if ( t.size() != 3 ) {
406  return (BADDATA);
407  }
408 
409  // Convert to seconds
410  double toSeconds(3600.0);
411  dtime = 0.0;
412  for ( unsigned int i = 0 ; i < 3 ; i++ ) {
413  dtime += toDouble(t[i]) * toSeconds;
414  toSeconds /= 60.0;
415  }
416 
417  return (VALID);
418 }
419 
420 /**
421  * @brief Compute analysis of program entry
422  *
423  * This method accepts a Pvl object that is assumed to orignate from an ISIS3
424  * print.prt log file and conforms to the format in the log file.
425  *
426  * Data is extracted from certain keywords in the object. Invalid objects or
427  * error conditions are detected and are indicated in the status of the program
428  * analysis structure, pdata. Other conditions of no time for runtimes or CPU
429  * times is also detected and indicated.
430  *
431  *
432  * @param obj Object containing program data
433  * @param pdata Structure to return derived values from the program data
434  *
435  * @return bool True if successful, false otherwise.
436  */
437 bool ProgramAnalyzer::accounting(PvlObject &obj,
438  ProgramAnalyzer::ProgramData &pdata) const {
439 
440  // Assume an error occured if the Accounting group is missing
441  if ( !obj.hasGroup("Accounting") ) {
442  pdata.status = ERRORS;
443  return (false);
444  }
445 
446  PvlGroup &acc = obj.findGroup("Accounting");
447  Status status = convertTime(findKey(acc,"ConnectTime"), pdata.connectTime);
448  pdata.status = convertTime(findKey(acc, "CpuTime"), pdata.cpuTime);
449 
450  // Test a few remaining times
451  if ( status != VALID ) pdata.status = status;
452  else if ( pdata.connectTime <= 0.0 ) pdata.status = ZEROTIME;
453 
454  return (pdata.status == VALID);
455 }
456 
457 
458 
459 /**
460  * @brief Performs the analysis of a program
461  *
462  * This method accepts a program data structure, determines validity for
463  * inclusion in the analysis and computes statistics from the data content.
464  *
465  * @param data Structure containing program data
466  *
467  * @return bool True if valid and included, false otherwize
468  */
469 bool ProgramAnalyzer::analyze(const ProgramAnalyzer::ProgramData &data) {
470 
471  bool good(false);
472  if ( data.status == VALID ) {
473  if ( !_programs.exists(data.name) ) {
474  _programs.add(data.name, RunTimeStats(data.name));
475  }
476  RunTimeStats &stats = _programs.get(data.name);
477  stats.contime.AddData(data.connectTime);
478  stats.cputime.AddData(data.cpuTime);
479  stats.iotime.AddData(data.connectTime - data.cpuTime);
480 
481  _totals.contime.AddData(data.connectTime);
482  _totals.cputime.AddData(data.cpuTime);
483  _totals.iotime.AddData(data.connectTime - data.cpuTime);
484 
485  good = true;
486  }
487  _pdata.push_back(data);
488  return (good);
489 }
490 
491 /**
492  * @brief Produces report of run time statistics for the given structure
493  *
494  *
495  * The data contained with the RunTimeStats is externalized to a Pvl group with
496  * some resonable formatting.
497  *
498  * @param stats Run time stats for the given data
499  * @param name Optional name of the PvlGroup results
500  *
501  * @return PvlGroup Pvl group of runtime statistics
502  */
503 PvlGroup ProgramAnalyzer::toPvl(const RunTimeStats &stats,
504  const QString &name) const {
505  PvlGroup pvl((name.isEmpty() ? stats.pname : name));
506 
507  pvl += PvlKeyword("Hits", toString(stats.contime.TotalPixels()));
508  pvl += PvlKeyword("ConnectTimeMinimum", DblToStr(stats.contime.Minimum(), 2));
509  pvl += PvlKeyword("ConnectTimeMaximum", DblToStr(stats.contime.Maximum(), 2));
510  pvl += PvlKeyword("ConnectTimeAverage", DblToStr(stats.contime.Average(), 2));
511  pvl += PvlKeyword("ConnectTimeStdDev", DblToStr(stats.contime.StandardDeviation(), 4));
512 
513  pvl += PvlKeyword("CpuTimeMinimum", DblToStr(stats.cputime.Minimum(), 2));
514  pvl += PvlKeyword("CpuTimeMaximum", DblToStr(stats.cputime.Maximum(), 2));
515  pvl += PvlKeyword("CpuTimeAverage", DblToStr(stats.cputime.Average(), 2));
516  pvl += PvlKeyword("CpuTimeStdDev", DblToStr(stats.cputime.StandardDeviation(), 4));
517 
518  pvl += PvlKeyword("IOTimeMinimum", DblToStr(stats.iotime.Minimum(), 2));
519  pvl += PvlKeyword("IOTimeMaximum", DblToStr(stats.iotime.Maximum(), 2));
520  pvl += PvlKeyword("IOTimeAverage", DblToStr(stats.iotime.Average(), 2));
521  pvl += PvlKeyword("IOTimeStdDev", DblToStr(stats.iotime.StandardDeviation(), 4));
522 
523  return (pvl);
524 }
525 
526 /**
527  * @brief Returns NULL for empty QStrings to ensure meaningful content
528  *
529  * @param s String to test for content
530  *
531  * @return QString Returns existing content if present, NULL if empty
532  */
533 QString ProgramAnalyzer::format(const QString &s) const {
534  if ( s.isEmpty() ) return (QString("NULL"));
535  return (s);
536 }
537 
538 
539  /**
540  * @brief Convert a double value to a QString subject to precision specs
541  *
542  * This method converts a double value to a QString that has a prefined digitis
543  * of precision. Fixed float form is used with the specified number of digits
544  * of precision.
545  *
546  * @param value Double value to convert to QString
547  *
548  * @return QString Returns the converted QString
549  */
550  QString ProgramAnalyzer::DblToStr(const double &value, const int precision)
551  const {
552  if(IsSpecial(value)) {
553  return (QString("0.0"));
554  }
555 
556  // Format the QString to specs
557  ostringstream strcnv;
558  strcnv.setf(std::ios::fixed);
559  strcnv << setprecision(precision) << value;
560  return (QString(strcnv.str().c_str()));
561  }
562 
563  /**
564  * @brief Returns the count of all programs in the log list
565  *
566  * This method computes a count of all programs that exist in the list of
567  * applications that encurred a limit in the analysis. It is not enough to
568  * just report the number of entries in the list - each list contains a count
569  * of occurances. These occurances are summed and return to the caller.
570  *
571  * @param limit List of applications incurring a limit
572  *
573  * @return int Total number of occurances all programs in the list
574  */
575  int ProgramAnalyzer::LimitTotals(const ProgramAnalyzer::LogList &limit)
576  const {
577  int total(0);
578  for ( int i = 0 ; i < limit.size() ; i++ ) {
579  total += limit.getNth(i);
580  }
581  return (total);
582  }
583 
584 } // namespace Isis
585 
586 
int g
Definition: butterworth.cpp:18
void init(FileList &inList)
This method will validate the input file list and set global variables: g_ccdFiles, g_ccdNumbers, g_numFiles, g_firstFilter.
Definition: hijitter.cpp:339
double add
Definition: poly.cpp:13
Statistics stats
Definition: cubediff.cpp:36
Cube * out
Definition: lrowac2pds.cpp:50
int size
Definition: fakecube.cpp:13
double toDouble(const T &value)
Helper function to convert values to doubles.
Definition: MdisCalUtils.h:49
QString format(const double &d, const int &precision, const QString &defValue="NULL")
Definition: sumspice.cpp:36
int errors
Definition: readmocisis.cpp:94