Isis 3 Programmer Reference
CubeIoHandler.cpp
Go to the documentation of this file.
1 
24 #include "IsisDebug.h"
25 #include "CubeIoHandler.h"
26 
27 #include <algorithm>
28 #include <cmath>
29 #include <iomanip>
30 
31 #include <QDebug>
32 #include <QFile>
33 #include <QList>
34 #include <QListIterator>
35 #include <QMapIterator>
36 #include <QMutex>
37 #include <QPair>
38 #include <QRect>
39 #include <QTime>
40 
41 #include "Area3D.h"
42 #include "Brick.h"
43 #include "CubeCachingAlgorithm.h"
44 #include "Displacement.h"
45 #include "Distance.h"
46 #include "Endian.h"
47 #include "EndianSwapper.h"
48 #include "IException.h"
49 #include "IString.h"
50 #include "PixelType.h"
51 #include "Preference.h"
52 #include "Pvl.h"
53 #include "PvlGroup.h"
54 #include "PvlObject.h"
55 #include "RawCubeChunk.h"
57 #include "SpecialPixel.h"
58 #include "Statistics.h"
59 
60 using namespace std;
61 
62 namespace Isis {
79  CubeIoHandler::CubeIoHandler(QFile * dataFile,
80  const QList<int> *virtualBandList, const Pvl &label, bool alreadyOnDisk) {
81  m_byteSwapper = NULL;
82  m_cachingAlgorithms = NULL;
83  m_dataIsOnDiskMap = NULL;
84  m_rawData = NULL;
85  m_virtualBands = NULL;
86  m_nullChunkData = NULL;
87  m_lastProcessByLineChunks = NULL;
88  m_writeCache = NULL;
89  m_ioThreadPool = NULL;
90  m_writeThreadMutex = NULL;
91 
92  try {
93  if (!dataFile) {
94  IString msg = "Cannot create a CubeIoHandler with a NULL data file";
95  throw IException(IException::Programmer, msg, _FILEINFO_);
96  }
97 
98  m_cachingAlgorithms = new QList<CubeCachingAlgorithm *>;
99 
100  PvlGroup &performancePrefs =
101  Preference::Preferences().findGroup("Performance");
102  IString cubeWritePerfOpt = performancePrefs["CubeWriteThread"][0];
103  m_useOptimizedCubeWrite = (cubeWritePerfOpt.DownCase() == "optimized");
104  if ((m_useOptimizedCubeWrite && !alreadyOnDisk) ||
105  cubeWritePerfOpt.DownCase() == "always") {
106  m_ioThreadPool = new QThreadPool;
107  m_ioThreadPool->setMaxThreadCount(1);
108  }
109 
110  m_consecutiveOverflowCount = 0;
111  m_lastOperationWasWrite = false;
112  m_rawData = new QMap<int, RawCubeChunk *>;
113  m_writeCache = new QPair< QMutex *, QList<Buffer *> >;
114  m_writeCache->first = new QMutex;
115  m_writeThreadMutex = new QMutex;
116 
117  m_idealFlushSize = 32;
118 
119  m_cachingAlgorithms->append(new RegionalCachingAlgorithm);
120 
121  m_dataFile = dataFile;
122 
123  const PvlObject &core = label.findObject("IsisCube").findObject("Core");
124  const PvlGroup &pixelGroup = core.findGroup("Pixels");
125 
126  QString byteOrderStr = pixelGroup.findKeyword("ByteOrder")[0];
127  m_byteSwapper = new EndianSwapper(
128  byteOrderStr.toUpper());
129  m_base = pixelGroup.findKeyword("Base");
130  m_multiplier = pixelGroup.findKeyword("Multiplier");
131  m_pixelType = PixelTypeEnumeration(pixelGroup.findKeyword("Type"));
132 
133  // If the byte swapper isn't going to do anything, then get rid of it
134  // because it's quicker to check for a NULL byte swapper member than to
135  // call a swap that won't do anything.
136  if(!m_byteSwapper->willSwap()) {
137  delete m_byteSwapper;
138  m_byteSwapper = NULL;
139  }
140 
141  const PvlGroup &dimensions = core.findGroup("Dimensions");
142  m_numSamples = dimensions.findKeyword("Samples");
143  m_numLines = dimensions.findKeyword("Lines");
144  m_numBands = dimensions.findKeyword("Bands");
145 
146  m_startByte = (int)core.findKeyword("StartByte") - 1;
147 
148  m_samplesInChunk = -1;
149  m_linesInChunk = -1;
150  m_bandsInChunk = -1;
151 
152  if(!alreadyOnDisk) {
153  m_dataIsOnDiskMap = new QMap<int, bool>;
154  }
155 
156  setVirtualBands(virtualBandList);
157  }
158  catch(IException &e) {
159  IString msg = "Constructing CubeIoHandler failed";
160  throw IException(e, IException::Programmer, msg, _FILEINFO_);
161  }
162  catch(...) {
163  IString msg = "Constructing CubeIoHandler failed";
164  throw IException(IException::Programmer, msg, _FILEINFO_);
165  }
166  }
167 
168 
173  CubeIoHandler::~CubeIoHandler() {
174  ASSERT( m_rawData ? m_rawData->size() == 0 : 1 );
175 
176  if (m_ioThreadPool)
177  m_ioThreadPool->waitForDone();
178 
179  delete m_ioThreadPool;
180  m_ioThreadPool = NULL;
181 
182  delete m_dataIsOnDiskMap;
183  m_dataIsOnDiskMap = NULL;
184 
185  if (m_cachingAlgorithms) {
186  QListIterator<CubeCachingAlgorithm *> it(*m_cachingAlgorithms);
187  while (it.hasNext()) {
188  delete it.next();
189  }
190  delete m_cachingAlgorithms;
191  m_cachingAlgorithms = NULL;
192  }
193 
194  if (m_rawData) {
195  QMapIterator<int, RawCubeChunk *> it(*m_rawData);
196  while (it.hasNext()) {
197  // Unwritten data here means it cannot be written :(
198  ASSERT(0);
199  it.next();
200 
201  if(it.value())
202  delete it.value();
203  }
204  delete m_rawData;
205  m_rawData = NULL;
206  }
207 
208  if (m_writeCache) {
209  delete m_writeCache->first;
210  m_writeCache->first = NULL;
211 
212  for (int i = 0; i < m_writeCache->second.size(); i++) {
213  delete m_writeCache->second[i];
214  }
215  m_writeCache->second.clear();
216 
217  delete m_writeCache;
218  m_writeCache = NULL;
219  }
220 
221  delete m_byteSwapper;
222  m_byteSwapper = NULL;
223 
224  delete m_virtualBands;
225  m_virtualBands = NULL;
226 
227  delete m_nullChunkData;
228  m_nullChunkData = NULL;
229 
230  delete m_lastProcessByLineChunks;
231  m_lastProcessByLineChunks = NULL;
232 
233  delete m_writeThreadMutex;
234  m_writeThreadMutex = NULL;
235  }
236 
237 
246  void CubeIoHandler::read(Buffer &bufferToFill) const {
247  // We need to record the current chunk count size so we can use
248  // it to evaluate if the cache should be minimized
249  int lastChunkCount = m_rawData->size();
250 
251  if (m_lastOperationWasWrite) {
252  // Do the remaining writes
253  flushWriteCache(true);
254 
255  m_lastOperationWasWrite = false;
256 
257  // Stop backgrounding writes now, we don't want to keep incurring this
258  // penalty.
259  if (m_useOptimizedCubeWrite) {
260  delete m_ioThreadPool;
261  m_ioThreadPool = NULL;
262  }
263  }
264 
265  QMutexLocker lock(m_writeThreadMutex);
266 
267  // NON-THREADED CUBE READ
268  QList<RawCubeChunk *> cubeChunks;
269  QList<int > chunkBands;
270 
271  int bufferSampleCount = bufferToFill.SampleDimension();
272  int bufferLineCount = bufferToFill.LineDimension();
273  int bufferBandCount = bufferToFill.BandDimension();
274 
275  // our chunk dimensions are same as buffer shape dimensions
276  if (bufferSampleCount == m_samplesInChunk &&
277  bufferLineCount == m_linesInChunk &&
278  bufferBandCount == m_bandsInChunk) {
279  int bufferStartSample = bufferToFill.Sample();
280  int bufferStartLine = bufferToFill.Line();
281  int bufferStartBand = bufferToFill.Band();
282 
283  int bufferEndSample = bufferStartSample + bufferSampleCount - 1;
284  int bufferEndLine = bufferStartLine + bufferLineCount - 1;
285  int bufferEndBand = bufferStartBand + bufferBandCount - 1;
286 
287  // make sure we access the correct band
288  int startBand = bufferStartBand - 1;
289  if (m_virtualBands)
290  startBand = m_virtualBands->at(bufferStartBand - 1);
291 
292  int expectedChunkIndex =
293  ((bufferStartSample - 1) / getSampleCountInChunk()) +
294  ((bufferStartLine - 1) / getLineCountInChunk()) *
295  getChunkCountInSampleDimension() +
296  ((startBand - 1) / getBandCountInChunk()) *
297  getChunkCountInSampleDimension() *
298  getChunkCountInLineDimension();
299 
300  int chunkStartSample, chunkStartLine, chunkStartBand,
301  chunkEndSample, chunkEndLine, chunkEndBand;
302 
303  getChunkPlacement(expectedChunkIndex,
304  chunkStartSample, chunkStartLine, chunkStartBand,
305  chunkEndSample, chunkEndLine, chunkEndBand);
306 
307  if (chunkStartSample == bufferStartSample &&
308  chunkStartLine == bufferStartLine &&
309  chunkStartBand == bufferStartBand &&
310  chunkEndSample == bufferEndSample &&
311  chunkEndLine == bufferEndLine &&
312  chunkEndBand == bufferEndBand) {
313  cubeChunks.append(getChunk(expectedChunkIndex, true));
314  chunkBands.append(cubeChunks.last()->getStartBand());
315  }
316  }
317 
318  if (cubeChunks.empty()) {
319  // We can't guarantee our cube chunks will encompass the buffer
320  // if the buffer goes beyond the cube bounds.
321  for(int i = 0; i < bufferToFill.size(); i++) {
322  bufferToFill[i] = Null;
323  }
324 
326  chunkInfo = findCubeChunks(
327  bufferToFill.Sample(), bufferToFill.SampleDimension(),
328  bufferToFill.Line(), bufferToFill.LineDimension(),
329  bufferToFill.Band(), bufferToFill.BandDimension());
330  cubeChunks = chunkInfo.first;
331  chunkBands = chunkInfo.second;
332  }
333 
334  for (int i = 0; i < cubeChunks.size(); i++) {
335  writeIntoDouble(*cubeChunks[i], bufferToFill, chunkBands[i]);
336  }
337 
338  // Minimize the cache if it changed in size
339  if (lastChunkCount != m_rawData->size()) {
340  minimizeCache(cubeChunks, bufferToFill);
341  }
342  }
343 
344 
354  void CubeIoHandler::write(const Buffer &bufferToWrite) {
355  m_lastOperationWasWrite = true;
356 
357  if (m_ioThreadPool) {
358  // THREADED CUBE WRITE
359  Buffer * copy = new Buffer(bufferToWrite);
360  {
361  QMutexLocker locker(m_writeCache->first);
362  m_writeCache->second.append(copy);
363  }
364 
365  flushWriteCache();
366  }
367  else {
368  QMutexLocker lock(m_writeThreadMutex);
369  // NON-THREADED CUBE WRITE
370  synchronousWrite(bufferToWrite);
371  }
372  }
373 
374 
384  void CubeIoHandler::addCachingAlgorithm(CubeCachingAlgorithm *algorithm) {
385  m_cachingAlgorithms->prepend(algorithm);
386  }
387 
388 
399  void CubeIoHandler::clearCache(bool blockForWriteCache) const {
400  if (blockForWriteCache) {
401  // Start the rest of the writes
402  flushWriteCache(true);
403  }
404 
405  // If this map is allocated, then this is a brand new cube and we need to
406  // make sure it's filled with data or NULLs.
407  if(m_dataIsOnDiskMap) {
408  writeNullDataToDisk();
409  }
410 
411  // This should be allocated. This is a list of the cached cube data.
412  // Write it all to disk.
413  if (m_rawData) {
414  QMapIterator<int, RawCubeChunk *> it(*m_rawData);
415  while (it.hasNext()) {
416  it.next();
417 
418  if(it.value()) {
419  if(it.value()->isDirty()) {
420  (const_cast<CubeIoHandler *>(this))->writeRaw(*it.value());
421  }
422 
423  delete it.value();
424  }
425  }
426 
427  m_rawData->clear();
428  }
429 
430  if(m_lastProcessByLineChunks) {
431  delete m_lastProcessByLineChunks;
432  m_lastProcessByLineChunks = NULL;
433  }
434  }
435 
436 
441  BigInt CubeIoHandler::getDataSize() const {
442  return (BigInt)getChunkCountInSampleDimension() *
443  (BigInt)getChunkCountInLineDimension() *
444  (BigInt)getChunkCountInBandDimension() *
445  (BigInt)getBytesPerChunk();
446  }
447 
448 
458  void CubeIoHandler::setVirtualBands(const QList<int> *virtualBandList) {
459  if(m_virtualBands) {
460  delete m_virtualBands;
461  m_virtualBands = NULL;
462  }
463 
464  if(virtualBandList && !virtualBandList->empty())
465  m_virtualBands = new QList<int>(*virtualBandList);
466  }
467 
475  QMutex *CubeIoHandler::dataFileMutex() {
476  return m_writeThreadMutex;
477  }
478 
482  int CubeIoHandler::bandCount() const {
483  return m_numBands;
484  }
485 
486 
490  int CubeIoHandler::getBandCountInChunk() const {
491  return m_bandsInChunk;
492  }
493 
494 
502  BigInt CubeIoHandler::getBytesPerChunk() const {
503  return m_samplesInChunk * m_linesInChunk * m_bandsInChunk *
504  SizeOf(m_pixelType);
505  }
506 
507 
512  int CubeIoHandler::getChunkCountInBandDimension() const {
513  return (int)ceil((double)m_numBands / (double)m_bandsInChunk);
514  }
515 
516 
521  int CubeIoHandler::getChunkCountInLineDimension() const {
522  return (int)ceil((double)m_numLines / (double)m_linesInChunk);
523  }
524 
525 
530  int CubeIoHandler::getChunkCountInSampleDimension() const {
531  return (int)ceil((double)m_numSamples / (double)m_samplesInChunk);
532  }
533 
534 
546  int CubeIoHandler::getChunkIndex(const RawCubeChunk &chunk) const {
547 // ASSERT(chunk.getStartSample() <= sampleCount());
548 // ASSERT(chunk.getStartLine() <= lineCount());
549 // ASSERT(chunk.getStartBand() <= bandCount());
550 
551  int sampleIndex = (chunk.getStartSample() - 1) / getSampleCountInChunk();
552  int lineIndex = (chunk.getStartLine() - 1) / getLineCountInChunk();
553  int bandIndex = (chunk.getStartBand() - 1) / getBandCountInChunk();
554 
555  int indexInBand =
556  sampleIndex + lineIndex * getChunkCountInSampleDimension();
557  int indexOffsetToBand = bandIndex * getChunkCountInSampleDimension() *
558  getChunkCountInLineDimension();
559 
560  return indexOffsetToBand + indexInBand;
561  }
562 
563 
567  BigInt CubeIoHandler::getDataStartByte() const {
568  return m_startByte;
569  }
570 
571 
578  QFile * CubeIoHandler::getDataFile() {
579  return m_dataFile;
580  }
581 
582 
587  int CubeIoHandler::lineCount() const {
588  return m_numLines;
589  }
590 
591 
595  int CubeIoHandler::getLineCountInChunk() const {
596  return m_linesInChunk;
597  }
598 
599 
603  PixelType CubeIoHandler::pixelType() const {
604  return m_pixelType;
605  }
606 
607 
612  int CubeIoHandler::sampleCount() const {
613  return m_numSamples;
614  }
615 
616 
620  int CubeIoHandler::getSampleCountInChunk() const {
621  return m_samplesInChunk;
622  }
623 
624 
637  void CubeIoHandler::setChunkSizes(
638  int numSamples, int numLines, int numBands) {
639  bool success = false;
640  IString msg;
641 
642  if(m_samplesInChunk != -1 || m_linesInChunk != -1 || m_bandsInChunk != -1) {
643  IString msg = "You cannot change the chunk sizes once set";
644  }
645  else if(numSamples < 1) {
646  msg = "Negative and zero chunk sizes are not supported, samples per chunk"
647  " cannot be [" + IString(numSamples) + "]";
648  }
649  else if(numLines < 1) {
650  msg = "Negative and zero chunk sizes are not supported, lines per chunk "
651  "cannot be [" + IString(numLines) + "]";
652  }
653  else if(numBands < 1) {
654  msg = "Negative and zero chunk sizes are not supported, lines per chunk "
655  "cannot be [" + IString(numBands) + "]";
656  }
657  else {
658  success = true;
659  }
660 
661  if(success) {
662  m_samplesInChunk = numSamples;
663  m_linesInChunk = numLines;
664  m_bandsInChunk = numBands;
665 
666  if(m_dataIsOnDiskMap) {
667  m_dataFile->resize(getDataStartByte() + getDataSize());
668  }
669  else if(m_dataFile->size() < getDataStartByte() + getDataSize()) {
670  success = false;
671  msg = "File size [" + IString((BigInt)m_dataFile->size()) +
672  " bytes] not big enough to hold data [" +
673  IString(getDataStartByte() + getDataSize()) + " bytes] where the "
674  "offset to the cube data is [" + IString(getDataStartByte()) +
675  " bytes]";
676  }
677  }
678  else {
679  throw IException(IException::Programmer, msg, _FILEINFO_);
680  }
681  }
682 
683 
690  void CubeIoHandler::blockUntilThreadPoolEmpty() const {
691  if (m_ioThreadPool) {
692  QMutexLocker lock(m_writeThreadMutex);
693  }
694  }
695 
696 
704  bool CubeIoHandler::bufferLessThan(Buffer * const &lhs, Buffer * const &rhs) {
705  bool lessThan = false;
706 
707  // If there is any overlap then we need to return false due to it being
708  // ambiguous.
709  Area3D lhsArea(
710  Displacement(lhs->Sample(), Displacement::Pixels),
711  Displacement(lhs->Line(), Displacement::Pixels),
712  Displacement(lhs->Band(), Displacement::Pixels),
713  Distance(lhs->SampleDimension() - 1, Distance::Pixels),
714  Distance(lhs->LineDimension() - 1, Distance::Pixels),
715  Distance(lhs->BandDimension() - 1, Distance::Pixels));
716  Area3D rhsArea(
717  Displacement(rhs->Sample(), Displacement::Pixels),
718  Displacement(rhs->Line(), Displacement::Pixels),
719  Displacement(rhs->Band(), Displacement::Pixels),
720  Distance(rhs->SampleDimension() - 1, Distance::Pixels),
721  Distance(rhs->LineDimension() - 1, Distance::Pixels),
722  Distance(rhs->BandDimension() - 1, Distance::Pixels));
723 
724  if (!lhsArea.intersect(rhsArea).isValid()) {
725  if (lhs->Band() != rhs->Band()) {
726  lessThan = lhs->Band() < rhs->Band();
727  }
728  else if (lhs->Line() != rhs->Line()) {
729  lessThan = lhs->Line() < rhs->Line();
730  }
731  else if (lhs->Sample() != rhs->Sample()) {
732  lessThan = lhs->Sample() < rhs->Sample();
733  }
734  }
735 
736  return lessThan;
737  }
738 
739 
753  QPair< QList<RawCubeChunk *>, QList<int> > CubeIoHandler::findCubeChunks(int startSample,
754  int numSamples, int startLine, int numLines, int startBand,
755  int numBands) const {
756  QList<RawCubeChunk *> results;
757  QList<int> resultBands;
758 /************************************************************************CHANGED THIS!!!!!!!!******/
759  int lastBand = startBand + numBands - 1;
760 // int lastBand = min(startBand + numBands - 1,
761 // bandCount());
762 /************************************************************************CHANGED THIS!!!!!!!!******/
763 
764  QRect areaInBand(
765  QPoint(max(startSample, 1),
766  max(startLine, 1)),
767  QPoint(min(startSample + numSamples - 1,
768  sampleCount()),
769  min(startLine + numLines - 1,
770  lineCount())));
771 
772  // We are considering only 1 band at a time.. we can't use m_bandsInChunk
773  // because of virtual bands, but every extra loop should not need extra
774  // IO.
775  for(int band = startBand; band <= lastBand; band ++) {
776  // This is the user-requested area in this band
777  QRect areaLeftInBand(areaInBand);
778 
779  int actualBand = band;
780 
781  if(m_virtualBands) {
782  if (band < 1 || band > m_virtualBands->size())
783  actualBand = 0;
784  else
785  actualBand = (m_virtualBands->at(band - 1) - 1) / m_bandsInChunk + 1;
786  }
787  // We will be consuming areaLeftInBand until we've got all of the area
788  // requested.
789  while(!areaLeftInBand.isEmpty()) {
837  int areaStartLine = areaLeftInBand.top();
838  int areaStartSample = areaLeftInBand.left();
839 
840  int initialChunkXPos = (areaStartSample - 1) / m_samplesInChunk;
841  int initialChunkYPos = (areaStartLine - 1) / m_linesInChunk;
842  int initialChunkZPos = (actualBand - 1) / m_bandsInChunk;
843  int initialChunkBand = initialChunkZPos * m_bandsInChunk + 1;
844 
845 
846  QRect chunkRect(initialChunkXPos * m_samplesInChunk + 1,
847  initialChunkYPos * m_linesInChunk + 1,
848  m_samplesInChunk, m_linesInChunk);
849 
850  // The chunk rectangle must intersect the remaining area that is in the
851  // current band, and the chunk's initial band must be between 1 and the
852  // number of physical bands in the cube (inclusive).
853  while(chunkRect.intersects(areaLeftInBand) &&
854  (initialChunkBand >= 1 && initialChunkBand <= bandCount())) {
855  int chunkXPos = (chunkRect.left() - 1) / m_samplesInChunk;
856  int chunkYPos = (chunkRect.top() - 1) / m_linesInChunk;
857  int chunkZPos = initialChunkZPos;
858 
859  // We now have an X,Y,Z position for the chunk. What's its index?
860  int chunkIndex = chunkXPos +
861  (chunkYPos * getChunkCountInSampleDimension()) +
862  (chunkZPos * getChunkCountInSampleDimension() *
863  getChunkCountInLineDimension());
864 
865  RawCubeChunk * newChunk = getChunk(chunkIndex, true);
866 
867  results.append(newChunk);
868  resultBands.append(band);
869 
870  chunkRect.moveLeft(chunkRect.right() + 1);
871  }
872 
873  areaLeftInBand.setTop(chunkRect.bottom() + 1);
874  }
875  }
876 
877  return QPair< QList<RawCubeChunk *>, QList<int> >(results, resultBands);
878  }
879 
880 
898  void CubeIoHandler::findIntersection(
899  const RawCubeChunk &cube1, const Buffer &cube2,
900  int &startX, int &startY, int &startZ,
901  int &endX, int &endY, int &endZ) const {
902  // So we have 2 3D "cubes" (not Cube cubes but 3d areas) we need to
903  // intersect in order to figure out what chunk data goes into the output
904  // buffer.
905 
906  // To find the band range we actually have to map all of the bands from
907  // virtual bands to physical bands
908  int startVBand = cube2.Band();
909  int endVBand = startVBand + cube2.BandDimension() - 1;
910 
911  int startPhysicalBand = 0;
912  int endPhysicalBand = 0;
913 
914  bool startVBandFound = false;
915  for(int virtualBand = startVBand; virtualBand <= endVBand; virtualBand ++) {
916  int physicalBand = virtualBand;
917 
918  bool bandExists = true;
919  if(m_virtualBands) {
920  if (virtualBand < 1 || virtualBand > m_virtualBands->size())
921  bandExists = false;
922  else {
923  physicalBand = m_virtualBands->at(virtualBand - 1);
924  }
925  }
926 
927  if (bandExists) {
928  if(!startVBandFound) {
929  startPhysicalBand = physicalBand;
930  endPhysicalBand = physicalBand;
931  startVBandFound = true;
932  }
933  else {
934  if(physicalBand < startPhysicalBand)
935  startPhysicalBand = physicalBand;
936 
937  if(physicalBand > endPhysicalBand)
938  endPhysicalBand = physicalBand;
939  }
940  }
941  }
942 
943  startX = max(cube1.getStartSample(), cube2.Sample());
944  startY = max(cube1.getStartLine(), cube2.Line());
945  startZ = max(cube1.getStartBand(), startPhysicalBand);
946  endX = min(cube1.getStartSample() + cube1.sampleCount() - 1,
947  cube2.Sample() + cube2.SampleDimension() - 1);
948  endY = min(cube1.getStartLine() + cube1.lineCount() - 1,
949  cube2.Line() + cube2.LineDimension() - 1);
950  endZ = min(cube1.getStartBand() + cube1.bandCount() - 1,
951  endPhysicalBand);
952  }
953 
954 
963  void CubeIoHandler::flushWriteCache(bool force) const {
964  if (m_ioThreadPool) {
965  bool shouldFlush = m_writeCache->second.size() >= m_idealFlushSize ||
966  force;
967  bool cacheOverflowing =
968  (m_writeCache->second.size() > m_idealFlushSize * 10);
969  bool shouldAndCanFlush = false;
970  bool forceStart = force;
971 
972  if (shouldFlush) {
973  shouldAndCanFlush = m_writeThreadMutex->tryLock();
974  if (shouldAndCanFlush) {
975  m_writeThreadMutex->unlock();
976  }
977  }
978 
979  if (cacheOverflowing && !shouldAndCanFlush) {
980  forceStart = true;
981  m_consecutiveOverflowCount++;
982  }
983 
984  if (forceStart) {
985  blockUntilThreadPoolEmpty();
986 
987  if (m_writeCache->second.size() != 0) {
988  m_idealFlushSize = m_writeCache->second.size();
989  shouldFlush = true;
990  shouldAndCanFlush = true;
991  }
992  }
993  else if (!cacheOverflowing && shouldAndCanFlush) {
994  m_consecutiveOverflowCount = 0;
995  }
996 
997  if (cacheOverflowing && m_useOptimizedCubeWrite) {
998  blockUntilThreadPoolEmpty();
999 
1000  // If the process is very I/O bound, then write caching isn't helping
1001  // anything. In fact, it hurts, so turn it off.
1002  if (m_consecutiveOverflowCount > 10) {
1003  delete m_ioThreadPool;
1004  m_ioThreadPool = NULL;
1005  }
1006 
1007  // Write it all synchronously.
1008  foreach (Buffer *bufferToWrite, m_writeCache->second) {
1009  const_cast<CubeIoHandler *>(this)->synchronousWrite(*bufferToWrite);
1010  delete bufferToWrite;
1011  }
1012 
1013  m_writeCache->second.clear();
1014  }
1015 
1016  if (shouldAndCanFlush && m_ioThreadPool) {
1017  QMutexLocker locker(m_writeCache->first);
1018  QRunnable *writer = new BufferToChunkWriter(
1019  const_cast<CubeIoHandler *>(this), m_writeCache->second);
1020 
1021  m_ioThreadPool->start(writer);
1022 
1023  m_writeCache->second.clear();
1024  m_lastOperationWasWrite = true;
1025  }
1026 
1027  if (force) {
1028  blockUntilThreadPoolEmpty();
1029  }
1030  }
1031  }
1032 
1033 
1040  void CubeIoHandler::freeChunk(RawCubeChunk *chunkToFree) const {
1041  if(chunkToFree && m_rawData) {
1042  int chunkIndex = getChunkIndex(*chunkToFree);
1043 
1044  m_rawData->erase(m_rawData->find(chunkIndex));
1045 
1046  if(chunkToFree->isDirty())
1047  (const_cast<CubeIoHandler *>(this))->writeRaw(*chunkToFree);
1048 
1049  delete chunkToFree;
1050 
1051  if(m_lastProcessByLineChunks) {
1052  delete m_lastProcessByLineChunks;
1053  m_lastProcessByLineChunks = NULL;
1054  }
1055  }
1056  }
1057 
1058 
1068  RawCubeChunk *CubeIoHandler::getChunk(int chunkIndex,
1069  bool allocateIfNecessary) const {
1070  RawCubeChunk *chunk = NULL;
1071 
1072  if(m_rawData) {
1073  chunk = m_rawData->value(chunkIndex);
1074  }
1075 
1076  if(allocateIfNecessary && !chunk) {
1077  if(m_dataIsOnDiskMap && !(*m_dataIsOnDiskMap)[chunkIndex]) {
1078  chunk = getNullChunk(chunkIndex);
1079  (*m_dataIsOnDiskMap)[chunkIndex] = true;
1080  }
1081  else {
1082  int startSample;
1083  int startLine;
1084  int startBand;
1085  int endSample;
1086  int endLine;
1087  int endBand;
1088  getChunkPlacement(chunkIndex, startSample, startLine, startBand,
1089  endSample, endLine, endBand);
1090  chunk = new RawCubeChunk(startSample, startLine, startBand,
1091  endSample, endLine, endBand,
1092  getBytesPerChunk());
1093 
1094  (const_cast<CubeIoHandler *>(this))->readRaw(*chunk);
1095  chunk->setDirty(false);
1096  }
1097 
1098  (*m_rawData)[chunkIndex] = chunk;
1099  }
1100 
1101  return chunk;
1102  }
1103 
1104 
1109  int CubeIoHandler::getChunkCount() const {
1110  return getChunkCountInSampleDimension() *
1111  getChunkCountInLineDimension() *
1112  getChunkCountInBandDimension();
1113  }
1114 
1115 
1127  void CubeIoHandler::getChunkPlacement(int chunkIndex,
1128  int &startSample, int &startLine, int &startBand,
1129  int &endSample, int &endLine, int &endBand) const {
1130  int chunkSampleIndex = chunkIndex % getChunkCountInSampleDimension();
1131 
1132  chunkIndex =
1133  (chunkIndex - chunkSampleIndex) / getChunkCountInSampleDimension();
1134 
1135  int chunkLineIndex = chunkIndex % getChunkCountInLineDimension();
1136  chunkIndex = (chunkIndex - chunkLineIndex) / getChunkCountInLineDimension();
1137 
1138  int chunkBandIndex = chunkIndex;
1139 
1140  startSample = chunkSampleIndex * getSampleCountInChunk() + 1;
1141  endSample = startSample + getSampleCountInChunk() - 1;
1142  startLine = chunkLineIndex * getLineCountInChunk() + 1;
1143  endLine = startLine + getLineCountInChunk() - 1;
1144  startBand = chunkBandIndex * getBandCountInChunk() + 1;
1145  endBand = startBand + getBandCountInChunk() - 1;
1146  }
1147 
1148 
1159  RawCubeChunk *CubeIoHandler::getNullChunk(int chunkIndex) const {
1160  // Shouldn't ask for null chunks when the area has already been allocated
1161 // ASSERT(getChunk(chunkIndex) == NULL);
1162 
1163  int startSample = 0;
1164  int startLine = 0;
1165  int startBand = 0;
1166 
1167  int endSample = 0;
1168  int endLine = 0;
1169  int endBand = 0;
1170 
1171  getChunkPlacement(chunkIndex, startSample, startLine, startBand,
1172  endSample, endLine, endBand);
1173 
1174  RawCubeChunk *result = new RawCubeChunk(startSample, startLine, startBand,
1175  endSample, endLine, endBand, getBytesPerChunk());
1176 
1177  if(!m_nullChunkData) {
1178  // the pixel type doesn't really matter, so pick something small
1179  Brick nullBuffer(result->sampleCount(),
1180  result->lineCount(),
1181  result->bandCount(),
1182  UnsignedByte);
1183 
1184  nullBuffer.SetBasePosition(result->getStartSample(),
1185  result->getStartLine(),
1186  result->getStartBand());
1187  for(int i = 0; i < nullBuffer.size(); i++) {
1188  nullBuffer[i] = Null;
1189  }
1190 
1191  writeIntoRaw(nullBuffer, *result, result->getStartBand());
1192  m_nullChunkData = new QByteArray(result->getRawData());
1193  }
1194  else {
1195  result->setRawData(*m_nullChunkData);
1196  }
1197 
1198  result->setDirty(true);
1199  return result;
1200  }
1201 
1202 
1212  void CubeIoHandler::minimizeCache(const QList<RawCubeChunk *> &justUsed,
1213  const Buffer &justRequested) const {
1214  // Since we have a lock on the cache, no newly created threads can utilize
1215  // or access any cache data until we're done.
1216  if (m_rawData->size() * getBytesPerChunk() > 1 * 1024 * 1024 ||
1217  m_cachingAlgorithms->size() > 1) {
1218  bool algorithmAccepted = false;
1219 
1220  int algorithmIndex = 0;
1221  while(!algorithmAccepted &&
1222  algorithmIndex < m_cachingAlgorithms->size()) {
1223  CubeCachingAlgorithm *algorithm = (*m_cachingAlgorithms)[algorithmIndex];
1224 
1226  algorithm->recommendChunksToFree(m_rawData->values(), justUsed,
1227  justRequested);
1228 
1229  algorithmAccepted = result.algorithmUnderstoodData();
1230 
1231  if(algorithmAccepted) {
1232  QList<RawCubeChunk *> chunksToFree = result.getChunksToFree();
1233 
1234  RawCubeChunk *chunkToFree;
1235  foreach(chunkToFree, chunksToFree) {
1236  freeChunk(chunkToFree);
1237  }
1238  }
1239 
1240  algorithmIndex ++;
1241  }
1242 
1243  // Fall back - no algorithms liked us :(
1244  if(!algorithmAccepted && m_rawData->size() > 100) {
1245  // This (minimizeCache()) is typically executed in the Runnable thread.
1246  // We don't want to wait for ourselves.
1247  clearCache(false);
1248  }
1249  }
1250  }
1251 
1252 
1261  void CubeIoHandler::synchronousWrite(const Buffer &bufferToWrite) {
1262  QList<RawCubeChunk *> cubeChunks;
1263  QList<int> cubeChunkBands;
1264 
1265  int bufferSampleCount = bufferToWrite.SampleDimension();
1266  int bufferLineCount = bufferToWrite.LineDimension();
1267  int bufferBandCount = bufferToWrite.BandDimension();
1268 
1269  // process by line optimization
1270  if(m_lastProcessByLineChunks &&
1271  m_lastProcessByLineChunks->size()) {
1272  // Not optimized yet, let's see if we can optimize
1273  if(bufferToWrite.Sample() == 1 &&
1274  bufferSampleCount == sampleCount() &&
1275  bufferLineCount == 1 &&
1276  bufferBandCount == 1) {
1277  // We look like a process by line, are we using the same chunks as
1278  // before?
1279  int bufferLine = bufferToWrite.Line();
1280  int chunkStartLine =
1281  (*m_lastProcessByLineChunks)[0]->getStartLine();
1282  int chunkLines =
1283  (*m_lastProcessByLineChunks)[0]->lineCount();
1284  int bufferBand = bufferToWrite.Band();
1285  int chunkStartBand =
1286  (*m_lastProcessByLineChunks)[0]->getStartBand();
1287  int chunkBands =
1288  (*m_lastProcessByLineChunks)[0]->bandCount();
1289 
1290  if(bufferLine >= chunkStartLine &&
1291  bufferLine <= chunkStartLine + chunkLines - 1 &&
1292  bufferBand >= chunkStartBand &&
1293  bufferBand <= chunkStartBand + chunkBands - 1) {
1294  cubeChunks = *m_lastProcessByLineChunks;
1295  for (int i = 0; i < cubeChunks.size(); i++) {
1296  cubeChunkBands.append( cubeChunks[i]->getStartBand() );
1297  }
1298  }
1299  }
1300  }
1301  // Processing by chunk size
1302  else if (bufferSampleCount == m_samplesInChunk &&
1303  bufferLineCount == m_linesInChunk &&
1304  bufferBandCount == m_bandsInChunk) {
1305  int bufferStartSample = bufferToWrite.Sample();
1306  int bufferStartLine = bufferToWrite.Line();
1307  int bufferStartBand = bufferToWrite.Band();
1308 
1309  int bufferEndSample = bufferStartSample + bufferSampleCount - 1;
1310  int bufferEndLine = bufferStartLine + bufferLineCount - 1;
1311  int bufferEndBand = bufferStartBand + bufferBandCount - 1;
1312 
1313  int expectedChunkIndex =
1314  ((bufferStartSample - 1) / getSampleCountInChunk()) +
1315  ((bufferStartLine - 1) / getLineCountInChunk()) *
1316  getChunkCountInSampleDimension() +
1317  ((bufferStartBand - 1) / getBandCountInChunk()) *
1318  getChunkCountInSampleDimension() *
1319  getChunkCountInLineDimension();
1320 
1321  int chunkStartSample, chunkStartLine, chunkStartBand,
1322  chunkEndSample, chunkEndLine, chunkEndBand;
1323 
1324  getChunkPlacement(expectedChunkIndex,
1325  chunkStartSample, chunkStartLine, chunkStartBand,
1326  chunkEndSample, chunkEndLine, chunkEndBand);
1327 
1328  if (chunkStartSample == bufferStartSample &&
1329  chunkStartLine == bufferStartLine &&
1330  chunkStartBand == bufferStartBand &&
1331  chunkEndSample == bufferEndSample &&
1332  chunkEndLine == bufferEndLine &&
1333  chunkEndBand == bufferEndBand) {
1334  cubeChunks.append(getChunk(expectedChunkIndex, true));
1335  cubeChunkBands.append(cubeChunks.last()->getStartBand());
1336  }
1337  }
1338 
1340  if(cubeChunks.empty()) {
1341  chunkInfo = findCubeChunks(
1342  bufferToWrite.Sample(), bufferSampleCount,
1343  bufferToWrite.Line(), bufferLineCount,
1344  bufferToWrite.Band(), bufferBandCount);
1345  cubeChunks = chunkInfo.first;
1346  cubeChunkBands = chunkInfo.second;
1347  }
1348 
1349  // process by line optimization
1350  if(bufferToWrite.Sample() == 1 &&
1351  bufferSampleCount == sampleCount() &&
1352  bufferLineCount == 1 &&
1353  bufferBandCount == 1) {
1354  if(!m_lastProcessByLineChunks)
1355  m_lastProcessByLineChunks =
1356  new QList<RawCubeChunk *>(cubeChunks);
1357  else
1358  *m_lastProcessByLineChunks = cubeChunks;
1359  }
1360 
1361  for(int i = 0; i < cubeChunks.size(); i++) {
1362  writeIntoRaw(bufferToWrite, *cubeChunks[i], cubeChunkBands[i]);
1363  }
1364 
1365  minimizeCache(cubeChunks, bufferToWrite);
1366  }
1367 
1368 
1376  void CubeIoHandler::writeIntoDouble(const RawCubeChunk &chunk,
1377  Buffer &output, int index) const {
1378  // The code in this method is highly optimized. Even the order of the if
1379  // statements will have a significant impact on performance if changed.
1380  // Also, there is a lot of duplicate code in both writeIntoDouble(...) and
1381  // writeIntoRaw(...). This is needed for performance gain. Any function
1382  // or method calls from within the x loop cause significant performance
1383  // decreases.
1384  int startX = 0;
1385  int startY = 0;
1386  int startZ = 0;
1387 
1388  int endX = 0;
1389  int endY = 0;
1390  int endZ = 0;
1391 
1392  findIntersection(chunk, output, startX, startY, startZ, endX, endY, endZ);
1393 
1394  int bufferBand = output.Band();
1395  int bufferBands = output.BandDimension();
1396  int chunkStartSample = chunk.getStartSample();
1397  int chunkStartLine = chunk.getStartLine();
1398  int chunkStartBand = chunk.getStartBand();
1399  int chunkLineSize = chunk.sampleCount();
1400  int chunkBandSize = chunkLineSize * chunk.lineCount();
1401  //double *buffersDoubleBuf = output.p_buf;
1402  double *buffersDoubleBuf = output.DoubleBuffer();
1403  const char *chunkBuf = chunk.getRawData().data();
1404  char *buffersRawBuf = (char *)output.RawBuffer();
1405 
1406  for(int z = startZ; z <= endZ; z++) {
1407  const int &bandIntoChunk = z - chunkStartBand;
1408  int virtualBand = z;
1409 
1410  virtualBand = index;
1411 
1412  if(virtualBand != 0 && virtualBand >= bufferBand &&
1413  virtualBand <= bufferBand + bufferBands - 1) {
1414 
1415  for(int y = startY; y <= endY; y++) {
1416  const int &lineIntoChunk = y - chunkStartLine;
1417  int bufferIndex = output.Index(startX, y, virtualBand);
1418 
1419  for(int x = startX; x <= endX; x++) {
1420  const int &sampleIntoChunk = x - chunkStartSample;
1421 
1422  const int &chunkIndex = sampleIntoChunk +
1423  (chunkLineSize * lineIntoChunk) +
1424  (chunkBandSize * bandIntoChunk);
1425 
1426  double &bufferVal = buffersDoubleBuf[bufferIndex];
1427 
1428  if(m_pixelType == Real) {
1429  float raw = ((float *)chunkBuf)[chunkIndex];
1430  if(m_byteSwapper)
1431  raw = m_byteSwapper->Float(&raw);
1432 
1433  if(raw >= VALID_MIN4) {
1434  bufferVal = (double) raw;
1435  }
1436  else {
1437  if(raw == NULL4)
1438  bufferVal = NULL8;
1439  else if(raw == LOW_INSTR_SAT4)
1440  bufferVal = LOW_INSTR_SAT8;
1441  else if(raw == LOW_REPR_SAT4)
1442  bufferVal = LOW_REPR_SAT8;
1443  else if(raw == HIGH_INSTR_SAT4)
1444  bufferVal = HIGH_INSTR_SAT8;
1445  else if(raw == HIGH_REPR_SAT4)
1446  bufferVal = HIGH_REPR_SAT8;
1447  else
1448  bufferVal = LOW_REPR_SAT8;
1449  }
1450 
1451  ((float *)buffersRawBuf)[bufferIndex] = raw;
1452  }
1453 
1454  else if(m_pixelType == SignedWord) {
1455  short raw = ((short *)chunkBuf)[chunkIndex];
1456  if(m_byteSwapper)
1457  raw = m_byteSwapper->ShortInt(&raw);
1458 
1459  if(raw >= VALID_MIN2) {
1460  bufferVal = (double) raw * m_multiplier + m_base;
1461  }
1462  else {
1463  if(raw == NULL2)
1464  bufferVal = NULL8;
1465  else if(raw == LOW_INSTR_SAT2)
1466  bufferVal = LOW_INSTR_SAT8;
1467  else if(raw == LOW_REPR_SAT2)
1468  bufferVal = LOW_REPR_SAT8;
1469  else if(raw == HIGH_INSTR_SAT2)
1470  bufferVal = HIGH_INSTR_SAT8;
1471  else if(raw == HIGH_REPR_SAT2)
1472  bufferVal = HIGH_REPR_SAT8;
1473  else
1474  bufferVal = LOW_REPR_SAT8;
1475  }
1476 
1477  ((short *)buffersRawBuf)[bufferIndex] = raw;
1478  }
1479 
1480 
1481  else if(m_pixelType == UnsignedWord) {
1482  unsigned short raw = ((unsigned short *)chunkBuf)[chunkIndex];
1483  if(m_byteSwapper)
1484  raw = m_byteSwapper->UnsignedShortInt(&raw);
1485 
1486  if(raw >= VALID_MINU2) {
1487  bufferVal = (double) raw * m_multiplier + m_base;
1488  }
1489  else if (raw > VALID_MAXU2) {
1490  if(raw == HIGH_INSTR_SATU2)
1491  bufferVal = HIGH_INSTR_SAT8;
1492  else if(raw == HIGH_REPR_SATU2)
1493  bufferVal = HIGH_REPR_SAT8;
1494  else
1495  bufferVal = LOW_REPR_SAT8;
1496  }
1497  else {
1498  if(raw == NULLU2)
1499  bufferVal = NULL8;
1500  else if(raw == LOW_INSTR_SATU2)
1501  bufferVal = LOW_INSTR_SAT8;
1502  else if(raw == LOW_REPR_SATU2)
1503  bufferVal = LOW_REPR_SAT8;
1504  else
1505  bufferVal = LOW_REPR_SAT8;
1506  }
1507 
1508  ((unsigned short *)buffersRawBuf)[bufferIndex] = raw;
1509  }
1510 
1511  else if(m_pixelType == UnsignedInteger) {
1512 
1513  unsigned int raw = ((unsigned int *)chunkBuf)[chunkIndex];
1514  if(m_byteSwapper)
1515  raw = m_byteSwapper->Uint32_t(&raw);
1516 
1517  if(raw >= VALID_MINUI4) {
1518  bufferVal = (double) raw * m_multiplier + m_base;
1519  }
1520  else if (raw > VALID_MAXUI4) {
1521  if(raw == HIGH_INSTR_SATUI4)
1522  bufferVal = HIGH_INSTR_SAT8;
1523  else if(raw == HIGH_REPR_SATUI4)
1524  bufferVal = HIGH_REPR_SAT8;
1525  else
1526  bufferVal = LOW_REPR_SAT8;
1527  }
1528  else {
1529  if(raw == NULLUI4)
1530  bufferVal = NULL8;
1531  else if(raw == LOW_INSTR_SATUI4)
1532  bufferVal = LOW_INSTR_SAT8;
1533  else if(raw == LOW_REPR_SATUI4)
1534  bufferVal = LOW_REPR_SAT8;
1535  else
1536  bufferVal = LOW_REPR_SAT8;
1537  }
1538 
1539  ((unsigned int *)buffersRawBuf)[bufferIndex] = raw;
1540 
1541 
1542 
1543  }
1544 
1545  else if(m_pixelType == UnsignedByte) {
1546  unsigned char raw = ((unsigned char *)chunkBuf)[chunkIndex];
1547 
1548  if(raw == NULL1) {
1549  bufferVal = NULL8;
1550  }
1551  else if(raw == HIGH_REPR_SAT1) {
1552  bufferVal = HIGH_REPR_SAT8;
1553  }
1554  else {
1555  bufferVal = (double) raw * m_multiplier + m_base;
1556  }
1557 
1558  ((unsigned char *)buffersRawBuf)[bufferIndex] = raw;
1559  }
1560 
1561  bufferIndex ++;
1562  }
1563  }
1564  }
1565  }
1566  }
1567 
1568 
1576  void CubeIoHandler::writeIntoRaw(const Buffer &buffer, RawCubeChunk &output, int index)
1577  const {
1578  // The code in this method is highly optimized. Even the order of the if
1579  // statements will have a significant impact on performance if changed.
1580  // Also, there is a lot of duplicate code in both writeIntoDouble(...) and
1581  // writeIntoRaw(...). This is needed for performance gain. Any function
1582  // or method calls from within the x loop cause significant performance
1583  // decreases.
1584  int startX = 0;
1585  int startY = 0;
1586  int startZ = 0;
1587 
1588  int endX = 0;
1589  int endY = 0;
1590  int endZ = 0;
1591 
1592  output.setDirty(true);
1593  findIntersection(output, buffer, startX, startY, startZ, endX, endY, endZ);
1594 
1595  int bufferBand = buffer.Band();
1596  int bufferBands = buffer.BandDimension();
1597  int outputStartSample = output.getStartSample();
1598  int outputStartLine = output.getStartLine();
1599  int outputStartBand = output.getStartBand();
1600  int lineSize = output.sampleCount();
1601  int bandSize = lineSize * output.lineCount();
1602  double *buffersDoubleBuf = buffer.DoubleBuffer();
1603  char *chunkBuf = output.getRawData().data();
1604 
1605  for(int z = startZ; z <= endZ; z++) {
1606  const int &bandIntoChunk = z - outputStartBand;
1607  int virtualBand = index;
1608 
1609 
1610  if(m_virtualBands) {
1611  virtualBand = m_virtualBands->indexOf(virtualBand) + 1;
1612  }
1613 
1614  if(virtualBand != 0 && virtualBand >= bufferBand &&
1615  virtualBand <= bufferBand + bufferBands - 1) {
1616 
1617  for(int y = startY; y <= endY; y++) {
1618  const int &lineIntoChunk = y - outputStartLine;
1619  int bufferIndex = buffer.Index(startX, y, virtualBand);
1620 
1621  for(int x = startX; x <= endX; x++) {
1622  const int &sampleIntoChunk = x - outputStartSample;
1623 
1624  const int &chunkIndex = sampleIntoChunk +
1625  (lineSize * lineIntoChunk) + (bandSize * bandIntoChunk);
1626 
1627  double bufferVal = buffersDoubleBuf[bufferIndex];
1628 
1629  if(m_pixelType == Real) {
1630  float raw = 0;
1631 
1632  if(bufferVal >= VALID_MIN8) {
1633  double filePixelValueDbl = (bufferVal - m_base) /
1634  m_multiplier;
1635 
1636  if(filePixelValueDbl < (double) VALID_MIN4) {
1637  raw = LOW_REPR_SAT4;
1638  }
1639  else if(filePixelValueDbl > (double) VALID_MAX4) {
1640  raw = HIGH_REPR_SAT4;
1641  }
1642  else {
1643  raw = (float) filePixelValueDbl;
1644  }
1645  }
1646  else {
1647  if(bufferVal == NULL8)
1648  raw = NULL4;
1649  else if(bufferVal == LOW_INSTR_SAT8)
1650  raw = LOW_INSTR_SAT4;
1651  else if(bufferVal == LOW_REPR_SAT8)
1652  raw = LOW_REPR_SAT4;
1653  else if(bufferVal == HIGH_INSTR_SAT8)
1654  raw = HIGH_INSTR_SAT4;
1655  else if(bufferVal == HIGH_REPR_SAT8)
1656  raw = HIGH_REPR_SAT4;
1657  else
1658  raw = LOW_REPR_SAT4;
1659  }
1660  ((float *)chunkBuf)[chunkIndex] =
1661  m_byteSwapper ? m_byteSwapper->Float(&raw) : raw;
1662  }
1663 
1664  else if(m_pixelType == SignedWord) {
1665  short raw;
1666 
1667  if(bufferVal >= VALID_MIN8) {
1668  double filePixelValueDbl = (bufferVal - m_base) /
1669  m_multiplier;
1670  if(filePixelValueDbl < VALID_MIN2 - 0.5) {
1671  raw = LOW_REPR_SAT2;
1672  }
1673  if(filePixelValueDbl > VALID_MAX2 + 0.5) {
1674  raw = HIGH_REPR_SAT2;
1675  }
1676  else {
1677  int filePixelValue = (int)round(filePixelValueDbl);
1678 
1679  if(filePixelValue < VALID_MIN2) {
1680  raw = LOW_REPR_SAT2;
1681  }
1682  else if(filePixelValue > VALID_MAX2) {
1683  raw = HIGH_REPR_SAT2;
1684  }
1685  else {
1686  raw = filePixelValue;
1687  }
1688  }
1689  }
1690  else {
1691  if(bufferVal == NULL8)
1692  raw = NULL2;
1693  else if(bufferVal == LOW_INSTR_SAT8)
1694  raw = LOW_INSTR_SAT2;
1695  else if(bufferVal == LOW_REPR_SAT8)
1696  raw = LOW_REPR_SAT2;
1697  else if(bufferVal == HIGH_INSTR_SAT8)
1698  raw = HIGH_INSTR_SAT2;
1699  else if(bufferVal == HIGH_REPR_SAT8)
1700  raw = HIGH_REPR_SAT2;
1701  else
1702  raw = LOW_REPR_SAT2;
1703  }
1704 
1705  ((short *)chunkBuf)[chunkIndex] =
1706  m_byteSwapper ? m_byteSwapper->ShortInt(&raw) : raw;
1707  }
1708 
1709  else if(m_pixelType == UnsignedInteger) {
1710 
1711  unsigned int raw;
1712 
1713  if(bufferVal >= VALID_MINUI4) {
1714  double filePixelValueDbl = (bufferVal - m_base) /
1715  m_multiplier;
1716  if(filePixelValueDbl < VALID_MINUI4 - 0.5) {
1717  raw = LOW_REPR_SATUI4;
1718  }
1719  if(filePixelValueDbl > VALID_MAXUI4) {
1720  raw = HIGH_REPR_SATUI4;
1721  }
1722  else {
1723  unsigned int filePixelValue = (unsigned int)round(filePixelValueDbl);
1724 
1725  if(filePixelValue < VALID_MINUI4) {
1726  raw = LOW_REPR_SATUI4;
1727  }
1728  else if(filePixelValue > VALID_MAXUI4) {
1729  raw = HIGH_REPR_SATUI4;
1730  }
1731  else {
1732  raw = filePixelValue;
1733  }
1734  }
1735  }
1736  else {
1737  if(bufferVal == NULL8)
1738  raw = NULLUI4;
1739  else if(bufferVal == LOW_INSTR_SAT8)
1740  raw = LOW_INSTR_SATUI4;
1741  else if(bufferVal == LOW_REPR_SAT8)
1742  raw = LOW_REPR_SATUI4;
1743  else if(bufferVal == HIGH_INSTR_SAT8)
1744  raw = HIGH_INSTR_SATUI4;
1745  else if(bufferVal == HIGH_REPR_SAT8)
1746  raw = HIGH_REPR_SATUI4;
1747  else
1748  raw = LOW_REPR_SATUI4;
1749  }
1750 
1751  ((unsigned int *)chunkBuf)[chunkIndex] =
1752  m_byteSwapper ? m_byteSwapper->Uint32_t(&raw) : raw;
1753 
1754  }
1755 
1756 
1757  else if(m_pixelType == UnsignedWord) {
1758  unsigned short raw;
1759 
1760  if(bufferVal >= VALID_MIN8) {
1761  double filePixelValueDbl = (bufferVal - m_base) /
1762  m_multiplier;
1763  if(filePixelValueDbl < VALID_MINU2 - 0.5) {
1764  raw = LOW_REPR_SATU2;
1765  }
1766  if(filePixelValueDbl > VALID_MAXU2 + 0.5) {
1767  raw = HIGH_REPR_SATU2;
1768  }
1769  else {
1770  int filePixelValue = (int)round(filePixelValueDbl);
1771 
1772  if(filePixelValue < VALID_MINU2) {
1773  raw = LOW_REPR_SATU2;
1774  }
1775  else if(filePixelValue > VALID_MAXU2) {
1776  raw = HIGH_REPR_SATU2;
1777  }
1778  else {
1779  raw = filePixelValue;
1780  }
1781  }
1782  }
1783  else {
1784  if(bufferVal == NULL8)
1785  raw = NULLU2;
1786  else if(bufferVal == LOW_INSTR_SAT8)
1787  raw = LOW_INSTR_SATU2;
1788  else if(bufferVal == LOW_REPR_SAT8)
1789  raw = LOW_REPR_SATU2;
1790  else if(bufferVal == HIGH_INSTR_SAT8)
1791  raw = HIGH_INSTR_SATU2;
1792  else if(bufferVal == HIGH_REPR_SAT8)
1793  raw = HIGH_REPR_SATU2;
1794  else
1795  raw = LOW_REPR_SATU2;
1796  }
1797 
1798  ((unsigned short *)chunkBuf)[chunkIndex] =
1799  m_byteSwapper ? m_byteSwapper->UnsignedShortInt(&raw) : raw;
1800  }
1801  else if(m_pixelType == UnsignedByte) {
1802  unsigned char raw;
1803 
1804  if(bufferVal >= VALID_MIN8) {
1805  double filePixelValueDbl = (bufferVal - m_base) /
1806  m_multiplier;
1807  if(filePixelValueDbl < VALID_MIN1 - 0.5) {
1808  raw = LOW_REPR_SAT1;
1809  }
1810  else if(filePixelValueDbl > VALID_MAX1 + 0.5) {
1811  raw = HIGH_REPR_SAT1;
1812  }
1813  else {
1814  int filePixelValue = (int)(filePixelValueDbl + 0.5);
1815  if(filePixelValue < VALID_MIN1) {
1816  raw = LOW_REPR_SAT1;
1817  }
1818  else if(filePixelValue > VALID_MAX1) {
1819  raw = HIGH_REPR_SAT1;
1820  }
1821  else {
1822  raw = (unsigned char)(filePixelValue);
1823  }
1824  }
1825  }
1826  else {
1827  if(bufferVal == NULL8)
1828  raw = NULL1;
1829  else if(bufferVal == LOW_INSTR_SAT8)
1830  raw = LOW_INSTR_SAT1;
1831  else if(bufferVal == LOW_REPR_SAT8)
1832  raw = LOW_REPR_SAT1;
1833  else if(bufferVal == HIGH_INSTR_SAT8)
1834  raw = HIGH_INSTR_SAT1;
1835  else if(bufferVal == HIGH_REPR_SAT8)
1836  raw = HIGH_REPR_SAT1;
1837  else
1838  raw = LOW_REPR_SAT1;
1839  }
1840 
1841  ((unsigned char *)chunkBuf)[chunkIndex] = raw;
1842  }
1843 
1844  bufferIndex ++;
1845  }
1846  }
1847  }
1848  }
1849  }
1850 
1851 
1855  void CubeIoHandler::writeNullDataToDisk() const {
1856  if(!m_dataIsOnDiskMap) {
1857  IString msg = "Cannot call CubeIoHandler::writeNullDataToDisk unless "
1858  "data is not already on disk (Cube::Create was called)";
1859  throw IException(IException::Programmer, msg, _FILEINFO_);
1860  }
1861 
1862  int numChunks = getChunkCount();
1863  for(int i = 0; i < numChunks; i++) {
1864  if(!(*m_dataIsOnDiskMap)[i]) {
1865  RawCubeChunk *nullChunk = getNullChunk(i);
1866  (const_cast<CubeIoHandler *>(this))->writeRaw(*nullChunk);
1867  (*m_dataIsOnDiskMap)[i] = true;
1868 
1869  delete nullChunk;
1870  nullChunk = NULL;
1871  }
1872  }
1873  }
1874 
1875 
1889  CubeIoHandler::BufferToChunkWriter::BufferToChunkWriter(
1890  CubeIoHandler * ioHandler, QList<Buffer *> buffersToWrite) {
1891  m_ioHandler = ioHandler;
1892  m_buffersToWrite = new QList<Buffer *>(buffersToWrite);
1893  m_timer = new QTime;
1894  m_timer->start();
1895 
1896  m_ioHandler->m_writeThreadMutex->lock();
1897  }
1898 
1899 
1910  CubeIoHandler::BufferToChunkWriter::~BufferToChunkWriter() {
1911  int elapsedMs = m_timer->elapsed();
1912  int idealFlushElapsedTime = 100; // ms
1913 
1914  // We want to aim our flush size at 100ms, so adjust accordingly to aim at
1915  // our target. This method seems to be extremely effective because
1916  // we maximize our I/O calls when caching is interfering and normalize
1917  // them otherwise.
1918  int msOffIdeal = elapsedMs - idealFlushElapsedTime;
1919  double percentOffIdeal = msOffIdeal / (double)idealFlushElapsedTime;
1920 
1921  // flush size is bounded to [32, 5000]
1922  int currentCacheSize = m_ioHandler->m_idealFlushSize;
1923  int desiredAdjustment = -1 * currentCacheSize * percentOffIdeal;
1924  int desiredCacheSize = (int)(currentCacheSize + desiredAdjustment);
1925  m_ioHandler->m_idealFlushSize = (int)(qMin(5000,
1926  qMax(32, desiredCacheSize)));
1927 
1928  delete m_timer;
1929  m_timer = NULL;
1930 
1931  m_ioHandler->m_writeThreadMutex->unlock();
1932  m_ioHandler = NULL;
1933 
1934  ASSERT(m_buffersToWrite->isEmpty());
1935  delete m_buffersToWrite;
1936  m_buffersToWrite = NULL;
1937  }
1938 
1939 
1945  void CubeIoHandler::BufferToChunkWriter::run() {
1946  // Sorting the buffers didn't seem to have a large positive impact on speed,
1947  // but does increase complexity so it's disabled.
1948 // QList<Buffer * > buffersToWrite(*m_buffersToWrite);
1949 // qSort(buffersToWrite.begin(), buffersToWrite.end(), bufferLessThan);
1950 
1951  // If the buffers have any overlap at all then we can't sort them and still
1952  // guarantee the last write() call makes it into the correct place. The
1953  // bufferLessThan is guaranteed to return false if there is overlap.
1954 // bool sortable = true;
1955 // for (int i = 1; sortable && i < buffersToWrite.size(); i++) {
1956 // if (!bufferLessThan(buffersToWrite[i - 1], buffersToWrite[i])) {
1957 // sortable = false;
1958 // }
1959 // }
1960 
1961 // if (!sortable) {
1962 // buffersToWrite = *m_buffersToWrite;
1963 // }
1964 
1965  foreach (Buffer * bufferToWrite, *m_buffersToWrite) {
1966  m_ioHandler->synchronousWrite(*bufferToWrite);
1967  delete bufferToWrite;
1968  }
1969 
1970  m_buffersToWrite->clear();
1971  m_ioHandler->m_dataFile->flush();
1972  }
1973 }
Buffer for reading and writing cube data.
Definition: Buffer.h:69
long long int BigInt
Big int.
Definition: Constants.h:65
int getStartLine() const
Definition: RawCubeChunk.h:75
void setRawData(QByteArray rawData)
Sets the chunk&#39;s raw data.
double * DoubleBuffer() const
Returns the value of the shape buffer.
Definition: Buffer.h:154
const double Null
Value for an Isis Null pixel.
Definition: SpecialPixel.h:110
int Band(const int index=0) const
Returns the band position associated with a shape buffer index.
Definition: Buffer.cpp:178
int bandCount() const
Definition: RawCubeChunk.h:103
PvlGroupIterator findGroup(const QString &name, PvlGroupIterator beg, PvlGroupIterator end)
Find a group with the specified name, within these indexes.
Definition: PvlObject.h:141
Represents a 3D area (a 3D "cube")
Definition: Area3D.h:41
void * RawBuffer() const
Returns a void pointer to the raw buffer.
Definition: Buffer.h:167
bool isValid() const
Returns true if all of the positions of the 3D area are valid (i.e.
Definition: Area3D.cpp:505
PvlObjectIterator findObject(const QString &name, PvlObjectIterator beg, PvlObjectIterator end)
Find the index of object with a specified name, between two indexes.
Definition: PvlObject.h:286
int SizeOf(Isis::PixelType pixelType)
Returns the number of bytes of the specified PixelType.
Definition: PixelType.h:62
Buffer for containing a three dimensional section of an image.
Definition: Brick.h:61
int lineCount() const
Definition: RawCubeChunk.h:96
Namespace for the standard library.
void SetBasePosition(const int start_sample, const int start_line, const int start_band)
This method is used to set the base position of the shape buffer.
Definition: Brick.h:136
int getStartSample() const
Definition: RawCubeChunk.h:68
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
int LineDimension() const
Returns the number of lines in the shape buffer.
Definition: Buffer.h:95
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
void setDirty(bool dirty)
Sets the chunk&#39;s dirty flag, indicating whether or not the chunk&#39;s data matches the data that is on d...
Distance measurement, usually in meters.
Definition: Distance.h:47
int size() const
Returns the total number of pixels in the shape buffer.
Definition: Buffer.h:113
PixelType
Enumerations for Isis Pixel Types.
Definition: PixelType.h:43
Handles converting buffers to and from disk.
virtual CacheResult recommendChunksToFree(QList< RawCubeChunk *> allocated, QList< RawCubeChunk *> justUsed, const Buffer &justRequested)=0
Call this to determine which chunks should be freed from memory.
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
int Sample(const int index=0) const
Returns the sample position associated with a shape buffer index.
Definition: Buffer.cpp:143
Isis::PixelType PixelTypeEnumeration(const QString &type)
Returns PixelType enumeration given a string.
Definition: PixelType.h:105
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
This algorithm recommends chunks to be freed that are not within the last IO.
QList< RawCubeChunk * > getChunksToFree() const
int BandDimension() const
Returns the number of bands in the shape buffer.
Definition: Buffer.h:104
int getStartBand() const
Definition: RawCubeChunk.h:82
PvlKeyword & findKeyword(const QString &kname, FindOptions opts)
Finds a keyword in the current PvlObject, or deeper inside other PvlObjects and Pvlgroups within this...
Definition: PvlObject.cpp:148
Contains multiple PvlContainers.
Definition: PvlGroup.h:57
#define _FILEINFO_
Macro for the filename and line number.
Definition: IException.h:40
int Line(const int index=0) const
Returns the line position associated with a shape buffer index.
Definition: Buffer.cpp:161
A section of raw data on the disk.
Definition: RawCubeChunk.h:42
Byte swapper.
Definition: EndianSwapper.h:55
Container for cube-like labels.
Definition: Pvl.h:135
PvlKeyword & findKeyword(const QString &name)
Find a keyword with a specified name.
int SampleDimension() const
Returns the number of samples in the shape buffer.
Definition: Buffer.h:86
IString DownCase()
Converts all upper case letters in the object IString into lower case characters. ...
Definition: IString.cpp:659
Displacement is a signed length, usually in meters.
Definition: Displacement.h:43
int sampleCount() const
Definition: RawCubeChunk.h:89
Isis exception class.
Definition: IException.h:107
Adds specific functionality to C++ strings.
Definition: IString.h:181
This stores the results of the caching algorithm.
Namespace for ISIS/Bullet specific routines.
Definition: Apollo.h:31
Area3D intersect(const Area3D &otherArea) const
Returns the intersection of this 3D area with another 3D area.
Definition: Area3D.cpp:478
bool isDirty() const
This class is designed to handle write() asynchronously.
int Index(const int i_samp, const int i_line, const int i_band) const
Given a sample, line, and band position, this returns the appropriate index in the shape buffer...
Definition: Buffer.cpp:213
Contains Pvl Groups and Pvl Objects.
Definition: PvlObject.h:74
QByteArray & getRawData() const
Definition: RawCubeChunk.h:53
bool algorithmUnderstoodData() const
If this is true, then the results (be them empty or not) should be considered valid.
This is the parent of the caching algorithms