24 #include "IsisDebug.h" 34 #include <QListIterator> 35 #include <QMapIterator> 79 CubeIoHandler::CubeIoHandler(QFile * dataFile,
80 const QList<int> *virtualBandList,
const Pvl &label,
bool alreadyOnDisk) {
82 m_cachingAlgorithms = NULL;
83 m_dataIsOnDiskMap = NULL;
85 m_virtualBands = NULL;
86 m_nullChunkData = NULL;
87 m_lastProcessByLineChunks = NULL;
89 m_ioThreadPool = NULL;
90 m_writeThreadMutex = NULL;
94 IString msg =
"Cannot create a CubeIoHandler with a NULL data file";
101 Preference::Preferences().findGroup(
"Performance");
102 IString cubeWritePerfOpt = performancePrefs[
"CubeWriteThread"][0];
103 m_useOptimizedCubeWrite = (cubeWritePerfOpt.
DownCase() ==
"optimized");
104 if ((m_useOptimizedCubeWrite && !alreadyOnDisk) ||
105 cubeWritePerfOpt.
DownCase() ==
"always") {
106 m_ioThreadPool =
new QThreadPool;
107 m_ioThreadPool->setMaxThreadCount(1);
110 m_consecutiveOverflowCount = 0;
111 m_lastOperationWasWrite =
false;
114 m_writeCache->first =
new QMutex;
115 m_writeThreadMutex =
new QMutex;
117 m_idealFlushSize = 32;
121 m_dataFile = dataFile;
126 QString byteOrderStr = pixelGroup.
findKeyword(
"ByteOrder")[0];
128 byteOrderStr.toUpper());
130 m_multiplier = pixelGroup.
findKeyword(
"Multiplier");
136 if(!m_byteSwapper->willSwap()) {
137 delete m_byteSwapper;
138 m_byteSwapper = NULL;
146 m_startByte = (int)core.
findKeyword(
"StartByte") - 1;
148 m_samplesInChunk = -1;
156 setVirtualBands(virtualBandList);
159 IString msg =
"Constructing CubeIoHandler failed";
163 IString msg =
"Constructing CubeIoHandler failed";
173 CubeIoHandler::~CubeIoHandler() {
174 ASSERT( m_rawData ? m_rawData->size() == 0 : 1 );
177 m_ioThreadPool->waitForDone();
179 delete m_ioThreadPool;
180 m_ioThreadPool = NULL;
182 delete m_dataIsOnDiskMap;
183 m_dataIsOnDiskMap = NULL;
185 if (m_cachingAlgorithms) {
186 QListIterator<CubeCachingAlgorithm *> it(*m_cachingAlgorithms);
187 while (it.hasNext()) {
190 delete m_cachingAlgorithms;
191 m_cachingAlgorithms = NULL;
195 QMapIterator<int, RawCubeChunk *> it(*m_rawData);
196 while (it.hasNext()) {
209 delete m_writeCache->first;
210 m_writeCache->first = NULL;
212 for (
int i = 0; i < m_writeCache->second.size(); i++) {
213 delete m_writeCache->second[i];
215 m_writeCache->second.clear();
221 delete m_byteSwapper;
222 m_byteSwapper = NULL;
224 delete m_virtualBands;
225 m_virtualBands = NULL;
227 delete m_nullChunkData;
228 m_nullChunkData = NULL;
230 delete m_lastProcessByLineChunks;
231 m_lastProcessByLineChunks = NULL;
233 delete m_writeThreadMutex;
234 m_writeThreadMutex = NULL;
246 void CubeIoHandler::read(
Buffer &bufferToFill)
const {
249 int lastChunkCount = m_rawData->size();
251 if (m_lastOperationWasWrite) {
253 flushWriteCache(
true);
255 m_lastOperationWasWrite =
false;
259 if (m_useOptimizedCubeWrite) {
260 delete m_ioThreadPool;
261 m_ioThreadPool = NULL;
265 QMutexLocker lock(m_writeThreadMutex);
276 if (bufferSampleCount == m_samplesInChunk &&
277 bufferLineCount == m_linesInChunk &&
278 bufferBandCount == m_bandsInChunk) {
279 int bufferStartSample = bufferToFill.
Sample();
280 int bufferStartLine = bufferToFill.
Line();
281 int bufferStartBand = bufferToFill.
Band();
283 int bufferEndSample = bufferStartSample + bufferSampleCount - 1;
284 int bufferEndLine = bufferStartLine + bufferLineCount - 1;
285 int bufferEndBand = bufferStartBand + bufferBandCount - 1;
288 int startBand = bufferStartBand - 1;
290 startBand = m_virtualBands->at(bufferStartBand - 1);
292 int expectedChunkIndex =
293 ((bufferStartSample - 1) / getSampleCountInChunk()) +
294 ((bufferStartLine - 1) / getLineCountInChunk()) *
295 getChunkCountInSampleDimension() +
296 ((startBand - 1) / getBandCountInChunk()) *
297 getChunkCountInSampleDimension() *
298 getChunkCountInLineDimension();
300 int chunkStartSample, chunkStartLine, chunkStartBand,
301 chunkEndSample, chunkEndLine, chunkEndBand;
303 getChunkPlacement(expectedChunkIndex,
304 chunkStartSample, chunkStartLine, chunkStartBand,
305 chunkEndSample, chunkEndLine, chunkEndBand);
307 if (chunkStartSample == bufferStartSample &&
308 chunkStartLine == bufferStartLine &&
309 chunkStartBand == bufferStartBand &&
310 chunkEndSample == bufferEndSample &&
311 chunkEndLine == bufferEndLine &&
312 chunkEndBand == bufferEndBand) {
313 cubeChunks.append(getChunk(expectedChunkIndex,
true));
314 chunkBands.append(cubeChunks.last()->getStartBand());
318 if (cubeChunks.empty()) {
321 for(
int i = 0; i < bufferToFill.
size(); i++) {
322 bufferToFill[i] =
Null;
326 chunkInfo = findCubeChunks(
330 cubeChunks = chunkInfo.first;
331 chunkBands = chunkInfo.second;
334 for (
int i = 0; i < cubeChunks.size(); i++) {
335 writeIntoDouble(*cubeChunks[i], bufferToFill, chunkBands[i]);
339 if (lastChunkCount != m_rawData->size()) {
340 minimizeCache(cubeChunks, bufferToFill);
354 void CubeIoHandler::write(
const Buffer &bufferToWrite) {
355 m_lastOperationWasWrite =
true;
357 if (m_ioThreadPool) {
361 QMutexLocker locker(m_writeCache->first);
362 m_writeCache->second.append(copy);
368 QMutexLocker lock(m_writeThreadMutex);
370 synchronousWrite(bufferToWrite);
385 m_cachingAlgorithms->prepend(algorithm);
399 void CubeIoHandler::clearCache(
bool blockForWriteCache)
const {
400 if (blockForWriteCache) {
402 flushWriteCache(
true);
407 if(m_dataIsOnDiskMap) {
408 writeNullDataToDisk();
414 QMapIterator<int, RawCubeChunk *> it(*m_rawData);
415 while (it.hasNext()) {
419 if(it.value()->isDirty()) {
430 if(m_lastProcessByLineChunks) {
431 delete m_lastProcessByLineChunks;
432 m_lastProcessByLineChunks = NULL;
441 BigInt CubeIoHandler::getDataSize()
const {
442 return (
BigInt)getChunkCountInSampleDimension() *
443 (
BigInt)getChunkCountInLineDimension() *
444 (
BigInt)getChunkCountInBandDimension() *
445 (
BigInt)getBytesPerChunk();
458 void CubeIoHandler::setVirtualBands(
const QList<int> *virtualBandList) {
460 delete m_virtualBands;
461 m_virtualBands = NULL;
464 if(virtualBandList && !virtualBandList->empty())
465 m_virtualBands =
new QList<int>(*virtualBandList);
475 QMutex *CubeIoHandler::dataFileMutex() {
476 return m_writeThreadMutex;
482 int CubeIoHandler::bandCount()
const {
490 int CubeIoHandler::getBandCountInChunk()
const {
491 return m_bandsInChunk;
502 BigInt CubeIoHandler::getBytesPerChunk()
const {
503 return m_samplesInChunk * m_linesInChunk * m_bandsInChunk *
512 int CubeIoHandler::getChunkCountInBandDimension()
const {
513 return (
int)ceil((
double)m_numBands / (
double)m_bandsInChunk);
521 int CubeIoHandler::getChunkCountInLineDimension()
const {
522 return (
int)ceil((
double)m_numLines / (
double)m_linesInChunk);
530 int CubeIoHandler::getChunkCountInSampleDimension()
const {
531 return (
int)ceil((
double)m_numSamples / (
double)m_samplesInChunk);
551 int sampleIndex = (chunk.
getStartSample() - 1) / getSampleCountInChunk();
552 int lineIndex = (chunk.
getStartLine() - 1) / getLineCountInChunk();
553 int bandIndex = (chunk.
getStartBand() - 1) / getBandCountInChunk();
556 sampleIndex + lineIndex * getChunkCountInSampleDimension();
557 int indexOffsetToBand = bandIndex * getChunkCountInSampleDimension() *
558 getChunkCountInLineDimension();
560 return indexOffsetToBand + indexInBand;
567 BigInt CubeIoHandler::getDataStartByte()
const {
578 QFile * CubeIoHandler::getDataFile() {
587 int CubeIoHandler::lineCount()
const {
595 int CubeIoHandler::getLineCountInChunk()
const {
596 return m_linesInChunk;
612 int CubeIoHandler::sampleCount()
const {
620 int CubeIoHandler::getSampleCountInChunk()
const {
621 return m_samplesInChunk;
637 void CubeIoHandler::setChunkSizes(
638 int numSamples,
int numLines,
int numBands) {
639 bool success =
false;
642 if(m_samplesInChunk != -1 || m_linesInChunk != -1 || m_bandsInChunk != -1) {
643 IString msg =
"You cannot change the chunk sizes once set";
645 else if(numSamples < 1) {
646 msg =
"Negative and zero chunk sizes are not supported, samples per chunk" 647 " cannot be [" +
IString(numSamples) +
"]";
649 else if(numLines < 1) {
650 msg =
"Negative and zero chunk sizes are not supported, lines per chunk " 651 "cannot be [" +
IString(numLines) +
"]";
653 else if(numBands < 1) {
654 msg =
"Negative and zero chunk sizes are not supported, lines per chunk " 655 "cannot be [" +
IString(numBands) +
"]";
662 m_samplesInChunk = numSamples;
663 m_linesInChunk = numLines;
664 m_bandsInChunk = numBands;
666 if(m_dataIsOnDiskMap) {
667 m_dataFile->resize(getDataStartByte() + getDataSize());
669 else if(m_dataFile->size() < getDataStartByte() + getDataSize()) {
672 " bytes] not big enough to hold data [" +
673 IString(getDataStartByte() + getDataSize()) +
" bytes] where the " 674 "offset to the cube data is [" +
IString(getDataStartByte()) +
690 void CubeIoHandler::blockUntilThreadPoolEmpty()
const {
691 if (m_ioThreadPool) {
692 QMutexLocker lock(m_writeThreadMutex);
704 bool CubeIoHandler::bufferLessThan(
Buffer *
const &lhs,
Buffer *
const &rhs) {
705 bool lessThan =
false;
726 lessThan = lhs->
Band() < rhs->
Band();
728 else if (lhs->
Line() != rhs->
Line()) {
729 lessThan = lhs->
Line() < rhs->
Line();
754 int numSamples,
int startLine,
int numLines,
int startBand,
755 int numBands)
const {
759 int lastBand = startBand + numBands - 1;
765 QPoint(max(startSample, 1),
767 QPoint(min(startSample + numSamples - 1,
769 min(startLine + numLines - 1,
775 for(
int band = startBand; band <= lastBand; band ++) {
777 QRect areaLeftInBand(areaInBand);
779 int actualBand = band;
782 if (band < 1 || band > m_virtualBands->size())
785 actualBand = (m_virtualBands->at(band - 1) - 1) / m_bandsInChunk + 1;
789 while(!areaLeftInBand.isEmpty()) {
837 int areaStartLine = areaLeftInBand.top();
838 int areaStartSample = areaLeftInBand.left();
840 int initialChunkXPos = (areaStartSample - 1) / m_samplesInChunk;
841 int initialChunkYPos = (areaStartLine - 1) / m_linesInChunk;
842 int initialChunkZPos = (actualBand - 1) / m_bandsInChunk;
843 int initialChunkBand = initialChunkZPos * m_bandsInChunk + 1;
846 QRect chunkRect(initialChunkXPos * m_samplesInChunk + 1,
847 initialChunkYPos * m_linesInChunk + 1,
848 m_samplesInChunk, m_linesInChunk);
853 while(chunkRect.intersects(areaLeftInBand) &&
854 (initialChunkBand >= 1 && initialChunkBand <= bandCount())) {
855 int chunkXPos = (chunkRect.left() - 1) / m_samplesInChunk;
856 int chunkYPos = (chunkRect.top() - 1) / m_linesInChunk;
857 int chunkZPos = initialChunkZPos;
860 int chunkIndex = chunkXPos +
861 (chunkYPos * getChunkCountInSampleDimension()) +
862 (chunkZPos * getChunkCountInSampleDimension() *
863 getChunkCountInLineDimension());
867 results.append(newChunk);
868 resultBands.append(band);
870 chunkRect.moveLeft(chunkRect.right() + 1);
873 areaLeftInBand.setTop(chunkRect.bottom() + 1);
898 void CubeIoHandler::findIntersection(
900 int &startX,
int &startY,
int &startZ,
901 int &endX,
int &endY,
int &endZ)
const {
908 int startVBand = cube2.
Band();
911 int startPhysicalBand = 0;
912 int endPhysicalBand = 0;
914 bool startVBandFound =
false;
915 for(
int virtualBand = startVBand; virtualBand <= endVBand; virtualBand ++) {
916 int physicalBand = virtualBand;
918 bool bandExists =
true;
920 if (virtualBand < 1 || virtualBand > m_virtualBands->size())
923 physicalBand = m_virtualBands->at(virtualBand - 1);
928 if(!startVBandFound) {
929 startPhysicalBand = physicalBand;
930 endPhysicalBand = physicalBand;
931 startVBandFound =
true;
934 if(physicalBand < startPhysicalBand)
935 startPhysicalBand = physicalBand;
937 if(physicalBand > endPhysicalBand)
938 endPhysicalBand = physicalBand;
963 void CubeIoHandler::flushWriteCache(
bool force)
const {
964 if (m_ioThreadPool) {
965 bool shouldFlush = m_writeCache->second.size() >= m_idealFlushSize ||
967 bool cacheOverflowing =
968 (m_writeCache->second.size() > m_idealFlushSize * 10);
969 bool shouldAndCanFlush =
false;
970 bool forceStart = force;
973 shouldAndCanFlush = m_writeThreadMutex->tryLock();
974 if (shouldAndCanFlush) {
975 m_writeThreadMutex->unlock();
979 if (cacheOverflowing && !shouldAndCanFlush) {
981 m_consecutiveOverflowCount++;
985 blockUntilThreadPoolEmpty();
987 if (m_writeCache->second.size() != 0) {
988 m_idealFlushSize = m_writeCache->second.size();
990 shouldAndCanFlush =
true;
993 else if (!cacheOverflowing && shouldAndCanFlush) {
994 m_consecutiveOverflowCount = 0;
997 if (cacheOverflowing && m_useOptimizedCubeWrite) {
998 blockUntilThreadPoolEmpty();
1002 if (m_consecutiveOverflowCount > 10) {
1003 delete m_ioThreadPool;
1004 m_ioThreadPool = NULL;
1008 foreach (
Buffer *bufferToWrite, m_writeCache->second) {
1009 const_cast<CubeIoHandler *
>(
this)->synchronousWrite(*bufferToWrite);
1010 delete bufferToWrite;
1013 m_writeCache->second.clear();
1016 if (shouldAndCanFlush && m_ioThreadPool) {
1017 QMutexLocker locker(m_writeCache->first);
1019 const_cast<CubeIoHandler *>(
this), m_writeCache->second);
1021 m_ioThreadPool->start(writer);
1023 m_writeCache->second.clear();
1024 m_lastOperationWasWrite =
true;
1028 blockUntilThreadPoolEmpty();
1041 if(chunkToFree && m_rawData) {
1042 int chunkIndex = getChunkIndex(*chunkToFree);
1044 m_rawData->erase(m_rawData->find(chunkIndex));
1047 (const_cast<CubeIoHandler *>(
this))->writeRaw(*chunkToFree);
1051 if(m_lastProcessByLineChunks) {
1052 delete m_lastProcessByLineChunks;
1053 m_lastProcessByLineChunks = NULL;
1069 bool allocateIfNecessary)
const {
1073 chunk = m_rawData->value(chunkIndex);
1076 if(allocateIfNecessary && !chunk) {
1077 if(m_dataIsOnDiskMap && !(*m_dataIsOnDiskMap)[chunkIndex]) {
1078 chunk = getNullChunk(chunkIndex);
1079 (*m_dataIsOnDiskMap)[chunkIndex] =
true;
1088 getChunkPlacement(chunkIndex, startSample, startLine, startBand,
1089 endSample, endLine, endBand);
1090 chunk =
new RawCubeChunk(startSample, startLine, startBand,
1091 endSample, endLine, endBand,
1092 getBytesPerChunk());
1098 (*m_rawData)[chunkIndex] = chunk;
1109 int CubeIoHandler::getChunkCount()
const {
1110 return getChunkCountInSampleDimension() *
1111 getChunkCountInLineDimension() *
1112 getChunkCountInBandDimension();
1127 void CubeIoHandler::getChunkPlacement(
int chunkIndex,
1128 int &startSample,
int &startLine,
int &startBand,
1129 int &endSample,
int &endLine,
int &endBand)
const {
1130 int chunkSampleIndex = chunkIndex % getChunkCountInSampleDimension();
1133 (chunkIndex - chunkSampleIndex) / getChunkCountInSampleDimension();
1135 int chunkLineIndex = chunkIndex % getChunkCountInLineDimension();
1136 chunkIndex = (chunkIndex - chunkLineIndex) / getChunkCountInLineDimension();
1138 int chunkBandIndex = chunkIndex;
1140 startSample = chunkSampleIndex * getSampleCountInChunk() + 1;
1141 endSample = startSample + getSampleCountInChunk() - 1;
1142 startLine = chunkLineIndex * getLineCountInChunk() + 1;
1143 endLine = startLine + getLineCountInChunk() - 1;
1144 startBand = chunkBandIndex * getBandCountInChunk() + 1;
1145 endBand = startBand + getBandCountInChunk() - 1;
1163 int startSample = 0;
1171 getChunkPlacement(chunkIndex, startSample, startLine, startBand,
1172 endSample, endLine, endBand);
1175 endSample, endLine, endBand, getBytesPerChunk());
1177 if(!m_nullChunkData) {
1187 for(
int i = 0; i < nullBuffer.size(); i++) {
1188 nullBuffer[i] =
Null;
1191 writeIntoRaw(nullBuffer, *result, result->
getStartBand());
1192 m_nullChunkData =
new QByteArray(result->
getRawData());
1213 const Buffer &justRequested)
const {
1216 if (m_rawData->size() * getBytesPerChunk() > 1 * 1024 * 1024 ||
1217 m_cachingAlgorithms->size() > 1) {
1218 bool algorithmAccepted =
false;
1220 int algorithmIndex = 0;
1221 while(!algorithmAccepted &&
1222 algorithmIndex < m_cachingAlgorithms->size()) {
1231 if(algorithmAccepted) {
1235 foreach(chunkToFree, chunksToFree) {
1236 freeChunk(chunkToFree);
1244 if(!algorithmAccepted && m_rawData->size() > 100) {
1261 void CubeIoHandler::synchronousWrite(
const Buffer &bufferToWrite) {
1270 if(m_lastProcessByLineChunks &&
1271 m_lastProcessByLineChunks->size()) {
1273 if(bufferToWrite.
Sample() == 1 &&
1274 bufferSampleCount == sampleCount() &&
1275 bufferLineCount == 1 &&
1276 bufferBandCount == 1) {
1279 int bufferLine = bufferToWrite.
Line();
1280 int chunkStartLine =
1281 (*m_lastProcessByLineChunks)[0]->getStartLine();
1283 (*m_lastProcessByLineChunks)[0]->lineCount();
1284 int bufferBand = bufferToWrite.
Band();
1285 int chunkStartBand =
1286 (*m_lastProcessByLineChunks)[0]->getStartBand();
1288 (*m_lastProcessByLineChunks)[0]->bandCount();
1290 if(bufferLine >= chunkStartLine &&
1291 bufferLine <= chunkStartLine + chunkLines - 1 &&
1292 bufferBand >= chunkStartBand &&
1293 bufferBand <= chunkStartBand + chunkBands - 1) {
1294 cubeChunks = *m_lastProcessByLineChunks;
1295 for (
int i = 0; i < cubeChunks.size(); i++) {
1296 cubeChunkBands.append( cubeChunks[i]->getStartBand() );
1302 else if (bufferSampleCount == m_samplesInChunk &&
1303 bufferLineCount == m_linesInChunk &&
1304 bufferBandCount == m_bandsInChunk) {
1305 int bufferStartSample = bufferToWrite.
Sample();
1306 int bufferStartLine = bufferToWrite.
Line();
1307 int bufferStartBand = bufferToWrite.
Band();
1309 int bufferEndSample = bufferStartSample + bufferSampleCount - 1;
1310 int bufferEndLine = bufferStartLine + bufferLineCount - 1;
1311 int bufferEndBand = bufferStartBand + bufferBandCount - 1;
1313 int expectedChunkIndex =
1314 ((bufferStartSample - 1) / getSampleCountInChunk()) +
1315 ((bufferStartLine - 1) / getLineCountInChunk()) *
1316 getChunkCountInSampleDimension() +
1317 ((bufferStartBand - 1) / getBandCountInChunk()) *
1318 getChunkCountInSampleDimension() *
1319 getChunkCountInLineDimension();
1321 int chunkStartSample, chunkStartLine, chunkStartBand,
1322 chunkEndSample, chunkEndLine, chunkEndBand;
1324 getChunkPlacement(expectedChunkIndex,
1325 chunkStartSample, chunkStartLine, chunkStartBand,
1326 chunkEndSample, chunkEndLine, chunkEndBand);
1328 if (chunkStartSample == bufferStartSample &&
1329 chunkStartLine == bufferStartLine &&
1330 chunkStartBand == bufferStartBand &&
1331 chunkEndSample == bufferEndSample &&
1332 chunkEndLine == bufferEndLine &&
1333 chunkEndBand == bufferEndBand) {
1334 cubeChunks.append(getChunk(expectedChunkIndex,
true));
1335 cubeChunkBands.append(cubeChunks.last()->getStartBand());
1340 if(cubeChunks.empty()) {
1341 chunkInfo = findCubeChunks(
1342 bufferToWrite.
Sample(), bufferSampleCount,
1343 bufferToWrite.
Line(), bufferLineCount,
1344 bufferToWrite.
Band(), bufferBandCount);
1345 cubeChunks = chunkInfo.first;
1346 cubeChunkBands = chunkInfo.second;
1350 if(bufferToWrite.
Sample() == 1 &&
1351 bufferSampleCount == sampleCount() &&
1352 bufferLineCount == 1 &&
1353 bufferBandCount == 1) {
1354 if(!m_lastProcessByLineChunks)
1355 m_lastProcessByLineChunks =
1358 *m_lastProcessByLineChunks = cubeChunks;
1361 for(
int i = 0; i < cubeChunks.size(); i++) {
1362 writeIntoRaw(bufferToWrite, *cubeChunks[i], cubeChunkBands[i]);
1365 minimizeCache(cubeChunks, bufferToWrite);
1377 Buffer &output,
int index)
const {
1392 findIntersection(chunk, output, startX, startY, startZ, endX, endY, endZ);
1394 int bufferBand = output.
Band();
1400 int chunkBandSize = chunkLineSize * chunk.
lineCount();
1403 const char *chunkBuf = chunk.
getRawData().data();
1404 char *buffersRawBuf = (
char *)output.
RawBuffer();
1406 for(
int z = startZ; z <= endZ; z++) {
1407 const int &bandIntoChunk = z - chunkStartBand;
1408 int virtualBand = z;
1410 virtualBand = index;
1412 if(virtualBand != 0 && virtualBand >= bufferBand &&
1413 virtualBand <= bufferBand + bufferBands - 1) {
1415 for(
int y = startY; y <= endY; y++) {
1416 const int &lineIntoChunk = y - chunkStartLine;
1417 int bufferIndex = output.
Index(startX, y, virtualBand);
1419 for(
int x = startX; x <= endX; x++) {
1420 const int &sampleIntoChunk = x - chunkStartSample;
1422 const int &chunkIndex = sampleIntoChunk +
1423 (chunkLineSize * lineIntoChunk) +
1424 (chunkBandSize * bandIntoChunk);
1426 double &bufferVal = buffersDoubleBuf[bufferIndex];
1428 if(m_pixelType == Real) {
1429 float raw = ((
float *)chunkBuf)[chunkIndex];
1431 raw = m_byteSwapper->Float(&raw);
1433 if(raw >= VALID_MIN4) {
1434 bufferVal = (double) raw;
1439 else if(raw == LOW_INSTR_SAT4)
1440 bufferVal = LOW_INSTR_SAT8;
1441 else if(raw == LOW_REPR_SAT4)
1442 bufferVal = LOW_REPR_SAT8;
1443 else if(raw == HIGH_INSTR_SAT4)
1444 bufferVal = HIGH_INSTR_SAT8;
1445 else if(raw == HIGH_REPR_SAT4)
1446 bufferVal = HIGH_REPR_SAT8;
1448 bufferVal = LOW_REPR_SAT8;
1451 ((
float *)buffersRawBuf)[bufferIndex] = raw;
1454 else if(m_pixelType == SignedWord) {
1455 short raw = ((
short *)chunkBuf)[chunkIndex];
1457 raw = m_byteSwapper->ShortInt(&raw);
1459 if(raw >= VALID_MIN2) {
1460 bufferVal = (double) raw * m_multiplier + m_base;
1465 else if(raw == LOW_INSTR_SAT2)
1466 bufferVal = LOW_INSTR_SAT8;
1467 else if(raw == LOW_REPR_SAT2)
1468 bufferVal = LOW_REPR_SAT8;
1469 else if(raw == HIGH_INSTR_SAT2)
1470 bufferVal = HIGH_INSTR_SAT8;
1471 else if(raw == HIGH_REPR_SAT2)
1472 bufferVal = HIGH_REPR_SAT8;
1474 bufferVal = LOW_REPR_SAT8;
1477 ((
short *)buffersRawBuf)[bufferIndex] = raw;
1481 else if(m_pixelType == UnsignedWord) {
1482 unsigned short raw = ((
unsigned short *)chunkBuf)[chunkIndex];
1484 raw = m_byteSwapper->UnsignedShortInt(&raw);
1486 if(raw >= VALID_MINU2) {
1487 bufferVal = (double) raw * m_multiplier + m_base;
1489 else if (raw > VALID_MAXU2) {
1490 if(raw == HIGH_INSTR_SATU2)
1491 bufferVal = HIGH_INSTR_SAT8;
1492 else if(raw == HIGH_REPR_SATU2)
1493 bufferVal = HIGH_REPR_SAT8;
1495 bufferVal = LOW_REPR_SAT8;
1500 else if(raw == LOW_INSTR_SATU2)
1501 bufferVal = LOW_INSTR_SAT8;
1502 else if(raw == LOW_REPR_SATU2)
1503 bufferVal = LOW_REPR_SAT8;
1505 bufferVal = LOW_REPR_SAT8;
1508 ((
unsigned short *)buffersRawBuf)[bufferIndex] = raw;
1511 else if(m_pixelType == UnsignedInteger) {
1513 unsigned int raw = ((
unsigned int *)chunkBuf)[chunkIndex];
1515 raw = m_byteSwapper->Uint32_t(&raw);
1517 if(raw >= VALID_MINUI4) {
1518 bufferVal = (double) raw * m_multiplier + m_base;
1520 else if (raw > VALID_MAXUI4) {
1521 if(raw == HIGH_INSTR_SATUI4)
1522 bufferVal = HIGH_INSTR_SAT8;
1523 else if(raw == HIGH_REPR_SATUI4)
1524 bufferVal = HIGH_REPR_SAT8;
1526 bufferVal = LOW_REPR_SAT8;
1531 else if(raw == LOW_INSTR_SATUI4)
1532 bufferVal = LOW_INSTR_SAT8;
1533 else if(raw == LOW_REPR_SATUI4)
1534 bufferVal = LOW_REPR_SAT8;
1536 bufferVal = LOW_REPR_SAT8;
1539 ((
unsigned int *)buffersRawBuf)[bufferIndex] = raw;
1545 else if(m_pixelType == UnsignedByte) {
1546 unsigned char raw = ((
unsigned char *)chunkBuf)[chunkIndex];
1551 else if(raw == HIGH_REPR_SAT1) {
1552 bufferVal = HIGH_REPR_SAT8;
1555 bufferVal = (double) raw * m_multiplier + m_base;
1558 ((
unsigned char *)buffersRawBuf)[bufferIndex] = raw;
1593 findIntersection(output, buffer, startX, startY, startZ, endX, endY, endZ);
1595 int bufferBand = buffer.
Band();
1601 int bandSize = lineSize * output.
lineCount();
1605 for(
int z = startZ; z <= endZ; z++) {
1606 const int &bandIntoChunk = z - outputStartBand;
1607 int virtualBand = index;
1610 if(m_virtualBands) {
1611 virtualBand = m_virtualBands->indexOf(virtualBand) + 1;
1614 if(virtualBand != 0 && virtualBand >= bufferBand &&
1615 virtualBand <= bufferBand + bufferBands - 1) {
1617 for(
int y = startY; y <= endY; y++) {
1618 const int &lineIntoChunk = y - outputStartLine;
1619 int bufferIndex = buffer.
Index(startX, y, virtualBand);
1621 for(
int x = startX; x <= endX; x++) {
1622 const int &sampleIntoChunk = x - outputStartSample;
1624 const int &chunkIndex = sampleIntoChunk +
1625 (lineSize * lineIntoChunk) + (bandSize * bandIntoChunk);
1627 double bufferVal = buffersDoubleBuf[bufferIndex];
1629 if(m_pixelType == Real) {
1632 if(bufferVal >= VALID_MIN8) {
1633 double filePixelValueDbl = (bufferVal - m_base) /
1636 if(filePixelValueDbl < (
double) VALID_MIN4) {
1637 raw = LOW_REPR_SAT4;
1639 else if(filePixelValueDbl > (
double) VALID_MAX4) {
1640 raw = HIGH_REPR_SAT4;
1643 raw = (float) filePixelValueDbl;
1647 if(bufferVal == NULL8)
1649 else if(bufferVal == LOW_INSTR_SAT8)
1650 raw = LOW_INSTR_SAT4;
1651 else if(bufferVal == LOW_REPR_SAT8)
1652 raw = LOW_REPR_SAT4;
1653 else if(bufferVal == HIGH_INSTR_SAT8)
1654 raw = HIGH_INSTR_SAT4;
1655 else if(bufferVal == HIGH_REPR_SAT8)
1656 raw = HIGH_REPR_SAT4;
1658 raw = LOW_REPR_SAT4;
1660 ((
float *)chunkBuf)[chunkIndex] =
1661 m_byteSwapper ? m_byteSwapper->Float(&raw) : raw;
1664 else if(m_pixelType == SignedWord) {
1667 if(bufferVal >= VALID_MIN8) {
1668 double filePixelValueDbl = (bufferVal - m_base) /
1670 if(filePixelValueDbl < VALID_MIN2 - 0.5) {
1671 raw = LOW_REPR_SAT2;
1673 if(filePixelValueDbl > VALID_MAX2 + 0.5) {
1674 raw = HIGH_REPR_SAT2;
1677 int filePixelValue = (int)round(filePixelValueDbl);
1679 if(filePixelValue < VALID_MIN2) {
1680 raw = LOW_REPR_SAT2;
1682 else if(filePixelValue > VALID_MAX2) {
1683 raw = HIGH_REPR_SAT2;
1686 raw = filePixelValue;
1691 if(bufferVal == NULL8)
1693 else if(bufferVal == LOW_INSTR_SAT8)
1694 raw = LOW_INSTR_SAT2;
1695 else if(bufferVal == LOW_REPR_SAT8)
1696 raw = LOW_REPR_SAT2;
1697 else if(bufferVal == HIGH_INSTR_SAT8)
1698 raw = HIGH_INSTR_SAT2;
1699 else if(bufferVal == HIGH_REPR_SAT8)
1700 raw = HIGH_REPR_SAT2;
1702 raw = LOW_REPR_SAT2;
1705 ((
short *)chunkBuf)[chunkIndex] =
1706 m_byteSwapper ? m_byteSwapper->ShortInt(&raw) : raw;
1709 else if(m_pixelType == UnsignedInteger) {
1713 if(bufferVal >= VALID_MINUI4) {
1714 double filePixelValueDbl = (bufferVal - m_base) /
1716 if(filePixelValueDbl < VALID_MINUI4 - 0.5) {
1717 raw = LOW_REPR_SATUI4;
1719 if(filePixelValueDbl > VALID_MAXUI4) {
1720 raw = HIGH_REPR_SATUI4;
1723 unsigned int filePixelValue = (
unsigned int)round(filePixelValueDbl);
1725 if(filePixelValue < VALID_MINUI4) {
1726 raw = LOW_REPR_SATUI4;
1728 else if(filePixelValue > VALID_MAXUI4) {
1729 raw = HIGH_REPR_SATUI4;
1732 raw = filePixelValue;
1737 if(bufferVal == NULL8)
1739 else if(bufferVal == LOW_INSTR_SAT8)
1740 raw = LOW_INSTR_SATUI4;
1741 else if(bufferVal == LOW_REPR_SAT8)
1742 raw = LOW_REPR_SATUI4;
1743 else if(bufferVal == HIGH_INSTR_SAT8)
1744 raw = HIGH_INSTR_SATUI4;
1745 else if(bufferVal == HIGH_REPR_SAT8)
1746 raw = HIGH_REPR_SATUI4;
1748 raw = LOW_REPR_SATUI4;
1751 ((
unsigned int *)chunkBuf)[chunkIndex] =
1752 m_byteSwapper ? m_byteSwapper->Uint32_t(&raw) : raw;
1757 else if(m_pixelType == UnsignedWord) {
1760 if(bufferVal >= VALID_MIN8) {
1761 double filePixelValueDbl = (bufferVal - m_base) /
1763 if(filePixelValueDbl < VALID_MINU2 - 0.5) {
1764 raw = LOW_REPR_SATU2;
1766 if(filePixelValueDbl > VALID_MAXU2 + 0.5) {
1767 raw = HIGH_REPR_SATU2;
1770 int filePixelValue = (int)round(filePixelValueDbl);
1772 if(filePixelValue < VALID_MINU2) {
1773 raw = LOW_REPR_SATU2;
1775 else if(filePixelValue > VALID_MAXU2) {
1776 raw = HIGH_REPR_SATU2;
1779 raw = filePixelValue;
1784 if(bufferVal == NULL8)
1786 else if(bufferVal == LOW_INSTR_SAT8)
1787 raw = LOW_INSTR_SATU2;
1788 else if(bufferVal == LOW_REPR_SAT8)
1789 raw = LOW_REPR_SATU2;
1790 else if(bufferVal == HIGH_INSTR_SAT8)
1791 raw = HIGH_INSTR_SATU2;
1792 else if(bufferVal == HIGH_REPR_SAT8)
1793 raw = HIGH_REPR_SATU2;
1795 raw = LOW_REPR_SATU2;
1798 ((
unsigned short *)chunkBuf)[chunkIndex] =
1799 m_byteSwapper ? m_byteSwapper->UnsignedShortInt(&raw) : raw;
1801 else if(m_pixelType == UnsignedByte) {
1804 if(bufferVal >= VALID_MIN8) {
1805 double filePixelValueDbl = (bufferVal - m_base) /
1807 if(filePixelValueDbl < VALID_MIN1 - 0.5) {
1808 raw = LOW_REPR_SAT1;
1810 else if(filePixelValueDbl > VALID_MAX1 + 0.5) {
1811 raw = HIGH_REPR_SAT1;
1814 int filePixelValue = (int)(filePixelValueDbl + 0.5);
1815 if(filePixelValue < VALID_MIN1) {
1816 raw = LOW_REPR_SAT1;
1818 else if(filePixelValue > VALID_MAX1) {
1819 raw = HIGH_REPR_SAT1;
1822 raw = (
unsigned char)(filePixelValue);
1827 if(bufferVal == NULL8)
1829 else if(bufferVal == LOW_INSTR_SAT8)
1830 raw = LOW_INSTR_SAT1;
1831 else if(bufferVal == LOW_REPR_SAT8)
1832 raw = LOW_REPR_SAT1;
1833 else if(bufferVal == HIGH_INSTR_SAT8)
1834 raw = HIGH_INSTR_SAT1;
1835 else if(bufferVal == HIGH_REPR_SAT8)
1836 raw = HIGH_REPR_SAT1;
1838 raw = LOW_REPR_SAT1;
1841 ((
unsigned char *)chunkBuf)[chunkIndex] = raw;
1855 void CubeIoHandler::writeNullDataToDisk()
const {
1856 if(!m_dataIsOnDiskMap) {
1857 IString msg =
"Cannot call CubeIoHandler::writeNullDataToDisk unless " 1858 "data is not already on disk (Cube::Create was called)";
1862 int numChunks = getChunkCount();
1863 for(
int i = 0; i < numChunks; i++) {
1864 if(!(*m_dataIsOnDiskMap)[i]) {
1867 (*m_dataIsOnDiskMap)[i] =
true;
1889 CubeIoHandler::BufferToChunkWriter::BufferToChunkWriter(
1891 m_ioHandler = ioHandler;
1893 m_timer =
new QTime;
1896 m_ioHandler->m_writeThreadMutex->lock();
1910 CubeIoHandler::BufferToChunkWriter::~BufferToChunkWriter() {
1911 int elapsedMs = m_timer->elapsed();
1912 int idealFlushElapsedTime = 100;
1918 int msOffIdeal = elapsedMs - idealFlushElapsedTime;
1919 double percentOffIdeal = msOffIdeal / (double)idealFlushElapsedTime;
1922 int currentCacheSize = m_ioHandler->m_idealFlushSize;
1923 int desiredAdjustment = -1 * currentCacheSize * percentOffIdeal;
1924 int desiredCacheSize = (int)(currentCacheSize + desiredAdjustment);
1925 m_ioHandler->m_idealFlushSize = (int)(qMin(5000,
1926 qMax(32, desiredCacheSize)));
1931 m_ioHandler->m_writeThreadMutex->unlock();
1934 ASSERT(m_buffersToWrite->isEmpty());
1935 delete m_buffersToWrite;
1936 m_buffersToWrite = NULL;
1945 void CubeIoHandler::BufferToChunkWriter::run() {
1965 foreach (
Buffer * bufferToWrite, *m_buffersToWrite) {
1966 m_ioHandler->synchronousWrite(*bufferToWrite);
1967 delete bufferToWrite;
1970 m_buffersToWrite->clear();
1971 m_ioHandler->m_dataFile->flush();
Buffer for reading and writing cube data.
long long int BigInt
Big int.
void setRawData(QByteArray rawData)
Sets the chunk's raw data.
double * DoubleBuffer() const
Returns the value of the shape buffer.
const double Null
Value for an Isis Null pixel.
int Band(const int index=0) const
Returns the band position associated with a shape buffer index.
PvlGroupIterator findGroup(const QString &name, PvlGroupIterator beg, PvlGroupIterator end)
Find a group with the specified name, within these indexes.
Represents a 3D area (a 3D "cube")
void * RawBuffer() const
Returns a void pointer to the raw buffer.
bool isValid() const
Returns true if all of the positions of the 3D area are valid (i.e.
PvlObjectIterator findObject(const QString &name, PvlObjectIterator beg, PvlObjectIterator end)
Find the index of object with a specified name, between two indexes.
int SizeOf(Isis::PixelType pixelType)
Returns the number of bytes of the specified PixelType.
Buffer for containing a three dimensional section of an image.
Namespace for the standard library.
void SetBasePosition(const int start_sample, const int start_line, const int start_band)
This method is used to set the base position of the shape buffer.
int getStartSample() const
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
int LineDimension() const
Returns the number of lines in the shape buffer.
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
void setDirty(bool dirty)
Sets the chunk's dirty flag, indicating whether or not the chunk's data matches the data that is on d...
Distance measurement, usually in meters.
int size() const
Returns the total number of pixels in the shape buffer.
PixelType
Enumerations for Isis Pixel Types.
Handles converting buffers to and from disk.
virtual CacheResult recommendChunksToFree(QList< RawCubeChunk *> allocated, QList< RawCubeChunk *> justUsed, const Buffer &justRequested)=0
Call this to determine which chunks should be freed from memory.
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
int Sample(const int index=0) const
Returns the sample position associated with a shape buffer index.
Isis::PixelType PixelTypeEnumeration(const QString &type)
Returns PixelType enumeration given a string.
Unless noted otherwise, the portions of Isis written by the USGS are public domain.
This algorithm recommends chunks to be freed that are not within the last IO.
QList< RawCubeChunk * > getChunksToFree() const
int BandDimension() const
Returns the number of bands in the shape buffer.
PvlKeyword & findKeyword(const QString &kname, FindOptions opts)
Finds a keyword in the current PvlObject, or deeper inside other PvlObjects and Pvlgroups within this...
Contains multiple PvlContainers.
#define _FILEINFO_
Macro for the filename and line number.
int Line(const int index=0) const
Returns the line position associated with a shape buffer index.
A section of raw data on the disk.
Container for cube-like labels.
PvlKeyword & findKeyword(const QString &name)
Find a keyword with a specified name.
int SampleDimension() const
Returns the number of samples in the shape buffer.
IString DownCase()
Converts all upper case letters in the object IString into lower case characters. ...
Displacement is a signed length, usually in meters.
Adds specific functionality to C++ strings.
This stores the results of the caching algorithm.
Namespace for ISIS/Bullet specific routines.
Area3D intersect(const Area3D &otherArea) const
Returns the intersection of this 3D area with another 3D area.
This class is designed to handle write() asynchronously.
int Index(const int i_samp, const int i_line, const int i_band) const
Given a sample, line, and band position, this returns the appropriate index in the shape buffer...
Contains Pvl Groups and Pvl Objects.
QByteArray & getRawData() const
bool algorithmUnderstoodData() const
If this is true, then the results (be them empty or not) should be considered valid.
This is the parent of the caching algorithms