Isis 3 Programmer Reference
CubeIoHandler.h
1 
6 /* SPDX-License-Identifier: CC0-1.0 */
7 
8 #ifndef CubeIoHandler_h
9 #define CubeIoHandler_h
10 
11 #include <QRunnable>
12 #include <QThreadPool>
13 
14 #include "Constants.h"
15 #include "Endian.h"
16 #include "PixelType.h"
17 
18 class QFile;
19 class QMutex;
20 class QTime;
21 template <typename A> class QList;
22 template <typename A, typename B> class QMap;
23 template <typename A, typename B> struct QPair;
24 
25 namespace Isis {
26  class Buffer;
27  class CubeCachingAlgorithm;
28  class EndianSwapper;
29  class Pvl;
30  class RawCubeChunk;
31 
110  public:
111  CubeIoHandler(QFile * dataFile, const QList<int> *virtualBandList,
112  const Pvl &label, bool alreadyOnDisk);
113  virtual ~CubeIoHandler();
114 
115  void read(Buffer &bufferToFill) const;
116  void write(const Buffer &bufferToWrite);
117 
118  void addCachingAlgorithm(CubeCachingAlgorithm *algorithm);
119  void clearCache(bool blockForWriteCache = true) const;
120  BigInt getDataSize() const;
121  void setVirtualBands(const QList<int> *virtualBandList);
127  virtual void updateLabels(Pvl &labels) = 0;
128 
129  QMutex *dataFileMutex();
130 
131  protected:
132  int bandCount() const;
133  int getBandCountInChunk() const;
134  BigInt getBytesPerChunk() const;
135  int getChunkCountInBandDimension() const;
136  int getChunkCountInLineDimension() const;
137  int getChunkCountInSampleDimension() const;
138  int getChunkIndex(const RawCubeChunk &) const;
139  BigInt getDataStartByte() const;
140  QFile * getDataFile();
141  int lineCount() const;
142  int getLineCountInChunk() const;
143  PixelType pixelType() const;
144  int sampleCount() const;
145  int getSampleCountInChunk() const;
146 
147  void setChunkSizes(int numSamples, int numLines, int numBands);
148 
157  virtual void readRaw(RawCubeChunk &chunkToFill) = 0;
158 
166  virtual void writeRaw(const RawCubeChunk &chunkToWrite) = 0;
167 
168  private:
186  public:
188  QList<Buffer *> buffersToWrite);
190 
191  void run();
192 
193  private:
205 
206  private:
212  QTime *m_timer;
213  };
214 
215 
222 
231 
232  void blockUntilThreadPoolEmpty() const;
233 
234  static bool bufferLessThan(Buffer * const &lhs, Buffer * const &rhs);
235 
236  QPair< QList<RawCubeChunk *>, QList<int> > findCubeChunks(int startSample, int numSamples,
237  int startLine, int numLines,
238  int startBand, int numBands) const;
239 
240  void findIntersection(const RawCubeChunk &cube1,
241  const Buffer &cube2, int &startX, int &startY, int &startZ,
242  int &endX, int &endY, int &endZ) const;
243 
244  void flushWriteCache(bool force = false) const;
245 
246  void freeChunk(RawCubeChunk *chunkToFree) const;
247 
248  RawCubeChunk *getChunk(int chunkIndex, bool allocateIfNecessary) const;
249 
250  int getChunkCount() const;
251 
252  void getChunkPlacement(int chunkIndex,
253  int &startSample, int &startLine, int &startBand,
254  int &endSample, int &endLine, int &endBand) const;
255 
256  RawCubeChunk *getNullChunk(int chunkIndex) const;
257 
258  void minimizeCache(const QList<RawCubeChunk *> &justUsed,
259  const Buffer &justRequested) const;
260 
261  void synchronousWrite(const Buffer &bufferToWrite);
262 
263  void writeIntoDouble(const RawCubeChunk &chunk, Buffer &output, int startIndex) const;
264 
265  void writeIntoRaw(const Buffer &buffer, RawCubeChunk &output, int index) const;
266 
267  void writeNullDataToDisk() const;
268 
269  private:
271  QFile * m_dataFile;
272 
279 
282 
284  double m_base;
285 
287  double m_multiplier;
288 
291 
294 
297 
300 
303 
306 
309 
312 
315 
318 
321 
324 
330 
332  mutable QByteArray *m_nullChunkData;
333 
336 
342  mutable QThreadPool *m_ioThreadPool;
343 
354 
357 
359  mutable volatile int m_idealFlushSize;
360 
363  };
364 }
365 
366 #endif
Isis::CubeIoHandler::m_ioThreadPool
QThreadPool * m_ioThreadPool
This contains threads for doing cube writes (and later maybe cube reads).
Definition: CubeIoHandler.h:342
Isis::RawCubeChunk
A section of raw data on the disk.
Definition: RawCubeChunk.h:27
Isis::CubeIoHandler::m_samplesInChunk
int m_samplesInChunk
The number of samples in a cube chunk.
Definition: CubeIoHandler.h:317
Isis::CubeIoHandler::addCachingAlgorithm
void addCachingAlgorithm(CubeCachingAlgorithm *algorithm)
This will add the given caching algorithm to the list of attempted caching algorithms.
Definition: CubeIoHandler.cpp:368
Isis::CubeIoHandler::m_numLines
int m_numLines
The number of lines in the cube.
Definition: CubeIoHandler.h:299
Isis::CubeIoHandler::m_base
double m_base
The additive offset of the data on disk.
Definition: CubeIoHandler.h:284
Isis::CubeIoHandler::m_lastProcessByLineChunks
QList< RawCubeChunk * > * m_lastProcessByLineChunks
This is an optimization for process by line.
Definition: CubeIoHandler.h:329
QList
This is free and unencumbered software released into the public domain.
Definition: BoxcarCachingAlgorithm.h:13
Isis::CubeIoHandler::m_nullChunkData
QByteArray * m_nullChunkData
A raw cube chunk's data when it was all NULL's. Used for speed.
Definition: CubeIoHandler.h:332
Isis::CubeIoHandler::sampleCount
int sampleCount() const
Definition: CubeIoHandler.cpp:596
Isis::CubeIoHandler::operator=
CubeIoHandler & operator=(const CubeIoHandler &other)
Disallow assignments of this object.
Isis::CubeIoHandler::getBandCountInChunk
int getBandCountInChunk() const
Definition: CubeIoHandler.cpp:474
Isis::CubeIoHandler::m_byteSwapper
EndianSwapper * m_byteSwapper
A helper that swaps byte order to and from file order.
Definition: CubeIoHandler.h:293
Isis::CubeIoHandler::updateLabels
virtual void updateLabels(Pvl &labels)=0
Function to update the labels with a Pvl object.
Isis::CubeIoHandler::pixelType
PixelType pixelType() const
Definition: CubeIoHandler.cpp:587
Isis::CubeIoHandler::BufferToChunkWriter::m_ioHandler
CubeIoHandler * m_ioHandler
The IO Handler instance to put the buffers into.
Definition: CubeIoHandler.h:208
Isis::CubeIoHandler::getBytesPerChunk
BigInt getBytesPerChunk() const
Definition: CubeIoHandler.cpp:486
Isis::CubeIoHandler::setVirtualBands
void setVirtualBands(const QList< int > *virtualBandList)
This changes the virtual band list.
Definition: CubeIoHandler.cpp:442
Isis::CubeIoHandler::writeIntoRaw
void writeIntoRaw(const Buffer &buffer, RawCubeChunk &output, int index) const
Write the intersecting area of the buffer into the chunk.
Definition: CubeIoHandler.cpp:1560
Isis::CubeIoHandler::m_dataFile
QFile * m_dataFile
The file containing cube data.
Definition: CubeIoHandler.h:271
Isis::CubeIoHandler::m_writeThreadMutex
QMutex * m_writeThreadMutex
This enables us to block while the write thread is working.
Definition: CubeIoHandler.h:356
Isis::Pvl
Container for cube-like labels.
Definition: Pvl.h:119
Isis::CubeIoHandler::m_numBands
int m_numBands
The number of physical bands in the cube.
Definition: CubeIoHandler.h:302
Isis::CubeIoHandler::getDataFile
QFile * getDataFile()
Definition: CubeIoHandler.cpp:562
Isis::CubeIoHandler::BufferToChunkWriter::BufferToChunkWriter
BufferToChunkWriter(const BufferToChunkWriter &other)
This is disabled.
Isis::CubeIoHandler::getSampleCountInChunk
int getSampleCountInChunk() const
Definition: CubeIoHandler.cpp:604
Isis::CubeIoHandler::m_multiplier
double m_multiplier
The multiplicative factor of the data on disk.
Definition: CubeIoHandler.h:287
Isis::CubeIoHandler::m_idealFlushSize
volatile int m_idealFlushSize
Ideal write cache flush size.
Definition: CubeIoHandler.h:359
Isis::ByteOrder
ByteOrder
Tests the current architecture for byte order.
Definition: Endian.h:42
Isis::Buffer
Buffer for reading and writing cube data.
Definition: Buffer.h:53
Isis::CubeIoHandler::CubeIoHandler
CubeIoHandler(QFile *dataFile, const QList< int > *virtualBandList, const Pvl &label, bool alreadyOnDisk)
Creates a new CubeIoHandler using a RegionalCachingAlgorithm.
Definition: CubeIoHandler.cpp:63
Isis::CubeIoHandler::freeChunk
void freeChunk(RawCubeChunk *chunkToFree) const
If the chunk is dirty, then we write it to disk.
Definition: CubeIoHandler.cpp:1024
Isis::CubeIoHandler::getChunkPlacement
void getChunkPlacement(int chunkIndex, int &startSample, int &startLine, int &startBand, int &endSample, int &endLine, int &endBand) const
Get the X/Y/Z (Sample,Line,Band) range of the chunk at the given index.
Definition: CubeIoHandler.cpp:1111
Isis::CubeIoHandler::BufferToChunkWriter::run
void run()
This is the asynchronous computation.
Definition: CubeIoHandler.cpp:1929
Isis::CubeIoHandler::lineCount
int lineCount() const
Definition: CubeIoHandler.cpp:571
Isis::CubeIoHandler::blockUntilThreadPoolEmpty
void blockUntilThreadPoolEmpty() const
This blocks (doesn't return) until the number of active runnables in the thread pool goes to 0.
Definition: CubeIoHandler.cpp:674
Isis::CubeIoHandler::getChunk
RawCubeChunk * getChunk(int chunkIndex, bool allocateIfNecessary) const
Retrieve the cached chunk at the given chunk index, if there is one.
Definition: CubeIoHandler.cpp:1052
Isis::CubeIoHandler::getNullChunk
RawCubeChunk * getNullChunk(int chunkIndex) const
This creates a chunk filled with NULLs whose placement is at chunkIndex's position.
Definition: CubeIoHandler.cpp:1143
Isis::CubeIoHandler::getDataStartByte
BigInt getDataStartByte() const
Definition: CubeIoHandler.cpp:551
Isis::CubeIoHandler::~CubeIoHandler
virtual ~CubeIoHandler()
Cleans up all allocated memory.
Definition: CubeIoHandler.cpp:157
Isis::CubeIoHandler::getChunkCountInBandDimension
int getChunkCountInBandDimension() const
Definition: CubeIoHandler.cpp:496
Isis::CubeIoHandler::dataFileMutex
QMutex * dataFileMutex()
Get the mutex that this IO handler is using around I/Os on the given data file.
Definition: CubeIoHandler.cpp:459
Isis::CubeCachingAlgorithm
This is the parent of the caching algorithms.
Definition: CubeCachingAlgorithm.h:31
Isis::CubeIoHandler
Handles converting buffers to and from disk.
Definition: CubeIoHandler.h:109
Isis::BigInt
long long int BigInt
Big int.
Definition: Constants.h:49
Isis::CubeIoHandler::m_dataIsOnDiskMap
QMap< int, bool > * m_dataIsOnDiskMap
The map from chunk index to on-disk status, all true if not allocated.
Definition: CubeIoHandler.h:311
Isis::CubeIoHandler::getChunkIndex
int getChunkIndex(const RawCubeChunk &) const
Given a chunk, what's its index in the file.
Definition: CubeIoHandler.cpp:530
Isis::CubeIoHandler::writeRaw
virtual void writeRaw(const RawCubeChunk &chunkToWrite)=0
This needs to write the chunkToWrite directly to disk with no modifications to the data itself.
Isis::CubeIoHandler::m_linesInChunk
int m_linesInChunk
The number of lines in a cube chunk.
Definition: CubeIoHandler.h:320
Isis::CubeIoHandler::writeIntoDouble
void writeIntoDouble(const RawCubeChunk &chunk, Buffer &output, int startIndex) const
Write the intersecting area of the chunk into the buffer.
Definition: CubeIoHandler.cpp:1360
Isis::CubeIoHandler::BufferToChunkWriter::BufferToChunkWriter
BufferToChunkWriter(CubeIoHandler *ioHandler, QList< Buffer * > buffersToWrite)
Create a BufferToChunkWriter which is designed to asynchronously move the given buffers into the cube...
Definition: CubeIoHandler.cpp:1873
Isis::CubeIoHandler::bandCount
int bandCount() const
Definition: CubeIoHandler.cpp:466
Isis::CubeIoHandler::getChunkCountInSampleDimension
int getChunkCountInSampleDimension() const
Definition: CubeIoHandler.cpp:514
Isis::CubeIoHandler::BufferToChunkWriter::m_timer
QTime * m_timer
Used to calculate the lifetime of an instance of this class.
Definition: CubeIoHandler.h:212
Isis::CubeIoHandler::m_numSamples
int m_numSamples
The number of samples in the cube.
Definition: CubeIoHandler.h:296
Isis::CubeIoHandler::m_consecutiveOverflowCount
int m_consecutiveOverflowCount
How many times the write cache has overflown in a row.
Definition: CubeIoHandler.h:362
Isis::CubeIoHandler::findIntersection
void findIntersection(const RawCubeChunk &cube1, const Buffer &cube2, int &startX, int &startY, int &startZ, int &endX, int &endY, int &endZ) const
Find the intersection between the buffer area and the cube chunk.
Definition: CubeIoHandler.cpp:882
Isis::CubeIoHandler::m_byteOrder
ByteOrder m_byteOrder
The byte order (endianness) of the data on disk.
Definition: CubeIoHandler.h:290
Isis::PixelType
PixelType
Enumerations for Isis Pixel Types.
Definition: PixelType.h:27
Isis::CubeIoHandler::read
void read(Buffer &bufferToFill) const
Read cube data from disk into the buffer.
Definition: CubeIoHandler.cpp:230
Isis::CubeIoHandler::m_bandsInChunk
int m_bandsInChunk
The number of physical bands in a cube chunk.
Definition: CubeIoHandler.h:323
Isis::CubeIoHandler::m_virtualBands
QList< int > * m_virtualBands
Converts from virtual band to physical band.
Definition: CubeIoHandler.h:314
Isis::CubeIoHandler::synchronousWrite
void synchronousWrite(const Buffer &bufferToWrite)
This method takes the given buffer and synchronously puts it into the Cube's cache.
Definition: CubeIoHandler.cpp:1245
QPair
This is free and unencumbered software released into the public domain.
Definition: CubeIoHandler.h:23
QRunnable
Isis::CubeIoHandler::BufferToChunkWriter::~BufferToChunkWriter
~BufferToChunkWriter()
We're done writing our buffers into the cube, clean up.
Definition: CubeIoHandler.cpp:1894
Isis::CubeIoHandler::minimizeCache
void minimizeCache(const QList< RawCubeChunk * > &justUsed, const Buffer &justRequested) const
Apply the caching algorithms and get rid of excess cube data in memory.
Definition: CubeIoHandler.cpp:1196
Isis::CubeIoHandler::bufferLessThan
static bool bufferLessThan(Buffer *const &lhs, Buffer *const &rhs)
This is used for sorting buffers into the most efficient write order.
Definition: CubeIoHandler.cpp:688
Isis::EndianSwapper
Byte swapper.
Definition: EndianSwapper.h:38
Isis::CubeIoHandler::m_rawData
QMap< int, RawCubeChunk * > * m_rawData
The map from chunk index to chunk for cached data.
Definition: CubeIoHandler.h:308
Isis::CubeIoHandler::flushWriteCache
void flushWriteCache(bool force=false) const
This attempts to write the so-far-unwritten buffers from m_writeCache into the cube's RawCubeChunk ca...
Definition: CubeIoHandler.cpp:947
Isis::CubeIoHandler::clearCache
void clearCache(bool blockForWriteCache=true) const
Free all cube chunks (cached cube data) from memory and write them to disk.
Definition: CubeIoHandler.cpp:383
Isis::CubeIoHandler::m_writeCache
QPair< QMutex *, QList< Buffer * > > * m_writeCache
These are the buffers we need to write to raw cube chunks.
Definition: CubeIoHandler.h:335
Isis::CubeIoHandler::findCubeChunks
QPair< QList< RawCubeChunk * >, QList< int > > findCubeChunks(int startSample, int numSamples, int startLine, int numLines, int startBand, int numBands) const
Get the cube chunks that correspond to the given cube area.
Definition: CubeIoHandler.cpp:737
Isis::CubeIoHandler::write
void write(const Buffer &bufferToWrite)
Write buffer data into the cube data on disk.
Definition: CubeIoHandler.cpp:338
QMap
This is free and unencumbered software released into the public domain.
Definition: CubeIoHandler.h:22
Isis::CubeIoHandler::BufferToChunkWriter
This class is designed to handle write() asynchronously.
Definition: CubeIoHandler.h:185
Isis::CubeIoHandler::getChunkCountInLineDimension
int getChunkCountInLineDimension() const
Definition: CubeIoHandler.cpp:505
Isis::CubeIoHandler::writeNullDataToDisk
void writeNullDataToDisk() const
Write all NULL cube chunks that have not yet been accessed to disk.
Definition: CubeIoHandler.cpp:1839
Isis::CubeIoHandler::BufferToChunkWriter::operator=
BufferToChunkWriter & operator=(const BufferToChunkWriter &rhs)
This is disabled.
Isis::CubeIoHandler::m_pixelType
PixelType m_pixelType
The format of each DN in the cube.
Definition: CubeIoHandler.h:281
Isis::CubeIoHandler::getChunkCount
int getChunkCount() const
Definition: CubeIoHandler.cpp:1093
Isis::CubeIoHandler::setChunkSizes
void setChunkSizes(int numSamples, int numLines, int numBands)
This should be called once from the child constructor.
Definition: CubeIoHandler.cpp:621
Isis::CubeIoHandler::CubeIoHandler
CubeIoHandler(const CubeIoHandler &other)
Disallow copying of this object.
Isis::CubeIoHandler::readRaw
virtual void readRaw(RawCubeChunk &chunkToFill)=0
This needs to populate the chunkToFill with unswapped raw bytes from the disk.
Isis::CubeIoHandler::m_startByte
BigInt m_startByte
The start byte of the cube data.
Definition: CubeIoHandler.h:278
Isis::CubeIoHandler::m_cachingAlgorithms
QList< CubeCachingAlgorithm * > * m_cachingAlgorithms
The caching algorithms to use, in order of priority.
Definition: CubeIoHandler.h:305
Isis::CubeIoHandler::BufferToChunkWriter::m_buffersToWrite
QList< Buffer * > * m_buffersToWrite
The buffers that need pushed into the IO handler; we own these.
Definition: CubeIoHandler.h:210
Isis::CubeIoHandler::m_useOptimizedCubeWrite
bool m_useOptimizedCubeWrite
This is true if the Isis preference for the cube write thread is optimized.
Definition: CubeIoHandler.h:353
Isis::CubeIoHandler::getLineCountInChunk
int getLineCountInChunk() const
Definition: CubeIoHandler.cpp:579
Isis::CubeIoHandler::m_lastOperationWasWrite
bool m_lastOperationWasWrite
If the last operation was a write then we need to flush the cache when reading.
Definition: CubeIoHandler.h:348
Isis
This is free and unencumbered software released into the public domain.
Definition: Apollo.h:16
Isis::CubeIoHandler::getDataSize
BigInt getDataSize() const
Definition: CubeIoHandler.cpp:425