Isis 3 Programmer Reference
CubeIoHandler.h
1
6/* SPDX-License-Identifier: CC0-1.0 */
7
8#ifndef CubeIoHandler_h
9#define CubeIoHandler_h
10
11#include <QRunnable>
12#include <QThreadPool>
13
14#include "Constants.h"
15#include "Endian.h"
16#include "PixelType.h"
17
18class QFile;
19class QMutex;
20class QTime;
21template <typename A> class QList;
22template <typename A, typename B> class QMap;
23template <typename A, typename B> struct QPair;
24
25namespace Isis {
26 class Buffer;
27 class CubeCachingAlgorithm;
28 class EndianSwapper;
29 class Pvl;
30 class RawCubeChunk;
31
110 public:
111 CubeIoHandler(QFile * dataFile, const QList<int> *virtualBandList,
112 const Pvl &label, bool alreadyOnDisk);
113 virtual ~CubeIoHandler();
114
115 void read(Buffer &bufferToFill) const;
116 void write(const Buffer &bufferToWrite);
117
119 void clearCache(bool blockForWriteCache = true) const;
120 BigInt getDataSize() const;
121 void setVirtualBands(const QList<int> *virtualBandList);
127 virtual void updateLabels(Pvl &labels) = 0;
128
129 QMutex *dataFileMutex();
130
131 protected:
132 int bandCount() const;
133 int getBandCountInChunk() const;
134 BigInt getBytesPerChunk() const;
138 int getChunkIndex(const RawCubeChunk &) const;
139 BigInt getDataStartByte() const;
140 QFile * getDataFile();
141 int lineCount() const;
142 int getLineCountInChunk() const;
143 PixelType pixelType() const;
144 int sampleCount() const;
145 int getSampleCountInChunk() const;
146
147 void setChunkSizes(int numSamples, int numLines, int numBands);
148
157 virtual void readRaw(RawCubeChunk &chunkToFill) = 0;
158
166 virtual void writeRaw(const RawCubeChunk &chunkToWrite) = 0;
167
168 private:
186 public:
188 QList<Buffer *> buffersToWrite);
190
191 void run();
192
193 private:
205
206 private:
210 QList<Buffer * > * m_buffersToWrite;
212 QTime *m_timer;
213 };
214
215
222
231
232 void blockUntilThreadPoolEmpty() const;
233
234 static bool bufferLessThan(Buffer * const &lhs, Buffer * const &rhs);
235
236 QPair< QList<RawCubeChunk *>, QList<int> > findCubeChunks(int startSample, int numSamples,
237 int startLine, int numLines,
238 int startBand, int numBands) const;
239
240 void findIntersection(const RawCubeChunk &cube1,
241 const Buffer &cube2, int &startX, int &startY, int &startZ,
242 int &endX, int &endY, int &endZ) const;
243
244 void flushWriteCache(bool force = false) const;
245
246 void freeChunk(RawCubeChunk *chunkToFree) const;
247
248 RawCubeChunk *getChunk(int chunkIndex, bool allocateIfNecessary) const;
249
250 int getChunkCount() const;
251
252 void getChunkPlacement(int chunkIndex,
253 int &startSample, int &startLine, int &startBand,
254 int &endSample, int &endLine, int &endBand) const;
255
256 RawCubeChunk *getNullChunk(int chunkIndex) const;
257
258 void minimizeCache(const QList<RawCubeChunk *> &justUsed,
259 const Buffer &justRequested) const;
260
261 void synchronousWrite(const Buffer &bufferToWrite);
262
263 void writeIntoDouble(const RawCubeChunk &chunk, Buffer &output, int startIndex) const;
264
265 void writeIntoRaw(const Buffer &buffer, RawCubeChunk &output, int index) const;
266
267 void writeNullDataToDisk() const;
268
269 private:
271 QFile * m_dataFile;
272
279
282
284 double m_base;
285
288
291
294
297
300
303
305 QList<CubeCachingAlgorithm *> * m_cachingAlgorithms;
306
308 mutable QMap<int, RawCubeChunk *> * m_rawData;
309
311 mutable QMap<int, bool> * m_dataIsOnDiskMap;
312
314 QList<int> * m_virtualBands;
315
318
321
324
329 mutable QList<RawCubeChunk *> *m_lastProcessByLineChunks;
330
332 mutable QByteArray *m_nullChunkData;
333
335 QPair< QMutex *, QList<Buffer *> > *m_writeCache;
336
342 mutable QThreadPool *m_ioThreadPool;
343
354
357
359 mutable volatile int m_idealFlushSize;
360
363 };
364}
365
366#endif
Buffer for reading and writing cube data.
Definition Buffer.h:53
This is the parent of the caching algorithms.
This class is designed to handle write() asynchronously.
BufferToChunkWriter & operator=(const BufferToChunkWriter &rhs)
This is disabled.
QTime * m_timer
Used to calculate the lifetime of an instance of this class.
BufferToChunkWriter(const BufferToChunkWriter &other)
This is disabled.
BufferToChunkWriter(CubeIoHandler *ioHandler, QList< Buffer * > buffersToWrite)
Create a BufferToChunkWriter which is designed to asynchronously move the given buffers into the cube...
~BufferToChunkWriter()
We're done writing our buffers into the cube, clean up.
QList< Buffer * > * m_buffersToWrite
The buffers that need pushed into the IO handler; we own these.
void run()
This is the asynchronous computation.
CubeIoHandler * m_ioHandler
The IO Handler instance to put the buffers into.
Handles converting buffers to and from disk.
bool m_useOptimizedCubeWrite
This is true if the Isis preference for the cube write thread is optimized.
QList< CubeCachingAlgorithm * > * m_cachingAlgorithms
The caching algorithms to use, in order of priority.
int m_consecutiveOverflowCount
How many times the write cache has overflown in a row.
int m_numBands
The number of physical bands in the cube.
int m_bandsInChunk
The number of physical bands in a cube chunk.
void clearCache(bool blockForWriteCache=true) const
Free all cube chunks (cached cube data) from memory and write them to disk.
int m_samplesInChunk
The number of samples in a cube chunk.
QMap< int, bool > * m_dataIsOnDiskMap
The map from chunk index to on-disk status, all true if not allocated.
void freeChunk(RawCubeChunk *chunkToFree) const
If the chunk is dirty, then we write it to disk.
int m_linesInChunk
The number of lines in a cube chunk.
CubeIoHandler & operator=(const CubeIoHandler &other)
Disallow assignments of this object.
virtual ~CubeIoHandler()
Cleans up all allocated memory.
void addCachingAlgorithm(CubeCachingAlgorithm *algorithm)
This will add the given caching algorithm to the list of attempted caching algorithms.
int m_numSamples
The number of samples in the cube.
QMutex * dataFileMutex()
Get the mutex that this IO handler is using around I/Os on the given data file.
void blockUntilThreadPoolEmpty() const
This blocks (doesn't return) until the number of active runnables in the thread pool goes to 0.
QPair< QList< RawCubeChunk * >, QList< int > > findCubeChunks(int startSample, int numSamples, int startLine, int numLines, int startBand, int numBands) const
Get the cube chunks that correspond to the given cube area.
PixelType pixelType() const
void writeNullDataToDisk() const
Write all NULL cube chunks that have not yet been accessed to disk.
QMutex * m_writeThreadMutex
This enables us to block while the write thread is working.
QList< int > * m_virtualBands
Converts from virtual band to physical band.
volatile int m_idealFlushSize
Ideal write cache flush size.
int getChunkCountInLineDimension() const
QList< RawCubeChunk * > * m_lastProcessByLineChunks
This is an optimization for process by line.
bool m_lastOperationWasWrite
If the last operation was a write then we need to flush the cache when reading.
void setChunkSizes(int numSamples, int numLines, int numBands)
This should be called once from the child constructor.
PixelType m_pixelType
The format of each DN in the cube.
QByteArray * m_nullChunkData
A raw cube chunk's data when it was all NULL's. Used for speed.
int getLineCountInChunk() const
double m_multiplier
The multiplicative factor of the data on disk.
int m_numLines
The number of lines in the cube.
int getChunkCountInSampleDimension() const
void minimizeCache(const QList< RawCubeChunk * > &justUsed, const Buffer &justRequested) const
Apply the caching algorithms and get rid of excess cube data in memory.
virtual void readRaw(RawCubeChunk &chunkToFill)=0
This needs to populate the chunkToFill with unswapped raw bytes from the disk.
QFile * m_dataFile
The file containing cube data.
int getChunkCountInBandDimension() const
QPair< QMutex *, QList< Buffer * > > * m_writeCache
These are the buffers we need to write to raw cube chunks.
double m_base
The additive offset of the data on disk.
CubeIoHandler(QFile *dataFile, const QList< int > *virtualBandList, const Pvl &label, bool alreadyOnDisk)
Creates a new CubeIoHandler using a RegionalCachingAlgorithm.
BigInt getBytesPerChunk() const
QMap< int, RawCubeChunk * > * m_rawData
The map from chunk index to chunk for cached data.
QThreadPool * m_ioThreadPool
This contains threads for doing cube writes (and later maybe cube reads).
BigInt getDataStartByte() const
void findIntersection(const RawCubeChunk &cube1, const Buffer &cube2, int &startX, int &startY, int &startZ, int &endX, int &endY, int &endZ) const
Find the intersection between the buffer area and the cube chunk.
void flushWriteCache(bool force=false) const
This attempts to write the so-far-unwritten buffers from m_writeCache into the cube's RawCubeChunk ca...
virtual void writeRaw(const RawCubeChunk &chunkToWrite)=0
This needs to write the chunkToWrite directly to disk with no modifications to the data itself.
void getChunkPlacement(int chunkIndex, int &startSample, int &startLine, int &startBand, int &endSample, int &endLine, int &endBand) const
Get the X/Y/Z (Sample,Line,Band) range of the chunk at the given index.
void read(Buffer &bufferToFill) const
Read cube data from disk into the buffer.
RawCubeChunk * getNullChunk(int chunkIndex) const
This creates a chunk filled with NULLs whose placement is at chunkIndex's position.
void write(const Buffer &bufferToWrite)
Write buffer data into the cube data on disk.
BigInt m_startByte
The start byte of the cube data.
void setVirtualBands(const QList< int > *virtualBandList)
This changes the virtual band list.
RawCubeChunk * getChunk(int chunkIndex, bool allocateIfNecessary) const
Retrieve the cached chunk at the given chunk index, if there is one.
static bool bufferLessThan(Buffer *const &lhs, Buffer *const &rhs)
This is used for sorting buffers into the most efficient write order.
virtual void updateLabels(Pvl &labels)=0
Function to update the labels with a Pvl object.
int getSampleCountInChunk() const
BigInt getDataSize() const
CubeIoHandler(const CubeIoHandler &other)
Disallow copying of this object.
int getBandCountInChunk() const
void writeIntoRaw(const Buffer &buffer, RawCubeChunk &output, int index) const
Write the intersecting area of the buffer into the chunk.
void synchronousWrite(const Buffer &bufferToWrite)
This method takes the given buffer and synchronously puts it into the Cube's cache.
EndianSwapper * m_byteSwapper
A helper that swaps byte order to and from file order.
void writeIntoDouble(const RawCubeChunk &chunk, Buffer &output, int startIndex) const
Write the intersecting area of the chunk into the buffer.
int getChunkIndex(const RawCubeChunk &) const
Given a chunk, what's its index in the file.
ByteOrder m_byteOrder
The byte order (endianness) of the data on disk.
Container for cube-like labels.
Definition Pvl.h:119
A section of raw data on the disk.
This is free and unencumbered software released into the public domain.
This is free and unencumbered software released into the public domain.
ByteOrder
Tests the current architecture for byte order.
Definition Endian.h:42
This is free and unencumbered software released into the public domain.
Definition Apollo.h:16
long long int BigInt
Big int.
Definition Constants.h:49
PixelType
Enumerations for Isis Pixel Types.
Definition PixelType.h:27
This is free and unencumbered software released into the public domain.