//# TSMCube.cc: Tiled Hypercube Storage Manager for tables //# Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002 //# Associated Universities, Inc. Washington DC, USA. //# //# This library is free software; you can redistribute it and/or modify it //# under the terms of the GNU Library General Public License as published by //# the Free Software Foundation; either version 2 of the License, or (at your //# option) any later version. //# //# This library is distributed in the hope that it will be useful, but WITHOUT //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public //# License for more details. //# //# You should have received a copy of the GNU Library General Public License //# along with this library; if not, write to the Free Software Foundation, //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. //# //# Correspondence concerning AIPS++ should be addressed as follows: //# Internet email: aips2-request@nrao.edu. //# Postal address: AIPS++ Project Office //# National Radio Astronomy Observatory //# 520 Edgemont Road //# Charlottesville, VA 22903-2475 USA //# //# $Id: TSMCube.cc 21521 2014-12-10 08:06:42Z gervandiepen $ //# Includes #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // for memcpy #include namespace casacore { //# NAMESPACE CASACORE - BEGIN // memcpy with constant argument is inlined #define TSM_COPY(a, b, n) case n: memcpy(a, b, n); break static void TSMCube_MoveData(char * a, char * b, int n) { switch (n) { case 0: break; TSM_COPY(a, b, 1); TSM_COPY(a, b, 2); TSM_COPY(a, b, 3); TSM_COPY(a, b, 4); TSM_COPY(a, b, 5); TSM_COPY(a, b, 6); TSM_COPY(a, b, 7); TSM_COPY(a, b, 8); TSM_COPY(a, b, 12); TSM_COPY(a, b, 16); TSM_COPY(a, b, 20); TSM_COPY(a, b, 24); TSM_COPY(a, b, 28); TSM_COPY(a, b, 32); default: memcpy(a, b, n); } } #undef TSM_COPY TSMCube::TSMCube (TiledStMan* stman, TSMFile* file, const IPosition& cubeShape, const IPosition& tileShape, const Record& values, Int64 fileOffset, Bool useDerived) : cachedTile_p (0), stmanPtr_p (stman), useDerived_p (useDerived), values_p (values), extensible_p (False), nrdim_p (0), nrTiles_p (0), tileSize_p (0), filePtr_p (file), fileOffset_p (0), cache_p (0), userSetCache_p (False), lastColAccess_p(NoAccess) { if (fileOffset < 0) { // TiledCellStMan uses an empty shape; setShape is called later. if (! cubeShape.empty()) { // A shape is given, so set it. extensible_p = cubeShape(cubeShape.nelements()-1) == 0; setShape (cubeShape, tileShape); } } else { // Meant for TiledFileAccess. nrdim_p = cubeShape.nelements(); cubeShape_p = cubeShape; tileShape_p = tileShape; fileOffset_p = fileOffset; setup(); } } TSMCube::TSMCube (TiledStMan* stman, AipsIO& ios, Bool useDerived) : cachedTile_p (0), stmanPtr_p (stman), useDerived_p (useDerived), filePtr_p (0), cache_p (0), userSetCache_p (False), lastColAccess_p(NoAccess) { Int fileSeqnr = getObject (ios); if (fileSeqnr >= 0) { filePtr_p = stmanPtr_p->getFile (fileSeqnr); } // Calculate the various variables. setup(); } TSMCube::~TSMCube() { delete cache_p; delete [] cachedTile_p; } void TSMCube::clearCache (Bool doFlush) { if (doFlush) { flushCache(); } if (cache_p != 0) { cache_p->clear (0, False); } } void TSMCube::emptyCache() { if (cache_p != 0) { cache_p->resize (0); } userSetCache_p = False; lastColAccess_p = NoAccess; } void TSMCube::showCacheStatistics (ostream& os) const { if (cache_p != 0) { os << ">>> TSMCube cache statistics:" << endl; os << "cubeShape: " << cubeShape_p << endl; os << "tileShape: " << tileShape_p << endl; os << "maxCacheSz:" << stmanPtr_p->maximumCacheSize() << " MiB" << endl; cache_p->showStatistics (os); os << "<<<" << endl; } } uInt TSMCube::coordinateSize (const String& coordinateName) const { if (! values_p.isDefined (coordinateName)) { return 0; } IPosition shape (values_p.shape (coordinateName)); if (shape.nelements() == 0) { return 0; } return shape(0); } IPosition TSMCube::cellShape() const { uInt nr = stmanPtr_p->nrCoordVector(); if (nr < cubeShape_p.nelements()) { return cubeShape_p.getFirst (nr); } return cubeShape_p; } IPosition TSMCube::adjustTileShape (const IPosition& cubeShape, const IPosition& tileShape) const { // Make this function independent of the length of tileShape, // so it can be shorter or longer than the cube shape. // The returned tile shape always has the length of the cube shape. uInt nrdim = cubeShape.nelements(); // Make length of tile shape equal to length of cube shape. // Fill with 0 (meaning undefined tile axes). IPosition tileShp (nrdim, 0); IPosition cubeUnk (nrdim); uInt nrunk = 0; uInt length = 1; for (uInt i=0; i cubeShape(i) && cubeShape(i) != 0) { tileShp(i) = cubeShape(i); } length *= tileShp(i); } } cubeUnk.resize (nrunk); // Calculate a default for the unknown tile axes. // Use the remainder of the 32768 for it. if (nrunk > 0) { Float rem = 32768. / length; Int leng = max(1, Int(rem + 0.5)); IPosition tileUnk = TiledStMan::makeTileShape (cubeUnk, 0.5, leng); length *= tileUnk.product(); uInt j = 0; for (uInt i=0; icheckCubeShape (this, cubeShape); // If the shape is redefined, the cache may already exist. // So delete it first. deleteCache(); fileOffset_p = filePtr_p->length(); nrdim_p = cubeShape.nelements(); // Resize the tile section member variables used in accessSection() resizeTileSections(); cubeShape_p = cubeShape; tileShape_p = adjustTileShape (cubeShape, tileShape); // Calculate the various variables. setup(); // If used directly, create the cache. // It has to be done here, otherwise the file does not get extended if // no explicit put is done. if (!useDerived_p) { makeCache(); } // Tell TSMFile that the file gets extended. filePtr_p->extend (nrTiles_p * bucketSize_p); // Initialize the coordinate columns (as far as needed). stmanPtr_p->initCoordinates (this); // Set flag if writing. stmanPtr_p->setDataChanged(); } void TSMCube::putObject (AipsIO& ios) { flushCache(); // If the offset is small enough, write it as an old style file, // so older software can still read it. Bool vers1 = (fileOffset_p < 2u*1024u*1024u*1024u); if (vers1) { ios << 1; // version 1 } else { ios << 2; // version 2 } ios << values_p; ios << extensible_p; ios << nrdim_p; ios << cubeShape_p; ios << tileShape_p; Int seqnr = -1; if (filePtr_p != 0) { seqnr = filePtr_p->sequenceNumber(); } ios << seqnr; if (vers1) { ios << uInt(fileOffset_p); } else { ios << fileOffset_p; } } Int TSMCube::getObject (AipsIO& ios) { uInt version; Int fileSeqnr; ios >> version; ios >> values_p; ios >> extensible_p; ios >> nrdim_p; ios >> cubeShape_p; ios >> tileShape_p; ios >> fileSeqnr; if (version == 1) { uInt offs; ios >> offs; fileOffset_p = offs; } else { ios >> fileOffset_p; } return fileSeqnr; } void TSMCube::resync (AipsIO& ios) { getObject (ios); setupNrTiles(); resyncCache(); } void TSMCube::setup() { // Determine the nr of tiles in all but the last dimension. // This is needed when extending the last dimension. // Also determine the nr of tiles needed (total and per dimension). setupNrTiles(); expandedTileShape_p = TSMShape (tileShape_p); expandedTilesPerDim_p = TSMShape (tilesPerDim_p); // Determine the bucket size for the cache. // Also determine the start offset for each data column // and the length of the tile if converted to local format. tileSize_p = tileShape_p.product(); bucketSize_p = stmanPtr_p->getLengthOffset (tileSize_p, externalOffset_p, localOffset_p, localTileLength_p); // Resize IPosition member variables used in accessSection() resizeTileSections(); } void TSMCube::setupNrTiles() { // Determine the nr of tiles in all but the last dimension. // This is needed when extending the last dimension. // Also determine the nr of tiles needed (total and per dimension). tilesPerDim_p.resize (nrdim_p); nrTiles_p = 1; for (uInt i=0; ibucketFile(), fileOffset_p, bucketSize_p, nrTiles_p, 1, this, readCallBack, writeCallBack, initCallBack, deleteCallBack); } } void TSMCube::flushCache() { if (cache_p != 0) { cache_p->flush(); } } void TSMCube::resyncCache() { if (cache_p != 0) { cache_p->resync (nrTiles_p, 0, -1); } } void TSMCube::deleteCache() { delete cache_p; cache_p = 0; } Bool TSMCube::isExtensible() const { return extensible_p; } void TSMCube::extend (uInt64 nr, const Record& coordValues, const TSMColumn* lastCoordColumn) { if (!extensible_p) { throw TSMError ("Hypercube in TSM " + stmanPtr_p->dataManagerName() + " is not extensible"); } // Make the cache here, otherwise nrTiles_p is too high. makeCache(); uInt lastDim = nrdim_p - 1; uInt nrold = nrTiles_p; cubeShape_p(lastDim) += nr; tilesPerDim_p(lastDim) = (cubeShape_p(lastDim) + tileShape_p(lastDim) - 1) / tileShape_p(lastDim); nrTiles_p = nrTilesSubCube_p * tilesPerDim_p(lastDim); getCache()->extend (nrTiles_p - nrold); filePtr_p->extend ((nrTiles_p - nrold) * bucketSize_p); // Update the last coordinate (if there). if (lastCoordColumn != 0) { extendCoordinates (coordValues, lastCoordColumn->columnName(), cubeShape_p(lastDim)); } } void TSMCube::extendCoordinates (const Record& coordValues, const String& name, uInt length) { //# Determine if the coordinate field is already defined. Bool defined = values_p.isDefined (name); //# Determine the extension length of the coordinate vector. //# This is the given length (which is the entire cube axis) //# minus already defined coordinate length. uInt vectorLength = length; if (defined) { vectorLength -= coordinateSize (name); } //# Exit if no extend. if (vectorLength == 0) { return; } //# Determine the start and end of the new coordinate values. //# If they are not defined, start will be > end. IPosition start(1, length); IPosition end(1, length-1); if (coordValues.isDefined (name)) { IPosition shape = coordValues.shape (name); if (shape.nelements() > 0) { start(0) -= shape(0); } } //# Now insert the new coordinate values. //# Note that the nr of new coordinate values can be less than //# the coordinate vector extension length. //# The algorithm is as follows: //# - Define coordinate values if not defined yet. //# - Extend the coordinate vector with default values. //# - Insert the new coordinate values. switch (stmanPtr_p->coordinateDataType (name)) { case TpBool: case TpArrayBool: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = False; Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayBool (name); } } break; case TpInt: case TpArrayInt: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = 0; Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayInt (name); } } break; case TpUInt: case TpArrayUInt: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = 0; Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayuInt (name); } } break; case TpFloat: case TpArrayFloat: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = 0; Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayFloat (name); } } break; case TpDouble: case TpArrayDouble: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = 0; Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayDouble (name); } } break; case TpComplex: case TpArrayComplex: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = Complex(0); Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayComplex (name); } } break; case TpDComplex: case TpArrayDComplex: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = DComplex(0); Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayDComplex (name); } } break; case TpString: case TpArrayString: { if (!defined) { values_p.define (name, Array()); } RecordFieldPtr > field (values_p, name); Array& array = *field; if (vectorLength > 0) { Vector vector(vectorLength); vector = ""; Array newArray (concatenateArray (array, vector)); array.reference (newArray); } if (start(0) < Int(length)) { array(start, end) = coordValues.toArrayString (name); } } break; default: throw DataManInvDT ("extendCoordinates in TSM " + stmanPtr_p->dataManagerName()); } } Bool TSMCube::matches (const PtrBlock& idColSet, const Record& idValues) { for (uInt i=0; icolumnName(); switch (values_p.dataType (name)) { case TpBool: if (idValues.asBool (name) != values_p.asBool (name)) { return False; } break; case TpString: if (idValues.asString (name) != values_p.asString (name)) { return False; } break; case TpComplex: case TpDComplex: { const DComplex& idVal = idValues.asDComplex (name); const DComplex& val = values_p.asDComplex (name); if (idVal != val) { return False; } } break; default: if (idValues.asdouble (name) != values_p.asdouble (name)) { return False; } break; } } return True; } char* TSMCube::readCallBack (void* owner, const char* external) { return ((TSMCube*)owner)->readTile (external); } char* TSMCube::readTile (const char* external) { char* local = 0; if (cachedTile_p != 0){ local = cachedTile_p; cachedTile_p = 0; } else { local = new char[localTileLength_p]; } stmanPtr_p->readTile (local, localOffset_p, external, externalOffset_p, tileSize_p); return local; } void TSMCube::writeCallBack (void* owner, char* external, const char* local) { ((TSMCube*)owner)->writeTile (external, local); } void TSMCube::writeTile (char* external, const char* local) { stmanPtr_p->writeTile (external, externalOffset_p, local, localOffset_p, tileSize_p); } void TSMCube::deleteCallBack (void* owner, char* buffer) { TSMCube * tsmCube = ((TSMCube*)owner); if (tsmCube->cachedTile_p == 0){ tsmCube->cachedTile_p = buffer; } else { delete [] buffer; } } char* TSMCube::initCallBack (void* owner) { uInt64 size = ((TSMCube*)owner)->localTileLength(); char* buffer = new char[size]; memset(buffer, 0, size); return buffer; } uInt TSMCube::cacheSize() const { if (cache_p == 0) { return 0; } return cache_p->cacheSize(); } uInt TSMCube::validateCacheSize (uInt cacheSize) const { return validateCacheSize (cacheSize, stmanPtr_p->maximumCacheSize(), bucketSize_p); } uInt TSMCube::validateCacheSize (uInt cacheSize, uInt maxSizeMiB, uInt bucketSize) { // An overdraft of 10% is allowed. uInt maxnb = std::max(1u, uInt(1024. * 1024. * maxSizeMiB / bucketSize)); if (maxSizeMiB > 0 && cacheSize > maxnb) { if (10 * cacheSize > 11 * maxnb) { return maxnb; } } return cacheSize; } void TSMCube::setCacheSize (uInt cacheSize, Bool forceSmaller, Bool userSet) { // Resize the cache in the expectation that this access is // the first of a bunch of accesses at the same tiles. // However, don't let the cache exceed the maximum, // unless it is only 10% more. BucketCache* cachePtr = getCache(); cacheSize = validateCacheSize (cacheSize); if (forceSmaller || cacheSize > cachePtr->cacheSize()) { cachePtr->resize (cacheSize); } //// cout << "cachesize=" << cacheSize << endl; userSetCache_p = userSet; } // Set the cache size for the given slice and access path. void TSMCube::setCacheSize (const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath, Bool forceSmaller, Bool userSet) { uInt cacheSize = calcCacheSize (sliceShape, windowStart, windowLength, axisPath); // If not userset, do not cache if more than 25% of the memory is needed. if (!userSet) { uInt maxSize = uInt(HostInfo::memoryTotal(True) * 1024.*0.25 / bucketSize_p); if (cacheSize > maxSize) { cacheSize = 1; } } setCacheSize (cacheSize, forceSmaller, userSet); } // Calculate the cache size for the given slice and access path. uInt TSMCube::calcCacheSize (const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath) const { return calcCacheSize (cubeShape_p, tileShape_p, extensible_p, sliceShape, windowStart, windowLength, axisPath, stmanPtr_p->maximumCacheSize(), bucketSize_p); } uInt TSMCube::calcCacheSize (const IPosition& cubeShape, const IPosition& tileShape, Bool extensible, const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath, uInt maxCacheSize, uInt bucketSize) { uInt nrdim = cubeShape.nelements(); if (sliceShape.nelements() > nrdim || windowStart.nelements() > nrdim || windowLength.nelements() > nrdim || axisPath.nelements() > nrdim) { throw TSMError ("calcCacheSize: invalid arguments"); } uInt i; // The unspecified sliceShape dimensions are 1. IPosition slice(nrdim, 1); for (i=0; i 0) { slice(i) = sliceShape(i); } } // The unspecified window start dimensions are 0. IPosition start(nrdim, 0); for (i=0; i 0) { nrd--; // Caching is needed if lower dimensions are reused because // a higher dimension loops through a tile. // So skip dimensions until a reused dimension is found. uInt nr = nrd; while (nr > 0 && reused(nr) == 0) { nr--; } // The cache needs to contain the tiles needed for the entire window // of the remaining axes. // If a tile is reused, we also need to take into account // the number of tiles needed for the slice. uInt64 cacheSize = 1; for (i=0; i 0) { for (i=nr+1; i<=nrd; i++) { cacheSize *= sliceTiles(i); } } if (cacheSize == validateCacheSize (cacheSize, maxCacheSize, bucketSize)) { return cacheSize; } nrd = nr; } return 1; } void TSMCube::resizeTileSections() { // Resize to dimension nrdim_p if (nrTileSection_p.nelements() != nrdim_p) { nrTileSection_p.resize(nrdim_p); startTile_p.resize(nrdim_p); endTile_p.resize(nrdim_p); startPixelInFirstTile_p.resize(nrdim_p); endPixelInFirstTile_p.resize(nrdim_p); endPixelInLastTile_p.resize(nrdim_p); } return; } void TSMCube::accessSection (const IPosition& start, const IPosition& end, char* section, uInt colnr, uInt localPixelSize, uInt, Bool writeFlag) { // Set flag if writing. if (writeFlag) { stmanPtr_p->setDataChanged(); } // Prepare for the iteration through the necessary tiles. uInt i, j; // Initialize the various variables and determine the number of // tiles needed (which will determine the cache size). // Also determine if the slice happens to be an entire slice // or if it is a line (this cases occur quite often and can be // handled in a more optimal way). Bool oneEntireTile = True; uInt lineIndex = 0; uInt nOneLong = 0; for (i=0; igetBucket (tileNr); // If writing, set cache slot to dirty. if (writeFlag) { memcpy (dataArray+pixelOffset, section, tileSize_p * localPixelSize); cachePtr->setDirty(); }else{ memcpy (section, dataArray+pixelOffset, tileSize_p * localPixelSize); } return; } // If the section is a line, call a specialized function. // Note that a single pixel is also handled as a line. if (nOneLong >= nrdim_p - 1) { accessLine (section, pixelOffset, localPixelSize, writeFlag, cachePtr, startTile_p, endTile_p(lineIndex), startPixelInFirstTile_p, endPixelInLastTile_p(lineIndex), lineIndex); return; } // At this point we start looping through all tiles. // startPixel and endPixel will contain the first and last pixels // needed in the current tile. // tilePos contains the position of the current tile. IPosition startSection (start); // start of section in cube IPosition sectionShape (end - start + 1); // section shape TSMShape expandedSectionShape (sectionShape); IPosition startPixel (startPixelInFirstTile_p); IPosition endPixel (endPixelInFirstTile_p); IPosition tilePos (startTile_p); IPosition tileIncr = expandedTilesPerDim_p.offsetIncrement (nrTileSection_p); IPosition dataLength(nrdim_p); IPosition dataPos (nrdim_p); IPosition sectionPos(nrdim_p); uInt dataOffset; size_t sectionOffset; uInt tileNr = expandedTilesPerDim_p.offset (tilePos); while (True) { // cout << "tilePos=" << tilePos << endl; // cout << "tileNr=" << tileNr << endl; // cout << "start=" << startPixel << endl; // cout << "end=" << endPixel << endl; // Get the tile from the cache. // Set it to dirty if we are writing. char* dataArray = cachePtr->getBucket (tileNr); if (writeFlag) { cachePtr->setDirty(); } // At this point we start looping through all pixels in the tile. // We do a vector at a time. // Calculate the start and end pixel in the tile. // Initialize the pixel position in the data and section. for (i=0; i