//# TiledStMan.h: Base class for Tiled Storage Managers //# Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003 //# Associated Universities, Inc. Washington DC, USA. //# //# This library is free software; you can redistribute it and/or modify it //# under the terms of the GNU Library General Public License as published by //# the Free Software Foundation; either version 2 of the License, or (at your //# option) any later version. //# //# This library is distributed in the hope that it will be useful, but WITHOUT //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public //# License for more details. //# //# You should have received a copy of the GNU Library General Public License //# along with this library; if not, write to the Free Software Foundation, //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. //# //# Correspondence concerning AIPS++ should be addressed as follows: //# Internet email: aips2-request@nrao.edu. //# Postal address: AIPS++ Project Office //# National Radio Astronomy Observatory //# 520 Edgemont Road //# Charlottesville, VA 22903-2475 USA //# //# $Id$ #ifndef TABLES_TILEDSTMAN_H #define TABLES_TILEDSTMAN_H //# Includes #include #include #include #include #include #include #include namespace casacore { //# NAMESPACE CASACORE - BEGIN //# Forward Declarations class TSMColumn; class TSMDataColumn; class TSMCube; class TSMFile; class TableDesc; class Record; // // Base class for Tiled Storage Manager classes // // // // // //# Classes you should understand before using this one. //
  • Description of Tiled Storage Manager in module file // Tables.h //
  • DataManager //
  • TSMColumn //
  • ROTiledStManAccessor // for a discussion of the maximum cache size // // // TiledStMan is the base class for Tiled Storage Managers. // A tiled storage manager is capable of storing a hypercolumn // (as defined by // TableDesc::defineHypercolumn) // in one or more hypercubes. //
    It is not necessary to define a hypercolumn. If not defined, // it is assumed that all columns bound to this storage manager are // data columns. At least one of the columns must have a fixed // dimensionality and is used to determine the hypercube dimnensionality. //
    The general concept of these storage managers is explained in the // Tables module description. //

    // TiledStMan contains all common functions for the different tiled // storage managers. In particular, it contains functions // to check if the definitions of the shapes of hypercubes, coordinates, and // data cells are consistent. // It also contains various data members and functions to make them // persistent by writing them into an AipsIO stream. // // // This base class contains the common functionality of all // tiled storage managers. The base class is still abstract. // Only concrete tiled storage managers derived from it can // be instantiated. //

    // Tiled storage managers make access to array data possible with // more or less the same efficiency for access along different axes. // //# //# A List of bugs, limitations, extensions or planned refinements. //# class TiledStMan : public DataManager { public: // Create a TiledStMan. TiledStMan(); // Create a TiledStMan storage manager. // The given maximum cache size (in MibiByte) is persistent, // thus will be reused when the table is read back. Note that the class // ROTiledStManAccessor // allows one to overwrite the maximum cache size temporarily. // Its description contains a discussion about the effects of // setting a maximum cache. TiledStMan (const String& hypercolumnName, uInt maximumCacheSizeMiB); virtual ~TiledStMan(); // Get the name given to the storage manager. // This is the name of the hypercolumn. virtual String dataManagerName() const; void setDataManagerName (const String& newHypercolumnName); // Return a record containing data manager specifications. virtual Record dataManagerSpec() const; // Get data manager properties that can be modified. // It is only MaxCacheSize (the maximum cache size in MibiByte). // It is a subset of the data manager specification. virtual Record getProperties() const; // Modify data manager properties. // Only MaxCacheSize can be used. It is similar to function setCacheSize // with canExceedNrBuckets=False. virtual void setProperties (const Record& spec); // Set the flag to "data has changed since last flush". void setDataChanged(); // Derive the tile shape from the hypercube shape for the given // number of pixels per tile. It is tried to get the same number // of tiles for each dimension. // When a weight vector is given, the number of tiles for a dimension // is proportional to the weight. //
    After the initial guess it tries to optimize it by trying // to waste as little space as possible, while trying to keep as close // to the initial guess. The given tolerance (possibly per axis) // gives the minimum and maximum possible length of a tile axis // (minimum = initial_guess*tolerance; maximum = initial_guess/tolerance). // The heuristic is such that a tile axis length dividing the cube length // exactly is always favoured. // The test program tTiledStMan can be used to see how // the algorithm works out for a given tile size and cube shape. // static IPosition makeTileShape (const IPosition& hypercubeShape, Double tolerance = 0.5, uInt64 maxNrPixelsPerTile = 4*1024*1024); static IPosition makeTileShape (const IPosition& hypercubeShape, const Vector& weight, const Vector& tolerance, uInt64 maxNrPixelsPerTile = 4*1024*1024); // // Set the maximum cache size (in MiB) in a non-persistent way. virtual void setMaximumCacheSize (uInt nMiB); // Get the current maximum cache size (in MiB (MibiByte)). uInt maximumCacheSize() const; // Get the current cache size (in buckets) for the hypercube in // the given row. uInt cacheSize (rownr_t rownr) const; // Get the hypercube shape of the data in the given row. const IPosition& hypercubeShape (rownr_t rownr) const; // Get the tile shape of the data in the given row. const IPosition& tileShape (rownr_t rownr) const; // Get the bucket size (in bytes) of the hypercube in the given row. uInt64 bucketSize (rownr_t rownr) const; // Can the tiled storage manager handle changing array shapes? // The default is no (but TiledCellStMan can). virtual Bool canChangeShape() const; // Can the tiled storage manager access an entire column. // TiledColumnStMan can always do that. // The others might be able to do it (for this time). // The default implementation returns True if there is only 1 hypercube. virtual Bool canAccessColumn() const; // The data manager supports use of MultiFile. virtual Bool hasMultiFileSupport() const; // Calculate the cache size (in buckets) for accessing the hypercube // containing the given row. It takes the maximum cache size into // account (allowing an overdraft of 10%). // It uses the given axisPath (i.e. traversal order) to determine // the optimum size. A window can be specified to indicate that only // the given subset of the hypercube will be accessed. //
    // The length of the slice and window arguments and axisPath // must be less or equal to the dimensionality of the hypercube. // The non-specified windowStart parts default to 0. // The non-specified windowLength parts default to // the hypercube shape. // The non-specified sliceShape parts default to 1. //
    // Axispath = [2,0,1] indicates that the z-axis changes most rapidly, // thereafter x and y. An axis can occur only once in the axisPath. // The non-specified axisPath parts get the natural order. // E.g. in the previous example axisPath=[2] defines the same path. //
    When forceSmaller is False, the cache is not resized when the // new size is smaller. //
    A flag is set indicating that the TSMDataColumn // access functions do not need to size the cache. uInt calcCacheSize (rownr_t rownr, const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath) const; // Set the cache size using the calcCacheSize // function mentioned above. void setCacheSize (rownr_t rownr, const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath, Bool forceSmaller); // Set the cache size for accessing the hypercube containing the given row. // When the give cache size exceeds the maximum cache size with more // than 10%, the maximum cache size is used instead. //
    When forceSmaller is False, the cache is not resized when the // new size is smaller. //
    A flag is set indicating that the TSMDataColumn // access functions do not need to size the cache. void setCacheSize (rownr_t rownr, uInt nbuckets, Bool forceSmaller); // Sets the cache size using the hypercube instead of the row number. // Useful for iterating over all hypercubes. void setHypercubeCacheSize (uInt hypercube, uInt nbuckets, Bool forceSmaller); // Determine if the user set the cache size (using setCacheSize). Bool userSetCache (rownr_t rownr) const; // Empty the caches used by the hypercubes in this storage manager. // It will flush the caches as needed and remove all buckets from them // resulting in a possibly large drop in memory used. // It also clears the userSetCache flag. void emptyCaches(); // Show the statistics of all caches used. void showCacheStatistics (ostream& os) const; // Get the length of the data for the given number of pixels. // This can be used to calculate the length of a tile. uInt64 getLengthOffset (uInt64 nrPixels, Block& dataOffset, Block& localOffset, uInt& localTileLength) const; // Get the number of coordinate vectors. uInt nrCoordVector() const; // Get the nr of rows in this storage manager. rownr_t nrow() const; // Does the storage manager allow to add rows? (yes) Bool canAddRow() const; // Get the default tile shape. // By default it returns a zero-length IPosition. virtual IPosition defaultTileShape() const; // Return the number of hypercubes. uInt nhypercubes() const; // Test if only one hypercube is used by this storage manager. // If not, throw an exception. Otherwise return the hypercube. virtual TSMCube* singleHypercube(); // Get the given hypercube. // const TSMCube* getTSMCube (uInt hypercube) const; TSMCube* getTSMCube (uInt hypercube); // // Get the hypercube in which the given row is stored. // const TSMCube* getHypercube (rownr_t rownr) const; virtual TSMCube* getHypercube (rownr_t rownr) = 0; // // Get the hypercube in which the given row is stored. // It also returns the position of the row in that hypercube. virtual TSMCube* getHypercube (rownr_t rownr, IPosition& position) = 0; // Make the correct TSMCube type (depending on tsmOption()). TSMCube* makeTSMCube (TSMFile* file, const IPosition& cubeShape, const IPosition& tileShape, const Record& values, Int64 fileOffset=-1); // Read a tile and convert the data to local format. void readTile (char* local, const Block& localOffset, const char* external, const Block& externalOffset, uInt nrpixels); // Write a tile after converting the data to external format. void writeTile (char* external, const Block& externalOffset, const char* local, const Block& localOffset, uInt nrpixels); // Get the TSMFile object with the given sequence number. TSMFile* getFile (uInt sequenceNumber); // Open the storage manager for an existing table. virtual rownr_t open64 (rownr_t nrrow, AipsIO&); // Resync the storage manager with the new file contents. virtual rownr_t resync64 (rownr_t nrrow); // Reopen all files used in this storage manager for read/write access. virtual void reopenRW(); // The data manager will be deleted (because all its columns are // requested to be deleted). // So clean up the things needed (e.g. delete files). virtual void deleteManager(); // Create a column in the storage manager on behalf of a table column. // // Create a scalar column. DataManagerColumn* makeScalarColumn (const String& name, int dataType, const String& dataTypeID); // Create a direct array column. DataManagerColumn* makeDirArrColumn (const String& name, int dataType, const String& dataTypeID); // Create an indirect array column. DataManagerColumn* makeIndArrColumn (const String& name, int dataType, const String& dataTypeID); // // The TiledStMan wants to do reallocateColumn. Bool canReallocateColumns() const; // Reallocate the column object if it is part of this data manager. // It returns a pointer to the new column object. // It is used to remove the indirection of the TSMColumn objects // resulting in only one iso. two virtual column calls to get the data. DataManagerColumn* reallocateColumn (DataManagerColumn* column); // Set the shape and tile shape of a hypercube. // By default it throws an "impossible" exception. virtual void setShape (rownr_t rownr, TSMCube* hypercube, const IPosition& shape, const IPosition& tileShape); // Check the shape to be set for a hypercube. // It checks if it matches predefined (fixed shape) columns // and the shape of already defined coordinate columns. void checkCubeShape (const TSMCube* hypercube, const IPosition& cubeShape) const; // Get the data type of the coordinate column with the given name. // An exception is thrown when the column is unknown. int coordinateDataType (const String& columnName) const; // Initialize the new coordinates for the given cube. void initCoordinates (TSMCube* hypercube); // Get pointer to data column object. const TSMDataColumn* getDataColumn (uInt colnr) const { return dataCols_p[colnr]; } protected: // Set the persistent maximum cache size (in MiB). void setPersMaxCacheSize (uInt nMiB); // Get the bindings of the columns with the given names. // If bound, the pointer to the TSMColumn object is stored in the block. // If mustExist is True, an exception is thrown if the column // is not bound. // It returns the number of bound columns. uInt getBindings (const Vector& columnNames, PtrBlock& colSet, Bool mustExist) const; // Function setup calls this function to allow the derived class // to check specific information. In case of errors, an exception // should be thrown. // By default it does nothing. virtual void setupCheck (const TableDesc& tableDesc, const Vector& dataNames) const; // Get the table description needed for the hypercolumn description. virtual const TableDesc& getDesc() const; // Check if values are given in the record for all columns in // the block. Also check if the data types are correct. // An exception is thrown if something is incorrect. void checkValues (const PtrBlock& colSet, const Record& values) const; // Check if the coordinate values are correct. // This calls checkValues and checks if their shapes match the // hypercube shape. // An exception is thrown if invalid. void checkCoordinates (const PtrBlock& coordColSet, const IPosition& cubeShape, const Record& values) const; // Check if the shapes of FixedShape data and coordinate columns match. // An exception is thrown if not. void checkShapeColumn (const IPosition& shape) const; // Check if the cube shape matches that of defined coordinates. void checkCoordinatesShapes (const TSMCube* hypercube, const IPosition& cubeShape) const; // Check if the hypercube to be added is correctly defined. void checkAddHypercube (const IPosition& cubeShape, const Record& values) const; // Make a new TSMCube object. TSMCube* makeHypercube (const IPosition& cubeShape, const IPosition& tileShape, const Record& values); // Get the index of the hypercube with the given id-values. // If not found, -1 is returned. Int getCubeIndex (const Record& idValues) const; // Determine how many rows need to be added for an extension // (in the last dimension) of a hypercube with the given shape. rownr_t addedNrrow (const IPosition& shape, uInt incrInLastDim) const; // Flush the caches of all hypercubes. // If data have put and fsync is set, fsync all files. Bool flushCaches (Bool fsync); // Let a derived class read the header info. // This is used by the open and resync function. virtual void readHeader (rownr_t nrrow, Bool firstTime) = 0; // Create the TSM header file. // It creates an AipsIO object for it. AipsIO* headerFileCreate(); // Open the TSM header file. // It creates an AipsIO object for it. AipsIO* headerFileOpen(); // Write the data into the header file. // The given number of TSMCube objects have to be written. void headerFilePut (AipsIO& headerFile, uInt64 nrCube); // Read the data from the header file. // When done for the first time, setup() is called to initialize // the various variables (using the extraNdim variable). // It returns the version of the AipsIO object in the header. uInt headerFileGet (AipsIO& headerFile, rownr_t tabNrrow, Bool firstTime, Int extraNdim); // Close the header file. // It deletes the AipsIO object. void headerFileClose (AipsIO* headerFile); // Set up the TiledStMan variables from the table description. // The argument specifies the number of extra dimensions for the // hypercube compared to the data array (usually 0 or 1). // It is only used if no hypercolumn definition exists. // -1 means that the hypercolumn definition has to be present. void setup (Int extraNdim=-1); // Create a TSMFile object and store its pointer at the given index // in the block. void createFile (uInt index); // Convert the scalar data type to an array data type. // This function is temporary and can disappear when the ColumnDesc // classes use type TpArray*. int arrayDataType (int dataType) const; //# Declare all data members. // The name of the hypercolumn. String hypercolumnName_p; // The number of rows in the columns. rownr_t nrrow_p; // The assembly of all columns. PtrBlock colSet_p; // The assembly of all data columns. PtrBlock dataCols_p; PtrBlock dataColSet_p; // The assembly of all id columns. PtrBlock idColSet_p; // The assembly of all coordinate columns. PtrBlock coordColSet_p; // The assembly of all TSMFile objects. // The first file is for all non-extensible cubes, while the others // are for one file per extensible cube. PtrBlock fileSet_p; // The assembly of all TSMCube objects. PtrBlock cubeSet_p; // The persistent maximum cache size (in MiB) for a hypercube. uInt persMaxCacheSize_p; // The actual maximum cache size for a hypercube (in MiB). uInt maxCacheSize_p; // The dimensionality of the hypercolumn. uInt nrdim_p; // The number of vector coordinates. uInt nrCoordVector_p; // The fixed cell shape. IPosition fixedCellShape_p; // Has any data changed since the last flush? Bool dataChanged_p; private: // Forbid copy constructor. TiledStMan (const TiledStMan&); // Forbid assignment. TiledStMan& operator= (const TiledStMan&); }; inline uInt TiledStMan::maximumCacheSize() const { return maxCacheSize_p; } inline uInt TiledStMan::nrCoordVector() const { return nrCoordVector_p; } inline rownr_t TiledStMan::nrow() const { return nrrow_p; } inline uInt TiledStMan::nhypercubes() const { return cubeSet_p.nelements(); } inline void TiledStMan::setDataChanged() { dataChanged_p = True; } inline const TSMCube* TiledStMan::getTSMCube (uInt hypercube) const { return const_cast(this)->getTSMCube (hypercube); } inline const TSMCube* TiledStMan::getHypercube (rownr_t rownr) const { return const_cast(this)->getHypercube (rownr); } inline void TiledStMan::setPersMaxCacheSize (uInt nMiB) { persMaxCacheSize_p = nMiB; maxCacheSize_p = nMiB; } } //# NAMESPACE CASACORE - END #endif