BinaryDataLoader Class Reference

A class to load a whole population from a binary data file. More...

#include <binarydataloader.h>

Collaboration diagram for BinaryDataLoader:

Public Member Functions
	BinaryDataLoader ()
	~BinaryDataLoader ()
off_t	extractOffsetTable (int FD)
Metapop *	extractPop (string &filename, unsigned int generation, SimBuilder *sim, Metapop *popPtr)
Metapop *	getPop () const
SimBuilder *	getSim () const
unsigned int	getGeneration () const
void	setGeneration (unsigned int gen)
const BinaryStorageBuffer *	getBuffer () const
void	clear ()

Private Attributes
SimBuilder *	_current_sim
	Pointer to the current (calling) simulation object. More...
BinaryStorageBuffer	_buff
	The data countainer into which the data will be copied. More...
Metapop *	_in_pop
	Pointer to the a newly created Metapop object used to retrieve the data. More...
SimBuilder *	_new_sim
	Pointer to a new simulation builder to load the new simulation. More...
unsigned int	_gen
std::string	_filename
off_t	_offsetDataStart
unsigned int	_generation_in_file
BinaryFileParser	_pExtractor

Detailed Description

A class to load a whole population from a binary data file.

Constructor & Destructor Documentation

BinaryDataLoader()

BinaryDataLoader::BinaryDataLoader ( )

inline

                     : _current_sim(0),_buff(),_in_pop(0),
                       _new_sim(0),_gen(0),_filename(),_offsetDataStart(0),
                      _generation_in_file(0),_pExtractor(0)
    { }

~BinaryDataLoader()

BinaryDataLoader::~BinaryDataLoader

(

)

{
  if(_in_pop != NULL) delete _in_pop;
  if(_new_sim != NULL) delete _new_sim;
}

References _in_pop, and _new_sim.

Member Function Documentation

clear()

void BinaryDataLoader::clear

(

)

{
  if(_in_pop != NULL) delete _in_pop;
  if(_new_sim != NULL) delete _new_sim;
  _in_pop = NULL;
  _new_sim = NULL;
  _buff.clear();
  
}

References _buff, _in_pop, _new_sim, and BinaryStorageBuffer::clear().

extractOffsetTable()

off_t BinaryDataLoader::extractOffsetTable

(

int

FD

)

{
  int rout;
 
  off_t current_pos = 0L;
  off_t stop_pos = 0L;
 
  current_pos = lseek(FD,0,SEEK_END); //this is eof
 
  stop_pos = lseek(FD,current_pos -sizeof(int) - sizeof(off_t) - 3,SEEK_SET); // -3 to get the separator
 
#ifdef _DEBUG_
  cout<<"\n+++ input binary file: "<<_filename<<flush;
  message(" position at the end of the file, at the position to read table info: %li\n", stop_pos);
#endif
 
  if(stop_pos == -1)
    fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable::lseek(1) failed on %s \n",strerror(errno));
  
#ifdef _DEBUG_
  message("  get the offset of the start of the recorded data: ");
#endif
 
//  unsigned int tot=0;
//  do{
//  if((rout = read(FD,&nb_recgen,sizeof(int))) == -1)// || rout != sizeof(int))
//    fatal("Binary file appears corrupted:\n \
//>>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
//    tot+=rout;
//    }while(tot < sizeof(int) && rout != 0);
 
//#ifdef _DEBUG_
//  current_pos = lseek(FD,0,SEEK_CUR);
//  message("%i, bytes read:%i, current offset %li\n",nb_recgen, rout, current_pos);
//  message("get offset of the beginning of the offset table: ");
//#endif
 
//  if((rout = read(FD, &off_table, sizeof(off_t))) == -1)// || rout != sizeof(int))
//    fatal("Binary file appears corrupted:\n \
//>>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
//
//  current_pos = lseek(FD,0,SEEK_CUR);
//  message("%li, bytes read:%i, current offset %li\n",off_table, rout, current_pos);
//  message("position at the beginning of the offset table:\n");
//
//  current_pos = lseek(FD, off_table, SEEK_SET);
//
//  if( current_pos == -1)
//    fatal("Binary file appears corrupted (pos = %li):\n \
//>>>> BinaryDataLoader::extractOffsetTable::lseek(2) failed on %s \n", current_pos, strerror(errno));
 
//#ifdef _DEBUG_ 
//  message("now at: %li\n",lseek(FD,0,SEEK_CUR));
//  message("check if we are correctly positioned, get the separator: ");
//#endif
  char separator[3];
  if((rout = read(FD, &separator[0], 3)) == -1)// || rout != 3)
    fatal("Binary file appears corrupted (rout = %d):\n \
>>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",rout,strerror(errno));
  
  if( separator[0] != '@' || separator[1] != 'O' || separator[2] != 'T')
    fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable:: wrong separator\n");
 
//#ifdef _DEBUG_ 
//  message(" ok\n read the table elements:\n");
//#endif
 
//  off_t table_elt[2];
 
//  _offset_table.clear();
 
//  unsigned int generation;
//
//  for(int i = 0; i < nb_recgen; ++i) {
 
    if((rout = read(FD, &_generation_in_file, sizeof(int))) == -1 || rout != sizeof(int))
      fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable::read of generation failed: %s\n",strerror(errno));
    
    if((rout = read(FD, &_offsetDataStart, sizeof(off_t))) == -1 || rout != sizeof(off_t))
      fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable::read of offset failed: %s\n",strerror(errno));
 
//    _offset_table[table_elt[0]] = table_elt[1];
    
#ifdef _DEBUG_
   message("gen %i at %li\n", _generation_in_file, _offsetDataStart);
#endif
//  }
 
//#ifdef _DEBUG_ 
//    message("returning %li\n",stop_pos);
//#endif
  return stop_pos;
}

References _filename, _generation_in_file, _offsetDataStart, fatal(), and message().

Referenced by extractPop().

Here is the caller graph for this function:

extractPop()

Metapop * BinaryDataLoader::extractPop	(	string &	filename,
		unsigned int	generation,
		SimBuilder *	sim,
		Metapop *	popPtr
	)

{
  int FD;
  off_t gen_offset=0, last_offset, byte_length;
  string cmd, magic_name, old_name;
  bool do_compress = 0;
  _filename = file;
  _gen = generation;
  _pExtractor.setName(file.c_str());
  
  if(_in_pop != NULL) delete _in_pop;
  _in_pop = new Metapop();
  _in_pop->setMPImanager( popPtr->_mpimgr );
  
  //create new sim builder from copy of existing one, will copy templates
  if(_new_sim != NULL) delete _new_sim;
  _new_sim = new SimBuilder(*sim);
  _current_sim = sim;
 
  //add the current metapop paramSet:
  _new_sim->add_paramset(_in_pop->get_paramset());
 
  message("\n>>>> Reading binary file \"%s\"\n",_filename.c_str());
 
  if((FD = open(_filename.c_str(),O_RDONLY)) == -1){
    //error -- save errno before it gets overwritten
    int open_errno = errno;
    //check if we have to uncompress it:
    string alt_name = _filename + ".bz2";
    int status;
    if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
      //error
      //check if gziped:
      alt_name = _filename + ".gz";
      if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
        //error
 
        fatal("BinaryDataLoader::extractPop::open failed on %s: %s\n",_filename.c_str(),strerror(open_errno));
      } else {
        //success
        //try to ungzip it:
        cmd = "gzip -cd " + alt_name + " > " + magic_name;
 
      }
    }
    //success on alt_name -> file is b-zipped
    close(FD);
    //now try to bunzip2 into a new file:
    magic_name = _filename + tstring::int2str(RAND::Uniform(5477871));
    cmd = "bunzip2 -ck " + alt_name + " > " + magic_name;
 
    status = system(cmd.c_str());
    if(status != 0)
      fatal("BinaryDataLoader::extractPop::bunzip2 failed (status %i) on %s\n",status, alt_name.c_str());
    
    //file is bunzip2-ed, open it to read
    if((FD = open(magic_name.c_str(),O_RDONLY)) == -1) {//this should work now...
      //error
      close(FD);
      fatal("BinaryDataLoader::extractPop::open failed on %s: %s\n", _filename.c_str(), strerror(errno));
    }
    //success
    old_name = _filename;
    _filename = magic_name;
    _pExtractor.setName(magic_name.c_str());
    do_compress = 1; //to remove the duplicated source file
  }
  
  //1. read the offset table:
  last_offset = extractOffsetTable(FD);
 
  //2. read the params and build the prototypes
  //extract params from binary file and set the sim params
  // !!! the param values in init part might not reflect pop state, esp if temporal arguments used !!!
  if( !(_new_sim->build_currentParams(_pExtractor.getParameters(NULL))) ){
    error("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::could not set parameters from binary file\n");
    return NULL;
  }
  //set Metapop params
  //build the list of the selected trait templates, will init() the prototypes
  //and load them into the pop, build the individual prototype
//  if( !(_in_pop->init(_new_sim->build_currentTraits(),_new_sim->build_currentLifeCycle())) )
//    return NULL;
  if(!_in_pop->setParameters()) return NULL;  
 
  //build the patch structure:
  _in_pop->buildPatchArray();
 
  //set the Individual and traits prototypes, will set the parameters (set_parameters called)
  _in_pop->makePrototype(_new_sim->build_currentTraits());
  
  //3. load the pop
  
  gen_offset = _offsetDataStart;
  _gen = _generation_in_file;
 
 
#ifdef _DEBUG_
  message("\nBinaryDataLoader::extractPop::generation offset is: %li\n  last offset is: %li ",gen_offset,last_offset);
#endif
 
  byte_length = last_offset - gen_offset;
  
  assert(byte_length > 0);
 
  //+ 2 bytes to get the next separator: (@OT ?)
  byte_length += 2;
 
#ifdef _DEBUG_
  message("--> nb bytes to read are: %li\n",byte_length);
#endif
 
  off_t current_pos = lseek(FD,_offsetDataStart,SEEK_SET);
 
  if(current_pos == -1) 
    fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::lseek failed on %s\n",strerror(errno));  
  
  off_t rout = -1, rcount = 0;
  off_t rest = byte_length;
  _buff.set_buff(byte_length);
 
  char *data = _buff.getBuffer(); // new char [byte_length];
  
  while(rest != 0 && rout != 0) {
    if( (rout = read(FD,&data[rcount],rest)) == -1)
      fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::read data %s (reading in %li bytes, read %li so far)\n",strerror(errno), rest, rout);
    rcount += rout;
    rest -= rout;
#ifdef _DEBUG_
    message("BinaryDataLoader::extractPop:read %i bytes from file\n",rout);
#endif
  }
  
  // close the file, we are done with it
  close(FD);
  
  //_buff.set_buff(data, byte_length);
  
  //delete [] data;
  
  // read some data from the storage to check for data integrity
  unsigned char separator[2];
  
  _buff.BSBread(&separator, 2 * sizeof(unsigned char));
  
//  cout<<"BinaryDataLoader::extractPop::generation separator: "<<separator<<endl;
 
  if( separator[0] != '@' || separator[1] != 'G')
    fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop:: wrong generation separator\n");
  
//  cout<<"  retrieve generation number from the buffer: "<<flush;
 
  unsigned int dummy;
  _buff.BSBread(&dummy, sizeof(unsigned int));
 
//  cout<<dummy<<endl;
 
  if(dummy != _gen)
    fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop:: wrong generation in file\n");
  
  // RETRIEVE THE DATA FROM BUFFER INTO POPULATION --------------------------------
  bool status = _in_pop->retrieve_data(&_buff);
  
  
  // clean up; remove temporary uncompressed file
  if(do_compress){
    cmd = "rm -f " + magic_name;
    if( system(cmd.c_str()) == 1)
      warning("BinaryDataLoader::extractPop:: deleting duplicated source file failed on %s\n", _filename.c_str());
  }
 
  // clean the load, and empty the storage, return the pop
//  delete _new_sim;  //don't do this here!! it will free the trait prototypes and forbid downstream param checks!
//  _new_sim = NULL;
  
  _buff.clear();
  
  if( !status ) return NULL;
 
  else return _in_pop;
 
}

References _buff, _current_sim, _filename, _gen, _generation_in_file, _in_pop, Metapop::_mpimgr, _new_sim, _offsetDataStart, _pExtractor, ParamManager::add_paramset(), BinaryStorageBuffer::BSBread(), SimBuilder::build_currentParams(), SimBuilder::build_currentTraits(), Metapop::buildPatchArray(), BinaryStorageBuffer::clear(), error(), extractOffsetTable(), fatal(), SimComponent::get_paramset(), BinaryStorageBuffer::getBuffer(), ParamsParser::getParameters(), tstring::int2str(), IndFactory::makePrototype(), message(), Metapop::retrieve_data(), BinaryStorageBuffer::set_buff(), Metapop::setMPImanager(), ParamsParser::setName(), Metapop::setParameters(), RAND::Uniform(), and warning().

Referenced by Metapop::loadPopFromBinarySource().

Here is the caller graph for this function:

getBuffer()

const BinaryStorageBuffer* BinaryDataLoader::getBuffer ( ) const

inline

{return &_buff;}

References _buff.

getGeneration()

unsigned int BinaryDataLoader::getGeneration ( ) const

inline

{return _gen;}

References _gen.

getPop()

Metapop* BinaryDataLoader::getPop ( ) const

inline

{return _in_pop;}

References _in_pop.

Referenced by Metapop::loadPopFromBinarySource().

Here is the caller graph for this function:

getSim()

SimBuilder* BinaryDataLoader::getSim ( ) const

inline

{return _new_sim;}

References _new_sim.

Referenced by Metapop::loadPopFromBinarySource().

Here is the caller graph for this function:

setGeneration()

void BinaryDataLoader::setGeneration ( unsigned int  gen )

inline

{_gen = gen;}

References _gen.

Member Data Documentation

_buff

BinaryStorageBuffer BinaryDataLoader::_buff

private

The data countainer into which the data will be copied.

Referenced by clear(), extractPop(), and getBuffer().

_current_sim

SimBuilder* BinaryDataLoader::_current_sim

private

Pointer to the current (calling) simulation object.

Referenced by extractPop().

_filename

std::string BinaryDataLoader::_filename

private

Referenced by extractOffsetTable(), and extractPop().

_gen

unsigned int BinaryDataLoader::_gen

private

Referenced by extractPop(), getGeneration(), and setGeneration().

_generation_in_file

unsigned int BinaryDataLoader::_generation_in_file

private

Referenced by extractOffsetTable(), and extractPop().

_in_pop

Metapop* BinaryDataLoader::_in_pop

private

Pointer to the a newly created Metapop object used to retrieve the data.

Referenced by clear(), extractPop(), getPop(), and ~BinaryDataLoader().

_new_sim

SimBuilder* BinaryDataLoader::_new_sim

private

Pointer to a new simulation builder to load the new simulation.

Referenced by clear(), extractPop(), getSim(), and ~BinaryDataLoader().

_offsetDataStart

off_t BinaryDataLoader::_offsetDataStart

private

Referenced by extractOffsetTable(), and extractPop().

_pExtractor

BinaryFileParser BinaryDataLoader::_pExtractor

private

Referenced by extractPop().

---

The documentation for this class was generated from the following files:

• binarydataloader.h
• binarydataloader.cc