A class to load a whole population from a binary data file. More...

#include <binarydataloader.h>

Collaboration diagram for BinaryDataLoader:

Public Member Functions
	BinaryDataLoader ()

	~BinaryDataLoader ()

off_t	extractOffsetTable (int FD)

Metapop *	extractPop (string &filename, unsigned int generation, SimBuilder sim, Metapop popPtr)

Metapop *	getPop () const

SimBuilder *	getSim () const

unsigned int	getGeneration () const

void	setGeneration (unsigned int gen)

const BinaryStorageBuffer *	getBuffer () const

void	clear ()

Private Attributes
SimBuilder *	_current_sim
	Pointer to the current (calling) simulation object. More...

BinaryStorageBuffer	_buff
	The data countainer into which the data will be copied. More...

Metapop *	_in_pop
	Pointer to the a newly created Metapop object used to retrieve the data. More...

SimBuilder *	_new_sim
	Pointer to a new simulation builder to load the new simulation. More...

unsigned int	_gen

std::string	_filename

off_t	_offsetDataStart

unsigned int	_generation_in_file

BinaryFileParser	_pExtractor

Detailed Description

A class to load a whole population from a binary data file.

Constructor & Destructor Documentation

◆ BinaryDataLoader()

BinaryDataLoader::BinaryDataLoader ( )

inline

                     : _current_sim(0),_buff(),_in_pop(0),
                       _new_sim(0),_gen(0),_filename(),_offsetDataStart(0),
                      _generation_in_file(0),_pExtractor(0)
    { }

◆ ~BinaryDataLoader()

BinaryDataLoader::~BinaryDataLoader ( )

{
  if(_in_pop != NULL) delete _in_pop;
  if(_new_sim != NULL) delete _new_sim;
}

References _in_pop, and _new_sim.

Member Function Documentation

◆ clear()

void BinaryDataLoader::clear ( )

{
  if(_in_pop != NULL) delete _in_pop;
  if(_new_sim != NULL) delete _new_sim;
  _in_pop = NULL;
  _new_sim = NULL;
  _buff.clear();
  
}

References _buff, _in_pop, _new_sim, and BinaryStorageBuffer::clear().

◆ extractOffsetTable()

off_t BinaryDataLoader::extractOffsetTable ( int FD )

{
  int rout;
 
  off_t current_pos = 0L;
  off_t stop_pos = 0L;
 
  current_pos = lseek(FD,0,SEEK_END); //this is eof
 
  stop_pos = lseek(FD,current_pos -sizeof(int) - sizeof(off_t) - 3,SEEK_SET); // -3 to get the separator
 
#ifdef _DEBUG_
  cout<<"\n+++ input binary file: "<<_filename<<flush;
  message(" position at the end of the file, at the position to read table info: %li\n", stop_pos);
#endif
 
  if(stop_pos == -1)
    fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable::lseek(1) failed on %s \n",strerror(errno));
  
#ifdef _DEBUG_
  message("  get the offset of the start of the recorded data: ");
#endif
 
//  unsigned int tot=0;
//  do{
//  if((rout = read(FD,&nb_recgen,sizeof(int))) == -1)// || rout != sizeof(int))
//    fatal("Binary file appears corrupted:\n \
//>>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
//    tot+=rout;
//    }while(tot < sizeof(int) && rout != 0);
 
//#ifdef _DEBUG_
//  current_pos = lseek(FD,0,SEEK_CUR);
//  message("%i, bytes read:%i, current offset %li\n",nb_recgen, rout, current_pos);
//  message("get offset of the beginning of the offset table: ");
//#endif
 
//  if((rout = read(FD, &off_table, sizeof(off_t))) == -1)// || rout != sizeof(int))
//    fatal("Binary file appears corrupted:\n \
//>>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
//
//  current_pos = lseek(FD,0,SEEK_CUR);
//  message("%li, bytes read:%i, current offset %li\n",off_table, rout, current_pos);
//  message("position at the beginning of the offset table:\n");
//
//  current_pos = lseek(FD, off_table, SEEK_SET);
//
//  if( current_pos == -1)
//    fatal("Binary file appears corrupted (pos = %li):\n \
//>>>> BinaryDataLoader::extractOffsetTable::lseek(2) failed on %s \n", current_pos, strerror(errno));
 
//#ifdef _DEBUG_ 
//  message("now at: %li\n",lseek(FD,0,SEEK_CUR));
//  message("check if we are correctly positioned, get the separator: ");
//#endif
  char separator[3];
  if((rout = read(FD, &separator[0], 3)) == -1)// || rout != 3)
    fatal("Binary file appears corrupted (rout = %d):\n \
>>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",rout,strerror(errno));
  
  if( separator[0] != '@' || separator[1] != 'O' || separator[2] != 'T')
    fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable:: wrong separator\n");
 
//#ifdef _DEBUG_ 
//  message(" ok\n read the table elements:\n");
//#endif
 
//  off_t table_elt[2];
 
//  _offset_table.clear();
 
//  unsigned int generation;
//
//  for(int i = 0; i < nb_recgen; ++i) {
 
    if((rout = read(FD, &_generation_in_file, sizeof(int))) == -1 || rout != sizeof(int))
      fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable::read of generation failed: %s\n",strerror(errno));
    
    if((rout = read(FD, &_offsetDataStart, sizeof(off_t))) == -1 || rout != sizeof(off_t))
      fatal("Binary file appears corrupted:\n \
>>>> BinaryDataLoader::extractOffsetTable::read of offset failed: %s\n",strerror(errno));
 
//    _offset_table[table_elt[0]] = table_elt[1];
    
#ifdef _DEBUG_
   message("gen %i at %li\n", _generation_in_file, _offsetDataStart);
#endif
//  }
 
//#ifdef _DEBUG_ 
//    message("returning %li\n",stop_pos);
//#endif
  return stop_pos;
}

References _filename, _generation_in_file, _offsetDataStart, fatal(), and message().

Referenced by extractPop().

Here is the caller graph for this function:

◆ extractPop()

Metapop * BinaryDataLoader::extractPop	(	string &	filename,
		unsigned int	generation,
		SimBuilder *	sim,
		Metapop *	popPtr
	)

{
  int FD;
  off_t gen_offset=0, last_offset, byte_length;
  string cmd, magic_name, old_name;
  bool do_compress = 0;
  _filename = file;
  _gen = generation;
  _pExtractor.setName(file.c_str());
  
  if(_in_pop != NULL) delete _in_pop;
  _in_pop = new Metapop();
  _in_pop->setMPImanager( popPtr->_mpimgr );
  
  //create new sim builder from copy of existing one, will copy templates
  if(_new_sim != 0) delete _new_sim;
  _new_sim = new SimBuilder();
  SIMenv::loadDefaultComponents(_new_sim);
  _current_sim = sim;
 
  //add the current metapop paramSet:
  _new_sim->add_paramset(_in_pop->get_paramset());
 
  message("\n>>>> Reading binary file \"%s\"\n",_filename.c_str());
 
  if((FD = open(_filename.c_str(),O_RDONLY)) == -1){
    close(FD);
    //check if we have to uncompress it:
    string alt_name = _filename + ".bz2";
    magic_name = _filename + tstring::int2str(RAND::Uniform(5477871));
    int status;
    if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
      close(FD);
      //check if gziped:
      alt_name = _filename + ".gz";
      if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
        close(FD);
        fatal("BinaryDataLoader::extractPop::open failed on %s: %s\n",_filename.c_str(),strerror(errno));
      } else {
        //try to ungzip it:
        cmd = "gzip -cd " + alt_name + " > " + magic_name;
 
      }
    }
    close(FD);
    //try to bunzip2:
    cmd = "bunzip2 -ck " + alt_name + " > " + magic_name;
 
    status = system(cmd.c_str());
    if(status != 0)
      fatal("BinaryDataLoader::extractPop::bunzip2 failed (status %i) on %s\n",status, alt_name.c_str());
    
    if((FD = open(magic_name.c_str(),O_RDONLY)) == -1) {//this should work now...
      close(FD);
      fatal("BinaryDataLoader::extractPop::open failed on %s: %s\n", _filename.c_str(), strerror(errno));
    }
    old_name = _filename;
    _filename = magic_name;
    _pExtractor.setName(magic_name.c_str());
    do_compress = 1; //to remove the duplicated source file
  }
  
  //1. read the offset table:
  last_offset = extractOffsetTable(FD);
 
  //2. read the params and build the prototypes
  //extract params from binary file and set the sim params
  // !!! the param values in init part might not reflect pop state, esp if temporal arguments used !!!
  if( !(_new_sim->build_currentParams(_pExtractor.getParameters(NULL))) ){
    error("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::could not set parameters from binary file\n");
    return NULL;
  }
  //set Metapop params
  //build the list of the selected trait templates, will init() the prototypes
  //and load them into the pop, build the individual prototype
//  if( !(_in_pop->init(_new_sim->build_currentTraits(),_new_sim->build_currentLifeCycle())) )
//    return NULL;
  if(!_in_pop->setParameters()) return NULL;  
 
  //build the patch structure:
  _in_pop->buildPatchArray();
 
  //set the Individual and traits prototypes, will set the parameters (set_parameters called)
  _in_pop->makePrototype(_new_sim->build_currentTraits());
  
  //3. check prototypes coherence with existing pop?
  /* check prototypes coherence with existing pop, might have to add or remove traits? */
  
  //4. load the pop
  //compute the number of bytes to read:
//  std::map<unsigned int, unsigned int>::iterator OT = _offset_table.end();
 
  //position within file, right after the parameters:
  //check:
//  cout<<"  currently at "<<lseek(FD,0,SEEK_CUR)<<"B in data file\n";
  //starting point:
//  if(generation == 0) {
//    OT--;
 
    gen_offset = _offsetDataStart; //load last generation recorded
    _gen = _generation_in_file;
 
//  } else {
//    if( generation != _generation_in_file )
//      fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::generation %i not found in file \"%s\"\n",generation,_filename.c_str());
//    else
//       gen_offset = _offset_table;
//  }
#ifdef _DEBUG_
  message("\nBinaryDataLoader::extractPop::generation offset is: %li\n  last offset is: %li ",gen_offset,last_offset);
#endif
 
  //length
//  if(_offset_table.size() == 1 || (++OT) == _offset_table.end())
  byte_length = last_offset - gen_offset;  //always only one generation recorded
 
//  else
//    gen_length = OT->second - gen_offset;
  
  assert(byte_length > 0);
 
  //+ 2 bytes to get the next separator: (@OT ?)
  byte_length += 2;
 
#ifdef _DEBUG_
  message("--> nb bytes to read are: %li\n",byte_length);
#endif
 
  off_t current_pos = lseek(FD,_offsetDataStart,SEEK_SET);
 
  if(current_pos == -1) 
    fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::lseek failed on %s\n",strerror(errno));  
  
  off_t rout = -1, rcount = 0;
  off_t rest = byte_length;
  char *data = new char [byte_length];
  
  while(rest != 0 && rout != 0) {
    if( (rout = read(FD,&data[rcount],rest)) == -1)
      fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::read data %s (reading in %li bytes, read %li so far)\n",strerror(errno), rest, rout);
    rcount += rout;
    rest -= rout;
#ifdef _DEBUG_
    message("BinaryDataLoader::extractPop:read %i bytes from file\n",rout);
#endif
  }
  
  close(FD);
  
  _buff.set_buff(data, byte_length);
  
  delete [] data;
  
  unsigned char separator[2];
  
  _buff.BSBread(&separator, 2 * sizeof(unsigned char));
  
//  cout<<"BinaryDataLoader::extractPop::generation separator: "<<separator<<endl;
 
  if( separator[0] != '@' || separator[1] != 'G')
    fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop:: wrong generation separator\n");
  
//  cout<<"  retrieve generation number from the buffer: "<<flush;
 
  unsigned int dummy;
  _buff.BSBread(&dummy, sizeof(unsigned int));
 
//  cout<<dummy<<endl;
 
  if(dummy != _gen)
    fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop:: wrong generation in file\n");
  
  bool status = _in_pop->retrieve_data(&_buff);
  
  
  if(do_compress){
    cmd = "rm -f " + magic_name;
    if( system(cmd.c_str()) == 1)
      warning("BinaryDataLoader::extractPop:: deleting duplicated source file failed on %s\n", _filename.c_str());
  }
  
  if( !status ) return NULL;
 
  else return _in_pop;
 
}