sparse_series.hpp

#ifndef HAVE_TOM__SPARSE_SERIES_HPP
#define HAVE_TOM__SPARSE_SERIES_HPP
 
#include <map>
#include <tom-util/time_series.hpp>
 
// template specialization for ordering timespec structs
// used by the nano_sparse_series_impl<T> : sparse_series class
template <>
struct std::less<timespec>
{
  bool
  operator()( const timespec & lhs, const timespec & rhs ) const {
    if ( lhs.tv_sec < rhs.tv_sec ) return true;
    if ( lhs.tv_sec > rhs.tv_sec ) return false;
    if ( lhs.tv_nsec < rhs.tv_nsec ) return true;
    return false;
  }
};
 
 
namespace { // anonymous namespace
 
    tom::calendars::date
    timespec2date( const timespec & ts )
    {
      struct tm tm1;
      localtime_r( & ts.tv_sec, & tm1 );
      return tom::calendars::date( tm1.tm_year + 1900, tm1.tm_mon + 1, tm1.tm_mday );
 
    }
    tom::calendars::date_int_type
    timespec2date_int_type( const timespec & ts, tom::calendars::calendar &cal )
    {
      return cal( timespec2date( ts ) );
    }
 
} // end anonymous namespace
 
namespace tom {
 
namespace collections {
    
using namespace tom::calendars;
 
template <typename ContainerT>
class TOM_UTIL_API standard_sparse_series_iterator : public time_series_iterator
{
public:
   typedef ContainerT                           value_type;
   typedef typename value_type::const_iterator  iterator;
   
   // pass the end iter, so we don't go past end
   standard_sparse_series_iterator ( time_series &ts, iterator iter, iterator begin, iterator end ) :
        m_ts( ts ), m_iter(iter), m_iter_begin(begin), m_iter_end(end)
        { }
 
    virtual 
    ~standard_sparse_series_iterator() { }
 
    virtual
    time_series_iterator &
    operator++() { if ( m_iter != m_iter_end ) ++m_iter; return *this; }
 
    virtual
    time_series_iterator &
    operator++(int) { if ( m_iter != m_iter_end ) m_iter++; return *this; }
 
    virtual
    time_series_iterator &
    operator--() { if ( m_iter != m_iter_begin ) --m_iter; return *this; }
 
    virtual
    time_series_iterator &
    operator--(int) { if ( m_iter != m_iter_begin ) m_iter--; return *this; }
 
    virtual
    bool
    operator==( const tom::collections::time_series_iterator& rhs) const
    {
        return rhs.get_calendar() == get_calendar() &&
            rhs.get_date_int() == get_date_int();
    }
 
    virtual
    bool
    operator!=( const tom::collections::time_series_iterator& rhs) const 
    {
        return ! ( *this == rhs );
    }
 
    virtual
    const observation &
    operator*() const
    {
        return m_ts.get_observation( m_iter->first );   
    }
 
    virtual
    const observation &
    operator->() const
    {
        return m_ts.get_observation( m_iter->first );           
    }
 
    virtual
    date
    get_date() const { return get_calendar()( get_date_int() ); }
 
    virtual
    date_int_type
    get_date_int() const 
    { 
        if ( m_iter != m_iter_end )
            return m_iter->first; 
        else
            return 0;
    }
 
    virtual
    timespec
    get_timespec() const
    {
        date d = get_date();
        struct tm tm;
        tm.tm_sec = tm.tm_min = tm.tm_hour = 0;
        tm.tm_year = d.year() - 1900;
        tm.tm_mon = d.month();
        tm.tm_mday = d.day();
        struct timespec spec {0,0};
        spec.tv_sec = mktime( &tm );
        spec.tv_nsec = 0;
        return spec;
    }
 
    virtual
    calendar &
    get_calendar() const { return m_ts.get_calendar(); }
 
protected:
    time_series &   m_ts;
    iterator        m_iter;
    iterator        m_iter_begin;
    iterator        m_iter_end;
};
 
/*
 * CLASS:  sparse_series
 *
 *
 *
 *
 *
*/
class TOM_UTIL_API sparse_series : public tom::collections::time_series
{
public:
   sparse_series( calendar &c ) : m_calendar( c ),
        m_ignore_missing_values(false), m_ignore_same_values(true),
        m_fill_forward(true) { }
   
   sparse_series( calendar &c, bool ignore_same_value, bool ignore_missing_value ) : 
        m_calendar( c ),
        m_ignore_missing_values( ignore_missing_value ),
        m_ignore_same_values(ignore_same_value),
        m_fill_forward( true ) { }
 
   sparse_series( calendar &c, bool ignore_same_value, bool ignore_missing_value,
                    bool fill_forward ) : 
        m_calendar( c ),
        m_ignore_missing_values( ignore_missing_value ),
        m_ignore_same_values(ignore_same_value),
        m_fill_forward( fill_forward ) { }
 
   virtual ~sparse_series() { }
   
    virtual
    void 
    set_observation( date_int_type d, const observation & val ) = 0;
 
    // this should suffice for any derived classes, and anyone who
    // needs can just override it...who me? ( I do just that for efficiency in
    // the sparse_series_impl<> class... )
    virtual
    void
    set_observation( date d, const observation & val )
    {
      set_observation( get_calendar()(d), val );
    }
    // provide specific handlers for dates since they will need to 
    // be properly converted if time-series hold date_interval_observation
    // values...which most(ALL!) date series will!
    virtual void set_observation( date_int_type d, const Date & ) = 0;
    virtual void set_observation( date d, const Date & ) = 0;
 
    virtual
    tom::calendars::calendar &
    get_calendar() const { return m_calendar; }
    
    void
    set_ignore_missing_values( bool b )
    { m_ignore_missing_values = b; }
    
    bool
    get_ignore_missing_values( ) const
    { return m_ignore_missing_values; }
    
    void
    set_ignore_same_values( bool b )
    { m_ignore_same_values = b; }
    
    bool
    get_ignore_same_values() const
    { return m_ignore_same_values; }
 
    bool
    get_fill_forward() const
    { return m_fill_forward; }
    
    void
    set_fill_forward( bool b )
    {  m_fill_forward = b; }
    
    // useful helpers for native formats
    
    // native float setters
    void set_observation( date_int_type d, float f )
        { set_observation( d, static_cast<const observation &>( Float(f) )); }
    void set_observation( date d, float f )
        { set_observation(get_calendar()( d ), static_cast<const observation &>( Float(f) )); }
    
    // native double setters
    void set_observation( date_int_type d, double f )
        { set_observation(d, static_cast<const observation &>( Double(f) )); }
    void set_observation( date d, double f )
        { set_observation( get_calendar()( d ), static_cast<const observation &>( Double(f) )); }
 
    // native std::string setters
    void set_observation( date_int_type d, const std::string &f )
        { set_observation(d, static_cast<const observation &>( String(f) ) ); }
    void set_observation( date d, const std::string &f )
        { set_observation(get_calendar()( d ), static_cast<const observation &>( String(f) ) ); }
 
    // native const char * setters
    void set_observation( date_int_type d, const char *f )
        { set_observation(d, static_cast<const observation &>( String(f) )); }
    void set_observation( date d, const char *f )
        { set_observation( get_calendar()( d ), static_cast<const observation &>( String(f) )); }
 
    // native int setters
    void set_observation( date_int_type d, int f )
        { set_observation(d, static_cast<const observation &>( Int(f) )); }
    void set_observation( date d, int f )
        { set_observation( get_calendar()( d ), static_cast<const observation &>( Int(f) )); }
 
    // native bool setters
    void set_observation( date_int_type d, bool f )
        { set_observation(d, static_cast<const observation &>( Bool(f) ) ); }
    void set_observation( date d, bool f )
        { set_observation( get_calendar()( d ), static_cast<const observation &>( Bool(f) ) ); }
 
    // native category_type setters
    void set_observation(   date_int_type d, 
                            tom::value_type f )
        { set_observation(d, static_cast<const observation &>( Int(f) ) ); }
    void set_observation( date d, tom::value_type f )
        { set_observation( get_calendar()( d ), static_cast<const observation &>( Int(f) ) ); }
 
protected:
    calendar &      m_calendar;
    bool            m_ignore_missing_values;
    bool            m_ignore_same_values;
    bool            m_fill_forward;
};
 
/*
 * CLASS:  sparse_series_impl<T>
 *
 *
 *
 *
 *
*/
template <typename T>
class TOM_UTIL_API sparse_series_impl : public sparse_series, 
                  private std::map<date_int_type, typename T::value_type, std::less<date_int_type> >
{
public:
    typedef     T                                       value_type;
    typedef     typename std::map<date_int_type, typename T::value_type, std::less<date_int_type> >
                                                        map_type;
    typedef     typename map_type::pointer              pointer;
    typedef     typename map_type::reference            reference;
    typedef     typename map_type::difference_type      difference_type;
    typedef     std::random_access_iterator_tag         iterator_category;
 
    typedef     typename map_type::iterator             iterator;
    typedef     typename map_type::const_iterator       const_iterator;
 
    sparse_series_impl( calendar &cal ) : 
      sparse_series( cal ), map_type()  {   }
 
    sparse_series_impl( calendar &cal,  bool ignore_same_value, bool ignore_missing_value,
                        bool fill_forward ) : 
      sparse_series( cal, ignore_same_value, ignore_missing_value, fill_forward ), map_type()  {    }
 
    template <typename Iter>
    sparse_series_impl( calendar &cal, Iter begin, Iter end ) :
        sparse_series( cal ), map_type( begin, end ){   }
 
    template <typename Iter>
    sparse_series_impl( calendar &cal, bool ignore_same_value, bool ignore_missing_value,
                         Iter begin, Iter end ) :
        sparse_series( cal, ignore_same_value, ignore_missing_value ), map_type( begin, end ){  }
 
    virtual ~sparse_series_impl() { }
 
    virtual time_series_iterator &
    begin() const
    {
        return *new standard_sparse_series_iterator< sparse_series_impl<T> >( 
            const_cast<sparse_series_impl<T> &>(*this), real_begin(), real_begin(), real_end() );
    }
    virtual time_series_iterator &
    end() const
    {
        return *new standard_sparse_series_iterator< sparse_series_impl<T> >( 
            const_cast<sparse_series_impl<T> &>(*this), real_end(), real_begin(), real_end() ); 
    }
 
    virtual
    const observation &
    get_observation( date_int_type d ) const
    {
        using namespace std;
        if ( ! is_empty() && get_fill_forward() )
        {
            const_iterator it = this->map_type::upper_bound(d);
            if ( it != real_begin() )
            {
                --it;
                m_return_value = it->second;
                return m_return_value;
            }
            else
                return Int::HAS_NO_DATA;
        }
        else if ( ! is_empty() && ! get_fill_forward() )
        {
            const_iterator it = this->map_type::find(d);
            if ( it != real_end() )
            {
                m_return_value = it->second;
                return m_return_value;
            }
            else
                return Int::HAS_NO_DATA;
        }
        else
            return Int::HAS_NO_DATA;    // ND in real life
    }
    virtual const observation &get_observation( const date d ) const;
 
    void set_observation( const date_int_type d, const typename T::value_type & val );
    void set_observation( const date d, const typename T::value_type & val );
    void set_observation( const date_int_type d, tom::value_type val );
    void set_observation( const date d, tom::value_type val );
 
    void
    set_observation_( date_int_type d, const T &v )
    {
      std::pair<iterator,bool> result = this->map_type::insert( typename map_type::value_type(d, v.value()) );
      if ( ! result.second )
      {
         this->map_type::erase( result.first );
         this->map_type::insert( typename map_type::value_type(d, v.value()) );
      }
      clean_missing_values();
      clean_same_values();      
    }
    virtual
    void
    set_observation( date d, const observation & val )
    {
      set_observation_( get_calendar()(d), T(val) );
    }
    virtual
    void
    set_observation( date_int_type d, const observation &val )
    {
        set_observation_( d, T(val) );
    }
    virtual void set_observation( date_int_type d, const Date &v )
    {
        set_observation_( d, T(v) );
    }
    virtual void set_observation( date d, const Date &v )
    {
        set_observation_( get_calendar()(d), T(v) );
    }
    
    virtual const observation & operator[ ](  const date_int_type d ) const ;
    virtual observation & operator[ ](  const date_int_type d );
    virtual const observation & operator[ ] ( const date d ) const ;
    virtual observation & operator[ ] ( const date d ) ;
 
    virtual std::ostream &print( std::ostream & ) const;
 
    date_int_type
    get_first_date_int() const 
    { 
      if ( is_empty() ) return 0;
      const_iterator it = real_begin(); return it->first; 
    }
 
    date_int_type
    get_last_date_int() const 
    {
      if ( is_empty() ) return 0; 
      const_iterator it = real_end(); it--; return it->first; 
    }
 
    bool
    is_empty() const
    {
        return this->map_type::empty();
    }
 
protected:
 
   void   
   clean_same_values()
   {
      if ( ! get_ignore_same_values( ) )
         return;
 
      std::vector<date_int_type> getRidOfEm;
      typename T::value_type prev = real_begin()->second;
      
      for( const_iterator it = real_begin(); it != real_end(); it++)
      {
         if (it == real_begin())
         {
            prev = it->second;
         }
         else if(it->second == prev)
         {
            getRidOfEm.push_back(it->first);
        }
         prev = it->second;
      }
      for ( std::vector<date_int_type>::iterator it = getRidOfEm.begin();
        it != getRidOfEm.end(); it++ )
            this->map_type::erase(*it);
 
   }
   void
   clean_missing_values()
   {
      if ( ! get_ignore_missing_values( ) )
         return;
 
      std::vector<date_int_type> deleteThem;
 
      for ( const_iterator it = real_begin(); it != real_end(); it++)
         if ( ! T(it->second).is_normal() )
            deleteThem.push_back( it->first );
 
      for ( std::vector<date_int_type>::iterator it = deleteThem.begin(); it != deleteThem.end(); it++ )
         this->map_type::erase(*it);
   }
    iterator real_begin() { return this->map_type::begin(); }
    iterator real_end() { return this->map_type::end(); }
    const_iterator real_begin() const { return this->map_type::begin(); }
    const_iterator real_end() const { return this->map_type::end(); }
 
    private:
    // This acts as a observation to be returned where one is needed
    mutable value_type          m_return_value;
    
};
 
template <typename T>
inline
const observation &
sparse_series_impl<T>::get_observation( const date d ) const 
{
    date_int_type di = get_calendar()(d);
    return this->get_observation( di ); 
}
 
template <typename T>
inline
void 
sparse_series_impl<T>::set_observation( const date_int_type d, const typename T::value_type & val )
{
    set_observation_( d , T(val) );
};
template <typename T>
inline
void 
sparse_series_impl<T>::set_observation( const date d, const typename T::value_type & val )
{
    set_observation_( get_calendar()( d ), T(val) );
}
template <typename T>
inline
void 
sparse_series_impl<T>::set_observation( const date_int_type d, tom::value_type val )
{
    set_observation_( d, T(val) );
}
template <typename T>
inline
void 
sparse_series_impl<T>::set_observation( const date d, tom::value_type val )
{
    set_observation_( get_calendar()( d ), T(val) );
}
 
template <typename T>
const observation & 
sparse_series_impl<T>::operator[ ](  const date_int_type d ) const 
{
   return get_observation(d);
}
template <typename T>
observation & 
sparse_series_impl<T>::operator[ ](  const date_int_type d )
{
   return *new value_type( get_observation(d) );
}
template <typename T>
const observation & 
sparse_series_impl<T>::operator[ ] ( const date d ) const 
{
    date_int_type di = get_calendar()(d);
    return get_observation(di);
}
template <typename T>
observation & 
sparse_series_impl<T>::operator[ ] ( const date d ) 
{
    date_int_type di = get_calendar()(d);
    return *new value_type( get_observation( di ) );
}
template <typename T>
inline
std::ostream &
sparse_series_impl<T>::print( std::ostream & os) const
{
    calendar &cal = get_calendar();
    if ( cal != ordinal_calendar::Instance() )
        for( time_series_iterator &it = begin(); it != end(); it++ )
            os << it.get_date() << ": " << *it << std::endl;
    else
        for( time_series_iterator &it = begin(); it != end(); it++ )
            os << it.get_date_int() << ": " << *it << std::endl;
    
    return os;
}
 
 
 
/*
 * CLASS:  nano_sparse_series_impl<T>
 *
 *
 *
 *
 *
*/
template <typename T>
class TOM_UTIL_API nano_sparse_series_impl : public sparse_series,
                  private std::map<timespec, typename T::value_type, std::less<timespec> >
{
public:
    typedef     T                                       value_type;
    typedef     typename std::map<timespec, typename T::value_type, std::less<timespec> >
                                                        map_type;
    typedef     typename map_type::pointer              pointer;
    typedef     typename map_type::reference            reference;
    typedef     typename map_type::difference_type      difference_type;
    typedef     std::random_access_iterator_tag         iterator_category;
 
    typedef     typename map_type::iterator             iterator;
    typedef     typename map_type::const_iterator       const_iterator;
 
    nano_sparse_series_impl( calendar &cal ) :
        sparse_series( cal ), map_type()  { }
 
    nano_sparse_series_impl( calendar &cal,  bool ignore_same_value, bool ignore_missing_value,
                        bool fill_forward ) :
        sparse_series( cal, ignore_same_value, ignore_missing_value, fill_forward ), map_type()  {  }
 
    template <typename Iter>
    nano_sparse_series_impl( calendar &cal, Iter begin, Iter end ) :
        sparse_series( cal ), map_type( begin, end ){   }
 
    template <typename Iter>
    nano_sparse_series_impl( calendar &cal, bool ignore_same_value, bool ignore_missing_value,
                         Iter begin, Iter end ) :
        sparse_series( cal, ignore_same_value, ignore_missing_value ), map_type( begin, end ){  }
 
    virtual ~nano_sparse_series_impl() { }
 
    virtual time_series_iterator &
    begin() const
    {
        return *new standard_sparse_series_iterator< nano_sparse_series_impl<T> >(
            const_cast<nano_sparse_series_impl<T> &>(*this), real_begin(), real_begin(), real_end() );
    }
    virtual time_series_iterator &
    end() const
    {
        return *new standard_sparse_series_iterator< nano_sparse_series_impl<T> >(
            const_cast<nano_sparse_series_impl<T> &>(*this), real_end(), real_begin(), real_end() );
    }
 
    virtual
    const observation &
    get_observation( date_int_type d ) const
    {
        using namespace std;
        if ( ! is_empty() && get_fill_forward() )
        {
            const_iterator it = this->map_type::upper_bound(d);
            if ( it != real_begin() )
            {
                --it;
                m_return_value = it->second;
                return m_return_value;
            }
            else
                return Int::HAS_NO_DATA;
        }
        else if ( ! is_empty() && ! get_fill_forward() )
        {
            const_iterator it = this->map_type::find(d);
            if ( it != real_end() )
            {
                m_return_value = it->second;
                return m_return_value;
            }
            else
                return Int::HAS_NO_DATA;
        }
        else
            return Int::HAS_NO_DATA;    // ND in real life
    }
    virtual const observation &get_observation( const date d ) const;
 
    void set_observation( const date_int_type d, const typename T::value_type & val );
    void set_observation( const date d, const typename T::value_type & val );
    void set_observation( const date_int_type d, tom::value_type val );
    void set_observation( const date d, tom::value_type val );
 
    void
    set_observation_( date_int_type d, const T &v )
    {
      std::pair<iterator,bool> result = this->map_type::insert( typename map_type::value_type(d, v.value()) );
      if ( ! result.second )
      {
         this->map_type::erase( result.first );
         this->map_type::insert( typename map_type::value_type(d, v.value()) );
      }
      clean_missing_values();
      clean_same_values();
    }
    virtual
    void
    set_observation( date d, const observation & val )
    {
      set_observation_( get_calendar()(d), T(val) );
    }
    virtual
    void
    set_observation( date_int_type d, const observation &val )
    {
        set_observation_( d, T(val) );
    }
    virtual void set_observation( date_int_type d, const Date &v )
    {
        set_observation_( d, T(v) );
    }
    virtual void set_observation( date d, const Date &v )
    {
        set_observation_( get_calendar()(d), T(v) );
    }
 
    virtual const observation & operator[ ](  const date_int_type d ) const ;
    virtual observation & operator[ ](  const date_int_type d );
    virtual const observation & operator[ ] ( const date d ) const ;
    virtual observation & operator[ ] ( const date d ) ;
 
    virtual std::ostream &print( std::ostream & ) const;
 
    date_int_type
    get_first_date_int() const
    {
      if ( is_empty() ) return 0;
      const_iterator it = real_begin(); return it->first;
    }
 
    date_int_type
    get_last_date_int() const
    {
      if ( is_empty() ) return 0;
      const_iterator it = real_end(); it--; return it->first;
    }
 
    bool
    is_empty() const
    {
        return this->map_type::empty();
    }
 
protected:
 
   void
   clean_same_values()
   {
      if ( ! get_ignore_same_values( ) )
         return;
 
      std::vector<date_int_type> getRidOfEm;
      typename T::value_type prev;
 
      for( const_iterator it = real_begin(); it != real_end(); it++)
      {
         if (it == real_begin())
         {
            prev = it->second;
         }
         else if(it->second == prev)
         {
            getRidOfEm.push_back(it->first);
        }
         prev = it->second;
      }
      for ( std::vector<date_int_type>::iterator it = getRidOfEm.begin();
        it != getRidOfEm.end(); it++ )
            this->map_type::erase(*it);
 
   }
   void
   clean_missing_values()
   {
      if ( ! get_ignore_missing_values( ) )
         return;
 
      std::vector<date_int_type> deleteThem;
 
      for ( const_iterator it = real_begin(); it != real_end(); it++)
         if ( ! T(it->second).is_normal() )
            deleteThem.push_back( it->first );
 
      for ( std::vector<date_int_type>::iterator it = deleteThem.begin(); it != deleteThem.end(); it++ )
         this->map_type::erase(*it);
   }
    iterator real_begin() { return this->map_type::begin(); }
    iterator real_end() { return this->map_type::end(); }
    const_iterator real_begin() const { return this->map_type::begin(); }
    const_iterator real_end() const { return this->map_type::end(); }
 
    private:
    // This acts as a observation to be returned where one is needed
    mutable value_type          m_return_value;
 
};
 
template <typename T>
inline
const observation &
nano_sparse_series_impl<T>::get_observation( const date d ) const
{
    date_int_type di = get_calendar()(d);
    return this->get_observation( di );
}
 
template <typename T>
inline
void
nano_sparse_series_impl<T>::set_observation( const date_int_type d, const typename T::value_type & val )
{
    set_observation_( d , T(val) );
};
template <typename T>
inline
void
nano_sparse_series_impl<T>::set_observation( const date d, const typename T::value_type & val )
{
    set_observation_( get_calendar()( d ), T(val) );
}
template <typename T>
inline
void
nano_sparse_series_impl<T>::set_observation( const date_int_type d, tom::value_type val )
{
    set_observation_( d, T(val) );
}
template <typename T>
inline
void
nano_sparse_series_impl<T>::set_observation( const date d, tom::value_type val )
{
    set_observation_( get_calendar()( d ), T(val) );
}
 
template <typename T>
const observation &
nano_sparse_series_impl<T>::operator[ ](  const date_int_type d ) const
{
   return get_observation(d);
}
template <typename T>
observation &
nano_sparse_series_impl<T>::operator[ ](  const date_int_type d )
{
   return *new value_type( get_observation(d) );
}
template <typename T>
const observation &
nano_sparse_series_impl<T>::operator[ ] ( const date d ) const
{
    date_int_type di = get_calendar()(d);
    return get_observation(di);
}
template <typename T>
observation &
nano_sparse_series_impl<T>::operator[ ] ( const date d )
{
    date_int_type di = get_calendar()(d);
    return *new value_type( get_observation( di ) );
}
template <typename T>
inline
std::ostream &
nano_sparse_series_impl<T>::print( std::ostream & os) const
{
    calendar &cal = get_calendar();
    if ( cal != ordinal_calendar::Instance() )
        for( time_series_iterator &it = begin(); it != end(); it++ )
            os << it.get_date() << ": " << *it << std::endl;
    else
        for( time_series_iterator &it = begin(); it != end(); it++ )
            os << it.get_date_int() << ": " << *it << std::endl;
 
    return os;
}
 
} // end collections namespace
} // end tom namespace
 
#endif /*HAVE_TOM__SPARSE_SERIES_HPP*/