mpqsStatistics.cc

Go to the documentation of this file.

namespace statistical_data
{

 class CDynamicFactorRating
  {
   private:
    static const int mysize = 7;
    int distribution[mysize];
    /* for statistical evaluation on how the dynamic factorbase
       is responsible for the finding of relations:
        [0]-> no dynamic factor is involved,
        [1]-> one dynamic factor is involved
        [2]-> two dynamic factors are involved
        [3]-> three dynamic factors are involved
        [n]-> n dynamic factors are involved
        [mysize-1]-> at least mysize-1 dynamic factors are involved
     */

   public:
    CDynamicFactorRating()
    {
      for (int i=0; i<mysize; ++i) distribution[i]=0;
    }
    ~CDynamicFactorRating() { }

    int size() const
    {
      return mysize;
    }

    int entry(const int pos) const
    {
      return distribution[pos];
    }

    void increment_Hits_at_position(const int pos)
    {
      if (pos<size()) ++distribution[pos]; else ++distribution[size()-1];
    }
  };    

 CDynamicFactorRating DynamicFactorRating;
 int directly_sieved_DynamicFactors = 0;
 int directly_sieved_SpecialFactors = 0;
 unsigned int relations_sieved_so_far = 0; // total counter

#ifndef IS_CLIENT
 // the following statistical data can only be collected during
 // a detailed analysis of the relations. Therefore these data
 // are restricted to servers and stand-alone versions.
 
 // Whenever it is possible, DynamicFactorRating is also updated.
 // Due to program restrictions/complexity, some "special hits" are
 // not counted accurately in DynamicFactorRating. (But only stats are affected!) 

 int Special_hit = 0;
 int Special_to_dynamic_Factor_hit = 0;
 int DLP_cycle_hits = 0;
#endif

#ifdef IS_SERVER
 // only for a server: collect statistical data about clients

 class CClientStats : protected CmpqsFactortypes
 {
  private:
   static CMutex Mutex;
   static unsigned int overall_counter[Factortype_size];
   unsigned int counter[Factortype_size];
   time_t first_connect, timestamp;
  protected:
   static void lock() { Mutex.lock(); }
   static void unlock() { Mutex.unlock(); }
  public:

   CClientStats() : counter(), first_connect(time(NULL)), timestamp(0) { }
   ~CClientStats() { }

   void PutTimeStamp(const time_t t = time(NULL))
   {
     lock();
     timestamp=t;
     unlock();
   }

   void increment(const CmpqsFactortypes::Factortype FType)
   {
     lock();
     ++counter[FType]; ++overall_counter[FType];
     unlock();  
   }
  
   const string info() const
   {
     ostringstream os;
     char buf [32];
     lock();
     const double percentage =
      100.0*static_cast<double>(counter[static_prime]+counter[single_large_prime]+counter[double_large_prime])
       /static_cast<double>(overall_counter[static_prime]+overall_counter[single_large_prime]+overall_counter[double_large_prime]);
     ctime_r(&first_connect,buf); // get localtime string for first_connect
     //buf[24]='\0'; // get rid of trailing newline
     buf[16]='\0'; // cut seconds, year and trailing newline
     char* const datestr = &buf[4]; // be less verbose
     os << datestr << " - ";
     ctime_r(&timestamp,buf);
     buf[16]='\0'; // cut seconds, year and trailing newline
     os << datestr << ":"
        << setw(5) << counter[static_prime] << ","
        << setw(7) << counter[single_large_prime] << ","
        << setw(8) << counter[double_large_prime] << "|";
     os.setf(ios::right, ios::adjustfield);
     os.setf(ios::fixed, ios::floatfield);
     os << setw(5) << setprecision(1) << percentage << "%" << flush;
     unlock();
     return os.str();
   }

   const string XML_info() const
   {
     ostringstream os;
     char buf [32];
     lock();
     const double percentage =
      100.0*static_cast<double>(counter[static_prime]+counter[single_large_prime]+counter[double_large_prime])
       /static_cast<double>(overall_counter[static_prime]+overall_counter[single_large_prime]+overall_counter[double_large_prime]);
     ctime_r(&first_connect,buf); // get localtime string for first_connect
     //buf[24]='\0'; // get rid of trailing newline
     buf[16]='\0'; // cut seconds, year and trailing newline
     char* const datestr = &buf[4]; // be less verbose
     os << " FIRSTCONNECT=\"" << datestr << "\"";
     ctime_r(&timestamp,buf);
     buf[16]='\0'; // cut seconds, year and trailing newline
     os << " LASTCONNECT=\"" << datestr << "\""
        << " STATICRELATIONS=\"" << counter[static_prime] << "\""
        << " DYNAMICRELATIONS=\"" << counter[single_large_prime] << "\""
        << " DLPRELATIONS=\"" << counter[double_large_prime] << "\"";
     os.setf(ios::right, ios::adjustfield);
     os.setf(ios::fixed, ios::floatfield);
     os << " PERCENTAGE=\"" << setw(6) << setprecision(2) << percentage
        << "%" << "\"" << flush;
     unlock();
     return os.str();
   }

 };
 CMutex CClientStats::Mutex;
 unsigned int CClientStats::overall_counter[Factortype_size] = {0};

 typedef map<string,CClientStats> TAllClientStats;
 TAllClientStats AllClientStats; // this is a list of all clients ever connected to the server
 CMutex AllClientStats_Mutex;

#endif /* IS_SERVER */

 class CProgressStats : private ForbidAssignment
  {
   private:
    double granularity;
    double next_sample_d;
    int next_sample_at;

    class TSample
     {
      private:
       unsigned int static_relations, SLP_relations, DLP_relations;
       time_t Time;
      public:
#ifdef IS_CLIENT
       TSample() : static_relations(StaticRelations::Count()),
          SLP_relations(DynamicFactorRelations.size()),
          DLP_relations(directly_sieved_SpecialFactors), Time(time(NULL)) { }
#else
       TSample() : static_relations(StaticRelations::Count()),
          SLP_relations(DynamicFactorRelations.size()),
          DLP_relations(SpecialRelations::Count()), Time(time(NULL)) { }
#endif

       ~TSample() { }
       time_t get_Time() const { return Time; }

       void XML(std::ostream &os, const time_t PrevTime) const
        {
          os << "<tr>"
             << "<td>" << setw(5) << setprecision(4) << 100.0*static_cast<double>(static_relations)/static_cast<double>(StaticFactorbase::Size()) << "%</td>"
             << "<td>" << static_relations << "</td>"
             << "<td>" << SLP_relations << "</td>"
             << "<td>" << DLP_relations << "</td>"
             << "<td><TIMESTAMP TIME=\"" << get_Time() << "\" /></td>"
             << "<td><DIFFTIME TIME=\"" << get_Time()-PrevTime << "\" /></td>"
             << "</tr>";
        }
     };

    typedef std::vector<TSample> TSamples;
    TSamples Samples;

   public:
    explicit CProgressStats(const double _granularity = 0.01)
     : granularity(_granularity), next_sample_d(0.0), next_sample_at(0) { }

    ~CProgressStats() { }

   public:
    void take_sample()
     {
       if (StaticRelations::Count()<next_sample_at) return;
       Samples.push_back(TSample()); // create & append new sample
       do {
         next_sample_d += granularity;
         next_sample_at = static_cast<int>(round(StaticFactorbase::Size()*next_sample_d));
        } while (next_sample_at<StaticRelations::Count());
     }

    void XML(std::ostream &os) const
     {
       if (Samples.empty()) return;
       os << "<TABLE CONTENT=\"progress\" >" << endl;
       os << "<tr><td>Rate</td><td>Static Relations</td><td>SLP Relations</td><td>DLP Relations</td><td>Time</td><td>Difftime</td></tr>" << endl;

       time_t prevTime = Samples.begin()->get_Time();
       for (TSamples::const_iterator pos=Samples.begin(); pos!=Samples.end(); ++pos)
        {
          pos->XML(os,prevTime); os << endl;
          prevTime=pos->get_Time();
        }
       if (time(NULL)>prevTime+5)
        {
          // use a temporary sample to emit current state
          TSample().XML(os,prevTime); os << endl;
        }
       os << "</TABLE>" << endl;
     }
  };

 CProgressStats ProgressStats;

 time_t StartTimestamp = 0; // time, at which sieving got started
 time_t PreviousTimestamp = 0; // time, at which previous status message was issued
 int StartFilling; // filling of the equation system when sieving (re-)started
 struct tsample { time_t Zeitpunkt; int Filling; int total_sieved; };
 const int samples=1<<4;
 tsample sample[samples]; int i_newest_sample=0;

 void calc_ETA(void)
 {
   // Actually it is rather the average time to find the relations (based on
   // the current speed) than an accurate estimation of the remaining time
   // to arrival that is calculated in this function...
   // Extrapolating the intermediate results is quite difficult and almost
   // impossible. In a multiuser and multitasking system with a changing
   // number of clients you cannot foresee what will happen in the future. 
   // (If any fileclients are connected, the situation would become even
   // more complicated!)

   if (StaticRelations::Count()==StartFilling) return;

   /*
      now.x - old.x = delta.x  // relations
      now.y - old.y = delta.y  // time
      delta.y/delta.x = y per x  -> seconds per static relation

      wanted.x = missing relations = StaticFactorbase::Size() - StaticRelations::Count(); known
      wanted.y -> (estimated) time till arrival; to compute

      wanted.y = (delta.y/delta.x)*wanted.x
      ETA = estimated time of arrival = now.y + wanted.y

   */

#ifdef VERBOSE_INFO
   const time_t now=time(NULL);
   const int i_oldest_sample=(i_newest_sample+1)%samples;
   double t=difftime(now,sample[i_oldest_sample].Zeitpunkt);
   t/=static_cast<double>(StaticRelations::Count()-sample[i_oldest_sample].Filling);
   double Restzeit=(StaticFactorbase::Size()-StaticRelations::Count())*t;

   {
     // print estimated time of arrival
     char buf[32]; struct tm my_tm;
     const time_t eta = now+static_cast<time_t>(Restzeit);
     localtime_r(&eta,&my_tm); asctime_r(&my_tm,buf);
     buf[24]='\0'; // get rid of trailing newline
     cout_status << "ETA: " << buf << ", ";
   }
   {
     double days = floor(Restzeit/(60*60*24));
     int hours,mins,secs;
     Restzeit=fmod(Restzeit,60*60*24);
     secs=static_cast<int>(Restzeit); mins=secs/60; hours=mins/60;
     mins%=60; secs%=60;

     if (days)
      cout_status << "time left (estimated): " << days << "d " << hours << "h" << endl;
     else
      cout_status << "time left (estimated): " << hours << "h " << mins << "m "
                  << secs << "s" << endl;

   }
#endif /* VERBOSE_INFO */
 }


 void display_StatusLegend(void)
 {
#ifdef VERBOSE_INFO
   cout_status << "Legend of the status information:" << endl;
   cout_status << "#static relations{distribution}/#static factorbase," << endl
               << "#directly sieved dynamic factors (single large primes)/#dynamic-FB," << endl;
 #ifdef IS_CLIENT
   cout_status << "#special factors (double large primes) ->{distribution}" << endl;
 #else
   cout_status << "#special factors (double large primes)" << endl;
 #endif
 #ifndef USE_NETWORK
   // sieve offset is only useful for standalone-version
   cout_status << " [sievepos]" << endl;
 #endif
   cout_status << " -> % progress" << endl;
#endif

   if (!StartTimestamp)
    {
     StartTimestamp=time(NULL); StartFilling=StaticRelations::Count(); // initialize these for calculating ETA and other stats
     PreviousTimestamp=StartTimestamp; // memorize the point in time
     for (int i=0; i<samples; ++i)
      {
        sample[i].Zeitpunkt=StartTimestamp;
        sample[i].Filling=StaticRelations::Count();
        sample[i].total_sieved=relations_sieved_so_far;
      }
    }
 }

 bool StatusReport(const bool force_now = false)
 {
   /*
      output a status message, if at least 15 seconds have passed since the
      previous status message.
      return true, if any output was issued. (false if output was skipped.)
    */ 

   if (!force_now && difftime(time(NULL),PreviousTimestamp)<15.0) return false;
   // we want to output a status message now
   
   ProgressStats.take_sample();

   const time_t now=time(NULL);

#ifdef VERBOSE_NOTICE
   {
     // print estimated time of arrival
     char buf[32]; struct tm my_tm;
     localtime_r(&now,&my_tm); asctime_r(&my_tm,buf);
     buf[24]='\0'; // get rid of trailing newline
     cout_status << buf << ": ";
   }
#endif

#ifdef VERBOSE_INFO
   cout_status << "Status for " << n << endl;

 #ifdef IS_SERVER
   {
    // print a sheet with all clients that ever have connected to this server
    AllClientStats_Mutex.lock();
    for (TAllClientStats::const_iterator pos=AllClientStats.begin(); pos!=AllClientStats.end(); ++pos)
      cout_status << (pos->first+"                    ").substr(0,20) << ":" << pos->second.info() << endl;
    AllClientStats_Mutex.unlock();
   }
 #endif
#endif /* VERBOSE_INFO */

#ifdef VERBOSE_NOTICE
   // overall statistics
   cout_status << StaticRelations::Count()
        << "{"
        << DynamicFactorRating.entry(0);

   for(int i=1; i<DynamicFactorRating.size(); ++i)
    cout_status << "," << DynamicFactorRating.entry(i);

   cout_status << "}/" << StaticFactorbase::Size() << ","
        << directly_sieved_DynamicFactors << "/"
        << DynamicFactorRelations.size();

 #ifdef IS_SERVER
   cout_status << "{active:" << DynamicFactorRelations.size_active()
      << ",passive:" << DynamicFactorRelations.size_passive()
      << "}";
 #endif

   cout_status << ",";
 #ifdef IS_CLIENT
   cout_status << directly_sieved_SpecialFactors;
 #else
   cout_status
        << SpecialRelations::Count()
           // BUG/remark: SpecialFactorRelations is not protected by a
           // mutex, we actually can get wrong results, if we read
           // SpecialFactorRelations.size() while another thread is
           // modifying the set. But since we do read-only access, this
           // should not cause any harm. (this actually happened once while
           // Zyklensuche was active)
        << "->{" << Special_to_dynamic_Factor_hit << "/" << Special_hit
        << "/" << DLP_cycle_hits << "}";
 #endif
 #ifndef USE_NETWORK
   // Sieve offset only useful for standalone version
   cout_status << " [" << SieveOffset << "]";
 #endif
   cout_status.setf(ios::left, ios::adjustfield);
   cout_status << " -> ca. " << setw(5) << setprecision(4)
        << 100.0*StaticRelations::Count()/StaticFactorbase::Size()
        << "%" << endl;
#ifdef VERBOSE_INFO
   cout_status << "mpqs interval ";
   Polynom.save(cout_status); // output mpqs polynomial data
#endif

#endif /* VERBOSE_NOTICE */

   PreviousTimestamp=now; // memorize timestamp
 #ifndef IS_CLIENT
   calc_ETA();
 #endif

   // only update ringbuffer, if
   // at least 1 minute has passed and the filling has increased.
   if (now-sample[i_newest_sample].Zeitpunkt>=60 && StaticRelations::Count()>sample[i_newest_sample].Filling)
    {
      const int i_oldest_sample=(i_newest_sample+1)%samples;
      i_newest_sample=i_oldest_sample; // because of ringbuffer!
      sample[i_newest_sample].Zeitpunkt=now;
      sample[i_newest_sample].Filling=StaticRelations::Count();
      sample[i_newest_sample].total_sieved=relations_sieved_so_far;
    }

#ifdef VERBOSE_NOTICE
#ifndef IS_SERVER
   // this info is only relevant to clients (and standalone version)...
   {
     // calculate and print DLP/minute performance
      const int i_oldest_sample=(i_newest_sample+1)%samples;
      const double t=difftime(now,sample[i_oldest_sample].Zeitpunkt);
      const double sieved_relations_per_sec = (relations_sieved_so_far-sample[i_oldest_sample].total_sieved)/t;
      cout << "sieved relations: " << relations_sieved_so_far
           << " [" << sieved_relations_per_sec*60.0 << " per minute]"
           << " #DYNFB: " << DynamicFactorArrays::DynamicFactorsInUse << ", countdown now: " << DynamicFactorArrays::DYNFB_threshold
           << endl;
 #if 0 || defined(VERBOSE)
      cout_status << "oldest sample: " << sample[i_oldest_sample].Zeitpunkt << ": " << sample[i_oldest_sample].Filling << endl;
      cout_status << "newest sample: " << sample[i_newest_sample].Zeitpunkt << ": " << sample[i_newest_sample].Filling << endl;
 #endif
   }
#endif
#endif /* VERBOSE_NOTICE */

   return true;
 }


#ifndef IS_CLIENT
 void XML_StatusReport(ostream &os)
 {
   {
     time_t t = time(NULL);
     char buf [32];
     ctime_r(&t,buf); // get localtime
     buf[24]='\0'; // get rid of trailing newline
     char* const datestr = &buf[4]; // be less verbose
     os << "<QSIEVESTATISTICS TIME=\"" << datestr
        << "\" VERSION=\"" << VERSION << "\" >" << endl;
   }

   os << "<NUMBERSECTION>" << endl;
   if (FoundFactors.regarding!="") os << "<ITEM CONTENT=\"input: \">" << FoundFactors.regarding << "</ITEM>" << endl;
   if (mpz_cmp_ui(n,1)!=0) os << "<ITEM CONTENT=\"remaining: \">" << n << "</ITEM>" << endl;
   os << "</NUMBERSECTION>" << endl;

   if (!FoundFactors.empty())
    {
// FIXME! TFoundFactors needs to be wrapped (not threadsafe!)
      os << "<TABLE CONTENT=\"FACTORS\" >";
       for (TFoundFactors::const_iterator pos=FoundFactors.begin(); pos!=FoundFactors.end(); ++pos)
        os << "<FACTOR>" << (*pos) << "</FACTOR>";
      os << "</TABLE>" << endl;
    }

   if (ecm_curves_processed)
    {
      os << "<TABLE CONTENT=\"Elliptic Curves\" >"
         << "<tr><td>ecm status</td><td>" << ecm_curves_processed << " / " << elcu_Kurven << "</td></tr>" << endl
         << "<tr><td>Phase 1</td><td>" << elcu_Phase1 << "</td></tr>" << endl
         << "<tr><td>Phase 2</td><td>" << elcu_Phase2 << "</td></tr>" << endl
         << "</TABLE>" << endl;
    }

   ProgressStats.XML(os); // a table with progress statistics
 #ifdef IS_SERVER
   {
    // print a sheet with all clients that ever have connected to this server
    AllClientStats_Mutex.lock();
    if (!AllClientStats.empty())
     {
       os << "<TABLE CONTENT=\"NETCLIENTS\" >" << endl;
       for (TAllClientStats::const_iterator pos=AllClientStats.begin(); pos!=AllClientStats.end(); ++pos)
        os << "<NETCLIENT "
           << " HOSTNAME=\"" << pos->first << "\""
           << pos->second.XML_info()
           << " />" << endl;
       os << "</TABLE>" << endl;
     }
    AllClientStats_Mutex.unlock();
   }
 #endif

   // overall statistics
   os << "<STATUSINFO>" << endl;
   os << "<RELFOUND>" << StaticRelations::Count()
      << "<RELDISTRIBUTION>"
      << DynamicFactorRating.entry(0);

   for(int i=1; i<DynamicFactorRating.size(); ++i)
    os << "," << DynamicFactorRating.entry(i);

   os << " </RELDISTRIBUTION>"
      << "</RELFOUND>" << endl
      << "<RELTOTAL>" << StaticFactorbase::Size() << "</RELTOTAL>" << endl
      << "<SLPFOUND>" << directly_sieved_DynamicFactors << "</SLPFOUND>"
      << "<SLPTOTAL>" << DynamicFactorRelations.size();

 #ifdef IS_SERVER
   os << "<RELDISTRIBUTION>" << "active:" << DynamicFactorRelations.size_active()
      << ", passive:" << DynamicFactorRelations.size_passive()
      << " </RELDISTRIBUTION>";
 #endif

   os << "</SLPTOTAL>";

 #ifndef IS_CLIENT
   os << "<DLPTOTAL>" << SpecialRelations::Count() << "</DLPTOTAL>" << endl;
           // BUG/remark: SpecialFactorRelations is not protected by a
           // mutex, we actually can get wrong results, if we read
           // SpecialFactorRelations.size() while another thread is
           // modifying the set. But since we do read-only access, this
           // should not cause any harm. (this actually happened once while
           // Zyklensuche was active)

   os << "<DLP2SLP>" << Special_to_dynamic_Factor_hit << "</DLP2SLP>"
      << "<DLP2STATIC>" << Special_hit << "</DLP2STATIC>"
      << "<DLP2CYCLE>" << DLP_cycle_hits << "</DLP2CYCLE>" << endl;
 #endif

   os.setf(ios::left, ios::adjustfield);
   os << "<PERCENTAGEDONE>" << setw(5) << setprecision(4)
        << 100.0*StaticRelations::Count()/StaticFactorbase::Size()
        << "%" << "</PERCENTAGEDONE>" << endl;
   os << "</STATUSINFO>" << endl;
   os << "</QSIEVESTATISTICS>" << endl;
 }
#endif /* ndef IS_CLIENT */


} // end of namespace statistical_data
using statistical_data::StatusReport;
using statistical_data::display_StatusLegend;

---