openAdmin/stats.cpp

Go to the documentation of this file.

#include<string>
#include<iomanip>
#include "cgiClass.h"
#include <sys/types.h>
#include <unistd.h>
// #include "ocAdminProxy.h"
#include "ocString.h"

#include "ocTypes.h"
#include "cgiTemplates.h"
#include "ocXML.h"
// #define DO_OPEN_LOGGING
#include "openLogger.h"
#include "ocFileSys.h"
#include "ocImportParser.h"

typedef map<string,long> cocounts;
typedef map<string,cocounts> aggregates;
typedef map<string,string> lookups;

class summer 
{
public:
  aggregates aggies;
  lookups httpstats;
  
  time_date tstamp;  // time work done here
  
  int ipcol, timecol, tzcol, 
      uricol, resultcol,
      referercol, sizecol,  
      agentcol, aggregatecol;  // agregate col is one we are doing stats on
  
  string parseFormat; // [07/Mar/2004:07:47:21 
  string printFormat; // to the day "%m/%d/%Y"
  
  string supportResources;
  
  // defaults based on typical combined httpd log format
  summer():ipcol(0),timecol(3)
  ,tzcol(4),uricol(5),resultcol(6)
  ,referercol(8),sizecol(7)
  ,agentcol(9),aggregatecol(timecol) // default stats on time
  ,parseFormat("[%d/%b/%Y:%H:%M:%S") // read it just as it's parsed
  ,printFormat("%m/%d/%Y") // day at a time
  ,supportResources(".jpg|.gif|.png|.bmp|.css|.js") // egrep style reg exp to filter out files
  {
    httpstats["200"]="&nbsp; OK";
    httpstats["301"]="&nbsp; MOVED_PERMANENTLY";
    httpstats["302"]="&nbsp; FOUND";
    httpstats["304"]="&nbsp; NOT_MODIFIED";
    httpstats["400"]="&nbsp; BAD_REQUEST";
    httpstats["401"]="&nbsp; UNAUTHORIZED";
    httpstats["403"]="&nbsp; FORBIDDEN";
    httpstats["404"]="&nbsp; NOT_FOUND";
    httpstats["500"]="&nbsp; INTERNAL_SERVER_ERROR";
  }
  ~summer(){;}
  
  bool add( ocFileCols & coldata )
  {
    bool bret=true;
    // set the timestamp
    if( timecol != -1 )
    {
      tstamp.parse( coldata[timecol].name.c_str(), parseFormat.c_str() );
      if( tstamp.year() == 1900) bret=false;
    }
    // get the browser ip
    string ip = coldata[ipcol].name;
    // get the key
    string key;
    // special handlers for aggregation and presentation
    if( aggregatecol == timecol )
    {
      key = tstamp.format( printFormat.c_str() );
    }
    else if( aggregatecol == tzcol )
    {
      ocString tz = coldata[aggregatecol].name;
      key = tz.remove("]");
    } 
    else if( aggregatecol == uricol )   
    {
      ocString uri = coldata[aggregatecol].name;
      uri.parse(","); // pull off POST, GET,
      uri = uri.remainder();
      uri = uri.parse("?"); // remove any query string
      uri = uri.parse(" "); // remove Protocol
      // eliminate all resource in the support set
      if( uri.regExMatch( supportResources.c_str()) )  bret=false;
      key = uri;
    }
    else if ( aggregatecol == resultcol )
    {
      // you want to look at errors by ip AND referrer
      ip += coldata[referercol].name;
      key = coldata[aggregatecol].name;
      key += httpstats[coldata[aggregatecol].name];
    }
    else
    {
      key = coldata[aggregatecol].name;
    }
    // add it
    if( bret )
    {
      aggies[key][ip]++;
    }
    return bret;  
  }
  bool readFile ( ocString filename, string & filter )
  {
    char linebuffer[2048];
    // unwarranted optimism
    bool bret = true;  
    writelog2("filename",filename);
    // check for a wildcard
    if ( filename.regExMatch("[*]") )
    {
      writelog2( "found wildcard: ", filename );
      filename = filename.remove("*");
      string::size_type idx = filename.find_last_of('/');
      if( idx != string::npos )
      {
        string path = filename.substr(0,idx);
        writelog2( "path: ", path );
        ocString pattern = filename.substr(idx);
        writelog2( "pattern: ", pattern );
        ocFileSys fs;
        if( fs.openDir( path ) )
        {
          ocDirectory & dir = fs.getDirectoryEntries();
          for( int i=0; i<dir.size(); ++i)
          {
            ocString test = "/";
            test += dir[i].name;
            if( test.regExMatch( pattern.c_str() ))
            {
              // recursion!
              string fullpath = path;
              fullpath += test;
              writelog2( "read: ", fullpath );
              readFile ( fullpath, filter );
            }
          }
        }        
      }             
    }
    else
    { 
      ifstream ifile;
      ifile.open(filename.c_str());

      while( ifile )
      { 
        ocString line;
        if( getline(ifile,line ) )
        {
          // line += linebuffer;
          line += "\n";  
          if( filter.length() )
          {
            if( line.regExMatch(filter.c_str() ) )
            {
              Agget(line);
              writelog2("wrote",line);
            }
          }
          else
          { // write it out always  
            Agget(line);
            writelog2("wrote",line);
          }
        }
      }
      writelog("end of file found");
      ifile.close();
    }
    return bret;
  } 
  void Agget( string line )
  {
    // The parser to parse the response from the socket
    ocFileParser parser;

    // going after httpd logs, so this is the way to breakemdown
    parser.setCheckQuote(true);
    parser.setDelim(" "); 
    parser.setLineTokens(line.c_str());
    parser.parse(true);
    ocFileCols & coldata = parser.getCols();
    int cols = parser.getColCount();
    if(cols) add( coldata );    
  }  

};

// Figure out the power of 10 for the index
long indexIncrement( double in )
{
  double lola = log10(abs(in));
  double lolita = floor(lola);
  double inc = pow( 10.0, lolita );
  if( inc < 1 ) inc = 1;
  return (long) inc;  
}
string scale( int data, int datamax, int pixels )
{
  ocString ret;
  ret.append( ((data*pixels)/datamax) );
  return ret;
}
string toString( long data )
{
  ocString ret;
  ret.append(data);
  return ret;
}
 
int main( int argc, char ** argv )
{
  string logfile, filter, show;
  ocString logfiles;
  int i = 0;
  setuid( 0 );
  cgiScript script;
  
  // ocAdmin admin("/usr/bin/", "/usr/sbin/");
  // ocAdminProxy admin;  
  cgiTemplates tmplt;
  tmplt.load("adminchart.htmp");  
  script << tmplt.getParagraph("top");


  // figure out what to do based on the control section of the template
  xmlParser xparser( tmplt.getParagraph("control") );
  xparser.parse();
  node_vector & xnodes = xparser.nodeList();

  for(i=0;i<xnodes.size();i++)
  {
    xmlNode & node = xnodes[i];
    if( node.name == "filter" )
    {
      filter = node.data;
    }
    else if( node.name == "logfiles" )
    {
      logfiles = node.data; 
    }
    else if( node.name == "show" )
    {
      show = node.data;    
    }    
  }
  
  // Overwrite defaults on show and filter
  cgiInput & args = script.ClientArguments();
  if( args.count("show") ) show = args["show"].c_str();
  if( args.count("filter") ) filter = args["filter"].c_str();  

  // Begin Emmission
  ocString heading = tmplt.getParagraph("charttitle");
  script << heading.replace("$title",show.c_str() );

  // the report aggregator
  summer sums;
  // progam the aggregator based on "show"
  if( show == "Time Zone" )
  {
    sums.aggregatecol = sums.tzcol;      
  }
  else if( show == "Page" )
  {
    sums.aggregatecol = sums.uricol;
  }
  else if( show == "Referrer" )
  { 
    sums.aggregatecol = sums.referercol;
  }
  else if( show == "Request Status" )
  {
    sums.aggregatecol = sums.resultcol;
  }
  else if( show == "Browser" )
  {
    sums.aggregatecol = sums.agentcol;
  }
  
  logfile = logfiles.parse(",");
  while( logfile.length() )
  {    
    sums.readFile ( logfile, filter );    
    logfile = logfiles.parse(",");
  }
  
  {
    // the labels
    ocString label = tmplt.getParagraph("label");
    int maxSize = 0;

    aggregates::iterator pos;
    for( pos=sums.aggies.begin(); 
        pos !=sums.aggies.end();
        ++pos )
    {
      maxSize = pos->second.size() > maxSize ? pos->second.size() : maxSize;
      script << label.replace("$label", pos->first.c_str() );
    }

    // the data
    script << tmplt.getParagraph("chartdata");
    char * tags[] = { "reddata", "greendata", "bluedata" };
    int tagMax=3;
    int tagEl = 0;

    ocString dataEl;
    for( pos=sums.aggies.begin(); 
        pos !=sums.aggies.end();
        ++pos )
    {     
      dataEl = tmplt.getParagraph(tags[tagEl%tagMax]);
      script << dataEl.replace("$num$", scale( pos->second.size(), maxSize, 400 ).c_str())
                      .replace("$num$", toString(pos->second.size())); 
      tagEl++;        
    }

    // now do the tickmarks
    long inc = indexIncrement( maxSize );
    script << tmplt.getParagraph("scale"); 
    ocString tick = tmplt.getParagraph("tickmark");
    for( long incs=inc; incs<=maxSize; incs+=inc )
    {
      script << tick.replace("$num$", scale( inc, maxSize, 400 ).c_str() )
                    .replace("$num$", toString(incs).c_str() ) ;
    }
    tick = tmplt.getParagraph("ticklabel");
    script << tick.replace("$label","Visitors");
  }  
  script << tmplt.getParagraph("end");
  return 0;
}

---