package org.paneris.bibliomania.logs;

 import java.io.BufferedReader;

 import java.io.File;

 import java.io.FileReader;

 import java.io.FileWriter;

 import java.io.FilenameFilter;

 import java.io.InputStreamReader;

 import java.io.PrintWriter;

 import java.util.Calendar;

 import java.util.Date;

 import java.util.GregorianCalendar;

 import org.melati.poem.AccessToken;

 import org.melati.poem.NoSuchRowPoemException;

 import org.melati.poem.PoemTask;

 import org.melati.poem.UnexpectedExceptionPoemException;

 import org.melati.poem.util.StringUtils;

 import org.paneris.bibliomania.BibliomaniaDatabase;

 /*

  * This program will analyse the stats generated by the bibliomania.com site.

  * 

  * It is assumed to be run from cron at 2am EDT ie a crontab entry of

  * 0   2 * * *  /usr/local/logs/stats.sh 

  * where stats.sh runs this and outputs error and reports files.

  * 

  * It uses 2 analysis programs - weblaizer and analog.

  * It expects log files in a directory tree:

  * 

  * year/month/

  *

  * with each directory containing 1 log file per day.  

  * It will not do anything with today's file (as it will probably not be complete)

  * FIXME this means that the last day of every month is missed.

  *

  * Output for webalizer is according to the config file bibliomania.conf

  *

  * Output for analog is in the file analog.html in the directory for each month,

  * and the directory for the year.  The annual stats are recomputed every day.

  *

  * The stats are translated before being analysed stats to be trnaslated need

  * to be unzipped first.

  *

  * You can optionally supply a single parameter of the form yyyy/mm in order to 

  * process the stats for the specific month.  

  * Annual stats are still calculated.

  */  

 public class LogAnalysis {

   public static void main(final String[] args) throws Exception {

     // get today's date

     Calendar today = new GregorianCalendar();

     int year = today.get(Calendar.YEAR);

     int month = today.get(Calendar.MONTH) + 1;

     String monthString = month + "";

     if (month < 10) monthString = "0" + month;

     int day = today.get(Calendar.DAY_OF_MONTH);

     String dayString = day + "";

     if (day < 10) dayString = "0" + day;

     String todayYearmonth = year + File.separator + monthString;

     // default 

     String yearmonth = todayYearmonth;

     // allow us to run month by month

     if (args.length > 0) {

       yearmonth = args[0];

     }

     System.out.println("Running Stats for " + yearmonth);

     System.out.println("Started at: " + new Date());

     File dir = new File(yearmonth);

     Translator trans = new Translator();

     // find files that need translating, translate them,

     // analize them, and then zip up the output

     FilenameFilter webalizerFilter = new WebalizerFilter(todayYearmonth, dayString);

     File[] files = dir.listFiles(webalizerFilter);

     java.util.Arrays.sort(files);

     if (files != null) {

       for (int i=0; i<files.length; i++) {    

         System.out.println("Translating " + files[i]);

         File out = trans.doIt(files[i]);

         String outName = out.toString();

         out = null;

         System.out.println("Webalizering " + outName);

         run("webalizer -c bibliomania.conf " + outName);

         System.out.println("Gzipping " + files[i]);

         run("gzip " + files[i]);

         System.out.println("Gzipping " + outName);

         run("gzip " + outName);

       }

     }    

     // run the files for this month through analog

     File[] dirs = new File[1];

     dirs[0] = dir;

     analog(dirs, todayYearmonth, dayString, yearmonth);

     // do the annual stuff.

     int i = yearmonth.indexOf("/");

     if (i > 0) {

       String yeardir = yearmonth.substring(0,i);

       File updir =  new File(yeardir);

       files = updir.listFiles();

       analog(files,todayYearmonth,dayString,yeardir);

     }

     System.out.println("Completed at: " + new Date());

   }

   /**

    * Run analog on all translated files in the supplied list of directories.

    */

   static void analog(File[] dirs, String yearmonth, String dayString, String destination) throws Exception {

     FilenameFilter translatedFilter = new TranslatedFilter(yearmonth, dayString);

     String filenames = "";

     for (int i=0; i<dirs.length; i++) {    

       File[] files = dirs[i].listFiles(translatedFilter);

       if (files != null) {

         for (int j=0; j<files.length; j++) {    

           filenames += " " + files[j];

         }

       }

     }

     System.out.println("Analogging " + filenames);

     run("analog -G +ganalog.cfg -O" + destination + File.separator + "index.html" + filenames);

   }   

   static void run(String command) throws Exception {

     Process p = Runtime.getRuntime().exec(command);

     InputStreamReader error = new InputStreamReader(p.getErrorStream());

     InputStreamReader input = new InputStreamReader(p.getInputStream());

     boolean ok = true;

     while (ok) {

       ok = ((error.read() != -1) || (input.read() != -1));

     }

     p.waitFor();

   }

 }

 /**

  * Select all files that are not for today.

  */

 class NotTodayFilter implements FilenameFilter {

   String yearmonth, day;

   public NotTodayFilter(String yearmonth, String day) {

     this.yearmonth = yearmonth;

     this.day = day;

   }

   public boolean accept(File dir, String name) {

     if (!dir.toString().equals(yearmonth)) return true;

     if (name.length() > 1) {

       if (day.equals(name.substring(0,2))) return false;

     }

     return true;

   }

 }  

 /**

  * Select files that have been translated.

  */

 class TranslatedFilter extends NotTodayFilter {

   public TranslatedFilter(String yearmonth, String day) {

     super(yearmonth, day);

   }

   public boolean accept(File dir, String name) {

     if (name.endsWith("access.log.translated.gz") && super.accept(dir, name)) return true;

     return false;

   }

 }

 /**

  *  Select files that need translating and webalizing.

  */

 class WebalizerFilter extends NotTodayFilter {

   public WebalizerFilter(String yearmonth, String day) {

     super(yearmonth, day);

   }

   public boolean accept(File dir, String name) {

     File translated1 = new File(dir, name + ".translated.gz");

     File translated2 = new File(dir, name + ".translated");

     if (!translated1.exists() && !translated2.exists() && name.endsWith("access.log") && super.accept(dir, name)) return true;

     return false;

   }

 }  

 /**

  * Translate a log file.

  */

 class Translator {

   File fileoutput;

   final BibliomaniaDatabase db;

   public Translator() {

     db = new BibliomaniaDatabase(false);

     db.connect("bibliomania", "org.melati.poem.dbms.Postgresql",

     "jdbc:postgresql:bibliomania", "postgres", "*",4);

   }

   public File doIt(final File file) throws Exception {

     db.inSession(

     AccessToken.root,       // FIXME

     new PoemTask() {

       public void run() {

         try {

           doTranslate(db, file);

         }

         catch (Exception e) {

           throw new UnexpectedExceptionPoemException(e);

         }

       }

     }

     );

     return fileoutput;

   }

   private void doTranslate(BibliomaniaDatabase dbP, File fileinput) throws Exception {

     System.out.println("Started translation at: " + new Date());

     fileoutput = new File(fileinput.toString() + ".translated");

     FileReader filein = new FileReader(fileinput);

     PrintWriter fileout = new PrintWriter(new FileWriter(fileoutput));

     BufferedReader in = new BufferedReader(filein);

     int totalLines = 0;

     int translatedLines = 0;

     int failedLines = 0;

     int failedSectionGroup = 0;

     int failedSection = 0;

     int failedAuthor = 0;

     int failedBook = 0;

     String line = "";

     while ((line = in.readLine()) != null) {

       totalLines++;

       LogLine ln = new LogLine(dbP,line, totalLines);

       fileout.println(ln.translate());

       if (ln.translatedAll == LogLine.TRANSLATED) translatedLines++;

       if (ln.translatedAll == LogLine.FAILED_TRANSLATED) failedLines++;

       if (ln.translatedSectionGroup == LogLine.FAILED_TRANSLATED) failedSectionGroup++;

       if (ln.translatedSection == LogLine.FAILED_TRANSLATED) failedSection++;

       if (ln.translatedAuthor == LogLine.FAILED_TRANSLATED) failedAuthor++;

       if (ln.translatedBook == LogLine.FAILED_TRANSLATED) failedBook++;

     }

     in.close();

     fileout.close();

     System.out.println("Found " + totalLines + " lines, translated " + translatedLines + " lines, failed on "+ failedLines + " lines.");

     System.out.println("Failed:  SectionGroup " + failedSectionGroup + ", Section " + failedSection + ", Author " + failedAuthor + ", Book " + failedBook);

   }

 }

   class LogLine {

     static String lookForStart = "GET ";

     static String lookForEnd = " HTTP"; 

     static int lookForStartLength = lookForStart.length();

     static int NOT_TRANSLATED = 0;

     static int TRANSLATED = 1;

     static int FAILED_TRANSLATED = 2;

     int translatedAll = NOT_TRANSLATED;

     int translatedSectionGroup = NOT_TRANSLATED;

     int translatedSection = NOT_TRANSLATED;

     int translatedAuthor = NOT_TRANSLATED;

     int translatedBook = NOT_TRANSLATED;

     int lineno;

     String in;

     String middle = "";

     String end = "";

     BibliomaniaDatabase db;

     public LogLine(BibliomaniaDatabase db, String in, int lineno) {

       this.in = in;

       this.db = db;

       this.lineno = lineno;

     }

     public String translate() {

       String start = in;

       String endBit = "";

       String middleBit = "";

       int length = in.length();

       int startPoint = in.indexOf(lookForStart);

       if (startPoint > -1) {

         startPoint += lookForStartLength;

         start = in.substring(0,startPoint);

         endBit = in.substring(startPoint, length);

         int middlePoint = in.indexOf(lookForEnd, startPoint);

         if (middlePoint > -1) {

           middleBit = in.substring(startPoint,middlePoint);

           middleBit = lookup(middleBit);

           endBit = in.substring(middlePoint, length);

         }

       }

       if (translatedSectionGroup == TRANSLATED || 

           translatedSection == TRANSLATED ||

           translatedAuthor == TRANSLATED ||

           translatedBook == TRANSLATED) {

         translatedAll = TRANSLATED;

       }

       if (translatedSectionGroup == FAILED_TRANSLATED || 

           translatedSection == FAILED_TRANSLATED ||

           translatedAuthor == FAILED_TRANSLATED ||

           translatedBook == FAILED_TRANSLATED) {

         translatedAll = FAILED_TRANSLATED;

       }

       return start + middleBit + endBit;

     }

     public String lookup(String inP) {

       if (inP.startsWith("/")) inP = inP.substring(1,inP.length());

       String[] split = StringUtils.split(inP, '/');

       int len = split.length;

       if (len > 4) len = 4;

       try {

         for (int i=0; i<len; i++) {

           split[i] = look(i,split[i]);

         }

       } catch (NumberFormatException e) {}

       return join(split);

     }

     public String look(int place,String inP) throws NumberFormatException {

       Integer i = new Integer(inP);

       switch (place) {

         case 0: 

           return lookupSectionGroup(i);

         case 1: 

           return lookupSection(i);

         case 2: 

           return lookupAuthor(i);

         case 3: 

           return lookupBook(i);

         default:

           return inP;

       }

     }

     public String lookupSectionGroup(Integer i) {

       try {

         translatedSectionGroup = TRANSLATED;

         return nospaces(db.getSectionGroupTable().getSectionGroupObject(i).getDisplayname());

       } catch (NoSuchRowPoemException e) {

         translatedSectionGroup = FAILED_TRANSLATED;

         System.err.println("Line no " + lineno + "  Not Found: Section Group " + i);

         System.err.println("        " + in);

         return i + "";

       }

     }

     public String lookupSection(Integer i) throws NoSuchRowPoemException {

       try {

         translatedSection = TRANSLATED;

         return nospaces(db.getSectionTable().getSectionObject(i).getDisplayname());

       } catch (NoSuchRowPoemException e) {

         translatedSection = FAILED_TRANSLATED;

         System.err.println("Line no " + lineno + "  Not Found:       Section " + i);

         System.err.println("        " + in);

         return i + "";

       }

     }

     public String lookupAuthor(Integer i) throws NoSuchRowPoemException {

       try {

         translatedAuthor = TRANSLATED;

         return nospaces(db.getAuthorTable().getAuthorObject(i).getSortname());

       } catch (NoSuchRowPoemException e) {

         translatedAuthor = FAILED_TRANSLATED;

         System.err.println("Line no " + lineno + "  Not Found:        Author " + i);

         System.err.println("        " + in);

         return i + "";

       }

     }

     public String lookupBook(Integer i) throws NoSuchRowPoemException {

       try {

         translatedBook = TRANSLATED;

         return nospaces(db.getBookTable().getBookObject(i).getTitle());

       } catch (NoSuchRowPoemException e) {

         translatedBook = FAILED_TRANSLATED;

         System.err.println("Line no " + lineno + "  Not Found:          Book " + i);

         System.err.println("        " + in);

         return i + "";

       }

     }

     public String join(String[] split) {

       String out = "";

       for (int i=0; i<split.length; i++) {

         out += "/" + split[i];

       }

       return out;

     }

     public static String nospaces(String s) {

       if (s == null) return null;

       String out = s.replace(' ','_');

       return out.replace('&','n');

     }

   }