[BACK]Return to parse_activity_log.pl CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.8 and 1.12

version 1.8, 2004/12/20 19:53:36 version 1.12, 2005/02/09 21:24:33
Line 34 Line 34
 #   2     Activity log does not exist  #   2     Activity log does not exist
 #   3     Unable to connect to database  #   3     Unable to connect to database
 #   4     Unable to create database tables  #   4     Unable to create database tables
 #   5     Unspecified error?  #   5     Unable to open log file
   #   6     Unable to get lock on activity log
 #  #
   
 #  #
Line 50 Line 51
   
 use strict;  use strict;
 use DBI;  use DBI;
   use lib '/home/httpd/lib/perl/Apache';
 use lib '/home/httpd/lib/perl/';  use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration();  use LONCAPA::Configuration();
 use Apache::lonmysql();  use Apache::lonmysql();
Line 62  use Fcntl qw(:flock); Line 64  use Fcntl qw(:flock);
   
 #  #
 # Determine parameters  # Determine parameters
 my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);  my ($help,$course,$domain,$drop_when_done,$srcfile,$logfile,$time_run,$nocleanup,$log,$backup);
 &Getopt::Long::GetOptions( "course=s"  => \$course,  &Getopt::Long::GetOptions( "course=s"  => \$course,
                            "domain=s"  => \$domain,                             "domain=s"  => \$domain,
                            "backup"    => \$backup,                             "backup"    => \$backup,
                            "help"      => \$help,                             "help"      => \$help,
                            "logfile=s" => \$file,                             "logfile=s" => \$logfile,
                              "srcfile=s" => \$srcfile,
                            "timerun"   => \$time_run,                             "timerun"   => \$time_run,
                            "nocleanup" => \$nocleanup,                             "nocleanup" => \$nocleanup,
                            "drop"      => \$drop,                             "dropwhendone" => \$drop_when_done,
                            "log"       => \$log);                             "log"       => \$log);
 if (! defined($course) || $help) {  if (! defined($course) || $help) {
     print<<USAGE;      print<<USAGE;
Line 79  parse_activity_log.pl Line 82  parse_activity_log.pl
 Process a lon-capa activity log into a database.  Process a lon-capa activity log into a database.
 Parameters:  Parameters:
    course             Required     course             Required
    domain             Optional     domain             optional
    backup             optional   if present, backup the activity log file     backup             optional   if present, backup the activity log file
                                  before processing it                                   before processing it
    drop               optional   if present, drop all course      dropwhendone       optional   if present, drop all course 
                                  specific activity log tables.                                   specific activity log tables after processing.
    file               optional   Specify the file to parse, including path     srcfile            optional   Specify the file to parse, including path
    time               optional   if present, print out timing data     time               optional   if present, print out timing data
    nocleanup          optional   if present, do not remove old files     nocleanup          optional   if present, do not remove old files
    log                optional   if present, prepare log file of activity     log                optional   if present, prepare log file of activity
      logfile            optional   specifies the logfile to use
 Examples:  Examples:
   $0 -course=123456abcdef -domain=msu    $0 -course=123456abcdef -domain=msu
   $0 -course=123456abcdef -file=activity.log    $0 -course=123456abcdef -srcfile=activity.log
     $0 -course-123456abcdef -log -logfile=/tmp/logfile -dropwhendone
 USAGE  USAGE
     exit;      exit;
 }  }
Line 116  if (! defined($domain) || $domain eq '') Line 121  if (! defined($domain) || $domain eq '')
 ##  ##
 ## Set up logging code  ## Set up logging code
 my $logthis = \&nothing;  my $logthis = \&nothing;
   
 if ($log) {  if ($log) {
     my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;      if (! $logfile) {
           $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;
       }
     print STDERR "$0: logging to $logfile".$/;      print STDERR "$0: logging to $logfile".$/;
     if (! open(LOGFILE,">$logfile")) {      if (! open(LOGFILE,">$logfile")) {
         die "Unable to open $logfile for writing.  Run aborted.";          warn("Unable to open $logfile for writing.  Run aborted.");
           exit 5;
     } else {      } else {
         $logthis = \&log_to_file;          $logthis = \&log_to_file;
     }      }
Line 134  my $sourcefilename;   # activity log dat Line 143  my $sourcefilename;   # activity log dat
 my $newfilename;      # $sourcefilename will be renamed to this  my $newfilename;      # $sourcefilename will be renamed to this
 my $gz_sql_filename;  # the gzipped mysql backup data file name.  my $gz_sql_filename;  # the gzipped mysql backup data file name.
 my $error_filename;   # Errors in parsing the activity log will be written here  my $error_filename;   # Errors in parsing the activity log will be written here
 if ($file) {  if ($srcfile) {
     $sourcefilename = $file;      $sourcefilename = $srcfile;
 } else {  } else {
     $sourcefilename = &get_filename($course,$domain);      $sourcefilename = &get_filename($course,$domain);
 }  }
Line 151  $logthis->('Beginning logging '.time); Line 160  $logthis->('Beginning logging '.time);
 # Wait for a lock on the lockfile to avoid collisions  # Wait for a lock on the lockfile to avoid collisions
 my $lockfilename = $sourcefilename.'.lock';  my $lockfilename = $sourcefilename.'.lock';
 open(LOCKFILE,'>'.$lockfilename);  open(LOCKFILE,'>'.$lockfilename);
 flock(LOCKFILE,LOCK_EX) || die("Unable to lock $lockfilename.  Aborting".$/);  if (!flock(LOCKFILE,LOCK_EX)) {
       warn("Unable to lock $lockfilename.  Aborting".$/);
       exit 6;
   }
   
 ##  ##
 ## There will only be a $newfilename file if a copy of this program is already  ## There will only be a $newfilename file if a copy of this program is already
Line 181  close(LOCKFILE); Line 193  close(LOCKFILE);
 ## Table definitions  ## Table definitions
 ##  ##
 my $prefix = $course.'_'.$domain.'_';  my $prefix = $course.'_'.$domain.'_';
 my $student_table = $prefix.'students';  my $student_table = &Apache::lonmysql::fix_table_name($prefix.'students');
 my $student_table_def =   my $student_table_def = 
 { id => $student_table,  { id => $student_table,
   permanent => 'no',    permanent => 'no',
Line 197  my $student_table_def = Line 209  my $student_table_def =
       'PRIMARY KEY' => ['student_id',],        'PRIMARY KEY' => ['student_id',],
           };            };
   
 my $res_table = $prefix.'resource';  my $res_table = &Apache::lonmysql::fix_table_name($prefix.'resource');
 my $res_table_def =   my $res_table_def = 
 { id => $res_table,  { id => $res_table,
   permanent => 'no',    permanent => 'no',
Line 212  my $res_table_def = Line 224  my $res_table_def =
   'PRIMARY KEY' => ['res_id'],    'PRIMARY KEY' => ['res_id'],
 };  };
   
 #my $action_table = $prefix.'actions';  #my $action_table = &Apache::lonmysql::fix_table_name($prefix.'actions');
 #my $action_table_def =  #my $action_table_def =
 #{ id => $action_table,  #{ id => $action_table,
 #  permanent => 'no',  #  permanent => 'no',
Line 227  my $res_table_def = Line 239  my $res_table_def =
 #  'PRIMARY KEY' => ['action_id',],   #  'PRIMARY KEY' => ['action_id',], 
 #};  #};
   
 my $machine_table = $prefix.'machine_table';  my $machine_table = &Apache::lonmysql::fix_table_name($prefix.'machine_table');
 my $machine_table_def =  my $machine_table_def =
 { id => $machine_table,  { id => $machine_table,
   permanent => 'no',    permanent => 'no',
Line 242  my $machine_table_def = Line 254  my $machine_table_def =
   'PRIMARY KEY' => ['machine_id',],    'PRIMARY KEY' => ['machine_id',],
  };   };
   
 my $activity_table = $prefix.'activity';  my $activity_table = &Apache::lonmysql::fix_table_name($prefix.'activity');
 my $activity_table_def =   my $activity_table_def = 
 { id => $activity_table,  { id => $activity_table,
   permanent => 'no',    permanent => 'no',
Line 277  my $activity_table_def = Line 289  my $activity_table_def =
 my @Activity_Table = ($activity_table_def);  my @Activity_Table = ($activity_table_def);
 my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);  my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);
 ##  ##
 ## End of table definitions  ## End of table definitionsOB
 ##  ##
   
 $logthis->('Connectiong to mysql');  $logthis->('Connectiong to mysql');
Line 290  if (!&Apache::lonmysql::verify_sql_conne Line 302  if (!&Apache::lonmysql::verify_sql_conne
 }  }
 $logthis->('SQL connection is up');  $logthis->('SQL connection is up');
   
 if ($drop) { &drop_tables(); $logthis->('dropped tables'); }  
   
 if (-s $gz_sql_filename) {  if (-s $gz_sql_filename) {
     my $backup_modification_time = (stat($gz_sql_filename))[9];      my $backup_modification_time = (stat($gz_sql_filename))[9];
     $logthis->($gz_sql_filename.' was last modified '.      $logthis->($gz_sql_filename.' was last modified '.
Line 376  if (-s $newfilename) { Line 386  if (-s $newfilename) {
         &backup_tables($gz_sql_filename);          &backup_tables($gz_sql_filename);
         $time_this->('write backup tables');          $time_this->('write backup tables');
     }      }
       if ($drop_when_done) { &drop_tables(); $logthis->('dropped tables'); }
 }  }
 close($error_fh);  close($error_fh);
   
Line 399  if ($log) { Line 410  if ($log) {
     close LOGFILE;      close LOGFILE;
 }  }
   
   foreach my $file ($lockfilename, $error_filename,$logfile) {
       if (-z $file) { 
           unlink($file); 
       }
   }
   
   
 exit 0;   # Everything is okay, so end here before it gets worse.  exit 0;   # Everything is okay, so end here before it gets worse.
   
 ########################################################  ########################################################
Line 474  sub process_courselog { Line 492  sub process_courselog {
                          'uname= '.$uname.$/.                           'uname= '.$uname.$/.
                          'udom = '.$udom.$/.                           'udom = '.$udom.$/.
                          'action='.$action.$/.                           'action='.$action.$/.
                          '@values = '.join(':',@values));                           '@values = '.join('&',@values));
                 next; #skip it if we cannot understand what is happening.                  next; #skip it if we cannot understand what is happening.
             }              }
             if (! defined($student) || $student eq ':') {              if (! defined($student) || $student eq ':') {
Line 518  sub process_courselog { Line 536  sub process_courselog {
                 next; # skip this chunk                  next; # skip this chunk
             }              }
             #              #
             my $values = $dbh->quote(join('',@values));              my $store_values;
               if ($action eq 'POST') {
                   $store_values = 
                       $dbh->quote(join('&',map { &escape($_); } @values));
               } else {
                   $store_values = $dbh->quote(join('&',@values));
               }
             $time_this->('get_ids');              $time_this->('get_ids');
             #              #
             my $row = [$res_id,              my $row = [$res_id,
Line 528  sub process_courselog { Line 552  sub process_courselog {
 #                       $action_id,  #                       $action_id,
                        qq{''},        # idx                         qq{''},        # idx
                        $machine_id,                         $machine_id,
                        $values];                         $store_values];
             push(@RowData,$row);              push(@RowData,$row);
             $time_this->('push_row');              $time_this->('push_row');
             $prevchunk = $chunk;              $prevchunk = $chunk;
Line 625  sub outputtimes { Line 649  sub outputtimes {
 ##  ##
 sub backup_tables {  sub backup_tables {
     my ($gz_sql_filename) = @_;      my ($gz_sql_filename) = @_;
     my $command = qq{mysqldump --opt loncapa };      my $command = qq{mysqldump --quote-names --opt loncapa };
                                
     foreach my $table (@ID_Tables,@Activity_Table) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $tablename = $table->{'id'};          my $tablename = $table->{'id'};
           $tablename =~ s/\`//g;
         $command .= $tablename.' ';          $command .= $tablename.' ';
     }      }
     $command .= '| gzip >'.$gz_sql_filename;      $command .= '| gzip >'.$gz_sql_filename;
Line 668  sub get_filename { Line 692  sub get_filename {
 sub create_tables {  sub create_tables {
     foreach my $table (@ID_Tables,@Activity_Table) {      foreach my $table (@ID_Tables,@Activity_Table) {
         my $table_id = &Apache::lonmysql::create_table($table);          my $table_id = &Apache::lonmysql::create_table($table);
 #        print STDERR "Unable to create table ".$table->{'id'}.$/;  
 #        print STDERR join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;  
         if (! defined($table_id)) {          if (! defined($table_id)) {
             warn "Unable to create table ".$table->{'id'}.$/;              warn "Unable to create table ".$table->{'id'}.$/;
             warn join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;              $logthis->('Unable to create table '.$table->{'id'});
               $logthis->(join($/,&Apache::lonmysql::build_table_creation_request($table)));
             return 0;              return 0;
         }          }
     }      }
Removed from v.1.8  
changed lines
  Added in v.1.12

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>