loncom/metadata_database/searchcat.pl - diff

Return to searchcat.pl CVS log

Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/searchcat.pl between versions 1.46 and 1.53

version 1.46, 2003/12/24 20:41:32	version 1.53, 2003/12/27 23:04:28
Line 56 This script also does general database m	Line 56 This script also does general database m
the C<loncapa:metadata> table if it is deprecated.	the C<loncapa:metadata> table if it is deprecated.

This script evaluates dynamic metadata from the authors'	This script evaluates dynamic metadata from the authors'
F<nohist_resevaldata.db> database file in order to store it in MySQL, as	F<nohist_resevaldata.db> database file in order to store it in MySQL.
well as to compress the filesize (add up all "count"-type metadata).

This script is playing an increasingly important role for a loncapa	This script is playing an increasingly important role for a loncapa
library server. The proper operation of this script is critical for a smooth	library server. The proper operation of this script is critical for a smooth
Line 131 sub escape {	Line 130 sub escape {
sub dynamicmeta {	sub dynamicmeta {
my $url=&declutter(shift);	my $url=&declutter(shift);
$url=~s/\.meta$//;	$url=~s/\.meta$//;
my %returnhash=();	my %returnhash=(
	'count' => 0,
	'course' => 0,
	'course_list' => '',
	'avetries' => 'NULL',
	'avetries_list' => '',
	'stdno' => 0,
	'stdno_list' => '',
	'usage' => 0,
	'usage_list' => '',
	'goto' => 0,
	'goto_list' => '',
	'comefrom' => 0,
	'comefrom_list' => '',
	'difficulty' => 'NULL',
	'difficulty_list' => '',
	'clear' => 'NULL',
	'technical' => 'NULL',
	'correct' => 'NULL',
	'helpful' => 'NULL',
	'depth' => 'NULL',
	'comments' => ''
	);
my ($adomain,$aauthor)=($url=~/^(\w+)\/(\w+)\//);	my ($adomain,$aauthor)=($url=~/^(\w+)\/(\w+)\//);
my $prodir=&propath($adomain,$aauthor);	my $prodir=&propath($adomain,$aauthor);

Line 167 sub dynamicmeta {	Line 188 sub dynamicmeta {
if (defined($cnt{$cat})) { $cnt{$cat}++; } else { $cnt{$cat}=1; }	if (defined($cnt{$cat})) { $cnt{$cat}++; } else { $cnt{$cat}=1; }
unless ($listitems{$cat} eq 'app') {	unless ($listitems{$cat} eq 'app') {
if (defined($sum{$cat})) {	if (defined($sum{$cat})) {
$sum{$cat}+=$evaldata{$esckey};	$sum{$cat}+=&unescape($evaldata{$esckey});
$concat{$cat}.=','.$item;	$concat{$cat}.=','.$item;
} else {	} else {
$sum{$cat}=$evaldata{$esckey};	$sum{$cat}=&unescape($evaldata{$esckey});
$concat{$cat}=$item;	$concat{$cat}=$item;
}	}
} else {	} else {
if (defined($sum{$cat})) {	if (defined($sum{$cat})) {
if ($evaldata{$esckey}=~/\w/) {	if ($evaldata{$esckey}=~/\w/) {
$sum{$cat}.='<hr>'.$evaldata{$esckey};	$sum{$cat}.='<hr />'.&unescape($evaldata{$esckey});
}	}
} else {	} else {
$sum{$cat}=''.$evaldata{$esckey};	$sum{$cat}=''.&unescape($evaldata{$esckey});
}	}
}	}
}	}
Line 187 sub dynamicmeta {	Line 208 sub dynamicmeta {
untie(%evaldata);	untie(%evaldata);
# transfer gathered data to returnhash, calculate averages where applicable	# transfer gathered data to returnhash, calculate averages where applicable
while (my $cat=each(%cnt)) {	while (my $cat=each(%cnt)) {
	if ($cnt{$cat} eq 'nan') { next; }
	if ($sum{$cat} eq 'nan') { next; }
if ($listitems{$cat} eq 'avg') {	if ($listitems{$cat} eq 'avg') {
$returnhash{$cat}=int(($sum{$cat}/$cnt{$cat})*100.0+0.5)/100.0;	if ($cnt{$cat}) {
	$returnhash{$cat}=int(($sum{$cat}/$cnt{$cat})*100.0+0.5)/100.0;
	} else {
	$returnhash{$cat}='NULL';
	}
} elsif ($listitems{$cat} eq 'cnt') {	} elsif ($listitems{$cat} eq 'cnt') {
$returnhash{$cat}=$cnt{$cat};	$returnhash{$cat}=$cnt{$cat};
} else {	} else {
Line 214 sub dynamicmeta {	Line 241 sub dynamicmeta {
# --------------- Read loncapa_apache.conf and loncapa.conf and get variables	# --------------- Read loncapa_apache.conf and loncapa.conf and get variables
my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');	my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
my %perlvar=%{$perlvarref};	my %perlvar=%{$perlvarref};
undef $perlvarref; # remove since sensitive and not needed	undef $perlvarref;
delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed	delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed

# ------------------------------------- Only run if machine is a library server	# ------------------------------------- Only run if machine is a library server
Line 249 my $dbh;	Line 276 my $dbh;
exit;	exit;
}	}

# Create table for static metadata, unless exists	# Make temporary table
my $make_metadata_table = "CREATE TABLE IF NOT EXISTS metadata (".	$dbh->do("DROP TABLE IF EXISTS newmetadata");
	my $make_metadata_table = "CREATE TABLE IF NOT EXISTS newmetadata (".
"title TEXT, author TEXT, subject TEXT, url TEXT, keywords TEXT, ".	"title TEXT, author TEXT, subject TEXT, url TEXT, keywords TEXT, ".
"version TEXT, notes TEXT, abstract TEXT, mime TEXT, language TEXT, ".	"version TEXT, notes TEXT, abstract TEXT, mime TEXT, language TEXT, ".
"creationdate DATETIME, lastrevisiondate DATETIME, owner TEXT, ".	"creationdate DATETIME, lastrevisiondate DATETIME, owner TEXT, ".
"copyright TEXT, FULLTEXT idx_title (title), ".	"copyright TEXT, dependencies TEXT, ".
"FULLTEXT idx_author (author), FULLTEXT idx_subject (subject), ".	"modifyinguser TEXT, authorspace TEXT, ".
"FULLTEXT idx_url (url), FULLTEXT idx_keywords (keywords), ".	"lowestgradelevel INTEGER UNSIGNED, highestgradelevel INTEGER UNSIGNED, ".
"FULLTEXT idx_version (version), FULLTEXT idx_notes (notes), ".	"standards TEXT, ".
"FULLTEXT idx_abstract (abstract), FULLTEXT idx_mime (mime), ".	"count INTEGER UNSIGNED, ".
"FULLTEXT idx_language (language), FULLTEXT idx_owner (owner), ".
"FULLTEXT idx_copyright (copyright)) TYPE=MYISAM";
# It would sure be nice to have some logging mechanism.
$dbh->do($make_metadata_table);

# Create table for dynamic metadata, unless exists
my $make_dynmetadata_table = "CREATE TABLE IF NOT EXISTS dynmetadata (".
"url TEXT, count INTEGER UNSIGNED, ".
"course INTEGER UNSIGNED, course_list TEXT, ".	"course INTEGER UNSIGNED, course_list TEXT, ".
"goto INTEGER UNSIGNED, goto_list TEXT, ".	"goto INTEGER UNSIGNED, goto_list TEXT, ".
"comefrom INTEGER UNSIGNED, comefrom_list TEXT, ".	"comefrom INTEGER UNSIGNED, comefrom_list TEXT, ".
"usage INTEGER UNSIGNED, usage_list TEXT, ".	"sequsage INTEGER UNSIGNED, sequsage_list TEXT, ".
"stdno INTEGER UNSIGNED, stdno_list TEXT, ".	"stdno INTEGER UNSIGNED, stdno_list TEXT, ".
"avetries FLOAT, avetries_list TEXT, ".	"avetries FLOAT, avetries_list TEXT, ".
"difficulty FLOAT, difficulty_list TEXT ".	"difficulty FLOAT, difficulty_list TEXT, ".
"TYPE=MYISAM";	"clear FLOAT, technical FLOAT, correct FLOAT, helpful FLOAT, depth FLOAT, ".
	"comments TEXT, ".
	# For backward compatibility, only insert new fields below
	# ...
	# For backward compatibility, end new fields above
	"FULLTEXT idx_title (title), ".
	"FULLTEXT idx_author (author), FULLTEXT idx_subject (subject), ".
	"FULLTEXT idx_url (url), FULLTEXT idx_keywords (keywords), ".
	"FULLTEXT idx_notes (notes), ".
	"FULLTEXT idx_abstract (abstract), FULLTEXT idx_mime (mime), ".
	"FULLTEXT idx_owner (owner), ".
	"FULLTEXT idx_standards (standards))".
	"TYPE=MyISAM";
# It would sure be nice to have some logging mechanism.	# It would sure be nice to have some logging mechanism.
#### $dbh->do($make_dynmetadata_table);	unless ($dbh->do($make_metadata_table)) {
	print LOG "\nMySQL Error Create: ".$dbh->errstr."\n";
	die $dbh->errstr;
	}
}	}

# ------------------------------------------------------------- get .meta files	# ------------------------------------------------------------- get .meta files
Line 289 closedir RESOURCES;	Line 323 closedir RESOURCES;

#	#
# Create the statement handlers we need	# Create the statement handlers we need
my $delete_sth = $dbh->prepare
("DELETE FROM metadata WHERE url LIKE BINARY ?");

my $insert_sth = $dbh->prepare	my $insert_sth = $dbh->prepare
("INSERT INTO metadata VALUES (".	("INSERT INTO newmetadata VALUES (".
"?,". # title	"?,". # title
"?,". # author	"?,". # author
"?,". # subject	"?,". # subject
"?,". # m2???	"?,". # declutter url
"?,". # version	"?,". # version
"?,". # current	"?,". # current
"?,". # notes	"?,". # notes
Line 307 my $insert_sth = $dbh->prepare	Line 339 my $insert_sth = $dbh->prepare
"?,". # creationdate	"?,". # creationdate
"?,". # revisiondate	"?,". # revisiondate
"?,". # owner	"?,". # owner
"?)" # copyright	"?,". # copyright
	"?,". # dependencies
	"?,". # modifyinguser
	"?,". # authorspace
	"?,". # lowestgradelevel
	"?,". # highestgradelevel
	"?,". # standards
	"?,". # count
	"?,". # course
	"?,". # course_list
	"?,". # goto
	"?,". # goto_list
	"?,". # comefrom
	"?,". # comefrom_list
	"?,". # usage
	"?,". # usage_list
	"?,". # stdno
	"?,". # stdno_list
	"?,". # avetries
	"?,". # avetries_list
	"?,". # difficulty
	"?,". # difficulty_list
	"?,". # clear
	"?,". # technical
	"?,". # correct
	"?,". # helpful
	"?,". # depth
	"?". # comments
	")"
);	);

foreach my $user (@homeusers) {	foreach my $user (@homeusers) {
Line 328 foreach my $user (@homeusers) {	Line 388 foreach my $user (@homeusers) {
$m2=~s/\.meta$//;	$m2=~s/\.meta$//;
if ($ref->{'obsolete'}) { print LOG "obsolete\n"; next; }	if ($ref->{'obsolete'}) { print LOG "obsolete\n"; next; }
if ($ref->{'copyright'} eq 'private') { print LOG "private\n"; next; }	if ($ref->{'copyright'} eq 'private') { print LOG "private\n"; next; }
&dynamicmeta($m2);	my %dyn=&dynamicmeta($m2);
&count($m2);	&count($m2);
$delete_sth->execute($m2);	unless ($insert_sth->execute(
$insert_sth->execute($ref->{'title'},	$ref->{'title'},
$ref->{'author'},	$ref->{'author'},
$ref->{'subject'},	$ref->{'subject'},
$m2,	$m2,
Line 344 foreach my $user (@homeusers) {	Line 404 foreach my $user (@homeusers) {
sqltime($ref->{'creationdate'}),	sqltime($ref->{'creationdate'}),
sqltime($ref->{'lastrevisiondate'}),	sqltime($ref->{'lastrevisiondate'}),
$ref->{'owner'},	$ref->{'owner'},
$ref->{'copyright'});	$ref->{'copyright'},
# if ($dbh->err()) {	$ref->{'dependencies'},
# print STDERR "Error:".$dbh->errstr()."\n";	$ref->{'modifyinguser'},
# }	$ref->{'authorspace'},
	$ref->{'lowestgradelevel'},
	$ref->{'highestgradelevel'},
	$ref->{'standards'},
	$dyn{'count'},
	$dyn{'course'},
	$dyn{'course_list'},
	$dyn{'goto'},
	$dyn{'goto_list'},
	$dyn{'comefrom'},
	$dyn{'comefrom_list'},
	$dyn{'usage'},
	$dyn{'usage_list'},
	$dyn{'stdno'},
	$dyn{'stdno_list'},
	$dyn{'avetries'},
	$dyn{'avetries_list'},
	$dyn{'difficulty'},
	$dyn{'difficulty_list'},
	$dyn{'clear'},
	$dyn{'technical'},
	$dyn{'correct'},
	$dyn{'helpful'},
	$dyn{'depth'},
	$dyn{'comments'}
	)) {
	print LOG "\nMySQL Error Insert: ".$dbh->errstr."\n";
	die $dbh->errstr;
	}
$ref = undef;	$ref = undef;
}	}

# --------------------------------------------------- Clean up database
# Need to, perhaps, remove stale SQL database records.
# ... not yet implemented

}	}
# --------------------------------------------------- Close database connection	# --------------------------------------------------- Close database connection
$dbh->disconnect;	$dbh->do("DROP TABLE IF EXISTS metadata");
	unless ($dbh->do("RENAME TABLE newmetadata TO metadata")) {
	print LOG "\nMySQL Error Rename: ".$dbh->errstr."\n";
	die $dbh->errstr;
	}
	unless ($dbh->disconnect) {
	print LOG "\nMySQL Error Disconnect: ".$dbh->errstr."\n";
	die $dbh->errstr;
	}
print LOG "\n==== Searchcat completed ".localtime()." ====\n";	print LOG "\n==== Searchcat completed ".localtime()." ====\n";
close(LOG);	close(LOG);
&writesimple();	&writesimple();
Line 372 exit 0;	Line 463 exit 0;
# significantly altered from subroutine present in lonnet	# significantly altered from subroutine present in lonnet
sub metadata {	sub metadata {
my ($uri,$what)=@_;	my ($uri,$what)=@_;
my %metacache;	my %metacache=();
$uri=&declutter($uri);	$uri=&declutter($uri);
my $filename=$uri;	my $filename=$uri;
$uri=~s/\.meta$//;	$uri=~s/\.meta$//;

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.46
changed lines
	Added in v.1.53