[Koha-cvs] CVS: koha/misc build_marc_Tword.pl,1.1,1.2

[Paul POULAIN]

Update of /cvsroot/koha/koha/misc
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv12257/misc

Modified Files:
	build_marc_Tword.pl 
Log Message:
new version, doing tag by tag. Should be slower, but requires a lot less memory

Index: build_marc_Tword.pl
===================================================================
RCS file: /cvsroot/koha/koha/misc/build_marc_Tword.pl,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -r1.1 -r1.2
*** build_marc_Tword.pl	27 May 2005 09:30:24 -0000	1.1
--- build_marc_Tword.pl	1 Jun 2005 18:47:38 -0000	1.2
***************
*** 55,124 ****
  
  # parse every line
! my $query="SELECT biblio.biblionumber,tag,subfieldcode,subfieldvalue,biblio.title FROM marc_subfield_table left join marc_biblio on marc_biblio.bibid=marc_subfield_table.bibid left join biblio on marc_biblio.biblionumber=biblio.biblionumber";
  my $sth=$dbh->prepare($query);
  
! print "******** SELECTING \n";
! $sth->execute;
! print "******** DONE \n";
! $|=1; # flushes output
! 
! my $sthT=$dbh->prepare("select usedin from marc_Tword where tagsubfield=? and word=?");
! my $updateT=$dbh->prepare("update marc_Tword set usedin=? where tagsubfield=? and word=?");
! my $insertT=$dbh->prepare("insert into marc_Tword (tagsubfield,word,usedin) values (?,?,?)");
! my $i=0;
! my $timeneeded;
! # 1st version, slower, but less RAM consumming
! # while (my ($biblionumber, $tag, $subfieldcode, $subfieldvalue, $title) = $sth->fetchrow) {
! # 	next if $ignore_list{"$tag.$subfieldcode"};
! #     $subfieldvalue =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g;
! # 	# remove useless chars in the title.
! #     $title =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g;
! #     my @words = split / /, $subfieldvalue;
! # 	# and retrieve the reversed entry
! # 	foreach my $word (@words) {
! # 		$sthT->execute($tag.$subfieldcode,$word);
! # 		if (my ($usedin) = $sthT->fetchrow) {
! # 			# add the field & save it once again.
! # 			$usedin.=",$biblionumber-$title";
! # 			$updateT->execute($usedin,$tag.$subfieldcode,$word);
! # 		} else {
! # 			$insertT->execute($tag.$subfieldcode,$word,",$title-$biblionumber");
! # 		}
! # 	}
! # 	$timeneeded = gettimeofday - $starttime unless ($i % 100);
! # 	print "$i in $timeneeded s\n" unless ($i % 100);
! # 	print ".";
! # 	$i++;
! # }
! 
! # 2nd version : faster (about 100 times !), bug maybe too much RAM consumming...
! my %largehash;
! print "READING\n";
! while (my ($biblionumber, $tag, $subfieldcode, $subfieldvalue, $title) = $sth->fetchrow) {
! 	next unless $subfieldvalue;
! 	next if $ignore_list{$tag.$subfieldcode};
!     $subfieldvalue =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g;
! 	# remove useless chars in the title.
!     $title =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g;
!     my @words = split / /, $subfieldvalue;
! 	# and retrieve the reversed entry
! 	foreach my $word (@words) {
! 		my $localkey = $tag.$subfieldcode.'|'.uc($word);
! 		$largehash{$localkey}.=",$title-$biblionumber";
  	}
- 	$timeneeded = gettimeofday - $starttime unless ($i % 30000);
- 	print "$i in $timeneeded s\n" unless ($i % 30000);
- 	print "." unless ($i % 500);
- 	$i++;
- }
- $i=0;
- print "WRITING\n";
- foreach my $k (keys %largehash) {
- 	$k =~ /(.*)\|(.*)/;
- 	$insertT->execute($1,$2,$largehash{$k});
- 	$timeneeded = gettimeofday - $starttime unless ($i % 30000);
- 	print "$i in $timeneeded s\n" unless ($i % 30000);
- 	print "." unless ($i % 500);
- 	$i++;
  }
  
--- 55,126 ----
  
  # parse every line
! my $query="SELECT biblio.biblionumber,tag,subfieldcode,subfieldvalue,biblio.title FROM marc_subfield_table left join marc_biblio on marc_biblio.bibid=marc_subfield_table.bibid left join biblio on marc_biblio.biblionumber=biblio.biblionumber and tag=?";
  my $sth=$dbh->prepare($query);
  
! for (my $looptag=0;$looptag<=999;$looptag++) {
! 	print "******** SELECTING ".(sprintf "%03s",$looptag)."\n";
! 	$sth->execute(sprintf "%03s",$looptag);
! 	print "******** DONE \n";
! 	$|=1; # flushes output
! 	
! 	my $sthT=$dbh->prepare("select usedin from marc_Tword where tagsubfield=? and word=?");
! 	my $updateT=$dbh->prepare("update marc_Tword set usedin=? where tagsubfield=? and word=?");
! 	my $insertT=$dbh->prepare("insert into marc_Tword (tagsubfield,word,usedin) values (?,?,?)");
! 	my $i=0;
! 	my $timeneeded;
! 	# 1st version, slower, but less RAM consumming
! 	# while (my ($biblionumber, $tag, $subfieldcode, $subfieldvalue, $title) = $sth->fetchrow) {
! 	# 	next if $ignore_list{"$tag.$subfieldcode"};
! 	#     $subfieldvalue =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g;
! 	# 	# remove useless chars in the title.
! 	#     $title =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g;
! 	#     my @words = split / /, $subfieldvalue;
! 	# 	# and retrieve the reversed entry
! 	# 	foreach my $word (@words) {
! 	# 		$sthT->execute($tag.$subfieldcode,$word);
! 	# 		if (my ($usedin) = $sthT->fetchrow) {
! 	# 			# add the field & save it once again.
! 	# 			$usedin.=",$biblionumber-$title";
! 	# 			$updateT->execute($usedin,$tag.$subfieldcode,$word);
! 	# 		} else {
! 	# 			$insertT->execute($tag.$subfieldcode,$word,",$title-$biblionumber");
! 	# 		}
! 	# 	}
! 	# 	$timeneeded = gettimeofday - $starttime unless ($i % 100);
! 	# 	print "$i in $timeneeded s\n" unless ($i % 100);
! 	# 	print ".";
! 	# 	$i++;
! 	# }
! 	
! 	# 2nd version : faster (about 100 times !), bug maybe too much RAM consumming...
! 	my %largehash;
! 	print "READING\n";
! 	while (my ($biblionumber, $tag, $subfieldcode, $subfieldvalue, $title) = $sth->fetchrow) {
! 		next unless $subfieldvalue;
! 		next if $ignore_list{$tag.$subfieldcode};
! 		$subfieldvalue =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g if $subfieldvalue;
! 		# remove useless chars in the title.
! 		$title =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\}|\/)/ /g if $title;
! 		my @words = split / /, $subfieldvalue;
! 		# and retrieve the reversed entry
! 		foreach my $word (@words) {
! 			my $localkey = $tag.$subfieldcode.'|'.uc($word);
! 			$largehash{$localkey}.=",".substr($title,0,15)."-$biblionumber";
! 		}
! 		$timeneeded = gettimeofday - $starttime unless ($i % 30000);
! 		print "$i in $timeneeded s\n" unless ($i % 30000);
! 		print "." unless ($i % 500);
! 		$i++;
! 	}
! 	$i=0;
! 	print "WRITING\n";
! 	foreach my $k (keys %largehash) {
! 		$k =~ /(.*)\|(.*)/;
! 		$insertT->execute($1,$2,$largehash{$k});
! 		$timeneeded = gettimeofday - $starttime unless ($i % 30000);
! 		print "$i in $timeneeded s\n" unless ($i % 30000);
! 		print "." unless ($i % 500);
! 		$i++;
  	}
  }