[Koha-cvs] koha/misc/migration_tools bulkmarcimport.pl

[Joshua Ferraro]

CVSROOT:	/sources/koha
Module name:	koha
Branch: 	
Changes by:	Joshua Ferraro <kados at savannah.gnu.org>	06/02/25 21:53:48

Modified files:
	misc/migration_tools: bulkmarcimport.pl 

Log message:
	A bulkmarcimport.pl that is based on the new Biblio.pm Zebra routines.
	It now responds to:
	
	-n : the number of records to import.
	-commit : the number of records to wait before performing a 'commit' operation
	
	ALSO: IMPORTANT: I took out the char_encoding as this should be handled by
	MARC::File::XML now, unless I'm mistaken.

CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/koha/koha/misc/migration_tools/bulkmarcimport.pl.diff?tr1=1.2&tr2=1.3&r1=text&r2=text

Patches:
Index: koha/misc/migration_tools/bulkmarcimport.pl
diff -u koha/misc/migration_tools/bulkmarcimport.pl:1.2 koha/misc/migration_tools/bulkmarcimport.pl:1.3
--- koha/misc/migration_tools/bulkmarcimport.pl:1.2	Thu Aug  4 14:14:53 2005
+++ koha/misc/migration_tools/bulkmarcimport.pl	Sat Feb 25 21:53:48 2006
@@ -12,11 +12,16 @@
 use Time::HiRes qw(gettimeofday);
 
 use Getopt::Long;
+
+my $Zconn = C4::Context->Zconn or die "unable to set Zconn";
+
 my ( $input_marc_file, $number) = ('',0);
-my ($version, $delete, $test_parameter,$char_encoding, $verbose);
+my ($version, $delete, $test_parameter,$char_encoding, $verbose, $commit);
+
 GetOptions(
+	'commit:f'	=> \$commit,
     'file:s'    => \$input_marc_file,
-    'n' => \$number,
+    'n:f' => \$number,
     'h' => \$version,
     'd' => \$delete,
     't' => \$test_parameter,
@@ -31,17 +36,20 @@
 \th : this version/help screen
 \tfile /path/to/file/to/dump : the file to dump
 \tv : verbose mode. 1 means "some infos", 2 means "MARC dumping"
-\tn : the number of the record to import. If missing, all the file is imported
+\tn : the number of records to import. If missing, all the file is imported
+\tcommit : the number of records to wait before performing a 'commit' operation
 \tt : test mode : parses the file, saying what he would do, but doing nothing.
 \tc : the char encoding. At the moment, only MARC21 and UNIMARC supported. MARC21 by default.
-\d : delete EVERYTHING related to biblio in koha-DB before import  :tables :
+\td : delete EVERYTHING related to biblio in koha-DB before import  :tables :
 \t\tbiblio, \t\tbiblioitems, \t\tsubjects,\titems
 \t\tadditionalauthors, \tbibliosubtitles, \tmarc_biblio,
 \t\tmarc_subfield_table, \tmarc_word, \t\tmarc_blob_subfield
-IMPORTANT : don't use this script before you've entered and checked twice (or more) your  MARC parameters tables.
-If you fail this, the import won't work correctly and you will get invalid datas.
+IMPORTANT : don't use this script before you've entered and checked your MARC parameters tables twice (or more!).
+Otherwise, the import won't work correctly and you will get invalid data.
 
-SAMPLE : ./bulkmarcimport.pl -file /home/paul/koha.dev/local/npl -n 1
+SAMPLE : 
+\t\$ export KOHA_CONF=/etc/koha.conf
+\t\$ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 -file /home/jmf/koha.mrc -n 3000
 EOF
 ;#'
 die;
@@ -73,11 +81,29 @@
 $batch->warnings_off();
 $batch->strict_off();
 my $i=0;
+$commit = 50 unless ($commit);
+
 #1st of all, find item MARC tag.
 my ($tagfield,$tagsubfield) = &MARCfind_marc_from_kohafield($dbh,"items.itemnumber",'');
 # $dbh->do("lock tables biblio write, biblioitems write, items write, marc_biblio write, marc_subfield_table write, marc_blob_subfield write, marc_word write, marc_subfield_structure write, stopwords write");
 while ( my $record = $batch->next() ) {
+warn "I:".$i;
+warn "NUM:".$number;
 	$i++;
+
+	if ($i==$number) {
+		z3950_extended_services($Zconn,'commit',set_service_options('commit'));
+		print "COMMIT OPERATION SUCCESSFUL\n" if $verbose;
+
+		my $timeneeded = gettimeofday - $starttime;
+		die "$i MARC records imported in $timeneeded seconds\n";
+	}
+	# perform the commit operation ever so often
+	if ($i==$commit) {
+		z3950_extended_services($Zconn,'commit',set_service_options('commit'));
+		$commit*=2;
+		print "COMMIT OPERATION SUCCESSFUL\n" if $verbose;
+	}
 	#now, parse the record, extract the item fields, and store them in somewhere else.
 
     ## create an empty record object to populate
@@ -98,9 +124,9 @@
 
 	# go through each subfield code/data pair
 	foreach my $pair ( $oldField->subfields() ) { 
-		$pair->[1] =~ s/\<//g;
-		$pair->[1] =~ s/\>//g;
-		push( @newSubfields, $pair->[0], char_decode($pair->[1],$char_encoding) );
+		#$pair->[1] =~ s/\<//g;
+		#$pair->[1] =~ s/\>//g;
+		push( @newSubfields, $pair->[0], $pair->[1] ); #char_decode($pair->[1],$char_encoding) );
 	}
 
 	# add the new field to our new record
@@ -115,7 +141,6 @@
 
     }
 
-
 	warn "$i ==>".$newRecord->as_formatted() if $verbose eq 2;
 	my @fields = $newRecord->field($tagfield);
 	my @items;
@@ -131,13 +156,16 @@
 	print "$i : $nbitems items found\n" if $verbose;
 	# now, create biblio and items with NEWnewXX call.
 	unless ($test_parameter) {
-		my ($bibid,$oldbibnum,$oldbibitemnum) = NEWnewbiblio($dbh,$newRecord,'');
+		my ($bibid,$oldbibnum,$oldbibitemnum) = NEWnewbiblio($dbh,$Zconn,$newRecord,'');
 		warn "ADDED biblio NB $bibid in DB\n" if $verbose;
 		for (my $i=0;$i<=$#items;$i++) {
-			NEWnewitem($dbh,$items[$i],$bibid);
+			NEWnewitem($dbh,$Zconn,$items[$i],$bibid);
 		}
 	}
 }
-# $dbh->do("unlock tables");
+# final commit of the changes
+z3950_extended_services($Zconn,'commit',set_service_options('commit'));
+print "COMMIT OPERATION SUCCESSFUL\n" if $verbose;
+
 my $timeneeded = gettimeofday - $starttime;
-print "$i MARC record done in $timeneeded seconds";
+print "$i MARC records done in $timeneeded seconds\n";