[Koha-cvs] koha/misc/migration_tools bulkmarcimport.pl

[Antoine Farnault]

CVSROOT:	/sources/koha
Module name:	koha
Changes by:	Antoine Farnault <toins>	06/07/04 15:06:35

Modified files:
	misc/migration_tools: bulkmarcimport.pl 

Log message:
	Head & rel_2_2 merged

CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/koha/misc/migration_tools/bulkmarcimport.pl?cvsroot=koha&r1=1.6&r2=1.7

Patches:
Index: bulkmarcimport.pl
===================================================================
RCS file: /sources/koha/koha/misc/migration_tools/bulkmarcimport.pl,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -b -r1.6 -r1.7
--- bulkmarcimport.pl	1 Apr 2006 22:10:50 -0000	1.6
+++ bulkmarcimport.pl	4 Jul 2006 15:06:35 -0000	1.7
@@ -2,14 +2,21 @@
 # small script that import an iso2709 file into koha 2.0
 
 use strict;
+# use warnings;
 
 # Koha modules used
 use MARC::File::USMARC;
+# Uncomment the line below and use MARC::File::XML again when it works better.
+# -- thd
+# use MARC::File::XML;
 use MARC::Record;
 use MARC::Batch;
+use MARC::Charset;
 use C4::Context;
 use C4::Biblio;
 use Time::HiRes qw(gettimeofday);
+use Getopt::Long;
+binmode(STDOUT, ":utf8");
 
 use Getopt::Long;
 
@@ -27,6 +34,110 @@
     'v:s' => \$verbose,
 );
 
+# FIXME:  Management of error conditions needed for record parsing problems
+# and MARC8 character sets with mappings to Unicode not yet included in 
+# MARC::Charset.  The real world rarity of these problems is not fully tested.
+# Unmapped character sets will throw a warning currently and processing will 
+# continue with the error condition.  A fairly trivial correction should 
+# address some record parsing and unmapped character set problems but I need 
+# time to implement a test and correction for undef subfields and revert to 
+# MARC8 if mappings are missing. -- thd
+sub fMARC8ToUTF8($$) {
+	my ($record) = shift;
+	my ($verbose) = shift;
+	if ($verbose) {
+		if ($verbose >= 2) {
+			my $leader = $record->leader();
+			$leader =~ s/ /#/g;
+			print "\n000 " . $leader;
+		}
+	}
+	foreach my $field ($record->fields()) {
+		if ($field->is_control_field()) {
+			if ($verbose) {
+				if ($verbose >= 2) {
+					my $fieldName = $field->tag();
+					my $fieldValue = $field->data();
+					$fieldValue =~ s/ /#/g;
+					print "\n" . $fieldName;
+					print ' ' . $fieldValue;
+				}
+			}
+		} else {
+			my @subfieldsArray;
+			my $fieldName = $field->tag();
+			my $indicator1Value = $field->indicator(1);
+			my $indicator2Value = $field->indicator(2);
+			if ($verbose) {
+				if ($verbose >= 2) {
+					$indicator1Value =~ s/ /#/;
+					$indicator2Value =~ s/ /#/;
+					print "\n" . $fieldName . ' ' . 
+							$indicator1Value . 
+					$indicator2Value;
+				}
+			}
+			foreach my $subfield ($field->subfields()) {
+				my $subfieldName = $subfield->[0];
+				my $subfieldValue = $subfield->[1];
+				$subfieldValue = MARC::Charset::marc8_to_utf8($subfieldValue);
+				
+				# Alas, MARC::Field::update() does not work correctly.
+				## push (@subfieldsArray, $subfieldName, $subfieldValue);
+				
+				push @subfieldsArray, [$subfieldName, $subfieldValue];
+				if ($verbose) {
+					if ($verbose >= 2) {
+						print " \$" . $subfieldName . ' ' . $subfieldValue;
+					}
+				}
+			}
+			
+			# Alas, MARC::Field::update() does not work correctly.
+			# 
+			# The first instance in the field of a of a repeated subfield 
+			# overwrites the content from later instances with the content 
+			# from the first instance.
+			## $field->update(@subfieldsArray);
+			
+			foreach my $subfieldRow(@subfieldsArray) {
+				my $subfieldName = $subfieldRow->[0];
+				$field->delete_subfields($subfieldName);
+			}
+			foreach my $subfieldRow(@subfieldsArray) {
+				$field->add_subfields(@$subfieldRow);
+			}
+			
+			if ($verbose) {
+				if ($verbose >= 2) {
+					# Reading the indicator values again is not necessary.  
+					# They were not converted.
+					# $indicator1Value = $field->indicator(1);
+					# $indicator2Value = $field->indicator(2);
+					# $indicator1Value =~ s/ /#/;
+					# $indicator2Value =~ s/ /#/;
+					print "\nCONVERTED TO UTF-8:\n" . $fieldName . ' ' . 
+							$indicator1Value . 
+					$indicator2Value;
+					foreach my $subfield ($field->subfields()) {
+						my $subfieldName = $subfield->[0];
+						my $subfieldValue = $subfield->[1];
+						print " \$" . $subfieldName . ' ' . $subfieldValue;
+					}
+				}
+			}
+			if ($verbose) {
+				if ($verbose >= 2) {
+					print "\n" if $verbose;
+				}
+			}
+		}
+	}
+	$record->encoding('UTF-8');
+	return $record;
+}
+
+
 if ($version || ($input_marc_file eq '')) {
 	print <<EOF
 small script to import an iso2709 file into Koha.
@@ -37,7 +148,8 @@
 \tn : the number of records to import. If missing, all the file is imported
 \tcommit : the number of records to wait before performing a 'commit' operation
 \tt : test mode : parses the file, saying what he would do, but doing nothing.
-\tc : the char encoding. At the moment, only MARC21 and UNIMARC supported. MARC21 by default.
+\tc : the characteristic MARC flavour. At the moment, only MARC21 and UNIMARC 
+\tsupported. MARC21 by default.
 \td : delete EVERYTHING related to biblio in koha-DB before import  :tables :
 \t\tbiblio, \t\tbiblioitems, \t\tsubjects,\titems
 \t\tadditionalauthors, \tbibliosubtitles, \tmarc_biblio,
@@ -72,8 +184,8 @@
 	print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
 }
 
-$char_encoding = 'MARC21' unless ($char_encoding);
-print "CHAR : $char_encoding\n" if $verbose;
+$marcFlavour = 'MARC21' unless ($marcFlavour);
+print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
 my $starttime = gettimeofday;
 my $batch = MARC::Batch->new( 'USMARC', $input_marc_file );
 $batch->warnings_off();
@@ -118,7 +230,12 @@
     foreach my $oldField ( $record->fields() ) {
 
 	# just reproduce tags < 010 in our new record
-	if ( $oldField->tag() < 10 ) {
+	# 
+	# Fields are not necessarily only numeric in the actual world of records 
+	# nor in what I would recommend for additonal safe non-interfering local
+	# use fields.  The following regular expression match is much safer than 
+	# a numeric evaluation. -- thd
+	if ( $oldField->tag() =~ m/^00/ ) {
 	    $newRecord->append_fields( $oldField );
 	    next();
 	}