[Koha-patches] [PATCH] Bug 5635: bulkmarcimport new parameters & features
Stéphane Delaune
stephane.delaune at biblibre.com
Tue Feb 7 17:25:27 CET 2012
---
C4/AuthoritiesMarc.pm | 4 +-
misc/migration_tools/bulkmarcimport.pl | 216 ++++++++++++++++++++++++--------
2 files changed, 166 insertions(+), 54 deletions(-)
diff --git a/C4/AuthoritiesMarc.pm b/C4/AuthoritiesMarc.pm
index b3c5070..80178b9 100644
--- a/C4/AuthoritiesMarc.pm
+++ b/C4/AuthoritiesMarc.pm
@@ -412,9 +412,9 @@ Get the record and tries to guess the adequate authtypecode from its content.
=cut
sub GuessAuthTypeCode {
- my ($record) = @_;
+ my ($record, $heading_fields) = @_;
return unless defined $record;
-my $heading_fields = {
+ $heading_fields //= {
"MARC21"=>{
'100'=>{authtypecode=>'PERSO_NAME'},
'110'=>{authtypecode=>'CORPO_NAME'},
diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl
index 70d5eb7..6efd1e0 100755
--- a/misc/migration_tools/bulkmarcimport.pl
+++ b/misc/migration_tools/bulkmarcimport.pl
@@ -24,16 +24,19 @@ use C4::Koha;
use C4::Debug;
use C4::Charset;
use C4::Items;
+use YAML;
use Unicode::Normalize;
use Time::HiRes qw(gettimeofday);
use Getopt::Long;
use IO::File;
use Pod::Usage;
-binmode STDOUT, ':encoding(UTF-8)';
+use open qw( :std :utf8 );
+binmode( STDOUT, ":utf8" );
my ( $input_marc_file, $number, $offset) = ('',0,0);
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
my ($sourcetag,$sourcesubfield,$idmapfl);
+my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
my $cleanisbn = 1;
$|=1;
@@ -45,7 +48,7 @@ GetOptions(
'o|offset:f' => \$offset,
'h' => \$version,
'd' => \$delete,
- 't' => \$test_parameter,
+ 't|test' => \$test_parameter,
's' => \$skip_marc8_conversion,
'c:s' => \$char_encoding,
'v:s' => \$verbose,
@@ -55,20 +58,34 @@ GetOptions(
'k|keepids:s' => \$keepids,
'b|biblios' => \$biblios,
'a|authorities' => \$authorities,
+ 'authtypes:s' => \$authtypes,
+ 'filter=s@' => \$filters,
+ 'insert' => \$insert,
+ 'update' => \$update,
+ 'all' => \$all,
'match=s@' => \$match,
'i|isbn' => \$isbn_check,
'x:s' => \$sourcetag,
'y:s' => \$sourcesubfield,
'idmap:s' => \$idmapfl,
'cleanisbn!' => \$cleanisbn,
+ 'yaml:s' => \$yamlfile,
);
-$biblios=!$authorities||$biblios;
+$biblios ||= !$authorities;
+$insert ||= !$update;
+
+if ($all) {
+ $insert = 1;
+ $update = 1;
+}
if ($version || ($input_marc_file eq '')) {
pod2usage( -verbose => 2 );
exit;
}
+my $heading_fields=get_heading_fields();
+
if (defined $idmapfl) {
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
}
@@ -139,7 +156,7 @@ $batch->warnings_off();
$batch->strict_off();
my $i=0;
my $commitnum = $commit ? $commit : 50;
-
+my $yamlhash;
# Skip file offset
if ( $offset ) {
@@ -196,6 +213,7 @@ RECORD: while ( ) {
next RECORD;
}
}
+ SetUTF8Flag($record);
my $isbn;
# remove trailing - in isbn (only for biblios, of course)
if ($biblios && $cleanisbn) {
@@ -209,45 +227,76 @@ RECORD: while ( ) {
}
my $id;
# search for duplicates (based on Local-number)
- if ($match){
- require C4::Search;
- my $query=build_query($match,$record);
- my $server=($authorities?'authorityserver':'biblioserver');
- my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
- die "unable to search the database for duplicates : $error" if (defined $error);
- #warn "$query $server : $totalhits";
- if ( @{$results} == 1 ){
- my $marcrecord = MARC::File::USMARC::decode($results->[0]);
- $id=GetRecordId($marcrecord,$tagid,$subfieldid);
- }
- elsif ( @{$results} > 1){
- $debug && warn "more than one match for $query";
- }
- else {
- $debug && warn "nomatch for $query";
- }
+ my $originalid;
+ $originalid = GetRecordId( $record, $tagid, $subfieldid );
+ if ($match) {
+ require C4::Search;
+ my $query = build_query( $match, $record );
+ my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
+ $debug && warn $query;
+ my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
+ die "unable to search the database for duplicates : $error" if ( defined $error );
+ $debug && warn "$query $server : $totalhits";
+ if ( $results && scalar(@$results) == 1 ) {
+ my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
+ SetUTF8Flag($marcrecord);
+ $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
+ if ( $authorities && $marcFlavour ) {
+ #Skip if authority in database is the same as the on in database
+ if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
+ if ($yamlfile) {
+ $yamlhash->{$originalid}->{'authid'} = $id;
+
+ # On récupère tous les souschamps des champs vedettes d'autorités
+ my @subfields;
+ foreach my $field ( $marcrecord->field("2..") ) {
+ push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
+ }
+ $yamlhash->{$originalid}->{'subfields'} = \@subfields;
+ }
+ next;
+ }
+ }
+ } elsif ( $results && scalar(@$results) > 1 ) {
+ $debug && warn "more than one match for $query";
+ } else {
+ $debug && warn "nomatch for $query";
+ }
}
- my $originalid;
- if ($keepids){
- $originalid=GetRecordId($record,$tagid,$subfieldid);
- if ($originalid){
- my $storeidfield;
- if (length($keepids)==3){
- $storeidfield=MARC::Field->new($keepids,$originalid);
- }
- else {
- $storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
- }
- $record->insert_fields_ordered($storeidfield);
- $record->delete_field($record->field($tagid));
- }
+ if ($keepids && $originalid) {
+ my $storeidfield;
+ if ( length($keepids) == 3 ) {
+ $storeidfield = MARC::Field->new( $keepids, $originalid );
+ } else {
+ $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
+ }
+ $record->insert_fields_ordered($storeidfield);
+ $record->delete_field( $record->field($tagid) );
+ }
+ foreach my $stringfilter (@$filters) {
+ if ( length($stringfilter) == 3 ) {
+ foreach my $field ( $record->field($stringfilter) ) {
+ $record->delete_field($field);
+ $debug && warn "removed : ", $field->as_string;
+ }
+ } else {
+ my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 )
+ if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/;
+ if ( ( $removetag > "010" ) && $removesubfield ) {
+ foreach my $field ( $record->field($removetag) ) {
+ $field->delete_subfield( code => "$removesubfield", match => $removematch );
+ $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
+ }
+ }
+ }
}
+
unless ($test_parameter) {
if ($authorities){
use C4::AuthoritiesMarc;
- my $authtypecode=GuessAuthTypeCode($record);
+ my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
my $authid= ($id?$id:GuessAuthId($record));
- if ($authid && GetAuthority($authid)){
+ if ($authid && GetAuthority($authid) && $update ){
## Authority has an id and is in database : Replace
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
if ($@){
@@ -280,6 +329,14 @@ RECORD: while ( ) {
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
}
}
+ if ($yamlfile) {
+ $yamlhash->{$originalid}->{'authid'} = $authid;
+ my @subfields;
+ foreach my $field ( $record->field("2..") ) {
+ push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
+ }
+ $yamlhash->{$originalid}->{'subfields'} = \@subfields;
+ }
}
else {
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
@@ -303,20 +360,33 @@ RECORD: while ( ) {
}
# create biblio, unless we already have it ( either match or isbn )
if ($biblionumber) {
- eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
- }
- else
- {
- eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) };
+ eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
+ if ($update) {
+ eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
+ if ($@) {
+ warn "ERROR: Edit biblio $biblionumber failed: $@\n";
+ printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
+ next RECORD;
+ } else {
+ printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
+ }
+ } else {
+ printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
+ }
+ } else {
+ if ($insert) {
+ eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
+ if ($@) {
+ warn "ERROR: Adding biblio $biblionumber failed: $@\n";
+ printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
+ next RECORD;
+ } else {
+ printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
+ }
+ } else {
+ printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
+ }
}
- if ( $@ ) {
- warn "ERROR: Adding biblio $biblionumber failed: $@\n";
- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
- next RECORD;
- }
- else{
- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
- }
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
if ( $@ ) {
warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
@@ -335,6 +405,7 @@ RECORD: while ( ) {
if ($#{ $errors_ref } > -1) {
report_item_errors($biblionumber, $errors_ref);
}
+ $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
}
$dbh->commit() if (0 == $i % $commitnum);
}
@@ -358,6 +429,10 @@ if ($logfile){
print $loghandle "$i MARC records done in $timeneeded seconds\n";
$loghandle->close;
}
+if ($yamlfile) {
+ open YAML, "> $yamlfile" or die "cannot open $yamlfile \n";
+ print YAML Dump($yamlhash);
+}
exit 0;
sub GetRecordId{
@@ -412,9 +487,22 @@ sub report_item_errors {
}
sub printlog{
my $logelements=shift;
- print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
+ print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
}
+sub get_heading_fields{
+ my $headingfields;
+ if ($authtypes){
+ $headingfields=YAML::LoadFile($authtypes);
+ $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
+ $debug && warn YAML::Dump($headingfields);
+ }
+ unless ($headingfields){
+ $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
+ $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
+ }
+ return $headingfields;
+}
=head1 NAME
@@ -476,7 +564,7 @@ The I<NUMBER> of records to wait before performing a 'commit' operation
File logs actions done for each record and their status into file
-=item B<-t>
+=item B<-t, -test>
Test mode: parses the file, saying what he would do, but doing nothing.
@@ -499,6 +587,30 @@ biblioitems, items
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
+=item B<-authtypes>
+
+file yamlfile with authoritiesTypes and distinguishable record field in order
+to store the correct authtype
+
+=item B<-yaml>
+
+yaml file format a yaml file with ids
+keepids field store ids in field (usefull for authorities,
+where 001 contains the authid for Koha, that can contain a very valuable
+info for authorities coming from LOC or BNF. useless for biblios probably
+
+=item B<-insert>
+
+if set, only insert when possible
+
+=item B<-update>
+
+if set, only updates (any biblio should have a matching record)
+
+=item B<-all>
+
+if set, do whatever is required
+
=item B<-k, -keepids>=<FIELD>
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
--
1.7.0.4
More information about the Koha-patches
mailing list