[Koha-patches] [PATCH] Bug 7475: Enable authority overlay
Jared Camins-Esakov
jcamins at cpbibliography.com
Thu Jun 14 00:24:30 CEST 2012
This patch does the following:
* Completes authority support within Koha's import batch code
* Completes authority support in Koha's matching rule code
* Exposes the ability to stage and commit batches of authorities using
the renamed scripts misc/stage_file.pl and misc/commit_file.pl
To test:
1. Find a file of authorities (a sample file with MARC21 authorities
is attached to bug 7475) and download it to your server
2. Stage the file using the following command (replace <filename> with
the name of the file you saved in step 1):
> misc/stage_file.pl --file <filename> --authorities
3. Note the batch number the script assigns to your batch
4. Commit the records using the following command (replace <batchnumber>
with the batch number you made note of in step 3):
> misc/commit_file.pl --batch-number <batchnumber>
5. Index the authorities Zebraqueue (or wait)
6. Confirm that the new authorities appear.
7. Revert the import with the following command:
> misc/commit_file.pl --batch-number <batchnumber> --revert
8. Index the authorities Zebraqueue (or wait)
9. Confirm that the records have been removed
---
C4/ImportBatch.pm | 304 +++++++++++++-------
C4/Matcher.pm | 73 ++++-
admin/matching-rules.pl | 12 +-
.../data/mysql/atomicupdate/importauthorities.pl | 19 ++
installer/data/mysql/kohastructure.sql | 15 +
.../prog/en/modules/admin/matching-rules.tt | 11 +
misc/commit_biblios_file.pl | 119 --------
misc/commit_file.pl | 151 ++++++++++
misc/cronjobs/import_webservice_batch.pl | 2 +-
misc/stage_biblios_file.pl | 185 ------------
misc/stage_file.pl | 197 +++++++++++++
svc/import_bib | 2 +-
tools/manage-marc-import.pl | 4 +-
13 files changed, 667 insertions(+), 427 deletions(-)
create mode 100755 installer/data/mysql/atomicupdate/importauthorities.pl
delete mode 100755 misc/commit_biblios_file.pl
create mode 100755 misc/commit_file.pl
delete mode 100755 misc/stage_biblios_file.pl
create mode 100755 misc/stage_file.pl
diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm
index 5bf79bf..a62f15d 100644
--- a/C4/ImportBatch.pm
+++ b/C4/ImportBatch.pm
@@ -1,6 +1,6 @@
package C4::ImportBatch;
-# Copyright (C) 2007 LibLime
+# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
#
# This file is part of Koha.
#
@@ -40,14 +40,16 @@ BEGIN {
GetImportRecordMarcXML
AddImportBatch
GetImportBatch
+ AddAuthorityToBatch
AddBiblioToBatch
AddItemsToImportBiblio
+ ModAuthorityInBatch
ModBiblioInBatch
BatchStageMarcRecords
- BatchFindBibDuplicates
- BatchCommitBibRecords
- BatchRevertBibRecords
+ BatchFindDuplicates
+ BatchCommitRecords
+ BatchRevertRecords
CleanBatch
GetAllImportBatches
@@ -272,10 +274,43 @@ sub ModBiblioInBatch {
}
+=head2 AddAuthToBatch
+
+ my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
+ $marc_record, $encoding, $z3950random, $update_counts);
+
+=cut
+
+sub AddAuthToBatch {
+ my $batch_id = shift;
+ my $record_sequence = shift;
+ my $marc_record = shift;
+ my $encoding = shift;
+ my $z3950random = shift;
+ my $update_counts = @_ ? shift : 1;
+
+ my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random);
+ _update_batch_record_counts($batch_id) if $update_counts;
+ return $import_record_id;
+}
+
+=head2 ModAuthInBatch
+
+ ModAuthInBatch($import_record_id, $marc_record);
+
+=cut
+
+sub ModAuthInBatch {
+ my ($import_record_id, $marc_record) = @_;
+
+ _update_import_record_marc($import_record_id, $marc_record);
+
+}
+
=head2 BatchStageMarcRecords
($batch_id, $num_records, $num_items, @invalid_records) =
- BatchStageMarcRecords($encoding, $marc_records, $file_name,
+ BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name,
$comments, $branch_code, $parse_items,
$leave_as_staging,
$progress_interval, $progress_callback);
@@ -283,6 +318,7 @@ sub ModBiblioInBatch {
=cut
sub BatchStageMarcRecords {
+ my $record_type = shift;
my $encoding = shift;
my $marc_records = shift;
my $file_name = shift;
@@ -338,10 +374,14 @@ sub BatchStageMarcRecords {
push @invalid_records, $marc_blob;
} else {
$num_valid++;
- $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
- if ($parse_items) {
- my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
- $num_items += scalar(@import_items_ids);
+ if ($record_type eq 'biblio') {
+ $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
+ if ($parse_items) {
+ my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
+ $num_items += scalar(@import_items_ids);
+ }
+ } elsif ($record_type eq 'auth') {
+ $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
}
}
}
@@ -392,9 +432,9 @@ sub AddItemsToImportBiblio {
return @import_items_ids;
}
-=head2 BatchFindBibDuplicates
+=head2 BatchFindDuplicates
- my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher,
+ my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
$max_matches, $progress_interval, $progress_callback);
Goes through the records loaded in the batch and attempts to
@@ -412,7 +452,7 @@ singular argument.
=cut
-sub BatchFindBibDuplicates {
+sub BatchFindDuplicates {
my $batch_id = shift;
my $matcher = shift;
my $max_matches = @_ ? shift : 10;
@@ -430,9 +470,10 @@ sub BatchFindBibDuplicates {
my $dbh = C4::Context->dbh;
- my $sth = $dbh->prepare("SELECT import_record_id, marc
+ my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, marc
FROM import_records
- JOIN import_biblios USING (import_record_id)
+ LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
+ LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
WHERE import_batch_id = ?");
$sth->execute($batch_id);
my $num_with_matches = 0;
@@ -460,15 +501,15 @@ sub BatchFindBibDuplicates {
return $num_with_matches;
}
-=head2 BatchCommitBibRecords
+=head2 BatchCommitRecords
- my ($num_added, $num_updated, $num_items_added, $num_items_errored,
- $num_ignored) = BatchCommitBibRecords($batch_id, $framework,
- $progress_interval, $progress_callback);
+ my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
+ BatchCommitRecords($batch_id, $framework,
+ $progress_interval, $progress_callback);
=cut
-sub BatchCommitBibRecords {
+sub BatchCommitRecords {
my $batch_id = shift;
my $framework = shift;
@@ -483,25 +524,29 @@ sub BatchCommitBibRecords {
$progress_interval = 0 unless 'CODE' eq ref $progress_callback;
}
+ my $record_type;
my $num_added = 0;
my $num_updated = 0;
my $num_items_added = 0;
my $num_items_errored = 0;
my $num_ignored = 0;
# commit (i.e., save, all records in the batch)
- # FIXME biblio only at the moment
SetImportBatchStatus('importing');
my $overlay_action = GetImportBatchOverlayAction($batch_id);
my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
my $item_action = GetImportBatchItemAction($batch_id);
+ my $item_tag;
+ my $item_subfield;
my $dbh = C4::Context->dbh;
- my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding
+ my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
FROM import_records
- JOIN import_biblios USING (import_record_id)
+ LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
+ LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
WHERE import_batch_id = ?");
$sth->execute($batch_id);
my $rec_num = 0;
while (my $rowref = $sth->fetchrow_hashref) {
+ $record_type = $rowref->{'record_type'};
$rec_num++;
if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
&$progress_callback($rec_num);
@@ -513,67 +558,87 @@ sub BatchCommitBibRecords {
my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
- # remove any item tags - rely on BatchCommitItems
- my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
- foreach my $item_field ($marc_record->field($item_tag)) {
- $marc_record->delete_field($item_field);
+ if ($record_type eq 'biblio') {
+ # remove any item tags - rely on BatchCommitItems
+ ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
+ foreach my $item_field ($marc_record->field($item_tag)) {
+ $marc_record->delete_field($item_field);
+ }
}
- # decide what what to do with the bib and item records
- my ($bib_result, $item_result, $bib_match) =
+ my ($record_result, $item_result, $record_match) =
_get_commit_action($overlay_action, $nomatch_action, $item_action,
- $rowref->{'overlay_status'}, $rowref->{'import_record_id'});
+ $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
- if ($bib_result eq 'create_new') {
+ my $recordid;
+ my $query;
+ if ($record_result eq 'create_new') {
$num_added++;
- my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, $framework);
- my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
- $sth->execute($biblionumber, $rowref->{'import_record_id'});
- $sth->finish();
- if ($item_result eq 'create_new') {
- my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
- $num_items_added += $bib_items_added;
- $num_items_errored += $bib_items_errored;
+ if ($record_type eq 'biblio') {
+ my $biblioitemnumber;
+ ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework);
+ $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
+ if ($item_result eq 'create_new') {
+ my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
+ $num_items_added += $bib_items_added;
+ $num_items_errored += $bib_items_errored;
+ }
+ } else {
+ my $authid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
+ $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
}
+ my $sth = $dbh->prepare_cached($query);
+ $sth->execute($recordid, $rowref->{'import_record_id'});
+ $sth->finish();
SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
- } elsif ($bib_result eq 'replace') {
+ } elsif ($record_result eq 'replace') {
$num_updated++;
- my $biblionumber = $bib_match;
- my ($count, $oldbiblio) = GetBiblio($biblionumber);
- my $oldxml = GetXmlBiblio($biblionumber);
-
- # remove item fields so that they don't get
- # added again if record is reverted
- my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'});
- foreach my $item_field ($old_marc->field($item_tag)) {
- $old_marc->delete_field($item_field);
- }
+ $recordid = $record_match;
+ my $oldxml;
+ if ($record_type eq 'biblio') {
+ my ($count, $oldbiblio) = GetBiblio($recordid);
+ $oldxml = GetXmlBiblio($recordid);
+
+ # remove item fields so that they don't get
+ # added again if record is reverted
+ my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'});
+ foreach my $item_field ($old_marc->field($item_tag)) {
+ $old_marc->delete_field($item_field);
+ }
+ $oldxml = $old_marc->as_xml();
+
+ ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'});
+ $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
+
+ if ($item_result eq 'create_new') {
+ my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
+ $num_items_added += $bib_items_added;
+ $num_items_errored += $bib_items_errored;
+ }
+ } else {
+ my $oldxml = GetAuthorityXML($recordid);
- ModBiblio($marc_record, $biblionumber, $oldbiblio->{'frameworkcode'});
+ ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
+ $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
+ }
my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
- $sth->execute($old_marc->as_xml(), $rowref->{'import_record_id'});
+ $sth->execute($oldxml, $rowref->{'import_record_id'});
$sth->finish();
- my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
- $sth2->execute($biblionumber, $rowref->{'import_record_id'});
+ my $sth2 = $dbh->prepare_cached($query);
+ $sth2->execute($recordid, $rowref->{'import_record_id'});
$sth2->finish();
- if ($item_result eq 'create_new') {
- my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
- $num_items_added += $bib_items_added;
- $num_items_errored += $bib_items_errored;
- }
SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
- } elsif ($bib_result eq 'ignore') {
+ } elsif ($record_result eq 'ignore') {
$num_ignored++;
- my $biblionumber = $bib_match;
- if (defined $biblionumber and $item_result eq 'create_new') {
- my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
+ if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') {
+ my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
$num_items_added += $bib_items_added;
$num_items_errored += $bib_items_errored;
# still need to record the matched biblionumber so that the
# items can be reverted
my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
- $sth2->execute($biblionumber, $rowref->{'import_record_id'});
+ $sth2->execute($recordid, $rowref->{'import_record_id'});
SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
}
SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
@@ -632,62 +697,82 @@ sub BatchCommitItems {
return ($num_items_added, $num_items_errored);
}
-=head2 BatchRevertBibRecords
+=head2 BatchRevertRecords
my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
- $num_ignored) = BatchRevertBibRecords($batch_id);
+ $num_ignored) = BatchRevertRecords($batch_id);
=cut
-sub BatchRevertBibRecords {
+sub BatchRevertRecords {
my $batch_id = shift;
+ my $record_type;
my $num_deleted = 0;
my $num_errors = 0;
my $num_reverted = 0;
- my $num_items_deleted = 0;
my $num_ignored = 0;
+ my $num_items_deleted = 0;
# commit (i.e., save, all records in the batch)
- # FIXME biblio only at the moment
SetImportBatchStatus('reverting');
my $overlay_action = GetImportBatchOverlayAction($batch_id);
my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
my $dbh = C4::Context->dbh;
- my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber
+ my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
FROM import_records
- JOIN import_biblios USING (import_record_id)
+ LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
+ LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
WHERE import_batch_id = ?");
$sth->execute($batch_id);
while (my $rowref = $sth->fetchrow_hashref) {
+ $record_type = $rowref->{'record_type'};
if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
$num_ignored++;
next;
}
- my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
+ my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
- if ($bib_result eq 'delete') {
- $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
- my $error = DelBiblio($rowref->{'matched_biblionumber'});
+ if ($record_result eq 'delete') {
+ my $error = undef;
+ if ($record_type eq 'biblio') {
+ $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
+ $error = DelBiblio($rowref->{'matched_biblionumber'});
+ } else {
+ my $deletedauthid = DelAuthority($rowref->{'matched_authid'});
+ }
if (defined $error) {
$num_errors++;
} else {
$num_deleted++;
SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
}
- } elsif ($bib_result eq 'restore') {
+ } elsif ($record_result eq 'restore') {
$num_reverted++;
my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'});
- my $biblionumber = $rowref->{'matched_biblionumber'};
- my ($count, $oldbiblio) = GetBiblio($biblionumber);
- $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
- ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
+ if ($record_type eq 'biblio') {
+ my $biblionumber = $rowref->{'matched_biblionumber'};
+ my ($count, $oldbiblio) = GetBiblio($biblionumber);
+ $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
+ ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
+ } else {
+ my $authid = $rowref->{'matched_authid'};
+ ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
+ }
SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
- } elsif ($bib_result eq 'ignore') {
- $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
+ } elsif ($record_result eq 'ignore') {
+ if ($record_type eq 'biblio') {
+ $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
+ }
SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
}
- my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?");
+ my $query;
+ if ($record_type eq 'biblio') {
+ $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?";
+ } else {
+ $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
+ }
+ my $sth2 = $dbh->prepare_cached($query);
$sth2->execute($rowref->{'import_record_id'});
}
@@ -1305,26 +1390,45 @@ sub _update_batch_record_counts {
}
sub _get_commit_action {
- my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_;
+ my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
- my ($bib_result, $bib_match, $item_result);
-
- if ($overlay_status ne 'no_match') {
- $bib_match = GetBestRecordMatch($import_record_id);
- if ($overlay_action eq 'replace') {
- $bib_result = defined($bib_match) ? 'replace' : 'create_new';
- } elsif ($overlay_action eq 'create_new') {
- $bib_result = 'create_new';
- } elsif ($overlay_action eq 'ignore') {
- $bib_result = 'ignore';
- }
- $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
- } else {
- $bib_result = $nomatch_action;
- $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
- }
+ if ($record_type eq 'biblio') {
+ my ($bib_result, $bib_match, $item_result);
+
+ if ($overlay_status ne 'no_match') {
+ $bib_match = GetBestRecordMatch($import_record_id);
+ if ($overlay_action eq 'replace') {
+ $bib_result = defined($bib_match) ? 'replace' : 'create_new';
+ } elsif ($overlay_action eq 'create_new') {
+ $bib_result = 'create_new';
+ } elsif ($overlay_action eq 'ignore') {
+ $bib_result = 'ignore';
+ }
+ $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
+ } else {
+ $bib_result = $nomatch_action;
+ $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
+ }
+ return ($bib_result, $item_result, $bib_match);
+ } else { # must be auths
+ my ($auth_result, $auth_match);
+
+ if ($overlay_status ne 'no_match') {
+ $auth_match = GetBestRecordMatch($import_record_id);
+ if ($overlay_action eq 'replace') {
+ $auth_result = defined($auth_match) ? 'replace' : 'create_new';
+ } elsif ($overlay_action eq 'create_new') {
+ $auth_result = 'create_new';
+ } elsif ($overlay_action eq 'ignore') {
+ $auth_result = 'ignore';
+ }
+ } else {
+ $auth_result = $nomatch_action;
+ }
+
+ return ($auth_result, undef, $auth_match);
- return ($bib_result, $item_result, $bib_match);
+ }
}
sub _get_revert_action {
diff --git a/C4/Matcher.pm b/C4/Matcher.pm
index 59b5876..c9f418d 100644
--- a/C4/Matcher.pm
+++ b/C4/Matcher.pm
@@ -1,6 +1,6 @@
package C4::Matcher;
-# Copyright (C) 2007 LibLime
+# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
#
# This file is part of Koha.
#
@@ -22,8 +22,6 @@ use warnings;
use C4::Context;
use MARC::Record;
-use C4::Search;
-use C4::Biblio;
use vars qw($VERSION);
@@ -384,6 +382,20 @@ sub delete {
$sth->execute($matcher_id); # relying on cascading deletes to clean up everything
}
+=head2 record_type
+
+ $matcher->record_type('biblio');
+ my $record_type = $matcher->record_type();
+
+Accessor method.
+
+=cut
+
+sub record_type {
+ my $self = shift;
+ @_ ? $self->{'record_type'} = shift : $self->{'record_type'};
+}
+
=head2 threshold
$matcher->threshold(1000);
@@ -582,7 +594,7 @@ sub add_simple_required_check {
);
}
-=head2 find_matches
+=head2 get_matches
my @matches = $matcher->get_matches($marc_record, $max_matches);
foreach $match (@matches) {
@@ -618,9 +630,34 @@ sub get_matches {
my @source_keys = _get_match_keys($source_record, $matchpoint);
next if scalar(@source_keys) == 0;
# build query
- my $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys);
- # FIXME only searching biblio index at the moment
- my ($error, $searchresults, $total_hits) = SimpleSearch($query, 0, $max_matches);
+ my $query;
+ my $error;
+ my $searchresults;
+ my $total_hits;
+ if ($self->{'record_type'} eq 'biblio') {
+ $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys);
+# FIXME only searching biblio index at the moment
+ require C4::Search;
+ ($error, $searchresults, $total_hits) = C4::Search::SimpleSearch($query, 0, $max_matches);
+ } elsif ($self->{'record_type'} eq 'authority') {
+ my $authresults;
+ my @marclist;
+ my @and_or;
+ my @excluding = [];
+ my @operator;
+ my @value;
+ foreach my $key (@source_keys) {
+ push @marclist, $matchpoint->{'index'};
+ push @and_or, 'or';
+ push @operator, 'exact';
+ push @value, $key;
+ }
+ require C4::AuthoritiesMarc;
+ ($searchresults, $total_hits) = C4::AuthoritiesMarc::SearchAuthorities(
+ \@marclist, \@and_or, \@excluding, \@operator,
+ \@value, 0, 20, undef, 'AuthidAsc', 1
+ );
+ }
if (defined $error ) {
warn "search failed ($query) $error";
@@ -639,13 +676,20 @@ sub get_matches {
keys %matches;
my @results = ();
- foreach my $marcblob (keys %matches) {
- my $target_record = MARC::Record->new_from_usmarc($marcblob);
- my $result = TransformMarcToKoha(C4::Context->dbh, $target_record, '');
- # FIXME - again, bibliospecific
- # also, can search engine be induced to give just the number in the first place?
- my $record_number = $result->{'biblionumber'};
- push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
+ if ($self->{'record_type'} eq 'biblio') {
+ require C4::Biblio;
+ foreach my $marcblob (keys %matches) {
+ my $target_record = MARC::Record->new_from_usmarc($marcblob);
+ my $record_number;
+ my $result = C4::Biblio::TransformMarcToKoha(C4::Context->dbh, $target_record, '');
+ $record_number = $result->{'biblionumber'};
+ push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
+ }
+ } elsif ($self->{'record_type'} eq 'authority') {
+ require C4::AuthoritiesMarc;
+ foreach my $authid (keys %matches) {
+ push @results, { 'record_id' => $authid, 'score' => $matches{$authid} };
+ }
}
@results = sort { $b->{'score'} cmp $a->{'score'} } @results;
if (scalar(@results) > $max_matches) {
@@ -673,6 +717,7 @@ sub dump {
$result->{'matcher_id'} = $self->{'id'};
$result->{'code'} = $self->{'code'};
$result->{'description'} = $self->{'description'};
+ $result->{'record_type'} = $self->{'record_type'};
$result->{'matchpoints'} = [];
foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
diff --git a/admin/matching-rules.pl b/admin/matching-rules.pl
index b167026..e682fb5 100755
--- a/admin/matching-rules.pl
+++ b/admin/matching-rules.pl
@@ -92,9 +92,10 @@ sub add_matching_rule_form {
sub add_update_matching_rule {
my $template = shift;
my $matcher_id = shift;
+ my $record_type = $input->param('record_type') || 'biblio';
# do parsing
- my $matcher = C4::Matcher->new('biblio', 1000); # FIXME biblio only for now
+ my $matcher = C4::Matcher->new($record_type, 1000);
$matcher->code($input->param('code'));
$matcher->description($input->param('description'));
$matcher->threshold($input->param('threshold'));
@@ -203,10 +204,11 @@ sub edit_matching_rule_form {
my $matcher = C4::Matcher->fetch($matcher_id);
- $template->param(matcher_id => $matcher_id);
- $template->param(code => $matcher->code());
- $template->param(description => $matcher->description());
- $template->param(threshold => $matcher->threshold());
+ $template->{VARS}->{'matcher_id'} = $matcher_id;
+ $template->{VARS}->{'code'} = $matcher->code();
+ $template->{VARS}->{'description'} = $matcher->description();
+ $template->{VARS}->{'threshold'} = $matcher->threshold();
+ $template->{VARS}->{'record_type'} = $matcher->record_type();
my $matcher_info = $matcher->dump();
my @matchpoints = ();
diff --git a/installer/data/mysql/atomicupdate/importauthorities.pl b/installer/data/mysql/atomicupdate/importauthorities.pl
new file mode 100755
index 0000000..f242f61
--- /dev/null
+++ b/installer/data/mysql/atomicupdate/importauthorities.pl
@@ -0,0 +1,19 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use C4::Context;
+my $dbh = C4::Context->dbh;
+
+$dbh->do(
+q|CREATE TABLE `import_auths` (
+ `import_record_id` int(11) NOT NULL,
+ `matched_authid` int(11) default NULL,
+ `control_number` varchar(25) default NULL,
+ `original_source` varchar(25) default NULL,
+ CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`)
+ REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ KEY `matched_authid` (`matched_authid`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;|
+);
+print "Upgrade done (Added support for staging authorities)\n";
diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql
index 54498c3..13864ce 100644
--- a/installer/data/mysql/kohastructure.sql
+++ b/installer/data/mysql/kohastructure.sql
@@ -909,6 +909,21 @@ CREATE TABLE `import_record_matches` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--
+-- Table structure for table `import_auths`
+--
+
+DROP TABLE IF EXISTS `import_auths`;
+CREATE TABLE `import_auths` (
+ `import_record_id` int(11) NOT NULL,
+ `matched_authid` int(11) default NULL,
+ `control_number` varchar(25) default NULL,
+ `original_source` varchar(25) default NULL,
+ CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`)
+ REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ KEY `matched_authid` (`matched_authid`),
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+--
-- Table structure for table `import_biblios`
--
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt
index 5835244..2bd8aaf 100644
--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt
+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt
@@ -220,6 +220,17 @@ function CheckRuleForm(f) {
<input type="text" id="threshold" name="threshold" size="5" maxlength="5"
value="[% threshold |html %]" /> <span class="required">Required</span>
</li>
+ <li><label for="record_type" class="required">Record type: </label>
+ <select id="record_type" name="record_type">
+ [% IF ( record_type == "authority" ) %]
+ <option value="biblio">Bibliographic record</option>
+ <option value="authority" selected="selected">Authority record</option>
+ [% ELSE %]
+ <option value="biblio" selected="selected">Bibliographic record</option>
+ <option value="authority">Authority record</option>
+ [% END %]
+ </select><span class="required">Required</span>
+ </li>
</ol>
</fieldset>
diff --git a/misc/commit_biblios_file.pl b/misc/commit_biblios_file.pl
deleted file mode 100755
index b4be670..0000000
--- a/misc/commit_biblios_file.pl
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-#use warnings; FIXME - Bug 2505
-BEGIN {
- # find Koha's Perl modules
- # test carefully before changing this
- use FindBin;
- eval { require "$FindBin::Bin/kohalib.pl" };
-}
-
-use C4::Context;
-use C4::ImportBatch;
-use Getopt::Long;
-
-$| = 1;
-
-# command-line parameters
-my $batch_number = "";
-my $list_batches = 0;
-my $want_help = 0;
-
-my $result = GetOptions(
- 'batch-number:s' => \$batch_number,
- 'list-batches' => \$list_batches,
- 'h|help' => \$want_help
-);
-
-if ($want_help or (not $batch_number and not $list_batches)) {
- print_usage();
- exit 0;
-}
-
-if ($list_batches) {
- list_batches();
- exit 0;
-}
-
-# FIXME dummy user so that logging won't fail
-# in future, probably should tie to a real user account
-C4::Context->set_userenv(0, 'batch', 0, 'batch', 'batch', 'batch', 'batch', 'batch');
-
-my $dbh = C4::Context->dbh;
-$dbh->{AutoCommit} = 0;
-if ($batch_number =~ /^\d+$/ and $batch_number > 0) {
- my $batch = GetImportBatch($batch_number);
- die "$0: import batch $batch_number does not exist in database\n" unless defined $batch;
- die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
- unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted";
- process_batch($batch_number);
- $dbh->commit();
-} else {
- die "$0: please specify a numeric batch ID\n";
-}
-
-exit 0;
-
-sub list_batches {
- my $results = GetAllImportBatches();
- print sprintf("%5.5s %-25.25s %-25.25s %-10.10s\n", "#", "File name", "Batch comments", "Status");
- print '-' x 5, ' ' , '-' x 25, ' ', '-' x 25, ' ', '-' x 10, "\n" ;
- foreach my $batch (@{ $results}) {
- if ($batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted") {
- print sprintf("%5.5s %-25.25s %-25.25s %-10.10s\n",
- $batch->{'import_batch_id'},
- $batch->{'file_name'},
- $batch->{'comments'},
- $batch->{'import_status'});
- }
- }
-}
-
-sub process_batch {
- my ($import_batch_id) = @_;
-
- print "... importing MARC records -- please wait\n";
- my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
- BatchCommitBibRecords($import_batch_id, '', 100, \&print_progress_and_commit);
- print "... finished importing MARC records\n";
-
- print <<_SUMMARY_;
-
-MARC record import report
-----------------------------------------
-Batch number: $import_batch_id
-Number of new bibs added: $num_added
-Number of bibs replaced: $num_updated
-Number of bibs ignored: $num_ignored
-Number of items added: $num_items_added
-Number of items ignored: $num_items_errored
-
-Note: an item is ignored if its barcode is a
-duplicate of one already in the database.
-_SUMMARY_
-}
-
-sub print_progress_and_commit {
- my $recs = shift;
- print "... processed $recs records\n";
- $dbh->commit();
-}
-
-sub print_usage {
- print <<_USAGE_;
-$0: import a batch of staged MARC records into database.
-
-Use this batch job to complete the import of a batch of
-MARC records that was staged either by the batch job
-stage_biblios_file.pl or by the Koha Tools option
-"Stage MARC Records for Import".
-
-Parameters:
- --batch-number <#> number of the record batch
- to import
- --list-batches print a list of record batches
- available to commit
- --help or -h show this message.
-_USAGE_
-}
diff --git a/misc/commit_file.pl b/misc/commit_file.pl
new file mode 100755
index 0000000..1f54faf
--- /dev/null
+++ b/misc/commit_file.pl
@@ -0,0 +1,151 @@
+#!/usr/bin/perl
+
+use strict;
+#use warnings; FIXME - Bug 2505
+BEGIN {
+ # find Koha's Perl modules
+ # test carefully before changing this
+ use FindBin;
+ eval { require "$FindBin::Bin/kohalib.pl" };
+}
+
+use C4::Context;
+use C4::ImportBatch;
+use Getopt::Long;
+
+$| = 1;
+
+# command-line parameters
+my $batch_number = "";
+my $list_batches = 0;
+my $revert = 0;
+my $want_help = 0;
+
+my $result = GetOptions(
+ 'batch-number:s' => \$batch_number,
+ 'list-batches' => \$list_batches,
+ 'revert' => \$revert,
+ 'h|help' => \$want_help
+);
+
+if ($want_help or (not $batch_number and not $list_batches)) {
+ print_usage();
+ exit 0;
+}
+
+if ($list_batches) {
+ list_batches();
+ exit 0;
+}
+
+# FIXME dummy user so that logging won't fail
+# in future, probably should tie to a real user account
+C4::Context->set_userenv(0, 'batch', 0, 'batch', 'batch', 'batch', 'batch', 'batch');
+
+my $dbh = C4::Context->dbh;
+$dbh->{AutoCommit} = 0;
+if ($batch_number =~ /^\d+$/ and $batch_number > 0) {
+ my $batch = GetImportBatch($batch_number);
+ die "$0: import batch $batch_number does not exist in database\n" unless defined $batch;
+ if ($revert) {
+ die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
+ unless $batch->{'import_status'} eq "imported";
+ revert_batch($batch_number);
+ } else {
+ die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
+ unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted";
+ process_batch($batch_number);
+ }
+ $dbh->commit();
+} else {
+ die "$0: please specify a numeric batch ID\n";
+}
+
+exit 0;
+
+sub list_batches {
+ my $results = GetAllImportBatches();
+ print sprintf("%5.5s %-25.25s %-25.25s %-10.10s\n", "#", "File name", "Batch comments", "Status");
+ print '-' x 5, ' ' , '-' x 25, ' ', '-' x 25, ' ', '-' x 10, "\n" ;
+ foreach my $batch (@{ $results}) {
+ if ($batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted") {
+ print sprintf("%5.5s %-25.25s %-25.25s %-10.10s\n",
+ $batch->{'import_batch_id'},
+ $batch->{'file_name'},
+ $batch->{'comments'},
+ $batch->{'import_status'});
+ }
+ }
+}
+
+sub process_batch {
+ my ($import_batch_id) = @_;
+
+ print "... importing MARC records -- please wait\n";
+ my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
+ BatchCommitRecords($import_batch_id, '', 100, \&print_progress_and_commit);
+ print "... finished importing MARC records\n";
+
+ print <<_SUMMARY_;
+
+MARC record import report
+----------------------------------------
+Batch number: $import_batch_id
+Number of new records added: $num_added
+Number of records replaced: $num_updated
+Number of records ignored: $num_ignored
+Number of items added: $num_items_added
+Number of items ignored: $num_items_errored
+
+Note: an item is ignored if its barcode is a
+duplicate of one already in the database.
+_SUMMARY_
+}
+
+sub revert_batch {
+ my ($import_batch_id) = @_;
+
+ print "... reverting batch -- please wait\n";
+ my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
+ BatchRevertRecords($import_batch_id, 100, \&print_progress_and_commit);
+ print "... finished reverting batch\n";
+
+ print <<_SUMMARY_;
+
+MARC record import report
+----------------------------------------
+Batch number: $import_batch_id
+Number of records deleted: $num_deleted
+Number of errors: $num_errors
+Number of records reverted: $num_reverted
+Number of records ignored: $num_ignored
+Number of items added: $num_items_deleted
+
+_SUMMARY_
+}
+
+
+sub print_progress_and_commit {
+ my $recs = shift;
+ print "... processed $recs records\n";
+ $dbh->commit();
+}
+
+sub print_usage {
+ print <<_USAGE_;
+$0: import a batch of staged MARC records into database.
+
+Use this batch job to complete the import of a batch of
+MARC records that was staged either by the batch job
+stage_file.pl or by the Koha Tools option
+"Stage MARC Records for Import".
+
+Parameters:
+ --batch-number <#> number of the record batch
+ to import
+ --list-batches print a list of record batches
+ available to commit
+ --revert revert a batch instead of importing it
+ --help or -h show this message.
+_USAGE_
+}
diff --git a/misc/cronjobs/import_webservice_batch.pl b/misc/cronjobs/import_webservice_batch.pl
index d1b30ef..3f57779 100755
--- a/misc/cronjobs/import_webservice_batch.pl
+++ b/misc/cronjobs/import_webservice_batch.pl
@@ -54,4 +54,4 @@ EOF
my $batch_ids = GetStagedWebserviceBatches() or exit;
$framework ||= '';
-BatchCommitBibRecords($_, $framework) foreach @$batch_ids;
+BatchCommitRecords($_, $framework) foreach @$batch_ids;
diff --git a/misc/stage_biblios_file.pl b/misc/stage_biblios_file.pl
deleted file mode 100755
index 6e2b30c..0000000
--- a/misc/stage_biblios_file.pl
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/perl
-
-# This file is part of Koha.
-#
-# Copyright (C) 2007 LibLime
-# Parts Copyright BSZ 2011
-#
-# Koha is free software; you can redistribute it and/or modify it under the
-# terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-use strict;
-use warnings;
-BEGIN {
- # find Koha's Perl modules
- # test carefully before changing this
- use FindBin;
- eval { require "$FindBin::Bin/kohalib.pl" };
-}
-
-use C4::Context;
-use C4::ImportBatch;
-use C4::Matcher;
-use Getopt::Long;
-
-$| = 1;
-
-# command-line parameters
-my $encoding = "";
-my $match_bibs = 0;
-my $add_items = 0;
-my $input_file = "";
-my $batch_comment = "";
-my $want_help = 0;
-my $no_replace ;
-
-my $result = GetOptions(
- 'encoding:s' => \$encoding,
- 'file:s' => \$input_file,
- 'match-bibs:s' => \$match_bibs,
- 'add-items' => \$add_items,
- 'no-replace' => \$no_replace,
- 'comment:s' => \$batch_comment,
- 'h|help' => \$want_help
-);
-
-if ($encoding eq "") {
- $encoding = "utf8";
-}
-
-if (not $result or $input_file eq "" or $want_help) {
- print_usage();
- exit 0;
-}
-
-unless (-r $input_file) {
- die "$0: cannot open input file $input_file: $!\n";
-}
-
-my $dbh = C4::Context->dbh;
-$dbh->{AutoCommit} = 0;
-process_batch($input_file, $match_bibs, $add_items, $batch_comment);
-$dbh->commit();
-
-exit 0;
-
-sub process_batch {
- my ($input_file, $match_bibs, $add_items, $batch_comment) = @_;
-
- open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
- my $marc_records = "";
- $/ = "\035";
- my $num_input_records = 0;
- while (<IN>) {
- s/^\s+//;
- s/\s+$//;
- next unless $_; # skip if record has only whitespace, as might occur
- # if file includes newlines between each MARC record
- $marc_records .= $_; # FIXME - this sort of string concatenation
- # is probably rather inefficient
- $num_input_records++;
- }
- close IN;
-
- print "... staging MARC records -- please wait\n";
- my ($batch_id, $num_valid, $num_items, @import_errors) =
- BatchStageMarcRecords($encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0,
- 100, \&print_progress_and_commit);
- print "... finished staging MARC records\n";
-
- my $num_with_matches = 0;
- if ($match_bibs) {
- my $matcher = C4::Matcher->fetch($match_bibs) ;
- if (! defined $matcher) {
- $matcher = C4::Matcher->new('biblio');
- $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
- $matcher->add_simple_required_check('245', 'a', -1, 0, '',
- '245', 'a', -1, 0, '');
- } else {
- SetImportBatchMatcher($batch_id, $match_bibs);
- }
- # set default record overlay behavior
- SetImportBatchOverlayAction($batch_id, ($no_replace) ? 'ignore' : 'replace');
- SetImportBatchNoMatchAction($batch_id, 'create_new');
- SetImportBatchItemAction($batch_id, 'always_add');
- print "... looking for matches with records already in database\n";
- $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit);
- print "... finished looking for matches\n";
- }
-
- my $num_invalid_bibs = scalar(@import_errors);
- print <<_SUMMARY_;
-
-MARC record staging report
-------------------------------------
-Input file: $input_file
-Number of input bibs: $num_input_records
-Number of valid bibs: $num_valid
-Number of invalid bibs: $num_invalid_bibs
-_SUMMARY_
- if ($match_bibs) {
- print "Number of bibs matched: $num_with_matches\n";
- } else {
- print "Incoming bibs not matched against existing bibs (--match-bibs option not supplied)\n";
- }
- if ($add_items) {
- print "Number of items parsed: $num_items\n";
- } else {
- print "No items parsed (--add-items option not supplied)\n";
- }
-
- print "\n";
- print "Batch number assigned: $batch_id\n";
- print "\n";
-}
-
-sub print_progress_and_commit {
- my $recs = shift;
- $dbh->commit();
- print "... processed $recs records\n";
-}
-
-sub print_usage {
- print <<_USAGE_;
-$0: stage MARC bib file into reservoir.
-
-Use this batch job to load a file of MARC bibliographic records
-(with optional item information) into the Koha reservoir.
-
-After running this program to stage your file, you can use
-either the batch job commit_biblios_file.pl or the Koha
-Tools option "Manage Staged MARC Records" to load the
-records into the main Koha database.
-
-Parameters:
- --file <file_name> name of input MARC bib file
- --encoding <encoding> encoding of MARC records, default is utf8.
- Other possible options are: MARC-8,
- ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
- --match-bibs <match_id> use this option to match bibs
- in the file with bibs already in
- the database for future overlay.
- If <match_id> isn't defined, a default
- MARC21 ISBN & title match rule will be applied.
- --add-items use this option to specify that
- item data is embedded in the MARC
- bibs and should be parsed.
- --no-replace overlay action for bib record: default is to
- replace extant bib with the imported record.
- --comment <comment> optional comment to describe
- the record batch; if the comment
- has spaces in it, surround the
- comment with quotation marks.
- --help or -h show this message.
-_USAGE_
-}
diff --git a/misc/stage_file.pl b/misc/stage_file.pl
new file mode 100755
index 0000000..3cf6309
--- /dev/null
+++ b/misc/stage_file.pl
@@ -0,0 +1,197 @@
+#!/usr/bin/perl
+
+# This file is part of Koha.
+#
+# Copyright (C) 2007 LibLime
+# Parts Copyright BSZ 2011
+# Parts Copyright C & P Bibliography Services 2012
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use strict;
+use warnings;
+BEGIN {
+ # find Koha's Perl modules
+ # test carefully before changing this
+ use FindBin;
+ eval { require "$FindBin::Bin/kohalib.pl" };
+}
+
+use C4::Context;
+use C4::ImportBatch;
+use C4::Matcher;
+use Getopt::Long;
+
+$| = 1;
+
+# command-line parameters
+my $record_type = "biblio";
+my $encoding = "";
+my $authorities = 0;
+my $match = 0;
+my $add_items = 0;
+my $input_file = "";
+my $batch_comment = "";
+my $want_help = 0;
+my $no_replace ;
+
+my $result = GetOptions(
+ 'encoding:s' => \$encoding,
+ 'file:s' => \$input_file,
+ 'match|match-bibs:s' => \$match,
+ 'add-items' => \$add_items,
+ 'no-replace' => \$no_replace,
+ 'comment:s' => \$batch_comment,
+ 'authorities' => \$authorities,
+ 'h|help' => \$want_help
+);
+
+$record_type = 'biblio' if ($authorities);
+
+if ($encoding eq "") {
+ $encoding = "utf8";
+}
+
+if (not $result or $input_file eq "" or $want_help) {
+ print_usage();
+ exit 0;
+}
+
+unless (-r $input_file) {
+ die "$0: cannot open input file $input_file: $!\n";
+}
+
+my $dbh = C4::Context->dbh;
+$dbh->{AutoCommit} = 0;
+process_batch($input_file, $record_type, $match, $add_items, $batch_comment);
+$dbh->commit();
+
+exit 0;
+
+sub process_batch {
+ my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_;
+
+ open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
+ my $marc_records = "";
+ $/ = "\035";
+ my $num_input_records = 0;
+ while (<IN>) {
+ s/^\s+//;
+ s/\s+$//;
+ next unless $_; # skip if record has only whitespace, as might occur
+ # if file includes newlines between each MARC record
+ $marc_records .= $_; # FIXME - this sort of string concatenation
+ # is probably rather inefficient
+ $num_input_records++;
+ }
+ close IN;
+
+ print "... staging MARC records -- please wait\n";
+ my ($batch_id, $num_valid_records, $num_items, @import_errors) =
+ BatchStageMarcRecords($record_type, $encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0,
+ 100, \&print_progress_and_commit);
+ print "... finished staging MARC records\n";
+
+ my $num_with_matches = 0;
+ if ($match) {
+ my $matcher = C4::Matcher->fetch($match) ;
+ if (defined $matcher) {
+ SetImportBatchMatcher($batch_id, $match);
+ } elsif ($record_type eq 'biblio') {
+ $matcher = C4::Matcher->new($record_type);
+ $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
+ $matcher->add_simple_required_check('245', 'a', -1, 0, '',
+ '245', 'a', -1, 0, '');
+ }
+ # set default record overlay behavior
+ SetImportBatchOverlayAction($batch_id, ($no_replace) ? 'ignore' : 'replace');
+ SetImportBatchNoMatchAction($batch_id, 'create_new');
+ SetImportBatchItemAction($batch_id, 'always_add');
+ print "... looking for matches with records already in database\n";
+ $num_with_matches = BatchFindDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit);
+ print "... finished looking for matches\n";
+ }
+
+ my $num_invalid_records = scalar(@import_errors);
+ print <<_SUMMARY_;
+
+MARC record staging report
+------------------------------------
+Input file: $input_file
+Record type: $record_type
+Number of input records: $num_input_records
+Number of valid records: $num_valid_records
+Number of invalid records: $num_invalid_records
+_SUMMARY_
+ if ($match) {
+ print "Number of records matched: $num_with_matches\n";
+ } else {
+ print "Incoming records not matched against existing records (--match option not supplied)\n";
+ }
+ if ($record_type eq 'biblio') {
+ if ($add_items) {
+ print "Number of items parsed: $num_items\n";
+ } else {
+ print "No items parsed (--add-items option not supplied)\n";
+ }
+ }
+
+ print "\n";
+ print "Batch number assigned: $batch_id\n";
+ print "\n";
+}
+
+sub print_progress_and_commit {
+ my $recs = shift;
+ $dbh->commit();
+ print "... processed $recs records\n";
+}
+
+sub print_usage {
+ print <<_USAGE_;
+$0: stage MARC file into reservoir.
+
+Use this batch job to load a file of MARC bibliographic
+(with optional item information) or authority records into
+the Koha reservoir.
+
+After running this program to stage your file, you can use
+either the batch job commit_file.pl or the Koha
+Tools option "Manage Staged MARC Records" to load the
+records into the main Koha database.
+
+Parameters:
+ --file <file_name> name of input MARC bib file
+ --authorities stage authority records instead of bibs
+ --encoding <encoding> encoding of MARC records, default is utf8.
+ Other possible options are: MARC-8,
+ ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
+ --match <match_id> use this option to match records
+ in the file with records already in
+ the database for future overlay.
+ If <match_id> isn't defined, a default
+ MARC21 ISBN & title match rule will be applied
+ for bib imports.
+ --add-items use this option to specify that
+ item data is embedded in the MARC
+ bibs and should be parsed.
+ --no-replace overlay action for record: default is to
+ replace extant with the imported record.
+ --comment <comment> optional comment to describe
+ the record batch; if the comment
+ has spaces in it, surround the
+ comment with quotation marks.
+ --help or -h show this message.
+_USAGE_
+}
diff --git a/svc/import_bib b/svc/import_bib
index 5e7e493..c130204 100755
--- a/svc/import_bib
+++ b/svc/import_bib
@@ -92,7 +92,7 @@ sub import_bib {
$marcxml =~ s/<\?xml.*?\?>//i;
# XXX we are ignoring the result of this;
- BatchCommitBibRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
+ BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
$result->{'status'} = "ok";
$result->{'import_batch_id'} = $batch_id;
diff --git a/tools/manage-marc-import.pl b/tools/manage-marc-import.pl
index 8a8d9e8..7346c59 100755
--- a/tools/manage-marc-import.pl
+++ b/tools/manage-marc-import.pl
@@ -244,7 +244,7 @@ sub commit_batch {
$callback = progress_callback($job, $dbh);
}
my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
- BatchCommitBibRecords($import_batch_id, $framework, 50, $callback);
+ BatchCommitRecords($import_batch_id, $framework, 50, $callback);
$dbh->commit();
my $results = {
@@ -273,7 +273,7 @@ sub revert_batch {
$callback = progress_callback($job, $dbh);
}
my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
- BatchRevertBibRecords($import_batch_id, 50, $callback);
+ BatchRevertRecords($import_batch_id, 50, $callback);
$dbh->commit();
my $results = {
--
1.7.2.5
More information about the Koha-patches
mailing list