[Koha-patches] [PATCH] Bug 2627: Allow Unicode data in import_borrowers, expand feedback on errors.
Joe Atzberger
joe.atzberger at liblime.com
Fri Sep 19 02:02:45 CEST 2008
Date fields are now checked against syspref and iso regexps before attempting
to convert or insert them. The problem characters were non-ASCII diacriticals.
Note: this may rely on improvements in the current 0.54 version of Text::CSV and
Text::CSV_XS, rather than the Koha minimum of 0.01 and 0.32, respectively.
---
.../prog/en/modules/tools/import_borrowers.tmpl | 59 +++++++++++++-----
tools/import_borrowers.pl | 62 +++++++++++++++----
2 files changed, 91 insertions(+), 30 deletions(-)
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl
index 0c5f5c5..47dc3a0 100644
--- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl
+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl
@@ -3,6 +3,7 @@
<!-- TMPL_INCLUDE NAME="doc-head-close.inc" -->
<style type="text/css">
.yui-u fieldset.rows label.widelabel { width: 12em; }
+ .line_error { width: 100%; }
code { background-color: yellow; }
</style>
</head>
@@ -13,12 +14,11 @@
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> › <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> › <a href="/cgi-bin/koha/tools/import_borrowers.pl">Import Patrons</a><!-- TMPL_IF name="uploadborrowers" --> › Results<!-- /TMPL_IF --></div>
<div id="doc3" class="yui-t2">
-
- <div id="bd">
- <div id="yui-main">
- <div class="yui-b">
-<div class="yui-g">
-<div class="yui-u first">
+ <div id="bd">
+ <div id="yui-main">
+ <div class="yui-b">
+ <div class="yui-g">
+ <div class="yui-u first">
<h1>Import Patrons</h1>
<!-- TMPL_IF name="uploadborrowers" -->
<h5>Import results :</h5>
@@ -30,6 +30,23 @@
<li><!-- TMPL_VAR name="total" --> records parsed</li>
<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
</ul>
+ <!-- TMPL_IF NAME="FEEDBACK" -->
+ <br /><br />
+ <div>
+ <h5>Feedback:</h5>
+ <ul class="feedback">
+ <!-- TMPL_LOOP NAME="FEEDBACK" -->
+ <li>
+ <!-- TMPL_IF NAME="filename" -->Parsing upload file <span class="filename"><!-- TMPL_VAR NAME="filename" --></span>
+ <!-- TMPL_ELSIF NAME="backend" -->Upload parsed using <!-- TMPL_VAR NAME="backend" -->
+ <!-- TMPL_ELSIF NAME="headerrow" -->These fields found: <!-- TMPL_VAR NAME="value" -->
+ <!-- TMPL_ELSE --><!-- TMPL_VAR NAME="name" --> : <!-- TMPL_VAR NAME="value" -->
+ <!-- /TMPL_IF -->
+ </li>
+ <!-- /TMPL_LOOP -->
+ </ul>
+ </div>
+ <!-- /TMPL_IF -->
<!-- TMPL_IF NAME="ERRORS" -->
<br /><br />
<div>
@@ -38,11 +55,18 @@
<!-- TMPL_LOOP NAME="ERRORS" -->
<!-- TMPL_IF NAME="badheader" --><li>Header row could not be parsed</li><!-- /TMPL_IF -->
<!-- TMPL_LOOP NAME="missing_criticals" -->
- <li>
+ <li class="line_error">
+ Line <span class="linenumber"><!-- TMPL_VAR NAME="line" --></span>
<!-- TMPL_IF NAME="badparse" -->
- Line <span class="linenumber"><!-- TMPL_VAR NAME="line" --></span> could not be parsed!
+ could not be parsed!
+ <!-- TMPL_ELSIF NAME="bad_date" -->
+ has "<!-- TMPL_VAR NAME="key" -->" in unrecognized format: "<!-- TMPL_VAR NAME="value" -->"
<!-- TMPL_ELSE -->
- Critical field "<!-- TMPL_VAR NAME="key" -->" missing on line <span class="linenumber"><!-- TMPL_VAR NAME="line" --></span>
+ Critical field "<!-- TMPL_VAR NAME="key" -->"
+ <!-- TMPL_IF NAME="branch_map" -->has unrecognized value "<!-- TMPL_VAR NAME="value" -->"
+ <!-- TMPL_ELSIF NAME="category_map" -->has unrecognized value "<!-- TMPL_VAR NAME="value" -->"
+ <!-- TMPL_ELSE -->missing
+ <!-- /TMPL_IF -->
(borrowernumber: <!-- TMPL_VAR NAME="borrowernumber" -->; surname: <!-- TMPL_VAR NAME="surname" -->).
<!-- /TMPL_IF -->
<br /><code><!-- TMPL_VAR NAME="lineraw" --></code>
@@ -105,9 +129,10 @@
<ul>
<li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b> Values are comma-separated.</li>
<li>OR format your file in CSV format with the following fields:</li>
-<ul><li>
+<li><ul><li>
<!-- TMPL_LOOP name="columnkeys" -->'<!-- TMPL_VAR name="key" -->', <!-- /TMPL_LOOP -->
</li></ul>
+</li>
<!-- TMPL_IF NAME="ExtendedPatronAttributes" -->
<li>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types
and values. The attribute type code and a ':' should precede each value. For example: "INSTID:12345,LANG:fr". This
@@ -119,13 +144,13 @@ means that if an input record has more than one attribute, the 'patron_attribute
<li>Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.</li>
<li>You may optionally include a header row, defining which columns you are supplying in the import file.</li>
</ul>
-</div>
-</div>
-</div>
-</div>
-<div class="yui-b noprint">
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="yui-b noprint">
<!-- TMPL_INCLUDE NAME="tools-menu.inc" -->
-</div>
-</div>
+ </div>
+ </div>
<!-- TMPL_INCLUDE NAME="intranet-bottom.inc" -->
diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl
index 5cdf3c1..662dfcc 100755
--- a/tools/import_borrowers.pl
+++ b/tools/import_borrowers.pl
@@ -34,6 +34,8 @@
# branchcode and categorycode need to be valid
use strict;
+use warnings;
+
use C4::Auth;
use C4::Output;
use C4::Dates qw(format_date_in_iso);
@@ -44,9 +46,14 @@ use C4::Members::Attributes;
use C4::Members::AttributeTypes;
use Text::CSV;
+# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
+# Ä—
+# Ä
+
use CGI;
+# use encoding 'utf8'; # don't do this
-my @errors;
+my (@errors, @feedback);
my $extended = C4::Context->preference('ExtendedPatronAttributes');
my @columnkeys = C4::Members->columns;
if ($extended) {
@@ -55,7 +62,8 @@ if ($extended) {
my $columnkeystpl = [ map { {'key' => $_} } grep {$_ ne 'borrowernumber' && $_ ne 'cardnumber'} @columnkeys ]; # ref. to array of hashrefs.
my $input = CGI->new();
-my $csv = Text::CSV->new();
+my $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicode
+# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend};
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
template_name => "tools/import_borrowers.tmpl",
@@ -77,8 +85,8 @@ if ($input->param('sample')) {
print $csv->string, "\n";
exit 1;
}
-my $uploadborrowers = $input->param('uploadborrowers');
-my $matchpoint = $input->param('matchpoint');
+my $uploadborrowers = $input->param('uploadborrowers');
+my $matchpoint = $input->param('matchpoint');
if ($matchpoint) {
$matchpoint =~ s/^patron_attribute_//;
}
@@ -89,6 +97,12 @@ $template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
($extended) and $template->param(ExtendedPatronAttributes => 1);
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
+ push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
+ my $handle = $input->upload('uploadborrowers');
+ my $uploadinfo = $input->uploadInfo($uploadborrowers);
+ foreach (keys %$uploadinfo) {
+ push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
+ }
my $imported = 0;
my $alreadyindb = 0;
my $overwritten = 0;
@@ -97,7 +111,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
my %defaults = $input->Vars;
# use header line to construct key to column map
- my $borrowerline = <$uploadborrowers>;
+ my $borrowerline = <$handle>;
my $status = $csv->parse($borrowerline);
($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline};
my @csvcolumns = $csv->fields();
@@ -113,9 +127,13 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
}
+ push @feedback, {feedback=>1, name=>'headerrow', value=>join(', ', @csvcolumns)};
+ my $today_iso = C4::Dates->new()->output('iso');
my @criticals = qw(cardnumber surname categorycode); # there probably should be others
- my @errors;
- LINE: while ( my $borrowerline = <$uploadborrowers> ) {
+ my @bad_dates; # I've had a few.
+ my $date_re = C4::Dates->new->regexp('syspref');
+ my $iso_re = C4::Dates->new->regexp('iso');
+ LINE: while ( my $borrowerline = <$handle> ) {
my %borrower;
my @missing_criticals;
my $patron_attributes;
@@ -141,8 +159,18 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
}
}
#warn join(':',%borrower);
- push @missing_criticals, {key=>'categorycode' , line=>$. , lineraw=>$borrowerline } unless( GetBorrowercategory($borrower{categorycode}) );
- push @missing_criticals, {key=>'branchcode' , line=>$. , lineraw=>$borrowerline } unless( GetBranchName($borrower{branchcode}) );
+ if ($borrower{categorycode}) {
+ push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}
+ unless GetBorrowercategory($borrower{categorycode});
+ } else {
+ push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline};
+ }
+ if ($borrower{branchcode}) {
+ push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}
+ unless GetBranchName($borrower{branchcode});
+ } else {
+ push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline};
+ }
if (@missing_criticals) {
foreach (@missing_criticals) {
$_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
@@ -162,12 +190,19 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
# FIXME error handling
$patron_attributes = [ map { map { my @arr = split /:/, $_, 2; { code => $arr[0], value => $arr[1] } } $_ } @list ];
}
- # FIXME date handling. Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
+ # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
foreach (qw(dateofbirth dateenrolled dateexpiry)) {
my $tempdate = $borrower{$_} or next;
- $borrower{$_} = format_date_in_iso($tempdate) || '';
+ if ($tempdate =~ /$date_re/) {
+ $borrower{$_} = format_date_in_iso($tempdate);
+ } elsif ($tempdate =~ /$iso_re/) {
+ $borrower{$_} = $tempdate;
+ } else {
+ $borrower{$_} = '';
+ push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1};
+ }
}
- $borrower{dateenrolled} = C4::Dates->new()->output('iso') unless $borrower{dateenrolled};
+ $borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
$borrower{dateexpiry} = GetExpiryDate($borrower{categorycode},$borrower{dateenrolled}) unless $borrower{dateexpiry};
my $borrowernumber;
my $member;
@@ -231,7 +266,8 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
}
}
}
- (@errors) and $template->param(ERRORS=>\@errors);
+ (@errors ) and $template->param( ERRORS=>\@errors );
+ (@feedback) and $template->param(FEEDBACK=>\@feedback);
$template->param(
'uploadborrowers' => 1,
'imported' => $imported,
--
1.5.5.GIT
More information about the Koha-patches
mailing list