[Koha-patches] [PATCH] Bug 3216 Add CHR/ICU Zebra tokenization choice to installation

[Frédéric Demians]

Word search with multi-part facets works properly only with Zebra ICU
tokenization. This patch add a new question to Koha command line
installer:

  Zebra has two methods to perform records tokenization
  and characters normalization: CHR and ICU. ICU is
  recommended for catalogs containing non-Latin
  characters. (chr, icu) [chr]

How to test:

  - perl ./Makefile.PL
  - Try each possible value for new parameter
  - Take a look at zebradb/etc/default.idx file.
    Depending of the parameter you get this line:
      icuchain words-icu.xml
    or this one:
      charmap word-phrase-utf.chr
---
 Makefile.PL                   |   22 +++++++++++++++++++---
 etc/zebradb/etc/default.idx   |    5 ++---
 etc/zebradb/etc/words-icu.xml |   12 ++++++++++++
 rewrite-config.PL             |    1 +
 4 files changed, 34 insertions(+), 6 deletions(-)
 create mode 100644 etc/zebradb/etc/words-icu.xml

diff --git a/Makefile.PL b/Makefile.PL
index 074ff55..5ca3103 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -409,6 +409,7 @@ my %config_defaults = (
   'AUTH_INDEX_MODE'   => 'dom',
   'ZEBRA_MARC_FORMAT' => 'marc21',
   'ZEBRA_LANGUAGE'    => 'en',
+  'ZEBRA_TOKENIZER'   => 'chr',
   'ZEBRA_USER'        => 'kohauser',
   'ZEBRA_PASS'        => 'zebrastripes',
   'ZEBRA_SRU_HOST'    => 'localhost',
@@ -460,6 +461,7 @@ my %valid_config_values = (
   'AUTH_INDEX_MODE' => { 'grs1' => 1, 'dom' => 1 },
   'ZEBRA_MARC_FORMAT' => { 'marc21' => 1, 'normarc' => 1, 'unimarc' => 1 }, # FIXME should generate from contents of distributation
   'ZEBRA_LANGUAGE'    => { 'en' => 1, 'fr' => 1, 'nb' => 1 }, # FIXME should generate from contents of distribution
+  'ZEBRA_TOKENIZER' => { chr => 1, icu => 1 },
   'RUN_DATABASE_TESTS' => { 'yes' => 1, 'no' => 1 },
   'USE_MEMCACHED'      => { 'yes' => 1, 'no' => 1 },
 );
@@ -486,11 +488,12 @@ my $pl_files = {
       'rewrite-config.PL' => [
          'blib/KOHA_CONF_DIR/koha-conf.xml',
          'blib/KOHA_CONF_DIR/koha-httpd.conf',
+         'blib/ZEBRA_CONF_DIR/etc/default.idx',
          'blib/MISC_DIR/koha-install-log'
          ],
 	  'fix-perl-path.PL' => [	# this script ensures the correct shebang line for the platform installed on...
 		 'blib'
-		 ]
+		 ],
 };
 
 if ($config{'INSTALL_ZEBRA'} eq "yes") {
@@ -527,6 +530,10 @@ if ($config{'INSTALL_MODE'} ne "dev") {
     );
 }
 
+$config{ZEBRA_TOKENIZER_STMT} = $config{ZEBRA_TOKENIZER} eq 'icu'
+    ? 'icuchain words-icu.xml'
+    : 'charmap word-phrase-utf.chr';
+
 my %test_suite_override_dirs = (
     KOHA_CONF_DIR  => ['etc'],
     ZEBRA_CONF_DIR => ['etc', 'zebradb'],
@@ -917,8 +924,8 @@ Since you've chosen to use Zebra with Koha,
 you must specify the primary MARC format of the
 records to be indexed by Zebra.
 
-Koha provides Zebra configuration files for MARC 21
-and UNIMARC.
+Koha provides Zebra configuration files for MARC21,
+NORMARC and UNIMARC.
 
 MARC format for Zebra indexing);
         $msg .= _add_valid_values_disp('ZEBRA_MARC_FORMAT', $valid_values);
@@ -946,6 +953,15 @@ Authorities indexing mode);
         $config{'AUTH_INDEX_MODE'} = _get_value('AUTH_INDEX_MODE', $msg, $defaults->{'AUTH_INDEX_MODE'}, $valid_values, $install_log_values);
 
         $msg = q(
+Zebra has two methods to perform records tokenization
+and characters normalization: CHR and ICU. ICU is
+recommended for catalogs containing non-Latin
+characters.);
+
+        $msg .= _add_valid_values_disp('ZEBRA_TOKENIZER', $valid_values);
+        $config{'ZEBRA_TOKENIZER'} = _get_value('ZEBRA_TOKENIZER', $msg, $defaults->{'ZEBRA_TOKENIZER'}, $valid_values, $install_log_values);
+
+        $msg = q(
 Please specify Zebra database user);
         $config{'ZEBRA_USER'} = _get_value('ZEBRA_USER', $msg, $defaults->{'ZEBRA_USER'}, $valid_values, $install_log_values);
 
diff --git a/etc/zebradb/etc/default.idx b/etc/zebradb/etc/default.idx
index 8ffbc59..d6314c6 100644
--- a/etc/zebradb/etc/default.idx
+++ b/etc/zebradb/etc/default.idx
@@ -10,16 +10,15 @@ completeness 0
 position 1
 alwaysmatches 1
 firstinfield 1
-charmap word-phrase-utf.chr
-#firstinfield 1
+__ZEBRA_TOKENIZER_STMT__
 
 # Phrase index
 # Used if completeness is 'complete {sub}field' (@attr 6=2, @attr 6=1)
 # and structure is word/phrase/word-list/free-form-text/document-text
 index p
 completeness 1
-charmap word-phrase-utf.chr
 firstinfield 1
+__ZEBRA_TOKENIZER_STMT__
 
 # URX (URL) index
 # Used if structure=urx (@attr 4=104)
diff --git a/etc/zebradb/etc/words-icu.xml b/etc/zebradb/etc/words-icu.xml
new file mode 100644
index 0000000..38af51f
--- /dev/null
+++ b/etc/zebradb/etc/words-icu.xml
@@ -0,0 +1,12 @@
+<icu_chain locale="">
+  <transliterate rule="\'>\ "/>
+  <transliterate rule="[:Number:] { '-' > '' "/>
+  <transform rule="[:Control:] Any-Remove"/>
+  <tokenize rule="l"/>
+  <transform rule="[[:WhiteSpace:][:Punctuation:]] Remove"/>
+  <transform rule="NFD"/>
+  <transform rule="[:Nonspacing Mark:] Remove"/>
+  <transform rule="NFC"/>
+  <display/>
+  <casemap rule="l"/>
+</icu_chain>
diff --git a/rewrite-config.PL b/rewrite-config.PL
index 7817f91..967a546 100644
--- a/rewrite-config.PL
+++ b/rewrite-config.PL
@@ -120,6 +120,7 @@ $prefix = $ENV{'INSTALL_BASE'} || "/usr";
   '__ZEBRA_RUN_DIR__' => "$prefix/var/run/zebradb",
   '__ZEBRA_MARC_FORMAT__' => 'marc21',
   '__ZEBRA_LANGUAGE__' => 'en',
+  '__ZEBRA_TOKENIZER_STMT__' => 'charmap word-phrase-utf.chr',
   '__ZEBRA_AUTH_CFG__' => 'zebra-authorities.cfg',
   '__AUTH_RETRIEVAL_CFG__' => 'retrieval-info-auth-grs1.xml',
   "__MERGE_SERVER_HOST__" => $myhost,
-- 
1.7.8