[Koha-cvs] koha/misc/migration_tools rebuild_zebra.pl [rel_3_0]
paul poulain
paul at koha-fr.org
Tue Oct 24 14:31:57 CEST 2006
CVSROOT: /sources/koha
Module name: koha
Branch: rel_3_0
Changes by: paul poulain <tipaul> 06/10/24 12:31:57
Modified files:
misc/migration_tools: rebuild_zebra.pl
Log message:
1st draft of a new rebuild_zebra.pl script, that will :
- read koha.xml config file
- create all directories needed for zebra
- copy what is needed (including record.abs for marc21 or unimarc depending on marcflavour)
- create the zebra-bioblio.cfg file
- export all your datas in a directory (an option can disable this export)
- reindex everything in zebra
- delete the directory where the datas were exporter (an option can disable this deletion)
WARNING : DON'T RUN IT WITHOUT CHECKING THAT YOUR koha.xml zebra directories are NOT in CVS scope !
CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/koha/misc/migration_tools/rebuild_zebra.pl?cvsroot=koha&only_with_tag=rel_3_0&r1=1.4&r2=1.4.2.1
Patches:
Index: rebuild_zebra.pl
===================================================================
RCS file: /sources/koha/koha/misc/migration_tools/rebuild_zebra.pl,v
retrieving revision 1.4
retrieving revision 1.4.2.1
diff -u -b -r1.4 -r1.4.2.1
--- rebuild_zebra.pl 9 Feb 2006 10:59:34 -0000 1.4
+++ rebuild_zebra.pl 24 Oct 2006 12:31:57 -0000 1.4.2.1
@@ -1,82 +1,212 @@
#!/usr/bin/perl
-use strict;
-
-# Koha modules used
-use MARC::File::USMARC;
-use MARC::Record;
-use MARC::Batch;
use C4::Context;
+use Getopt::Long;
use C4::Biblio;
-use ZOOM;
-use Time::HiRes qw(gettimeofday);
-use Getopt::Long;
-my ( $input_marc_file, $number) = ('',0);
-my ($confirm);
+#
+# script that checks zebradir structure & create directories & mandatory files if needed
+#
+#
+
+$|=1; # flushes output
+
+print "Zebra directory =>".C4::Context->zebraconfig('biblioserver')->{directory}."\n";
+print "Koha directory =>".C4::Context->config('intranetdir')."\n";
+
+my $zebradir = C4::Context->zebraconfig('biblioserver')->{directory};
+my $kohadir = C4::Context->config('intranetdir');
+my $directory;
+my $skip_export;
+my $keep_export;
GetOptions(
- 'c' => \$confirm,
-);
+ 'd:s' => \$directory,
+ 's' => \$skip_export,
+ 'k' => \$keep_export,
+ );
+
+$directory = "export" unless $directory;
+
+#
+# creating directory structure
+#
+my $created_dir_or_file = 0;
+print "====================\n";
+print "checking directories & files\n";
+print "====================\n";
+unless (-d "$zebradir") {
+ system("mkdir -p $zebradir");
+ print "created $zebradir\n";
+ $created_dir_or_file++;
+}
+unless (-d "$zebradir/lock") {
+ mkdir "$zebradir/lock";
+ print "created $zebradir/lock\n";
+ $created_dir_or_file++;
+}
+unless (-d "$zebradir/register") {
+ mkdir "$zebradir/register";
+ print "created $zebradir/register\n";
+ $created_dir_or_file++;
+}
+unless (-d "$zebradir/shadow") {
+ mkdir "$zebradir/shadow";
+ print "created $zebradir/shadow\n";
+ $created_dir_or_file++;
+}
+unless (-d "$zebradir/tab") {
+ mkdir "$zebradir/tab";
+ print "created $zebradir/tab\n";
+ $created_dir_or_file++;
+}
+unless (-d "$zebradir/key") {
+ mkdir "$zebradir/key";
+ print "created $zebradir/key\n";
+ $created_dir_or_file++;
+}
-unless ($confirm) {
- print <<EOF
+unless (-d "$zebradir/etc") {
+ mkdir "$zebradir/etc";
+ print "created $zebradir/etc\n";
+ $created_dir_or_file++;
+}
+
+#
+# copying mandatory files.
+#
+# the record model, depending on marc flavour
+unless (-f "$zebradir/tab/record.abs") {
+ if (C4::Context->preference("marcflavour") eq "UNIMARC") {
+ system("cp -f $kohadir/zebraplugin/zebradb/biblios/tab/record_for_unimarc.abs $zebradir/tab/record.abs");
+ print "copied record.abs for UNIMARC\n";
+ } else {
+ system("cp -f $kohadir/zebraplugin/zebradb/biblios/tab/record.abs $zebradir/tab/record.abs");
+ print "copied record.abs for MARC21\n";
+ }
+ $created_dir_or_file++;
+}
+unless (-f "$zebradir/tab/sort-string-utf.chr") {
+ system("cp -f $kohadir/zebraplugin/zebradb/biblios/tab/sort-string-utf.chr $zebradir/tab/sort-string-utf.chr");
+ print "copied sort-string-utf.chr\n";
+ $created_dir_or_file++;
+}
+unless (-f "$zebradir/tab/word-phrase-utf.chr") {
+ system("cp -f $kohadir/zebraplugin/zebradb/biblios/tab/word-phrase-utf.chr $zebradir/tab/word-phrase-utf.chr");
+ print "copied word-phase-utf.chr\n";
+ $created_dir_or_file++;
+}
+unless (-f "$zebradir/tab/bib1.att") {
+ system("cp -f $kohadir/zebraplugin/zebradb/biblios/tab/bib1.att $zebradir/tab/bib1.att");
+ print "copied bib1.att\n";
+ $created_dir_or_file++;
+}
+unless (-f "$zebradir/etc/ccl.properties") {
+ system("cp -f $kohadir/zebraplugin/etc/ccl.properties $zebradir/etc/ccl.properties");
+ print "copied ccl.properties\n";
+ $created_dir_or_file++;
+}
+unless (-f "$zebradir/etc/pqf.properties") {
+ system("cp -f $kohadir/zebraplugin/etc/pqf.properties $zebradir/etc/pqf.properties");
+ print "copied pqf.properties\n";
+ $created_dir_or_file++;
+}
-Script to create the zebra DB from a Koha DB
+#
+# creating zebra-biblios.cfg depending on systempreferences
+#
+
+unless (-f "$zebradir/etc/zebra-biblios.cfg") {
+open ZD,">$zebradir/etc/zebra-biblios.cfg";
+print ZD "
+# generated by KOHA/misc/migrtion_tools/rebuild_zebra.pl
+profilePath:\${srcdir:-.}:$zebradir/tab/:/usr/local/share/idzebra/tab/:\${srcdir:-.}/tab/
+
+encoding: UTF-8
+# Files that describe the attribute sets supported.
+attset: bib1.att
+attset: explain.att
+attset:gils.att
+
+systag sysno rank
+# Specify record type
+iso2709.recordType:grs.marcxml.record
+recordType:grs.xml
+recordId: (bib1,Identifier-standard)
+storeKeys:1
+storeData:1
+
+
+# Lock File Area
+lockDir: $zebradir/lock
+perm.anonymous:r
+perm.kohaadmin:rw
+passw.kohalis
+shadow
+register: $zebradir/register:4G
+shadow: $zebradir/shadow:4G
+
+# Temp File area for result sets
+setTmpDir: $zebradir/tmp
+
+# Temp File area for index program
+keyTmpDir: $zebradir/key
+
+# Approx. Memory usage during indexing
+memMax: 40M
+rank:rank-1
+";
+ print "creating zebra-biblios.cfg\n";
+ $created_dir_or_file++;
+}
-EOF
-;#'
-die;
+if ($created_dir_or_file) {
+ print "created : $created_dir_or_file directories & files\n";
+} else {
+ print "file & directories OK\n";
}
-$|=1; # flushes output
+#
+# exporting biblios
+#
+if ($skip_export) {
+ print "====================\n";
+ print "SKIPPING biblio export\n";
+ print "====================\n";
+} else {
+ print "====================\n";
+ print "exporting biblios\n";
+ print "====================\n";
+ mkdir "$directory" unless (-d $directory);
+ open(OUT,">:utf8","$directory/export") or die $!;
+ my $dbh=C4::Context->dbh;
+ my $sth;
+ $sth=$dbh->prepare("select biblionumber from biblioitems where biblionumber <1000 order by biblionumber");
+ $sth->execute();
+ my $i=0;
+ while (my ($biblionumber) = $sth->fetchrow) {
+ my $record = MARCgetbiblio($dbh,$biblionumber);
+ print ".";
+ print "\r$i" unless ($i++ %100);
+ print OUT $record->as_usmarc();
+ }
+ close(OUT);
+}
-my $dbh = C4::Context->dbh;
-my $Zconn;
-eval {
- $Zconn = new ZOOM::Connection('localhost','2100');
-};
-if ($@) {
- print "Error ", $@->code()," : ",$@->message()."\n";
- die;
-}
-
-# first, drop Zebra DB
-# eval {
-# my $Zpackage = $Zconn->package();
-# $Zpackage->option(databaseName => 'Koha');
-# # $Zpackage->send("drop");
-# };
-
-eval {
- my $Zpackage = $Zconn->package();
- $Zpackage->option(databaseName => 'Koha');
- $Zpackage->send("create");
-};
-my $cgidir = C4::Context->intranetdir ."/cgi-bin";
-unless (opendir(DIR, "$cgidir")) {
- $cgidir = C4::Context->intranetdir."/";
-}
-my $starttime = gettimeofday;
-my $sth = $dbh->prepare("select biblionumber from biblio");
-$sth->execute;
-my $i=0;
-while ((my $biblionumber) = $sth->fetchrow) {
- my $record = XMLgetbiblio($dbh,$biblionumber);
-# warn "\n==============\n$record\n==================\n";
- my $Zpackage = $Zconn->package();
- $Zpackage->option(databaseName => 'Koha');
- $Zpackage->option(action => "specialUpdate");
-# $Zpackage->option(recordIdNumber => $biblionumber);
- $Zpackage->option(record => $record);
- $Zpackage->send("update");
-# $Zpackage->destroy;
- $i++;
- print '.';
- print "$i\r" unless ($i % 100);
-# exit if $i>100;
-}
-my $Zpackage = $Zconn->package();
-$Zpackage->option(databaseName => 'Koha');
-$Zpackage->send("commit");
-my $timeneeded = gettimeofday - $starttime;
-print "\n\n$i MARC record done in $timeneeded seconds\n";
+#
+# and reindexing everything
+#
+print "====================\n";
+print "REINDEXING zebra\n";
+print "====================\n";
+system("zebraidx -g iso2709 -c $zebradir/etc/zebra-biblios.cfg -d biblios update $directory");
+system("zebraidx -g iso2709 -c $zebradir/etc/zebra-biblios.cfg -d biblios commit");
+
+print "====================\n";
+print "CLEANING\n";
+print "====================\n";
+if ($k) {
+ print "NOTHING cleaned : the $directory has been kept. You can re-run this script with the -s parameter if you just want to rebuild zebra after changing the record.abs or another zebra config file\n";
+} else {
+# system("rm -rf $export");
+# print "directory $export deleted\n";
+}
More information about the Koha-cvs
mailing list