[Koha-patches] [PATCH 2/2] Bug 7286: fix rebuild_zebra.pl to add rebuild_zebra_sliced.zsh

[Stéphane Delaune]

---
 misc/migration_tools/rebuild_zebra_sliced.zsh |  174 +++++++++++++++++++++++++
 1 files changed, 174 insertions(+), 0 deletions(-)
 create mode 100755 misc/migration_tools/rebuild_zebra_sliced.zsh

diff --git a/misc/migration_tools/rebuild_zebra_sliced.zsh b/misc/migration_tools/rebuild_zebra_sliced.zsh
new file mode 100755
index 0000000..c015343
--- /dev/null
+++ b/misc/migration_tools/rebuild_zebra_sliced.zsh
@@ -0,0 +1,174 @@
+#!/usr/bin/zsh
+
+# Copyright 2011 BibLibre SARL
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+INCREMENT=$1
+BIBLIOSTART=$2
+BIBLIOEND=$3
+TEMPDIRECTORY=$4
+RMLOGS=$5
+
+#echo " | $INCREMENT , $BIBLIOSTART , $BIBLIOEND , $TEMPDIRECTORY , $RMLOGS | ";
+# this script rebuild the zebra index recursively
+# INCREMENT specify how many records we must try at once
+# BIBLIOSTART is the record number to BIBLIOSTART on
+# BIBLIOEND is the record number to BIBLIOEND on, if it's greater than the last biblionumber it will be redefine
+# TEMPDIRECTORY define a temporary directory to rebuild_zebra
+# RMLOGS is used to remove files into logs/, it's use and define by script itself so you have not to define it
+# At the BIBLIOEND of each "chunk", this script checks if the indexing process has been OK
+# if it hasn't, the slice is splitted in 10, and the reindexing is called again on each smaller chunk
+# if the increment goes to 1, it means we tried to reindex 1 by 1, and the failing indexing concern wrong records
+
+# the logs are stored in a directory called logs/ that must be a subdirectory of reindex.zsh
+
+# at the BIBLIOEND of the script, just type :
+#grep -l "previous transaction" `ls rebuild1.*.err`
+# the result contains all the biblios that have not been indexed
+# WARNING : the numbers are not the biblionumber but the record number, they can be reached by :
+# SELECT biblionumber FROM biblio LIMIT YourNumberHere,1;
+
+# EXAMPLE to run the script on a 800 000 biblios database :
+# ./reindex.zsh 50000 0 800000
+# will reindex the DB, BIBLIOSTARTing with chunks of 50k biblios
+#if { grep -E "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; then
+
+lastbiblionumber=`perl -e '#!/usr/bin/perl
+use C4::Context;
+my $dbh = C4::Context->dbh;
+my $querylastbiblionumber = "SELECT max(biblionumber) FROM biblio;";
+my $sthlastbiblionumber   = $dbh->prepare($querylastbiblionumber);
+$sthlastbiblionumber->execute();
+my ( $lastbiblionumber ) = $sthlastbiblionumber->fetchrow_array; print $lastbiblionumber;'`
+#echo $lastbiblionumber;
+let "maxbiblionumber = $lastbiblionumber + 1"
+if [ $# = 4 ]
+then
+    RMLOGS=yes
+elif [ $# = 3 ]
+then
+    RMLOGS=yes
+    TEMPDIRECTORY=/tmp/rebuild
+elif [ $# = 2 ]
+then
+    RMLOGS=yes
+    TEMPDIRECTORY=/tmp/rebuild
+    BIBLIOEND=$lastbiblionumber
+elif [ $# = 1 ]
+then
+    RMLOGS=yes
+    TEMPDIRECTORY=/tmp/rebuild
+    BIBLIOSTART=0
+    BIBLIOEND=$lastbiblionumber
+elif [ $# = 0 ]
+then
+    RMLOGS=yes
+    TEMPDIRECTORY=/tmp/rebuild
+    INCREMENT=10000
+    BIBLIOSTART=0
+    BIBLIOEND=$lastbiblionumber
+fi
+if [[ $INCREMENT =~ ^10*$ ]]
+then
+else
+    echo "The first argument (INCREMENT) must be 1 or a multiple of 10"
+    exit 2
+fi
+if [[ $BIBLIOSTART =~ ^[0-9]*$ ]]
+then
+else
+    echo "The second argument (BIBLIOSTART) must be an integer"
+    exit 2
+fi
+if [[ $BIBLIOEND =~ ^[0-9]*$ ]]
+then
+else
+    echo "The third argument (BIBLIOEND) must be an integer"
+    exit 2
+fi
+if [ $BIBLIOSTART -ge $BIBLIOEND ]
+then
+    echo "The second argument (BIBLIOSTART) must be lower than the third argument (BIBLIOEND)"
+    exit 2
+fi
+if [[ $maxbiblionumber =~ ^[0-9]*$ ]]
+then
+else
+    echo "error : last biblionumber is not an integer"
+    exit 2
+fi
+if [ $BIBLIOEND -gt $lastbiblionumber ];
+then
+    BIBLIOEND=$lastbiblionumber
+fi
+ls `dirname $0`/logs/ >/dev/null 2>&1
+if [ $? != 0 ]
+then
+    mkdir `dirname $0`/logs
+else
+    if [ $RMLOGS = "yes" ]
+    then
+        rm `dirname $0`/logs/*
+    fi
+fi
+ls $TEMPDIRECTORY >/dev/null 2>&1
+if [ $? != 0 ]
+then
+    mkdir $TEMPDIRECTORY
+    if [ $? != 0 ]
+    then
+        echo "can't create dir $TEMPDIRECTORY"
+        exit 2
+    fi
+fi
+#/home/koha/src/misc/migration_tools/rebuild_zebra.pl -r -b -v -x -nosanitize -length 1 -offset 1
+for ((i=$BIBLIOSTART ; i<=$BIBLIOEND ; i=i+$INCREMENT))
+do
+    echo "I = " $i "with increment " $INCREMENT
+    #echo "BIBLIOSTART $BIBLIOSTART INCREMENT $INCREMENT BIBLIOEND $BIBLIOEND i $i"
+    #echo "`dirname $0`/rebuild_zebra.pl -b -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i -v log"
+    `dirname $0`/rebuild_zebra.pl -b -v -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i > `dirname $0`/logs/rebuild$INCREMENT.$i.log 2> `dirname $0`/logs/rebuild$INCREMENT.$i.err
+    if (($INCREMENT >1 ));
+    then
+        if { grep -q "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ;
+        then
+            echo "I must split $i (increment $INCREMENT) because previous transaction didn't reach commit"
+            ((subincrement=$INCREMENT/10))
+            if (($i+$INCREMENT >= $BIBLIOEND ));
+            then
+                ((newBIBLIOEND=$BIBLIOEND))
+            else
+                ((newBIBLIOEND=$i+$INCREMENT))
+            fi
+            echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no"
+            $0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no
+        elif { ! grep -q "Records: $INCREMENT" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ;
+        then
+            echo "I must split $i (increment $INCREMENT) because index was uncomplete, less than $INCREMENT records indexed"
+            ((subincrement=$INCREMENT/10))
+            if (($i+$INCREMENT >= $BIBLIOEND ));
+            then
+                ((newBIBLIOEND=$BIBLIOEND))
+            else
+                ((newBIBLIOEND=$i+$INCREMENT))
+            fi
+            echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no"
+            $0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no
+        fi
+    fi
+done
+exit 0
-- 
1.7.0.4