[Koha-cvs] CVS: koha/misc/translator TmplTokenizer.pm,1.4,1.5

[Ambrose Li]

Update of /cvsroot/koha/koha/misc/translator
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv13338

Modified Files:
	TmplTokenizer.pm 
Log Message:
Still more bugfixes for my own bugs.

$readahead is now an array @readahead which can contain TmplToken objects,
so "ungetting" tokens should not disturb the line number counter any more.


Index: TmplTokenizer.pm
===================================================================
RCS file: /cvsroot/koha/koha/misc/translator/TmplTokenizer.pm,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -r1.4 -r1.5
*** TmplTokenizer.pm	17 Feb 2004 02:45:27 -0000	1.4
--- TmplTokenizer.pm	17 Feb 2004 03:17:48 -0000	1.5
***************
*** 85,89 ****
  # End of the hideous stuff
  
! use vars qw( $readahead $lc_0 $lc $syntaxerror_p );
  use vars qw( $cdata_mode_p $cdata_close );
  
--- 85,89 ----
  # End of the hideous stuff
  
! use vars qw( @readahead $lc_0 $lc $syntaxerror_p );
  use vars qw( $cdata_mode_p $cdata_close );
  
***************
*** 152,171 ****
      my($it, $kind);
      my $eof_p = 0;
!     if (!defined $readahead || !length $readahead) {
  	my $next = scalar <$h>;
  	$eof_p = !defined $next;
  	if (!$eof_p) {
  	    $lc += 1;
! 	    $readahead .= $next;
  	}
      }
      $lc_0 = $lc;			# remember line number of first line
!     if ($eof_p && !length $readahead) {	# nothing left to do
  	;
!     } elsif ($readahead =~ /^\s+/s) {	# whitespace
! 	($kind, $it, $readahead) = (TmplTokenType::TEXT, $&, $');
      # FIXME the following (the [<\s] part) is an unreliable HACK :-(
!     } elsif ($readahead =~ /^(?:[^<]|<[<\s])+/s) {	# non-space normal text
! 	($kind, $it, $readahead) = (TmplTokenType::TEXT, $&, $');
  	warn_normal "Unescaped < in $it\n", $lc_0
  		if !$cdata_mode_p && $it =~ /</s;
--- 152,176 ----
      my($it, $kind);
      my $eof_p = 0;
!     pop @readahead if @readahead && !ref $readahead[$#readahead]
! 	    && !length $readahead[$#readahead];
!     if (!@readahead) {
  	my $next = scalar <$h>;
  	$eof_p = !defined $next;
  	if (!$eof_p) {
  	    $lc += 1;
! 	    push @readahead, $next;
  	}
      }
      $lc_0 = $lc;			# remember line number of first line
!     if (@readahead && ref $readahead[$#readahead]) {	# TmplToken object
! 	my $t = pop @readahead;
! 	($it, $kind, local $lc) = ($t->string, $t->type, $t->line_number);
!     } elsif ($eof_p && !@readahead) {	# nothing left to do
  	;
!     } elsif ($readahead[$#readahead] =~ /^\s+/s) {	# whitespace
! 	($kind, $it, $readahead[$#readahead]) = (TmplTokenType::TEXT, $&, $');
      # FIXME the following (the [<\s] part) is an unreliable HACK :-(
!     } elsif ($readahead[$#readahead] =~ /^(?:[^<]|<[<\s])+/s) {	# non-space normal text
! 	($kind, $it, $readahead[$#readahead]) = (TmplTokenType::TEXT, $&, $');
  	warn_normal "Unescaped < in $it\n", $lc_0
  		if !$cdata_mode_p && $it =~ /</s;
***************
*** 174,190 ****
  	for (;;) {
  	    if ($cdata_mode_p) {
! 		if ($readahead =~ /^$cdata_close/) {
! 		    ($kind, $it, $readahead) = (TmplTokenType::TAG, $&, $');
  		    $ok_p = 1;
  		} else {
! 		    ($kind, $it, $readahead) = (TmplTokenType::TEXT, $readahead, undef);
  		    $ok_p = 1;
  		}
! 	    } elsif ($readahead =~ /^$re_tag_compat/os) {
! 		($kind, $it, $readahead) = (TmplTokenType::TAG, "$1>", $3);
  		$ok_p = 1;
  		warn_normal "SGML \"closed start tag\" notation: $1<\n", $lc_0 if $2 eq '';
! 	    } elsif ($readahead =~ /^<!--(?:(?!-->).)*-->/s) {
! 		($kind, $it, $readahead) = (TmplTokenType::COMMENT, $&, $');
  		$ok_p = 1;
  		warn_normal "Syntax error in comment: $&\n", $lc_0;
--- 179,195 ----
  	for (;;) {
  	    if ($cdata_mode_p) {
! 		if ($readahead[$#readahead] =~ /^$cdata_close/) {
! 		    ($kind, $it, $readahead[$#readahead]) = (TmplTokenType::TAG, $&, $');
  		    $ok_p = 1;
  		} else {
! 		    ($kind, $it) = (TmplTokenType::TEXT, pop @readahead);
  		    $ok_p = 1;
  		}
! 	    } elsif ($readahead[$#readahead] =~ /^$re_tag_compat/os) {
! 		($kind, $it, $readahead[$#readahead]) = (TmplTokenType::TAG, "$1>", $3);
  		$ok_p = 1;
  		warn_normal "SGML \"closed start tag\" notation: $1<\n", $lc_0 if $2 eq '';
! 	    } elsif ($readahead[$#readahead] =~ /^<!--(?:(?!-->).)*-->/s) {
! 		($kind, $it, $readahead[$#readahead]) = (TmplTokenType::COMMENT, $&, $');
  		$ok_p = 1;
  		warn_normal "Syntax error in comment: $&\n", $lc_0;
***************
*** 196,200 ****
  	last if $eof_p;
  	    $lc += 1;
! 	    $readahead .= $next;
  	}
  	if ($kind ne TmplTokenType::TAG) {
--- 201,205 ----
  	last if $eof_p;
  	    $lc += 1;
! 	    $readahead[$#readahead] .= $next;
  	}
  	if ($kind ne TmplTokenType::TAG) {
***************
*** 210,214 ****
  	}
  	if (!$ok_p && $eof_p) {
! 	    ($kind, $it, $readahead) = (TmplTokenType::UNKNOWN, $readahead, undef);
  	    $syntaxerror_p = 1;
  	}
--- 215,219 ----
  	}
  	if (!$ok_p && $eof_p) {
! 	    ($kind, $it, $readahead[$#readahead]) = (TmplTokenType::UNKNOWN, $readahead[$#readahead], undef);
  	    $syntaxerror_p = 1;
  	}
***************
*** 235,239 ****
  	last if !defined $next;
  	    if (defined $next && $next->string =~ /$cdata_close/i) {
! 		($lc, $readahead) = ($lc_prev, $next->string . $readahead);
  		$cdata_mode_p = 0;
  	    }
--- 240,245 ----
  	last if !defined $next;
  	    if (defined $next && $next->string =~ /$cdata_close/i) {
! 		push @readahead, $next; # push the entire TmplToken object
! 		#$lc = $lc_prev; XXX
  		$cdata_mode_p = 0;
  	    }