[Koha-cvs] CVS: koha/misc/translator TmplTokenizer.pm,1.20,1.21

[Ambrose C. LI]

Update of /cvsroot/koha/koha/misc/translator
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv2759

Modified Files:
	TmplTokenizer.pm 
Log Message:
Try to relax the criteria for allowing groups of tokens without TMPL_VAR
to be combined together into one string. This seems to have the desired
effect (that "<b>foo</b> bar" type strings are now recognized in one piece).

However, "<h1>foo</h1>\nexplanation"-type things may now also be (arguably
wrongly) recognized as one piece.


Index: TmplTokenizer.pm
===================================================================
RCS file: /cvsroot/koha/koha/misc/translator/TmplTokenizer.pm,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -r1.20 -r1.21
*** TmplTokenizer.pm	22 Feb 2004 08:18:27 -0000	1.20
--- TmplTokenizer.pm	22 Feb 2004 09:04:53 -0000	1.21
***************
*** 460,466 ****
  	    && $structure[$#structure]->string =~ /^<\//s) {
  	my $has_other_tags_p = 0;
- 	    printf STDERR "last token %d is type %s: %s\n", $#structure, $structure[$#structure]->type->to_string, $structure[$#structure]->string;
  	for (my $i = 0; $i < $#structure; $i += 1) {
- 	    printf STDERR "token %d is type %s: %s\n", $i, $structure[$i]->type->to_string, $structure[$i]->string;
  	    $has_other_tags_p = 1 if $structure[$i]->type == TmplTokenType::TAG;
  	last if $has_other_tags_p;
--- 460,464 ----
***************
*** 479,482 ****
--- 477,504 ----
  }
  
+ sub looks_plausibly_like_groupable_text_p (@) {
+     my @structure = @_;
+     # The text would look plausibly groupable if all open tags are also closed.
+     my @tags = ();
+     my $error_p = 0;
+     for (my $i = 0; $i <= $#structure; $i += 1) {
+ 	if ($structure[$i]->type == TmplTokenType::TAG) {
+ 	    if ($structure[$i]->string =~ /^<([A-Z0-9]+)/i) {
+ 		push @tags, lc($1);
+ 	    } elsif ($structure[$i]->string =~ /^<\/([A-Z0-9]+)/i) {
+ 		if (@tags && lc($1) eq $tags[$#tags]) {
+ 		    pop @tags;
+ 		} else {
+ 		    $error_p = 1;
+ 		}
+ 	    }
+ 	} elsif ($structure[$i]->type != TmplTokenType::TEXT) {
+ 	    $error_p = 1;
+ 	}
+     last if $error_p;
+     }
+     return !$error_p && !@tags;
+ }
+ 
  sub next_token {
      my $this = shift;
***************
*** 484,487 ****
--- 506,512 ----
      my $it;
      $this->{_queue} = [] unless defined $this->{_queue};
+ 
+     # Don't reparse anything in the queue. We can put a parametrized token
+     # there if we need to, however.
      if (@{$this->{_queue}}) {
  	$it = pop @{$this->{_queue}};
***************
*** 533,537 ****
  		$it->set_form( $form );
  		$it->set_children( @structure );
! 	    } elsif ($nonblank_text_p && $structure[0]->type == TmplTokenType::TEXT && $structure[$#structure]->type == TmplTokenType::TEXT) {
  		# Combine the strings
  		my $string = join('', map { $_->string } @structure);
--- 558,564 ----
  		$it->set_form( $form );
  		$it->set_children( @structure );
! 	    } elsif ($nonblank_text_p
! 	    && looks_plausibly_like_groupable_text_p( @structure )
! 	    && $structure[$#structure]->type == TmplTokenType::TEXT) {
  		# Combine the strings
  		my $string = join('', map { $_->string } @structure);