web/translations/translate.pl

#!/usr/bin/perl
#
# Routino translation replacement Perl script
#
# Part of the Routino routing software.
#
# This file Copyright 2014 Andrew M. Bishop
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

# Use the perl encoding/decoding functions
use Encode qw(decode encode);

use strict;

# Constants

my @translation_files=(<translation.*.txt>);
my $xml_file="../data/translations.xml";
my $output_dir="../www/routino";

my @html_template_files=(<*.html>);

my %languages=();
my %translations=();


# Read in the translations (HTML)

foreach my $translation_file (@translation_files)
  {
   $translation_file =~ m%translation.([^.]+).txt%;

   # Add to list of languages

   my $language=$1;

   if(! defined $languages{$language})
     {
      $languages{$language}=1;

      $translations{$language}={};
      $translations{$language}->{codes}={};
      $translations{$language}->{html}=0;
      $translations{$language}->{xml}=0;
     }

   # Process the file

   open(FILE,"<$translation_file");

   while(<FILE>)
     {
      s%\r*\n%%;

      next if(m%^#%);
      next if(m%^$%);

      # Single line entries

      if(m%\@\@%)
        {
         my($code,$text)=split("\t");

         if(defined $translations{$language}->{codes}->{$code})
           {
            print STDERR "Language: $language DUPLICATED codeword '$code'\n";
           }
         else
           {
            $translations{$language}->{html}++;
            $translations{$language}->{codes}->{$code}={};
            $translations{$language}->{codes}->{$code}->{text}=encode('ascii',decode('UTF-8',$text),Encode::FB_HTMLCREF);
            $translations{$language}->{codes}->{$code}->{used}=0;
           }
        }

      # Multi-line entries

      if(m%\$\$([^\$]+)\$\$%)
        {
         my($code,$text)=("@@".$1."@@","");

         while(<FILE>)
           {
            last if(m%\$\$%);

            $text=$text.$_;
           }

         $text =~ s%\r*\n$%%;

         if(defined $translations{$language}->{codes}->{$code})
           {
            print STDERR "Language: $language DUPLICATED codeword '$code'\n";
           }
         else
           {
            $translations{$language}->{html}++;
            $translations{$language}->{codes}->{$code}={};
            $translations{$language}->{codes}->{$code}->{text}=encode('ascii',decode('UTF-8',$text),Encode::FB_HTMLCREF);
            $translations{$language}->{codes}->{$code}->{used}=0;
           }
        }
     }

   close(FILE);
  }


# Read in the translations (XML)

open(XML,"<$xml_file");

my $language="";

while(<XML>)
  {
   if(m%<language +lang=["']([^"']+)["'] *>%)
     {
      # Add to list of languages

      $language=$1;

      if(! defined $languages{$language})
        {
         $languages{$language}=1;

         $translations{$language}={};
         $translations{$language}->{codes}={};
         $translations{$language}->{html}=0;
         $translations{$language}->{xml}=0;
        }

      next;
     }

   $language="" if(m%</language *>%);

   next if(!$language);

   # Count the phrases

   if(m%<[a-z]+% && m% +(text|string)=["']([^"']+)["']%)
     {
      $translations{$language}->{xml}++;
     }
  }

close(XML);


# Sort out the languages

my @languages=();

push(@languages,"en");

foreach my $language (sort (keys %languages))
  {
   push(@languages,$language) if($language ne "en");
  }


# Create the files

foreach my $html_template_file (@html_template_files)
  {
   foreach my $language (@languages)
     {
      next if(!$translations{$language}->{html});

      print "Language: $language File: $html_template_file\n";

      my $language_meta=0;
      my $language_meta_string="";

      open(HTML_IN ,"<$html_template_file");
      open(HTML_OUT,">$output_dir/$html_template_file.$language");

      while(<HTML_IN>)
        {
         my $line=$_;

         # Language selection - special handling

         if($line =~ m%\$\$LANGUAGES-META\$\$%)
           {
            $language_meta=1-$language_meta;

            if($language_meta==0)
              {
               foreach my $language2 (@languages)
                 {
                  my $LANGUAGE2=$language2;
                  $LANGUAGE2 =~ tr%a-z%A-Z%;

                  $line=$language_meta_string;

                  if($language eq $language2)
                    {
                     $line =~ s%CHECKED-XX%checked%g;
                    }
                  else
                    {
                     $line =~ s%CHECKED-XX%%g;
                    }

                  $line =~ s%xx%$language2%g;
                  $line =~ s%XX%$LANGUAGE2%g;

                  if(!$translations{$language2}->{html})
                    {
                     $line =~ s%<a.+</a>%%;
                    }

                  if(!$translations{$language2}->{xml})
                    {
                     $line =~ s%<input .+>%%;
                    }

                  foreach my $code (keys $translations{$language}->{codes})
                    {
                     if($line =~ s%$code%$translations{$language2}->{codes}->{$code}->{text}%g)
                       {$translations{$language2}->{codes}->{$code}->{used} = 1;}
                    }

                  # Remove un-needed spaces

                  $line =~ s%[\t ]+% %g;
                  $line =~ s%\n %\n%g;
                  $line =~ s%^ %%g;

                  print HTML_OUT $line;
                 }
              }

            next;
           }

         if($language_meta)
           {
            $language_meta_string.=$line;
            next;
           }

         # Replace with translated phrases

         foreach my $code (keys $translations{$language}->{codes})
           {
            if($line =~ s%$code%$translations{$language}->{codes}->{$code}->{text}%g)
              {$translations{$language}->{codes}->{$code}->{used} = 1;}
           }

         # Replace what is left with English phrases

         foreach my $code (keys $translations{$languages[0]}->{codes})
           {
            $line =~ s%$code%$translations{$languages[0]}->{codes}->{$code}->{text}%g;
           }

         if($line =~ m%\@\@%)
           {
            print STDERR "   Unmatched codeword in line: $line";
           }

         # Remove un-needed spaces

         $line =~ s%[\t ]+% %g;
         $line =~ s%\n %\n%g;
         $line =~ s%^ %%g;

         print HTML_OUT $line;
        }

      close(HTML_IN);
      close(HTML_OUT);
     }
  }


# Check the languages

foreach my $language (@languages)
  {
   foreach my $code (keys $translations{$language}->{codes})
     {
      if(! $translations{$language}->{codes}->{$code}->{used})
        {
         print STDERR "Language: $language UNUSED codeword: $code\n";
        }
     }
  }


# Print the translation coverage

print "\n";

print "Translation Coverage\n";
print "====================\n";

print "\n";
print "            Number      Fraction\n";
print "Language  HTML   XML   HTML   XML\n";
print "--------  ----   ---   ----   ---\n";

foreach my $language (@languages)
  {
   printf("%-6s     %3d   %3d  %4.0f%% %4.0f%%\n",$language,
                                                  $translations{$language}->{html},
                                                  $translations{$language}->{xml},
                                                  100.0*$translations{$language}->{html}/$translations{$languages[0]}->{html},
                                                  100.0*$translations{$language}->{xml} /$translations{$languages[0]}->{xml});
  }
1	amb	1521	#!/usr/bin/perl
2			#
3			# Routino translation replacement Perl script
4			#
5			# Part of the Routino routing software.
6			#
7			# This file Copyright 2014 Andrew M. Bishop
8			#
9			# This program is free software: you can redistribute it and/or modify
10			# it under the terms of the GNU Affero General Public License as published by
11			# the Free Software Foundation, either version 3 of the License, or
12			# (at your option) any later version.
13			#
14			# This program is distributed in the hope that it will be useful,
15			# but WITHOUT ANY WARRANTY; without even the implied warranty of
16			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17			# GNU Affero General Public License for more details.
18			#
19			# You should have received a copy of the GNU Affero General Public License
20			# along with this program. If not, see <http://www.gnu.org/licenses/>.
21			#
22
23	amb	1524	# Use the perl encoding/decoding functions
24			use Encode qw(decode encode);
25
26	amb	1521	use strict;
27
28			# Constants
29
30			my @translation_files=(<translation.*.txt>);
31			my $xml_file="../data/translations.xml";
32			my $output_dir="../www/routino";
33
34			my @html_template_files=(<*.html>);
35
36			my %languages=();
37			my %translations=();
38
39
40			# Read in the translations (HTML)
41
42			foreach my $translation_file (@translation_files)
43			{
44			$translation_file =~ m%translation.([^.]+).txt%;
45
46			# Add to list of languages
47
48			my $language=$1;
49
50			if(! defined $languages{$language})
51			{
52			$languages{$language}=1;
53
54			$translations{$language}={};
55			$translations{$language}->{codes}={};
56			$translations{$language}->{html}=0;
57			$translations{$language}->{xml}=0;
58			}
59
60			# Process the file
61
62			open(FILE,"<$translation_file");
63
64			while(<FILE>)
65			{
66			s%\r*\n%%;
67
68			next if(m%^#%);
69			next if(m%^$%);
70
71			# Single line entries
72
73			if(m%\@\@%)
74			{
75			my($code,$text)=split("\t");
76
77			if(defined $translations{$language}->{codes}->{$code})
78			{
79			print STDERR "Language: $language DUPLICATED codeword '$code'\n";
80			}
81			else
82			{
83			$translations{$language}->{html}++;
84			$translations{$language}->{codes}->{$code}={};
85	amb	1524	$translations{$language}->{codes}->{$code}->{text}=encode('ascii',decode('UTF-8',$text),Encode::FB_HTMLCREF);
86	amb	1521	$translations{$language}->{codes}->{$code}->{used}=0;
87			}
88			}
89
90			# Multi-line entries
91
92			if(m%\$\$([^\$]+)\$\$%)
93			{
94			my($code,$text)=("@@".$1."@@","");
95
96			while(<FILE>)
97			{
98			last if(m%\$\$%);
99
100			$text=$text.$_;
101			}
102
103			$text =~ s%\r*\n$%%;
104
105			if(defined $translations{$language}->{codes}->{$code})
106			{
107			print STDERR "Language: $language DUPLICATED codeword '$code'\n";
108			}
109			else
110			{
111			$translations{$language}->{html}++;
112			$translations{$language}->{codes}->{$code}={};
113	amb	1524	$translations{$language}->{codes}->{$code}->{text}=encode('ascii',decode('UTF-8',$text),Encode::FB_HTMLCREF);
114	amb	1521	$translations{$language}->{codes}->{$code}->{used}=0;
115			}
116			}
117			}
118
119			close(FILE);
120			}
121
122
123			# Read in the translations (XML)
124
125			open(XML,"<$xml_file");
126
127			my $language="";
128
129			while(<XML>)
130			{
131			if(m%<language +lang=["']([^"']+)["'] *>%)
132			{
133			# Add to list of languages
134
135			$language=$1;
136
137			if(! defined $languages{$language})
138			{
139			$languages{$language}=1;
140
141			$translations{$language}={};
142			$translations{$language}->{codes}={};
143			$translations{$language}->{html}=0;
144			$translations{$language}->{xml}=0;
145			}
146
147			next;
148			}
149
150			$language="" if(m%</language *>%);
151
152			next if(!$language);
153
154			# Count the phrases
155
156			if(m%<[a-z]+% && m% +(text\|string)=["']([^"']+)["']%)
157			{
158			$translations{$language}->{xml}++;
159			}
160			}
161
162			close(XML);
163
164
165			# Sort out the languages
166
167			my @languages=();
168
169			push(@languages,"en");
170
171			foreach my $language (sort (keys %languages))
172			{
173			push(@languages,$language) if($language ne "en");
174			}
175
176
177			# Create the files
178
179			foreach my $html_template_file (@html_template_files)
180			{
181			foreach my $language (@languages)
182			{
183			next if(!$translations{$language}->{html});
184
185			print "Language: $language File: $html_template_file\n";
186
187			my $language_meta=0;
188			my $language_meta_string="";
189
190			open(HTML_IN ,"<$html_template_file");
191			open(HTML_OUT,">$output_dir/$html_template_file.$language");
192
193			while(<HTML_IN>)
194			{
195			my $line=$_;
196
197			# Language selection - special handling
198
199			if($line =~ m%\$\$LANGUAGES-META\$\$%)
200			{
201			$language_meta=1-$language_meta;
202
203			if($language_meta==0)
204			{
205			foreach my $language2 (@languages)
206			{
207			my $LANGUAGE2=$language2;
208			$LANGUAGE2 =~ tr%a-z%A-Z%;
209
210			$line=$language_meta_string;
211
212			if($language eq $language2)
213			{
214			$line =~ s%CHECKED-XX%checked%g;
215			}
216			else
217			{
218			$line =~ s%CHECKED-XX%%g;
219			}
220
221			$line =~ s%xx%$language2%g;
222			$line =~ s%XX%$LANGUAGE2%g;
223
224			if(!$translations{$language2}->{html})
225			{
226			$line =~ s%<a.+</a>%%;
227			}
228
229			if(!$translations{$language2}->{xml})
230			{
231			$line =~ s%<input .+>%%;
232			}
233
234			foreach my $code (keys $translations{$language}->{codes})
235			{
236			if($line =~ s%$code%$translations{$language2}->{codes}->{$code}->{text}%g)
237			{$translations{$language2}->{codes}->{$code}->{used} = 1;}
238			}
239
240			# Remove un-needed spaces
241
242			$line =~ s%[\t ]+% %g;
243			$line =~ s%\n %\n%g;
244			$line =~ s%^ %%g;
245
246			print HTML_OUT $line;
247			}
248			}
249
250			next;
251			}
252
253			if($language_meta)
254			{
255			$language_meta_string.=$line;
256			next;
257			}
258
259			# Replace with translated phrases
260
261			foreach my $code (keys $translations{$language}->{codes})
262			{
263			if($line =~ s%$code%$translations{$language}->{codes}->{$code}->{text}%g)
264			{$translations{$language}->{codes}->{$code}->{used} = 1;}
265			}
266
267			# Replace what is left with English phrases
268
269			foreach my $code (keys $translations{$languages[0]}->{codes})
270			{
271			$line =~ s%$code%$translations{$languages[0]}->{codes}->{$code}->{text}%g;
272			}
273
274			if($line =~ m%\@\@%)
275			{
276			print STDERR " Unmatched codeword in line: $line";
277			}
278
279			# Remove un-needed spaces
280
281			$line =~ s%[\t ]+% %g;
282			$line =~ s%\n %\n%g;
283			$line =~ s%^ %%g;
284
285			print HTML_OUT $line;
286			}
287
288			close(HTML_IN);
289			close(HTML_OUT);
290			}
291			}
292
293
294			# Check the languages
295
296			foreach my $language (@languages)
297			{
298			foreach my $code (keys $translations{$language}->{codes})
299			{
300			if(! $translations{$language}->{codes}->{$code}->{used})
301			{
302			print STDERR "Language: $language UNUSED codeword: $code\n";
303			}
304			}
305			}
306
307
308			# Print the translation coverage
309
310			print "\n";
311
312			print "Translation Coverage\n";
313			print "====================\n";
314
315			print "\n";
316			print " Number Fraction\n";
317			print "Language HTML XML HTML XML\n";
318			print "-------- ---- --- ---- ---\n";
319
320			foreach my $language (@languages)
321			{
322			printf("%-6s %3d %3d %4.0f%% %4.0f%%\n",$language,
323			$translations{$language}->{html},
324			$translations{$language}->{xml},
325			100.0*$translations{$language}->{html}/$translations{$languages[0]}->{html},
326			100.0*$translations{$language}->{xml} /$translations{$languages[0]}->{xml});
327			}