web/translations/translate.pl

#!/usr/bin/perl
#
# Routino translation replacement Perl script
#
# Part of the Routino routing software.
#
# This file Copyright 2014-2015 Andrew M. Bishop
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

use strict;

# Constants

my @translation_files=(<translation.*.txt>);
my $xml_output_file="../../xml/routino-translations.xml";
my $html_output_dir="../www/routino";

my @html_template_files=(<*.html>);

my %languages=();
my %translations=();


# Sort so that English is first

@translation_files=sort {if($a eq "translation.en.txt"){return -1;} if($b eq "translation.en.txt"){return 1;} return $a <=> $b;} @translation_files;

# Read in the translations

foreach my $translation_file (@translation_files)
  {
   $translation_file =~ m%translation.([^.]+).txt%;

   # Add to list of languages

   my $language=$1;

   if(! defined $languages{$language})
     {
      $languages{$language}=1;

      $translations{$language}={};
      $translations{$language}->{codes}={};
      $translations{$language}->{html}=0;
      $translations{$language}->{xml}=0;
     }

   # Process the file

   open(FILE,"<$translation_file");

   while(<FILE>)
     {
      s%\r*\n%%;

      next if(m%^#%);
      next if(m%^$%);

      # Single line HTML entries

      if(m%\@\@%)
        {
         my($code,$text)=split("\t");

         if(defined $translations{$language}->{codes}->{$code})
           {
            print STDERR "Language: $language DUPLICATED codeword '$code'\n";
           }
         else
           {
            $translations{$language}->{html}++;
            $translations{$language}->{codes}->{$code}={};
            $translations{$language}->{codes}->{$code}->{text}=$text;
            $translations{$language}->{codes}->{$code}->{usedX}=0;
            $translations{$language}->{codes}->{$code}->{usedR}=0;
            $translations{$language}->{codes}->{$code}->{usedV}=0;
           }
        }

      # Multi-line HTML entries

      if(m%(\$\$[^\$]+\$\$)%)
        {
         my($code,$text)=($1,"");

         while(<FILE>)
           {
            last if(m%\$\$%);

            $text=$text.$_;
           }

         $text =~ s%\r*\n$%%;

         if(defined $translations{$language}->{codes}->{$code})
           {
            print STDERR "Language: $language DUPLICATED codeword '$code'\n";
           }
         else
           {
            $translations{$language}->{html}++;
            $translations{$language}->{codes}->{$code}={};
            $translations{$language}->{codes}->{$code}->{text}=$text;
            $translations{$language}->{codes}->{$code}->{usedX}=0;
            $translations{$language}->{codes}->{$code}->{usedR}=0;
            $translations{$language}->{codes}->{$code}->{usedV}=0;
           }
        }

      # Single line XML entries

      if(m%\%\%%)
        {
         my($code,$text)=split("\t");

         if(defined $translations{$language}->{codes}->{$code})
           {
            print STDERR "Language: $language DUPLICATED codeword '$code'\n";
           }
         else
           {
            $translations{$language}->{xml}++;
            $translations{$language}->{codes}->{$code}={};
            $translations{$language}->{codes}->{$code}->{text}=$text;
            $translations{$language}->{codes}->{$code}->{usedX}=0;
            $translations{$language}->{codes}->{$code}->{usedR}=0;
            $translations{$language}->{codes}->{$code}->{usedV}=0;
           }

         my($n_strings_en)=$translations{en}->{codes}->{$code}->{text} =~ s/%s/%s/g;
         my($n_strings)   =$text =~ s/%s/%s/g;

         if($n_strings != $n_strings_en)
           {
            print STDERR "Language: $language WRONG number of '%s' in text '$text' ($translations{en}->{codes}->{$code}->{text})\n";
           }
        }
     }

   close(FILE);
  }


# Sort out the languages

my @languages=();

push(@languages,"en");

foreach my $language (sort (keys %languages))
  {
   push(@languages,$language) if($language ne "en");
  }


# Create the HTML files

foreach my $html_template_file (@html_template_files)
  {
   my $usedtype="";

   $usedtype="R" if($html_template_file =~ m%router%);
   $usedtype="V" if($html_template_file =~ m%visualiser%);

   foreach my $language (@languages)
     {
      next if(!$translations{$language}->{html});

      print "Language: $language File: $html_template_file\n";

      my $language_meta=0;
      my $language_meta_string="";

      open(HTML_IN ,"<$html_template_file");
      open(HTML_OUT,">$html_output_dir/$html_template_file.$language");

      while(<HTML_IN>)
        {
         my $line=$_;

         # Language selection - special handling

         if($line =~ m%\*\*LANGUAGES-META\*\*%)
           {
            $language_meta=1-$language_meta;

            if($language_meta==0)
              {
               foreach my $language2 (@languages)
                 {
                  my $LANGUAGE2=$language2;
                  $LANGUAGE2 =~ tr%a-z%A-Z%;

                  $line=$language_meta_string;

                  if($language eq $language2)
                    {
                     $line =~ s%~~CHECKED~~%checked%g;
                    }
                  else
                    {
                     $line =~ s%~~CHECKED~~%%g;
                    }

                  $line =~ s%~~lang~~%$language2%g;
                  $line =~ s%~~LANG~~%$LANGUAGE2%g;

                  if(!$translations{$language2}->{html})
                    {
                     $line =~ s%<a.+</a>%%;
                    }

                  if(!$translations{$language2}->{xml})
                    {
                     $line =~ s%<input .+>%%;
                    }

                  foreach my $code (keys %{$translations{$language2}->{codes}})
                    {
                     if($line =~ s%$code%$translations{$language2}->{codes}->{$code}->{text}%g)
                       {$translations{$language2}->{codes}->{$code}->{"used$usedtype"} = 1;}
                    }

                  if($line =~ m%((\@\@|\$\$|\*\*|\~\~)[^\@\$*~]+(\@\@|\$\$|\*\*|\~\~))%)
                    {
                     print STDERR "   Unmatched codeword '$1' in line: $line";
                    }

                  # Remove un-needed spaces

                  $line =~ s%[\t ]+% %g;
                  $line =~ s%\n %\n%g;
                  $line =~ s%^ %%g;

                  print HTML_OUT $line;
                 }
              }

            next;
           }

         if($language_meta)
           {
            $language_meta_string.=$line;
            next;
           }

         # Replace with translated phrases

         foreach my $code (keys %{$translations{$language}->{codes}})
           {
            if($line =~ s%\Q$code\E%$translations{$language}->{codes}->{$code}->{text}%g)
              {$translations{$language}->{codes}->{$code}->{"used$usedtype"} = 1;}
           }

         # Replace what is left with English phrases

         foreach my $code (keys %{$translations{$languages[0]}->{codes}})
           {
            $line =~ s%\Q$code\E%$translations{$languages[0]}->{codes}->{$code}->{text}%g;
           }

         if($line =~ m%((\@\@|\$\$|\*\*|\~\~)[^\@\$*~]+(\@\@|\$\$|\*\*|\~\~))%)
           {
            print STDERR "   Unmatched codeword '$1' in line: $line";
           }

         # Remove un-needed spaces

         $line =~ s%[\t ]+% %g;
         $line =~ s%\n %\n%g;
         $line =~ s%^ %%g;

         print HTML_OUT $line;
        }

      close(HTML_IN);
      close(HTML_OUT);
     }
  }


# Create the XML file

open(XML_OUT,">$xml_output_file");

open(XML_IN ,"<translations-head.xml");

while(<XML_IN>)
  {
   print XML_OUT;
  }

close(XML_IN);

foreach my $language (@languages)
  {
   next if(!$translations{$language}->{xml});

   print "Language: $language File: translations.xml\n";

   open(XML_IN ,"<translations-body.xml");

   while(<XML_IN>)
     {
      my $line=$_;

      $line =~ s%~~lang~~%$language%g;
      $line =~ s%~~language~~%$translations{$language}->{codes}->{'@@LANGUAGE@@'}->{text}%g;

      # Replace with translated phrases

      foreach my $code (keys %{$translations{$language}->{codes}})
        {
         if($line =~ s%$code%$translations{$language}->{codes}->{$code}->{text}%g)
           {$translations{$language}->{codes}->{$code}->{usedX} = 1;}
        }

      # Replace what is left with a note about missing translations

      if($line =~ m%\%\%%)
        {
         foreach my $code (keys %{$translations{$languages[0]}->{codes}})
           {
            $line =~ s%$code%$translations{$languages[0]}->{codes}->{$code}->{text}%g;
           }

         $line =~ s%<%<!-- TRANSLATION REQUIRED: %;
         $line =~ s%>% -->%;

         if($line =~ m%((\%\%|\~\~)[^\%~]+(\%\%|\~\~))%)
           {
            print STDERR "   Unmatched codeword '$1' in line: $line";
           }
        }

      print XML_OUT $line;
     }

   close(XML_IN);
  }

open(XML_IN ,"<translations-tail.xml");

while(<XML_IN>)
  {
   print XML_OUT;
  }

close(XML_IN);

close(XML_OUT);


# Check the languages and usage

my %usedX=();
my %usedR=();
my %usedV=();

foreach my $language (@languages)
  {
   $usedX{$language}=0;
   $usedR{$language}=0;
   $usedV{$language}=0;

   foreach my $code (keys %{$translations{$language}->{codes}})
     {
      $usedX{$language}+=$translations{$language}->{codes}->{$code}->{usedX};
      $usedR{$language}+=$translations{$language}->{codes}->{$code}->{usedR};
      $usedV{$language}+=$translations{$language}->{codes}->{$code}->{usedV};

      if(! $translations{$language}->{codes}->{$code}->{usedX} &&
         ! $translations{$language}->{codes}->{$code}->{usedR} &&
         ! $translations{$language}->{codes}->{$code}->{usedV})
        {
         print STDERR "Language: $language UNUSED codeword: $code\n";
        }
     }
  }


# Print the translation coverage

print "\n";

print "Translation Coverage\n";
print "====================\n";

print "\n";
print "           Number      Percentage Complete\n";
print "Language  XML HTML    XML router visualiser\n";
print "--------  --- ----    --- ------ ----------\n";

foreach my $language (@languages)
  {
   printf("%-6s    %3d  %3d  %4.0f%%  %4.0f%%  %4.0f%%\n",
          $language,
          $translations{$language}->{xml},
          $translations{$language}->{html},
          100.0*$usedX{$language}/$usedX{$languages[0]},
          100.0*$usedR{$language}/$usedR{$languages[0]},
          100.0*$usedV{$language}/$usedV{$languages[0]})
  }
1	#!/usr/bin/perl
2	#
3	# Routino translation replacement Perl script
4	#
5	# Part of the Routino routing software.
6	#
7	# This file Copyright 2014-2015 Andrew M. Bishop
8	#
9	# This program is free software: you can redistribute it and/or modify
10	# it under the terms of the GNU Affero General Public License as published by
11	# the Free Software Foundation, either version 3 of the License, or
12	# (at your option) any later version.
13	#
14	# This program is distributed in the hope that it will be useful,
15	# but WITHOUT ANY WARRANTY; without even the implied warranty of
16	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17	# GNU Affero General Public License for more details.
18	#
19	# You should have received a copy of the GNU Affero General Public License
20	# along with this program. If not, see <http://www.gnu.org/licenses/>.
21	#
22
23	use strict;
24
25	# Constants
26
27	my @translation_files=(<translation.*.txt>);
28	my $xml_output_file="../../xml/routino-translations.xml";
29	my $html_output_dir="../www/routino";
30
31	my @html_template_files=(<*.html>);
32
33	my %languages=();
34	my %translations=();
35
36
37	# Sort so that English is first
38
39	@translation_files=sort {if($a eq "translation.en.txt"){return -1;} if($b eq "translation.en.txt"){return 1;} return $a <=> $b;} @translation_files;
40
41	# Read in the translations
42
43	foreach my $translation_file (@translation_files)
44	{
45	$translation_file =~ m%translation.([^.]+).txt%;
46
47	# Add to list of languages
48
49	my $language=$1;
50
51	if(! defined $languages{$language})
52	{
53	$languages{$language}=1;
54
55	$translations{$language}={};
56	$translations{$language}->{codes}={};
57	$translations{$language}->{html}=0;
58	$translations{$language}->{xml}=0;
59	}
60
61	# Process the file
62
63	open(FILE,"<$translation_file");
64
65	while(<FILE>)
66	{
67	s%\r*\n%%;
68
69	next if(m%^#%);
70	next if(m%^$%);
71
72	# Single line HTML entries
73
74	if(m%\@\@%)
75	{
76	my($code,$text)=split("\t");
77
78	if(defined $translations{$language}->{codes}->{$code})
79	{
80	print STDERR "Language: $language DUPLICATED codeword '$code'\n";
81	}
82	else
83	{
84	$translations{$language}->{html}++;
85	$translations{$language}->{codes}->{$code}={};
86	$translations{$language}->{codes}->{$code}->{text}=$text;
87	$translations{$language}->{codes}->{$code}->{usedX}=0;
88	$translations{$language}->{codes}->{$code}->{usedR}=0;
89	$translations{$language}->{codes}->{$code}->{usedV}=0;
90	}
91	}
92
93	# Multi-line HTML entries
94
95	if(m%(\$\$[^\$]+\$\$)%)
96	{
97	my($code,$text)=($1,"");
98
99	while(<FILE>)
100	{
101	last if(m%\$\$%);
102
103	$text=$text.$_;
104	}
105
106	$text =~ s%\r*\n$%%;
107
108	if(defined $translations{$language}->{codes}->{$code})
109	{
110	print STDERR "Language: $language DUPLICATED codeword '$code'\n";
111	}
112	else
113	{
114	$translations{$language}->{html}++;
115	$translations{$language}->{codes}->{$code}={};
116	$translations{$language}->{codes}->{$code}->{text}=$text;
117	$translations{$language}->{codes}->{$code}->{usedX}=0;
118	$translations{$language}->{codes}->{$code}->{usedR}=0;
119	$translations{$language}->{codes}->{$code}->{usedV}=0;
120	}
121	}
122
123	# Single line XML entries
124
125	if(m%\%\%%)
126	{
127	my($code,$text)=split("\t");
128
129	if(defined $translations{$language}->{codes}->{$code})
130	{
131	print STDERR "Language: $language DUPLICATED codeword '$code'\n";
132	}
133	else
134	{
135	$translations{$language}->{xml}++;
136	$translations{$language}->{codes}->{$code}={};
137	$translations{$language}->{codes}->{$code}->{text}=$text;
138	$translations{$language}->{codes}->{$code}->{usedX}=0;
139	$translations{$language}->{codes}->{$code}->{usedR}=0;
140	$translations{$language}->{codes}->{$code}->{usedV}=0;
141	}
142
143	my($n_strings_en)=$translations{en}->{codes}->{$code}->{text} =~ s/%s/%s/g;
144	my($n_strings) =$text =~ s/%s/%s/g;
145
146	if($n_strings != $n_strings_en)
147	{
148	print STDERR "Language: $language WRONG number of '%s' in text '$text' ($translations{en}->{codes}->{$code}->{text})\n";
149	}
150	}
151	}
152
153	close(FILE);
154	}
155
156
157	# Sort out the languages
158
159	my @languages=();
160
161	push(@languages,"en");
162
163	foreach my $language (sort (keys %languages))
164	{
165	push(@languages,$language) if($language ne "en");
166	}
167
168
169	# Create the HTML files
170
171	foreach my $html_template_file (@html_template_files)
172	{
173	my $usedtype="";
174
175	$usedtype="R" if($html_template_file =~ m%router%);
176	$usedtype="V" if($html_template_file =~ m%visualiser%);
177
178	foreach my $language (@languages)
179	{
180	next if(!$translations{$language}->{html});
181
182	print "Language: $language File: $html_template_file\n";
183
184	my $language_meta=0;
185	my $language_meta_string="";
186
187	open(HTML_IN ,"<$html_template_file");
188	open(HTML_OUT,">$html_output_dir/$html_template_file.$language");
189
190	while(<HTML_IN>)
191	{
192	my $line=$_;
193
194	# Language selection - special handling
195
196	if($line =~ m%\\LANGUAGES-META\\%)
197	{
198	$language_meta=1-$language_meta;
199
200	if($language_meta==0)
201	{
202	foreach my $language2 (@languages)
203	{
204	my $LANGUAGE2=$language2;
205	$LANGUAGE2 =~ tr%a-z%A-Z%;
206
207	$line=$language_meta_string;
208
209	if($language eq $language2)
210	{
211	$line =~ s%~~CHECKED~~%checked%g;
212	}
213	else
214	{
215	$line =~ s%~~CHECKED~~%%g;
216	}
217
218	$line =~ s%~~lang~~%$language2%g;
219	$line =~ s%~~LANG~~%$LANGUAGE2%g;
220
221	if(!$translations{$language2}->{html})
222	{
223	$line =~ s%<a.+</a>%%;
224	}
225
226	if(!$translations{$language2}->{xml})
227	{
228	$line =~ s%<input .+>%%;
229	}
230
231	foreach my $code (keys %{$translations{$language2}->{codes}})
232	{
233	if($line =~ s%$code%$translations{$language2}->{codes}->{$code}->{text}%g)
234	{$translations{$language2}->{codes}->{$code}->{"used$usedtype"} = 1;}
235	}
236
237	if($line =~ m%((\@\@\|\$\$\|\\\|\~\~)[^\@\$~]+(\@\@\|\$\$\|\\*\|\~\~))%)
238	{
239	print STDERR " Unmatched codeword '$1' in line: $line";
240	}
241
242	# Remove un-needed spaces
243
244	$line =~ s%[\t ]+% %g;
245	$line =~ s%\n %\n%g;
246	$line =~ s%^ %%g;
247
248	print HTML_OUT $line;
249	}
250	}
251
252	next;
253	}
254
255	if($language_meta)
256	{
257	$language_meta_string.=$line;
258	next;
259	}
260
261	# Replace with translated phrases
262
263	foreach my $code (keys %{$translations{$language}->{codes}})
264	{
265	if($line =~ s%\Q$code\E%$translations{$language}->{codes}->{$code}->{text}%g)
266	{$translations{$language}->{codes}->{$code}->{"used$usedtype"} = 1;}
267	}
268
269	# Replace what is left with English phrases
270
271	foreach my $code (keys %{$translations{$languages[0]}->{codes}})
272	{
273	$line =~ s%\Q$code\E%$translations{$languages[0]}->{codes}->{$code}->{text}%g;
274	}
275
276	if($line =~ m%((\@\@\|\$\$\|\\\|\~\~)[^\@\$~]+(\@\@\|\$\$\|\\*\|\~\~))%)
277	{
278	print STDERR " Unmatched codeword '$1' in line: $line";
279	}
280
281	# Remove un-needed spaces
282
283	$line =~ s%[\t ]+% %g;
284	$line =~ s%\n %\n%g;
285	$line =~ s%^ %%g;
286
287	print HTML_OUT $line;
288	}
289
290	close(HTML_IN);
291	close(HTML_OUT);
292	}
293	}
294
295
296	# Create the XML file
297
298	open(XML_OUT,">$xml_output_file");
299
300	open(XML_IN ,"<translations-head.xml");
301
302	while(<XML_IN>)
303	{
304	print XML_OUT;
305	}
306
307	close(XML_IN);
308
309	foreach my $language (@languages)
310	{
311	next if(!$translations{$language}->{xml});
312
313	print "Language: $language File: translations.xml\n";
314
315	open(XML_IN ,"<translations-body.xml");
316
317	while(<XML_IN>)
318	{
319	my $line=$_;
320
321	$line =~ s%~~lang~~%$language%g;
322	$line =~ s%~~language~~%$translations{$language}->{codes}->{'@@LANGUAGE@@'}->{text}%g;
323
324	# Replace with translated phrases
325
326	foreach my $code (keys %{$translations{$language}->{codes}})
327	{
328	if($line =~ s%$code%$translations{$language}->{codes}->{$code}->{text}%g)
329	{$translations{$language}->{codes}->{$code}->{usedX} = 1;}
330	}
331
332	# Replace what is left with a note about missing translations
333
334	if($line =~ m%\%\%%)
335	{
336	foreach my $code (keys %{$translations{$languages[0]}->{codes}})
337	{
338	$line =~ s%$code%$translations{$languages[0]}->{codes}->{$code}->{text}%g;
339	}
340
341	$line =~ s%<%<!-- TRANSLATION REQUIRED: %;
342	$line =~ s%>% -->%;
343
344	if($line =~ m%((\%\%\|\~\~)[^\%~]+(\%\%\|\~\~))%)
345	{
346	print STDERR " Unmatched codeword '$1' in line: $line";
347	}
348	}
349
350	print XML_OUT $line;
351	}
352
353	close(XML_IN);
354	}
355
356	open(XML_IN ,"<translations-tail.xml");
357
358	while(<XML_IN>)
359	{
360	print XML_OUT;
361	}
362
363	close(XML_IN);
364
365	close(XML_OUT);
366
367
368	# Check the languages and usage
369
370	my %usedX=();
371	my %usedR=();
372	my %usedV=();
373
374	foreach my $language (@languages)
375	{
376	$usedX{$language}=0;
377	$usedR{$language}=0;
378	$usedV{$language}=0;
379
380	foreach my $code (keys %{$translations{$language}->{codes}})
381	{
382	$usedX{$language}+=$translations{$language}->{codes}->{$code}->{usedX};
383	$usedR{$language}+=$translations{$language}->{codes}->{$code}->{usedR};
384	$usedV{$language}+=$translations{$language}->{codes}->{$code}->{usedV};
385
386	if(! $translations{$language}->{codes}->{$code}->{usedX} &&
387	! $translations{$language}->{codes}->{$code}->{usedR} &&
388	! $translations{$language}->{codes}->{$code}->{usedV})
389	{
390	print STDERR "Language: $language UNUSED codeword: $code\n";
391	}
392	}
393	}
394
395
396	# Print the translation coverage
397
398	print "\n";
399
400	print "Translation Coverage\n";
401	print "====================\n";
402
403	print "\n";
404	print " Number Percentage Complete\n";
405	print "Language XML HTML XML router visualiser\n";
406	print "-------- --- ---- --- ------ ----------\n";
407
408	foreach my $language (@languages)
409	{
410	printf("%-6s %3d %3d %4.0f%% %4.0f%% %4.0f%%\n",
411	$language,
412	$translations{$language}->{xml},
413	$translations{$language}->{html},
414	100.0*$usedX{$language}/$usedX{$languages[0]},
415	100.0*$usedR{$language}/$usedR{$languages[0]},
416	100.0*$usedV{$language}/$usedV{$languages[0]})
417	}