Change the .lng files to contain strings from multiple users. Still hard-coded to only output the core strings for now. Should be the majority of the core changes needed for translatable plugins.

git-svn-id: svn://svn.rockbox.org/rockbox/trunk@23241 a1c6a512-1295-4272-9138-f99709370657

7 files changed, 108 insertions, 55 deletions

diff --git a/apps/filetree.c b/apps/filetree.c
index 8fbc39fa67..3468051b55 100644
--- a/apps/filetree.c
+++ b/apps/filetree.c
@@ -526,7 +526,7 @@ int ft_enter(struct tree_context* c)
 
             case FILE_ATTR_LNG:
                 splash(0, ID2P(LANG_WAIT));
-                if(!lang_load(buf)) {
+                if(!lang_core_load(buf)) {
                     set_file(buf, (char *)global_settings.lang_file,
                              MAX_FILENAME);
                     talk_init(); /* use voice of same language */
diff --git a/apps/lang/lang.make b/apps/lang/lang.make
index e32f084a29..f7366c9225 100644
--- a/apps/lang/lang.make
+++ b/apps/lang/lang.make
@@ -18,9 +18,14 @@ CLEANOBJS += $(BUILDDIR)/lang/max_language_size.h $(BUILDDIR)/lang/lang*
 # Therefore we create it here.
 #DUMMY := $(shell mkdir -p $(BUILDDIR)/apps/lang)
 
+# Calculate the maximum language size. Currently based on the file size
+# of the largest lng file. Subtract 10 due to HEADER_SIZE and 
+# SUBHEADER_SIZE.
+# TODO: In the future generate this file within genlang or another script
+# in order to only calculate the maximum size based on the core strings.
 $(BUILDDIR)/lang/max_language_size.h: $(LANGOBJ)
 	$(call PRINTS,Create $(notdir $@))
-	$(SILENT)echo "#define MAX_LANGUAGE_SIZE `ls -ln $(BUILDDIR)/apps/lang/* | awk '{print $$5}' | sort -n | tail -1`" > $@
+	$(SILENT)echo "#define MAX_LANGUAGE_SIZE `ls -ln $(BUILDDIR)/apps/lang/* | awk '{print $$5-10}' | sort -n | tail -1`" > $@
 
 $(BUILDDIR)/lang/lang_core.o: $(APPSDIR)/lang/$(LANGUAGE).lang $(BUILDDIR)/apps/features
 	$(SILENT)for f in `cat $(BUILDDIR)/apps/features`; do feat="$$feat:$$f" ; done; \
diff --git a/apps/language.c b/apps/language.c
index fe9ad5e97d..70549e194d 100644
--- a/apps/language.c
+++ b/apps/language.c
@@ -37,10 +37,11 @@
 /* These defines must match the initial bytes in the binary lang file */
 /* See tools/genlang (TODO: Use common include for both) */
 #define LANGUAGE_COOKIE   0x1a
-#define LANGUAGE_VERSION  0x05
+#define LANGUAGE_VERSION  0x06
 #define LANGUAGE_FLAG_RTL 0x01
 
 #define HEADER_SIZE 4
+#define SUBHEADER_SIZE 6
 
 static unsigned char language_buffer[MAX_LANGUAGE_SIZE];
 static unsigned char lang_options = 0;
@@ -54,52 +55,62 @@ void lang_init(const unsigned char *builtin, unsigned char **dest, int count)
     }
 }
 
-int lang_load(const char *filename)
+int lang_load(const char *filename, const unsigned char *builtin, 
+              unsigned char **dest, unsigned char *buffer, 
+              unsigned int user_num, int max_lang_size,
+              unsigned int max_id)
 {
-    int fsize;
+    int lang_size;
     int fd = open(filename, O_RDONLY);
     int retcode=0;
     unsigned char lang_header[HEADER_SIZE];
+    unsigned char sub_header[SUBHEADER_SIZE];
+    unsigned int id, num_strings, foffset;
+
     if(fd < 0)
         return 1;
-    fsize = filesize(fd) - HEADER_SIZE;
-    if(fsize <= MAX_LANGUAGE_SIZE) {
-        read(fd, lang_header, HEADER_SIZE);
-        if((lang_header[0] == LANGUAGE_COOKIE) &&
-           (lang_header[1] == LANGUAGE_VERSION) &&
-           (lang_header[2] == TARGET_ID)) {
-            read(fd, language_buffer, MAX_LANGUAGE_SIZE);
-            unsigned char *ptr = language_buffer;
-            int id;
+    read(fd, lang_header, HEADER_SIZE);
+    if((lang_header[0] == LANGUAGE_COOKIE) &&
+       (lang_header[1] == LANGUAGE_VERSION) &&
+       (lang_header[2] == TARGET_ID)) {
+        /* jump to the proper entry in the table of subheaders */
+        lseek(fd, user_num * SUBHEADER_SIZE, SEEK_CUR);
+        read(fd, sub_header, SUBHEADER_SIZE);
+        /* read in information about the requested lang */
+        num_strings = (sub_header[0]<<8) | sub_header[1];
+        lang_size = (sub_header[2]<<8) | sub_header[3];
+        foffset = (sub_header[4]<<8) | sub_header[5];
+        if(lang_size <= max_lang_size) {
             /* initialize with builtin */
-            lang_init(language_builtin, language_strings, 
-                      LANG_LAST_INDEX_IN_ARRAY);
+            lang_init(builtin, dest, num_strings);
+            lseek(fd, foffset, SEEK_SET);
+            read(fd, buffer, lang_size);
 
-            while(fsize>3) {
-                id = (ptr[0]<<8) | ptr[1];  /* get two-byte id */
-                ptr+=2;                     /* pass the id */
-                if(id < LANG_LAST_INDEX_IN_ARRAY) {
+            while(lang_size>3) {
+                id = ((buffer[0]<<8) | buffer[1]); /* get two-byte id */
+                buffer += 2;                       /* pass the id */
+                if(id < max_id) {
 #if 0
-                    DEBUGF("%2x New: %30s ", id, ptr);
-                    DEBUGF("Replaces: %s\n", language_strings[id]);
+                    DEBUGF("%2x New: %30s ", id, buffer);
+                    DEBUGF("Replaces: %s\n", dest[id]);
 #endif
-                    language_strings[id] = ptr; /* point to this string */
-                }
-                while(*ptr) {               /* pass the string */
-                    fsize--;
-                    ptr++;
+                    dest[id] = buffer; /* point to this string */
                 }
-                fsize-=3; /* the id and the terminating zero */
-                ptr++;       /* pass the terminating zero-byte */
+                while(*buffer) {               /* pass the string */
+                    lang_size--;
+                    buffer++;
+                 }
+                lang_size-=3; /* the id and the terminating zero */
+                buffer++;     /* pass the terminating zero-byte */
             }
         }
         else {
-            DEBUGF("Illegal language file\n");
+            DEBUGF("Language %s too large: %d\n", filename, lang_size);
             retcode = 2;
         }
     }
     else {
-        DEBUGF("Language %s too large: %d\n", filename, fsize);
+        DEBUGF("Illegal language file\n");
         retcode = 3;
     }
     close(fd);
@@ -107,10 +118,17 @@ int lang_load(const char *filename)
     return retcode;
 }
 
+int lang_core_load(const char *filename)
+{
+    return lang_load(filename, core_language_builtin, language_strings,
+                     language_buffer, 0, MAX_LANGUAGE_SIZE,
+                     LANG_LAST_INDEX_IN_ARRAY);
+}
+
 int lang_english_to_id(const char *english)
 {
     int i;
-    unsigned char *ptr = (unsigned char *) language_builtin;
+    unsigned char *ptr = (unsigned char *) core_language_builtin;
     
     for (i = 0; i < LANG_LAST_INDEX_IN_ARRAY; i++) {
         if (!strcmp(ptr, english))
diff --git a/apps/language.h b/apps/language.h
index 4cfe2b22ce..cbfa7e2c1d 100644
--- a/apps/language.h
+++ b/apps/language.h
@@ -25,7 +25,12 @@
 void lang_init(const unsigned char *builtin, unsigned char **dest, int count);
 
 /* load a given language file */
-int lang_load(const char *filename);
+int lang_core_load(const char *filename);
+
+int lang_load(const char *filename, const unsigned char *builtin, 
+              unsigned char **dest, unsigned char *buffer, 
+              unsigned int user_num, int max_lang_size,
+              unsigned int max_id);
 
 /* get the ID of an english string so it can be localised */
 int lang_english_to_id(const char *english);
diff --git a/apps/main.c b/apps/main.c
index 3c93b4fea4..ec4829189b 100644
--- a/apps/main.c
+++ b/apps/main.c
@@ -308,7 +308,8 @@ static void init(void)
     button_init();
     backlight_init();
     sim_tasks_init();
-    lang_init(language_builtin, language_strings, LANG_LAST_INDEX_IN_ARRAY);
+    lang_init(core_language_builtin, language_strings, 
+              LANG_LAST_INDEX_IN_ARRAY);
 #ifdef DEBUG
     debug_init();
 #endif
@@ -399,7 +400,8 @@ static void init(void)
     font_init();
 
     show_logo();
-    lang_init(language_builtin, language_strings, LANG_LAST_INDEX_IN_ARRAY);
+    lang_init(core_language_builtin, language_strings, 
+              LANG_LAST_INDEX_IN_ARRAY);
 
 #ifdef DEBUG
     debug_init();
diff --git a/apps/settings.c b/apps/settings.c
index 2a5e31824f..6cbd559f94 100644
--- a/apps/settings.c
+++ b/apps/settings.c
@@ -900,7 +900,7 @@ void settings_apply(bool read_disk)
         if ( global_settings.lang_file[0]) {
             snprintf(buf, sizeof buf, LANG_DIR "/%s.lng",
                      global_settings.lang_file);
-            lang_load(buf);
+            lang_core_load(buf);
             talk_init(); /* use voice of same language */
         }
 
diff --git a/tools/genlang b/tools/genlang
index c5fb403801..6f00365716 100755
--- a/tools/genlang
+++ b/tools/genlang
@@ -13,9 +13,12 @@
 # See apps/language.c (TODO: Use common include for both)
 # Cookie and binary version for the binary lang file
 my $LANGUAGE_COOKIE   = 0x1a;
-my $LANGUAGE_VERSION  = 0x05;
+my $LANGUAGE_VERSION  = 0x06;
 my $LANGUAGE_FLAG_RTL = 0x01;
 
+my $HEADER_SIZE       = 4;
+my $SUBHEADER_SIZE    = 6;
+
 # A note for future users and readers: The original v1 language system allowed
 # the build to create and use a different language than english built-in. We
 # removed that feature from our build-system, but the build scripts still had
@@ -385,7 +388,7 @@ sub compare {
 my @idcount;           # counter for lang ID numbers
 my @voiceid;           # counter for voice-only ID numbers
 
-foreach $i (keys %users) {
+for (keys %users) {
     push @idcount, 0;
     push @voiceid, 0x8000;
 }
@@ -613,7 +616,7 @@ if($prefix) {
    It will be initialized at runtime. */
 extern unsigned char *language_strings[];
 /* this contains the concatenation of all strings, separated by \\0 chars */
-extern const unsigned char language_builtin[];
+extern const unsigned char core_language_builtin[];
 
 /* The enum below contains all available strings */
 enum \{
@@ -627,18 +630,18 @@ MOO
 #include "$headername"
 
 unsigned char *language_strings[LANG_LAST_INDEX_IN_ARRAY];
-const unsigned char language_builtin[] =
+const unsigned char core_language_builtin[] =
 MOO
 ;
 
     # Output the ID names for the enum in the header file
     my $i;
-    for $i (1 .. $idcount[$users{"core"}]) {
-        my $name=$idnum[$users{"core"}][$i - 1]; # get the ID name
+    for $i (0 .. $idcount[$users{"core"}]-1) {
+        my $name=$idnum[$users{"core"}][$i]; # get the ID name
         
         $name =~ s/\"//g; # cut off the quotes
         
-        printf HFILE_CORE ("    %s, /* %d */\n", $name, $i-1);
+        printf HFILE_CORE ("    %s, /* %d */\n", $name, $i);
     }
 
 # Output separation marker for last string ID and the upcoming voice IDs
@@ -663,8 +666,8 @@ MOO
     print HFILE_CORE "\n};\n/* end of generated enum list */\n";
 
     # Output the target phrases for the source file
-    for $i (1 .. $idcount[$users{"core"}]) {
-        my $name=$idnum[$users{"core"}][$i - 1]; # get the ID
+    for $i (0 .. $idcount[$users{"core"}]-1) {
+        my $name=$idnum[$users{"core"}][$i]; # get the ID
         my $dest = $dest{$name}; # get the destination phrase
         
         $dest =~ s:\"$:\\0\":; # insert a \0 before the second quote
@@ -700,18 +703,32 @@ elsif($binary) {
     printf OUTF ("%c%c%c%c", $LANGUAGE_COOKIE, $LANGUAGE_VERSION, $target_id,
       $langoptions); # magic lang file header
 
-    # loop over the target phrases
-    for $i (1 .. $idcount[$users{"core"}]) {
-        my $name=$idnum[$users{"core"}][$i - 1]; # get the ID
-        my $dest = $dest{$name}; # get the destination phrase
+    # output the number of strings for each user
+    my $foffset = $HEADER_SIZE + $SUBHEADER_SIZE * keys(%users);
+    for (keys %users) {
+        my $size;
+        for $n (0 .. $idcount[$_]-1) {
+            $size += length(trim($dest{$idnum[$_][$n]})) + 1;
+        }
+        printf OUTF ("%c%c%c%c%c%c", ($idcount[$_] >> 8), ($idcount[$_] & 0xff),
+                     ($size >> 8), ($size & 0xff), ($foffset >> 8), ($foffset & 0xff));
+        $foffset += $size;
+    }
 
-        if($dest) {
-            $dest =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
+    for (keys %users) {
+        # loop over the target phrases
+        for $n (0 .. $idcount[$_]-1) {
+            my $name=$idnum[$_][$n]; # get the ID
+            my $dest = $dest{$name}; # get the destination phrase
 
-            # Now, make sure we get the number from the english sort order:
-            $idnum = $idmap[$users{"core"}]{$name};
+            if($dest) {
+                $dest =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
 
-            printf OUTF ("%c%c%s\x00", ($idnum>>8), ($idnum&0xff), $dest);
+                # Now, make sure we get the number from the english sort order:
+                $idnum = $idmap[$_]{$name};
+
+                printf OUTF ("%c%c%s\x00", ($idnum>>8), ($idnum&0xff), $dest);
+            }
         }
     }
 }
@@ -770,7 +787,13 @@ elsif($voiceout) {
 
 
 if($verbose) {
-    printf("%d ID strings scanned\n", $idcount[$users{"core"}]);
+    my $num_str = 0;
+    
+    for (keys %users) {
+        $num_str += $idcount[$_];
+    }
+    
+    printf("%d ID strings scanned\n", $num_str);
 
     print "* head *\n";
     for(keys %head) {