path: root/firmware/common/unicode.c

/***************************************************************************
 *             __________               __   ___.
 *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
 *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
 *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
 *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
 *                     \/            \/     \/    \/            \/
 * $Id$
 *
 * Copyright (c) 2004,2005 by Marcoen Hirschberg
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ****************************************************************************/
/*   Some conversion functions for handling UTF-8
 *
 *   I got all the info from:
 *   http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
 *   and
 *   http://en.wikipedia.org/wiki/Unicode
 */

#include <stdio.h>
#include "config.h"
#include "system.h"
#include "thread.h"
#include "file.h"
#include "debug.h"
#include "rbunicode.h"
#include "rbpaths.h"
#include "pathfuncs.h"
#include "core_alloc.h"

#ifndef O_BINARY
#define O_BINARY 0
#endif
#ifndef O_NOISODECODE
#define O_NOISODECODE 0
#endif

#define getle16(p) (p[0] | (p[1] << 8))
#define getbe16(p) ((p[0] << 8) | p[1])

#if !defined (__PCTOOL__) && (CONFIG_PLATFORM & PLATFORM_NATIVE)
/* Because file scanning uses the default CP table when matching entries,
   on-demand loading is not feasible; we also must use the filesystem lock */
#include "file_internal.h"
#else /* APPLICATION */
#ifdef __PCTOOL__
#define yield()
#define DEFAULT_CP_STATIC_ALLOC
#endif
#define open_noiso_internal open
#endif /* !APPLICATION */

#if 0 /* not needed just now (will probably end up a spinlock) */
#include "mutex.h"
static struct mutex cp_mutex SHAREDBSS_ATTR;
#define cp_lock_init()   mutex_init(&cp_mutex)
#define cp_lock_enter()  mutex_lock(&cp_mutex)
#define cp_lock_leave()  mutex_unlock(&cp_mutex)
#else
#define cp_lock_init()   do {} while (0)
#define cp_lock_enter()  asm volatile ("")
#define cp_lock_leave()  asm volatile ("")
#endif

enum cp_tid
{
    CP_TID_NONE = -1,
    CP_TID_ISO,
    CP_TID_932,
    CP_TID_936,
    CP_TID_949,
    CP_TID_950,
};

struct cp_info
{
    int8_t      tid;
    const char  *filename;
    const char  *name;
};

#ifdef HAVE_LCD_BITMAP

#define MAX_CP_TABLE_SIZE  32768

#define CPF_ISO "iso.cp"
#define CPF_932 "932.cp"  /* SJIS    */
#define CPF_936 "936.cp"  /* GB2312  */
#define CPF_949 "949.cp"  /* KSX1001 */
#define CPF_950 "950.cp"  /* BIG5    */

static const struct cp_info cp_info[NUM_CODEPAGES+1] =
{
    [0 ... NUM_CODEPAGES] = { CP_TID_NONE, NULL   , "unknown"     },
    [ISO_8859_1]          = { CP_TID_NONE, NULL   , "ISO-8859-1"  },
    [ISO_8859_7]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-7"  },
    [ISO_8859_8]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-8"  },
    [WIN_1251]            = { CP_TID_ISO , CPF_ISO, "CP1251"      },
    [ISO_8859_11]         = { CP_TID_ISO , CPF_ISO, "ISO-8859-11" },
    [WIN_1256]            = { CP_TID_ISO , CPF_ISO, "CP1256"      },
    [ISO_8859_9]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-9"  },
    [ISO_8859_2]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-2"  },
    [WIN_1250]            = { CP_TID_ISO , CPF_ISO, "CP1250"      },
    [WIN_1252]            = { CP_TID_ISO , CPF_ISO, "CP1252"      },
    [SJIS]                = { CP_TID_932 , CPF_932, "SJIS"        },
    [GB_2312]             = { CP_TID_936 , CPF_936, "GB-2312"     },
    [KSX_1001]            = { CP_TID_949 , CPF_949, "KSX-1001"    },
    [BIG_5]               = { CP_TID_950 , CPF_950, "BIG5"        },
    [UTF_8]               = { CP_TID_NONE, NULL   , "UTF-8"       },
};

#else /* !HAVE_LCD_BITMAP, reduced support */

#define MAX_CP_TABLE_SIZE  768

#define CPF_ISOMINI "isomini.cp"

static const struct cp_info cp_info[NUM_CODEPAGES+1] =
{
    [0 ... NUM_CODEPAGES] = { CP_TID_NONE, NULL       , "unknown"    },
    [ISO_8859_1]          = { CP_TID_NONE, NULL       , "ISO-8859-1" },
    [ISO_8859_7]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-7" },
    [WIN_1251]            = { CP_TID_ISO , CPF_ISOMINI, "CP1251"     },
    [ISO_8859_9]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-9" },
    [ISO_8859_2]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-2" },
    [WIN_1250]            = { CP_TID_ISO , CPF_ISOMINI, "CP1250"     },
    [WIN_1252]            = { CP_TID_ISO , CPF_ISOMINI, "CP1252"     },
    [UTF_8]               = { CP_TID_ISO , NULL       , "UTF-8"      },
};

#endif /* HAVE_LCD_BITMAP */

static int default_cp = INIT_CODEPAGE;
static int default_cp_tid = CP_TID_NONE;
static int default_cp_handle = 0;
static int volatile default_cp_table_ref = 0;

static int loaded_cp_tid = CP_TID_NONE;
static int volatile cp_table_ref = 0;
#define CP_LOADING BIT_N(sizeof(int)*8-1) /* guard against multi loaders */

/* non-default codepage table buffer (cannot be bufalloced! playback itself
   may be making the load request) */
static unsigned short codepage_table[MAX_CP_TABLE_SIZE+1];

#if defined(APPLICATION) && defined(__linux__)
static const char * const name_codepages_linux[NUM_CODEPAGES+1] =
{
    [0 ... NUM_CODEPAGES] = "unknown",
    [ISO_8859_1]          = "iso8859-1",
    [ISO_8859_7]          = "iso8859-7",
    [ISO_8859_8]          = "iso8859-8",
    [WIN_1251]            = "cp1251",
    [ISO_8859_11]         = "iso8859-11",
    [WIN_1256]            = "cp1256",
    [ISO_8859_9]          = "iso8859-9",
    [ISO_8859_2]          = "iso8859-2",
    [WIN_1250]            = "cp1250",
    /* iso8859-15 is closest, linux doesnt have a codepage named cp1252 */
    [WIN_1252]            = "iso8859-15",
    [SJIS]                = "cp932",
    [GB_2312]             = "cp936",
    [KSX_1001]            = "cp949",
    [BIG_5]               = "cp950",
    [UTF_8]               = "utf8",
};

const char *get_current_codepage_name_linux(void)
{
    int cp = default_cp;
    if (cp < 0 || cp>= NUM_CODEPAGES)
        cp = NUM_CODEPAGES;
    return name_codepages_linux[cp];
}
#endif /* defined(APPLICATION) && defined(__linux__) */

#ifdef DEFAULT_CP_STATIC_ALLOC
static unsigned short default_cp_table_buf[MAX_CP_TABLE_SIZE+1];
#define cp_table_get_data(handle) \
    default_cp_table_buf
#define cp_table_free(handle) \
    do {} while (0)
#define cp_table_alloc(filename, size, opsp) \
    ({ (void)(opsp); 1; })
#else
#define cp_table_alloc(filename, size, opsp) \
    core_alloc_ex((filename), (size), (opsp))
#define cp_table_free(handle) \
    core_free(handle)
#define cp_table_get_data(handle) \
    core_get_data(handle)
#endif

static const unsigned char utf8comp[6] =
{
    0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
};

static inline void cptable_tohw16(uint16_t *buf, unsigned int count)
{
#ifdef ROCKBOX_BIG_ENDIAN
    for (unsigned int i = 0; i < count; i++)
        buf[i] = letoh16(buf[i]);
#endif
    (void)buf; (void)count;
}

static int move_callback(int handle, void *current, void *new)
{
    /* we don't keep a pointer but we have to stop it if this applies to a
       buffer not yet swapped-in since it will likely be in use in an I/O
       call */
    return (handle != default_cp_handle || default_cp_table_ref != 0) ?
                BUFLIB_CB_CANNOT_MOVE : BUFLIB_CB_OK;
    (void)current; (void)new;
}

static int alloc_and_load_cp_table(int cp, void *buf)
{
    static struct buflib_callbacks ops =
        { .move_callback = move_callback };

    /* alloc and read only if there is an associated file */
    const char *filename = cp_info[cp].filename;
    if (!filename)
        return 0;

    char path[MAX_PATH];
    if (path_append(path, CODEPAGE_DIR, filename, sizeof (path))
        >= sizeof (path)) {
        return -1;
    }

    /* must be opened without a chance of reentering from FS code */
    int fd = open_noiso_internal(path, O_RDONLY);
    if (fd < 0)
        return -1;

    off_t size = filesize(fd);

    if (size > 0 && size <= MAX_CP_TABLE_SIZE*2 &&
        !(size % (off_t)sizeof (uint16_t))) {

        /* if the buffer is provided, use that but don't alloc */
        int handle = buf ? 0 : cp_table_alloc(filename, size, &ops);
        if (handle > 0)
            buf = cp_table_get_data(handle);

        if (buf && read(fd, buf, size) == size) {
            close(fd);
            cptable_tohw16(buf, size / sizeof (uint16_t));
            return handle;
        }

        if (handle > 0)
            cp_table_free(handle);
    }

    close(fd);
    return -1;
}

/* Encode a UCS value as UTF-8 and return a pointer after this UTF-8 char. */
unsigned char* utf8encode(unsigned long ucs, unsigned char *utf8)
{
    int tail = 0;

    if (ucs > 0x7F)
        while (ucs >> (5*tail + 6))
            tail++;

    *utf8++ = (ucs >> (6*tail)) | utf8comp[tail];
    while (tail--)
        *utf8++ = ((ucs >> (6*tail)) & (MASK ^ 0xFF)) | COMP;

    return utf8;
}

/* Recode an iso encoded string to UTF-8 */
unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
                          int cp, int count)
{
    uint16_t *table = NULL;

    cp_lock_enter();

    if (cp < 0 || cp >= NUM_CODEPAGES)
        cp = default_cp;

    int tid = cp_info[cp].tid;

    while (1) {
        if (tid == default_cp_tid) {
            /* use default table */
            if (default_cp_handle > 0) {
                table = cp_table_get_data(default_cp_handle);
                default_cp_table_ref++;
            }

            break;
        }

        bool load = false;

        if (tid == loaded_cp_tid) {
            /* use loaded table */
            if (!(cp_table_ref & CP_LOADING)) {
                if (tid != CP_TID_NONE) {
                    table = codepage_table;
                    cp_table_ref++;
                }

                break;
            }
        } else if (cp_table_ref == 0) {
            load = true;
            cp_table_ref |= CP_LOADING;
        }

        /* alloc and load must be done outside the lock */
        cp_lock_leave();

        if (!load) {
            yield();
        } else if (alloc_and_load_cp_table(cp, codepage_table) < 0) {
            cp = INIT_CODEPAGE; /* table may be clobbered now */
            tid = cp_info[cp].tid;
        }

        cp_lock_enter();

        if (load) {
            loaded_cp_tid = tid;
            cp_table_ref &= ~CP_LOADING;
        }
    }

    cp_lock_leave();

    while (count--) {
        unsigned short ucs, tmp;

        if (*iso < 128 || cp == UTF_8) /* Already UTF-8 */
            *utf8++ = *iso++;

        else {
            /* tid tells us which table to use and how */
            switch (tid) {
                case CP_TID_ISO: /* Greek */
                                 /* Hebrew */
                                 /* Cyrillic */
                                 /* Thai */
                                 /* Arabic */
                                 /* Turkish */
                                 /* Latin Extended */
                                 /* Central European */
                                 /* Western European */
                    tmp = ((cp-1)*128) + (*iso++ - 128);
                    ucs = table[tmp];
                    break;

#ifdef HAVE_LCD_BITMAP
                case CP_TID_932: /* Japanese */
                    if (*iso > 0xA0 && *iso < 0xE0) {
                        tmp = *iso++ | (0xA100 - 0x8000);
                        ucs = table[tmp];
                        break;
                    }

                case CP_TID_936: /* Simplified Chinese */
                case CP_TID_949: /* Korean */
                case CP_TID_950: /* Traditional Chinese */
                    if (count < 1 || !iso[1]) {
                        ucs = *iso++;
                        break;
                    }

                    /* we assume all cjk strings are written
                       in big endian order */
                    tmp = *iso++ << 8;
                    tmp |= *iso++;
                    tmp -= 0x8000;
                    ucs = table[tmp];
                    count--;
                    break;
#endif /* HAVE_LCD_BITMAP */

                default:
                    ucs = *iso++;
                    break;
            }

            if (ucs == 0) /* unknown char, use replacement char */
                ucs = 0xfffd;
            utf8 = utf8encode(ucs, utf8);
        }
    }

    if (table) {
        cp_lock_enter();
        if (table == codepage_table) {
            cp_table_ref--;
        } else {
            default_cp_table_ref--;
        }
        cp_lock_leave();
    }

    return utf8;
}

/* Recode a UTF-16 string with little-endian byte ordering to UTF-8 */
unsigned char* utf16LEdecode(const unsigned char *utf16, unsigned char *utf8,
        int count)
{
    unsigned long ucs;

    while (count > 0) {
        /* Check for a surrogate pair */
        if (utf16[1] >= 0xD8 && utf16[1] < 0xE0) {
            ucs = 0x10000 + ((utf16[0] << 10) | ((utf16[1] - 0xD8) << 18)
                    | utf16[2] | ((utf16[3] - 0xDC) << 8));
            utf16 += 4;
            count -= 2;
        } else {
            ucs = getle16(utf16);
            utf16 += 2;
            count -= 1;
        }
        utf8 = utf8encode(ucs, utf8);
    }
    return utf8;
}

/* Recode a UTF-16 string with big-endian byte ordering to UTF-8 */
unsigned char* utf16BEdecode(const unsigned char *utf16, unsigned char *utf8,
        int count)
{
    unsigned long ucs;

    while (count > 0) {
        if (*utf16 >= 0xD8 && *utf16 < 0xE0) { /* Check for a surrogate pair */
            ucs = 0x10000 + (((utf16[0] - 0xD8) << 18) | (utf16[1] << 10)
                    | ((utf16[2] - 0xDC) << 8) | utf16[3]);
            utf16 += 4;
            count -= 2;
        } else {
            ucs = getbe16(utf16);
            utf16 += 2;
            count -= 1;
        }
        utf8 = utf8encode(ucs, utf8);
    }
    return utf8;
}

#if 0 /* currently unused */
/* Recode any UTF-16 string to UTF-8 */
unsigned char* utf16decode(const unsigned char *utf16, unsigned char *utf8,
        unsigned int count)
{
    unsigned long ucs;

    ucs = *(utf16++) << 8;
    ucs |= *(utf16++);

    if (ucs == 0xFEFF) /* Check for BOM */
        return utf16BEdecode(utf16, utf8, count-1);
    else if (ucs == 0xFFFE)
        return utf16LEdecode(utf16, utf8, count-1);
    else { /* ADDME: Should default be LE or BE? */
        utf16 -= 2;
        return utf16BEdecode(utf16, utf8, count);
    }
}
#endif

/* Return the number of UTF-8 chars in a string */
unsigned long utf8length(const unsigned char *utf8)
{
    unsigned long l = 0;

    while (*utf8 != 0)
        if ((*utf8++ & MASK) != COMP)
            l++;

    return l;
}

/* Decode 1 UTF-8 char and return a pointer to the next char. */
const unsigned char* utf8decode(const unsigned char *utf8, unsigned short *ucs)
{
    unsigned char c = *utf8++;
    unsigned long code;
    int tail = 0;

    if ((c <= 0x7f) || (c >= 0xc2)) {
        /* Start of new character. */
        if (c < 0x80) {        /* U-00000000 - U-0000007F, 1 byte */
            code = c;
        } else if (c < 0xe0) { /* U-00000080 - U-000007FF, 2 bytes */
            tail = 1;
            code = c & 0x1f;
        } else if (c < 0xf0) { /* U-00000800 - U-0000FFFF, 3 bytes */
            tail = 2;
            code = c & 0x0f;
        } else if (c < 0xf5) { /* U-00010000 - U-001FFFFF, 4 bytes */
            tail = 3;
            code = c & 0x07;
        } else {
            /* Invalid size. */
            code = 0xfffd;
        }

        while (tail-- && ((c = *utf8++) != 0)) {
            if ((c & 0xc0) == 0x80) {
                /* Valid continuation character. */
                code = (code << 6) | (c & 0x3f);

            } else {
                /* Invalid continuation char */
                code = 0xfffd;
                utf8--;
                break;
            }
        }
    } else {
        /* Invalid UTF-8 char */
        code = 0xfffd;
    }
    /* currently we don't support chars above U-FFFF */
    *ucs = (code < 0x10000) ? code : 0xfffd;
    return utf8;
}

void set_codepage(int cp)
{
    if (cp < 0 || cp >= NUM_CODEPAGES)
        cp = NUM_CODEPAGES;

    /* load first then swap if load is successful, else just leave it; if
       handle is 0 then we just free the current one; this won't happen often
       thus we don't worry about reusing it and consequently avoid possible
       clobbering of the existing one */

    int handle = -1;
    int tid = cp_info[cp].tid;

    while (1) {
        cp_lock_enter();

        if (default_cp_tid == tid)
            break;

        if (handle >= 0 && default_cp_table_ref == 0) {
            int hold = default_cp_handle;
            default_cp_handle = handle;
            handle = hold;
            default_cp_tid = tid;
            break;
        }

        /* alloc and load must be done outside the lock */
        cp_lock_leave();

        if (handle < 0 && (handle = alloc_and_load_cp_table(cp, NULL)) < 0)
            return; /* OOM; change nothing */

        yield();
    }

    default_cp = cp;
    cp_lock_leave();

    if (handle > 0)
        cp_table_free(handle);
}

int get_codepage(void)
{
    return default_cp;
}

/* seek to a given char in a utf8 string and
   return its start position in the string */
int utf8seek(const unsigned char* utf8, int offset)
{
    int pos = 0;

    while (offset--) {
        pos++;
        while ((utf8[pos] & MASK) == COMP)
            pos++;
    }
    return pos;
}

const char * get_codepage_name(int cp)
{
    if (cp < 0 || cp >= NUM_CODEPAGES)
        cp = NUM_CODEPAGES;
    return cp_info[cp].name;
}

#if 0 /* not needed just now */
void unicode_init(void)
{
    cp_lock_init();
}
#endif