wireshark/epan/charsets.c

/* charsets.c
 * Routines for handling character sets
 *
 * $Id$
 *
 * Wireshark - Network traffic analyzer
 * By Gerald Combs <gerald@wireshark.org>
 * Copyright 1998 Gerald Combs
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include "config.h"

#include <glib.h>

#include <epan/proto.h>
#include <epan/tvbuff.h>
#include <epan/value_string.h>

#include "charsets.h"

/*
 * Wikipedia's "Character encoding" template, giving a pile of character encodings and
 * Wikipedia pages for them:
 *
 *	http://en.wikipedia.org/wiki/Template:Character_encoding
 *
 * Unicode character encoding model:
 *
 *	http://www.unicode.org/reports/tr17/
 *
 * International Components for Unicode character set mapping tables:
 *
 *	http://site.icu-project.org/charts/charset
 *
 * MSDN information on code pages:
 *
 *	http://msdn.microsoft.com/en-us/library/dd317752(v=VS.85).aspx
 *
 * ASCII-based code pages, from IBM:
 *
 *	http://www-01.ibm.com/software/globalization/cp/cp_cpgid.html
 *
 * EBCDIC code pages, from IBM:
 *
 *	http://www-03.ibm.com/systems/i/software/globalization/codepages.html
 */

/* ASCII/EBCDIC conversion tables from
 * http://www.room42.com/store/computer_center/code_tables.shtml
 */
#if 0
static guint8 ASCII_translate_EBCDIC [ 256 ] = {
    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
    0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
    0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
    0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,
    0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D,
    0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
    0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
    0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
    0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
    0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
    0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D,
    0x7D, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
    0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
    0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
    0xA8, 0xA9, 0xC0, 0x6A, 0xD0, 0xA1, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B,
    0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B, 0x4B
};

void
ASCII_to_EBCDIC(guint8 *buf, guint bytes)
{
	guint	i;
	guint8	*bufptr;

	bufptr = buf;

	for (i = 0; i < bytes; i++, bufptr++) {
		*bufptr = ASCII_translate_EBCDIC[*bufptr];
	}
}

guint8
ASCII_to_EBCDIC1(guint8 c)
{
	return ASCII_translate_EBCDIC[c];
}
#endif

static guint8 EBCDIC_translate_ASCII [ 256 ] = {
    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
    0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
    0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
    0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,
    0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28,
    0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
    0x2E, 0x2E, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38,
    0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x2E, 0x3F,
    0x20, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x2E, 0x2E, 0x2E, 0x3C, 0x28, 0x2B, 0x7C,
    0x26, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x2E, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
    0x2D, 0x2F, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x2E, 0x7C, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
    0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x2E, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
    0x2E, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
    0x69, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x2E, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71,
    0x72, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x2E, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79,
    0x7A, 0x2E, 0x2E, 0x2E, 0x5B, 0x2E, 0x2E,
    0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x2E, 0x2E, 0x2E, 0x2E, 0x5D, 0x2E, 0x2E,
    0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48,
    0x49, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51,
    0x52, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x5C, 0x2E, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
    0x5A, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E,
    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38,
    0x39, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E
};

void
EBCDIC_to_ASCII(guint8 *buf, guint bytes)
{
	guint	i;
	guint8	*bufptr;

	bufptr = buf;

	for (i = 0; i < bytes; i++, bufptr++) {
		*bufptr = EBCDIC_translate_ASCII[*bufptr];
	}
}

guint8
EBCDIC_to_ASCII1(guint8 c)
{
	return EBCDIC_translate_ASCII[c];
}

/*
 * Translation tables that map the upper 128 code points in single-byte
 * "extended ASCII" character encodings to Unicode code points in the
 * Basic Multilingual Plane.
 */

/* REPLACEMENT CHARACTER */
#define UNREPL 0xFFFD

/* ISO-8859-2 (http://en.wikipedia.org/wiki/ISO/IEC_8859-2#Code_page_layout) */
const gunichar2 charset_table_iso_8859_2[0x80] = {
	0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,		/* 0x80 -      */
	0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,		/*      - 0x8F */
	0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,		/* 0x90 -      */
	0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,		/*      - 0x9F */
	0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7,		/* 0xA0 -      */
	0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b,		/*      - 0xAF */
	0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7,		/* 0xB0 -      */
	0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c,		/*      - 0xBF */
	0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7,		/* 0xC0 -      */
	0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,		/*      - 0xCF */
	0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7,		/* 0xD0 -      */
	0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,		/*      - 0xDF */
	0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7,		/* 0xE0 -      */
	0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,		/*      - 0xEF */
	0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7,		/* 0xF0 -      */
	0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9		/*      - 0xFF */
};

/* Windows-1250 (http://en.wikipedia.org/wiki/Windows-1250) */
const gunichar2 charset_table_cp1250[0x80] = {
	0x20ac, UNREPL, 0x201a, UNREPL, 0x201e, 0x2026, 0x2020, 0x2021,		/* 0x80 -      */
	UNREPL, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179,		/*      - 0x8F */
	UNREPL, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014,		/* 0x90 -      */
	UNREPL, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a,		/*      - 0x9F */
	0x00a0, 0x02c7, 0x02d8, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7,		/* 0xA0 -      */
	0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b, 	/*      - 0xAF */
	0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7,		/* 0xB0 -      */
	0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c, 	/*      - 0xBF */
	0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7,		/* 0xC0 -      */
	0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e, 	/*      - 0xCF */
	0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7,		/* 0xD0 -      */
	0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df, 	/*      - 0xDF */
	0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7,		/* 0xE0 -      */
	0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f, 	/*      - 0xEF */
	0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7,		/* 0xF0 -      */
	0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9, 	/*      - 0xFF */
};


static dvb_encoding_e
dvb_analyze_string_charset0(guint8 byte0)
{
    switch (byte0) {
        case 0x01:
         return DVB_ENCODING_ISO_8859_5;
        case 0x02:
         return DVB_ENCODING_ISO_8859_6;
        case 0x03:
         return DVB_ENCODING_ISO_8859_7;
        case 0x04:
         return DVB_ENCODING_ISO_8859_8;
        case 0x05:
         return DVB_ENCODING_ISO_8859_9;
        case 0x06:
         return DVB_ENCODING_ISO_8859_10;
        case 0x07:
         return DVB_ENCODING_ISO_8859_11;
        case 0x08:
         return DVB_ENCODING_RESERVED; /* was reserved for ISO-8859-12 */
        case 0x09: /* 0x09 */
         return DVB_ENCODING_ISO_8859_13;
        case 0x0A: /* 0x0A */
         return DVB_ENCODING_ISO_8859_14;
        case 0x0B: /* 0x0B */
         return DVB_ENCODING_ISO_8859_15;

    /* XXX 0x11 ... 0x15 */

        default:
            return DVB_ENCODING_UNKNOWN;
    }
}

static dvb_encoding_e
dvb_analyze_string_charset0_10(guint16 byte12)
{
    switch (byte12) {
        case 0x0000:
             return DVB_ENCODING_RESERVED;
        case 0x0001:
         return DVB_ENCODING_EXT_ISO_8859_1;
        case 0x0002:
         return DVB_ENCODING_EXT_ISO_8859_2;
        case 0x0003:
         return DVB_ENCODING_EXT_ISO_8859_3;
        case 0x0004:
         return DVB_ENCODING_EXT_ISO_8859_4;
        case 0x0005:
         return DVB_ENCODING_EXT_ISO_8859_5;
        case 0x0006:
         return DVB_ENCODING_EXT_ISO_8859_6;
        case 0x0007:
         return DVB_ENCODING_EXT_ISO_8859_7;
        case 0x0008:
         return DVB_ENCODING_EXT_ISO_8859_8;
        case 0x0009:
         return DVB_ENCODING_EXT_ISO_8859_9;
        case 0x000A:
         return DVB_ENCODING_EXT_ISO_8859_10;
        case 0x000B:
         return DVB_ENCODING_EXT_ISO_8859_11;
        case 0x000C:
         return DVB_ENCODING_RESERVED;
        case 0x000D:
         return DVB_ENCODING_EXT_ISO_8859_13;
        case 0x000E:
         return DVB_ENCODING_EXT_ISO_8859_14;
        case 0x000F:
         return DVB_ENCODING_EXT_ISO_8859_15;

        default: /* 0x10 XX XX */
            return DVB_ENCODING_UNKNOWN;
    }
}

static dvb_encoding_e
dvb_analyze_string_charset0_1F(guint8 byte1)
{
   /* http://www.dvbservices.com/identifiers/encoding_type_id */

    switch (byte1) {
       case 0x00: /* 0x1F 0x00 */
          return DVB_ENCODING_RESERVED;
       case 0x01:
       case 0x02:
       case 0x03:
       case 0x04:
          /* XXX: BBC */
          return DVB_ENCODING_RESERVED;
       case 0x05:
       case 0x06:
          /* XXX: Malaysian Technical Standards Forum Bhd */
          return DVB_ENCODING_RESERVED;

       default: /* 0x1F XX */
          return DVB_ENCODING_RESERVED;
    }
}


guint
dvb_analyze_string_charset(tvbuff_t *tvb, int offset, int length, dvb_encoding_e *encoding)
{
   if (length >= 1) {
      guint8 byte0 = tvb_get_guint8(tvb, offset + 0);

      if (byte0 >= 0x20) {
         /* the first byte is a normal character, not the number of a character table */
         *encoding = DVB_ENCODING_LATIN;
         return 0;

      } else if (byte0 == 0x1F) {
         if (length >= 2) {
            *encoding = dvb_analyze_string_charset0_1F(tvb_get_guint8(tvb, offset + 1));
            return 2;
         }
         *encoding = DVB_ENCODING_INVALID;
         return 1;

      } else if (byte0 >= 0x16) { /* 16 ... 1E */
         *encoding = DVB_ENCODING_RESERVED;
         return 1;

      } else if (byte0 == 0x10) {
         if (length >= 3) {
            *encoding = dvb_analyze_string_charset0_10(tvb_get_ntohs(tvb, offset + 1));
            return 3;
         }
         *encoding = DVB_ENCODING_INVALID;
         return 1;

      } else if ((byte0 >= 0x0C && byte0 <= 0x0F)) {
         *encoding = DVB_ENCODING_RESERVED;
         return 1;
      } else {
         *encoding = dvb_analyze_string_charset0(byte0);
         return 1;
      }
   } else
      *encoding = DVB_ENCODING_LATIN;

   return 0;
}


guint
dvb_enc_to_item_enc(dvb_encoding_e encoding)
{
   /* XXX: take ISO control codes into account,
      e.g. 0x86 - turn emphasis on ; 0x87 - turn emphasis off */

   switch (encoding) {
      case DVB_ENCODING_EXT_ISO_8859_2:
         return ENC_ISO_8859_2 | ENC_NA;

      default: /* not supported */
         return ENC_ASCII | ENC_NA;
   }
}


const value_string dvb_string_encoding_vals[] = {
   { DVB_ENCODING_INVALID,  "Incorrect length for encoding" },
   { DVB_ENCODING_RESERVED, "Reserved for future use" },
   { DVB_ENCODING_UNKNOWN,  "Value not specified by spec" },

   { DVB_ENCODING_LATIN,    "Latin (default table)" },

   { DVB_ENCODING_ISO_8859_5,  "ISO/IEC 8859-5 (Latin/Cyrillic)" },
   { DVB_ENCODING_ISO_8859_6,  "ISO/IEC 8859-6 (Latin/Arabic)" },
   { DVB_ENCODING_ISO_8859_7,  "ISO/IEC 8859-7 (Latin/Greek)" },
   { DVB_ENCODING_ISO_8859_8,  "ISO/IEC 8859-8 (Latin/Hebrew)" },
   { DVB_ENCODING_ISO_8859_9,  "ISO/IEC 8859-9 (West European & Turkish)" },
   { DVB_ENCODING_ISO_8859_10, "ISO/IEC 8859-10 (North European)" },
   { DVB_ENCODING_ISO_8859_11, "ISO/IEC 8859-11 (Thai)" },
   { DVB_ENCODING_ISO_8859_13, "ISO/IEC 8859-13 (Baltic)" },
   { DVB_ENCODING_ISO_8859_14, "ISO/IEC 8859-14 (Celtic)" },
   { DVB_ENCODING_ISO_8859_15, "ISO/IEC 8859-15 (West European)" },

   { DVB_ENCODING_EXT_ISO_8859_1,  "ISO/IEC 8859-1 (West European)" },
   { DVB_ENCODING_EXT_ISO_8859_2,  "ISO/IEC 8859-2 (East European)" },
   { DVB_ENCODING_EXT_ISO_8859_3,  "ISO/IEC 8859-3 (South European)" },
   { DVB_ENCODING_EXT_ISO_8859_4,  "ISO/IEC 8859-4 (North and North-East European)" },
   { DVB_ENCODING_EXT_ISO_8859_5,  "ISO/IEC 8859-5 (Latin/Cyrillic)" },
   { DVB_ENCODING_EXT_ISO_8859_6,  "ISO/IEC 8859-6 (Latin/Arabic)" },
   { DVB_ENCODING_EXT_ISO_8859_7,  "ISO/IEC 8859-7 (Latin/Greek)" },
   { DVB_ENCODING_EXT_ISO_8859_8,  "ISO/IEC 8859-8 (Latin/Hebrew)" },
   { DVB_ENCODING_EXT_ISO_8859_9,  "ISO/IEC 8859-9 (West European & Turkish)" },
   { DVB_ENCODING_EXT_ISO_8859_10, "ISO/IEC 8859-10 (North European)" },
   { DVB_ENCODING_EXT_ISO_8859_11, "ISO/IEC 8859-11 (Thai)" },
   { DVB_ENCODING_EXT_ISO_8859_13, "ISO/IEC 8859-13 (Baltic)" },
   { DVB_ENCODING_EXT_ISO_8859_14, "ISO/IEC 8859-14 (Celtic)" },
   { DVB_ENCODING_EXT_ISO_8859_15, "ISO/IEC 8859-15 (West European)" },

   { 0, NULL }
};