/* | |
* cjkcodecs.h: common header for cjkcodecs | |
* | |
* Written by Hye-Shik Chang <perky@FreeBSD.org> | |
*/ | |
#ifndef _CJKCODECS_H_ | |
#define _CJKCODECS_H_ | |
#define PY_SSIZE_T_CLEAN | |
#include "Python.h" | |
#include "multibytecodec.h" | |
/* a unicode "undefined" codepoint */ | |
#define UNIINV 0xFFFE | |
/* internal-use DBCS codepoints which aren't used by any charsets */ | |
#define NOCHAR 0xFFFF | |
#define MULTIC 0xFFFE | |
#define DBCINV 0xFFFD | |
/* shorter macros to save source size of mapping tables */ | |
#define U UNIINV | |
#define N NOCHAR | |
#define M MULTIC | |
#define D DBCINV | |
struct dbcs_index { | |
const ucs2_t *map; | |
unsigned char bottom, top; | |
}; | |
typedef struct dbcs_index decode_map; | |
struct widedbcs_index { | |
const ucs4_t *map; | |
unsigned char bottom, top; | |
}; | |
typedef struct widedbcs_index widedecode_map; | |
struct unim_index { | |
const DBCHAR *map; | |
unsigned char bottom, top; | |
}; | |
typedef struct unim_index encode_map; | |
struct unim_index_bytebased { | |
const unsigned char *map; | |
unsigned char bottom, top; | |
}; | |
struct dbcs_map { | |
const char *charset; | |
const struct unim_index *encmap; | |
const struct dbcs_index *decmap; | |
}; | |
struct pair_encodemap { | |
ucs4_t uniseq; | |
DBCHAR code; | |
}; | |
static const MultibyteCodec *codec_list; | |
static const struct dbcs_map *mapping_list; | |
#define CODEC_INIT(encoding) \ | |
static int encoding##_codec_init(const void *config) | |
#define ENCODER_INIT(encoding) \ | |
static int encoding##_encode_init( \ | |
MultibyteCodec_State *state, const void *config) | |
#define ENCODER(encoding) \ | |
static Py_ssize_t encoding##_encode( \ | |
MultibyteCodec_State *state, const void *config, \ | |
const Py_UNICODE **inbuf, Py_ssize_t inleft, \ | |
unsigned char **outbuf, Py_ssize_t outleft, int flags) | |
#define ENCODER_RESET(encoding) \ | |
static Py_ssize_t encoding##_encode_reset( \ | |
MultibyteCodec_State *state, const void *config, \ | |
unsigned char **outbuf, Py_ssize_t outleft) | |
#define DECODER_INIT(encoding) \ | |
static int encoding##_decode_init( \ | |
MultibyteCodec_State *state, const void *config) | |
#define DECODER(encoding) \ | |
static Py_ssize_t encoding##_decode( \ | |
MultibyteCodec_State *state, const void *config, \ | |
const unsigned char **inbuf, Py_ssize_t inleft, \ | |
Py_UNICODE **outbuf, Py_ssize_t outleft) | |
#define DECODER_RESET(encoding) \ | |
static Py_ssize_t encoding##_decode_reset( \ | |
MultibyteCodec_State *state, const void *config) | |
#if Py_UNICODE_SIZE == 4 | |
#define UCS4INVALID(code) \ | |
if ((code) > 0xFFFF) \ | |
return 1; | |
#else | |
#define UCS4INVALID(code) \ | |
if (0) ; | |
#endif | |
#define NEXT_IN(i) \ | |
(*inbuf) += (i); \ | |
(inleft) -= (i); | |
#define NEXT_OUT(o) \ | |
(*outbuf) += (o); \ | |
(outleft) -= (o); | |
#define NEXT(i, o) \ | |
NEXT_IN(i) NEXT_OUT(o) | |
#define REQUIRE_INBUF(n) \ | |
if (inleft < (n)) \ | |
return MBERR_TOOFEW; | |
#define REQUIRE_OUTBUF(n) \ | |
if (outleft < (n)) \ | |
return MBERR_TOOSMALL; | |
#define IN1 ((*inbuf)[0]) | |
#define IN2 ((*inbuf)[1]) | |
#define IN3 ((*inbuf)[2]) | |
#define IN4 ((*inbuf)[3]) | |
#define OUT1(c) ((*outbuf)[0]) = (c); | |
#define OUT2(c) ((*outbuf)[1]) = (c); | |
#define OUT3(c) ((*outbuf)[2]) = (c); | |
#define OUT4(c) ((*outbuf)[3]) = (c); | |
#define WRITE1(c1) \ | |
REQUIRE_OUTBUF(1) \ | |
(*outbuf)[0] = (c1); | |
#define WRITE2(c1, c2) \ | |
REQUIRE_OUTBUF(2) \ | |
(*outbuf)[0] = (c1); \ | |
(*outbuf)[1] = (c2); | |
#define WRITE3(c1, c2, c3) \ | |
REQUIRE_OUTBUF(3) \ | |
(*outbuf)[0] = (c1); \ | |
(*outbuf)[1] = (c2); \ | |
(*outbuf)[2] = (c3); | |
#define WRITE4(c1, c2, c3, c4) \ | |
REQUIRE_OUTBUF(4) \ | |
(*outbuf)[0] = (c1); \ | |
(*outbuf)[1] = (c2); \ | |
(*outbuf)[2] = (c3); \ | |
(*outbuf)[3] = (c4); | |
#if Py_UNICODE_SIZE == 2 | |
# define WRITEUCS4(c) \ | |
REQUIRE_OUTBUF(2) \ | |
(*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10); \ | |
(*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff); \ | |
NEXT_OUT(2) | |
#else | |
# define WRITEUCS4(c) \ | |
REQUIRE_OUTBUF(1) \ | |
**outbuf = (Py_UNICODE)(c); \ | |
NEXT_OUT(1) | |
#endif | |
#define _TRYMAP_ENC(m, assi, val) \ | |
((m)->map != NULL && (val) >= (m)->bottom && \ | |
(val)<= (m)->top && ((assi) = (m)->map[(val) - \ | |
(m)->bottom]) != NOCHAR) | |
#define TRYMAP_ENC_COND(charset, assi, uni) \ | |
_TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff) | |
#define TRYMAP_ENC(charset, assi, uni) \ | |
if TRYMAP_ENC_COND(charset, assi, uni) | |
#define _TRYMAP_DEC(m, assi, val) \ | |
((m)->map != NULL && (val) >= (m)->bottom && \ | |
(val)<= (m)->top && ((assi) = (m)->map[(val) - \ | |
(m)->bottom]) != UNIINV) | |
#define TRYMAP_DEC(charset, assi, c1, c2) \ | |
if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2) | |
#define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val) \ | |
((m)->map != NULL && (val) >= (m)->bottom && \ | |
(val)<= (m)->top && \ | |
((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \ | |
(((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \ | |
(((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1)) | |
#define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni) \ | |
if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \ | |
assplane, asshi, asslo, (uni) & 0xff) | |
#define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2) \ | |
if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2) | |
#if Py_UNICODE_SIZE == 2 | |
#define DECODE_SURROGATE(c) \ | |
if (c >> 10 == 0xd800 >> 10) { /* high surrogate */ \ | |
REQUIRE_INBUF(2) \ | |
if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \ | |
c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \ | |
((ucs4_t)(IN2) - 0xdc00); \ | |
} \ | |
} | |
#define GET_INSIZE(c) ((c) > 0xffff ? 2 : 1) | |
#else | |
#define DECODE_SURROGATE(c) {;} | |
#define GET_INSIZE(c) 1 | |
#endif | |
#define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = { | |
#define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL}, | |
#define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap}, | |
#define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap}, | |
#define END_MAPPINGS_LIST \ | |
{"", NULL, NULL} }; \ | |
static const struct dbcs_map *mapping_list = \ | |
(const struct dbcs_map *)_mapping_list; | |
#define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = { | |
#define _STATEFUL_METHODS(enc) \ | |
enc##_encode, \ | |
enc##_encode_init, \ | |
enc##_encode_reset, \ | |
enc##_decode, \ | |
enc##_decode_init, \ | |
enc##_decode_reset, | |
#define _STATELESS_METHODS(enc) \ | |
enc##_encode, NULL, NULL, \ | |
enc##_decode, NULL, NULL, | |
#define CODEC_STATEFUL(enc) { \ | |
#enc, NULL, NULL, \ | |
_STATEFUL_METHODS(enc) \ | |
}, | |
#define CODEC_STATELESS(enc) { \ | |
#enc, NULL, NULL, \ | |
_STATELESS_METHODS(enc) \ | |
}, | |
#define CODEC_STATELESS_WINIT(enc) { \ | |
#enc, NULL, \ | |
enc##_codec_init, \ | |
_STATELESS_METHODS(enc) \ | |
}, | |
#define END_CODECS_LIST \ | |
{"", NULL,} }; \ | |
static const MultibyteCodec *codec_list = \ | |
(const MultibyteCodec *)_codec_list; | |
static PyObject * | |
getmultibytecodec(void) | |
{ | |
static PyObject *cofunc = NULL; | |
if (cofunc == NULL) { | |
PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec"); | |
if (mod == NULL) | |
return NULL; | |
cofunc = PyObject_GetAttrString(mod, "__create_codec"); | |
Py_DECREF(mod); | |
} | |
return cofunc; | |
} | |
static PyObject * | |
getcodec(PyObject *self, PyObject *encoding) | |
{ | |
PyObject *codecobj, *r, *cofunc; | |
const MultibyteCodec *codec; | |
const char *enc; | |
if (!PyString_Check(encoding)) { | |
PyErr_SetString(PyExc_TypeError, | |
"encoding name must be a string."); | |
return NULL; | |
} | |
cofunc = getmultibytecodec(); | |
if (cofunc == NULL) | |
return NULL; | |
enc = PyString_AS_STRING(encoding); | |
for (codec = codec_list; codec->encoding[0]; codec++) | |
if (strcmp(codec->encoding, enc) == 0) | |
break; | |
if (codec->encoding[0] == '\0') { | |
PyErr_SetString(PyExc_LookupError, | |
"no such codec is supported."); | |
return NULL; | |
} | |
codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL); | |
if (codecobj == NULL) | |
return NULL; | |
r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL); | |
Py_DECREF(codecobj); | |
return r; | |
} | |
static struct PyMethodDef __methods[] = { | |
{"getcodec", (PyCFunction)getcodec, METH_O, ""}, | |
{NULL, NULL}, | |
}; | |
static int | |
register_maps(PyObject *module) | |
{ | |
const struct dbcs_map *h; | |
for (h = mapping_list; h->charset[0] != '\0'; h++) { | |
char mhname[256] = "__map_"; | |
int r; | |
strcpy(mhname + sizeof("__map_") - 1, h->charset); | |
r = PyModule_AddObject(module, mhname, | |
PyCapsule_New((void *)h, PyMultibyteCodec_CAPSULE_NAME, NULL)); | |
if (r == -1) | |
return -1; | |
} | |
return 0; | |
} | |
#ifdef USING_BINARY_PAIR_SEARCH | |
static DBCHAR | |
find_pairencmap(ucs2_t body, ucs2_t modifier, | |
const struct pair_encodemap *haystack, int haystacksize) | |
{ | |
int pos, min, max; | |
ucs4_t value = body << 16 | modifier; | |
min = 0; | |
max = haystacksize; | |
for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) | |
if (value < haystack[pos].uniseq) { | |
if (max == pos) break; | |
else max = pos; | |
} | |
else if (value > haystack[pos].uniseq) { | |
if (min == pos) break; | |
else min = pos; | |
} | |
else | |
break; | |
if (value == haystack[pos].uniseq) | |
return haystack[pos].code; | |
else | |
return DBCINV; | |
} | |
#endif | |
#ifdef USING_IMPORTED_MAPS | |
#define IMPORT_MAP(locale, charset, encmap, decmap) \ | |
importmap("_codecs_" #locale, "__map_" #charset, \ | |
(const void**)encmap, (const void**)decmap) | |
static int | |
importmap(const char *modname, const char *symbol, | |
const void **encmap, const void **decmap) | |
{ | |
PyObject *o, *mod; | |
mod = PyImport_ImportModule((char *)modname); | |
if (mod == NULL) | |
return -1; | |
o = PyObject_GetAttrString(mod, (char*)symbol); | |
if (o == NULL) | |
goto errorexit; | |
else if (!PyCapsule_IsValid(o, PyMultibyteCodec_CAPSULE_NAME)) { | |
PyErr_SetString(PyExc_ValueError, | |
"map data must be a Capsule."); | |
goto errorexit; | |
} | |
else { | |
struct dbcs_map *map; | |
map = PyCapsule_GetPointer(o, PyMultibyteCodec_CAPSULE_NAME); | |
if (encmap != NULL) | |
*encmap = map->encmap; | |
if (decmap != NULL) | |
*decmap = map->decmap; | |
Py_DECREF(o); | |
} | |
Py_DECREF(mod); | |
return 0; | |
errorexit: | |
Py_DECREF(mod); | |
return -1; | |
} | |
#endif | |
#define I_AM_A_MODULE_FOR(loc) \ | |
void \ | |
init_codecs_##loc(void) \ | |
{ \ | |
PyObject *m = Py_InitModule("_codecs_" #loc, __methods);\ | |
if (m != NULL) \ | |
(void)register_maps(m); \ | |
} | |
#endif |