# Note:

# This *is* now explicitly RPython.

# Please make sure not to break this.

# XXX RUSTPYTHON: this was originally from PyPy and has been updated to run on

# Python 3. It's currently in the process of being rewritten

# to a native Rust module in vm/src/stdlib/codecs.rs

"""

_codecs -- Provides access to the codec registry and the builtin

codecs.

This module should never be imported directly. The standard library

module "codecs" wraps this builtin module for use within Python.

The codec registry is accessible via:

register(search_function) -> None

lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer)

The builtin Unicode codecs use the following interface:

<encoding>_encode(Unicode_object[,errors='strict']) ->

(string object, bytes consumed)

<encoding>_decode(char_buffer_obj[,errors='strict']) ->

(Unicode object, bytes consumed)

<encoding>_encode() interfaces also accept non-Unicode object as

input. The objects are then converted to Unicode using

PyUnicode_FromObject() prior to applying the conversion.

These <encoding>s are available: utf_8, unicode_escape,

raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit),

mbcs (on win32).

Written by Marc-Andre Lemburg ([email protected]).

Copyright (c) Corporation for National Research Initiatives.

From PyPy v1.0.0

"""

#from unicodecodec import *

__all__ = ['register', 'lookup', 'lookup_error', 'register_error', 'encode', 'decode',

'latin_1_encode', 'mbcs_decode', 'readbuffer_encode', 'escape_encode',

'utf_8_decode', 'raw_unicode_escape_decode', 'utf_7_decode',

'unicode_escape_encode', 'latin_1_decode', 'utf_16_decode',

'unicode_escape_decode', 'ascii_decode', 'charmap_encode', 'charmap_build',

'unicode_internal_encode', 'unicode_internal_decode', 'utf_16_ex_decode',

'escape_decode', 'charmap_decode', 'utf_7_encode', 'mbcs_encode',

'ascii_encode', 'utf_16_encode', 'raw_unicode_escape_encode', 'utf_8_encode',

'utf_16_le_encode', 'utf_16_be_encode', 'utf_16_le_decode', 'utf_16_be_decode',]

import sys

import warnings

from _codecs import *

def latin_1_encode( obj, errors='strict'):

"""None

"""

res = PyUnicode_EncodeLatin1(obj, len(obj), errors)

res = bytes(res)

return res, len(obj)

# XXX MBCS codec might involve ctypes ?

def mbcs_decode():

"""None

"""

pass

def readbuffer_encode( obj, errors='strict'):

"""None

"""

if isinstance(obj, str):

res = obj.encode()

else:

res = bytes(obj)

return res, len(obj)

def escape_encode( obj, errors='strict'):

"""None

"""

if not isinstance(obj, bytes):

raise TypeError("must be bytes")

s = repr(obj).encode()

v = s[2:-1]

if s[1] == ord('"'):

v = v.replace(b"'", b"\\'").replace(b'\\"', b'"')

return v, len(obj)

def raw_unicode_escape_decode( data, errors='strict'):

"""None

"""

res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors)

res = ''.join(res)

return res, len(data)

def utf_7_decode( data, errors='strict'):

"""None

"""

res = PyUnicode_DecodeUTF7(data, len(data), errors)

res = ''.join(res)

return res, len(data)

def unicode_escape_encode( obj, errors='strict'):

"""None

"""

res = unicodeescape_string(obj, len(obj), 0)

res = b''.join(res)

return res, len(obj)

def latin_1_decode( data, errors='strict'):

"""None

"""

res = PyUnicode_DecodeLatin1(data, len(data), errors)

res = ''.join(res)

return res, len(data)

def utf_16_decode( data, errors='strict', final=False):

"""None

"""

consumed = len(data)

if final:

consumed = 0

res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final)

res = ''.join(res)

return res, consumed

def unicode_escape_decode( data, errors='strict'):

"""None

"""

res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors)

res = ''.join(res)

return res, len(data)

def ascii_decode( data, errors='strict'):

"""None

"""

res = PyUnicode_DecodeASCII(data, len(data), errors)

res = ''.join(res)

return res, len(data)

def charmap_encode(obj, errors='strict', mapping='latin-1'):

"""None

"""

res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors)

res = bytes(res)

return res, len(obj)

def charmap_build(s):

return {ord(c): i for i, c in enumerate(s)}

if sys.maxunicode == 65535:

unicode_bytes = 2

else:

unicode_bytes = 4

def unicode_internal_encode( obj, errors='strict'):

"""None

"""

if type(obj) == str:

p = bytearray()

t = [ord(x) for x in obj]

for i in t:

b = bytearray()

for j in range(unicode_bytes):

b.append(i%256)

i >>= 8

if sys.byteorder == "big":

b.reverse()

p += b

res = bytes(p)

return res, len(res)

else:

res = "You can do better than this" # XXX make this right

return res, len(res)

def unicode_internal_decode( unistr, errors='strict'):

"""None

"""

if type(unistr) == str:

return unistr, len(unistr)

else:

p = []

i = 0

if sys.byteorder == "big":

start = unicode_bytes - 1

stop = -1

step = -1

else:

start = 0

stop = unicode_bytes

step = 1

while i < len(unistr)-unicode_bytes+1:

t = 0

h = 0

for j in range(start, stop, step):

t += ord(unistr[i+j])<<(h*8)

h += 1

i += unicode_bytes

p += chr(t)

res = ''.join(p)

return res, len(res)

def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0):

"""None

"""

if byteorder == 0:

bm = 'native'

elif byteorder == -1:

bm = 'little'

else:

bm = 'big'

consumed = len(data)

if final:

consumed = 0

res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final)

res = ''.join(res)

return res, consumed, byteorder

# XXX needs error messages when the input is invalid

def escape_decode(data, errors='strict'):

"""None

"""

l = len(data)

i = 0

res = bytearray()

while i < l:

if data[i] == '\\':

i += 1

if i >= l:

raise ValueError("Trailing \\ in string")

else:

if data[i] == '\\':

res += b'\\'

elif data[i] == 'n':

res += b'\n'

elif data[i] == 't':

res += b'\t'

elif data[i] == 'r':

res += b'\r'

elif data[i] == 'b':

res += b'\b'

elif data[i] == '\'':

res += b'\''

elif data[i] == '\"':

res += b'\"'

elif data[i] == 'f':

res += b'\f'

elif data[i] == 'a':

res += b'\a'

elif data[i] == 'v':

res += b'\v'

elif '0' <= data[i] <= '9':

# emulate a strange wrap-around behavior of CPython:

# \400 is the same as \000 because 0400 == 256

octal = data[i:i+3]

res.append(int(octal, 8) & 0xFF)

i += 2

elif data[i] == 'x':

hexa = data[i+1:i+3]

res.append(int(hexa, 16))

i += 2

else:

res.append(data[i])

i += 1

res = bytes(res)

return res, len(res)

def charmap_decode( data, errors='strict', mapping=None):

"""None

"""

res = PyUnicode_DecodeCharmap(data, len(data), mapping, errors)

res = ''.join(res)

return res, len(data)

def utf_7_encode( obj, errors='strict'):

"""None

"""

res = PyUnicode_EncodeUTF7(obj, len(obj), 0, 0, errors)

res = b''.join(res)

return res, len(obj)

def mbcs_encode( obj, errors='strict'):

"""None

"""

pass

## return (PyUnicode_EncodeMBCS(

## (obj),

## len(obj),

## errors),

## len(obj))

def ascii_encode( obj, errors='strict'):

"""None

"""

res = PyUnicode_EncodeASCII(obj, len(obj), errors)

res = bytes(res)

return res, len(obj)

def utf_16_encode( obj, errors='strict'):

"""None

"""

res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native')

res = bytes(res)

return res, len(obj)

def raw_unicode_escape_encode( obj, errors='strict'):

"""None

"""

res = PyUnicode_EncodeRawUnicodeEscape(obj, len(obj))

res = bytes(res)

return res, len(obj)

def utf_16_le_encode( obj, errors='strict'):

"""None

"""

res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little')

res = bytes(res)

return res, len(obj)

def utf_16_be_encode( obj, errors='strict'):

"""None

"""

res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big')

res = bytes(res)

return res, len(obj)

def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0):

"""None

"""

consumed = len(data)

if final:

consumed = 0

res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final)

res = ''.join(res)

return res, consumed

def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0):

"""None

"""

consumed = len(data)

if final:

consumed = 0

res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final)

res = ''.join(res)

return res, consumed

# ----------------------------------------------------------------------

##import sys

##""" Python implementation of CPythons builtin unicode codecs.

##

## Generally the functions in this module take a list of characters an returns

## a list of characters.

##

## For use in the PyPy project"""

## indicate whether a UTF-7 character is special i.e. cannot be directly

## encoded:

## 0 - not special

## 1 - special

## 2 - whitespace (optional)

## 3 - RFC2152 Set O (optional)

utf7_special = [

1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,

3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,

3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,

]

unicode_latin1 = [None]*256

def SPECIAL(c, encodeO, encodeWS):

c = ord(c)

return (c>127 or utf7_special[c] == 1) or \

(encodeWS and (utf7_special[(c)] == 2)) or \

(encodeO and (utf7_special[(c)] == 3))

def B64(n):

return bytes([b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f]])

def B64CHAR(c):

return (c.isalnum() or (c) == b'+' or (c) == b'/')

def UB64(c):

if (c) == b'+' :

return 62

elif (c) == b'/':

return 63

elif (c) >= b'a':

return ord(c) - 71

elif (c) >= b'A':

return ord(c) - 65

else:

return ord(c) + 4

def ENCODE( ch, bits) :

out = []

while (bits >= 6):

out += B64(ch >> (bits-6))

bits -= 6

return out, bits

def PyUnicode_DecodeUTF7(s, size, errors):

starts = s

errmsg = ""

inShift = 0

bitsleft = 0

charsleft = 0

surrogate = 0

p = []

errorHandler = None

exc = None

if (size == 0):

return ''

i = 0

while i < size:

ch = bytes([s[i]])

if (inShift):

if ((ch == b'-') or not B64CHAR(ch)):

inShift = 0

i += 1

while (bitsleft >= 16):

outCh = ((charsleft) >> (bitsleft-16)) & 0xffff

bitsleft -= 16

if (surrogate):

## We have already generated an error for the high surrogate

## so let's not bother seeing if the low surrogate is correct or not

surrogate = 0

elif (0xDC00 <= (outCh) and (outCh) <= 0xDFFF):

## This is a surrogate pair. Unfortunately we can't represent

## it in a 16-bit character

surrogate = 1

msg = "code pairs are not supported"

out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i)

p.append(out)

bitsleft = 0

break

else:

p.append(chr(outCh ))

#p += out

if (bitsleft >= 6):

## /* The shift sequence has a partial character in it. If

## bitsleft < 6 then we could just classify it as padding

## but that is not the case here */

msg = "partial character in shift sequence"

out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i)

## /* According to RFC2152 the remaining bits should be zero. We

## choose to signal an error/insert a replacement character

## here so indicate the potential of a misencoded character. */

## /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */

## if (bitsleft and (charsleft << (sizeof(charsleft) * 8 - bitsleft))):

## raise UnicodeDecodeError, "non-zero padding bits in shift sequence"

if (ch == b'-') :

if ((i < size) and (s[i] == '-')) :

p += '-'

inShift = 1

elif SPECIAL(ch, 0, 0) :

raise UnicodeDecodeError("unexpected special character")

else:

p.append(chr(ord(ch)))

else:

charsleft = (charsleft << 6) | UB64(ch)

bitsleft += 6

i += 1

## /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);

elif ( ch == b'+' ):

startinpos = i

i += 1

if (i<size and s[i] == '-'):

i += 1

p.append(b'+')

else:

inShift = 1

bitsleft = 0

elif (SPECIAL(ch, 0, 0)):

i += 1

raise UnicodeDecodeError("unexpected special character")

else:

p.append(chr(ord(ch)))

i += 1

if (inShift) :

#XXX This aint right

endinpos = size

raise UnicodeDecodeError("unterminated shift sequence")

return p

def PyUnicode_EncodeUTF7(s, size, encodeSetO, encodeWhiteSpace, errors):

# /* It might be possible to tighten this worst case */

inShift = False

i = 0

bitsleft = 0

charsleft = 0

out = []

for ch in s:

if (not inShift) :

if (ch == '+'):

out.append(b'+-')

elif (SPECIAL(ch, encodeSetO, encodeWhiteSpace)):

charsleft = ord(ch)

bitsleft = 16

out.append(b'+')

p, bitsleft = ENCODE( charsleft, bitsleft)

out.append(p)

inShift = bitsleft > 0

else:

out.append(bytes([ord(ch)]))

else:

if (not SPECIAL(ch, encodeSetO, encodeWhiteSpace)):

out.append(B64((charsleft) << (6-bitsleft)))

charsleft = 0

bitsleft = 0

## /* Characters not in the BASE64 set implicitly unshift the sequence

## so no '-' is required, except if the character is itself a '-' */

if (B64CHAR(ch) or ch == '-'):

out.append(b'-')

inShift = False

out.append(bytes([ord(ch)]))

else:

bitsleft += 16

charsleft = (((charsleft) << 16) | ord(ch))

p, bitsleft = ENCODE(charsleft, bitsleft)

out.append(p)

## /* If the next character is special then we dont' need to terminate

## the shift sequence. If the next character is not a BASE64 character

## or '-' then the shift sequence will be terminated implicitly and we

## don't have to insert a '-'. */

if (bitsleft == 0):

if (i + 1 < size):

ch2 = s[i+1]

if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)):

pass

elif (B64CHAR(ch2) or ch2 == '-'):

out.append(b'-')

inShift = False

else:

inShift = False

else:

out.append(b'-')

inShift = False

i += 1

if (bitsleft):

out.append(B64(charsleft << (6-bitsleft) ) )

out.append(b'-')

return out

unicode_empty = ''

def unicodeescape_string(s, size, quotes):

p = []

if (quotes) :

if (s.find('\'') != -1 and s.find('"') == -1):

p.append(b'"')

else:

p.append(b'\'')

pos = 0

while (pos < size):

ch = s[pos]

#/* Escape quotes */

if (quotes and (ch == p[1] or ch == '\\')):

p.append(b'\\%c' % ord(ch))

pos += 1

continue

#ifdef Py_UNICODE_WIDE

#/* Map 21-bit characters to '\U00xxxxxx' */

elif (ord(ch) >= 0x10000):

p.append(b'\\U%08x' % ord(ch))

pos += 1

continue

#endif

#/* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */

elif (ord(ch) >= 0xD800 and ord(ch) < 0xDC00):

pos += 1

ch2 = s[pos]

if (ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF):

ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000

p.append(b'\\U%08x' % ucs)

pos += 1

continue

#/* Fall through: isolated surrogates are copied as-is */

pos -= 1

#/* Map 16-bit characters to '\uxxxx' */

if (ord(ch) >= 256):

p.append(b'\\u%04x' % ord(ch))

#/* Map special whitespace to '\t', \n', '\r' */

elif (ch == '\t'):

p.append(b'\\t')

elif (ch == '\n'):

p.append(b'\\n')

elif (ch == '\r'):

p.append(b'\\r')

elif (ch == '\\'):

p.append(b'\\\\')

#/* Map non-printable US ASCII to '\xhh' */

elif (ch < ' ' or ch >= chr(0x7F)) :

p.append(b'\\x%02x' % ord(ch))

#/* Copy everything else as-is */

else:

p.append(bytes([ord(ch)]))

pos += 1

if (quotes):

p.append(p[0])

return p

def PyUnicode_DecodeASCII(s, size, errors):

# /* ASCII is equivalent to the first 128 ordinals in Unicode. */

if (size == 1 and ord(s) < 128) :

return [chr(ord(s))]

if (size == 0):

return [''] #unicode('')

p = []

pos = 0

while pos < len(s):

c = s[pos]

if c < 128:

p += chr(c)

pos += 1

else:

res = unicode_call_errorhandler(

errors, "ascii", "ordinal not in range(128)",

s, pos, pos+1)

p += res[0]

pos = res[1]

return p

def PyUnicode_EncodeASCII(p, size, errors):

return unicode_encode_ucs1(p, size, errors, 128)

def PyUnicode_AsASCIIString(unistr):

if not type(unistr) == str:

raise TypeError

return PyUnicode_EncodeASCII(str(unistr),

len(str),

None)

def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True):

bo = 0 #/* assume native ordering by default */

consumed = 0

errmsg = ""

if sys.byteorder == 'little':

ihi = 1

ilo = 0

else:

ihi = 0

ilo = 1

#/* Unpack UTF-16 encoded data */

## /* Check for BOM marks (U+FEFF) in the input and adjust current

## byte order setting accordingly. In native mode, the leading BOM

## mark is skipped, in all other modes, it is copied to the output

## stream as-is (giving a ZWNBSP character). */

q = 0

p = []

if byteorder == 'native':

if (size >= 2):

bom = (s[ihi] << 8) | s[ilo]

#ifdef BYTEORDER_IS_LITTLE_ENDIAN

if sys.byteorder == 'little':

if (bom == 0xFEFF):

q += 2

bo = -1

elif bom == 0xFFFE:

q += 2

bo = 1

else:

if bom == 0xFEFF:

q += 2

bo = 1

elif bom == 0xFFFE:

q += 2

bo = -1

elif byteorder == 'little':

bo = -1

else:

bo = 1

if (size == 0):

return [''], 0, bo

if (bo == -1):

#/* force LE */

ihi = 1

ilo = 0

elif (bo == 1):

#/* force BE */

ihi = 0

ilo = 1

while (q < len(s)):

#/* remaining bytes at the end? (size should be even) */

if (len(s)-q<2):

if not final:

break

errmsg = "truncated data"

startinpos = q

endinpos = len(s)

unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)

# /* The remaining input chars are ignored if the callback

## chooses to skip the input */

ch = (s[q+ihi] << 8) | s[q+ilo]

q += 2

if (ch < 0xD800 or ch > 0xDFFF):

p.append(chr(ch))

continue

#/* UTF-16 code pair: */

if (q >= len(s)):

errmsg = "unexpected end of data"

startinpos = q-2

endinpos = len(s)

unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)

if (0xD800 <= ch and ch <= 0xDBFF):

ch2 = (s[q+ihi] << 8) | s[q+ilo]

q += 2

if (0xDC00 <= ch2 and ch2 <= 0xDFFF):

#ifndef Py_UNICODE_WIDE

if sys.maxunicode < 65536:

p += [chr(ch), chr(ch2)]

else:

p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000))

#endif

continue

else:

errmsg = "illegal UTF-16 surrogate"

startinpos = q-4

endinpos = startinpos+2

unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)

errmsg = "illegal encoding"

startinpos = q-2

endinpos = startinpos+2

unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)

return p, q, bo

# moved out of local scope, especially because it didn't

# have any nested variables.

def STORECHAR(CH, byteorder):

hi = (CH >> 8) & 0xff

lo = CH & 0xff

if byteorder == 'little':

return [lo, hi]

else:

return [hi, lo]

def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'):

# /* Offsets from p for storing byte pairs in the right order. */

p = []

bom = sys.byteorder

if (byteorder == 'native'):

bom = sys.byteorder

p += STORECHAR(0xFEFF, bom)

if (size == 0):

return ""

if (byteorder == 'little' ):

bom = 'little'

elif (byteorder == 'big'):

bom = 'big'

for c in s:

ch = ord(c)

ch2 = 0

if (ch >= 0x10000) :

ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF)

ch = 0xD800 | ((ch-0x10000) >> 10)

p += STORECHAR(ch, bom)

if (ch2):

p += STORECHAR(ch2, bom)

return p

def PyUnicode_DecodeMBCS(s, size, errors):

pass

def PyUnicode_EncodeMBCS(p, size, errors):

pass

def unicode_call_errorhandler(errors, encoding,

reason, input, startinpos, endinpos, decode=True):

errorHandler = lookup_error(errors)

if decode:

exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason)

else:

exceptionObject = UnicodeEncodeError(encoding, input, startinpos, endinpos, reason)

res = errorHandler(exceptionObject)

if isinstance(res, tuple) and isinstance(res[0], str) and isinstance(res[1], int):

newpos = res[1]

if (newpos < 0):

newpos = len(input) + newpos

if newpos < 0 or newpos > len(input):

raise IndexError( "position %d from error handler out of bounds" % newpos)

return res[0], newpos

else:

raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res))

#/* --- Latin-1 Codec ------------------------------------------------------ */

def PyUnicode_DecodeLatin1(s, size, errors):

#/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */

## if (size == 1):

## return [PyUnicode_FromUnicode(s, 1)]

pos = 0

p = []

while (pos < size):

p += chr(s[pos])

pos += 1

return p

def unicode_encode_ucs1(p, size, errors, limit):

if limit == 256:

reason = "ordinal not in range(256)"

encoding = "latin-1"

else:

reason = "ordinal not in range(128)"

encoding = "ascii"

if (size == 0):

return []

res = bytearray()

pos = 0

while pos < len(p):

#for ch in p:

ch = p[pos]

if ord(ch) < limit:

res.append(ord(ch))

pos += 1

else:

#/* startpos for collecting unencodable chars */

collstart = pos

collend = pos+1

while collend < len(p) and ord(p[collend]) >= limit:

collend += 1

x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False)

res += x[0].encode()

pos = x[1]

return res

def PyUnicode_EncodeLatin1(p, size, errors):

res = unicode_encode_ucs1(p, size, errors, 256)

return res

hexdigits = [ord(hex(i)[-1]) for i in range(16)]+[ord(hex(i)[-1].upper()) for i in range(10, 16)]

def hex_number_end(s, pos, digits):

target_end = pos + digits

while pos < target_end and pos < len(s) and s[pos] in hexdigits:

pos += 1

return pos

def hexescape(s, pos, digits, message, errors):

ch = 0

p = []

number_end = hex_number_end(s, pos, digits)

if number_end - pos != digits:

x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, number_end)

p.append(x[0])

pos = x[1]

else:

ch = int(s[pos:pos+digits], 16)

#/* when we get here, ch is a 32-bit unicode character */

if ch <= sys.maxunicode:

p.append(chr(ch))

pos += digits

elif (ch <= 0x10ffff):

ch -= 0x10000

p.append(chr(0xD800 + (ch >> 10)))

p.append(chr(0xDC00 + (ch & 0x03FF)))

pos += digits

else:

message = "illegal Unicode character"

x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2,

pos+digits)

p.append(x[0])

pos = x[1]

res = p

return res, pos

def PyUnicode_DecodeUnicodeEscape(s, size, errors):

if (size == 0):

return ''

if isinstance(s, str):

s = s.encode()

found_invalid_escape = False

p = []

pos = 0

while (pos < size):

## /* Non-escape characters are interpreted as Unicode ordinals */

if (chr(s[pos]) != '\\') :

p.append(chr(s[pos]))

pos += 1

continue

## /* \ - Escapes */

else:

pos += 1

if pos >= len(s):

errmessage = "\\ at end of string"

unicode_call_errorhandler(errors, "unicodeescape", errmessage, s, pos-1, size)

ch = chr(s[pos])

pos += 1

## /* \x escapes */

if ch == '\n': pass

elif ch == '\\': p += '\\'

elif ch == '\'': p += '\''

elif ch == '\"': p += '\"'

elif ch == 'b' : p += '\b'

elif ch == 'f' : p += '\014' #/* FF */

elif ch == 't' : p += '\t'

elif ch == 'n' : p += '\n'

elif ch == 'r' : p += '\r'

elif ch == 'v' : p += '\013' #break; /* VT */

elif ch == 'a' : p += '\007' # break; /* BEL, not classic C */

elif '0' <= ch <= '7':

x = ord(ch) - ord('0')

if pos < size:

ch = chr(s[pos])

if '0' <= ch <= '7':

pos += 1

x = (x<<3) + ord(ch) - ord('0')

if pos < size:

ch = chr(s[pos])

if '0' <= ch <= '7':

pos += 1

x = (x<<3) + ord(ch) - ord('0')