Coverage for gws-app/gws/lib/vendor/umsgpack.py: 23%

1# type: ignore

3# #@@gws: patched _pack3 because we desperately need the 'default' option like in json

5# u-msgpack-python v2.7.2 - v at sergeev.io

6# https://github.com/vsergeev/u-msgpack-python

8# u-msgpack-python is a lightweight MessagePack serializer and deserializer

9# module, compatible with both Python 2 and 3, as well CPython and PyPy

10# implementations of Python. u-msgpack-python is fully compliant with the

11# latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In

12# particular, it supports the new binary, UTF-8 string, and application ext

13# types.

14#

15# MIT License

16#

18#

19# Permission is hereby granted, free of charge, to any person obtaining a copy

20# of this software and associated documentation files (the "Software"), to deal

21# in the Software without restriction, including without limitation the rights

22# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

23# copies of the Software, and to permit persons to whom the Software is

24# furnished to do so, subject to the following conditions:

25#

26# The above copyright notice and this permission notice shall be included in

27# all copies or substantial portions of the Software.

28#

29# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

30# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

31# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

32# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

33# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

34# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

35# THE SOFTWARE.

36#

37"""

38u-msgpack-python v2.7.2 - v at sergeev.io

39https://github.com/vsergeev/u-msgpack-python

41u-msgpack-python is a lightweight MessagePack serializer and deserializer

42module, compatible with both Python 2 and 3, as well CPython and PyPy

43implementations of Python. u-msgpack-python is fully compliant with the

44latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In

45particular, it supports the new binary, UTF-8 string, and application ext

46types.

48License: MIT

49"""

50import struct

51import collections

52import datetime

53import sys

54import io

56if sys.version_info[0:2] >= (3, 3):

57 from collections.abc import Hashable

58else:

59 from collections import Hashable

61__version__ = "2.7.2"

62"Module version string"

64version = (2, 7, 2)

65"Module version tuple"

68##############################################################################

69# Ext Class

70##############################################################################

72# Extension type for application-defined types and data

73class Ext(object):

74 """

75 The Ext class facilitates creating a serializable extension object to store

76 an application-defined type and data byte array.

77 """

79 def __init__(self, type, data):

80 """

81 Construct a new Ext object.

83 Args:

84 type: application-defined type integer

85 data: application-defined data byte array

87 TypeError:

88 Type is not an integer.

89 ValueError:

90 Type is out of range of -128 to 127.

91 TypeError::

92 Data is not type 'bytes' (Python 3) or not type 'str' (Python 2).

94 Example:

95 >>> foo = umsgpack.Ext(5, b"\x01\x02\x03")

96 >>> umsgpack.packb({u"special stuff": foo, u"awesome": True})

97 '\x82\xa7awesome\xc3\xadspecial stuff\xc7\x03\x05\x01\x02\x03'

98 >>> bar = umsgpack.unpackb(_)

99 >>> print(bar["special stuff"])

100 Ext Object (Type: 5, Data: 01 02 03)

101 >>>

102 """

103 # Check type is type int and in range

104 if not isinstance(type, int):

105 raise TypeError("ext type is not type integer")

106 elif not (-2**7 <= type <= 2**7 - 1):

107 raise ValueError("ext type value {:d} is out of range (-128 to 127)".format(type))

108 # Check data is type bytes or str

109 elif sys.version_info[0] == 3 and not isinstance(data, bytes):

110 raise TypeError("ext data is not type \'bytes\'")

111 elif sys.version_info[0] == 2 and not isinstance(data, str):

112 raise TypeError("ext data is not type \'str\'")

113

114 self.type = type

115 self.data = data

116

117 def __eq__(self, other):

118 """

119 Compare this Ext object with another for equality.

120 """

121 return isinstance(other, self.__class__) \

122 and self.type == other.type and self.data == other.data

123

124 def __ne__(self, other):

125 """

126 Compare this Ext object with another for inequality.

127 """

128 return not self.__eq__(other)

129

130 def __str__(self):

131 """

132 String representation of this Ext object.

133 """

134 s = "Ext Object (Type: {:d}, Data: ".format(self.type)

135 s += " ".join(["0x{:02x}".format(ord(self.data[i:i + 1]))

136 for i in xrange(min(len(self.data), 8))])

137 if len(self.data) > 8:

138 s += " ..."

139 s += ")"

140 return s

141

142 def __hash__(self):

143 """

144 Provide a hash of this Ext object.

145 """

146 return hash((self.type, self.data))

147

148

149class InvalidString(bytes):

150 """Subclass of bytes to hold invalid UTF-8 strings."""

151

152

153##############################################################################

154# Ext Serializable Decorator

155##############################################################################

156

157_ext_class_to_type = {}

158_ext_type_to_class = {}

159

160

161def ext_serializable(ext_type):

162 """

163 Return a decorator to register a class for automatic packing and unpacking

164 with the specified Ext type code. The application class should implement a

165 `packb()` method that returns serialized bytes, and an `unpackb()` class

166 method or static method that accepts serialized bytes and returns an

167 instance of the application class.

168

169 Args:

170 ext_type: application-defined Ext type code

171

172 Raises:

173 TypeError:

174 Ext type is not an integer.

175 ValueError:

176 Ext type is out of range of -128 to 127.

177 ValueError:

178 Ext type or class already registered.

179 """

180 def wrapper(cls):

181 if not isinstance(ext_type, int):

182 raise TypeError("Ext type is not type integer")

183 elif not (-2**7 <= ext_type <= 2**7 - 1):

184 raise ValueError("Ext type value {:d} is out of range of -128 to 127".format(ext_type))

185 elif ext_type in _ext_type_to_class:

186 raise ValueError("Ext type {:d} already registered with class {:s}".format(ext_type, repr(_ext_type_to_class[ext_type])))

187 elif cls in _ext_class_to_type:

188 raise ValueError("Class {:s} already registered with Ext type {:d}".format(repr(cls), ext_type))

189

190 _ext_type_to_class[ext_type] = cls

191 _ext_class_to_type[cls] = ext_type

193 return cls

195 return wrapper

198##############################################################################

199# Exceptions

200##############################################################################

201

202

203# Base Exception classes

204class PackException(Exception):

205 "Base class for exceptions encountered during packing."

206

207

208class UnpackException(Exception):

209 "Base class for exceptions encountered during unpacking."

210

211

212# Packing error

213class UnsupportedTypeException(PackException):

214 "Object type not supported for packing."

215

216

217# Unpacking error

218class InsufficientDataException(UnpackException):

219 "Insufficient data to unpack the serialized object."

220

221

222class InvalidStringException(UnpackException):

223 "Invalid UTF-8 string encountered during unpacking."

224

225

226class UnsupportedTimestampException(UnpackException):

227 "Unsupported timestamp format encountered during unpacking."

228

229

230class ReservedCodeException(UnpackException):

231 "Reserved code encountered during unpacking."

232

233

234class UnhashableKeyException(UnpackException):

235 """

236 Unhashable key encountered during map unpacking.

237 The serialized map cannot be deserialized into a Python dictionary.

238 """

239

240

241class DuplicateKeyException(UnpackException):

242 "Duplicate key encountered during map unpacking."

243

244

245# Backwards compatibility

246KeyNotPrimitiveException = UnhashableKeyException

247KeyDuplicateException = DuplicateKeyException

248

249#############################################################################

250# Exported Functions and Glob

251#############################################################################

252

253# Exported functions and variables, set up in __init()

254pack = None

255packb = None

256unpack = None

257unpackb = None

258dump = None

259dumps = None

260load = None

261loads = None

262

263compatibility = False

264"""

265Compatibility mode boolean.

266

267When compatibility mode is enabled, u-msgpack-python will serialize both

268unicode strings and bytes into the old "raw" msgpack type, and deserialize the

269"raw" msgpack type into bytes. This provides backwards compatibility with the

270old MessagePack specification.

271

272Example:

273>>> umsgpack.compatibility = True

274>>>

275>>> umsgpack.packb([u"some string", b"some bytes"])

276b'\x92\xabsome string\xaasome bytes'

277>>> umsgpack.unpackb(_)

278[b'some string', b'some bytes']

279>>>

280"""

281

282##############################################################################

283# Packing

284##############################################################################

285

286# You may notice struct.pack("B", obj) instead of the simpler chr(obj) in the

287# code below. This is to allow for seamless Python 2 and 3 compatibility, as

288# chr(obj) has a str return type instead of bytes in Python 3, and

289# struct.pack(...) has the right return type in both versions.

290

291

292def _pack_integer(obj, fp, options):

293 if obj < 0:

294 if obj >= -32:

295 fp.write(struct.pack("b", obj))

296 elif obj >= -2**(8 - 1):

297 fp.write(b"\xd0" + struct.pack("b", obj))

298 elif obj >= -2**(16 - 1):

299 fp.write(b"\xd1" + struct.pack(">h", obj))

300 elif obj >= -2**(32 - 1):

301 fp.write(b"\xd2" + struct.pack(">i", obj))

302 elif obj >= -2**(64 - 1):

303 fp.write(b"\xd3" + struct.pack(">q", obj))

304 else:

305 raise UnsupportedTypeException("huge signed int")

306 else:

307 if obj < 128:

308 fp.write(struct.pack("B", obj))

309 elif obj < 2**8:

310 fp.write(b"\xcc" + struct.pack("B", obj))

311 elif obj < 2**16:

312 fp.write(b"\xcd" + struct.pack(">H", obj))

313 elif obj < 2**32:

314 fp.write(b"\xce" + struct.pack(">I", obj))

315 elif obj < 2**64:

316 fp.write(b"\xcf" + struct.pack(">Q", obj))

317 else:

318 raise UnsupportedTypeException("huge unsigned int")

319

320

321def _pack_nil(obj, fp, options):

322 fp.write(b"\xc0")

323

324

325def _pack_boolean(obj, fp, options):

326 fp.write(b"\xc3" if obj else b"\xc2")

327

328

329def _pack_float(obj, fp, options):

330 float_precision = options.get('force_float_precision', _float_precision)

331

332 if float_precision == "double":

333 fp.write(b"\xcb" + struct.pack(">d", obj))

334 elif float_precision == "single":

335 fp.write(b"\xca" + struct.pack(">f", obj))

336 else:

337 raise ValueError("invalid float precision")

338

339

340def _pack_string(obj, fp, options):

341 obj = obj.encode('utf-8')

342 obj_len = len(obj)

343 if obj_len < 32:

344 fp.write(struct.pack("B", 0xa0 | obj_len) + obj)

345 elif obj_len < 2**8:

346 fp.write(b"\xd9" + struct.pack("B", obj_len) + obj)

347 elif obj_len < 2**16:

348 fp.write(b"\xda" + struct.pack(">H", obj_len) + obj)

349 elif obj_len < 2**32:

350 fp.write(b"\xdb" + struct.pack(">I", obj_len) + obj)

351 else:

352 raise UnsupportedTypeException("huge string")

353

354

355def _pack_binary(obj, fp, options):

356 obj_len = len(obj)

357 if obj_len < 2**8:

358 fp.write(b"\xc4" + struct.pack("B", obj_len) + obj)

359 elif obj_len < 2**16:

360 fp.write(b"\xc5" + struct.pack(">H", obj_len) + obj)

361 elif obj_len < 2**32:

362 fp.write(b"\xc6" + struct.pack(">I", obj_len) + obj)

363 else:

364 raise UnsupportedTypeException("huge binary string")

365

366

367def _pack_oldspec_raw(obj, fp, options):

368 obj_len = len(obj)

369 if obj_len < 32:

370 fp.write(struct.pack("B", 0xa0 | obj_len) + obj)

371 elif obj_len < 2**16:

372 fp.write(b"\xda" + struct.pack(">H", obj_len) + obj)

373 elif obj_len < 2**32:

374 fp.write(b"\xdb" + struct.pack(">I", obj_len) + obj)

375 else:

376 raise UnsupportedTypeException("huge raw string")

377

378

379def _pack_ext(obj, fp, options):

380 obj_len = len(obj.data)

381 if obj_len == 1:

382 fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data)

383 elif obj_len == 2:

384 fp.write(b"\xd5" + struct.pack("B", obj.type & 0xff) + obj.data)

385 elif obj_len == 4:

386 fp.write(b"\xd6" + struct.pack("B", obj.type & 0xff) + obj.data)

387 elif obj_len == 8:

388 fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data)

389 elif obj_len == 16:

390 fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data)

391 elif obj_len < 2**8:

392 fp.write(b"\xc7" + struct.pack("BB", obj_len, obj.type & 0xff) + obj.data)

393 elif obj_len < 2**16:

394 fp.write(b"\xc8" + struct.pack(">HB", obj_len, obj.type & 0xff) + obj.data)

395 elif obj_len < 2**32:

396 fp.write(b"\xc9" + struct.pack(">IB", obj_len, obj.type & 0xff) + obj.data)

397 else:

398 raise UnsupportedTypeException("huge ext data")

399

400

401def _pack_ext_timestamp(obj, fp, options):

402 if not obj.tzinfo:

403 # Object is naive datetime, convert to aware date time,

404 # assuming UTC timezone

405 delta = obj.replace(tzinfo=_utc_tzinfo) - _epoch

406 else:

407 # Object is aware datetime

408 delta = obj - _epoch

409

410 seconds = delta.seconds + delta.days * 86400

411 microseconds = delta.microseconds

412

413 if microseconds == 0 and 0 <= seconds <= 2**32 - 1:

414 # 32-bit timestamp

415 fp.write(b"\xd6\xff" + struct.pack(">I", seconds))

416 elif 0 <= seconds <= 2**34 - 1:

417 # 64-bit timestamp

418 value = ((microseconds * 1000) << 34) | seconds

419 fp.write(b"\xd7\xff" + struct.pack(">Q", value))

420 elif -2**63 <= abs(seconds) <= 2**63 - 1:

421 # 96-bit timestamp

422 fp.write(b"\xc7\x0c\xff" + struct.pack(">Iq", microseconds * 1000, seconds))

423 else:

424 raise UnsupportedTypeException("huge timestamp")

425

426

427def _pack_array(obj, fp, options):

428 obj_len = len(obj)

429 if obj_len < 16:

430 fp.write(struct.pack("B", 0x90 | obj_len))

431 elif obj_len < 2**16:

432 fp.write(b"\xdc" + struct.pack(">H", obj_len))

433 elif obj_len < 2**32:

434 fp.write(b"\xdd" + struct.pack(">I", obj_len))

435 else:

436 raise UnsupportedTypeException("huge array")

437

438 for e in obj:

439 pack(e, fp, **options)

440

441

442def _pack_map(obj, fp, options):

443 obj_len = len(obj)

444 if obj_len < 16:

445 fp.write(struct.pack("B", 0x80 | obj_len))

446 elif obj_len < 2**16:

447 fp.write(b"\xde" + struct.pack(">H", obj_len))

448 elif obj_len < 2**32:

449 fp.write(b"\xdf" + struct.pack(">I", obj_len))

450 else:

451 raise UnsupportedTypeException("huge array")

452

453 for k, v in obj.items():

454 pack(k, fp, **options)

455 pack(v, fp, **options)

456

457########################################

458

459

460# Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type

461def _pack2(obj, fp, **options):

462 """

463 Serialize a Python object into MessagePack bytes.

464

465 Args:

466 obj: a Python object

467 fp: a .write()-supporting file-like object

468

469 Kwargs:

470 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type

471 to a callable that packs an instance of the type

472 into an Ext object

473 force_float_precision (str): "single" to force packing floats as

474 IEEE-754 single-precision floats,

475 "double" to force packing floats as

476 IEEE-754 double-precision floats.

477

478 Returns:

479 None.

480

481 Raises:

482 UnsupportedType(PackException):

483 Object type not supported for packing.

484

485 Example:

486 >>> f = open('test.bin', 'wb')

487 >>> umsgpack.pack({u"compact": True, u"schema": 0}, f)

488 >>>

489 """

490 global compatibility

491

492 ext_handlers = options.get("ext_handlers")

493

494 if obj is None:

495 _pack_nil(obj, fp, options)

496 elif ext_handlers and obj.__class__ in ext_handlers:

497 _pack_ext(ext_handlers[obj.__class__](obj), fp, options)

498 elif obj.__class__ in _ext_class_to_type:

499 try:

500 _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options)

501 except AttributeError:

502 raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__)))

503 elif isinstance(obj, bool):

504 _pack_boolean(obj, fp, options)

505 elif isinstance(obj, (int, long)):

506 _pack_integer(obj, fp, options)

507 elif isinstance(obj, float):

508 _pack_float(obj, fp, options)

509 elif compatibility and isinstance(obj, unicode):

510 _pack_oldspec_raw(bytes(obj), fp, options)

511 elif compatibility and isinstance(obj, bytes):

512 _pack_oldspec_raw(obj, fp, options)

513 elif isinstance(obj, unicode):

514 _pack_string(obj, fp, options)

515 elif isinstance(obj, str):

516 _pack_binary(obj, fp, options)

517 elif isinstance(obj, (list, tuple)):

518 _pack_array(obj, fp, options)

519 elif isinstance(obj, dict):

520 _pack_map(obj, fp, options)

521 elif isinstance(obj, datetime.datetime):

522 _pack_ext_timestamp(obj, fp, options)

523 elif isinstance(obj, Ext):

524 _pack_ext(obj, fp, options)

525 elif ext_handlers:

526 # Linear search for superclass

527 t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None)

528 if t:

529 _pack_ext(ext_handlers[t](obj), fp, options)

530 else:

531 raise UnsupportedTypeException(

532 "unsupported type: {:s}".format(str(type(obj))))

533 elif _ext_class_to_type:

534 # Linear search for superclass

535 t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None)

536 if t:

537 try:

538 _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options)

539 except AttributeError:

540 raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t)))

541 else:

542 raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj))))

543 else:

544 raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj))))

545

546

547# Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type

548def _pack3(obj, fp, **options):

549 """

550 Serialize a Python object into MessagePack bytes.

551

552 Args:

553 obj: a Python object

554 fp: a .write()-supporting file-like object

555

556 Kwargs:

557 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type

558 to a callable that packs an instance of the type

559 into an Ext object

560 force_float_precision (str): "single" to force packing floats as

561 IEEE-754 single-precision floats,

562 "double" to force packing floats as

563 IEEE-754 double-precision floats.

564

565 Returns:

566 None.

567

568 Raises:

569 UnsupportedType(PackException):

570 Object type not supported for packing.

571

572 Example:

573 >>> f = open('test.bin', 'wb')

574 >>> umsgpack.pack({u"compact": True, u"schema": 0}, f)

575 >>>

576 """

577 global compatibility

578

579 ext_handlers = options.get("ext_handlers")

580

581 if obj is None:

582 _pack_nil(obj, fp, options)

583 elif ext_handlers and obj.__class__ in ext_handlers:

584 _pack_ext(ext_handlers[obj.__class__](obj), fp, options)

585 elif obj.__class__ in _ext_class_to_type:

586 try:

587 _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options)

588 except AttributeError:

589 raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__)))

590 elif isinstance(obj, bool):

591 _pack_boolean(obj, fp, options)

592 elif isinstance(obj, int):

593 _pack_integer(obj, fp, options)

594 elif isinstance(obj, float):

595 _pack_float(obj, fp, options)

596 elif compatibility and isinstance(obj, str):

597 _pack_oldspec_raw(obj.encode('utf-8'), fp, options)

598 elif compatibility and isinstance(obj, bytes):

599 _pack_oldspec_raw(obj, fp, options)

600 elif isinstance(obj, str):

601 _pack_string(obj, fp, options)

602 elif isinstance(obj, bytes):

603 _pack_binary(obj, fp, options)

604 elif isinstance(obj, (list, tuple)):

605 _pack_array(obj, fp, options)

606 elif isinstance(obj, dict):

607 _pack_map(obj, fp, options)

608 elif isinstance(obj, datetime.datetime):

609 _pack_ext_timestamp(obj, fp, options)

610 elif isinstance(obj, Ext):

611 _pack_ext(obj, fp, options)

612 elif ext_handlers:

613 # Linear search for superclass

614 t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None)

615 if t:

616 _pack_ext(ext_handlers[t](obj), fp, options)

617 else:

618 raise UnsupportedTypeException(

619 "unsupported type: {:s}".format(str(type(obj))))

620 elif _ext_class_to_type:

621 # Linear search for superclass

622 t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None)

623 if t:

624 try:

625 _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options)

626 except AttributeError:

627 raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t)))

628 else:

629 raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj))))

630 #@@gws

631 elif options.get("default"):

632 _pack3(options.get("default")(obj), fp, **options)

633 #@@gws

634

635 else:

636 raise UnsupportedTypeException(

637 "unsupported type: {:s}".format(str(type(obj))))

638

639

640def _packb2(obj, **options):

641 """

642 Serialize a Python object into MessagePack bytes.

643

644 Args:

645 obj: a Python object

646

647 Kwargs:

648 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type

649 to a callable that packs an instance of the type

650 into an Ext object

651 force_float_precision (str): "single" to force packing floats as

652 IEEE-754 single-precision floats,

653 "double" to force packing floats as

654 IEEE-754 double-precision floats.

655

656 Returns:

657 A 'str' containing serialized MessagePack bytes.

658

659 Raises:

660 UnsupportedType(PackException):

661 Object type not supported for packing.

662

663 Example:

664 >>> umsgpack.packb({u"compact": True, u"schema": 0})

665 '\x82\xa7compact\xc3\xa6schema\x00'

666 >>>

667 """

668 fp = io.BytesIO()

669 _pack2(obj, fp, **options)

670 return fp.getvalue()

671

672

673def _packb3(obj, **options):

674 """

675 Serialize a Python object into MessagePack bytes.

676

677 Args:

678 obj: a Python object

679

680 Kwargs:

681 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type

682 to a callable that packs an instance of the type

683 into an Ext object

684 force_float_precision (str): "single" to force packing floats as

685 IEEE-754 single-precision floats,

686 "double" to force packing floats as

687 IEEE-754 double-precision floats.

688

689 Returns:

690 A 'bytes' containing serialized MessagePack bytes.

691

692 Raises:

693 UnsupportedType(PackException):

694 Object type not supported for packing.

695

696 Example:

697 >>> umsgpack.packb({u"compact": True, u"schema": 0})

698 b'\x82\xa7compact\xc3\xa6schema\x00'

699 >>>

700 """

701 fp = io.BytesIO()

702 _pack3(obj, fp, **options)

703 return fp.getvalue()

704

705#############################################################################

706# Unpacking

707#############################################################################

708

709

710def _read_except(fp, n):

711 if n == 0:

712 return b""

713

714 data = fp.read(n)

715 if len(data) == 0:

716 raise InsufficientDataException()

717

718 while len(data) < n:

719 chunk = fp.read(n - len(data))

720 if len(chunk) == 0:

721 raise InsufficientDataException()

723 data += chunk

725 return data

728def _unpack_integer(code, fp, options):

729 if (ord(code) & 0xe0) == 0xe0:

730 return struct.unpack("b", code)[0]

731 elif code == b'\xd0':

732 return struct.unpack("b", _read_except(fp, 1))[0]

733 elif code == b'\xd1':

734 return struct.unpack(">h", _read_except(fp, 2))[0]

735 elif code == b'\xd2':

736 return struct.unpack(">i", _read_except(fp, 4))[0]

737 elif code == b'\xd3':

738 return struct.unpack(">q", _read_except(fp, 8))[0]

739 elif (ord(code) & 0x80) == 0x00:

740 return struct.unpack("B", code)[0]

741 elif code == b'\xcc':

742 return struct.unpack("B", _read_except(fp, 1))[0]

743 elif code == b'\xcd':

744 return struct.unpack(">H", _read_except(fp, 2))[0]

745 elif code == b'\xce':

746 return struct.unpack(">I", _read_except(fp, 4))[0]

747 elif code == b'\xcf':

748 return struct.unpack(">Q", _read_except(fp, 8))[0]

749 raise Exception("logic error, not int: 0x{:02x}".format(ord(code)))

750

751

752def _unpack_reserved(code, fp, options):

753 if code == b'\xc1':

754 raise ReservedCodeException(

755 "encountered reserved code: 0x{:02x}".format(ord(code)))

756 raise Exception(

757 "logic error, not reserved code: 0x{:02x}".format(ord(code)))

758

759

760def _unpack_nil(code, fp, options):

761 if code == b'\xc0':

762 return None

763 raise Exception("logic error, not nil: 0x{:02x}".format(ord(code)))

764

765

766def _unpack_boolean(code, fp, options):

767 if code == b'\xc2':

768 return False

769 elif code == b'\xc3':

770 return True

771 raise Exception("logic error, not boolean: 0x{:02x}".format(ord(code)))

772

773

774def _unpack_float(code, fp, options):

775 if code == b'\xca':

776 return struct.unpack(">f", _read_except(fp, 4))[0]

777 elif code == b'\xcb':

778 return struct.unpack(">d", _read_except(fp, 8))[0]

779 raise Exception("logic error, not float: 0x{:02x}".format(ord(code)))

780

781

782def _unpack_string(code, fp, options):

783 if (ord(code) & 0xe0) == 0xa0:

784 length = ord(code) & ~0xe0

785 elif code == b'\xd9':

786 length = struct.unpack("B", _read_except(fp, 1))[0]

787 elif code == b'\xda':

788 length = struct.unpack(">H", _read_except(fp, 2))[0]

789 elif code == b'\xdb':

790 length = struct.unpack(">I", _read_except(fp, 4))[0]

791 else:

792 raise Exception("logic error, not string: 0x{:02x}".format(ord(code)))

793

794 # Always return raw bytes in compatibility mode

795 global compatibility

796 if compatibility:

797 return _read_except(fp, length)

798

799 data = _read_except(fp, length)

800 try:

801 return bytes.decode(data, 'utf-8')

802 except UnicodeDecodeError:

803 if options.get("allow_invalid_utf8"):

804 return InvalidString(data)

805 raise InvalidStringException("unpacked string is invalid utf-8")

806

807

808def _unpack_binary(code, fp, options):

809 if code == b'\xc4':

810 length = struct.unpack("B", _read_except(fp, 1))[0]

811 elif code == b'\xc5':

812 length = struct.unpack(">H", _read_except(fp, 2))[0]

813 elif code == b'\xc6':

814 length = struct.unpack(">I", _read_except(fp, 4))[0]

815 else:

816 raise Exception("logic error, not binary: 0x{:02x}".format(ord(code)))

817

818 return _read_except(fp, length)

819

820

821def _unpack_ext(code, fp, options):

822 if code == b'\xd4':

823 length = 1

824 elif code == b'\xd5':

825 length = 2

826 elif code == b'\xd6':

827 length = 4

828 elif code == b'\xd7':

829 length = 8

830 elif code == b'\xd8':

831 length = 16

832 elif code == b'\xc7':

833 length = struct.unpack("B", _read_except(fp, 1))[0]

834 elif code == b'\xc8':

835 length = struct.unpack(">H", _read_except(fp, 2))[0]

836 elif code == b'\xc9':

837 length = struct.unpack(">I", _read_except(fp, 4))[0]

838 else:

839 raise Exception("logic error, not ext: 0x{:02x}".format(ord(code)))

840

841 ext_type = struct.unpack("b", _read_except(fp, 1))[0]

842 ext_data = _read_except(fp, length)

843

844 # Unpack with ext handler, if we have one

845 ext_handlers = options.get("ext_handlers")

846 if ext_handlers and ext_type in ext_handlers:

847 return ext_handlers[ext_type](Ext(ext_type, ext_data))

848

849 # Unpack with ext classes, if type is registered

850 if ext_type in _ext_type_to_class:

851 try:

852 return _ext_type_to_class[ext_type].unpackb(ext_data)

853 except AttributeError:

854 raise NotImplementedError("Ext serializable class {:s} is missing implementation of unpackb()".format(repr(_ext_type_to_class[ext_type])))

855

856 # Timestamp extension

857 if ext_type == -1:

858 return _unpack_ext_timestamp(ext_data, options)

859

860 return Ext(ext_type, ext_data)

861

862

863def _unpack_ext_timestamp(ext_data, options):

864 obj_len = len(ext_data)

865 if obj_len == 4:

866 # 32-bit timestamp

867 seconds = struct.unpack(">I", ext_data)[0]

868 microseconds = 0

869 elif obj_len == 8:

870 # 64-bit timestamp

871 value = struct.unpack(">Q", ext_data)[0]

872 seconds = value & 0x3ffffffff

873 microseconds = (value >> 34) // 1000

874 elif obj_len == 12:

875 # 96-bit timestamp

876 seconds = struct.unpack(">q", ext_data[4:12])[0]

877 microseconds = struct.unpack(">I", ext_data[0:4])[0] // 1000

878 else:

879 raise UnsupportedTimestampException(

880 "unsupported timestamp with data length {:d}".format(len(ext_data)))

881

882 return _epoch + datetime.timedelta(seconds=seconds,

883 microseconds=microseconds)

884

885

886def _unpack_array(code, fp, options):

887 if (ord(code) & 0xf0) == 0x90:

888 length = (ord(code) & ~0xf0)

889 elif code == b'\xdc':

890 length = struct.unpack(">H", _read_except(fp, 2))[0]

891 elif code == b'\xdd':

892 length = struct.unpack(">I", _read_except(fp, 4))[0]

893 else:

894 raise Exception("logic error, not array: 0x{:02x}".format(ord(code)))

895

896 if options.get('use_tuple'):

897 return tuple((_unpack(fp, options) for i in xrange(length)))

898

899 return [_unpack(fp, options) for i in xrange(length)]

900

901

902def _deep_list_to_tuple(obj):

903 if isinstance(obj, list):

904 return tuple([_deep_list_to_tuple(e) for e in obj])

905 return obj

906

907

908def _unpack_map(code, fp, options):

909 if (ord(code) & 0xf0) == 0x80:

910 length = (ord(code) & ~0xf0)

911 elif code == b'\xde':

912 length = struct.unpack(">H", _read_except(fp, 2))[0]

913 elif code == b'\xdf':

914 length = struct.unpack(">I", _read_except(fp, 4))[0]

915 else:

916 raise Exception("logic error, not map: 0x{:02x}".format(ord(code)))

917

918 d = {} if not options.get('use_ordered_dict') else collections.OrderedDict()

919 for _ in xrange(length):

920 # Unpack key

921 k = _unpack(fp, options)

922

923 if isinstance(k, list):

924 # Attempt to convert list into a hashable tuple

925 k = _deep_list_to_tuple(k)

926 elif not isinstance(k, Hashable):

927 raise UnhashableKeyException(

928 "encountered unhashable key: \"{:s}\" ({:s})".format(str(k), str(type(k))))

929 elif k in d:

930 raise DuplicateKeyException(

931 "encountered duplicate key: \"{:s}\" ({:s})".format(str(k), str(type(k))))

932

933 # Unpack value

934 v = _unpack(fp, options)

935

936 try:

937 d[k] = v

938 except TypeError:

939 raise UnhashableKeyException(

940 "encountered unhashable key: \"{:s}\"".format(str(k)))

941 return d

942

943

944def _unpack(fp, options):

945 code = _read_except(fp, 1)

946 return _unpack_dispatch_table[code](code, fp, options)

947

948########################################

949

950

951def _unpack2(fp, **options):

952 """

953 Deserialize MessagePack bytes into a Python object.

954

955 Args:

956 fp: a .read()-supporting file-like object

957

958 Kwargs:

959 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext

960 type to a callable that unpacks an instance of

961 Ext into an object

962 use_ordered_dict (bool): unpack maps into OrderedDict, instead of

963 unordered dict (default False)

964 use_tuple (bool): unpacks arrays into tuples, instead of lists (default

965 False)

966 allow_invalid_utf8 (bool): unpack invalid strings into instances of

967 InvalidString, for access to the bytes

968 (default False)

969

970 Returns:

971 A Python object.

972

973 Raises:

974 InsufficientDataException(UnpackException):

975 Insufficient data to unpack the serialized object.

976 InvalidStringException(UnpackException):

977 Invalid UTF-8 string encountered during unpacking.

978 UnsupportedTimestampException(UnpackException):

979 Unsupported timestamp format encountered during unpacking.

980 ReservedCodeException(UnpackException):

981 Reserved code encountered during unpacking.

982 UnhashableKeyException(UnpackException):

983 Unhashable key encountered during map unpacking.

984 The serialized map cannot be deserialized into a Python dictionary.

985 DuplicateKeyException(UnpackException):

986 Duplicate key encountered during map unpacking.

987

988 Example:

989 >>> f = open('test.bin', 'rb')

990 >>> umsgpack.unpackb(f)

991 {u'compact': True, u'schema': 0}

992 >>>

993 """

994 return _unpack(fp, options)

995

996

997def _unpack3(fp, **options):

998 """

999 Deserialize MessagePack bytes into a Python object.

1000

1001 Args:

1002 fp: a .read()-supporting file-like object

1003

1004 Kwargs:

1005 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext

1006 type to a callable that unpacks an instance of

1007 Ext into an object

1008 use_ordered_dict (bool): unpack maps into OrderedDict, instead of

1009 unordered dict (default False)

1010 use_tuple (bool): unpacks arrays into tuples, instead of lists (default

1011 False)

1012 allow_invalid_utf8 (bool): unpack invalid strings into instances of

1013 InvalidString, for access to the bytes

1014 (default False)

1015

1016 Returns:

1017 A Python object.

1018

1019 Raises:

1020 InsufficientDataException(UnpackException):

1021 Insufficient data to unpack the serialized object.

1022 InvalidStringException(UnpackException):

1023 Invalid UTF-8 string encountered during unpacking.

1024 UnsupportedTimestampException(UnpackException):

1025 Unsupported timestamp format encountered during unpacking.

1026 ReservedCodeException(UnpackException):

1027 Reserved code encountered during unpacking.

1028 UnhashableKeyException(UnpackException):

1029 Unhashable key encountered during map unpacking.

1030 The serialized map cannot be deserialized into a Python dictionary.

1031 DuplicateKeyException(UnpackException):

1032 Duplicate key encountered during map unpacking.

1033

1034 Example:

1035 >>> f = open('test.bin', 'rb')

1036 >>> umsgpack.unpackb(f)

1037 {'compact': True, 'schema': 0}

1038 >>>

1039 """

1040 return _unpack(fp, options)

1041

1042

1043# For Python 2, expects a str object

1044def _unpackb2(s, **options):

1045 """

1046 Deserialize MessagePack bytes into a Python object.

1047

1048 Args:

1049 s: a 'str' or 'bytearray' containing serialized MessagePack bytes

1050

1051 Kwargs:

1052 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext

1053 type to a callable that unpacks an instance of

1054 Ext into an object

1055 use_ordered_dict (bool): unpack maps into OrderedDict, instead of

1056 unordered dict (default False)

1057 use_tuple (bool): unpacks arrays into tuples, instead of lists (default

1058 False)

1059 allow_invalid_utf8 (bool): unpack invalid strings into instances of

1060 InvalidString, for access to the bytes

1061 (default False)

1062

1063 Returns:

1064 A Python object.

1065

1066 Raises:

1067 TypeError:

1068 Packed data type is neither 'str' nor 'bytearray'.

1069 InsufficientDataException(UnpackException):

1070 Insufficient data to unpack the serialized object.

1071 InvalidStringException(UnpackException):

1072 Invalid UTF-8 string encountered during unpacking.

1073 UnsupportedTimestampException(UnpackException):

1074 Unsupported timestamp format encountered during unpacking.

1075 ReservedCodeException(UnpackException):

1076 Reserved code encountered during unpacking.

1077 UnhashableKeyException(UnpackException):

1078 Unhashable key encountered during map unpacking.

1079 The serialized map cannot be deserialized into a Python dictionary.

1080 DuplicateKeyException(UnpackException):

1081 Duplicate key encountered during map unpacking.

1082

1083 Example:

1084 >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00')

1085 {u'compact': True, u'schema': 0}

1086 >>>

1087 """

1088 if not isinstance(s, (str, bytearray)):

1089 raise TypeError("packed data must be type 'str' or 'bytearray'")

1090 return _unpack(io.BytesIO(s), options)

1091

1092

1093# For Python 3, expects a bytes object

1094def _unpackb3(s, **options):

1095 """

1096 Deserialize MessagePack bytes into a Python object.

1097

1098 Args:

1099 s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes

1100

1101 Kwargs:

1102 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext

1103 type to a callable that unpacks an instance of

1104 Ext into an object

1105 use_ordered_dict (bool): unpack maps into OrderedDict, instead of

1106 unordered dict (default False)

1107 use_tuple (bool): unpacks arrays into tuples, instead of lists (default

1108 False)

1109 allow_invalid_utf8 (bool): unpack invalid strings into instances of

1110 InvalidString, for access to the bytes

1111 (default False)

1112

1113 Returns:

1114 A Python object.

1115

1116 Raises:

1117 TypeError:

1118 Packed data type is neither 'bytes' nor 'bytearray'.

1119 InsufficientDataException(UnpackException):

1120 Insufficient data to unpack the serialized object.

1121 InvalidStringException(UnpackException):

1122 Invalid UTF-8 string encountered during unpacking.

1123 UnsupportedTimestampException(UnpackException):

1124 Unsupported timestamp format encountered during unpacking.

1125 ReservedCodeException(UnpackException):

1126 Reserved code encountered during unpacking.

1127 UnhashableKeyException(UnpackException):

1128 Unhashable key encountered during map unpacking.

1129 The serialized map cannot be deserialized into a Python dictionary.

1130 DuplicateKeyException(UnpackException):

1131 Duplicate key encountered during map unpacking.

1132

1133 Example:

1134 >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00')

1135 {'compact': True, 'schema': 0}

1136 >>>

1137 """

1138 if not isinstance(s, (bytes, bytearray)):

1139 raise TypeError("packed data must be type 'bytes' or 'bytearray'")

1140 return _unpack(io.BytesIO(s), options)

1141

1142#############################################################################

1143# Module Initialization

1144#############################################################################

1145

1146

1147def __init():

1148 global pack

1149 global packb

1150 global unpack

1151 global unpackb

1152 global dump

1153 global dumps

1154 global load

1155 global loads

1156 global compatibility

1157 global _epoch

1158 global _utc_tzinfo

1159 global _float_precision

1160 global _unpack_dispatch_table

1161 global xrange

1162

1163 # Compatibility mode for handling strings/bytes with the old specification

1164 compatibility = False

1165

1166 if sys.version_info[0] == 3:

1167 _utc_tzinfo = datetime.timezone.utc

1168 else:

1169 class UTC(datetime.tzinfo):

1170 ZERO = datetime.timedelta(0)

1171

1172 def utcoffset(self, dt):

1173 return UTC.ZERO

1174

1175 def tzname(self, dt):

1176 return "UTC"

1177

1178 def dst(self, dt):

1179 return UTC.ZERO

1180

1181 _utc_tzinfo = UTC()

1182

1183 # Calculate an aware epoch datetime

1184 _epoch = datetime.datetime(1970, 1, 1, tzinfo=_utc_tzinfo)

1185

1186 # Auto-detect system float precision

1187 if sys.float_info.mant_dig == 53:

1188 _float_precision = "double"

1189 else:

1190 _float_precision = "single"

1191

1192 # Map packb and unpackb to the appropriate version

1193 if sys.version_info[0] == 3:

1194 pack = _pack3

1195 packb = _packb3

1196 dump = _pack3

1197 dumps = _packb3

1198 unpack = _unpack3

1199 unpackb = _unpackb3

1200 load = _unpack3

1201 loads = _unpackb3

1202 xrange = range

1203 else:

1204 pack = _pack2

1205 packb = _packb2

1206 dump = _pack2

1207 dumps = _packb2

1208 unpack = _unpack2

1209 unpackb = _unpackb2

1210 load = _unpack2

1211 loads = _unpackb2

1212

1213 # Build a dispatch table for fast lookup of unpacking function

1214

1215 _unpack_dispatch_table = {}

1216 # Fix uint

1217 for code in range(0, 0x7f + 1):

1218 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer

1219 # Fix map

1220 for code in range(0x80, 0x8f + 1):

1221 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_map

1222 # Fix array

1223 for code in range(0x90, 0x9f + 1):

1224 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_array

1225 # Fix str

1226 for code in range(0xa0, 0xbf + 1):

1227 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string

1228 # Nil

1229 _unpack_dispatch_table[b'\xc0'] = _unpack_nil

1230 # Reserved

1231 _unpack_dispatch_table[b'\xc1'] = _unpack_reserved

1232 # Boolean

1233 _unpack_dispatch_table[b'\xc2'] = _unpack_boolean

1234 _unpack_dispatch_table[b'\xc3'] = _unpack_boolean

1235 # Bin

1236 for code in range(0xc4, 0xc6 + 1):

1237 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_binary

1238 # Ext

1239 for code in range(0xc7, 0xc9 + 1):

1240 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext

1241 # Float

1242 _unpack_dispatch_table[b'\xca'] = _unpack_float

1243 _unpack_dispatch_table[b'\xcb'] = _unpack_float

1244 # Uint

1245 for code in range(0xcc, 0xcf + 1):

1246 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer

1247 # Int

1248 for code in range(0xd0, 0xd3 + 1):

1249 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer

1250 # Fixext

1251 for code in range(0xd4, 0xd8 + 1):

1252 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext

1253 # String

1254 for code in range(0xd9, 0xdb + 1):

1255 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string

1256 # Array

1257 _unpack_dispatch_table[b'\xdc'] = _unpack_array

1258 _unpack_dispatch_table[b'\xdd'] = _unpack_array

1259 # Map

1260 _unpack_dispatch_table[b'\xde'] = _unpack_map

1261 _unpack_dispatch_table[b'\xdf'] = _unpack_map

1262 # Negative fixint

1263 for code in range(0xe0, 0xff + 1):

1264 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer

1265

1266

1267__init()