Coverage for gws-app/gws/spec/reader.py: 18%

1"""Read and validate values according to spec types.""" 

2 

3import re 

4 

5import gws 

6import gws.gis.crs 

7import gws.lib.datetimex 

8import gws.lib.osx 

9import gws.lib.uom 

10 

11from . import core 

12 

13 

14class Reader: 

15 atom = core.Type(c=core.C.ATOM) 

16 

17 def __init__(self, runtime, path, options): 

18 self.runtime = runtime 

19 self.path = path 

20 

21 options = set(options or []) 

22 

23 self.accept_extra_props = gws.SpecReadOption.acceptExtraProps in options 

24 self.case_insensitive = gws.SpecReadOption.caseInsensitive in options 

25 self.convert_values = gws.SpecReadOption.convertValues in options 

26 self.ignore_extra_props = gws.SpecReadOption.ignoreExtraProps in options 

27 self.allow_skip_required = gws.SpecReadOption.allowMissing in options 

28 self.verbose_errors = gws.SpecReadOption.verboseErrors in options 

29 

30 self.stack = None 

31 self.push = lambda _: ... 

32 self.pop = lambda: ... 

33 

34 def read(self, value, type_uid): 

35 

36 if not self.verbose_errors: 

37 return self.read2(value, type_uid) 

38 

39 self.stack = [('', value, type_uid)] 

40 self.push = self.stack.append 

41 self.pop = self.stack.pop 

42 

43 try: 

44 return self.read2(value, type_uid) 

45 except core.ReadError as exc: 

46 raise self.add_error_details(exc) 

47 

48 def read2(self, value, type_uid): 

49 typ = self.runtime.get_type(type_uid) 

50 

51 if type_uid in _READERS: 

52 return _READERS[type_uid](self, value, typ or self.atom) 

53 

54 if not typ: 

55 raise core.ReadError(f'unknown type {type_uid!r}', value) 

56 

57 if typ.c not in _READERS: 

58 raise core.ReadError(f'unknown type category {typ.c!r}', value) 

59 

60 return _READERS[typ.c](self, value, typ) 

61 

62 def add_error_details(self, exc: Exception): 

63 details = { 

64 'formatted_value': _format_error_value(exc), 

65 'path': self.path, 

66 'formatted_stack': _format_error_stack(self.stack or []) 

67 } 

68 exc.args = (exc.args[0], exc.args[1], details) 

69 return exc 

70 

71 

72# atoms 

73 

74def _read_any(r: Reader, val, typ: core.Type): 

75 return val 

76 

77 

78def _read_bool(r: Reader, val, typ: core.Type): 

79 if not r.convert_values: 

80 return _ensure(val, bool) 

81 try: 

82 return bool(val) 

83 except: 

84 raise core.ReadError('must be true or false', val) 

85 

86 

87def _read_bytes(r: Reader, val, typ: core.Type): 

88 try: 

89 if isinstance(val, str): 

90 return val.encode('utf8', errors='strict') 

91 return bytes(val) 

92 except: 

93 raise core.ReadError('must be a byte buffer', val) 

94 

95 

96def _read_float(r: Reader, val, typ: core.Type): 

97 if not r.convert_values: 

98 if isinstance(val, int): 

99 return float(val) 

100 return _ensure(val, float) 

101 try: 

102 return float(val) 

103 except: 

104 raise core.ReadError('must be a float', val) 

105 

106 

107def _read_int(r: Reader, val, typ: core.Type): 

108 if isinstance(val, bool): 

109 raise core.ReadError('must be an integer', val) 

110 if not r.convert_values: 

111 return _ensure(val, int) 

112 try: 

113 return int(val) 

114 except: 

115 raise core.ReadError('must be an integer', val) 

116 

117 

118def _read_str(r: Reader, val, typ: core.Type): 

119 if not r.convert_values: 

120 return _ensure(val, str) 

121 try: 

122 return _to_string(val) 

123 except: 

124 raise core.ReadError('must be a string', val) 

125 

126 

127# built-ins 

128 

129def _read_raw_dict(r: Reader, val, typ: core.Type): 

130 return _ensure(val, dict) 

131 

132 

133def _read_dict(r: Reader, val, typ: core.Type): 

134 dct = {} 

135 for k, v in _ensure(val, dict).items(): 

136 dct[k] = r.read2(v, typ.tValue) 

137 return dct 

138 

139 

140def _read_raw_list(r: Reader, val, typ: core.Type): 

141 return _ensure(val, list) 

142 

143 

144def _read_list(r: Reader, val, typ: core.Type): 

145 lst = _read_any_list(r, val) 

146 res = [] 

147 for n, v in enumerate(lst): 

148 r.push((n, v, typ.tItem)) 

149 res.append(r.read2(v, typ.tItem)) 

150 r.pop() 

151 return res 

152 

153 

154def _read_set(r: Reader, val, typ: core.Type): 

155 lst = _read_list(r, val, typ) 

156 return set(lst) 

157 

158 

159def _read_tuple(r: Reader, val, typ: core.Type): 

160 lst = _read_any_list(r, val) 

161 

162 if len(lst) != len(typ.tItems): 

163 raise core.ReadError(f"expected: {_comma(typ.tItems)}", val) 

164 

165 res = [] 

166 for n, v in enumerate(lst): 

167 r.push((n, v, typ.tItems[n])) 

168 res.append(r.read2(v, typ.tItems[n])) 

169 r.pop() 

170 return res 

171 

172 

173def _read_any_list(r, val): 

174 if r.convert_values and isinstance(val, str): 

175 val = val.strip() 

176 val = [v.strip() for v in val.split(',')] if val else [] 

177 return _ensure(val, list) 

178 

179 

180def _read_literal(r: Reader, val, typ: core.Type): 

181 s = _read_any(r, val, typ) 

182 if s not in typ.literalValues: 

183 raise core.ReadError(f"invalid value: {s!r}, expected: {_comma(typ.literalValues)}", val) 

184 return s 

185 

186 

187def _read_optional(r: Reader, val, typ: core.Type): 

188 if val is None: 

189 return val 

190 return r.read2(val, typ.tTarget) 

191 

192 

193def _read_union(r: Reader, val, typ: core.Type): 

194 # @TODO no untyped unions yet 

195 raise core.ReadError('unions are not supported yet', val) 

196 

197 

198# our types 

199 

200def _read_type(r: Reader, val, typ: core.Type): 

201 return r.read2(val, typ.tTarget) 

202 

203 

204def _read_enum(r: Reader, val, typ: core.Type): 

205 # NB: our Enums accept both names (for configs) and values (for api calls) 

206 # this prevents silly things like Enum{foo=bar bar=123} but we don't care 

207 # 

208 # the comparison is also case-insensitive 

209 # 

210 # this reader returns a value, it's up to the caller to convert it to the actual enum 

211 

212 def _lower(s): 

213 return s.lower() if isinstance(s, str) else s 

214 

215 lv = _lower(val) 

216 

217 for k, v in typ.enumValues.items(): 

218 if lv == _lower(k) or lv == _lower(v): 

219 return v 

220 raise core.ReadError(f"invalid value: {val!r}, expected: {_comma(typ.enumValues)}", val) 

221 

222 

223def _read_object(r: Reader, val, typ: core.Type): 

224 val = _ensure(val, dict) 

225 

226 if r.case_insensitive: 

227 val = {k.lower(): v for k, v in val.items()} 

228 else: 

229 val = dict(val) 

230 

231 res = {} 

232 

233 for prop_name, prop_type_uid in typ.tProperties.items(): 

234 prop_val = val.pop(prop_name.lower() if r.case_insensitive else prop_name, None) 

235 r.push((prop_name, prop_val, prop_type_uid)) 

236 res[prop_name] = r.read2(prop_val, prop_type_uid) 

237 r.pop() 

238 

239 unknown = [] 

240 

241 for k in val: 

242 if k not in typ.tProperties: 

243 if r.accept_extra_props: 

244 res[k] = val[k] 

245 elif r.ignore_extra_props: 

246 continue 

247 else: 

248 unknown.append(k) 

249 

250 if unknown: 

251 raise core.ReadError(f"unknown keys: {_comma(unknown)}, expected: {_comma(typ.tProperties)} for {typ.uid!r}", val) 

252 

253 return gws.Data(res) 

254 

255 

256def _read_property(r: Reader, val, typ: core.Type): 

257 if val is not None: 

258 return r.read2(val, typ.tValue) 

259 

260 if not typ.hasDefault: 

261 if r.allow_skip_required: 

262 return None 

263 raise core.ReadError(f"required property missing: {typ.ident!r} for {typ.tOwner!r}", None) 

264 

265 if typ.defaultValue is None: 

266 return None 

267 

268 # the default, if given, must match the type 

269 # NB, for Data objects, default={} will create an object with defaults 

270 return r.read2(typ.defaultValue, typ.tValue) 

271 

272 

273def _read_variant(r: Reader, val, typ: core.Type): 

274 val = _ensure(val, dict) 

275 if r.case_insensitive: 

276 val = {k.lower(): v for k, v in val.items()} 

277 

278 type_name = val.get(core.VARIANT_TAG, core.DEFAULT_VARIANT_TAG) 

279 target_type_uid = typ.tMembers.get(type_name) 

280 if not target_type_uid: 

281 raise core.ReadError(f"illegal type: {type_name!r}, expected: {_comma(typ.tMembers)}", val) 

282 return r.read2(val, target_type_uid) 

283 

284 

285# custom types 

286 

287def _read_acl_str(r: Reader, val, typ: core.Type): 

288 try: 

289 return gws.u.parse_acl(val) 

290 except ValueError: 

291 raise core.ReadError(f'invalid ACL', val) 

292 

293 

294def _read_color(r: Reader, val, typ: core.Type): 

295 # @TODO: parse color values 

296 return _read_str(r, val, typ) 

297 

298 

299def _read_crs(r: Reader, val, typ: core.Type): 

300 crs = gws.gis.crs.get(val) 

301 if not crs: 

302 raise core.ReadError(f'invalid crs: {val!r}', val) 

303 return crs.srid 

304 

305 

306def _read_date(r: Reader, val, typ: core.Type): 

307 try: 

308 return gws.lib.datetimex.from_string(str(val)) 

309 except ValueError: 

310 raise core.ReadError(f'invalid date: {val!r}', val) 

311 

312 

313def _read_datetime(r: Reader, val, typ: core.Type): 

314 try: 

315 return gws.lib.datetimex.from_iso_string(str(val)) 

316 except ValueError: 

317 raise core.ReadError(f'invalid date: {val!r}', val) 

318 

319 

320def _read_dirpath(r: Reader, val, typ: core.Type): 

321 path = gws.lib.osx.abs_path(val, r.path) 

322 if not gws.u.is_dir(path): 

323 raise core.ReadError(f'directory not found: {path!r}, base {r.path!r}', val) 

324 return path 

325 

326 

327def _read_duration(r: Reader, val, typ: core.Type): 

328 try: 

329 return gws.lib.datetimex.parse_duration(val) 

330 except ValueError: 

331 raise core.ReadError(f'invalid duration: {val!r}', val) 

332 

333 

334def _read_filepath(r: Reader, val, typ: core.Type): 

335 path = gws.lib.osx.abs_path(val, r.path) 

336 if not gws.lib.osx.is_abs_path(val): 

337 gws.log.warning(f'relative path, assuming {path!r} for {val!r}') 

338 if not gws.u.is_file(path): 

339 raise core.ReadError(f'file not found: {path!r}, base {r.path!r}', val) 

340 return path 

341 

342 

343def _read_formatstr(r: Reader, val, typ: core.Type): 

344 # @TODO validate 

345 return _read_str(r, val, typ) 

346 

347 

348def _read_metadata(r: Reader, val, typ: core.Type): 

349 rr = r.allow_skip_required 

350 r.allow_skip_required = True 

351 res = gws.u.compact(_read_object(r, val, typ)) 

352 r.allow_skip_required = rr 

353 return res 

354 

355 

356def _read_uom_value(r: Reader, val, typ: core.Type): 

357 try: 

358 return gws.lib.uom.parse(val) 

359 except ValueError as e: 

360 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

361 

362 

363def _read_uom_value_2(r: Reader, val, typ: core.Type): 

364 try: 

365 ls = [gws.lib.uom.parse(s) for s in gws.u.to_list(val)] 

366 u = set(p[1] for p in ls) 

367 if len(ls) != 2 or len(u) != 1: 

368 raise ValueError('invalid length or unit') 

369 return tuple(p[0] for p in ls) + tuple(u) 

370 except ValueError as e: 

371 raise core.ReadError(f'invalid point: {val!r}: {e!r}', val) 

372 

373 

374def _read_uom_value_4(r: Reader, val, typ: core.Type): 

375 try: 

376 ls = [gws.lib.uom.parse(s) for s in gws.u.to_list(val)] 

377 u = set(p[1] for p in ls) 

378 if len(ls) != 4 or len(u) != 1: 

379 raise ValueError('invalid length or unit') 

380 return tuple(p[0] for p in ls) + tuple(u) 

381 except ValueError as e: 

382 raise core.ReadError(f'invalid extent: {val!r}: {e!r}', val) 

383 

384 

385def _read_regex(r: Reader, val, typ: core.Type): 

386 try: 

387 re.compile(val) 

388 return val 

389 except re.error as e: 

390 raise core.ReadError(f'invalid regular expression: {val!r}: {e!r}', val) 

391 

392 

393def _read_url(r: Reader, val, typ: core.Type): 

394 u = _read_str(r, val, typ) 

395 if u.startswith(('http://', 'https://')): 

396 return u 

397 raise core.ReadError(f'invalid url: {val!r}', val) 

398 

399 

400# utils 

401 

402 

403def _ensure(val, cls): 

404 if isinstance(val, cls): 

405 return val 

406 if cls == list and isinstance(val, tuple): 

407 return list(val) 

408 if cls == dict and gws.u.is_data_object(val): 

409 return vars(val) 

410 raise core.ReadError(f"wrong type: {_classname(type(val))!r}, expected: {_classname(cls)!r}", val) 

411 

412 

413def _to_string(x): 

414 if isinstance(x, str): 

415 return x 

416 if isinstance(x, (bytes, bytearray)): 

417 return x.decode('utf8') 

418 raise ValueError() 

419 

420 

421def _classname(cls): 

422 try: 

423 return cls.__name__ 

424 except: 

425 return str(cls) 

426 

427 

428def _comma(ls): 

429 return repr(', '.join(sorted(str(x) for x in ls))) 

430 

431 

432## 

433 

434 

435def _format_error_value(exc): 

436 try: 

437 val = exc.args[1] 

438 except (AttributeError, IndexError): 

439 return '' 

440 

441 s = repr(val) 

442 if len(s) > 600: 

443 s = s[:600] + '...' 

444 return s 

445 

446 

447def _format_error_stack(stack): 

448 f = [] 

449 

450 for name, value, type_uid in reversed(stack): 

451 line = '' 

452 

453 if name: 

454 name = repr(name) 

455 line = f'item {name}' if name.isdigit() else name 

456 

457 obj = type_uid or 'object' 

458 for p in 'uid', 'title', 'type': 

459 try: 

460 s = value.get(p) 

461 if s is not None: 

462 obj += f' {p}={s!r}' 

463 break 

464 except AttributeError: 

465 pass 

466 

467 f.append(f'in {line} <{obj}>') 

468 

469 return f 

470 

471 

472# 

473 

474_READERS = { 

475 'any': _read_any, 

476 'bool': _read_bool, 

477 'bytes': _read_bytes, 

478 'float': _read_float, 

479 'int': _read_int, 

480 'str': _read_str, 

481 

482 'list': _read_raw_list, 

483 'dict': _read_raw_dict, 

484 

485 core.C.CLASS: _read_object, 

486 core.C.DICT: _read_dict, 

487 core.C.ENUM: _read_enum, 

488 core.C.LIST: _read_list, 

489 core.C.LITERAL: _read_literal, 

490 core.C.OPTIONAL: _read_optional, 

491 core.C.PROPERTY: _read_property, 

492 core.C.SET: _read_set, 

493 core.C.TUPLE: _read_tuple, 

494 core.C.TYPE: _read_type, 

495 core.C.UNION: _read_union, 

496 core.C.VARIANT: _read_variant, 

497 core.C.CONFIG: _read_object, 

498 core.C.PROPS: _read_object, 

499 

500 'gws.AclStr': _read_acl_str, 

501 'gws.Color': _read_color, 

502 'gws.CrsName': _read_crs, 

503 'gws.DateStr': _read_date, 

504 'gws.DateTimeStr': _read_datetime, 

505 'gws.DirPath': _read_dirpath, 

506 'gws.Duration': _read_duration, 

507 'gws.FilePath': _read_filepath, 

508 'gws.FormatStr': _read_formatstr, 

509 

510 'gws.UomValueStr': _read_uom_value, 

511 'gws.UomPointStr': _read_uom_value_2, 

512 'gws.UomSizeStr': _read_uom_value_2, 

513 'gws.UomExtentStr': _read_uom_value_4, 

514 

515 'gws.Metadata': _read_metadata, 

516 'gws.Regex': _read_regex, 

517 'gws.Url': _read_url, 

518}