Coverage for gws-app/gws/plugin/qgis/caps.py: 0%
408 statements
« prev ^ index » next coverage.py v7.8.0, created at 2025-04-17 01:37 +0200
« prev ^ index » next coverage.py v7.8.0, created at 2025-04-17 01:37 +0200
1"""QGIS project xml parser."""
3from typing import Optional
5import math
6import re
8import gws
9import gws.gis.bounds
10import gws.gis.extent
11import gws.gis.crs
12import gws.gis.source
13import gws.lib.metadata
14import gws.lib.net
15import gws.lib.xmlx
18class PrintTemplateElement(gws.Data):
19 type: str
20 uuid: str
21 attributes: dict
22 position: gws.UomPoint
23 size: gws.UomSize
26class PrintTemplate(gws.Data):
27 title: str
28 index: int
29 attributes: dict
30 elements: list[PrintTemplateElement]
33class Caps(gws.Data):
34 metadata: gws.Metadata
35 printTemplates: list[PrintTemplate]
36 projectCrs: gws.Crs
37 projectBounds: Optional[gws.Bounds]
38 projectCanvasBounds: Optional[gws.Bounds]
39 properties: dict
40 sourceLayers: list[gws.SourceLayer]
41 version: str
42 visibilityPresets: dict[str, list[str]]
45def parse(xml: str) -> Caps:
46 el = gws.lib.xmlx.from_string(xml)
47 return parse_element(el)
50def parse_element(root_el: gws.XmlElement) -> Caps:
51 caps = Caps()
53 caps.version = root_el.get('version')
54 caps.properties = parse_properties(root_el.find('properties'))
55 caps.metadata = _project_metadata(root_el)
56 caps.printTemplates = _print_templates(root_el)
57 caps.visibilityPresets = _visibility_presets(root_el)
59 srid = root_el.textof('projectCrs/spatialrefsys/authid') or '4326'
60 caps.projectCrs = gws.gis.crs.get(srid)
61 if not caps.projectCrs:
62 raise gws.Error(f'invalid CRS in qgis project')
64 ext = _extent_from_tag(root_el.find('properties/WMSExtent'))
65 if ext:
66 caps.projectBounds = gws.gis.bounds.from_extent(ext, caps.projectCrs)
68 ext = _extent_from_tag(root_el.find('mapcanvas/extent'))
69 if ext:
70 caps.projectCanvasBounds = gws.gis.bounds.from_extent(ext, caps.projectCrs)
72 layers_dct = _map_layers(root_el, caps)
73 root_group = _layer_tree(root_el.find('layer-tree-group'), layers_dct)
74 caps.sourceLayers = gws.gis.source.check_layers(root_group.layers)
76 return caps
79##
82def _project_metadata(root_el) -> gws.Metadata:
83 md = gws.Metadata()
85 el = root_el.find('projectMetadata')
86 if el:
87 _metadata(el, md)
89 # @TODO supplementary metadata
90 return md
93_meta_mapping = [
94 ('authorityIdentifier', 'identifier'),
95 ('parentIdentifier', 'parentidentifier'),
96 ('language', 'language'),
97 ('type', 'type'),
98 ('title', 'title'),
99 ('abstract', 'abstract'),
100 ('dateCreated', 'creation'),
101 ('fees', 'fees'),
102]
104_contact_mapping = [
105 ('contactEmail', 'email'),
106 ('contactFax', 'fax'),
107 ('contactOrganization', 'organization'),
108 ('contactPerson', 'name'),
109 ('contactPhone', 'voice'),
110 ('contactPosition', 'position'),
111 ('contactRole', 'role'),
112 ('contactAddress', 'address'),
113 ('contactAddressType', 'type'),
114 ('contactArea', 'administrativearea'),
115 ('contactCity', 'city'),
116 ('contactCountry', 'country'),
117 ('contactZip', 'postalcode'),
118]
121def _add_dict(dst, src, mapping):
122 for dkey, skey in mapping:
123 if skey in src:
124 setattr(dst, dkey, src[skey])
127def _metadata(el: gws.XmlElement, md: gws.Metadata):
128 # extract metadata from projectMetadata/resourceMetadata
130 _add_dict(md, el.textdict(), _meta_mapping)
132 md.keywords = []
133 for kw in el.findall('keywords'):
134 keywords = kw.textlist('keyword')
135 if kw.get('vocabulary') == 'gmd:topicCategory':
136 md.isoTopicCategories = keywords
137 else:
138 md.keywords.extend(keywords)
140 contact_el = el.find('contact')
141 if contact_el:
142 _add_dict(md, contact_el.textdict(), _contact_mapping)
143 for e in contact_el.findall('contactAddress'):
144 _add_dict(md, e.textdict(), _contact_mapping)
145 break # NB we only support one contact address
147 md.metaLinks = []
148 for e in el.findall('links/link'):
149 # @TODO clarify
150 md.metaLinks.append(gws.MetadataLink(
151 url=e.get('url'),
152 description=e.get('description'),
153 mimeType=e.get('mimeType'),
154 format=e.get('format'),
155 title=e.get('name'),
156 scheme=e.get('type'),
157 ))
159 md.accessConstraints = []
160 for e in el.findall('constraints'):
161 md.accessConstraints.append(gws.MetadataAccessConstraint(
162 type=e.get('type'),
163 text=e.text,
164 ))
166 for e in el.findall('license'):
167 md.license = gws.MetadataLicense(name=e.text)
168 break
170 e = el.find('extent/temporal')
171 if e:
172 md.dateBegin = e.textof('period/start')
173 md.dateEnd = e.textof('period/end')
176# see QGIS/src/core/layout/qgslayoutitemregistry.h
178_QGraphicsItem_UserType = 65536 # https://doc.qt.io/qtforpython/PySide2/QtWidgets/QGraphicsItem.html
180_LT0 = _QGraphicsItem_UserType + 100
182_LAYOUT_TYPES = {
183 _LT0 + 0: 'item', # LayoutItem
184 _LT0 + 1: 'group', # LayoutGroup
185 _LT0 + 2: 'page', # LayoutPage
186 _LT0 + 3: 'map', # LayoutMap
187 _LT0 + 4: 'picture', # LayoutPicture
188 _LT0 + 5: 'label', # LayoutLabel
189 _LT0 + 6: 'legend', # LayoutLegend
190 _LT0 + 7: 'shape', # LayoutShape
191 _LT0 + 8: 'polygon', # LayoutPolygon
192 _LT0 + 9: 'polyline', # LayoutPolyline
193 _LT0 + 10: 'scalebar', # LayoutScaleBar
194 _LT0 + 11: 'frame', # LayoutFrame
195 _LT0 + 12: 'html', # LayoutHtml
196 _LT0 + 13: 'attributetable', # LayoutAttributeTable
197 _LT0 + 14: 'texttable', # LayoutTextTable
198 _LT0 + 15: '3dmap', # Layout3DMap
199 _LT0 + 16: 'manualtable', # LayoutManualTable
200 _LT0 + 17: 'marker', # LayoutMarker
201}
204# print templates in qgis-3:
205#
206# <Layouts>
207# <Layout name="..." <- template 1
208# <PageCollection
209# <LayoutItem <- pages
210# <LayoutItem type="<int, see below>" ...
211# <LayoutMultiFrame type="<int>" ...
212# <Layout??? <- evtl. other item tags
213#
214# <Layout name="..." <- template 2
215# etc
218def _print_templates(root_el: gws.XmlElement):
219 templates = []
221 for layout_el in root_el.findall('Layouts/Layout'):
222 tpl = PrintTemplate(
223 title=layout_el.get('name', ''),
224 attributes=layout_el.attrib,
225 index=len(templates),
226 elements=[],
227 )
229 pc_el = layout_el.find('PageCollection')
230 if pc_el:
231 tpl.elements.extend(gws.u.compact(_layout_element(c) for c in pc_el))
233 tpl.elements.extend(gws.u.compact(_layout_element(c) for c in layout_el))
235 templates.append(tpl)
237 return templates
240def _layout_element(item_el: gws.XmlElement):
241 type = _LAYOUT_TYPES.get(_parse_int(item_el.get('type')))
242 uuid = item_el.get('uuid')
243 if type and uuid:
244 return PrintTemplateElement(
245 type=type,
246 uuid=uuid,
247 attributes=item_el.attrib,
248 position=_parse_msize(item_el.get('position')),
249 size=_parse_msize(item_el.get('size')),
250 )
253##
256def _map_layers(root_el: gws.XmlElement, caps: Caps) -> dict[str, gws.SourceLayer]:
257 no_wms_layers = set(caps.properties.get('WMSRestrictedLayers', []))
258 use_layer_ids = caps.properties.get('WMSUseLayerIDs', False)
260 layers_dct = {}
262 for el in root_el.findall('projectlayers/maplayer'):
263 sl = _map_layer(el, caps, use_layer_ids)
264 if not sl:
265 continue
266 # no_wms_layers always contains titles, not ids (=names)
267 if sl.title in no_wms_layers:
268 continue
269 layers_dct[sl.sourceId] = sl
271 return layers_dct
274def _map_layer(layer_el: gws.XmlElement, caps: Caps, use_layer_ids: bool) -> gws.SourceLayer:
275 sl = gws.SourceLayer(
276 supportedCrs=[],
277 )
279 sl.metadata = _map_layer_metadata(layer_el)
281 crs = gws.gis.crs.get(layer_el.textof('srs/spatialrefsys/authid'))
282 if crs:
283 sl.supportedCrs.append(crs)
285 if layer_el.get('hasScaleBasedVisibilityFlag') == '1':
286 # in qgis, maxScale < minScale
287 a = _parse_float(layer_el.get('maxScale'))
288 z = _parse_float(layer_el.get('minScale'))
289 if z > a:
290 sl.scaleRange = [a, z]
292 sl.dataSource = _map_layer_datasource(layer_el)
293 sl.opacity = _parse_float(layer_el.textof('layerOpacity') or '1')
294 sl.isQueryable = layer_el.textof('flags/Identifiable') == '1'
295 sl.properties = parse_properties(layer_el.find('customproperties'))
297 uid = layer_el.textof('id')
298 if use_layer_ids:
299 layer_name = uid
300 else:
301 layer_name = layer_el.textof('shortname') or layer_el.textof('layername')
303 sl.title = sl.metadata.get('title')
304 sl.name = layer_name
305 sl.sourceId = uid
307 ext = _map_layer_wgs_extent(layer_el, caps.projectCrs)
308 if ext:
309 sl.wgsExtent = ext
311 return sl
314def _map_layer_metadata(layer_el) -> gws.Metadata:
315 # Layer metadata is either Layer->Properties->Metadata (stored in maplayer/resourceMetadata),
316 # or Layer->Properties->QGIS Server (stored directly under maplayer/abstract, maplayer/keywordList and so on.
318 md = gws.Metadata()
320 el = layer_el.find('resourceMetadata')
321 if el:
322 _metadata(el, md)
324 # fill in missing props from direct metadata
326 d = layer_el.textdict()
328 md.title = md.title or d.get('title') or d.get('shortname') or d.get('layername')
329 md.abstract = md.abstract or d.get('abstract')
331 if not md.attribution and d.get('attribution'):
332 md.attribution = gws.MetadataAttribution(title=d.get('attribution'))
334 if not md.keywords:
335 md.keywords = layer_el.textlist('keywordList/value')
337 if not md.metaLinks:
338 md.metaLinks = []
339 for e in layer_el.findall('metadataUrls/metadataUrl'):
340 md.metaLinks.append(gws.MetadataLink(
341 type=e.get('type'),
342 format=e.get('format'),
343 title=e.text,
344 ))
346 return md
349def _map_layer_datasource(layer_el: gws.XmlElement) -> dict:
350 prov = layer_el.textof('provider')
351 ds_text = layer_el.textof('datasource')
353 if ds_text:
354 return parse_datasource((prov or '').lower(), ds_text)
355 if prov:
356 return {'provider': prov.lower()}
357 return {}
360def _map_layer_wgs_extent(layer_el: gws.XmlElement, project_crs: gws.Crs):
361 # extent explicitly defined in metadata (Layer Props -> Metadata -> Extent)
362 el = layer_el.find('resourceMetadata/extent/spatial')
363 if el:
364 ext = _extent_from_tag(el)
365 crs = gws.gis.crs.get(el.get('crs'))
366 if ext and crs:
367 ext = gws.gis.extent.transform(ext, crs, gws.gis.crs.WGS84)
368 if gws.gis.extent.is_valid_wgs(ext):
369 gws.log.debug(f"_map_layer_wgs_extent: {layer_el.textof('id')}: spatial: {ext}")
370 return ext
372 # extent in <maplayer>/<wgs84extent>
373 el = layer_el.find('wgs84extent')
374 if el:
375 ext = _extent_from_tag(el)
376 if gws.gis.extent.is_valid_wgs(ext):
377 gws.log.debug(f"_map_layer_wgs_extent: {layer_el.textof('id')}: wgs84extent: {ext}")
378 return ext
380 # extent in <maplayer>/<extent>, assume the project CRS
381 el = layer_el.find('extent')
382 if el:
383 ext = _extent_from_tag(el)
384 if ext:
385 ext = gws.gis.extent.transform(ext, project_crs, gws.gis.crs.WGS84)
386 if gws.gis.extent.is_valid_wgs(ext):
387 gws.log.debug(f"_map_layer_wgs_extent: {layer_el.textof('id')}: extent: {ext}")
388 return ext
390 gws.log.warning(f"_map_layer_wgs_extent: {layer_el.textof('id')}: NOT FOUND")
393# layer trees:
395# <layer-tree-group>
396# <layer-tree-group checked="Qt::Checked" expanded="1" name="...">
397# <layer-tree-layer ... checked="Qt::Checked" expanded="1" id="...">
398# ...
401def _layer_tree(el: gws.XmlElement, layers_dct):
402 visible = el.get('checked') != 'Qt::Unchecked'
403 expanded = el.get('expanded') == '1'
405 if el.tag == 'layer-tree-group':
406 title = el.get('name')
407 # qgis doesn't write 'id' for groups but our generators might
408 name = el.get('id') or title
410 return gws.SourceLayer(
411 title=title,
412 name=name,
413 metadata=gws.Metadata(title=title, name=name),
414 isVisible=visible,
415 isExpanded=expanded,
416 isGroup=True,
417 isQueryable=False,
418 isImage=False,
419 layers=gws.u.compact(_layer_tree(c, layers_dct) for c in el)
420 )
422 if el.tag == 'layer-tree-layer':
423 sl = layers_dct.get(el.get('id'))
424 if sl:
425 sl.isVisible = visible
426 sl.isExpanded = expanded
427 sl.isGroup = False
428 sl.isImage = True
429 return sl
432##
434def _visibility_presets(root_el: gws.XmlElement):
435 """Parse the global ``visibility-presets`` block.
437 We're only interested in which layers are visible.
439 Overall structure::
441 <visibility-presets>
442 <visibility-preset .... name="...">
443 <layer id="..." visible="1" ... />
444 <layer id="..." visible="1" ... />
445 <visibility-preset .... name="...">
446 <layer id="..." visible="1" ... />
447 <layer id="..." visible="1" ... />
449 """
451 d = {}
453 for el in root_el.findall('visibility-presets/visibility-preset'):
454 ls = []
455 for la in el.findall('layer'):
456 if la.attr('visible') == '1':
457 ls.append(la.attr('id'))
458 d[el.attr('name')] = ls
460 return d
463##
466def parse_datasource(prov, text):
467 ds = gws.u.to_lower_dict(_parse_datasource(text) or {})
468 ds['provider'] = (ds.get('provider') or prov).lower()
470 if ds['provider'] == 'wms' and 'tilematrixset' in ds:
471 ds['provider'] = 'wmts'
472 elif ds['provider'] == 'wms' and ds.get('type') == 'xyz':
473 ds['provider'] = 'xyz'
475 # @TODO classify ogr's based on a file extension
477 return ds
480def _parse_datasource(text):
481 # Datasources are very versatile and the format depends on the provider.
482 # For some hints see `decodedSource` in qgsvectorlayer.cpp/qgsrasterlayer.cpp.
483 # We don't have ambition to parse them all, just do some ad-hoc parsing
484 # of the most common flavors, and return the rest as `{'text': text}`.
486 text = text.strip()
488 if re.match(r'^\w+=[^&]*&', text):
489 # key=value, amp-separated, uri-encoded
490 # used for WMS, e.g.
491 # contextualWMSLegend=0&crs=EPSG:31468&...&url=...?SERVICE%3DWMTS%26REQUEST%3DGetCapabilities
492 return _datasource_amp_delimited(text)
494 if re.match(r'^\w+=\S+ ', text):
495 # key=value, space separated
496 # used for postgres/WFS, e.g.
497 # dbname='...' host=... port=...
498 # pagingEnabled='...' preferCoordinatesForWfsT11=...
499 return _datasource_space_delimited(text)
501 if text.startswith(('http://', 'https://')):
502 # just an url
503 return {'url': text}
505 if text.startswith(('.', '/')):
506 # path or path|options
507 # used for Geojson, GPKG, e.g.
508 # ../rel/path/test.gpkg|layername=name|subset=... etc
509 return _datasource_pipe_delimited(text)
511 return {'text': text}
514def _datasource_amp_delimited(text):
515 ds = {}
517 for p in text.split('&'):
518 if '=' not in p:
519 continue
520 k, v = p.split('=', maxsplit=1)
522 v = gws.lib.net.unquote(v)
524 if k in {'layers', 'styles'}:
525 ds.setdefault(k, []).append(v)
526 else:
527 ds[k] = v
529 if 'url' not in ds:
530 return ds
532 # extract params from the url
534 url, params = gws.lib.net.extract_params(ds['url'])
536 if 'typename' in params:
537 ds['typename'] = params.pop('typename')
538 if 'layers' in params:
539 ds.setdefault('layers', []).extend(params.pop('layers').split(','))
540 if 'styles' in params:
541 ds.setdefault('styles', []).extend(params.pop('styles').split(','))
543 params.pop('service', None)
544 params.pop('request', None)
546 ds['params'] = params
548 # {x} placeholders shouldn't be encoded
549 url = url.replace('%7B', '{')
550 url = url.replace('%7D', '}')
552 ds['url'] = url
554 return ds
557def _datasource_space_delimited(text):
558 key_re = r'^\w+\s*=\s*'
560 value_re = r'''(?x)
561 " (?: \\. | [^"])* " |
562 ' (?: \\. | [^'])* ' |
563 \S+
564 '''
566 parens_re = r'\(.*?\)'
568 def _cut(u, rx):
569 m = re.match(rx, u)
570 if not m:
571 raise ValueError(f'datasource uri error, expected {rx!r}, found {u[:25]!r}')
572 v = m.group(0)
573 return v, u[len(v):].strip()
575 def _unesc(s):
576 return re.sub(r'\\(.)', '\1', s)
578 def _mid(s):
579 return s[1:-1].strip()
581 def _value(v):
582 if v.startswith(('\'', '\"')):
583 return _unesc(_mid(v))
584 return v
586 ds = {}
588 while text:
589 # keyword=
590 key, text = _cut(text, key_re)
591 key = key.strip('= ')
593 if key == 'sql':
594 # 'sql=' is special and can contain whatever, it's always the last one
595 ds[key] = text
596 break
598 elif key == 'table':
599 # 'table=' is special, it can be `table="foo"` or `table="foo"."bar"` or table=`"foo"."bar" (geom)`
600 v, text = _cut(text, value_re)
601 ds['table'] = _value(v)
603 if text.startswith('.'):
604 v, text = _cut(text[1:], value_re)
605 ds['table'] += '.' + _value(v)
607 if text.startswith('('):
608 v, text = _cut(text, parens_re)
609 ds['geometryColumn'] = _mid(v)
611 else:
612 # just param=val
613 v, text = _cut(text, value_re)
614 ds[key] = _value(v)
616 return ds
619def _datasource_pipe_delimited(text):
620 if '|' not in text:
621 return {'path': text}
623 path, rest = text.split('|', maxsplit=1)
625 if '=' not in rest:
626 return {'path': path, 'options': rest}
628 ds = {'path': path}
630 for p in rest.split('|'):
631 k, v = p.split('=', maxsplit=1)
632 ds[k] = v
634 return ds
637##
640def parse_properties(el: gws.XmlElement):
641 """Parse qgis property blocks.
643 There are following forms:
645 Scalar property::
647 <WMSContactPhone type="QString">...
649 Dict::
651 <QFieldSync>
652 <dirsToCopy type="QString">...
653 <exportDirectoryProject type="QString">...
654 </QFieldSync>
656 Option map::
658 <data-defined-properties>
659 <Option type="Map">
660 <Option type="QString" name="..." value="..."/>
661 <Option name="properties"/>
662 </Option>
663 </data-defined-properties>
666 """
668 if not el:
669 return {}
671 _, val = _parse_property_tag(el)
672 return val
675def _parse_property_tag(el: gws.XmlElement):
676 typ = el.get('type')
677 name = el.tag
678 is_opt = el.tag == 'Option'
680 if is_opt and el.attr('name'):
681 name = el.attr('name')
683 if not typ or typ == 'Map':
684 d = {}
686 for c in el:
687 k, v = _parse_property_tag(c)
688 if k:
689 d[k] = v
691 if len(d) == 1 and 'Option' in d:
692 return name, d['Option']
694 return name, d
696 if typ == 'List':
697 ls = []
698 for c in el:
699 _, v = _parse_property_tag(c)
700 ls.append(v)
701 return name, ls
703 if typ == 'QStringList':
704 val = [c.text for c in el.findall('value')]
705 return name, val
707 if typ == 'QString':
708 val = el.attr('value') if is_opt else el.text
709 return name, val
711 if typ == 'bool':
712 val = el.attr('value') if is_opt else el.text
713 return name, val.lower() == 'true'
715 if typ == 'int':
716 val = el.attr('value') if is_opt else el.text
717 return name, _parse_int(val)
719 if typ == 'double':
720 val = el.attr('value') if is_opt else el.text
721 return name, _parse_float(val)
723 return '', None
726##
729def _extent_from_tag(el: gws.XmlElement):
731 if not el:
732 return
734 # <spatial dimensions="2" miny="0" maxz="0" maxx="0" crs="EPSG:25832" minx="0" minz="0" maxy="0"/>
735 if el.get('minx'):
736 return _extent_from_list([
737 el.get('minx'),
738 el.get('miny'),
739 el.get('maxx'),
740 el.get('maxy'),
741 ])
743 # <WMSExtent type="QStringList">
744 # <value>1</value>
745 # <value>2</value>
746 # <value>3</value>
747 # <value>4</value>
748 # </WMSExtent>
749 #
750 if el.get('type') == 'QStringList':
751 return _extent_from_list([
752 v.text for v in el.children()
753 ])
755 # <wgs84extent>
756 # <xmin>1</xmin>
757 # <ymin>2</ymin>
758 # <xmax>3</xmax>
759 # <ymax>4</ymax>
760 # </wgs84extent>
762 if el.find('xmin'):
763 return _extent_from_list([
764 el.textof('xmin'),
765 el.textof('ymin'),
766 el.textof('xmax'),
767 el.textof('ymax'),
768 ])
771def _extent_from_list(ls):
772 if len(ls) != 4:
773 return
774 try:
775 e = [float(p) for p in ls]
776 except ValueError:
777 return
778 if not all(math.isfinite(p) for p in e):
779 return
780 # all coordinates = 0, consider invalid
781 if all(abs(p) < 0.0001 for p in e):
782 return
783 e = [
784 min(e[0], e[2]),
785 min(e[1], e[3]),
786 max(e[0], e[2]),
787 max(e[1], e[3]),
788 ]
789 # for single-point extents, add 0.0001 (~10 m)
790 c = 0.0001
791 if e[0] == e[2]:
792 e[0] -= c
793 e[2] += c
794 if e[1] == e[3]:
795 e[1] -= c
796 e[3] += c
797 return gws.Extent(e)
800def _parse_msize(s):
801 # e.g. 'position': '228.477,27.8455,mm'
802 try:
803 x, y, u = s.split(',')
804 return float(x), float(y), u
805 except Exception:
806 return None
809def _parse_int(s):
810 try:
811 return int(s)
812 except Exception:
813 return 0
816def _parse_float(s):
817 try:
818 x = float(s)
819 except Exception:
820 return 0
821 if math.isnan(x) or math.isinf(x):
822 return 0
823 return x