Coverage for gws-app/gws/plugin/qfield/core.py: 0%

1"""Qfield reader and writer."""

3from typing import Optional, cast

5import shutil

7import gws

8import gws.base.database

9import gws.base.shape

10import gws.gis.crs

11import gws.gis.extent

12import gws.gis.gdalx

13import gws.gis.render

14import gws.gis.source

15import gws.lib.image

16import gws.lib.jsonx

17import gws.lib.osx

18import gws.lib.sa as sa

19import gws.plugin.model_field.file

20import gws.plugin.qgis

22GPKG_EXT = 'gpkg'

25class PackageConfig(gws.ConfigWithAccess):

26 qgisProvider: gws.plugin.qgis.provider.Config

27 """QGis provider settings."""

28 models: Optional[list[gws.ext.config.model]]

29 """Data models."""

30 mapCacheLifeTime: gws.Duration = 0

31 """Cache life time for base map layers."""

34class Package(gws.Node):

35 qgisProvider: gws.plugin.qgis.provider.Object

36 models: list[gws.DatabaseModel]

37 mapCacheLifeTime: int

39 def configure(self):

40 self.qgisProvider = self.create_child(gws.plugin.qgis.provider.Object, self.cfg('qgisProvider'))

41 self.models = self.create_children(gws.ext.object.model, self.cfg('models'))

42 self.mapCacheLifeTime = self.cfg('mapCacheLifeTime') or 0

45class ExportArgs(gws.Data):

46 package: Package

47 project: gws.Project

48 user: gws.User

49 baseDir: str

50 qgisFileName: str

51 dbFileName: str

52 dbPath: str

53 withBaseMap: bool

54 withData: bool

55 withMedia: bool

56 withQgis: bool

59class ImportArgs(gws.Data):

60 package: Package

61 project: gws.Project

62 user: gws.User

63 baseDir: str

64 dbFileName: str

67class LayerAction(gws.Enum):

68 remove = 'remove'

69 edit = 'edit'

70 baseMap = 'baseMap'

73class EditAction(gws.Enum):

74 update = 'update'

75 geometryUpdate = 'geometryUpdate'

76 insert = 'insert'

77 delete = 'delete'

80class EditOperation(gws.Data):

81 action: EditAction

82 fid: int

83 pkey: str

84 columnName: str

85 attributes: dict

88class ModelEntry(gws.Data):

89 gpId: int

90 gpName: str

91 model: gws.DatabaseModel

92 fidToPkey: dict

93 columnIndex: dict

94 editOperations: list[EditOperation]

95 features: list[gws.Feature]

98class LayerEntry(gws.Data):

99 action: LayerAction

100 qgisId: str

101 modelEntry: ModelEntry

102 readOnly: bool

103 sqlFilter: str

104 dataSourceFileName: str

105 dataSourcePath: str

106 dataSource: str

107 dataProvider: str

108 sourceLayer: gws.SourceLayer

109

110

111class QFieldCaps(gws.Data):

112 qgisPath: str

113 layerMap: dict[str, LayerEntry]

114 modelMap: dict[str, ModelEntry]

115 globalProps: dict

116 dirsToCopy: list[str]

117 baseMapLayerIds: list[str]

118 areaOfInterest: Optional[gws.Bounds]

119 offlineCopyOnlyAoi: bool

120

121

122class OfflineLog(gws.Data):

123 log_added_attrs: list[dict]

124 log_added_features: list[dict]

125 log_feature_updates: list[dict]

126 log_fids: list[dict]

127 log_geometry_updates: list[dict]

128 log_indices: list[dict]

129 log_layer_ids: list[dict]

130 log_removed_features: list[dict]

131 log_gws_columns: list[dict]

132 log_gws_tables: list[dict]

133

134

135_GP_ATTRIBUTE_TYPES = {

136 gws.AttributeType.bool,

137 gws.AttributeType.date,

138 gws.AttributeType.datetime,

139 gws.AttributeType.float,

140 gws.AttributeType.int,

141 gws.AttributeType.str,

142 gws.AttributeType.time,

143}

144

145

146class Exporter:

147 package: Package

148 project: gws.Project

149 user: gws.User

150 args: ExportArgs

151

152 sourceQgisProject: gws.plugin.qgis.project.Object

153

154 targetQgisPath: str

155

156 deviceDbPath: str

157 localDbPath: str

158

159 qfCaps: QFieldCaps

160

161 def run(self, args: ExportArgs):

162 self.prepare(args)

163

164 if self.args.withData:

165 self.write_data()

166

167 if self.args.withBaseMap:

168 self.write_base_map()

169

170 if self.args.withMedia:

171 self.write_media()

172

173 if self.args.withQgis:

174 self.write_qgis_project()

175

176 def prepare(self, args: ExportArgs):

177 self.args = args

178 self.package = self.args.package

179 self.project = self.args.project

180 self.user = self.args.user

181

182 self.sourceQgisProject = self.package.qgisProvider.qgis_project()

183

184 self.targetQgisPath = f'{self.args.baseDir}/{self.args.qgisFileName or self.package.uid}.qgs'

185

186 self.localDbPath = f'{self.args.baseDir}/{args.dbPath}'

187 self.deviceDbPath = f'./{args.dbPath}'

188

189 self.qfCaps = QFieldCapsParser().run(self.package)

190

191 for le in self.qfCaps.layerMap.values():

192 if le.action == LayerAction.edit:

193 le.dataSourceFileName = args.dbPath

194 le.dataSourcePath = self.localDbPath

195 le.dataSource = f'{self.deviceDbPath}|layername={le.modelEntry.gpName}'

196 if le.sqlFilter:

197 le.dataSource += f'|subset={le.sqlFilter}'

198 le.dataProvider = f'ogr'

199 if le.action == LayerAction.baseMap:

200 le.dataSourceFileName = f'{le.qgisId}.{GPKG_EXT}'

201 le.dataSourcePath = f'{self.args.baseDir}/{le.dataSourceFileName}'

202 le.dataSource = f'./{le.dataSourceFileName}'

203 le.dataProvider = f'gdal'

204

205 def write_data(self):

206 with gws.gis.gdalx.open_vector(self.localDbPath, 'w') as ds:

207 for me in self.qfCaps.modelMap.values():

208 self.write_features(me, ds)

209 self.write_offline_log()

210

211 def write_base_map(self):

212 # @TODO options for flattened base maps

213 for le in self.qfCaps.layerMap.values():

214 if le.action == LayerAction.baseMap:

215 self.write_base_map_layer(le)

216

217 def write_media(self):

218 for d in self.qfCaps.dirsToCopy:

219 if self.qfCaps.qgisPath:

220 rel_dir = gws.lib.osx.rel_path(d, self.qfCaps.qgisPath)

221 else:

222 # @TODO absolute dir with a postgres-based qgis project?

223 rel_dir = d.split('/')[-1]

224 shutil.copytree(d, f'{self.args.baseDir}/{rel_dir}')

225

226 def write_features(self, me: ModelEntry, ds: gws.gis.gdalx.VectorDataSet):

227 # see qgis/src/core/qgsofflineediting.cpp convertToOfflineLayer()

228

229 gws.log.debug(f'{self.args.baseDir}: BEGIN write_features: {self.package.uid}::{me.gpName!r}')

230

231 gp_fields = [f for f in me.model.fields if f.attributeType in _GP_ATTRIBUTE_TYPES]

232

233 gp_layer = ds.create_layer(

234 me.gpName,

235 columns={f.name: f.attributeType for f in gp_fields},

236 geometry_type=me.model.geometryType,

237 crs=me.model.geometryCrs,

238 overwrite=True

239 )

240

241 mc = gws.ModelContext(user=self.user, project=self.project, op=gws.ModelOperation.read)

242

243 features = me.features

244 if not features:

245 q = gws.SearchQuery()

246 if self.qfCaps.offlineCopyOnlyAoi:

247 q.bounds = self.qfCaps.areaOfInterest or self.package.qgisProvider.bounds

248 features = me.model.find_features(q, mc)

249

250 records = []

251

252 for feature in features:

253 records.append(gws.FeatureRecord(

254 attributes={f.name: feature.get(f.name) for f in gp_fields},

255 shape=feature.shape(),

256 meta={}

257 ))

258

259 with ds.transaction():

260 fids = gp_layer.insert(records)

261

262 me.columnIndex = {}

263 col_idx = 1 # column 0 == fid

264 for field in gp_fields:

265 me.columnIndex[col_idx] = field.name

266 col_idx += 1

267

268 me.fidToPkey = {}

269 if me.model.uidName:

270 for rec, fid in zip(records, fids):

271 me.fidToPkey[fid] = rec.attributes.get(me.model.uidName)

272

273 gws.log.debug(f'{self.args.baseDir}: END write_features: {self.package.uid}::{me.gpName!r} count={gp_layer.count()}')

274

275 def write_base_map_layer(self, le: LayerEntry):

276 cache_path = gws.u.ensure_dir(gws.c.CACHE_DIR + '/qfield') + '/' + le.dataSourceFileName

277

278 if self.package.mapCacheLifeTime > 0:

279 if 0 < gws.lib.osx.file_age(cache_path) < self.package.mapCacheLifeTime:

280 gws.log.debug(f'base map {le.qgisId} cached')

281 shutil.copy(cache_path, le.dataSourcePath)

282 return

283

284 bounds = self.qfCaps.areaOfInterest or self.package.qgisProvider.bounds

285 resolution = int(self.qfCaps.globalProps.get('baseMapMupp', 10))

286 w, h = gws.gis.extent.size(bounds.extent)

287 px_size = (w / resolution, h / resolution, gws.Uom.px)

288

289 flat_layer = cast(gws.Layer, self.package.root.create_temporary(

290 gws.ext.object.layer,

291 type='qgisflat',

292 _parentBounds=bounds,

293 _parentResolutions=[1],

294 _defaultProvider=self.package.qgisProvider,

295 _defaultSourceLayers=[le.sourceLayer],

296 ))

297

298 mv = gws.gis.render.map_view_from_bbox(

299 size=px_size,

300 bbox=bounds.extent,

301 crs=bounds.crs,

302 dpi=96,

303 rotation=0,

304 )

305

306 lri = gws.LayerRenderInput(

307 type=gws.LayerRenderInputType.box,

308 user=self.user,

309 view=mv,

310 )

311

312 lro = flat_layer.render(lri)

313 img = gws.lib.image.from_bytes(lro.content)

314

315 with gws.gis.gdalx.open_from_image(img, bounds) as src:

316 src.create_copy(le.dataSourcePath)

317

318 if self.package.mapCacheLifeTime > 0:

319 shutil.copy(le.dataSourcePath, cache_path)

320

321 def write_offline_log(self):

322

323 ol = OfflineLog()

324

325 ol.log_fids = [

326 {'layer_id': me.gpId, 'offline_fid': fid, 'remote_fid': fid, 'remote_pk': str(pk)}

327 for me in self.qfCaps.modelMap.values()

328 for fid, pk in me.fidToPkey.items()

329 ]

330 ol.log_indices = [

331 {'name': 'commit_no', 'last_index': 0},

332 {'name': 'layer_id', 'last_index': len(self.qfCaps.modelMap)}

333 ]

334

335 ol.log_layer_ids = []

336 for le in self.qfCaps.layerMap.values():

337 if le.action == LayerAction.edit:

338 ol.log_layer_ids.append({'id': le.modelEntry.gpId, 'qgis_id': le.qgisId})

339

340 ol.log_gws_columns = []

341 for me in self.qfCaps.modelMap.values():

342 for col_id, col_name in me.columnIndex.items():

343 ol.log_gws_columns.append({'layer_id': me.gpId, 'attr': col_id, 'name': col_name})

344

345 ol.log_gws_tables = [

346 {'layer_id': me.gpId, 'name': me.gpName}

347 for me in self.qfCaps.modelMap.values()

348 ]

349

350 engine = sa.create_engine(f'sqlite:///{self.localDbPath}', echo=False, future=True)

351 with engine.begin() as conn:

352 _write_offline_log(ol, conn)

353

354 def write_qgis_project(self):

355 root_el = self.sourceQgisProject.xml_root()

356 QgisXmlTransformer().run(self, root_el)

357 xml = root_el.to_string()

358 xml = self.replace_vars(xml)

359 gws.u.write_file(self.targetQgisPath, xml)

360

361 def replace_vars(self, s: str) -> str:

362 # @TODO render attributes as templates

363 s = s.replace('{user.authToken}', self.user.authToken)

364 s = s.replace('{user.loginName}', self.user.loginName)

365 s = s.replace('{user.displayName}', self.user.displayName)

366 return s

367

368

369class QgisXmlTransformer:

370 ex: Exporter

371 root: gws.XmlElement

372 remove: list[gws.XmlElement]

373

374 def run(self, ex: Exporter, root_el: gws.XmlElement):

375 self.ex = ex

376 self.root = root_el

377 self.remove = []

378

379 self.change_global_props()

380 self.update_layer_tree()

381 self.update_map_layers()

382 self.update_referenced_layers()

383 self.update_referenced_layers()

384 self.update_edit_widgets()

385

386 self.cleanup_layer_group(root_el.find('layer-tree-group'))

387 self.remove_elements(root_el, None)

388

389 def change_global_props(self):

390 # change global properties

391

392 properties = self.root.find('properties') or self.root.add('properties')

393

394 # this is added by the Sync plugin

395 p = properties.add('OfflineEditingPlugin').add('OfflineDbPath', type='QString')

396 p.text = f'{self.ex.deviceDbPath}'

397

398 # ensure relative paths

399 p = properties.find('Paths/Absolute')

400 if not p:

401 p = properties.add('Paths').add('Absolute', type='bool')

402 p.text = 'false'

403

404 def update_layer_tree(self):

405 for el in self.root.findall('.//layer-tree-layer'):

406 le = self.ex.qfCaps.layerMap.get(el.get('id'))

407 if not le:

408 continue

409

410 if le.action == LayerAction.remove:

411 self.remove.append(el)

412 continue

413

414 el.set('source', le.dataSource)

415 el.set('providerKey', le.dataProvider)

416

417 def update_map_layers(self):

418 for el in self.root.findall('.//maplayer'):

419 le = self.ex.qfCaps.layerMap.get(el.textof('id'))

420 if not le:

421 continue

422

423 if le.action == LayerAction.remove:

424 self.remove.append(el)

425 continue

426

427 el.find('datasource').text = le.dataSource

428 el.find('provider').text = le.dataProvider

429

430 if le.action == LayerAction.edit:

431 el.remove(el.find('customproperties'))

432 opt = el.add('customproperties').add('Option', type='Map')

433

434 opt.add('Option', type='QString', name='QFieldSync/action', value='offline')

435 opt.add('Option', type='QString', name='QFieldSync/attachment_naming', value='{}')

436 opt.add('Option', type='QString', name='QFieldSync/photo_naming', value='{}')

437 opt.add('Option', type='QString', name='QFieldSync/sourceDataPrimaryKeys', value='fid')

438

439 if le.action == LayerAction.edit:

440 if le.readOnly:

441 opt.add('Option', type='bool', name='QFieldSync/is_geometry_locked', value='true')

442 else:

443 opt.add('Option', type='bool', name='isOfflineEditable', value='true')

444

445 def update_referenced_layers(self):

446 for el in self.root.findall('.//referencedLayers/relation'):

447 """

448 <referencedLayers>

449 <relation

450 strength="Association"

451 referencingLayer="..."

452 layerId="..."

453 referencedLayer="REF_ID"

454 providerKey="<REPLACE THIS>"

455 dataSource="<REPLACE THIS>"

456 >

457 ...

458 """

459

460 ref_id = el.get('referencedLayer')

461

462 le = self.ex.qfCaps.layerMap.get(ref_id)

463 if not le or le.action != LayerAction.edit:

464 gws.log.warning(f'relation: referenced layer not found: {ref_id!r}')

465 continue

466

467 if 'dataSource' in el.attrib:

468 el.set('dataSource', le.dataSource)

469 if 'providerKey' in el.attrib:

470 el.set('providerKey', le.dataProvider)

471

472 def update_edit_widgets(self):

473 for el in self.root.findall('.//editWidget'):

474 """

475 <editWidget type="RelationReference">

476 <config>

477 <Option type="Map">

478 ...

479 <Option value="<REF_ID>" name="ReferencedLayerId" type="QString"/>

480 <Option value="<REPLACE THIS>" name="ReferencedLayerDataSource" type="QString"/>

481 <Option value="<REPLACE THIS>" name="ReferencedLayerProviderKey" type="QString"/>

482 ...

483 """

484

485 if el.get('type') != 'RelationReference':

486 continue

487

488 ref_id = None

489 for opt in el.findall('.//Option'):

490 if opt.get('name') == 'ReferencedLayerId':

491 ref_id = opt.get('value')

492 break

493

494 if not ref_id:

495 continue

496

497 le = self.ex.qfCaps.layerMap.get(ref_id)

498 if not le or le.action != LayerAction.edit:

499 gws.log.warning(f'editWidget: referenced layer not found: {ref_id!r}')

500 continue

501

502 for opt in el.findall('.//Option'):

503 if opt.get('name') == 'ReferencedLayerDataSource':

504 opt.set('value', le.dataSource)

505 if opt.get('name') == 'ReferencedLayerProviderKey':

506 opt.set('value', le.dataProvider)

507

508 def cleanup_layer_group(self, group_el):

509 is_empty = True

510

511 for sub in group_el.children():

512 if sub.tag == 'layer-tree-group':

513 if self.cleanup_layer_group(sub):

514 is_empty = False

515 if sub.tag == 'layer-tree-layer' and sub not in self.remove:

516 is_empty = False

517

518 if is_empty:

519 self.remove.append(group_el)

520

521 return not is_empty

522

523 def remove_elements(self, el, parent_el):

524 if el in self.remove:

525 parent_el.remove(el)

526 return

527 ns = el.children()

528 for n in ns:

529 self.remove_elements(n, el)

530

531

532##

533

534class Importer:

535 package: Package

536 project: gws.Project

537 user: gws.User

538 args: ImportArgs

539

540 localDbPath: str

541 localImagePaths: list[str]

542

543 updatedModels: list[ModelEntry]

544

545 qfCaps: QFieldCaps

546

547 def run(self, args: ImportArgs):

548 self.prepare(args)

549 if self.localDbPath:

550 self.read_data()

551

552 def prepare(self, args: ImportArgs):

553 self.args = args

554 self.package = self.args.package

555 self.project = self.args.project

556 self.user = self.args.user

557

558 self.localDbPath = ''

559 self.localImagePaths = []

560

561 for path in gws.lib.osx.find_files(self.args.baseDir):

562 if path.endswith(args.dbFileName):

563 self.localDbPath = path

564 if path.lower().endswith(('.jpg', '.jpeg', '.png', '.webp')):

565 self.localImagePaths.append(path)

566

567 gws.log.debug(f'{self.localDbPath=} {self.localImagePaths=}')

568 self.qfCaps = QFieldCapsParser().run(self.package)

569

570 self.updatedModels = []

571

572 def read_data(self):

573 self.enumerate_updated_models()

574 self.read_features()

575 self.read_images()

576 self.commit_edits()

577

578 def enumerate_updated_models(self):

579 engine = sa.create_engine(f'sqlite:///{self.localDbPath}', echo=False, future=True)

580

581 with engine.begin() as conn:

582 ol = _read_offline_log(conn)

583

584 for rec in ol.log_gws_tables:

585 gp_name = rec.get('name')

586

587 me = self.qfCaps.modelMap.get(gp_name)

588 if not me:

589 gws.log.warning(f'offline model {gp_name!r}: not found')

590 continue

591

592 me.gpId = rec.get('layer_id')

593 has_updates = self.read_offline_log_for_model(me, ol)

594 if has_updates:

595 self.updatedModels.append(me)

596

597 def read_features(self):

598 with gws.gis.gdalx.open_vector(self.localDbPath) as ds:

599 for me in self.updatedModels:

600 self.read_features_for_model(me, ds)

601

602 def read_images(self):

603 for me in self.updatedModels:

604 self.read_images_for_model(me)

605

606 def read_offline_log_for_model(self, me: ModelEntry, ol: OfflineLog) -> bool:

607

608 me.columnIndex = {}

609 for rec in ol.log_gws_columns:

610 if rec['layer_id'] == me.gpId:

611 me.columnIndex[rec.get('attr')] = rec.get('name')

612

613 me.fidToPkey = {}

614 for rec in ol.log_fids:

615 if rec['layer_id'] == me.gpId:

616 me.fidToPkey[rec.get('offline_fid')] = rec.get('remote_pk')

617

618 me.editOperations = []

619 self.read_edit_operations_for_model(me, ol)

620

621 return len(me.editOperations) > 0

622

623 def read_edit_operations_for_model(self, me: ModelEntry, ol: OfflineLog):

624 # NB the order is inserts -> updates (sorted by commit_no) -> deletes

625

626 for rec in ol.log_added_features:

627 if rec['layer_id'] == me.gpId:

628 fid = rec.get('fid')

629 me.editOperations.append(EditOperation(action=EditAction.insert, fid=fid))

630

631 ops = {}

632 for rec in ol.log_feature_updates:

633 if rec['layer_id'] == me.gpId:

634 fid = rec.get('fid')

635 pkey = me.fidToPkey.get(fid)

636 cname = me.columnIndex.get(rec.get('attr'))

637 if not pkey or not cname:

638 gws.log.warning(f'invalid update record {rec!r} {pkey=} {cname=}')

639 continue

640 ops[pkey, cname] = EditOperation(action=EditAction.update, fid=fid, pkey=pkey, columnName=cname)

641 me.editOperations.extend(ops.values())

642

643 ops = {}

644 for rec in ol.log_geometry_updates:

645 if rec['layer_id'] == me.gpId:

646 fid = rec.get('fid')

647 pkey = me.fidToPkey.get(fid)

648 if not pkey:

649 gws.log.warning(f'invalid update record {rec!r}')

650 continue

651 ops[pkey] = EditOperation(action=EditAction.geometryUpdate, fid=fid, pkey=pkey)

652 me.editOperations.extend(ops.values())

653

654 for rec in ol.log_removed_features:

655 if rec['layer_id'] == me.gpId:

656 pkey = me.fidToPkey.get(rec.get('fid'))

657 if not pkey:

658 gws.log.warning(f'invalid delete record {rec!r}')

659 continue

660 me.editOperations.append(EditOperation(action=EditAction.delete, pkey=pkey))

661

662 def read_features_for_model(self, me: ModelEntry, ds: gws.gis.gdalx.VectorDataSet):

663

664 gp_layer = ds.layer(me.gpId)

665

666 for eo in me.editOperations:

667 eo.attributes = {me.model.uidName: eo.pkey}

668

669 if eo.action == EditAction.delete:

670 continue

671

672 feature_rec = gp_layer.get(eo.fid)

673 if not feature_rec:

674 gws.log.warning(f'{self.args.baseDir}: operation {eo!r}: fid not found')

675 continue

676

677 if eo.action == EditAction.update:

678 eo.attributes[eo.columnName] = feature_rec.attributes.get(eo.columnName)

679 continue

680

681 if eo.action == EditAction.geometryUpdate:

682 if not me.model.geometryName:

683 gws.log.warning(f'{self.args.baseDir}: operation {eo!r}: geometry not defined')

684 continue

685 if not feature_rec.shape:

686 gws.log.warning(f'{self.args.baseDir}: operation {eo!r}: geometry not found')

687 continue

688 eo.attributes[me.model.geometryName] = feature_rec.shape

689 continue

690

691 if eo.action == EditAction.insert:

692 eo.attributes.update(feature_rec.attributes)

693 if me.model.geometryName and feature_rec.shape:

694 eo.attributes[me.model.geometryName] = feature_rec.shape

695 # remove primary key if it is null

696 if eo.attributes.get(me.model.uidName) is None:

697 eo.attributes.pop(me.model.uidName, None)

698 continue

699

700 def read_images_for_model(self, me: ModelEntry):

701 file_field_map = {}

702

703 for field in me.model.fields:

704 if field.extType == 'file':

705 col_name = cast(gws.plugin.model_field.file.Object, field).cols.name.name

706 file_field_map[col_name] = field

707

708 if not file_field_map:

709 return

710

711 for eo in me.editOperations:

712 eo.attributes = self.update_file_attributes(eo, file_field_map)

713

714 def update_file_attributes(self, eo: EditOperation, file_field_map):

715 atts = {}

716

717 for name, val in eo.attributes.items():

718 field = file_field_map.get(name)

719 if not field:

720 atts[name] = val

721 continue

722

723 fv = self.file_value_for_path(val)

724 if not fv:

725 gws.log.warning(f'file not found: {val!r}')

726 continue

727

728 atts[field.name] = fv

729

730 return atts

731

732 def file_value_for_path(self, path):

733 file_name = gws.lib.osx.file_name(path)

734 for p in self.localImagePaths:

735 if gws.lib.osx.file_name(p) == file_name:

736 return gws.plugin.model_field.file.FileValue(

737 content=gws.u.read_file_b(p),

738 name=file_name,

739 path=path,

740 size=gws.lib.osx.file_size(p),

741 )

742

743 def commit_edits(self):

744 for me in self.updatedModels:

745 self.commit_edits_for_model(me)

746

747 def commit_edits_for_model(self, me: ModelEntry):

748 for eo in me.editOperations:

749 gws.log.debug(f'edit: {self.localDbPath}: {me.gpName=}: {eo}')

750

751 if eo.action in {EditAction.update, EditAction.geometryUpdate}:

752 mc = gws.ModelContext(user=self.user, op=gws.ModelOperation.update)

753 model = self.check_model(me, self.user, gws.Access.write)

754 feature = model.feature_from_props(gws.FeatureProps(attributes=eo.attributes), mc)

755 if not model.validate_feature(feature, mc):

756 gws.log.info(f'validation errors: {feature.errors}')

757 raise gws.BadRequestError('Validation error')

758 model.update_feature(feature, mc)

759

760 if eo.action == EditAction.insert:

761 mc = gws.ModelContext(user=self.user, op=gws.ModelOperation.create)

762 model = self.check_model(me, self.user, gws.Access.create)

763 feature = model.feature_from_props(gws.FeatureProps(attributes=eo.attributes), mc)

764 if not model.validate_feature(feature, mc):

765 gws.log.info(f'validation errors: {feature.errors}')

766 raise gws.BadRequestError('Validation error')

767 model.create_feature(feature, mc)

768

769 if eo.action == EditAction.delete:

770 mc = gws.ModelContext(user=self.user, op=gws.ModelOperation.delete)

771 model = self.check_model(me, self.user, gws.Access.delete)

772 feature = model.feature_from_props(gws.FeatureProps(attributes=eo.attributes), mc)

773 model.delete_feature(feature, mc)

774

775 def check_model(self, me: ModelEntry, user: gws.User, access: gws.Access) -> gws.Model:

776 if not me.model:

777 raise gws.ForbiddenError(f'{me.gpName}: model: not found, {access=} {user=}')

778 if not me.model.isEditable:

779 raise gws.ForbiddenError(f'{me.gpName}: model not editable, {access=} {user=}')

780 if not user.can(access, me.model):

781 raise gws.ForbiddenError(f'{me.gpName}: model forbidden, {access=} {user=}')

782 return me.model

783

784

785class QFieldCapsParser:

786 """Read qf-related capabilities from the qgis project."""

787

788 package: Package

789 caps: QFieldCaps

790 qgisCaps: gws.plugin.qgis.caps.Caps

791

792 def run(self, package: Package) -> QFieldCaps:

793 self.package = package

794 self.caps = QFieldCaps(

795 layerMap={},

796 modelMap={},

797 globalProps={},

798 )

799 self.qgisCaps = package.qgisProvider.qgis_project().caps()

800

801 # for some reason, there are two of them

802 self.caps.globalProps.update(self.qgisCaps.properties.get('qfieldsync', {}))

803 self.caps.globalProps.update(self.qgisCaps.properties.get('QFieldSync', {}))

804

805 self.caps.qgisPath = ''

806 if self.package.qgisProvider.store.type == gws.plugin.qgis.project.StoreType.file:

807 self.caps.qgisPath = self.package.qgisProvider.store.path

808

809 self.caps.dirsToCopy = self.check_dirs_to_copy()

810 self.caps.baseMapLayerIds = self.base_map_layer_ids()

811

812 for sl in gws.gis.source.filter_layers(self.qgisCaps.sourceLayers, is_group=False):

813 le = self.layer_entry(sl)

814 if le:

815 le.qgisId = sl.sourceId

816 le.gpId = len(self.caps.layerMap)

817 le.sourceLayer = sl

818 self.caps.layerMap[le.qgisId] = le

819

820 self.caps.offlineCopyOnlyAoi = self.caps.globalProps.get('offlineCopyOnlyAoi') == 1

821

822 aoi = self.caps.globalProps.get('areaOfInterest')

823 if aoi:

824 crs = self.caps.globalProps.get('areaOfInterestCrs')

825 sh = gws.base.shape.from_wkt(

826 aoi,

827 gws.gis.crs.get(crs) if crs else self.qgisCaps.projectCrs

828 )

829 self.caps.areaOfInterest = sh.bounds()

830

831 return self.caps

832

833 def check_dirs_to_copy(self):

834 dc_str = self.caps.globalProps.get('dirsToCopy')

835 if not dc_str:

836 return []

837

838 try:

839 dc = gws.lib.jsonx.from_string(dc_str)

840 except gws.lib.jsonx.Error:

841 gws.log.warning(f'dirsToCopy: invalid JSON')

842 return []

843

844 dirs = []

845

846 for dir_name, flag in sorted(dc.items()):

847 if flag is not True:

848 continue

849

850 if not dir_name.startswith('/'):

851 if not self.caps.qgisPath:

852 gws.log.warning(f'dirsToCopy: cannot determine an absolute path for {dir_name!r}')

853 continue

854 dir_name = gws.lib.osx.abs_path(dir_name, self.caps.qgisPath)

855

856 if any(dir_name.startswith(d) for d in dirs):

857 continue

858 dirs.append(dir_name)

859

860 return dirs

861

862 def base_map_layer_ids(self) -> list[str]:

863 f = self.caps.globalProps.get('createBaseMap')

864 if f != 1:

865 return []

866

867 bt = self.caps.globalProps.get('baseMapType')

868

869 if bt == 'mapTheme':

870 th = self.caps.globalProps.get('baseMapTheme', '')

871 vp = self.qgisCaps.visibilityPresets.get(th)

872 if not vp:

873 gws.log.warning(f'map theme {th!r} not found')

874 return []

875 return vp

876

877 if bt == 'singleLayer':

878 uid = self.caps.globalProps.get('baseMapLayer', '')

879 if not uid:

880 return []

881 return [uid]

882

883 return []

884

885 def layer_entry(self, sl: gws.SourceLayer) -> Optional[LayerEntry]:

886 qf_props = {}

887

888 for k, v in sl.properties.items():

889 if k.startswith('QFieldSync/'):

890 qf_props[k.split('/').pop()] = v

891

892 # 'offline', 'no_action' or 'remove'

893 qf_action = qf_props.get('action', 'no_action')

894

895 prov = sl.dataSource.get('provider')

896

897 if prov == 'postgres':

898 if qf_action == 'remove':

899 return LayerEntry(action=LayerAction.remove)

900

901 if qf_action == 'offline':

902 return self.postgres_layer_entry(sl, qf_props)

903

904 else:

905 if sl.sourceId in self.caps.baseMapLayerIds:

906 return LayerEntry(action=LayerAction.baseMap)

907

908 if qf_action == 'remove':

909 return LayerEntry(action=LayerAction.remove)

910

911 if qf_action == 'offline':

912 gws.log.warning(f'layer {sl.sourceId!r}: offline editing of {prov!r} not supported')

913 return LayerEntry(action=LayerAction.remove)

914

915 def postgres_layer_entry(self, sl, qf_props):

916 read_only = qf_props.get('is_geometry_locked')

917

918 table_name = sl.dataSource.get('table')

919 if not table_name or table_name.startswith('(') or table_name.upper().startswith('SELECT '):

920 gws.log.warning(f'layer {sl.sourceId!r}: no table name')

921 return LayerEntry(action=LayerAction.remove)

922

923 me = self.model_entry_for_table(sl)

924 if not me:

925 gws.log.warning(f'layer {sl.sourceId!r}: no model')

926 return LayerEntry(action=LayerAction.remove)

927

928 if not read_only and not me.model.isEditable:

929 gws.log.warning(f'layer {sl.sourceId!r}: table {table_name!r} is not editable')

930 return LayerEntry(action=LayerAction.remove)

931

932 return LayerEntry(

933 action=LayerAction.edit,

934 readOnly=read_only,

935 modelEntry=me,

936 sqlFilter=sl.dataSource.get('sql', ''),

937 )

938

939 def model_entry_for_table(self, sl: gws.SourceLayer) -> Optional[ModelEntry]:

940 table_name = sl.dataSource.get('table')

941

942 for model in self.package.models:

943 full_name = model.db.join_table_name('', model.tableName)

944 if full_name == model.db.join_table_name('', table_name):

945 gp_name = self.gp_name_for_model(full_name)

946 if gp_name not in self.caps.modelMap:

947 self.caps.modelMap[gp_name] = self.model_entry(gp_name, model)

948 return self.caps.modelMap[gp_name]

949

950 db = self.package.qgisProvider.postgres_provider_from_datasource(sl.dataSource)

951 if not db.has_table(table_name):

952 gws.log.warning(f'layer {sl.sourceId!r}: table {table_name!r} not found')

953 return

954

955 gp_name = self.gp_name_for_model(table_name)

956

957 if gp_name not in self.caps.modelMap:

958 model = self.package.root.create_shared(

959 gws.ext.object.model,

960 gws.Config(

961 uid=f'qfield_model_{table_name}',

962 type='postgres',

963 # NB: permissions are checked in the public export/import functions above

964 permissions=gws.Config(read=gws.c.PUBLIC, edit=gws.c.PUBLIC),

965 tableName=table_name,

966 isEditable=True,

967 _defaultDb=db,

968 )

969 )

970 self.caps.modelMap[gp_name] = self.model_entry(gp_name, model)

971

972 return self.caps.modelMap[gp_name]

973

974 def model_entry(self, gp_name: str, model: gws.DatabaseModel) -> ModelEntry:

975 return ModelEntry(

976 gpId=len(self.caps.modelMap),

977 gpName=gp_name,

978 model=model,

979 fidToPkey={},

980 columnIndex={},

981 editOperations=[],

982 features=[],

983 )

984

985 def gp_name_for_model(self, table_name):

986 if '.' not in table_name:

987 table_name = 'public.' + table_name

988 return 'qm_' + table_name.replace('.', '_')

991##

994def _write_offline_log(ol: OfflineLog, conn: sa.Connection):

995 """Create logging tables for the Qgis Offline editing feature.

996

997 see qgis/src/core/qgsofflineediting.cpp createLoggingTables()

998 """

999

1000 meta = sa.MetaData()

1001 tables = _offline_log_tables(meta)

1002

1003 meta.create_all(conn)

1004

1005 for name, tab in tables.items():

1006 data = getattr(ol, name, [])

1007 if data:

1008 conn.execute(sa.insert(tab), data)

1009

1010

1011def _read_offline_log(conn: sa.Connection) -> OfflineLog:

1012 meta = sa.MetaData()

1013 tables = _offline_log_tables(meta)

1014

1015 ol = OfflineLog()

1016

1017 for name, tab in tables.items():

1018 sel = sa.select(tab)

1019 setattr(ol, name, [gws.u.to_dict(r) for r in conn.execute(sel)])

1020

1021 by_commit = lambda r: r['commit_no']

1022

1023 ol.log_added_attrs.sort(key=by_commit)

1024 ol.log_geometry_updates.sort(key=by_commit)

1025 ol.log_geometry_updates.sort(key=by_commit)

1026

1027 return ol

1028

1029

1030def _offline_log_tables(meta: sa.MetaData) -> dict[str, sa.Table]:

1031 ddl = '''

1032 log_added_attrs (layer_id INT, commit_no INT, name TEXT, type INT, length INT, precision INT, comment TEXT)

1033 log_added_features (layer_id INT, fid INT)

1034 log_feature_updates (layer_id INT, commit_no INT, fid INT, attr INT, value TEXT)

1035 log_fids (layer_id INT, offline_fid INT, remote_fid INT, remote_pk TEXT)

1036 log_geometry_updates (layer_id INT, commit_no INT, fid INT, geom_wkt TEXT)

1037 log_indices (name TEXT, last_index INT)

1038 log_layer_ids (id INT, qgis_id TEXT)

1039 log_removed_features (layer_id INT, fid INT)

1040 '''

1041

1042 # our extensions

1043

1044 ddl += 'log_gws_columns (layer_id INT, attr INT, name TEXT)\n'

1045 ddl += 'log_gws_tables (layer_id INT, name TEXT)\n'

1046

1047 tables = {}

1048

1049 for ln in ddl.strip().split('\n'):

1050 name, rest = ln.split(maxsplit=1)

1051 args = [name, meta]

1052 for f in rest.strip()[1:-1].split(','):

1053 fname, ftype = f.split()

1054 fn = None

1055 if ftype == 'INT':

1056 fn = sa.Integer

1057 if ftype == 'TEXT':

1058 fn = sa.String

1059 args.append(sa.Column(fname, fn))

1060 tables[name] = sa.Table(*args)

1061

1062 return tables