Coverage for pygeoapi/provider/ogr.py: 63%

1# =================================================================

3# Authors: Just van den Broecke <justb4@gmail.com>

4# Francesco Bartoli <xbartolone@gmail.com>

10# Permission is hereby granted, free of charge, to any person

11# obtaining a copy of this software and associated documentation

12# files (the "Software"), to deal in the Software without

13# restriction, including without limitation the rights to use,

14# copy, modify, merge, publish, distribute, sublicense, and/or sell

15# copies of the Software, and to permit persons to whom the

16# Software is furnished to do so, subject to the following

17# conditions:

18#

19# The above copyright notice and this permission notice shall be

20# included in all copies or substantial portions of the Software.

21#

22# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

23# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES

24# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

25# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT

26# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,

27# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

28# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

29# OTHER DEALINGS IN THE SOFTWARE.

30#

31# =================================================================

33from copy import deepcopy

34import functools

35import importlib

36import logging

37import os

38from typing import Any

40from osgeo import gdal as osgeo_gdal

41from osgeo import ogr as osgeo_ogr

42from osgeo import osr as osgeo_osr

44from pygeoapi.crs import get_crs

45from pygeoapi.provider.base import (

46 BaseProvider, ProviderGenericError,

47 ProviderQueryError, ProviderConnectionError,

48 ProviderItemNotFoundError)

50LOGGER = logging.getLogger(__name__)

53class OGRProvider(BaseProvider):

54 """

55 OGR Provider. Uses GDAL/OGR Python-bindings to access OGR

56 Vector sources. References:

57 https://pcjericks.github.io/py-gdalogr-cookbook/

58 https://gdal.org/ogr_formats.html (per-driver specifics).

60 In theory any OGR source type (Driver) could be used, although

61 some Source Types are Driver-specific handling. This is handled

62 in Source Helper classes, instantiated per Source-Type.

64 The following Source Types have been tested to work:

65 GeoPackage (GPKG), SQLite, GeoJSON, ESRI Shapefile, WFS v2.

66 """

68 # To deal with some OGR Source-Driver specifics.

69 SOURCE_HELPERS = {

70 'ESRIJSON': 'pygeoapi.provider.ogr.ESRIJSONHelper',

71 'WFS': 'pygeoapi.provider.ogr.WFSHelper',

72 '*': 'pygeoapi.provider.ogr.CommonSourceHelper'

73 }

74 os.environ['OGR_GEOJSON_MAX_OBJ_SIZE'] = os.environ.get(

75 'OGR_GEOJSON_MAX_OBJ_SIZE', '20MB')

77 def __init__(self, provider_def):

78 """

79 Initialize object

81 # Typical OGRProvider YAML config:

83 provider:

84 name: OGR

85 data:

86 source_type: WFS

87 source: WFS:https://service.pdok.nl/kadaster/rdinfo/wfs/v1_0?

88 source_capabilities:

89 paging: True

90 source_options:

91 OGR_WFS_LOAD_MULTIPLE_LAYER_DEFN: NO

92 # open_options:

93 # EXPOSE_GML_ID: NO

94 gdal_ogr_options:

95 EMPTY_AS_NULL: NO

96 GDAL_CACHEMAX: 64

97 # GDAL_HTTP_PROXY: (optional proxy)

98 # GDAL_PROXY_AUTH: (optional auth for remote WFS)

99 CPL_DEBUG: NO

100

101 id_field: gml_id

102 layer: rdinfo:stations

103

104

105 :param provider_def: provider definition

106

107 :returns: pygeoapi.provider.ogr.OGRProvider

108 """

109

110 super().__init__(provider_def)

111

112 self.ogr = osgeo_ogr

113 # http://trac.osgeo.org/gdal/wiki/PythonGotchas

114 self.gdal = osgeo_gdal

115 LOGGER.info("Using GDAL/OGR version: %d"

116 % int(osgeo_gdal.VersionInfo('VERSION_NUM')))

117

118 # install error handler

119 err = GdalErrorHandler()

120 self.handler = err.handler

121 self.gdal.PushErrorHandler(self.handler)

122 # Exceptions will get raised on anything >= gdal.CE_Failure

123 self.gdal.UseExceptions()

124 LOGGER.debug('Setting OGR properties')

125

126 self.data_def = provider_def['data']

127

128 # Generic GDAL/OGR options (optional)

129 gdal_ogr_options = self.data_def.get('gdal_ogr_options', {})

130 for key in gdal_ogr_options:

131 self.gdal.SetConfigOption(key, str(gdal_ogr_options[key]))

132

133 # Driver-specific options (optional)

134 source_options = self.data_def.get('source_options', {})

135 for key in source_options: 135 ↛ 136line 135 didn't jump to line 136 because the loop on line 135 never started

136 self.gdal.SetConfigOption(key, str(source_options[key]))

137 # Open options

138 self.open_options = self.data_def.get('open_options', {})

139

140 self.source_capabilities = self.data_def.get('source_capabilities',

141 {'paging': False})

142

143 if self.data_def.get('source_srs') or self.data_def.get('target_srs'): 143 ↛ 144line 143 didn't jump to line 144 because the condition on line 143 was never true

144 LOGGER.warning('source/target_srs no longer supported in OGRProvider') # noqa

145 LOGGER.warning('Use crs and storage_crs in config, see docs')

146

147 self._load_source_helper(self.data_def['source_type'])

148

149 self.geom_field = provider_def.get('geom_field')

150

151 # ID field is required

152 self.id_field = provider_def.get('id_field')

153 if not self.id_field: 153 ↛ 154line 153 didn't jump to line 154 because the condition on line 153 was never true

154 msg = 'Need explicit \'id_field\' attr in provider config'

155 LOGGER.error(msg)

156 raise Exception(msg)

157

158 # Layer name is required

159 self.layer_name = provider_def.get('layer')

160 if not self.layer_name: 160 ↛ 161line 160 didn't jump to line 161 because the condition on line 160 was never true

161 msg = 'Need explicit \'layer\' attr in provider config'

162 LOGGER.error(msg)

163 raise Exception(msg)

164

165 # Init driver and Source connection

166 self.driver = None

167 self.conn = None

168

169 LOGGER.debug('Grabbing field information')

170 self.get_fields()

171

172 def _list_open_options(self):

173 return [

174 f"{key}={str(value)}" for key, value in self.open_options.items()]

175

176 def _open(self):

177 source_type = self.data_def['source_type']

178 self.driver = self.ogr.GetDriverByName(source_type)

179 if not self.driver: 179 ↛ 180line 179 didn't jump to line 180 because the condition on line 179 was never true

180 msg = f'No Driver for Source: {source_type}'

181 LOGGER.error(msg)

182 raise Exception(msg)

183 if self.open_options: 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true

184 try:

185 self.conn = self.gdal.OpenEx(

186 self.data_def['source'],

187 self.gdal.OF_VECTOR,

188 open_options=self._list_open_options())

189 except RuntimeError as err:

190 LOGGER.error(err)

191 raise ProviderConnectionError(err)

192 except Exception:

193 msg = f'Ignore errors during the connection for Driver {source_type}' # noqa

194 LOGGER.error(msg)

195 self.conn = _ignore_gdal_error(

196 self.gdal, 'OpenEx', self.data_def['source'],

197 self.gdal.OF_VECTOR,

198 open_options=self._list_open_options())

199 else:

200 try:

201 self.conn = self.driver.Open(self.data_def['source'], 0)

202 except RuntimeError as err:

203 LOGGER.error(err)

204 raise ProviderConnectionError(err)

205 except Exception:

206 msg = f'Ignore errors during the connection for Driver {source_type}' # noqa

207 LOGGER.error(msg)

208 # ignore errors for ESRIJSON not having geometry member

209 # see https://github.com/OSGeo/gdal/commit/38b0feed67f80ded32be6c508323d862e1a14474 # noqa

210 self.conn = _ignore_gdal_error(

211 self.driver, 'Open', self.data_def['source'], 0)

212 if not self.conn: 212 ↛ 213line 212 didn't jump to line 213 because the condition on line 212 was never true

213 msg = 'Cannot open OGR Source: %s' % self.data_def['source']

214 LOGGER.error(msg)

215 raise Exception(msg)

216

217 # Always need to disable paging immediately after Open!

218 if self.source_capabilities['paging']: 218 ↛ exitline 218 didn't return from function '_open' because the condition on line 218 was always true

219 self.source_helper.disable_paging()

220

221 def _close(self):

222 self.source_helper.close()

223 self.conn = None

224 LOGGER.debug('closed self.conn')

225

226 self.driver = None

227

228 def _get_layer(self):

229 if not self.conn: 229 ↛ 233line 229 didn't jump to line 233 because the condition on line 229 was always true

230 self._open()

231

232 # Delegate getting Layer to SourceHelper

233 return self.source_helper.get_layer()

234

235 def get_fields(self):

236 """

237 Get provider field information (names, types)

238

239 :returns: dict of fields

240 """

241

242 if not self._fields: 242 ↛ 278line 242 didn't jump to line 278 because the condition on line 242 was always true

243 try:

244 layer_defn = self._get_layer().GetLayerDefn()

245 for fld in range(layer_defn.GetFieldCount()):

246 field_defn = layer_defn.GetFieldDefn(fld)

247 fieldName = field_defn.GetName()

248 fieldTypeCode = field_defn.GetType()

249 fieldType = field_defn.GetFieldTypeName(fieldTypeCode)

250

251 fieldName2 = fieldType.lower()

252

253 if fieldName2 == 'integer64':

254 fieldName2 = 'integer'

255 elif fieldName2 == 'real': 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true

256 fieldName2 = 'number'

257

258 self._fields[fieldName] = {'type': fieldName2}

259

260 if fieldName2 == 'datetime': 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true

261 self._fields[fieldName] = {

262 'type': 'string',

263 'format': 'date-time'

264 }

265

266 # fieldWidth = layer_defn.GetFieldDefn(fld).GetWidth()

267 # GetPrecision = layer_defn.GetFieldDefn(fld).GetPrecision() # noqa

268

269 except RuntimeError as err:

270 LOGGER.error(err)

271 raise ProviderConnectionError(err)

272 except Exception as err:

273 LOGGER.error(err)

274

275 finally:

276 self._close()

277

278 return self._fields

279

280 def query(self, offset=0, limit=10, resulttype='results',

281 bbox=[], datetime_=None, properties=[], sortby=[],

282 select_properties=[], skip_geometry=False, q=None,

283 crs_transform_spec=None, **kwargs):

284 """

285 Query OGR source

286

287 :param offset: starting record to return (default 0)

288 :param limit: number of records to return (default 10)

289 :param resulttype: return results or hit limit (default results)

290 :param bbox: bounding box [minx,miny,maxx,maxy]

291 :param datetime_: temporal (datestamp or extent)

292 :param properties: list of tuples (name, value)

293 :param sortby: list of dicts (property, order)

294 :param select_properties: list of property names

295 :param skip_geometry: bool of whether to skip geometry (default False)

296 :param q: full-text search term(s)

297 :param crs_transform_spec: `CrsTransformSpec` instance, optional

298

299 :returns: dict of 0..n GeoJSON features

300 """

301 result = None

302 try:

303 if self.source_capabilities['paging']: 303 ↛ 306line 303 didn't jump to line 306 because the condition on line 303 was always true

304 self.source_helper.enable_paging(offset, limit)

305

306 layer = self._get_layer()

307

308 if bbox:

309 LOGGER.debug('processing bbox parameter')

310 minx, miny, maxx, maxy = [float(b) for b in bbox]

311

312 wkt = f"POLYGON (({minx} {miny},{minx} {maxy},{maxx} {maxy}," \

313 f"{maxx} {miny},{minx} {miny}))"

314

315 polygon = self.ogr.CreateGeometryFromWkt(wkt)

316 layer.SetSpatialFilter(polygon)

317

318 # layer.SetSpatialFilterRect(

319 # float(minx), float(miny), float(maxx), float(maxy))

320

321 if properties:

322 LOGGER.debug('processing properties')

323

324 attribute_filter = ' and '.join(

325 map(lambda x: f'{x[0]} = {sanitize_attribute_value(x[1])}', properties) # noqa

326 )

327

328 LOGGER.debug(attribute_filter)

329

330 layer.SetAttributeFilter(attribute_filter)

331

332 # Make response based on resulttype specified

333 if resulttype == 'hits': 333 ↛ 334line 333 didn't jump to line 334 because the condition on line 333 was never true

334 LOGGER.debug('hits only specified')

335 result = self._response_feature_hits(layer)

336 elif resulttype == 'results': 336 ↛ 345line 336 didn't jump to line 345 because the condition on line 336 was always true

337 LOGGER.debug('results specified')

338 result = self._response_feature_collection(

339 layer,

340 limit,

341 skip_geometry=skip_geometry,

342 crs_transform_spec=crs_transform_spec,

343 )

344 else:

345 LOGGER.error('Invalid resulttype: %s' % resulttype)

346

347 except RuntimeError as err:

348 LOGGER.error(err)

349 raise ProviderQueryError(err)

350 except ProviderConnectionError as err:

351 LOGGER.error(err)

352 raise ProviderConnectionError(err)

353 except Exception as err:

354 LOGGER.error(err)

355 raise ProviderGenericError(err)

356

357 finally:

358 self._close()

359

360 return result

361

362 def _get_spatial_ref_from_epsg(self, epsg_code, force_auth_comply=False):

363 axis_order = osgeo_osr.OAMS_AUTHORITY_COMPLIANT

364 # Assume http://www.opengis.net/def/crs/OGC/1.3/CRS84

365 # for EPSG:4326, GeoJSON Compliant

366 if epsg_code == 4326 and not force_auth_comply:

367 axis_order = osgeo_osr.OAMS_TRADITIONAL_GIS_ORDER

368 spatial_ref = osgeo_osr.SpatialReference()

369 spatial_ref.SetAxisMappingStrategy(axis_order)

370 spatial_ref.ImportFromEPSG(epsg_code)

371 return spatial_ref

372

373 def _get_spatial_ref_from_uri(self, crs_uri):

374 # Assume http://www.opengis.net/def/crs/OGC/1.3/CRS84

375 # is EPSG:4326, with lon/lat order

376 if crs_uri == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84':

377 epsg_code = 4326

378 force_auth_comply = False

379 else:

380 pyproj_crs = get_crs(crs_uri)

381 epsg_code = int(pyproj_crs.srs.split(':')[1])

382 force_auth_comply = True

383 return self._get_spatial_ref_from_epsg(

384 epsg_code, force_auth_comply=force_auth_comply)

385

386 def _get_crs_transform(self, crs_transform_spec=None):

387 if crs_transform_spec is not None: 387 ↛ 388line 387 didn't jump to line 388 because the condition on line 387 was never true

388 source = self._get_spatial_ref_from_uri(

389 crs_transform_spec.source_crs_uri)

390 target = self._get_spatial_ref_from_uri(

391 crs_transform_spec.target_crs_uri)

392 crs_transform = osgeo_osr.CoordinateTransformation(source, target)

393 else:

394 crs_transform = None

395 return crs_transform

396

397 def get(self, identifier, crs_transform_spec=None, **kwargs):

398 """

399 Get Feature by id

400

401 :param identifier: feature id

402 :param crs_transform_spec: `CrsTransformSpec` instance, optional

403

404 :returns: feature collection

405 """

406 result = None

407 crs_transform_out = self._get_crs_transform(crs_transform_spec)

408

409 try:

410 LOGGER.debug(f'Fetching identifier {identifier}')

411 layer = self._get_layer()

412

413 identifier2 = sanitize_attribute_value(identifier)

414

415 layer.SetAttributeFilter(f'{self.id_field} = {identifier2}')

416

417 ogr_feature = self._get_next_feature(layer, identifier)

418 result = self._ogr_feature_to_json(

419 ogr_feature, crs_transform_out=crs_transform_out,

420 )

421

422 except RuntimeError as err:

423 LOGGER.error(err)

424 raise ProviderQueryError(err)

425 except ProviderConnectionError as err:

426 LOGGER.error(err)

427 raise ProviderConnectionError(err)

428 except ProviderItemNotFoundError as err:

429 LOGGER.error(err)

430 raise ProviderItemNotFoundError(err)

431 except Exception as err:

432 LOGGER.error(err)

433 raise ProviderGenericError(err)

434

435 finally:

436 self._close()

437

438 return result

439

440 def __repr__(self):

441 return f'<OGRProvider> {self.data}'

442

443 def _load_source_helper(self, source_type):

444 """

445 Loads Source Helper by name.

446

447 :param Source type: Source type name

448

449 :returns: Source Helper object

450 """

451 helper_type = source_type

452 if source_type not in OGRProvider.SOURCE_HELPERS.keys(): 452 ↛ 456line 452 didn't jump to line 456 because the condition on line 452 was always true

453 helper_type = '*'

454

455 # Create object from full package.class name string.

456 source_helper_class = OGRProvider.SOURCE_HELPERS[helper_type]

457

458 packagename, classname = source_helper_class.rsplit('.', 1)

459 module = importlib.import_module(packagename)

460 class_ = getattr(module, classname)

461 self.source_helper = class_(self)

462

463 def _get_next_feature(self, layer, feature_id):

464 try:

465 if layer.GetFeatureCount() == 0:

466 msg = f"item {feature_id} is not found"

467 LOGGER.error(msg)

468 raise ProviderItemNotFoundError(msg)

469 # Ignore gdal error

470 next_feature = _ignore_gdal_error(layer, 'GetNextFeature')

471 if next_feature: 471 ↛ 478line 471 didn't jump to line 478 because the condition on line 471 was always true

472 if all(val is None for val in next_feature.items().values()): 472 ↛ 473line 472 didn't jump to line 473 because the condition on line 472 was never true

473 self.gdal.Error(

474 self.gdal.CE_Failure, 1,

475 "Object properties are all null"

476 )

477 else:

478 raise RuntimeError(

479 f"GDAL has returned a null feature for item {feature_id}")

480 return next_feature

481 except RuntimeError as gdalerr:

482 LOGGER.error(self.gdal.GetLastErrorMsg())

483 raise gdalerr

484

485 def _ogr_feature_to_json(

486 self, ogr_feature, skip_geometry=False, crs_transform_out=None,

487 ):

488 if self.geom_field is not None: 488 ↛ 489line 488 didn't jump to line 489 because the condition on line 488 was never true

489 geom = ogr_feature.GetGeomFieldRef(self.geom_field)

490 else:

491 geom = ogr_feature.GetGeometryRef()

492

493 if crs_transform_out is not None: 493 ↛ 495line 493 didn't jump to line 495 because the condition on line 493 was never true

494 # Optionally reproject the geometry

495 geom.Transform(crs_transform_out)

496

497 # NB With GDAL >= 3.3 seems that Axis is swapped for e.g.

498 # EPSG:4258 in ExportToJson where it shouldn't. See #1174.

499 # Suppress swapping by unassigning SpatialReference

500 geom.AssignSpatialReference(None)

501 json_feature = ogr_feature.ExportToJson(as_object=True)

502

503 if skip_geometry: 503 ↛ 504line 503 didn't jump to line 504 because the condition on line 503 was never true

504 json_feature['geometry'] = None

505

506 # Drop non-defined properties

507 if self.properties: 507 ↛ 508line 507 didn't jump to line 508 because the condition on line 507 was never true

508 props = json_feature['properties']

509 dropping_keys = deepcopy(props).keys()

510 for item in dropping_keys:

511 if item not in self.properties:

512 props.pop(item)

513

514 try:

515 json_feature['id'] = json_feature['properties'].pop(

516 self.id_field, json_feature['id']

517 )

518 except KeyError as err:

519 LOGGER.error(f"Cannot use configured id_field nor fid as id, err={err}") # noqa

520

521 return json_feature

522

523 def _response_feature_collection(

524 self, layer, limit, skip_geometry=False, crs_transform_spec=None,

525 ):

526 """

527 Assembles output from Layer query as

528 GeoJSON FeatureCollection structure.

529

530 :returns: GeoJSON FeatureCollection

531 """

532

533 feature_collection = {

534 'type': 'FeatureCollection',

535 'features': []

536 }

537

538 # See https://github.com/OSGeo/gdal/blob/master/autotest/

539 # ogr/ogr_wfs.py#L313

540 layer.ResetReading()

541 crs_transform_out = self._get_crs_transform(crs_transform_spec)

542

543 try:

544 # Ignore gdal error

545 ogr_feature = _ignore_gdal_error(layer, 'GetNextFeature')

546 count = 0

547 while ogr_feature is not None:

548 json_feature = self._ogr_feature_to_json(

549 ogr_feature,

550 skip_geometry=skip_geometry,

551 crs_transform_out=crs_transform_out,

552 )

553

554 feature_collection['features'].append(json_feature)

555

556 count += 1

557 if count == limit:

558 break

559

560 # Ignore gdal error

561 ogr_feature = _ignore_gdal_error(layer, 'GetNextFeature')

562

563 return feature_collection

564 except RuntimeError as gdalerr:

565 LOGGER.error(self.gdal.GetLastErrorMsg())

566 raise gdalerr

567

568 def _response_feature_hits(self, layer):

569 """

570 Assembles GeoJSON hits from OGR Feature count

571 e.g: http://localhost:5000/collections/

572 hotosm_bdi_waterways/items?resulttype=hits

573

574 :returns: GeoJSON FeaturesCollection

575 """

576

577 return {

578 'type': 'FeatureCollection',

579 'numberMatched': layer.GetFeatureCount(),

580 'features': []

581 }

582

583

584class InvalidHelperError(Exception):

585 """Invalid helper"""

586 pass

587

588

589class SourceHelper:

590 """

591 Helper classes for OGR-specific Source Types (Drivers).

592 For some actions Driver-specific settings or processing is

593 required. This is delegated to the OGR SourceHelper classes.

594 """

595

596 def __init__(self, provider):

597 """

598 Initialize object with related OGRProvider object.

599

600 :param provider: provider instance

601

602 :returns: pygeoapi.provider.ogr.SourceHelper

603 """

604 self.provider = provider

605

606 def close(self):

607 """

608 OGR Driver-specific handling of closing dataset.

609 Default is no specific handling.

610

611 """

612

613 pass

614

615 def get_layer(self):

616 """

617 Default action to get a Layer object from opened OGR Driver.

618 :return:

619 """

620 layer = self.provider.conn.GetLayerByName(self.provider.layer_name)

621

622 if not layer: 622 ↛ 623line 622 didn't jump to line 623 because the condition on line 622 was never true

623 msg = f'Cannot get Layer {self.provider.layer_name} from OGR Source' # noqa

624 LOGGER.error(msg)

625 raise Exception(msg)

626

627 return layer

628

629 def enable_paging(self, offset=-1, limit=-1):

630 """

631 Enable paged access to dataset (OGR Driver-specific)

632

633 """

634

635 pass

636

637 def disable_paging(self):

638 """

639 Disable paged access to dataset (OGR Driver-specific)

640 """

641

642 pass

643

644

645class CommonSourceHelper(SourceHelper):

646 """

647 SourceHelper for most common OGR Source types:

648 Shapefile, GeoPackage, SQLite, GeoJSON etc.

649 """

650 def __init__(self, provider):

651 """

652 Initialize object

653

654 :param provider: provider instance

655

656 :returns: pygeoapi.provider.ogr.SourceHelper

657 """

658

659 super().__init__(provider)

660

661 self.offset = -1

662 self.limit = -1

663 self.result_set = None

664

665 def close(self):

666 """

667 OGR Driver-specific handling of closing dataset.

668 If ExecuteSQL has been (successfully) called

669 must close ResultSet explicitly.

670 https://gis.stackexchange.com/questions/114112/explicitly-close-a-ogr-result-object-from-a-call-to-executesql # noqa

671 """

672

673 if not self.result_set:

674 return

675

676 try:

677 self.provider.conn.ReleaseResultSet(self.result_set)

678 except Exception as err:

679 msg = f'ReleaseResultSet exception for Layer {self.provider.layer_name}' # noqa

680 LOGGER.error(msg, err)

681 finally:

682 self.result_set = None

683

684 def enable_paging(self, offset=-1, limit=-1):

685 """

686 Enable paged access to dataset (OGR Driver-specific)

687 using OGR SQL https://gdal.org/user/ogr_sql_dialect.html

688 e.g. SELECT * FROM poly LIMIT 10 OFFSET 30

689

690 """

691 self.offset = offset

692 self.limit = limit

693

694 def disable_paging(self):

695 """

696 Disable paged access to dataset (OGR Driver-specific)

697 """

698

699 pass

700

701 def get_layer(self):

702 """

703 Gets OGR Layer from opened OGR dataset.

704 When offset defined 1 or greater will invoke

705 OGR SQL SELECT with LIMIT and OFFSET and return

706 as Layer as ResultSet from ExecuteSQL on dataset.

707 :return: OGR layer object

708 """

709 if self.offset <= 0:

710 return SourceHelper.get_layer(self)

711

712 self.close()

713

714 sql = f'SELECT * FROM "{self.provider.layer_name}" LIMIT {self.limit} OFFSET {self.offset}' # noqa

715 self.result_set = self.provider.conn.ExecuteSQL(sql)

716

717 # Reset since needs to be set each time explicitly

718 self.offset = -1

719 self.limit = -1

720

721 if not self.result_set: 721 ↛ 722line 721 didn't jump to line 722 because the condition on line 721 was never true

722 msg = f'Cannot get Layer {self.provider.layer_name} via ExecuteSQL'

723 LOGGER.error(msg)

724 raise Exception(msg)

725

726 return self.result_set

727

728

729class ESRIJSONHelper(CommonSourceHelper):

730

731 def __init__(self, provider):

732 """

733 Initialize object

734

735 :param provider: provider instance

736

737 :returns: pygeoapi.provider.ogr.SourceHelper

738 """

739

740 super().__init__(provider)

741

742 def enable_paging(self, offset=-1, limit=-1):

743 """

744 Enable paged access to dataset (OGR Driver-specific)

745

746 """

747 if offset < 0:

748 return

749

750 self.provider.open_options.update(FEATURE_SERVER_PAGING=True)

751 self.offset = offset

752 self.limit = limit

753

754 def disable_paging(self):

755 """

756 Disable paged access to dataset (OGR Driver-specific)

757 """

758

759 self.provider.open_options.update(FEATURE_SERVER_PAGING=False)

760

761 def get_layer(self):

762 """

763 Gets OGR Layer from opened OGR dataset.

764 When offset defined 1 or greater will invoke

765 OGR SQL SELECT with LIMIT and OFFSET and return

766 as Layer as ResultSet from ExecuteSQL on dataset.

767 :return: OGR layer object

768 """

769 if self.offset <= 0:

770 return CommonSourceHelper.get_layer(self)

771

772 self.close()

773

774 sql = f"SELECT * FROM {self.provider.layer_name} LIMIT {self.limit} OFFSET {self.offset}" # noqa

775 self.result_set = self.provider.conn.ExecuteSQL(sql)

776

777 # Reset since needs to be set each time explicitly

778 self.offset = -1

779 self.limit = -1

780

781 if not self.result_set:

782 msg = f'Cannot get Layer {self.provider.layer_name} via ExecuteSQL'

783 LOGGER.error(msg)

784 raise Exception(msg)

785

786 return self.result_set

787

788

789class WFSHelper(SourceHelper):

790

791 def __init__(self, provider):

792 """

793 Initialize object

794

795 :param provider: provider instance

796

797 :returns: pygeoapi.provider.ogr.SourceHelper

798 """

799

800 super().__init__(provider)

801

802 def enable_paging(self, offset=-1, limit=-1):

803 """

804 Enable paged access to dataset (OGR Driver-specific)

805

806 """

807

808 if offset < 0:

809 return

810

811 self.provider.gdal.SetConfigOption(

812 'OGR_WFS_PAGING_ALLOWED', 'ON')

813 self.provider.gdal.SetConfigOption(

814 'OGR_WFS_BASE_START_INDEX', str(offset))

815 self.provider.gdal.SetConfigOption(

816 'OGR_WFS_PAGE_SIZE', str(limit))

817

818 def disable_paging(self):

819 """

820 Disable paged access to dataset (OGR Driver-specific)

821 """

822

823 self.provider.gdal.SetConfigOption(

824 'OGR_WFS_PAGING_ALLOWED', None)

825 self.provider.gdal.SetConfigOption(

826 'OGR_WFS_PAGE_SIZE', None)

827

828

829class GdalErrorHandler:

830

831 def __init__(self):

832 """

833 Initialize the error handler

834

835 :returns: pygeoapi.provider.ogr.GdalErrorHandler

836 """

837 self.err_level = osgeo_gdal.CE_None

838 self.err_num = 0

839 self.err_msg = ''

840

841 def handler(self, err_level, err_num, err_msg):

842 """

843 Define custom GDAL error handler function

844

845 :param err_level: error level

846 :param err_num: internal gdal error number

847 :param err_msg: error message

848

849 :returns: pygeoapi.provider.ogr.GdalErrorHandler

850 """

851

852 err_type = {

853 osgeo_gdal.CE_None: 'None',

854 osgeo_gdal.CE_Debug: 'Debug',

855 osgeo_gdal.CE_Warning: 'Warning',

856 osgeo_gdal.CE_Failure: 'Failure',

857 osgeo_gdal.CE_Fatal: 'Fatal'

858 }

859 err_msg = err_msg.replace('\n', ' ')

860 level = err_type.get(err_level, 'None')

861

862 self.err_level = err_level

863 self.err_num = err_num

864 self.err_msg = err_msg

865

866 LOGGER.error('Error Number: %s, Type: %s, Msg: %s' % (

867 self.err_num, level, self.err_msg))

868 last_error = osgeo_gdal.GetLastErrorMsg()

869 if self.err_level >= osgeo_gdal.CE_Failure: 869 ↛ 870line 869 didn't jump to line 870 because the condition on line 869 was never true

870 if 'HTTP error code' in last_error:

871 # 500 <= http error ode <=599

872 for i in list(range(500, 599)):

873 if str(i) in last_error:

874 raise ProviderConnectionError(last_error)

875 else:

876 raise ProviderGenericError(last_error)

877

878

879def _silent_gdal_error(f):

880 """

881 Decorator function for gdal

882 """

883 @functools.wraps(f)

884 def wrapper(*args, **kwargs):

885 osgeo_gdal.PushErrorHandler('CPLQuietErrorHandler')

886 v = f(*args, **kwargs)

887 osgeo_gdal.PopErrorHandler()

888 return v

889

890 return wrapper

891

892

893@_silent_gdal_error

894def _ignore_gdal_error(inst, fn, *args, **kwargs) -> Any:

895 """

896 Evaluate the function with the object instance.

897

898 :param inst: Object instance

899 :param fn: String function name

900 :param args: List of positional arguments

901 :param kwargs: Keyword arguments

902

903 :returns: Any function evaluation result

904 """

905 value = getattr(inst, fn)(*args, **kwargs)

906 return value

907

908

909def sanitize_attribute_value(value) -> str:

910 """

911 Sanitize an attribute value used in an

912 OGR layer SetAttributeFilter function

913

914 :param value: `str` of attribute value

915

916 :returns: `str` of sanitized attribute value

917 """

918

919 if value is None: 919 ↛ 920line 919 didn't jump to line 920 because the condition on line 919 was never true

920 return 'NULL'

921

922 if isinstance(value, bool): 922 ↛ 923line 922 didn't jump to line 923 because the condition on line 922 was never true

923 return '1' if value else '0'

924

925 if isinstance(value, (int, float)): 925 ↛ 926line 925 didn't jump to line 926 because the condition on line 925 was never true

926 return f"'{value}'"

927

928 return "'" + str(value).replace("'", "''") + "'"