Coverage for brodata/gmw.py: 78%

1import json

2import logging

3import os

4from functools import partial

5from zipfile import ZipFile

7import numpy as np

8import pandas as pd

10from . import bro, gld, gar, frd, gmn, util

12logger = logging.getLogger(__name__)

15def get_well_code(bro_id):

16 """

17 Retrieve the well code based on a given BRO-ID and return it as plain text.

19 This function sends a GET request to fetch the well code associated with the

20 specified BRO-ID. If the request fails, it logs an error message and returns `None`.

22 Parameters

23 ----------

24 bro_id : str

25 The BRO-ID for which to retrieve the associated well code.

27 Returns

28 -------

29 well_code : str or None

30 The well code as plain text if the request is successful. Returns `None` if

31 the request fails.

32 """

34 url = f"{GroundwaterMonitoringWell._rest_url}/well-code/{bro_id}"

35 req = bro.util.get_with_rate_limit(url)

36 if req.status_code > 200:

37 logger.error(req.reason)

38 return

39 well_code = req.text

40 return well_code

43class GroundwaterMonitoringWell(bro.FileOrUrl):

44 """

45 Class to represent a Groundwater Monitoring Well (GMW) from the BRO.

47 This class parses XML data related to a groundwater monitoring well (GMW).

48 It extracts details such as location, monitoring tube data, and well history

49 and stores these in attributes.

51 Notes

52 -----

53 This class extends `bro.XmlFileOrUrl` and is designed to work with GMW XML data,

54 either from a file or URL.

55 """

57 _rest_url = "https://publiek.broservices.nl/gm/gmw/v1"

58 _xmlns = "http://www.broservices.nl/xsd/dsgmw/1.1"

59 _char = "GMW_C"

61 def _read_contents(self, tree):

62 ns = {

63 "brocom": "http://www.broservices.nl/xsd/brocommon/3.0",

64 "xmlns": self._xmlns,

65 }

67 object_names = ["GMW_PO", "GMW_PPO", "BRO_DO"]

68 gmw = self._get_main_object(tree, object_names, ns)

70 for key in gmw.attrib:

71 setattr(self, key.split("}", 1)[1], gmw.attrib[key])

72 for child in gmw:

73 key = self._get_tag(child)

74 if len(child) == 0:

75 setattr(self, key, child.text)

76 elif key == "standardizedLocation":

77 self._read_standardized_location(child)

78 elif key == "deliveredLocation":

79 self._read_delivered_location(child)

80 elif key == "wellHistory":

81 for grandchild in child:

82 key = self._get_tag(grandchild)

83 if key in ["wellConstructionDate", "wellRemovalDate"]:

84 setattr(self, key, self._read_date(grandchild))

85 elif key == "intermediateEvent":

86 if not hasattr(self, key):

87 self.intermediateEvent = []

88 event = self._read_intermediate_event(grandchild)

89 self.intermediateEvent.append(event)

90 else:

91 self._warn_unknown_tag(key)

93 elif key in ["deliveredVerticalPosition", "registrationHistory"]:

94 to_float = ["offset", "groundLevelPosition"]

95 self._read_children_of_children(child, to_float=to_float)

96 elif key in ["monitoringTube"]:

97 if not hasattr(self, key):

98 self.monitoringTube = []

99 tube = {}

100 to_float = [

101 "tubeTopDiameter",

102 "tubeTopPosition",

103 "screenLength",

104 "screenTopPosition",

105 "screenBottomPosition",

106 "plainTubePartLength",

107 ]

108 self._read_children_of_children(child, tube, to_float=to_float)

109 self.monitoringTube.append(tube)

110 else:

111 self._warn_unknown_tag(key)

112 if hasattr(self, "monitoringTube"):

113 self.monitoringTube = pd.DataFrame(self.monitoringTube)

114 tubeNumber = self.monitoringTube["tubeNumber"].astype(int)

115 self.monitoringTube["tubeNumber"] = tubeNumber

116 self.monitoringTube = self.monitoringTube.set_index("tubeNumber")

117 if hasattr(self, "intermediateEvent"):

118 self.intermediateEvent = pd.DataFrame(self.intermediateEvent)

119

120 def _read_intermediate_event(self, node):

121 d = {}

122 for child in node:

123 key = self._get_tag(child)

124 if key == "eventName":

125 d[key] = child.text

126 elif key == "eventDate":

127 d[key] = self._read_date(child)

128 else:

129 self._warn_unknown_tag(key)

130 return d

131

132

133def get_observations(

134 bro_ids,

135 kind="gld",

136 drop_references=True,

137 silent=False,

138 tmin=None,

139 tmax=None,

140 as_csv=False,

141 tube_number=None,

142 status=None,

143 observation_type=None,

144 qualifier=None,

145 to_path=None,

146 to_zip=None,

147 redownload=False,

148 zipfile=None,

149 continue_on_error=False,

150 sort=True,

151 drop_duplicates=True,

152 progress_callback=None,

153 _files=None,

154):

155 """

156 Retrieve groundwater observations for the specified monitoring wells (bro_ids).

157

158 This function fetches groundwater data for monitoring wells based on the provided

159 parameters. It supports different types of observations, allows filtering by tube

160 number, and can request the data in CSV format for groundwater level observations.

161

162 Parameters

163 ----------

164 bro_ids : str or list or pd.DataFrame

165 The BRO IDs of the monitoring wells for which to retrieve the data. If a

166 DataFrame is provided, its index is used as the list of BRO IDs.

167 kind : str, optional

168 The type of observations to retrieve. Can be one of {'gmn', 'gld', 'gar', 'frd'}.

169 Defaults to 'gld' (groundwater level dossier).

170 drop_references : bool or list of str, optional

171 Specifies whether to drop reference fields in the returned data. Defaults to True,

172 in which case 'gmnReferences', 'gldReferences', 'garReferences' and

173 'frdReferences' are removed. Only used when as_csv=True.

174 silent : bool, optional

175 If True, suppresses progress logging. Defaults to False.

176 tmin : str or datetime, optional

177 The minimum time filter for the observations. Defaults to None.

178 tmax : str or datetime, optional

179 The maximum time filter for the observations. Defaults to None.

180 as_csv : bool, optional

181 If True, requests the observations as CSV files instead of XML-files. Only valid

182 if `kind` is 'gld'. Defaults to False.

183 tube_number : int, optional

184 Filters observations to a specific tube number. Defaults to None.

185 status : str, optional

186 A status string for additional filtering. Possible values are

187 "volledigBeoordeeld", "voorlopig" and "onbekend" Only valid if `kind` is 'gld'.

188 Defaults to None.

189 observation_type : str, optional

190 An observation type string for additional filtering. Possible values are

191 "reguliereMeting" and "controleMeting". Only valid if `kind` is 'gld'. Defaults

192 to None.

193 qualifier : str or list of str, optional

194 A qualifier string for additional filtering. Only valid if `kind` is 'gld'.

195 Defaults to None.

196 to_path : str, optional

197 If not None, save the downloaded files in the directory named to_path. The

198 default is None.

199 to_zip : str, optional

200 If not None, save the downloaded files in a zip-file named to_zip. The default

201 is None.

202 redownload : bool, optional

203 When downloaded files exist in to_path or to_zip, read from these files when

204 redownload is False. If redownload is True, download the data again from the

205 BRO-servers. The default is False.

206 zipfile : zipfile.ZipFile, optional

207 A zipfile-object. When not None, zipfile is used to read previously downloaded

208 data from. The default is None.

209 continue_on_error : bool, optional

210 If True, continue after an error occurs during downloading or processing of

211 individual observation data. Defaults to False.

212 sort : bool, optional

213 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

214 drop_duplicates : bool, optional

215 If True, drop duplicate observations based on their timestamp. Only used if

216 `kind` is 'gld'. Defaults to True.

217 progress_callback : function, optional

218 A callback function that takes two arguments (current, total) to report

219 progress. If None, no progress reporting is done. Defaults to None.

220

221

222 Returns

223 -------

224 pd.DataFrame

225 A DataFrame containing the observations for the specified monitoring wells,

226 where each row corresponds to an individual observation. The index is the BRO-id

227 of the observation-objects.

228

229 Raises

230 ------

231 ValueError

232 If the specified `kind` is not supported, or if `as_csv=True` and `kind` is not

233 'gld', or if `qualifier` is provided for a kind other than 'gld'.

234 """

235 tubes = []

236

237 if isinstance(bro_ids, str):

238 bro_ids = [bro_ids]

239 silent = True

240

241 if isinstance(bro_ids, pd.DataFrame):

242 bro_ids = bro_ids.index

243

244 if as_csv and isinstance(drop_references, bool):

245 if drop_references:

246 drop_references = [

247 "gmnReferences",

248 "gldReferences",

249 "garReferences",

250 "frdReferences",

251 ]

252 else:

253 drop_references = []

254

255 if to_zip is not None:

256 if not redownload and os.path.isfile(to_zip):

257 raise (NotImplementedError("Redownload=False is not suppported yet"))

258 if to_path is None:

259 to_path = os.path.splitext(to_zip)[0]

260 remove_path_again = not os.path.isdir(to_path)

261 if _files is None:

262 _files = []

263

264 desc = f"Downloading {kind}-observations"

265 if as_csv and kind != "gld":

266 raise (ValueError("as_csv=True is only supported for kind=='gld'"))

267 if qualifier is not None and kind != "gld":

268 raise (ValueError("A qualifier is only supported for kind=='gld'"))

269 if to_path is not None and not os.path.isdir(to_path):

270 os.makedirs(to_path)

271

272 if kind == "gld":

273 meas_cl = gld.GroundwaterLevelDossier

274 elif kind == "gar":

275 meas_cl = gar.GroundwaterAnalysisReport

276 elif kind == "frd":

277 meas_cl = frd.FormationResistanceDossier

278 elif kind == "gmn":

279 meas_cl = gmn.GroundwaterMonitoringNetwork

280 else:

281 raise (ValueError(f"kind='{kind}' not supported"))

282

283 gld_kwargs = _get_gld_kwargs(

284 kind, tmin, tmax, qualifier, status, observation_type, sort, drop_duplicates

285 )

286

287 for igmw, bro_id in enumerate(

288 util.tqdm(np.unique(bro_ids), disable=silent, desc=desc)

289 ):

290 to_rel_file = util._get_to_file(

291 f"gmw_relations_{bro_id}.json", zipfile, to_path, _files

292 )

293 if zipfile is None and (

294 redownload or to_rel_file is None or not os.path.isfile(to_rel_file)

295 ):

296 url = f"https://publiek.broservices.nl/gm/v1/gmw-relations/{bro_id}"

297 req = bro.util.get_with_rate_limit(url)

298 if req.status_code > 200:

299 logger.error(req.json()["errors"][0]["message"])

300 return

301 if to_rel_file is not None:

302 with open(to_rel_file, "w") as f:

303 f.write(req.text)

304 data = req.json()

305 else:

306 if zipfile is not None:

307 with zipfile.open(to_rel_file) as f:

308 data = json.load(f)

309 else:

310 with open(to_rel_file) as f:

311 data = json.load(f)

312 for tube_ref in data["monitoringTubeReferences"]:

313 tube_ref["groundwaterMonitoringWell"] = data["gmwBroId"]

314 if tube_number is not None:

315 if tube_ref["tubeNumber"] != tube_number:

316 continue

317 ref_key = f"{kind}References"

318 for ref in tube_ref[ref_key]:

319 obsdata = _download_observations_for_bro_id(

320 ref["broId"],

321 meas_cl,

322 as_csv,

323 zipfile,

324 to_path,

325 _files,

326 gld_kwargs,

327 redownload=redownload,

328 continue_on_error=continue_on_error,

329 )

330 if as_csv:

331 gld_dict = tube_ref.copy()

332 gld_dict["observation"] = obsdata

333 # drop references, as these are dictionaries of no interest to the user

334 for key in drop_references:

335 if key in gld_dict:

336 gld_dict.pop(key)

337 # add fields of the ref to gld_dict, like the broId of the GLD

338 for key in ref:

339 gld_dict[key] = ref[key]

340 tubes.append(gld_dict)

341 else:

342 tubes.append(obsdata.to_dict())

343

344 if progress_callback is not None:

345 progress_callback(igmw + 1, len(bro_ids))

346 if to_zip is not None:

347 util._save_data_to_zip(to_zip, _files, remove_path_again, to_path)

348 return pd.DataFrame(tubes).set_index("broId")

349

350

351def _download_observations_for_bro_id(

352 bro_id,

353 meas_cl,

354 as_csv,

355 zipfile,

356 to_path,

357 _files,

358 gld_kwargs,

359 redownload=False,

360 continue_on_error=False,

361):

362 if as_csv:

363 fname = f"{bro_id}.csv"

364 observatietype = None

365 if "status" in gld_kwargs and gld_kwargs["status"] == "voorlopig":

366 observatietype = "regulier_voorlopig"

367 elif "status" in gld_kwargs and gld_kwargs["status"] == "volledigBeoordeeld":

368 observatietype = "regulier_beoordeeld"

369 elif "status" in gld_kwargs and gld_kwargs["status"] == "onbekend":

370 observatietype = "onbekend"

371 elif (

372 "observation_type" in gld_kwargs

373 and gld_kwargs["observation_type"] == "controleMeting"

374 ):

375 observatietype = "controle"

376 else:

377 fname = f"{bro_id}.xml"

378 to_file = util._get_to_file(fname, zipfile, to_path, _files)

379 if zipfile is None and (

380 redownload or to_file is None or not os.path.isfile(to_file)

381 ): # download the data

382 if as_csv:

383 try:

384 data = gld.get_objects_as_csv(

385 bro_id,

386 observatietype=observatietype,

387 to_file=to_file,

388 **gld_kwargs,

389 )

390 except Exception as e:

391 if not continue_on_error:

392 raise e

393 logger.error(

394 "Error processing %s csv for broid %s: %s",

395 meas_cl.__name__,

396 bro_id,

397 e,

398 )

399 else:

400 try:

401 data = meas_cl.from_bro_id(bro_id, to_file=to_file, **gld_kwargs)

402 except Exception as e:

403 if not continue_on_error:

404 raise e

405 logger.error(

406 "Error processing %s xml for broid %s: %s",

407 meas_cl.__name__,

408 bro_id,

409 e,

410 )

411 else:

412 # read the data from a file

413 if as_csv:

414 if zipfile is not None:

415 to_file = zipfile.open(to_file)

416 data = gld.read_gld_csv(

417 to_file,

418 bro_id,

419 observatietype=observatietype,

420 **gld_kwargs,

421 )

422 else:

423 data = meas_cl(to_file, zipfile=zipfile, **gld_kwargs)

424 return data

425

426

427def _get_gld_kwargs(

428 kind, tmin, tmax, qualifier, status, observation_type, sort, drop_duplicates

429):

430 gld_kwargs = {}

431 if kind == "gld":

432 if tmin is not None:

433 gld_kwargs["tmin"] = tmin

434 if tmax is not None:

435 gld_kwargs["tmax"] = tmax

436 if qualifier is not None:

437 gld_kwargs["qualifier"] = qualifier

438 if status is not None:

439 gld_kwargs["status"] = status

440 if observation_type is not None:

441 gld_kwargs["observation_type"] = observation_type

442 gld_kwargs["sort"] = sort

443 gld_kwargs["drop_duplicates"] = drop_duplicates

444 return gld_kwargs

445

446

447def get_tube_observations(

448 gmw_id, tube_number, kind="gld", sort=True, drop_duplicates=True, **kwargs

449):

450 """

451 Get the observations of a single groundwater monitoring tube.

452

453 Parameters

454 ----------

455 gmw_id : str

456 The bro_id of the groundwater monitoring well.

457 tube_number : int

458 The tube number.

459 kind : str, optional

460 The type of observations to retrieve. Can be one of {'gmn', 'gld', 'gar', 'frd'}.

461 Defaults to 'gld' (groundwater level dossier).

462 sort : bool, optional

463 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

464 drop_duplicates : bool, optional

465 If True, drop duplicate observations based on their timestamp. Only used if

466 `kind` is 'gld'. Defaults to True.

467 **kwargs : dict

468 Kwargs are passed onto get_observations.

469

470 Returns

471 -------

472 pd.DataFrame

473 A DataFrame containing the observations.

474

475 """

476 # sorting and dropping duplicates is done after combining the observations

477 # to avoid doing this multiple times

478 df = get_observations(

479 gmw_id,

480 tube_number=tube_number,

481 kind=kind,

482 sort=False,

483 drop_duplicates=False,

484 **kwargs,

485 )

486 if df.empty:

487 return _get_empty_observation_df(kind)

488 else:

489 data_column = _get_data_column(kind)

490 return _combine_observations(

491 df[data_column],

492 kind=kind,

493 bro_id=f"{gmw_id}_{tube_number}",

494 sort=sort,

495 drop_duplicates=drop_duplicates,

496 )

497

498

499def get_tube_gdf(gmws, index=None):

500 """

501 Create a GeoDataFrame of tube properties combined with well metadata.

502

503 This function processes a DataFrame of well properties, extracts the relevant

504 tube information, and combines them into a GeoDataFrame. The resulting GeoDataFrame

505 contains metadata for each monitoring well and its associated tubes, with optional

506 spatial information (coordinates) and relevant physical properties.

507

508 Parameters

509 ----------

510 gmws : list or dict of GroundwaterMonitoringWell, or pd.DataFrame Well and tube data

511 in one of the following formats: a list of `GroundwaterMonitoringWell` objects,

512 a dictionary of these objects, or a DataFrame with the bro-ids of the

513 GroundwaterMonitoringWells as the index and the column monitoringTube containing

514 tube properties.

515 index : str or list of str, optional

516 The column or columns to use for indexing the resulting GeoDataFrame. Defaults

517 to ['groundwaterMonitoringWell', 'tubeNumber'] if not provided.

518

519 Returns

520 -------

521 gdf : gpd.GeoDataFrame

522 A GeoDataFrame containing the combined well and tube properties, with the

523 specified index and optional geometry (spatial data) if 'x' and 'y' columns are

524 present.

525

526 Notes

527 -----

528 If 'x' and 'y' columns are present, the function creates a GeoDataFrame with point

529 geometries based on these coordinates, assuming the EPSG:28992 (Dutch National

530 Coordinate System) CRS.

531 """

532 if isinstance(gmws, list):

533 gmws = pd.DataFrame([x.to_dict() for x in gmws])

534 if "broId" in gmws.columns:

535 gmws = gmws.set_index("broId")

536 elif isinstance(gmws, dict):

537 gmws = pd.DataFrame([gmws[x].to_dict() for x in gmws])

538 if "broId" in gmws.columns:

539 gmws = gmws.set_index("broId")

540 tubes = []

541 for bro_id in gmws.index:

542 tube_df = gmws.loc[bro_id, "monitoringTube"]

543 if not isinstance(tube_df, pd.DataFrame):

544 continue

545 for tube_number in tube_df.index:

546 # combine properties of well and tube

547 tube = pd.concat(

548 (

549 gmws.loc[bro_id].drop("monitoringTube"),

550 tube_df.loc[tube_number],

551 )

552 )

553 tube["groundwaterMonitoringWell"] = bro_id

554 tube["tubeNumber"] = tube_number

555

556 tubes.append(tube)

557

558 if index is None:

559 index = ["groundwaterMonitoringWell", "tubeNumber"]

560 gdf = bro.objects_to_gdf(tubes, index=index)

561

562 gdf = gdf.sort_index()

563 return gdf

564

565

566def get_data_in_extent(

567 extent,

568 kind="gld",

569 tmin=None,

570 tmax=None,

571 combine=None,

572 index=None,

573 as_csv=False,

574 qualifier=None,

575 to_zip=None,

576 to_path=None,

577 redownload=False,

578 silent=False,

579 continue_on_error=False,

580 sort=True,

581 drop_duplicates=True,

582 progress_callback=None,

583):

584 """

585 Retrieve metadata and observations within a specified spatial extent.

586

587 This function fetches monitoring well characteristics, groundwater observations,

588 and tube properties within the given spatial extent. It can combine the data

589 for specific observation types and return either individual dataframes or a

590 combined dataframe.

591

592 Parameters

593 ----------

594 extent : str or sequence

595 The spatial extent ([xmin, xmax, ymin, ymax]) to filter the data.

596 kind : str, optional

597 The type of observations to retrieve. Valid values are {'gld', 'gar'} for

598 groundwater level dossier or groundwater analysis report. When kind is None, no

599 observations are downloaded. Defaults to 'gld'.

600 tmin : str or datetime, optional

601 The minimum time for filtering observations. Defaults to None.

602 tmax : str or datetime, optional

603 The maximum time for filtering observations. Defaults to None.

604 combine : bool, optional

605 If True, combines the metadata, tube properties, and observations into a single

606 dataframe. Defaults to False, which will change to True in a future version.

607 index : str, optional

608 The column to use for indexing in the resulting dataframe. Defaults to None.

609 as_csv : bool, optional

610 If True, the measurement data is requested as CSV files instead of XML files

611 (only supported for 'gld'). Defaults to False.

612 qualifier : str or list of str, optional

613 A string or list of strings used to filter the observations. Only valid if

614 `kind` is 'gld'. Defaults to None.

615 to_path : str, optional

616 If not None, save the downloaded files in the directory named to_path. The

617 default is None.

618 to_zip : str, optional

619 If not None, save the downloaded files in a zip-file named to_zip. The default

620 is None.

621 redownload : bool, optional

622 When downloaded files exist in to_path or to_zip, read from these files when

623 redownload is False. If redownload is True, download the data again from the

624 BRO-server. The default is False.

625 silent : bool, optional

626 If True, suppresses progress logging. Defaults to False.

627 continue_on_error : bool, optional

628 If True, continue after an error occurs during downloading or processing of

629 individual observation data. Defaults to False.

630 sort : bool, optional

631 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

632 drop_duplicates : bool, optional

633 If True, drop duplicate observations based on their timestamp. Only used if

634 `kind` is 'gld'. Defaults to True.

635 progress_callback : function, optional

636 A callback function that takes two arguments (current, total) to report

637 progress. If None, no progress reporting is done. Defaults to None.

638

639 Returns

640 -------

641 gdf : pd.DataFrame

642 A dataframe containing tube properties and metadata within the specified extent.

643

644 obs_df : pd.DataFrame, optional

645 A dataframe containing the observations for the specified wells. Returned only if

646 `combine` is False.

647

648 Raises

649 ------

650 Exception

651 If `as_csv=True` and `kind` is not 'gld', or if other parameters are invalid.

652 """

653 if combine is None:

654 logger.warning(

655 "The default of `combine=False` will change to True in a future version of "

656 "brodata. Pass combine=False to retain current behavior or combine=True to "

657 "adopt the future default and silence this warning."

658 )

659 combine = False

660 if isinstance(extent, str):

661 if to_zip is not None:

662 raise (Exception("When extent is a string, do not supply to_zip"))

663 to_zip = extent

664 extent = None

665 redownload = False

666

667 zipfile = None

668 _files = None

669 if to_zip is not None:

670 if not redownload and os.path.isfile(to_zip):

671 logger.info(f"Reading data from {to_zip}")

672 zipfile = ZipFile(to_zip)

673 else:

674 if to_path is None:

675 to_path = os.path.splitext(to_zip)[0]

676 remove_path_again = not os.path.isdir(to_path)

677 _files = []

678

679 if to_path is not None and not os.path.isdir(to_path):

680 os.makedirs(to_path)

681

682 # get gwm characteristics

683 logger.info(f"Getting gmw-characteristics in extent: {extent}")

684

685 to_file = util._get_to_file("gmw_characteristics.xml", zipfile, to_path, _files)

686 gmw = get_characteristics(

687 extent=extent, to_file=to_file, redownload=redownload, zipfile=zipfile

688 )

689

690 if kind is None:

691 obs_df = pd.DataFrame()

692 combine = False

693 else:

694 # get observations

695 logger.info(f"Downloading {kind}-observations")

696 obs_df = get_observations(

697 gmw,

698 kind=kind,

699 tmin=tmin,

700 tmax=tmax,

701 as_csv=as_csv,

702 qualifier=qualifier,

703 to_path=to_path,

704 redownload=redownload,

705 zipfile=zipfile,

706 _files=_files,

707 silent=silent,

708 continue_on_error=continue_on_error,

709 sort=sort,

710 drop_duplicates=drop_duplicates,

711 progress_callback=progress_callback,

712 )

713

714 # only keep wells with observations

715 if "groundwaterMonitoringWell" in obs_df.columns:

716 gmw = gmw[gmw.index.isin(obs_df["groundwaterMonitoringWell"])]

717

718 logger.info("Downloading tube-properties")

719

720 # get the properties of the monitoringTubes

721 gdf = get_tube_gdf_from_characteristics(

722 gmw,

723 index=index,

724 to_path=to_path,

725 redownload=redownload,

726 zipfile=zipfile,

727 _files=_files,

728 silent=silent,

729 )

730

731 if zipfile is not None:

732 zipfile.close()

733 if zipfile is None and to_zip is not None:

734 util._save_data_to_zip(to_zip, _files, remove_path_again, to_path)

735

736 if not obs_df.empty:

737 obs_df = (

738 obs_df.reset_index()

739 .set_index(["groundwaterMonitoringWell", "tubeNumber"])

740 .sort_index()

741 )

742

743 if combine and kind in ["gld", "gar"]:

744 gdf = add_observations_to_tubes(

745 gdf, obs_df, kind=kind, sort=sort, drop_duplicates=drop_duplicates

746 )

747 return gdf

748 else:

749 if kind is None:

750 return gdf

751 else:

752 return gdf, obs_df

753

754

755def add_observations_to_tubes(gdf, obs_df, kind="gld", sort=True, drop_duplicates=True):

756 """

757 Add observations to a GeoDataFrame of tube properties.

758

759 This function takes a GeoDataFrame containing tube properties and a DataFrame of

760 observations, and adds the observations to the GeoDataFrame based on matching

761 'groundwaterMonitoringWell' and 'tubeNumber' indices. The observations are combined

762 for each tube and added as a new column in the GeoDataFrame.

763

764 Parameters

765 ----------

766 gdf : gpd.GeoDataFrame

767 A GeoDataFrame containing tube properties with a MultiIndex of

768 ['groundwaterMonitoringWell', 'tubeNumber'].

769 obs_df : pd.DataFrame

770 A DataFrame containing observations with a MultiIndex of

771 ['groundwaterMonitoringWell', 'tubeNumber'] and a column containing the

772 observation data.

773 kind : str, optional

774 The type of observations to add. Can be one of {'gld', 'gar'}. Defaults to 'gld'.

775 sort : bool, optional

776 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

777 drop_duplicates : bool, optional

778 If True, drop duplicate observations based on their timestamp. Only used if

779 `kind` is 'gld'. Defaults to True.

780

781 Returns

782 -------

783 gpd.GeoDataFrame

784 The input GeoDataFrame with an additional column containing the combined observations.

785

786 Raises

787 ------

788 ValueError

789 If `kind` is not one of the supported types ('gld', 'gar').

790 """

791 logger.info("Adding observations to tube-properties")

792 if kind == "gld":

793 idcol = "groundwaterLevelDossier"

794 elif kind == "gar":

795 idcol = "groundwaterAnalysisReport"

796 datcol = _get_data_column(kind)

797

798 # check if all indices of obs_df are in gdf

799 if not obs_df.index.isin(gdf.index).all():

800 missing = obs_df.index[~obs_df.index.isin(gdf.index)]

801 logger.warning(

802 "Not all indices of obs_df are in gdf: %s. Only adding observations for tubes "

803 "with matching indices.",

804 missing,

805 )

806

807 data = {}

808 ids = {}

809 for index in gdf.index:

810 if index not in obs_df.index:

811 data[index] = _get_empty_observation_df(kind)

812 continue

813

814 data[index] = _combine_observations(

815 obs_df.loc[[index], datcol],

816 kind=kind,

817 bro_id=f"{index[0]}_{index[1]}",

818 sort=sort,

819 drop_duplicates=drop_duplicates,

820 )

821 ids[index] = list(obs_df.loc[[index], "broId"])

822 gdf[datcol] = data

823 gdf[idcol] = ids

824 return gdf

825

826

827def _get_data_column(kind):

828 if kind == "gld":

829 return "observation"

830 elif kind == "gar":

831 return "laboratoryAnalysis"

832 else:

833 raise (NotImplementedError(f"Measurement-kind {kind} not supported yet"))

834

835

836def _get_empty_observation_df(kind):

837 if kind == "gld":

838 return gld._get_empty_observation_df()

839 elif kind == "gar":

840 return gar._get_empty_observation_df()

841 else:

842 raise (NotImplementedError(f"Measurement-kind {kind} not supported yet"))

843

844

845def _combine_observations(

846 observations, kind, bro_id=None, sort=True, drop_duplicates=True

847):

848 obslist = []

849 for observation in observations:

850 if not isinstance(observation, pd.DataFrame) or observation.empty:

851 continue

852 obslist.append(observation)

853 if len(obslist) == 0:

854 return _get_empty_observation_df(kind)

855 else:

856 df = pd.concat(obslist).sort_index()

857 if kind == "gld":

858 if sort:

859 df = gld.sort_observations(df)

860 if drop_duplicates:

861 df = gld.drop_duplicate_observations(df, bro_id=bro_id)

862 return df

863

864

865def get_tube_gdf_from_characteristics(characteristics_gdf, **kwargs):

866 """

867 Generate a GeoDataFrame of tube properties based on well characteristics.

868

869 This function downloads the GroundwaterMonitoringWell-objects to retreive data about

870 the groundwater monitoring tubes, and combined this information in a new

871 GeoDataFrame.

872

873 Parameters

874 ----------

875 characteristics_gdf : gpd.GeoDataFrame

876 GeoDataFrame of well characteristics with bro-ids of the

877 GroundwaterMonitoringWells as the index, retreived with

878 `brodata.gmw.get_characteristics`.

879 index : str or list of str, optional

880 Column(s) to use as the index for the resulting GeoDataFrame. Defaults

881 to ['groundwaterMonitoringWell', 'tubeNumber'] if not provided.

882

883 Returns

884 -------

885 gpd.GeoDataFrame

886 GeoDataFrame of combined well and tube properties

887 """

888 bro_ids = characteristics_gdf.index.unique()

889 return get_tube_gdf_from_bro_ids(bro_ids, **kwargs)

890

891

892def get_tube_gdf_from_bro_ids(

893 bro_ids,

894 index=None,

895 **kwargs,

896):

897 """

898 Generate a GeoDataFrame of tube properties based on an iterable of gmw bro-ids.

899

900 This function downloads the GroundwaterMonitoringWell-objects to retreive data about

901 the groundwater monitoring tubes, and combined this information in a new

902 GeoDataFrame.

903

904 Parameters

905 ----------

906 bro_ids : gpd.GeoDataFrame

907 GeoDataFrame of well characteristics with bro-ids of the

908 GroundwaterMonitoringWells as the index, retreived with

909 `brodata.gmw.get_characteristics`.

910 index : str or list of str, optional

911 Column(s) to use as the index for the resulting GeoDataFrame. Defaults

912 to ['groundwaterMonitoringWell', 'tubeNumber'] if not provided.

913

914 Returns

915 -------

916 gpd.GeoDataFrame

917 GeoDataFrame of combined well and tube properties

918 """

919 desc = "Downloading Groundwater Monitoring Wells"

920 gmws = bro._get_data_for_bro_ids(

921 GroundwaterMonitoringWell, bro_ids, desc=desc, **kwargs

922 )

923 gdf = get_tube_gdf(gmws, index=index)

924 return gdf

925

926

927cl = GroundwaterMonitoringWell

928

929get_bro_ids_of_bronhouder = partial(bro._get_bro_ids_of_bronhouder, cl)

930get_bro_ids_of_bronhouder.__doc__ = bro._get_bro_ids_of_bronhouder.__doc__

931

932get_data_for_bro_ids = partial(bro._get_data_for_bro_ids, cl)

933get_data_for_bro_ids.__doc__ = bro._get_data_for_bro_ids.__doc__

934

935get_characteristics = partial(bro._get_characteristics, cl)

936get_characteristics.__doc__ = bro._get_characteristics.__doc__