Coverage for brodata/gmw.py: 77%

1import json

2import logging

3import os

4from functools import partial

5from zipfile import ZipFile

7import numpy as np

8import pandas as pd

10from . import bro, frd, gar, gld, gmn, util

12logger = logging.getLogger(__name__)

15def get_well_code(bro_id):

16 """

17 Retrieve the well code based on a given BRO-ID and return it as plain text.

19 This function sends a GET request to fetch the well code associated with the

20 specified BRO-ID. If the request fails, it logs an error message and returns `None`.

22 Parameters

23 ----------

24 bro_id : str

25 The BRO-ID for which to retrieve the associated well code.

27 Returns

28 -------

29 well_code : str or None

30 The well code as plain text if the request is successful. Returns `None` if

31 the request fails.

32 """

34 url = f"{GroundwaterMonitoringWell._rest_url}/well-code/{bro_id}"

35 req = bro.util.get_with_rate_limit(url)

36 if req.status_code > 200:

37 logger.error(req.reason)

38 return

39 well_code = req.text

40 return well_code

43class GroundwaterMonitoringWell(bro.FileOrUrl):

44 """

45 Class to represent a Groundwater Monitoring Well (GMW) from the BRO.

47 This class parses XML data related to a groundwater monitoring well (GMW).

48 It extracts details such as location, monitoring tube data, and well history

49 and stores these in attributes.

51 Notes

52 -----

53 This class extends `bro.XmlFileOrUrl` and is designed to work with GMW XML data,

54 either from a file or URL.

55 """

57 _rest_url = "https://publiek.broservices.nl/gm/gmw/v1"

58 _xmlns = "http://www.broservices.nl/xsd/dsgmw/1.1"

59 _char = "GMW_C"

61 def _read_contents(self, tree):

62 ns = {

63 "brocom": "http://www.broservices.nl/xsd/brocommon/3.0",

64 "xmlns": self._xmlns,

65 }

67 object_names = ["GMW_PO", "GMW_PPO", "BRO_DO"]

68 gmw = self._get_main_object(tree, object_names, ns)

70 for key in gmw.attrib:

71 setattr(self, key.split("}", 1)[1], gmw.attrib[key])

72 for child in gmw:

73 key = self._get_tag(child)

74 if len(child) == 0:

75 setattr(self, key, child.text)

76 elif key == "standardizedLocation":

77 self._read_standardized_location(child)

78 elif key == "deliveredLocation":

79 self._read_delivered_location(child)

80 elif key == "wellHistory":

81 for grandchild in child:

82 key = self._get_tag(grandchild)

83 if key in ["wellConstructionDate", "wellRemovalDate"]:

84 setattr(self, key, self._read_date(grandchild))

85 elif key == "intermediateEvent":

86 if not hasattr(self, key):

87 self.intermediateEvent = []

88 event = self._read_intermediate_event(grandchild)

89 self.intermediateEvent.append(event)

90 else:

91 self._warn_unknown_tag(key)

93 elif key in ["deliveredVerticalPosition", "registrationHistory"]:

94 to_float = ["offset", "groundLevelPosition"]

95 self._read_children_of_children(child, to_float=to_float)

96 elif key in ["monitoringTube"]:

97 if not hasattr(self, key):

98 self.monitoringTube = []

99 tube = {}

100 to_float = [

101 "tubeTopDiameter",

102 "tubeTopPosition",

103 "screenLength",

104 "screenTopPosition",

105 "screenBottomPosition",

106 "plainTubePartLength",

107 ]

108 self._read_children_of_children(child, tube, to_float=to_float)

109 self.monitoringTube.append(tube)

110 else:

111 self._warn_unknown_tag(key)

112 if hasattr(self, "monitoringTube"):

113 self.monitoringTube = pd.DataFrame(self.monitoringTube)

114 tubeNumber = self.monitoringTube["tubeNumber"].astype(int)

115 self.monitoringTube["tubeNumber"] = tubeNumber

116 self.monitoringTube = self.monitoringTube.set_index("tubeNumber")

117 if hasattr(self, "intermediateEvent"):

118 self.intermediateEvent = pd.DataFrame(self.intermediateEvent)

119

120 def _read_intermediate_event(self, node):

121 d = {}

122 for child in node:

123 key = self._get_tag(child)

124 if key == "eventName":

125 d[key] = child.text

126 elif key == "eventDate":

127 d[key] = self._read_date(child)

128 else:

129 self._warn_unknown_tag(key)

130 return d

131

132

133def get_observations(

134 bro_ids,

135 kind="gld",

136 drop_references=True,

137 silent=False,

138 tmin=None,

139 tmax=None,

140 as_csv=False,

141 tube_number=None,

142 status=None,

143 observation_type=None,

144 qualifier=None,

145 to_path=None,

146 to_zip=None,

147 redownload=False,

148 zipfile=None,

149 continue_on_error=False,

150 sort=True,

151 drop_duplicates=True,

152 progress_callback=None,

153 _files=None,

154):

155 """

156 Retrieve groundwater observations for the specified monitoring wells (bro_ids).

157

158 This function fetches groundwater data for monitoring wells based on the provided

159 parameters. It supports different types of observations, allows filtering by tube

160 number, and can request the data in CSV format for groundwater level observations.

161

162 Parameters

163 ----------

164 bro_ids : str or list or pd.DataFrame

165 The BRO IDs of the monitoring wells for which to retrieve the data. If a

166 DataFrame is provided, its index is used as the list of BRO IDs.

167 kind : str, optional

168 The type of observations to retrieve. Can be one of {'gmn', 'gld', 'gar', 'frd'}.

169 Defaults to 'gld' (groundwater level dossier).

170 drop_references : bool or list of str, optional

171 Specifies whether to drop reference fields in the returned data. Defaults to True,

172 in which case 'gmnReferences', 'gldReferences', 'garReferences' and

173 'frdReferences' are removed. Only used when as_csv=True.

174 silent : bool, optional

175 If True, suppresses progress logging. Defaults to False.

176 tmin : str or datetime, optional

177 The minimum time filter for the observations. Defaults to None.

178 tmax : str or datetime, optional

179 The maximum time filter for the observations. Defaults to None.

180 as_csv : bool, optional

181 If True, requests the observations as CSV files instead of XML-files. Only valid

182 if `kind` is 'gld'. Defaults to False.

183 tube_number : int, optional

184 Filters observations to a specific tube number. Defaults to None.

185 status : str, optional

186 A status string for additional filtering. Possible values are

187 "volledigBeoordeeld", "voorlopig" and "onbekend" Only valid if `kind` is 'gld'.

188 Defaults to None.

189 observation_type : str, optional

190 An observation type string for additional filtering. Possible values are

191 "reguliereMeting" and "controleMeting". Only valid if `kind` is 'gld'. Defaults

192 to None.

193 qualifier : str or list of str, optional

194 A qualifier string for additional filtering. Only valid if `kind` is 'gld'.

195 Defaults to None.

196 to_path : str, optional

197 If not None, save the downloaded files in the directory named to_path. The

198 default is None.

199 to_zip : str, optional

200 If not None, save the downloaded files in a zip-file named to_zip. The default

201 is None.

202 redownload : bool, optional

203 When downloaded files exist in to_path or to_zip, read from these files when

204 redownload is False. If redownload is True, download the data again from the

205 BRO-servers. The default is False.

206 zipfile : zipfile.ZipFile, optional

207 A zipfile-object. When not None, zipfile is used to read previously downloaded

208 data from. The default is None.

209 continue_on_error : bool, optional

210 If True, continue after an error occurs during downloading or processing of

211 individual observation data. Defaults to False.

212 sort : bool, optional

213 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

214 drop_duplicates : bool, optional

215 If True, drop duplicate observations based on their timestamp. Only used if

216 `kind` is 'gld'. Defaults to True.

217 progress_callback : function, optional

218 A callback function that takes two arguments (current, total) to report

219 progress. If None, no progress reporting is done. Defaults to None.

220

221

222 Returns

223 -------

224 pd.DataFrame

225 A DataFrame containing the observations for the specified monitoring wells,

226 where each row corresponds to an individual observation. The index is the BRO-id

227 of the observation-objects.

228

229 Raises

230 ------

231 ValueError

232 If the specified `kind` is not supported, or if `as_csv=True` and `kind` is not

233 'gld', or if `qualifier` is provided for a kind other than 'gld'.

234 """

235 tubes = []

236

237 if isinstance(bro_ids, str):

238 bro_ids = [bro_ids]

239 silent = True

240

241 if isinstance(bro_ids, pd.DataFrame):

242 bro_ids = bro_ids.index

243

244 if as_csv and isinstance(drop_references, bool):

245 if drop_references:

246 drop_references = [

247 "gmnReferences",

248 "gldReferences",

249 "garReferences",

250 "frdReferences",

251 ]

252 else:

253 drop_references = []

254

255 if to_zip is not None:

256 if not redownload and os.path.isfile(to_zip):

257 raise (NotImplementedError("Redownload=False is not suppported yet"))

258 if to_path is None:

259 to_path = os.path.splitext(to_zip)[0]

260 remove_path_again = not os.path.isdir(to_path)

261 if _files is None:

262 _files = []

263

264 desc = f"Downloading {kind}-observations"

265 if as_csv and kind != "gld":

266 raise (ValueError("as_csv=True is only supported for kind=='gld'"))

267 if qualifier is not None and kind != "gld":

268 raise (ValueError("A qualifier is only supported for kind=='gld'"))

269 if to_path is not None and not os.path.isdir(to_path):

270 os.makedirs(to_path)

271

272 if kind == "gld":

273 meas_cl = gld.GroundwaterLevelDossier

274 elif kind == "gar":

275 meas_cl = gar.GroundwaterAnalysisReport

276 elif kind == "frd":

277 meas_cl = frd.FormationResistanceDossier

278 elif kind == "gmn":

279 meas_cl = gmn.GroundwaterMonitoringNetwork

280 else:

281 raise (ValueError(f"kind='{kind}' not supported"))

282

283 gld_kwargs = _get_gld_kwargs(

284 kind, tmin, tmax, qualifier, status, observation_type, sort, drop_duplicates

285 )

286

287 for igmw, bro_id in enumerate(

288 util.tqdm(np.unique(bro_ids), disable=silent, desc=desc)

289 ):

290 to_rel_file = util._get_to_file(

291 f"gmw_relations_{bro_id}.json", zipfile, to_path, _files

292 )

293 if zipfile is None and (

294 redownload or to_rel_file is None or not os.path.isfile(to_rel_file)

295 ):

296 url = f"https://publiek.broservices.nl/gm/v1/gmw-relations/{bro_id}"

297 req = bro.util.get_with_rate_limit(url)

298 if req.status_code > 200:

299 logger.error(req.json()["errors"][0]["message"])

300 return

301 if to_rel_file is not None:

302 with open(to_rel_file, "w") as f:

303 f.write(req.text)

304 data = req.json()

305 else:

306 if zipfile is not None:

307 with zipfile.open(to_rel_file) as f:

308 data = json.load(f)

309 else:

310 with open(to_rel_file) as f:

311 data = json.load(f)

312 for tube_ref in data["monitoringTubeReferences"]:

313 tube_ref["groundwaterMonitoringWell"] = data["gmwBroId"]

314 if tube_number is not None and tube_ref["tubeNumber"] != tube_number:

315 continue

316 ref_key = f"{kind}References"

317 for ref in tube_ref[ref_key]:

318 obsdata = _download_observations_for_bro_id(

319 ref["broId"],

320 meas_cl,

321 as_csv,

322 zipfile,

323 to_path,

324 _files,

325 gld_kwargs,

326 redownload=redownload,

327 continue_on_error=continue_on_error,

328 )

329 if as_csv:

330 gld_dict = tube_ref.copy()

331 gld_dict["observation"] = obsdata

332 # drop references, as these are dictionaries of no interest to the user

333 for key in drop_references:

334 if key in gld_dict:

335 gld_dict.pop(key)

336 # add fields of the ref to gld_dict, like the broId of the GLD

337 for key in ref:

338 gld_dict[key] = ref[key]

339 tubes.append(gld_dict)

340 else:

341 tubes.append(obsdata.to_dict())

342

343 if progress_callback is not None:

344 progress_callback(igmw + 1, len(bro_ids))

345 if to_zip is not None:

346 util._save_data_to_zip(to_zip, _files, remove_path_again, to_path)

347 return pd.DataFrame(tubes).set_index("broId")

348

349

350def _download_observations_for_bro_id(

351 bro_id,

352 meas_cl,

353 as_csv,

354 zipfile,

355 to_path,

356 _files,

357 gld_kwargs,

358 redownload=False,

359 continue_on_error=False,

360):

361 if as_csv:

362 fname = f"{bro_id}.csv"

363 observatietype = None

364 if "status" in gld_kwargs and gld_kwargs["status"] == "voorlopig":

365 observatietype = "regulier_voorlopig"

366 elif "status" in gld_kwargs and gld_kwargs["status"] == "volledigBeoordeeld":

367 observatietype = "regulier_beoordeeld"

368 elif "status" in gld_kwargs and gld_kwargs["status"] == "onbekend":

369 observatietype = "onbekend"

370 elif (

371 "observation_type" in gld_kwargs

372 and gld_kwargs["observation_type"] == "controleMeting"

373 ):

374 observatietype = "controle"

375 else:

376 fname = f"{bro_id}.xml"

377 to_file = util._get_to_file(fname, zipfile, to_path, _files)

378 if zipfile is None and (

379 redownload or to_file is None or not os.path.isfile(to_file)

380 ): # download the data

381 if as_csv:

382 try:

383 data = gld.get_objects_as_csv(

384 bro_id,

385 observatietype=observatietype,

386 to_file=to_file,

387 **gld_kwargs,

388 )

389 except Exception as e:

390 if not continue_on_error:

391 raise

392 logger.error(

393 "Error processing %s csv for broid %s: %s",

394 meas_cl.__name__,

395 bro_id,

396 e,

397 )

398 return None

399 else:

400 try:

401 data = meas_cl.from_bro_id(bro_id, to_file=to_file, **gld_kwargs)

402 except Exception as e:

403 if not continue_on_error:

404 raise

405 logger.error(

406 "Error processing %s xml for broid %s: %s",

407 meas_cl.__name__,

408 bro_id,

409 e,

410 )

411 return None

412 else:

413 # read the data from a file

414 if as_csv:

415 if zipfile is not None:

416 to_file = zipfile.open(to_file)

417 data = gld.read_gld_csv(

418 to_file,

419 bro_id,

420 observatietype=observatietype,

421 **gld_kwargs,

422 )

423 else:

424 data = meas_cl(to_file, zipfile=zipfile, **gld_kwargs)

425 return data

426

427

428def _get_gld_kwargs(

429 kind, tmin, tmax, qualifier, status, observation_type, sort, drop_duplicates

430):

431 gld_kwargs = {}

432 if kind == "gld":

433 if tmin is not None:

434 gld_kwargs["tmin"] = tmin

435 if tmax is not None:

436 gld_kwargs["tmax"] = tmax

437 if qualifier is not None:

438 gld_kwargs["qualifier"] = qualifier

439 if status is not None:

440 gld_kwargs["status"] = status

441 if observation_type is not None:

442 gld_kwargs["observation_type"] = observation_type

443 gld_kwargs["sort"] = sort

444 gld_kwargs["drop_duplicates"] = drop_duplicates

445 return gld_kwargs

446

447

448def get_tube_observations(

449 gmw_id, tube_number, kind="gld", sort=True, drop_duplicates=True, **kwargs

450):

451 """

452 Get the observations of a single groundwater monitoring tube.

453

454 Parameters

455 ----------

456 gmw_id : str

457 The bro_id of the groundwater monitoring well.

458 tube_number : int

459 The tube number.

460 kind : str, optional

461 The type of observations to retrieve. Can be one of {'gmn', 'gld', 'gar', 'frd'}.

462 Defaults to 'gld' (groundwater level dossier).

463 sort : bool, optional

464 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

465 drop_duplicates : bool, optional

466 If True, drop duplicate observations based on their timestamp. Only used if

467 `kind` is 'gld'. Defaults to True.

468 **kwargs : dict

469 Kwargs are passed onto get_observations.

470

471 Returns

472 -------

473 pd.DataFrame

474 A DataFrame containing the observations.

475

476 """

477 # sorting and dropping duplicates is done after combining the observations

478 # to avoid doing this multiple times

479 df = get_observations(

480 gmw_id,

481 tube_number=tube_number,

482 kind=kind,

483 sort=False,

484 drop_duplicates=False,

485 **kwargs,

486 )

487 if df.empty:

488 return _get_empty_observation_df(kind)

489 else:

490 data_column = _get_data_column(kind)

491 return _combine_observations(

492 df[data_column],

493 kind=kind,

494 bro_id=f"{gmw_id}_{tube_number}",

495 sort=sort,

496 drop_duplicates=drop_duplicates,

497 )

498

499

500def get_tube_gdf(gmws, index=None):

501 """

502 Create a GeoDataFrame of tube properties combined with well metadata.

503

504 This function processes a DataFrame of well properties, extracts the relevant

505 tube information, and combines them into a GeoDataFrame. The resulting GeoDataFrame

506 contains metadata for each monitoring well and its associated tubes, with optional

507 spatial information (coordinates) and relevant physical properties.

508

509 Parameters

510 ----------

511 gmws : list or dict of GroundwaterMonitoringWell, or pd.DataFrame Well and tube data

512 in one of the following formats: a list of `GroundwaterMonitoringWell` objects,

513 a dictionary of these objects, or a DataFrame with the bro-ids of the

514 GroundwaterMonitoringWells as the index and the column monitoringTube containing

515 tube properties.

516 index : str or list of str, optional

517 The column or columns to use for indexing the resulting GeoDataFrame. Defaults

518 to ['groundwaterMonitoringWell', 'tubeNumber'] if not provided.

519

520 Returns

521 -------

522 gdf : gpd.GeoDataFrame

523 A GeoDataFrame containing the combined well and tube properties, with the

524 specified index and optional geometry (spatial data) if 'x' and 'y' columns are

525 present.

526

527 Notes

528 -----

529 If 'x' and 'y' columns are present, the function creates a GeoDataFrame with point

530 geometries based on these coordinates, assuming the EPSG:28992 (Dutch National

531 Coordinate System) CRS.

532 """

533 if isinstance(gmws, list):

534 gmws = pd.DataFrame([x.to_dict() for x in gmws])

535 if "broId" in gmws.columns:

536 gmws = gmws.set_index("broId")

537 elif isinstance(gmws, dict):

538 gmws = pd.DataFrame([gmws[x].to_dict() for x in gmws])

539 if "broId" in gmws.columns:

540 gmws = gmws.set_index("broId")

541 tubes = []

542 for bro_id in gmws.index:

543 tube_df = gmws.loc[bro_id, "monitoringTube"]

544 if not isinstance(tube_df, pd.DataFrame):

545 continue

546 for tube_number in tube_df.index:

547 # combine properties of well and tube

548 tube = pd.concat(

549 (

550 gmws.loc[bro_id].drop("monitoringTube"),

551 tube_df.loc[tube_number],

552 )

553 )

554 tube["groundwaterMonitoringWell"] = bro_id

555 tube["tubeNumber"] = tube_number

556

557 tubes.append(tube)

558

559 if index is None:

560 index = ["groundwaterMonitoringWell", "tubeNumber"]

561 gdf = bro.objects_to_gdf(tubes, index=index)

562

563 gdf = gdf.sort_index()

564 return gdf

565

566

567def get_data_in_extent(

568 extent,

569 kind="gld",

570 tmin=None,

571 tmax=None,

572 combine=None,

573 index=None,

574 as_csv=False,

575 qualifier=None,

576 to_zip=None,

577 to_path=None,

578 redownload=False,

579 silent=False,

580 continue_on_error=False,

581 sort=True,

582 drop_duplicates=True,

583 progress_callback=None,

584):

585 """

586 Retrieve metadata and observations within a specified spatial extent.

587

588 This function fetches monitoring well characteristics, groundwater observations,

589 and tube properties within the given spatial extent. It can combine the data

590 for specific observation types and return either individual dataframes or a

591 combined dataframe.

592

593 Parameters

594 ----------

595 extent : str or sequence

596 The spatial extent ([xmin, xmax, ymin, ymax]) to filter the data.

597 kind : str, optional

598 The type of observations to retrieve. Valid values are {'gld', 'gar'} for

599 groundwater level dossier or groundwater analysis report. When kind is None, no

600 observations are downloaded. Defaults to 'gld'.

601 tmin : str or datetime, optional

602 The minimum time for filtering observations. Defaults to None.

603 tmax : str or datetime, optional

604 The maximum time for filtering observations. Defaults to None.

605 combine : bool, optional

606 If True, combines the metadata, tube properties, and observations into a single

607 dataframe. Defaults to False, which will change to True in a future version.

608 index : str, optional

609 The column to use for indexing in the resulting dataframe. Defaults to None.

610 as_csv : bool, optional

611 If True, the measurement data is requested as CSV files instead of XML files

612 (only supported for 'gld'). Defaults to False.

613 qualifier : str or list of str, optional

614 A string or list of strings used to filter the observations. Only valid if

615 `kind` is 'gld'. Defaults to None.

616 to_path : str, optional

617 If not None, save the downloaded files in the directory named to_path. The

618 default is None.

619 to_zip : str, optional

620 If not None, save the downloaded files in a zip-file named to_zip. The default

621 is None.

622 redownload : bool, optional

623 When downloaded files exist in to_path or to_zip, read from these files when

624 redownload is False. If redownload is True, download the data again from the

625 BRO-server. The default is False.

626 silent : bool, optional

627 If True, suppresses progress logging. Defaults to False.

628 continue_on_error : bool, optional

629 If True, continue after an error occurs during downloading or processing of

630 individual observation data. Defaults to False.

631 sort : bool, optional

632 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

633 drop_duplicates : bool, optional

634 If True, drop duplicate observations based on their timestamp. Only used if

635 `kind` is 'gld'. Defaults to True.

636 progress_callback : function, optional

637 A callback function that takes two arguments (current, total) to report

638 progress. If None, no progress reporting is done. Defaults to None.

639

640 Returns

641 -------

642 gdf : pd.DataFrame

643 A dataframe containing tube properties and metadata within the specified extent.

644

645 obs_df : pd.DataFrame, optional

646 A dataframe containing the observations for the specified wells. Returned only if

647 `combine` is False.

648

649 Raises

650 ------

651 Exception

652 If `as_csv=True` and `kind` is not 'gld', or if other parameters are invalid.

653 """

654 if combine is None:

655 logger.warning(

656 "The default of `combine=False` will change to True in a future version of "

657 "brodata. Pass combine=False to retain current behavior or combine=True to "

658 "adopt the future default and silence this warning."

659 )

660 combine = False

661 if isinstance(extent, str):

662 if to_zip is not None:

663 raise (ValueError("When extent is a string, do not supply to_zip"))

664 to_zip = extent

665 extent = None

666 redownload = False

667

668 zipfile = None

669 _files = None

670 if to_zip is not None:

671 if not redownload and os.path.isfile(to_zip):

672 logger.info(f"Reading data from {to_zip}")

673 zipfile = ZipFile(to_zip)

674 else:

675 if to_path is None:

676 to_path = os.path.splitext(to_zip)[0]

677 remove_path_again = not os.path.isdir(to_path)

678 _files = []

679

680 if to_path is not None and not os.path.isdir(to_path):

681 os.makedirs(to_path)

682

683 # get gwm characteristics

684 logger.info(f"Getting gmw-characteristics in extent: {extent}")

685

686 to_file = util._get_to_file("gmw_characteristics.xml", zipfile, to_path, _files)

687 gmw = get_characteristics(

688 extent=extent, to_file=to_file, redownload=redownload, zipfile=zipfile

689 )

690

691 if kind is None:

692 obs_df = pd.DataFrame()

693 combine = False

694 else:

695 # get observations

696 logger.info(f"Downloading {kind}-observations")

697 obs_df = get_observations(

698 gmw,

699 kind=kind,

700 tmin=tmin,

701 tmax=tmax,

702 as_csv=as_csv,

703 qualifier=qualifier,

704 to_path=to_path,

705 redownload=redownload,

706 zipfile=zipfile,

707 _files=_files,

708 silent=silent,

709 continue_on_error=continue_on_error,

710 sort=sort,

711 drop_duplicates=drop_duplicates,

712 progress_callback=progress_callback,

713 )

714

715 # only keep wells with observations

716 if "groundwaterMonitoringWell" in obs_df.columns:

717 gmw = gmw[gmw.index.isin(obs_df["groundwaterMonitoringWell"])]

718

719 logger.info("Downloading tube-properties")

720

721 # get the properties of the monitoringTubes

722 gdf = get_tube_gdf_from_characteristics(

723 gmw,

724 index=index,

725 to_path=to_path,

726 redownload=redownload,

727 zipfile=zipfile,

728 _files=_files,

729 silent=silent,

730 )

731

732 if zipfile is not None:

733 zipfile.close()

734 if zipfile is None and to_zip is not None:

735 util._save_data_to_zip(to_zip, _files, remove_path_again, to_path)

736

737 if not obs_df.empty:

738 obs_df = (

739 obs_df.reset_index()

740 .set_index(["groundwaterMonitoringWell", "tubeNumber"])

741 .sort_index()

742 )

743

744 if combine and kind in ["gld", "gar"]:

745 gdf = add_observations_to_tubes(

746 gdf, obs_df, kind=kind, sort=sort, drop_duplicates=drop_duplicates

747 )

748 return gdf

749 else:

750 if kind is None:

751 return gdf

752 else:

753 return gdf, obs_df

754

755

756def add_observations_to_tubes(gdf, obs_df, kind="gld", sort=True, drop_duplicates=True):

757 """

758 Add observations to a GeoDataFrame of tube properties.

759

760 This function takes a GeoDataFrame containing tube properties and a DataFrame of

761 observations, and adds the observations to the GeoDataFrame based on matching

762 'groundwaterMonitoringWell' and 'tubeNumber' indices. The observations are combined

763 for each tube and added as a new column in the GeoDataFrame.

764

765 Parameters

766 ----------

767 gdf : gpd.GeoDataFrame

768 A GeoDataFrame containing tube properties with a MultiIndex of

769 ['groundwaterMonitoringWell', 'tubeNumber'].

770 obs_df : pd.DataFrame

771 A DataFrame containing observations with a MultiIndex of

772 ['groundwaterMonitoringWell', 'tubeNumber'] and a column containing the

773 observation data.

774 kind : str, optional

775 The type of observations to add. Can be one of {'gld', 'gar'}. Defaults to 'gld'.

776 sort : bool, optional

777 If True, sort the observations. Only used if `kind` is 'gld'. Defaults to True.

778 drop_duplicates : bool, optional

779 If True, drop duplicate observations based on their timestamp. Only used if

780 `kind` is 'gld'. Defaults to True.

781

782 Returns

783 -------

784 gpd.GeoDataFrame

785 The input GeoDataFrame with an additional column containing the combined observations.

786

787 Raises

788 ------

789 ValueError

790 If `kind` is not one of the supported types ('gld', 'gar').

791 """

792 logger.info("Adding observations to tube-properties")

793 if kind == "gld":

794 idcol = "groundwaterLevelDossier"

795 elif kind == "gar":

796 idcol = "groundwaterAnalysisReport"

797 datcol = _get_data_column(kind)

798

799 # check if all indices of obs_df are in gdf

800 if not obs_df.index.isin(gdf.index).all():

801 missing = obs_df.index[~obs_df.index.isin(gdf.index)]

802 logger.warning(

803 "Not all indices of obs_df are in gdf: %s. Only adding observations for tubes "

804 "with matching indices.",

805 missing,

806 )

807

808 data = {}

809 ids = {}

810 for index in gdf.index:

811 if index not in obs_df.index:

812 data[index] = _get_empty_observation_df(kind)

813 continue

814

815 data[index] = _combine_observations(

816 obs_df.loc[[index], datcol],

817 kind=kind,

818 bro_id=f"{index[0]}_{index[1]}",

819 sort=sort,

820 drop_duplicates=drop_duplicates,

821 )

822 ids[index] = list(obs_df.loc[[index], "broId"])

823 gdf[datcol] = data

824 gdf[idcol] = ids

825 return gdf

826

827

828def _get_data_column(kind):

829 if kind == "gld":

830 return "observation"

831 elif kind == "gar":

832 return "laboratoryAnalysis"

833 else:

834 raise (NotImplementedError(f"Measurement-kind {kind} not supported yet"))

835

836

837def _get_empty_observation_df(kind):

838 if kind == "gld":

839 return gld._get_empty_observation_df()

840 elif kind == "gar":

841 return gar._get_empty_observation_df()

842 else:

843 raise (NotImplementedError(f"Measurement-kind {kind} not supported yet"))

844

845

846def _combine_observations(

847 observations, kind, bro_id=None, sort=True, drop_duplicates=True

848):

849 obslist = []

850 for observation in observations:

851 if not isinstance(observation, pd.DataFrame) or observation.empty:

852 continue

853 obslist.append(observation)

854 if len(obslist) == 0:

855 return _get_empty_observation_df(kind)

856 else:

857 df = pd.concat(obslist).sort_index()

858 if kind == "gld":

859 if sort:

860 df = gld.sort_observations(df)

861 if drop_duplicates:

862 df = gld.drop_duplicate_observations(df, bro_id=bro_id)

863 return df

864

865

866def get_tube_gdf_from_characteristics(characteristics_gdf, **kwargs):

867 """

868 Generate a GeoDataFrame of tube properties based on well characteristics.

869

870 This function downloads the GroundwaterMonitoringWell-objects to retreive data about

871 the groundwater monitoring tubes, and combined this information in a new

872 GeoDataFrame.

873

874 Parameters

875 ----------

876 characteristics_gdf : gpd.GeoDataFrame

877 GeoDataFrame of well characteristics with bro-ids of the

878 GroundwaterMonitoringWells as the index, retreived with

879 `brodata.gmw.get_characteristics`.

880 index : str or list of str, optional

881 Column(s) to use as the index for the resulting GeoDataFrame. Defaults

882 to ['groundwaterMonitoringWell', 'tubeNumber'] if not provided.

883

884 Returns

885 -------

886 gpd.GeoDataFrame

887 GeoDataFrame of combined well and tube properties

888 """

889 bro_ids = characteristics_gdf.index.unique()

890 return get_tube_gdf_from_bro_ids(bro_ids, **kwargs)

891

892

893def get_tube_gdf_from_bro_ids(

894 bro_ids,

895 index=None,

896 **kwargs,

897):

898 """

899 Generate a GeoDataFrame of tube properties based on an iterable of gmw bro-ids.

900

901 This function downloads the GroundwaterMonitoringWell-objects to retreive data about

902 the groundwater monitoring tubes, and combined this information in a new

903 GeoDataFrame.

904

905 Parameters

906 ----------

907 bro_ids : gpd.GeoDataFrame

908 GeoDataFrame of well characteristics with bro-ids of the

909 GroundwaterMonitoringWells as the index, retreived with

910 `brodata.gmw.get_characteristics`.

911 index : str or list of str, optional

912 Column(s) to use as the index for the resulting GeoDataFrame. Defaults

913 to ['groundwaterMonitoringWell', 'tubeNumber'] if not provided.

914

915 Returns

916 -------

917 gpd.GeoDataFrame

918 GeoDataFrame of combined well and tube properties

919 """

920 desc = "Downloading Groundwater Monitoring Wells"

921 gmws = bro._get_data_for_bro_ids(

922 GroundwaterMonitoringWell, bro_ids, desc=desc, **kwargs

923 )

924 gdf = get_tube_gdf(gmws, index=index)

925 return gdf

926

927

928cl = GroundwaterMonitoringWell

929

930get_bro_ids_of_bronhouder = partial(bro._get_bro_ids_of_bronhouder, cl)

931get_bro_ids_of_bronhouder.__doc__ = bro._get_bro_ids_of_bronhouder.__doc__

932

933get_data_for_bro_ids = partial(bro._get_data_for_bro_ids, cl)

934get_data_for_bro_ids.__doc__ = bro._get_data_for_bro_ids.__doc__

935

936get_characteristics = partial(bro._get_characteristics, cl)

937get_characteristics.__doc__ = bro._get_characteristics.__doc__