Coverage for brodata/dino.py: 75%

1import logging 

2import os 

3from io import BytesIO, StringIO, TextIOWrapper 

4from pathlib import Path 

5from zipfile import ZipFile 

6 

7import numpy as np 

8import pandas as pd 

9import geopandas as gpd 

10import requests 

11import json 

12from shapely.geometry import LineString 

13import matplotlib.pyplot as plt 

14 

15from . import util 

16from .webservices import get_configuration, get_gdf 

17 

18logger = logging.getLogger(__name__) 

19 

20 

21def objects_to_gdf( 

22 objects, 

23 x="X-coordinaat", 

24 y="Y-coordinaat", 

25 geometry=None, 

26 index=None, 

27 to_gdf=True, 

28): 

29 """ 

30 Convert a dictionary of dino-objects to a geopandas GeoDataFrame. 

31 

32 Parameters 

33 ---------- 

34 objects: dictionary of bro or dinoloket objects 

35 dictionary of objects to convert to (geo)dataframe 

36 geometry: str 

37 name of column of geometry 

38 x: str 

39 name of column of x-coordinate 

40 y: str 

41 name of column of y-coordinate 

42 index: str or list of str 

43 name of column to use as index 

44 to_gdf: bool 

45 convert to geodataframe 

46 

47 Returns 

48 ------- 

49 gdf: GeoDataFrame or DataFrame 

50 Returns a GeoDataFrame if to_gdf is True, otherwise a DataFrame 

51 """ 

52 

53 if not to_gdf: 

54 return objects 

55 

56 # convert a list of dino-objects to a geodataframe 

57 df = pd.DataFrame([objects[key].to_dict() for key in objects]) 

58 if geometry is not None: 

59 if geometry in df.columns: 

60 geometry = df[geometry] 

61 else: 

62 geometry = None 

63 else: 

64 if df.empty: 

65 logger.warning("no data found") 

66 else: 

67 if x not in df: 

68 logger.warning(f"{x} not found in data. No geometry column created.") 

69 elif y not in df: 

70 logger.warning(f"{y} not found in data. No geometry column created.") 

71 else: 

72 geometry = gpd.points_from_xy(df[x], df[y]) 

73 gdf = gpd.GeoDataFrame(df, geometry=geometry) 

74 if index is not None and not gdf.empty: 

75 if isinstance(index, str): 

76 if index in gdf.columns: 

77 gdf = gdf.set_index(index) 

78 elif np.all([x in gdf.columns for x in index]): 

79 # we assume index is an iterable (list), to form a MultiIndex 

80 gdf = gdf.set_index(index) 

81 return gdf 

82 

83 

84def _get_data_within_extent( 

85 dino_cl, 

86 kind, 

87 extent, 

88 config=None, 

89 timeout=5, 

90 silent=False, 

91 to_path=None, 

92 to_zip=None, 

93 redownload=False, 

94 x="X-coordinaat", 

95 y="Y-coordinaat", 

96 geometry=None, 

97 index="NITG-nr", 

98 to_gdf=True, 

99 max_retries=2, 

100 continue_on_error=False, 

101 progress_callback=None, 

102): 

103 """Retrieve DINO data within a specified geographical extent or from local files. 

104 

105 This is a core function used by various data retrieval methods in the DINO system. 

106 It can either load data from local files/archives or fetch it from the DINO server 

107 based on geographical extent. 

108 

109 Parameters 

110 ---------- 

111 dino_cl : class 

112 The DINO data class to instantiate for each location (e.g., Grondwaterstand). 

113 kind : str 

114 The type of DINO data to retrieve (e.g., "Grondwaterstand", "Boorgatmeting"). 

115 extent : str, Path, or sequence 

116 Either a path to local data, or a sequence of [xmin, xmax, ymin, ymax] 

117 coordinates. 

118 config : dict, optional 

119 Configuration mapping for DINO data kinds. Uses default if None. 

120 timeout : int or float, optional. 

121 Timeout in seconds for network requests when downloading data. The default is 5. 

122 silent : bool, default=False 

123 If True, suppress progress output. 

124 to_path : str, optional 

125 Directory to save downloaded files. Created if it doesn't exist. 

126 to_zip : str, optional 

127 Path to save downloaded files in a zip archive. 

128 redownload : bool, optional 

129 If True, redownload data even if local files exist. The default is False. 

130 x : str, optional 

131 Name of the x-coordinate column. The default is "X-coordinaat". 

132 y : str, optional 

133 Name of the y-coordinate column. The default is "Y-coordinaat". 

134 geometry : str, optional 

135 Name of the geometry column if different from creating from x,y coordinates. 

136 index : str, optional 

137 Column(s) to use as index in the output GeoDataFrame. The default is "NITG-nr". 

138 to_gdf : bool, optional 

139 If True, return a GeoDataFrame; if False, return raw dictionary of objects. The 

140 default is True 

141 max_retries : int, optional 

142 Maximum number of retries for failed network requests. The default is 2. 

143 continue_on_error : bool, optional 

144 If True, continue after an error occurs during downloading or processing of 

145 individual observation data. Defaults to False. 

146 progress_callback : function, optional 

147 A callback function that takes two arguments (current, total) to report 

148 progress. If None, no progress reporting is done. Defaults to None. 

149 

150 Returns 

151 ------- 

152 geopandas.GeoDataFrame or dict 

153 If to_gdf is True, returns a GeoDataFrame with the requested data. 

154 If to_gdf is False, returns a dictionary of DINO objects. 

155 """ 

156 if isinstance(extent, (str, Path)): 

157 data = _get_data_from_path(extent, dino_cl, silent=silent, progress_callback=progress_callback) 

158 return objects_to_gdf(data, x, y, geometry, index, to_gdf) 

159 

160 if to_zip is not None: 

161 if not redownload and os.path.isfile(to_zip): 

162 data = _get_data_from_zip(to_zip, dino_cl, silent=silent, extent=extent, progress_callback=progress_callback) 

163 return objects_to_gdf(data, x, y, geometry, index, to_gdf) 

164 if to_path is None: 

165 to_path = os.path.splitext(to_zip)[0] 

166 remove_path_again = not os.path.isdir(to_path) 

167 files = [] 

168 

169 if config is None: 

170 config = get_configuration() 

171 

172 if to_path is not None and not os.path.isdir(to_path): 

173 os.makedirs(to_path) 

174 

175 to_file = None 

176 gdf = None 

177 if to_path is not None: 

178 to_file = os.path.join(to_path, f"{dino_cl.__name__}.geojson") 

179 if to_zip is not None: 

180 files.append(to_file) 

181 if not redownload and os.path.isfile(to_file): 

182 gdf = gpd.read_file(to_file) 

183 if not gdf.empty and "DINO_NR" in gdf.columns: 

184 gdf = gdf.set_index("DINO_NR") 

185 if gdf is None: 

186 gdf = get_gdf( 

187 kind, 

188 config=config, 

189 extent=extent, 

190 timeout=timeout, 

191 ) 

192 if to_file is not None: 

193 gdf.to_file(to_file) 

194 

195 to_file = None 

196 

197 data = {} 

198 for i, dino_nr in util.tqdm(enumerate(gdf.index), disable=silent): 

199 if progress_callback is not None: 

200 progress_callback(i, len(gdf)) 

201 if to_path is not None: 

202 to_file = os.path.join(to_path, f"{dino_nr}.csv") 

203 if to_zip is not None: 

204 files.append(to_file) 

205 if not redownload and os.path.isfile(to_file): 

206 data[dino_nr] = dino_cl(to_file) 

207 continue 

208 try: 

209 data[dino_nr] = dino_cl.from_dino_nr( 

210 dino_nr, timeout=timeout, to_file=to_file, max_retries=max_retries 

211 ) 

212 except Exception as e: 

213 if not continue_on_error: 

214 raise e 

215 logger.error("Error retrieving %s %s: %s", kind, dino_nr, e) 

216 continue 

217 if to_zip is not None: 

218 util._save_data_to_zip(to_zip, files, remove_path_again, to_path) 

219 

220 return objects_to_gdf(data, x, y, geometry, index, to_gdf) 

221 

222 

223def _get_data_from_path(from_path, dino_class, silent=False, ext=".csv", progress_callback=None): 

224 if str(from_path).endswith(".zip"): 

225 return _get_data_from_zip(from_path, dino_class, silent=silent, progress_callback=progress_callback) 

226 files = os.listdir(from_path) 

227 files = [file for file in files if file.endswith(ext)] 

228 data = {} 

229 total = len(files) 

230 for i, file in util.tqdm(enumerate(files), total=total, disable=silent): 

231 if progress_callback is not None: 

232 progress_callback(i, total) 

233 fname = os.path.join(from_path, file) 

234 data[os.path.splitext(file)[0]] = dino_class(fname) 

235 return data 

236 

237 

238def _get_data_from_zip(to_zip, dino_class, silent=False, extent=None, progress_callback=None): 

239 # read data from zipfile 

240 data = {} 

241 with ZipFile(to_zip) as zf: 

242 names = zf.namelist() 

243 name = f"{dino_class.__name__}.geojson" 

244 has_location_file = name in names 

245 if has_location_file: 

246 names.remove(name) 

247 if has_location_file and extent is not None: 

248 gdf = gpd.read_file(zf.open(name)) 

249 gdf = gdf.set_index("DINO_NR") 

250 gdf = gdf.cx[extent[0] : extent[1], extent[2] : extent[3]] 

251 names = [f"{name}.csv" for name in gdf.index] 

252 total = len(names) 

253 for i, name in util.tqdm(enumerate(names), total=total, disable=silent): 

254 if progress_callback is not None: 

255 progress_callback(i, total) 

256 data[name] = dino_class(name, zipfile=zf) 

257 return data 

258 

259 

260def get_verticaal_elektrisch_sondeeronderzoek(extent, **kwargs): 

261 dino_class = VerticaalElektrischSondeeronderzoek 

262 kind = "Verticaal elektrisch sondeeronderzoek" 

263 return _get_data_within_extent( 

264 dino_class, kind, extent, geometry="geometry", **kwargs 

265 ) 

266 

267 

268def get_grondwaterstand( 

269 extent, 

270 config=None, 

271 timeout=5, 

272 silent=False, 

273 to_path=None, 

274 to_zip=None, 

275 redownload=False, 

276 to_gdf=True, 

277 skip=None, 

278 continue_on_error=False, 

279 progress_callback=None, 

280): 

281 """ 

282 Get groundwater level (Grondwaterstand) data as a GeoDataFrame or raw objects. 

283 

284 Fetch Grondwaterstand data for a given geographical extent or load it from local 

285 files. Data are retrieved per monitoring location and per piezometer. Results can 

286 be returned as a GeoDataFrame or as a dictionary of Grondwaterstand objects. 

287 

288 Parameters 

289 ---------- 

290 extent : str or sequence 

291 The spatial extent ([xmin, xmax, ymin, ymax]) to filter the data. 

292 config : dict, optional 

293 Configuration mapping for available DINO data kinds. If None, a default 

294 configuration is used. 

295 timeout : int or float, optional 

296 Timeout in seconds for network requests when downloading data. The default is 5. 

297 silent : bool, optional 

298 If True, suppress progress output. 

299 to_path : str, optional 

300 If not None, save the downloaded files in the directory named to_path. The 

301 default is None. 

302 to_zip : str, optional 

303 If not None, save the downloaded files in a zip-file named to_zip. The default 

304 is None. 

305 redownload : bool, optional 

306 When downloaded files exist in to_path or to_zip, read from these files when 

307 redownload is False. If redownload is True, download the data again from the 

308 DINO-server. The default is False. 

309 to_gdf : bool, optional 

310 If True (default), convert the loaded Grondwaterstand objects into a 

311 geopandas.GeoDataFrame. If False, return the raw mapping of objects. 

312 skip : str or iterable, optional 

313 Name or iterable of location names to skip during download or processing. 

314 continue_on_error : bool, optional 

315 If True, continue after an error occurs during downloading or processing of 

316 individual observation data. Defaults to False. 

317 progress_callback : function, optional 

318 A callback function that takes two arguments (current, total) to report 

319 progress. If None, no progress reporting is done. Defaults to None. 

320 

321 Returns 

322 ------- 

323 geopandas.GeoDataFrame or dict 

324 If `to_gdf` is True, returns a GeoDataFrame indexed by ['Locatie', 

325 'Filternummer']. If False, returns a dictionary with Grondwaterstand objects. 

326 

327 Notes 

328 ----- 

329 - When `extent` is a path string, this function loads local data. 

330 - When `to_zip` is provided, the function will create a temporary directory and 

331 archive files into the supplied ZIP. 

332 """ 

333 dino_class = Grondwaterstand 

334 index = ["Locatie", "Filternummer"] 

335 if skip is not None and isinstance(skip, str): 

336 skip = [skip] 

337 

338 if isinstance(extent, str): 

339 data = _get_data_from_path(extent, dino_class, silent=silent, progress_callback=progress_callback) 

340 return objects_to_gdf(data, index=index, to_gdf=to_gdf) 

341 

342 if to_zip is not None: 

343 if not redownload and os.path.isfile(to_zip): 

344 data = _get_data_from_zip(to_zip, dino_class, silent=silent, progress_callback=progress_callback) 

345 return objects_to_gdf(data, index=index, to_gdf=to_gdf) 

346 if to_path is None: 

347 to_path = os.path.splitext(to_zip)[0] 

348 remove_path_again = not os.path.isdir(to_path) 

349 files = [] 

350 

351 kind = "Grondwaterstand" 

352 if config is None: 

353 config = get_configuration() 

354 gdf = get_gdf( 

355 kind, 

356 config=config, 

357 extent=extent, 

358 timeout=timeout, 

359 ) 

360 download_url = config[kind]["download"] 

361 

362 to_file = None 

363 if to_path is not None and not os.path.isdir(to_path): 

364 os.makedirs(to_path) 

365 data = {} 

366 for i, name in util.tqdm(enumerate(gdf.index), disable=silent): 

367 if progress_callback is not None: 

368 progress_callback(i, len(gdf)) 

369 if skip is not None and name in skip: 

370 continue 

371 for i_st in range(1, gdf.at[name, "ST_CNT"] + 1): 

372 piezometer_nr = f"{i_st:03d}" 

373 url = f"{download_url}/{name}/{piezometer_nr}" 

374 if to_path is not None: 

375 to_file = os.path.join(to_path, f"{name}_{piezometer_nr}.csv") 

376 if to_zip is not None: 

377 files.append(to_file) 

378 if not redownload and os.path.isfile(to_file): 

379 data[f"{name}_{piezometer_nr}"] = dino_class(to_file) 

380 continue 

381 try: 

382 data[f"{name}_{piezometer_nr}"] = dino_class( 

383 url, timeout=timeout, to_file=to_file 

384 ) 

385 except Exception as e: 

386 if not continue_on_error: 

387 raise e 

388 logger.error( 

389 "Error retrieving %s %s piezometer %s: %s", 

390 kind, 

391 name, 

392 piezometer_nr, 

393 e, 

394 ) 

395 continue 

396 if to_zip is not None: 

397 util._save_data_to_zip(to_zip, files, remove_path_again, to_path) 

398 return objects_to_gdf( 

399 data, index=index, to_gdf=to_gdf, x="X-coordinaat", y="Y-coordinaat" 

400 ) 

401 

402 

403def get_grondwatersamenstelling(extent, **kwargs): 

404 dino_class = Grondwatersamenstelling 

405 kind = "Grondwatersamenstelling" 

406 return _get_data_within_extent(dino_class, kind, extent, **kwargs) 

407 

408 

409def get_geologisch_booronderzoek(extent, **kwargs): 

410 logger.warning( 

411 "`get_geologisch_booronderzoek` is deprecated. Use `get_boormonsterprofiel` instead" 

412 ) 

413 dino_class = GeologischBooronderzoek 

414 kind = "Geologisch booronderzoek" 

415 return _get_data_within_extent(dino_class, kind, extent, **kwargs) 

416 

417 

418def get_boormonsterprofiel(extent, **kwargs): 

419 dino_class = Boormonsterprofiel 

420 kind = "Boormonsterprofiel" 

421 return _get_data_within_extent(dino_class, kind, extent, **kwargs) 

422 

423 

424def get_boorgatmeting(extent, **kwargs): 

425 dino_class = Boorgatmeting 

426 kind = "Boorgatmeting" 

427 return _get_data_within_extent(dino_class, kind, extent, **kwargs) 

428 

429 

430def get_chemische_analyse(extent, **kwargs): 

431 dino_class = ChemischeAnalyse 

432 kind = "Chemische analyse" 

433 return _get_data_within_extent(dino_class, kind, extent, **kwargs) 

434 

435 

436def get_korrelgrootte_analyse(extent, **kwargs): 

437 dino_class = KorrelgrootteAnalyse 

438 kind = "Korrelgrootte analyse" 

439 return _get_data_within_extent(dino_class, kind, extent, **kwargs) 

440 

441 

442def get_oppervlaktewaterstand(extent, **kwargs): 

443 dino_class = Oppervlaktewaterstand 

444 kind = "Oppervlaktewateronderzoek" 

445 return _get_data_within_extent(dino_class, kind, extent, **kwargs) 

446 

447 

448class CsvFileOrUrl: 

449 def __init__( 

450 self, 

451 url_or_file, 

452 zipfile=None, 

453 timeout=5, 

454 to_file=None, 

455 redownload=True, 

456 max_retries=2, 

457 ): 

458 if zipfile is not None: 

459 with zipfile.open(url_or_file) as f: 

460 self._read_contents(TextIOWrapper(f)) 

461 elif url_or_file.startswith("http"): 

462 if redownload or to_file is None or not os.path.isfile(to_file): 

463 if max_retries > 1: 

464 adapter = requests.adapters.HTTPAdapter(max_retries=max_retries) 

465 session = requests.Session() 

466 session.mount("https://", adapter) 

467 req = session.get(url_or_file, timeout=timeout) 

468 else: 

469 req = requests.get(url_or_file, timeout=timeout) 

470 if not req.ok: 

471 raise (Exception((f"Retieving data from {url_or_file} failed"))) 

472 is_zipfile = False 

473 if "content-disposition" in req.headers: 

474 if req.headers["content-disposition"].endswith(".zip"): 

475 is_zipfile = True 

476 if is_zipfile: 

477 # BoorgatMetingen are las files that are delivered in a zip-file 

478 with ZipFile(BytesIO(req.content)) as myzip: 

479 files = [f for f in myzip.namelist() if f.endswith(".las")] 

480 if len(files) == 0: 

481 logger.warning("Zipfile contains no las files.") 

482 self._read_contents(None) 

483 else: 

484 if len(files) > 1: 

485 logger.warning(f"Zipfile contains {len(files)} las files. Only first file will be read.") 

486 with myzip.open(files[0]) as myfile: 

487 if to_file is not None: 

488 with open(to_file, "wb") as f: 

489 f.write(myfile.read()) 

490 self._read_contents(TextIOWrapper(myfile)) 

491 else: 

492 if to_file is not None: 

493 with open(to_file, "w") as f: 

494 f.write(req.text) 

495 self._read_contents(StringIO(req.text)) 

496 else: 

497 with open(to_file, "r") as f: 

498 self._read_contents(f) 

499 else: 

500 with open(url_or_file, "r") as f: 

501 self._read_contents(f) 

502 

503 def __repr__(self): 

504 # retrieve properties if they exist 

505 propdict = {"NITG-nr": "NITG-nr", "X-coordinaat": "x", "Y-coordinaat": "y"} 

506 props = {} 

507 for key in propdict: 

508 if hasattr(self, key): 

509 props[propdict[key]] = getattr(self, key) 

510 name = util._format_repr(self, props) 

511 return name 

512 

513 @classmethod 

514 def from_dino_nr(cls, dino_nr, **kwargs): 

515 if not hasattr(cls, "_download_url"): 

516 raise (NotImplementedError(f"No download-url defined for {cls.__name__}")) 

517 return cls(f"{cls._download_url}/{dino_nr}", **kwargs) 

518 

519 def _read_properties_csv_rows(self, f, merge_columns=False, **kwargs): 

520 # this is the new format of properties from dinoloket 

521 df, line = self._read_csv_part(f, header=None, index_col=0, **kwargs) 

522 # remove empty columns 

523 df = df.loc[:, ~df.isna().all(axis=0)] 

524 if merge_columns: 

525 for index in df.index: 

526 df.at[index, 1] = " ".join(df.loc[index, ~df.loc[index].isna()].values) 

527 df = df.loc[:, :1] 

528 else: 

529 assert df.shape[1] == 1 

530 d = df.squeeze().to_dict() 

531 return d, line 

532 

533 def _read_properties_csv_columns(self, f, **kwargs): 

534 df, line = self._read_csv_part(f, **kwargs) 

535 assert df.shape[0] == 1 

536 d = df.squeeze().to_dict() 

537 return d, line 

538 

539 def _read_csv_part(self, f, sep=",", header=0, index_col=False, **kwargs): 

540 strt = f.tell() 

541 if header is None: 

542 nrows = 0 

543 else: 

544 nrows = -1 # the header does not count 

545 line = f.readline() 

546 while line.replace(",", "") not in ["\n", ""]: 

547 nrows += 1 

548 line = f.readline() 

549 eind = f.tell() 

550 # go back to where we were before 

551 f.seek(strt) 

552 df = pd.read_csv( 

553 f, sep=sep, index_col=index_col, nrows=nrows, header=header, **kwargs 

554 ) 

555 if header is not None: 

556 df = df.loc[:, ~df.columns.str.startswith("Unnamed: ")] 

557 f.seek(eind) 

558 

559 if line != "": 

560 # read empty lines gat 

561 while line.replace(",", "") == "\n": 

562 new_start = f.tell() 

563 line = f.readline() 

564 f.seek(new_start) 

565 

566 return df, line 

567 

568 

569class Oppervlaktewaterstand(CsvFileOrUrl): 

570 _download_url = "https://www.dinoloket.nl/uitgifteloket/api/wo/owo/full" 

571 

572 def __repr__(self): 

573 # retrieve properties if they exist 

574 

575 props = {} 

576 if hasattr(self, "meta") and not self.meta.empty: 

577 s = self.meta.iloc[-1] 

578 propdict = {"Locatie": "Locatie", "X-coordinaat": "x", "Y-coordinaat": "y"} 

579 for key in propdict: 

580 if key in s: 

581 props[propdict[key]] = s[key] 

582 name = util._format_repr(self, props) 

583 return name 

584 

585 def _read_contents(self, f): 

586 self.props, line = self._read_properties_csv_rows(f, merge_columns=True) 

587 if line.startswith( 

588 '"Van deze put zijn geen standen opgenomen in de DINO-database"' 

589 ): 

590 return 

591 self.meta, line = self._read_csv_part(f) 

592 self.data, line = self._read_csv_part(f) 

593 for column in ["Peildatum"]: 

594 if column in self.data.columns: 

595 self.data[column] = pd.to_datetime(self.data[column], dayfirst=True) 

596 

597 def to_dict(self): 

598 d = {**self.props} 

599 if hasattr(self, "meta"): 

600 d["meta"] = self.meta 

601 for column in d["meta"]: 

602 d[column] = d["meta"][column].iloc[-1] 

603 if hasattr(self, "data"): 

604 d["data"] = self.data 

605 return d 

606 

607 

608class Grondwaterstand(CsvFileOrUrl): 

609 _download_url = "https://www.dinoloket.nl/uitgifteloket/api/wo/gwo/full" 

610 

611 @classmethod 

612 def from_dino_nr(cls, dino_nr, filter_nr, **kwargs): 

613 return cls(f"{cls._download_url}/{dino_nr}/{filter_nr:03d}", **kwargs) 

614 

615 def __repr__(self): 

616 # retrieve properties if they exist 

617 

618 props = {} 

619 if hasattr(self, "meta") and not self.meta.empty: 

620 s = self.meta.iloc[-1] 

621 propdict = { 

622 "Locatie": "Locatie", 

623 "Filternummer": "filter", 

624 "X-coordinaat": "x", 

625 "Y-coordinaat": "y", 

626 } 

627 for key in propdict: 

628 if key in s: 

629 props[propdict[key]] = s[key] 

630 name = util._format_repr(self, props) 

631 return name 

632 

633 def _read_contents(self, f): 

634 self.props, line = self._read_properties_csv_rows(f, merge_columns=True) 

635 self.props2, line = self._read_properties_csv_rows(f) 

636 if line.startswith( 

637 '"Van deze put zijn geen standen opgenomen in de DINO-database"' 

638 ): 

639 return 

640 if "Peildatum" not in line: 

641 self.meta, line = self._read_csv_part(f) 

642 self.data, line = self._read_csv_part(f) 

643 for column in ["Peildatum"]: 

644 if column in self.data.columns: 

645 self.data[column] = pd.to_datetime(self.data[column], dayfirst=True) 

646 

647 def to_dict(self): 

648 d = {**self.props, **self.props2} 

649 if hasattr(self, "meta"): 

650 d["meta"] = self.meta 

651 for column in d["meta"]: 

652 d[column] = d["meta"][column].iloc[-1] 

653 if hasattr(self, "data"): 

654 d["data"] = self.data 

655 return d 

656 

657 

658class Grondwatersamenstelling(CsvFileOrUrl): 

659 _download_url = "https://www.dinoloket.nl/uitgifteloket/api/wo/gwo/qua/report" 

660 

661 def _read_contents(self, f): 

662 # read first line and place cursor at start of document again 

663 start = f.tell() 

664 line = f.readline().rstrip("\n") 

665 f.seek(start) 

666 

667 # LOCATIE gegevens 

668 if line.startswith('"LOCATIE gegevens"'): 

669 line = f.readline() 

670 self.locatie_gegevens, line = self._read_properties_csv_columns(f) 

671 for key in self.locatie_gegevens: 

672 setattr(self, key, self.locatie_gegevens[key]) 

673 

674 # KWALITEIT gegevens VLOEIBAAR 

675 if line.startswith('"KWALITEIT gegevens VLOEIBAAR"'): 

676 line = f.readline() 

677 self.kwaliteit_gegevens_vloeibaar, line = self._read_csv_part(f) 

678 for column in ["Monster datum", "Analyse datum"]: 

679 if column in self.kwaliteit_gegevens_vloeibaar.columns: 

680 self.kwaliteit_gegevens_vloeibaar[column] = pd.to_datetime( 

681 self.kwaliteit_gegevens_vloeibaar[column], dayfirst=True 

682 ) 

683 

684 def to_dict(self): 

685 d = {**self.locatie_gegevens} 

686 if hasattr(self, "kwaliteit_gegevens_vloeibaar"): 

687 d["kwaliteit_gegevens_vloeibaar"] = self.kwaliteit_gegevens_vloeibaar 

688 return d 

689 

690 

691class Boormonsterprofiel(CsvFileOrUrl): 

692 _download_url = ( 

693 "https://www.dinoloket.nl/uitgifteloket/api/brh/sampledescription/csv" 

694 ) 

695 

696 def _read_contents(self, f): 

697 # read first line and place cursor at start of document again 

698 start = f.tell() 

699 line = f.readline().rstrip("\n") 

700 f.seek(start) 

701 if line.startswith('"ALGEMENE GEGEVENS BORING"'): 

702 line = f.readline() 

703 self.algemene_gegevens_boring, line = self._read_properties_csv_columns(f) 

704 for key in self.algemene_gegevens_boring: 

705 setattr(self, key, self.algemene_gegevens_boring[key]) 

706 if line.startswith('"ALGEMENE GEGEVENS LITHOLOGIE"'): 

707 line = f.readline() 

708 self.algemene_gegevens_lithologie, line = self._read_properties_csv_columns( 

709 f 

710 ) 

711 if line.startswith('"LITHOLOGIE LAGEN"'): 

712 line = f.readline() 

713 self.lithologie_lagen, line = self._read_csv_part(f) 

714 if line.startswith('"LITHOLOGIE SUBLAGEN"'): 

715 line = f.readline() 

716 self.lithologie_sublagen, line = self._read_csv_part(f) 

717 

718 def to_dict(self): 

719 d = {**self.algemene_gegevens_boring} 

720 if hasattr(self, "algemene_gegevens_lithologie"): 

721 for key in self.algemene_gegevens_boring: 

722 if key in self.algemene_gegevens_lithologie: 

723 # 'Datum boring' can be specified in algemene_gegevens_boring and algemene_gegevens_lithologie 

724 if pd.isna(self.algemene_gegevens_lithologie[key]): 

725 self.algemene_gegevens_lithologie.pop(key) 

726 d = {**d, **self.algemene_gegevens_lithologie} 

727 if hasattr(self, "lithologie_lagen"): 

728 d["lithologie_lagen"] = self.lithologie_lagen 

729 if hasattr(self, "lithologie_sublagen"): 

730 d["lithologie_sublagen"] = self.lithologie_sublagen 

731 return d 

732 

733 

734def get_drilling_from_dinoloket( 

735 name, 

736 column_type=None, 

737 depthReference="NAP", 

738 language="nl", 

739 return_response=False, 

740 ignore_exceptions=False, 

741): 

742 """ 

743 Get a drilling from dinoloket. 

744 

745 This method uses the information from the webservice used by dinoloket for 

746 displaying the drilling. In this way, also lithostratigraphy-data can be returned, 

747 which is not present in the data downloaded as a csv-file by `Boormonsterprofiel`. 

748 

749 Parameters 

750 ---------- 

751 name : str 

752 The name of the drilling. 

753 column_type : str, optional 

754 The type of data that is returned. Possible options are "LITHOLOGY" and 

755 "LITHOSTRATIGRAPHY" and None. If column_type is None, return a dictionary with 

756 all data. The default is None. 

757 depthReference : str, optional 

758 Possible values are "NAP" and "MV". The default is "NAP". 

759 language : str of length 2, optional 

760 Possible values are "nl" for Ducth and "en" for English. When language is not 

761 'nl' or 'en', english is returned. The default is "nl". 

762 return_response : bool, optional 

763 Return the json-respons of the web-service without any interpretation. The 

764 default is False. 

765 ignore_exceptions : bool, optional 

766 When True, ignore exceptions when things go wrong. This is usefull when 

767 requesting multiple drillings. The default is False. 

768 

769 Returns 

770 ------- 

771 df or dict 

772 A dictionary or a DataFarme (when column_type is set) containing the drilling 

773 data. 

774 """ 

775 # columnType is 'LITHOSTRATIGRAPHY' or 'LITHOLOGY' 

776 url = "https://www.dinoloket.nl/javascriptmapviewer-web/rest/brh/profile" 

777 payload = {"dinoId": name, "depthReference": depthReference, "language": language} 

778 req = requests.post( 

779 url, data=json.dumps(payload), headers={"content-type": "application/json"} 

780 ) 

781 if not req.ok: 

782 msg = f"Retieving data from {url} failed"