diff --git a/src/open_irceline/__init__.py b/src/open_irceline/__init__.py index 00e8314..d830345 100644 --- a/src/open_irceline/__init__.py +++ b/src/open_irceline/__init__.py @@ -1,6 +1,6 @@ from .api import IrcelineApiError from .rio import IrcelineRioClient from .forecast import IrcelineForecastClient -from .data import RioFeature, ForecastFeature, FeatureValue, BelAqiIndex +from .data import RioFeature, ForecastFeature, FeatureValue __version__ = '2.0.0' diff --git a/src/open_irceline/api.py b/src/open_irceline/api.py index bb374a2..a89c165 100644 --- a/src/open_irceline/api.py +++ b/src/open_irceline/api.py @@ -2,17 +2,18 @@ import asyncio import socket from abc import ABC, abstractmethod from typing import Tuple, List, Set +from xml.etree import ElementTree import aiohttp import async_timeout +from aiohttp import ClientResponse from .data import IrcelineFeature from .utils import SizedDict _rio_wfs_base_url = 'https://geo.irceline.be/wfs' _forecast_wms_base_url = 'https://geo.irceline.be/forecast/wms' -# noinspection HttpUrlsUsage -# There is not HTTPS version of this endpoint +_rio_ifdm_wms_base_url = 'https://geobelair.irceline.be/rioifdm/wms' _user_agent = 'github.com/jdejaegh/python-irceline' @@ -65,3 +66,38 @@ class IrcelineBaseClient(ABC): raise IrcelineApiError(f"Something really wrong happened! {exception}") from exception +class IrcelineBaseWmsClient(IrcelineBaseClient, ABC): + _default_querystring = {"service": "WMS", + "version": "1.1.1", + "request": "GetFeatureInfo", + "info_format": "application/json", + "width": "1", + "height": "1", + "srs": "EPSG:4326", + "X": "1", + "Y": "1"} + _epsilon = 0.00001 + _base_url = None + + @staticmethod + def _parse_capabilities(xml_string: str) -> Set[str]: + try: + root = ElementTree.fromstring(xml_string) + except ElementTree.ParseError: + return set() + + path = './/Capability/Layer/Layer/Name' + feature_type_names = {t.text for t in root.findall(path)} + return feature_type_names + + async def get_capabilities(self) -> Set[str]: + """ + Fetch the list of possible features from the WMS server + :return: set of features available on the WMS server + """ + querystring = {"service": "WMS", + "version": "1.1.1", + "request": "GetCapabilities"} + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) + + return self._parse_capabilities(await r.text()) diff --git a/src/open_irceline/data.py b/src/open_irceline/data.py index fd4cb41..7f8c4c7 100644 --- a/src/open_irceline/data.py +++ b/src/open_irceline/data.py @@ -1,5 +1,5 @@ from datetime import datetime, date -from enum import StrEnum, Enum +from enum import StrEnum from typing import TypedDict @@ -30,6 +30,14 @@ class RioFeature(IrcelineFeature): SO2_HMEAN = 'rio:so2_hmean' +class RioIfdmFeature(IrcelineFeature): + PM25_HMEAN = 'rioifdm:pm25_hmean' + NO2_HMEAN = 'rioifdm:no2_hmean' + PM10_HMEAN = 'rioifdm:pm10_hmean' + O3_HMEAN = 'rioifdm:o3_hmean' + BELAQI = 'rioifdm:belaqi' + + class ForecastFeature(IrcelineFeature): NO2_MAXHMEAN = 'forecast:no2_maxhmean' NO2_DMEAN = 'forecast:no2_dmean' @@ -40,20 +48,7 @@ class ForecastFeature(IrcelineFeature): BELAQI = 'forecast:belaqi' -class BelAqiIndex(Enum): - EXCELLENT = 1 - VERY_GOOD = 2 - GOOD = 3 - FAIRLY_GOOD = 4 - MODERATE = 5 - POOR = 6 - VERY_POOR = 7 - BAD = 8 - VERY_BAD = 9 - HORRIBLE = 10 - - class FeatureValue(TypedDict): # Timestamp at which the value was computed timestamp: datetime | date | None - value: int | float | BelAqiIndex | None + value: int | float | None diff --git a/src/open_irceline/forecast.py b/src/open_irceline/forecast.py index d4ba038..c2b5efd 100644 --- a/src/open_irceline/forecast.py +++ b/src/open_irceline/forecast.py @@ -1,16 +1,15 @@ from datetime import date, timedelta, datetime from itertools import product -from typing import List, Tuple, Dict, Set -from xml.etree import ElementTree +from typing import List, Tuple, Dict from aiohttp import ClientResponse, ClientResponseError -from .api import IrcelineBaseClient, _forecast_wms_base_url, IrcelineApiError +from .api import IrcelineApiError, IrcelineBaseWmsClient, _forecast_wms_base_url from .data import ForecastFeature, FeatureValue -class IrcelineForecastClient(IrcelineBaseClient): - _epsilon = 0.00001 +class IrcelineForecastClient(IrcelineBaseWmsClient): + _base_url = _forecast_wms_base_url async def get_data(self, features: List[ForecastFeature], @@ -26,22 +25,14 @@ class IrcelineForecastClient(IrcelineBaseClient): timestamp = date.today() result = dict() lat, lon = position - base_querystring = {"service": "WMS", - "version": "1.1.1", - "request": "GetFeatureInfo", - "info_format": "application/json", - "width": "1", - "height": "1", - "srs": "EPSG:4326", - "bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}", - "X": "1", - "Y": "1"} + base_querystring = (self._default_querystring | + {"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) for feature, d in product(features, range(4)): querystring = base_querystring | {"layers": f"{feature}_d{d}", "query_layers": f"{feature}_d{d}"} try: - r: ClientResponse = await self._api_wrapper(_forecast_wms_base_url, querystring) + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) r: dict = await r.json() result[(feature, timestamp + timedelta(days=d))] = FeatureValue( value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), @@ -50,26 +41,3 @@ class IrcelineForecastClient(IrcelineBaseClient): result[(feature, timestamp + timedelta(days=d))] = FeatureValue(value=None, timestamp=None) return result - - async def get_capabilities(self) -> Set[str]: - """ - Fetch the list of possible features from the WMS server - :return: set of features available on the WMS server - """ - querystring = {"service": "WMS", - "version": "1.1.1", - "request": "GetCapabilities"} - r: ClientResponse = await self._api_wrapper(_forecast_wms_base_url, querystring) - - return self._parse_capabilities(await r.text()) - - @staticmethod - def _parse_capabilities(xml_string: str) -> Set[str]: - try: - root = ElementTree.fromstring(xml_string) - except ElementTree.ParseError: - return set() - - path = './/Capability/Layer/Layer/Name' - feature_type_names = {t.text for t in root.findall(path)} - return feature_type_names diff --git a/src/open_irceline/rio.py b/src/open_irceline/rio.py index 93a839a..3cea0b5 100644 --- a/src/open_irceline/rio.py +++ b/src/open_irceline/rio.py @@ -2,15 +2,19 @@ from datetime import datetime, date, UTC, timedelta from typing import List, Tuple, Dict, Set from xml.etree import ElementTree -from aiohttp import ClientResponse +from aiohttp import ClientResponse, ClientResponseError -from .api import IrcelineBaseClient, _rio_wfs_base_url, IrcelineApiError -from .data import RioFeature, FeatureValue +from .api import IrcelineBaseClient, _rio_wfs_base_url, IrcelineApiError, _rio_ifdm_wms_base_url, IrcelineBaseWmsClient +from .data import RioFeature, FeatureValue, RioIfdmFeature from .utils import epsg_transform class IrcelineRioClient(IrcelineBaseClient): - """API client for RIO interpolated IRCEL - CELINE open data""" + """ + API client for RIO interpolated IRCEL - CELINE open data + RIO is more coarse grained for interpolation than RIO IFDM and allows to request multiple features in the same + request, which may be faster. + """ async def get_data(self, features: List[RioFeature], @@ -124,3 +128,41 @@ class IrcelineRioClient(IrcelineBaseClient): result[name] = FeatureValue(timestamp=timestamp, value=value) return result + + +class IrcelineRioIfdmClient(IrcelineBaseWmsClient): + """ + API client for RIO IFDM interpolated IRCEL - CELINE open data + RIO IFDM is more fine-grained for interpolation than RIO but only allows one feature to be request at a time, which + may be slower + """ + _base_url = _rio_ifdm_wms_base_url + + async def get_data(self, + features: List[RioIfdmFeature], + position: Tuple[float, float] + ) -> Dict[RioIfdmFeature, FeatureValue]: + """ + Get interpolated concentrations for the given features at the given position. + :param features: pollutants to get the forecasts for + :param position: (lat, long) + :return: dict where key is RioIfdmFeature and value is a FeatureValue + """ + result = dict() + lat, lon = position + base_querystring = (self._default_querystring | + {"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) + print({"bbox": f"{lon},{lat},{lon + self._epsilon},{lat + self._epsilon}"}) + + for feature in features: + querystring = base_querystring | {"layers": f"{feature}", "query_layers": f"{feature}"} + try: + r: ClientResponse = await self._api_wrapper(self._base_url, querystring) + r: dict = await r.json() + result[feature] = FeatureValue( + value=r.get('features', [{}])[0].get('properties', {}).get('GRAY_INDEX'), + timestamp=datetime.fromisoformat(r.get('timeStamp')) if 'timeStamp' in r else None) + except (IrcelineApiError, ClientResponseError, IndexError): + result[feature] = FeatureValue(value=None, timestamp=None) + + return result diff --git a/tests/fixtures/ifdm_interpolation_feature_info.json b/tests/fixtures/ifdm_interpolation_feature_info.json new file mode 100644 index 0000000..42b1fba --- /dev/null +++ b/tests/fixtures/ifdm_interpolation_feature_info.json @@ -0,0 +1,17 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "", + "geometry": null, + "properties": { + "GRAY_INDEX": 84.33950805664062 + } + } + ], + "totalFeatures": "unknown", + "numberReturned": 1, + "timeStamp": "2024-06-30T15:43:07.222Z", + "crs": null +} \ No newline at end of file diff --git a/tests/fixtures/capabilities.xml b/tests/fixtures/rio_capabilities.xml similarity index 100% rename from tests/fixtures/capabilities.xml rename to tests/fixtures/rio_capabilities.xml diff --git a/tests/fixtures/rio_ifdm_capabilities.xml b/tests/fixtures/rio_ifdm_capabilities.xml new file mode 100644 index 0000000..4bffb10 --- /dev/null +++ b/tests/fixtures/rio_ifdm_capabilities.xml @@ -0,0 +1,694 @@ + + + + + OGC:WMS + IRCEL - CELINE - Web Map Service + A compliant implementation of WMS plus most of the SLD extension (dynamic styling). Can also generate + PDF, SVG, KML, GeoRSS + + + WFS + WMS + GEOSERVER + + + + + IRCEL - CELINE + Belgian Interregional Environment Agency + + + + +
Gaucheretstraat 92-94 Rue Gaucheret
+ Brussels + + 1030 + Belgium + + +(32)(0)2 227 57 01 + + info@irceline.be + + NONE + NONE + + + + + application/vnd.ogc.wms_xml + text/xml + + + + + + + + + + + + + image/png + application/atom xml + application/atom+xml + application/json;type=utfgrid + application/openlayers + application/openlayers2 + application/openlayers3 + application/pdf + application/rss xml + application/rss+xml + application/vnd.google-earth.kml + application/vnd.google-earth.kml xml + application/vnd.google-earth.kml+xml + application/vnd.google-earth.kml+xml;mode=networklink + application/vnd.google-earth.kmz + application/vnd.google-earth.kmz xml + application/vnd.google-earth.kmz+xml + application/vnd.google-earth.kmz;mode=networklink + atom + image/geotiff + image/geotiff8 + image/gif + image/gif;subtype=animated + image/jpeg + image/png8 + image/png; mode=8bit + image/svg + image/svg xml + image/svg+xml + image/tiff + image/tiff8 + image/vnd.jpeg-png + image/vnd.jpeg-png8 + kml + kmz + openlayers + rss + text/html; subtype=openlayers + text/html; subtype=openlayers2 + text/html; subtype=openlayers3 + utfgrid + + + + + + + + + + text/plain + application/vnd.ogc.gml + text/xml + application/vnd.ogc.gml/3.1.1 + text/xml; subtype=gml/3.1.1 + text/html + application/json + + + + + + + + + + + + + application/vnd.ogc.wms_xml + + + + + + + + + + image/png + image/jpeg + application/json + image/gif + + + + + + + + + + application/vnd.ogc.sld+xml + + + + + + + + + + + application/vnd.ogc.se_xml + application/vnd.ogc.se_inimage + application/vnd.ogc.se_blank + application/json + + + + IRCEL - CELINE - Web Map Service + A compliant implementation of WMS plus most of the SLD extension (dynamic styling). Can also + generate PDF, SVG, KML, GeoRSS + + + EPSG:3857 + EPSG:4258 + EPSG:4326 + EPSG:31370 + EPSG:900913 + + + + + http://geo.irceline.be + + belaqi + belaqi + + + belaqi + WCS + GeoTIFF + + EPSG:31370 + + + + + + belaqi_dm1 + belaqi_dm1 + + + belaqi_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + belaqi_dm2 + belaqi_dm2 + + + belaqi_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + belaqi_dm3 + belaqi_dm3 + + + belaqi_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_dm1 + no2_dm1 + + + no2_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_dm2 + no2_dm2 + + + no2_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_dm3 + no2_dm3 + + + no2_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + no2_hmean + no2_hmean + + + no2_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_dm1 + o3_dm1 + + + o3_max8hmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_dm2 + o3_dm2 + + + o3_max8hmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_dm3 + o3_dm3 + + + o3_max8hmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + o3_hmean + o3_hmean + + + o3_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_dm1 + pm10_dm1 + + + pm10_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_dm2 + pm10_dm2 + + + pm10_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_dm3 + pm10_dm3 + + + pm10_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm10_hmean + pm10_hmean + + + pm10_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_dm1 + pm25_dm1 + + + pm25_dmean_dm1 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_dm2 + pm25_dm2 + + + pm25_dmean_dm2 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_dm3 + pm25_dm3 + + + pm25_dmean_dm3 + WCS + GeoTIFF + + EPSG:31370 + + + + + + pm25_hmean + pm25_hmean + + + pm25_hmean + WCS + GeoTIFF + + EPSG:31370 + + + + + + + diff --git a/tests/test_api_forecast.py b/tests/test_api_forecast.py index 628636a..4a5a703 100644 --- a/tests/test_api_forecast.py +++ b/tests/test_api_forecast.py @@ -95,7 +95,7 @@ async def test_api_forecast(): client = IrcelineForecastClient(session) features = [ForecastFeature.NO2_DMEAN, ForecastFeature.O3_MAXHMEAN] - _ = await client.get_data(features, pos) + result = await client.get_data(features, pos) base = {"service": "WMS", "version": "1.1.1", @@ -119,6 +119,9 @@ async def test_api_forecast(): session.request.assert_has_calls(calls, any_order=True) + for k, v in result.items(): + assert v['value'] == 10.853286743164062 + def test_parse_capabilities_with_error(): result = IrcelineForecastClient._parse_capabilities("wow there no valid XML") diff --git a/tests/test_api_rio.py b/tests/test_api_rio.py index ec41631..f7dc1f5 100644 --- a/tests/test_api_rio.py +++ b/tests/test_api_rio.py @@ -64,7 +64,7 @@ async def test_format_result_dmean(): def test_parse_capabilities(): - data = get_api_data('capabilities.xml', plain=True) + data = get_api_data('rio_capabilities.xml', plain=True) result = IrcelineRioClient._parse_capabilities(data) expected = {'rio:so2_anmean_be', 'rio:o3_hmean', 'rio:bc_anmean_vl', 'rio:o3_anmean_be', 'rio:pm10_hmean_vl', @@ -117,7 +117,7 @@ async def test_api_rio(): async def test_api_rio_get_capabilities(): - session = get_mock_session(text_file='capabilities.xml') + session = get_mock_session(text_file='rio_capabilities.xml') client = IrcelineRioClient(session) _ = await client.get_capabilities() diff --git a/tests/test_api_rio_ifdm.py b/tests/test_api_rio_ifdm.py new file mode 100644 index 0000000..7b92fdc --- /dev/null +++ b/tests/test_api_rio_ifdm.py @@ -0,0 +1,102 @@ +from datetime import datetime +from unittest.mock import call + +from freezegun import freeze_time + +from src.open_irceline.api import _rio_ifdm_wms_base_url, _user_agent +from src.open_irceline.data import RioIfdmFeature, FeatureValue +from src.open_irceline.rio import IrcelineRioIfdmClient +from tests.conftest import get_api_data, get_mock_session + + +def test_parse_capabilities(): + data = get_api_data('rio_ifdm_capabilities.xml', plain=True) + result = IrcelineRioIfdmClient._parse_capabilities(data) + + expected = {'no2_dm3', 'belaqi_dm2', 'pm10_hmean', 'belaqi_dm1', 'pm25_dm3', 'pm25_dm2', 'pm10_dm1', 'o3_dm3', + 'no2_dm1', 'pm10_dm3', 'pm25_dm1', 'belaqi', 'belaqi_dm3', 'pm10_dm2', 'o3_dm2', 'pm25_hmean', 'o3_dm1', + 'o3_hmean', 'no2_dm2', 'no2_hmean'} + + assert result == expected + + for f in RioIfdmFeature: + assert f"{f.split(':')[1]}" in result + + +async def test_aget_capabilities(): + session = get_mock_session(text_file='rio_ifdm_capabilities.xml') + + client = IrcelineRioIfdmClient(session) + _ = await client.get_capabilities() + + session.request.assert_called_once_with( + method='GET', + url=_rio_ifdm_wms_base_url, + params={"service": "WMS", + "version": "1.1.1", + "request": "GetCapabilities"}, + headers={'User-Agent': _user_agent} + ) + + +@freeze_time(datetime.fromisoformat("2024-06-30T13:00:21.520Z")) +async def test_api_forecast_error(): + pos = (50.4657, 4.8647) + session = get_mock_session('forecast_wms_feature_info_invalid.json') + + client = IrcelineRioIfdmClient(session) + + features = [RioIfdmFeature.NO2_HMEAN, RioIfdmFeature.O3_HMEAN] + result = await client.get_data(features, pos) + + for k, v in result.items(): + assert v == FeatureValue(timestamp=datetime.fromisoformat("2024-06-30T13:00:21.520Z"), value=None) + + +async def test_api_forecast(): + pos = (50.4657, 4.8647) + lat, lon = pos + session = get_mock_session('forecast_wms_feature_info.json') + + client = IrcelineRioIfdmClient(session) + + features = [RioIfdmFeature.NO2_HMEAN, RioIfdmFeature.O3_HMEAN] + result = await client.get_data(features, pos) + + base = {"service": "WMS", + "version": "1.1.1", + "request": "GetFeatureInfo", + "info_format": "application/json", + "width": "1", + "height": "1", + "srs": "EPSG:4326", + "bbox": f"{lon},{lat},{lon + 0.00001},{lat + 0.00001}", + "X": "1", + "Y": "1"} + + calls = [call( + method='GET', + url=_rio_ifdm_wms_base_url, + params=base | {"layers": f"{feature}", + "query_layers": f"{feature}"}, + headers={'User-Agent': _user_agent}, + ) + for feature in features] + + session.request.assert_has_calls(calls, any_order=True) + + for k, v in result.items(): + assert v['value'] == 10.853286743164062 + + +async def test_api_forecast_no_field(): + pos = (50.4657, 4.8647) + session = get_mock_session('forecast_wms_feature_info_no_field.json') + + client = IrcelineRioIfdmClient(session) + + features = [RioIfdmFeature.NO2_HMEAN, RioIfdmFeature.O3_HMEAN] + result = await client.get_data(features, pos) + + for k, v in result.items(): + assert v == FeatureValue(timestamp=None, value=None)