From 6be82c942a7eed293246fe4e7ebc1f87dc220bfa Mon Sep 17 00:00:00 2001 From: Jules Dejaeghere Date: Wed, 19 Jun 2024 23:05:18 +0200 Subject: [PATCH] Add tests to improve code coverage --- src/open_irceline/__init__.py | 2 +- src/open_irceline/api.py | 8 ++-- tests/conftest.py | 47 ++++++++++++++++++++ tests/test_api_forecasts.py | 80 ++++++++++++++++++++++++++++++++++- tests/test_api_rio.py | 46 +++++++++++++++++++- 5 files changed, 175 insertions(+), 8 deletions(-) diff --git a/src/open_irceline/__init__.py b/src/open_irceline/__init__.py index c255431..eae1097 100644 --- a/src/open_irceline/__init__.py +++ b/src/open_irceline/__init__.py @@ -1,7 +1,7 @@ from pyproj import Transformer project_transform = Transformer.from_crs('EPSG:4326', 'EPSG:31370', always_xy=False) -rio_wfs_base_url = 'https://geo.irceline.be/rio/wfs' +rio_wfs_base_url = 'https://geo.irceline.be/wfs' # noinspection HttpUrlsUsage # There is not HTTPS version of this endpoint forecast_base_url = 'http://ftp.irceline.be/forecast' diff --git a/src/open_irceline/api.py b/src/open_irceline/api.py index a3d9a29..b46f470 100644 --- a/src/open_irceline/api.py +++ b/src/open_irceline/api.py @@ -92,7 +92,6 @@ class IrcelineRioClient(IrcelineBaseClient): :param position: decimal degrees pair of coordinates :return: dict with the response (key is RioFeature, value is FeatureValue with actual value and timestamp) """ - # Remove one hour/day from timestamp to handle case where the hour just passed but the data is not yet there # (e.g. 5.01 PM, but the most recent data is for 4.00 PM) if isinstance(timestamp, datetime): @@ -106,7 +105,7 @@ class IrcelineRioClient(IrcelineBaseClient): else: raise IrcelineApiError(f"Wrong parameter type for timestamp: {type(timestamp)}") - coord = epsg_transform(position) + lat, lon = epsg_transform(position) querystring = {"service": "WFS", "version": "1.3.0", "request": "GetFeature", @@ -115,8 +114,7 @@ class IrcelineRioClient(IrcelineBaseClient): "cql_filter": f"{key}>='{timestamp}'" f" AND " - f"INTERSECTS(the_geom, POINT ({coord[0]} {coord[1]}))"} - + f"INTERSECTS(the_geom, POINT ({lat} {lon}))"} r: ClientResponse = await self._api_wrapper(rio_wfs_base_url, querystring) return self._format_result('rio', await r.json(), features) @@ -212,7 +210,6 @@ class IrcelineForecastClient(IrcelineBaseClient): :return: dict where key is (ForecastFeature, date of the forecast) and value is a FeatureValue """ x, y = round_coordinates(position[0], position[1]) - result = dict() for feature, d in product(features, range(5)): @@ -223,6 +220,7 @@ class IrcelineForecastClient(IrcelineBaseClient): except IrcelineApiError: # retry for the day before yesterday = day - timedelta(days=1) + print('here') url = f"{forecast_base_url}/BE_{feature}_{yesterday.strftime('%Y%m%d')}_d{d}.csv" try: r: ClientResponse = await self._api_cached_wrapper(url) diff --git a/tests/conftest.py b/tests/conftest.py index da3b13f..626601d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,8 @@ import json +from unittest.mock import Mock, AsyncMock + +import aiohttp + def get_api_data(fixture: str, plain=False) -> str | dict: @@ -6,3 +10,46 @@ def get_api_data(fixture: str, plain=False) -> str | dict: if plain: return file.read() return json.load(file) + + +def get_mock_session_json(json_file=None, text_file=None): + # Create the mock response + mock_response = Mock() + if json_file is not None: + mock_response.json = AsyncMock(return_value=get_api_data(json_file)) + if text_file is not None: + mock_response.text = AsyncMock(return_value=get_api_data(text_file, plain=True)) + + mock_response.status = 200 + mock_response.headers = dict() + + # Create the mock session + mock_session = Mock(aiohttp.ClientSession) + mock_session.request = AsyncMock(return_value=mock_response) + return mock_session + + +def create_mock_response(*args, **kwargs): + etag = 'my-etag-here' + mock_response = Mock() + if '20240619' not in kwargs.get('url', ''): + mock_response.status = 404 + mock_response.raise_for_status = Mock(side_effect=aiohttp.ClientResponseError(Mock(), Mock())) + elif etag in kwargs.get('headers', {}).get('If-None-Match', ''): + mock_response.text = AsyncMock(return_value='') + mock_response.status = 304 + else: + mock_response.text = AsyncMock(return_value=get_api_data('forecast.csv', plain=True)) + mock_response.status = 200 + + if '20240619' in kwargs.get('url', ''): + mock_response.headers = {'ETag': etag} + else: + mock_response.headers = dict() + return mock_response + + +def get_mock_session_many_csv(): + mock_session = Mock(aiohttp.ClientSession) + mock_session.request = AsyncMock(side_effect=create_mock_response) + return mock_session diff --git a/tests/test_api_forecasts.py b/tests/test_api_forecasts.py index ba27c98..df61357 100644 --- a/tests/test_api_forecasts.py +++ b/tests/test_api_forecasts.py @@ -1,5 +1,10 @@ +from datetime import date +from unittest.mock import call + +from src.open_irceline import forecast_base_url, user_agent from src.open_irceline.api import IrcelineForecastClient -from tests.conftest import get_api_data +from src.open_irceline.data import ForecastFeature +from tests.conftest import get_api_data, get_mock_session_many_csv def test_extract_from_csv(): @@ -11,3 +16,76 @@ def test_extract_from_csv(): result = IrcelineForecastClient.extract_result_from_csv(23, 4, data) assert result is None + + +async def test_cached_calls(): + session = get_mock_session_many_csv() + client = IrcelineForecastClient(session) + + _ = await client.get_forecasts( + day=date(2024, 6, 19), + features=[ForecastFeature.NO2_MAXHMEAN], + position=(50.45, 4.85) + ) + + calls = [ + call(method='GET', + url=f"{forecast_base_url}/BE_{ForecastFeature.NO2_MAXHMEAN}_20240619_d{i}.csv", + params=None, + headers={'User-Agent': user_agent} + ) for i in range(5) + ] + + assert session.request.call_count == 5 + session.request.assert_has_calls(calls) + + _ = await client.get_forecasts( + day=date(2024, 6, 19), + features=[ForecastFeature.NO2_MAXHMEAN], + position=(50.45, 4.85) + ) + + calls += [ + call(method='GET', + url=f"{forecast_base_url}/BE_{ForecastFeature.NO2_MAXHMEAN}_20240619_d{i}.csv", + params=None, + headers={'User-Agent': user_agent, 'If-None-Match': 'my-etag-here'} + ) for i in range(5) + ] + + assert session.request.call_count == 10 + session.request.assert_has_calls(calls) + + +async def test_missed_cached_calls(): + session = get_mock_session_many_csv() + client = IrcelineForecastClient(session) + + r = await client.get_forecasts( + day=date(2024, 6, 21), + features=[ForecastFeature.NO2_MAXHMEAN], + position=(50.45, 4.85) + ) + + calls = list() + + for i in range(5): + calls += [ + call(method='GET', + url=f"{forecast_base_url}/BE_{ForecastFeature.NO2_MAXHMEAN}_20240621_d{i}.csv", + params=None, + headers={'User-Agent': user_agent} + ), + call(method='GET', + url=f"{forecast_base_url}/BE_{ForecastFeature.NO2_MAXHMEAN}_20240620_d{i}.csv", + params=None, + headers={'User-Agent': user_agent} + ) + ] + + assert session.request.call_count == 10 + session.request.assert_has_calls(calls) + + for value in r.values(): + assert value['value'] is None + diff --git a/tests/test_api_rio.py b/tests/test_api_rio.py index 4fefa5a..df18014 100644 --- a/tests/test_api_rio.py +++ b/tests/test_api_rio.py @@ -2,9 +2,11 @@ from datetime import datetime, date from freezegun import freeze_time +from src.open_irceline import rio_wfs_base_url, user_agent from src.open_irceline.api import IrcelineRioClient from src.open_irceline.data import RioFeature, FeatureValue -from tests.conftest import get_api_data +from src.open_irceline.utils import epsg_transform +from tests.conftest import get_api_data, get_mock_session_json @freeze_time(datetime.fromisoformat("2024-06-15T16:55:03.419Z")) @@ -86,3 +88,45 @@ def test_parse_capabilities(): def test_parse_capabilities_with_error(): result = IrcelineRioClient._parse_capabilities("wow there no valid XML") assert result == set() + + +async def test_api_rio(): + pos = (50.4657, 4.8647) + x, y = epsg_transform(pos) + session = get_mock_session_json('rio_wfs.json') + + client = IrcelineRioClient(session) + + d = date(2024, 6, 18) + features = [RioFeature.NO2_HMEAN, RioFeature.O3_HMEAN] + _ = await client.get_rio_value(d, features, pos) + session.request.assert_called_once_with( + method='GET', + url=rio_wfs_base_url, + params={"service": "WFS", + "version": "1.3.0", + "request": "GetFeature", + "outputFormat": "application/json", + "typeName": ",".join(features), + "cql_filter": + f"date>='2024-06-17'" + f" AND " + f"INTERSECTS(the_geom, POINT ({x} {y}))"}, + headers={'User-Agent': user_agent} + ) + + +async def test_api_rio_get_capabilities(): + session = get_mock_session_json(text_file='capabilities.xml') + + client = IrcelineRioClient(session) + _ = await client.get_rio_capabilities() + + session.request.assert_called_once_with( + method='GET', + url=rio_wfs_base_url, + params={"service": "WFS", + "version": "1.3.0", + "request": "GetCapabilities"}, + headers={'User-Agent': user_agent} + )