Implement caching based on ETag

src/open_irceline/api.py

@@ -10,6 +10,7 @@ from aiohttp import ClientResponse
 
 from . import project_transform, rio_wfs_base_url, user_agent
 from .data import RioFeature, FeatureValue
+from .utils import SizedDict
 
 
 class IrcelineApiError(Exception):
@@ -19,8 +20,9 @@ class IrcelineApiError(Exception):
 class IrcelineBaseClient:
     def __init__(self, session: aiohttp.ClientSession) -> None:
         self._session = session
+        self._cache = SizedDict(20)
 
-    async def _api_wrapper(self, url: str, querystring: dict, method: str = 'GET'):
+    async def _api_wrapper(self, url: str, querystring: dict = None, headers: dict = None, method: str = 'GET'):
         """
         Call the URL with the specified query string. Raises exception for >= 400 response code
         :param url: base URL
@@ -28,7 +30,10 @@ class IrcelineBaseClient:
         :return: response from the client
         """
 
-        headers = {'User-Agent': user_agent}
+        if headers is None:
+            headers = dict()
+        if 'User-Agent' not in headers:
+            headers |= {'User-Agent': user_agent}
 
         try:
             async with async_timeout.timeout(60):
@@ -48,6 +53,20 @@ class IrcelineBaseClient:
         except Exception as exception:  # pylint: disable=broad-except
             raise IrcelineApiError(f"Something really wrong happened! {exception}") from exception
 
+    async def _api_cached_wrapper(self, url: str, method: str = 'GET'):
+        if url in self._cache:
+            headers = {"If-None-Match": f'{self._cache.get(url, {}).get("etag")}'}
+        else:
+            headers = None
+
+        r: ClientResponse = await self._api_wrapper(url, headers=headers, method=method)
+        if r.status == 304:
+            return self._cache.get(url, {}).get("response")
+        elif 'ETag' in r.headers:
+            self._cache[url] = {'etag': r.headers['ETag'],
+                                'response': r}
+        return r
+
 
 class IrcelineRioClient(IrcelineBaseClient):
     """API client for RIO interpolated IRCEL - CELINE open data"""

src/open_irceline/utils.py

@@ -12,11 +12,15 @@ class SizedDict(OrderedDict):
         super().__setitem__(key, value)
         self.move_to_end(key)
         if len(self) > self._size:
-            self.popitem(False)
+            print('drop', self.popitem(False)[0])
 
     def __getitem__(self, key):
-        super().__getitem__(key)
         self.move_to_end(key)
+        return super().__getitem__(key)
+
+    def get(self, __key, __default=None):
+        self.move_to_end(__key)
+        return super().get(__key, __default)
 
     def update(self, __m, **kwargs):
         raise NotImplementedError()

tests/test_sized_dict.py

@@ -14,14 +14,14 @@ def test_size_dict():
 
     s_dict['f'] = 6
     assert 'a' not in s_dict
-    assert 'f' in s_dict
+    assert s_dict['f'] == 6
     assert len(s_dict) == 5
 
     s_dict['b'] = 42
     s_dict['g'] = 7
-    assert 'f' in s_dict
+    assert s_dict['f'] == 6
-    assert 'g' in s_dict
+    assert s_dict['g'] == 7
-    assert 'b' in s_dict
+    assert s_dict['b'] == 42
     assert 'c' not in s_dict
     assert len(s_dict) == 5