From 4f0f111ed74fe3884f25bd047fbaeae2cacd2263 Mon Sep 17 00:00:00 2001 From: xelast418 Date: Thu, 7 Sep 2023 15:30:20 +1200 Subject: [PATCH 1/7] Adds support for redacting event details via a "redactAs" modify parameter --- .gitignore | 4 ++++ README.md | 1 + app/tools/tools.py | 22 +++++++++++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a054264..40a69eb 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,7 @@ dmypy.json .pyre/ app/config/calendar.json /app/cache/ + +#development directories +/app/config/ +.vscode diff --git a/README.md b/README.md index 259cab8..e8ac8f9 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Only the `url` and the `name` field are mandatory. - `location`: modification to apply to the location of the events - `addPrefix`: string to add at the beginning of the field - `addSuffix`: string to add at the end of the field +- `redactAs`: Replaces the content of the field with the specified string If multiple calendars are specified in the configuration list, their events will be merged in the resulting ics feed. diff --git a/app/tools/tools.py b/app/tools/tools.py index e0aa8e2..262666b 100644 --- a/app/tools/tools.py +++ b/app/tools/tools.py @@ -34,15 +34,18 @@ The JSON configuration file used in this module has the following structure }, "name":{ "addPrefix":"str", - "addSuffix":"str" + "addSuffix":"str", + "redactAs":"str" }, "description":{ "addPrefix":"str", - "addSuffix":"str" + "addSuffix":"str", + "redactAs":"str" }, "location":{ "addPrefix":"str", - "addSuffix":"str" + "addSuffix":"str", + "redactAs":"str" } } } @@ -69,6 +72,7 @@ Only the url and the name field are mandatory. - location: modification to apply to the location of the events - addPrefix: string to add at the beginning of the field - addSuffix: string to add at the end of the field +- redactAs: string to replace the field with """ import json @@ -250,6 +254,18 @@ def modify_text(cal: Calendar, modify: dict, field_name: str) -> Calendar: elif field_name == "location": event.location = event.location + change["addSuffix"] \ if event.location is not None else change["addSuffix"] + + if "redactAs" in change: + for event in cal.events: + + if field_name == "name": + event.name = change["redactAs"] + + elif field_name == "description": + event.description = change["redactAs"] + + elif field_name == "location": + event.location = change["redactAs"] return cal -- 2.45.3 From e5acc4fd1df8518a044d36b05138304eab6a2146 Mon Sep 17 00:00:00 2001 From: xelast418 Date: Mon, 18 Sep 2023 10:29:44 +1200 Subject: [PATCH 2/7] Minor updates: - Updates README.md with redactAs - Updates .gitignore to exclude developement files - Updates tools.py to reword the horodate prefix from "Downloaded at" to "Event last fetched" --- .gitignore | 2 +- README.md | 9 ++++++--- app/tools/tools.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 40a69eb..72cc436 100644 --- a/.gitignore +++ b/.gitignore @@ -133,6 +133,6 @@ dmypy.json app/config/calendar.json /app/cache/ -#development directories +# Development directories /app/config/ .vscode diff --git a/README.md b/README.md index e8ac8f9..4059092 100644 --- a/README.md +++ b/README.md @@ -59,15 +59,18 @@ The JSON configuration file should look like the following. }, "name":{ "addPrefix":"str", - "addSuffix":"str" + "addSuffix":"str", + "redactAs":"str" }, "description":{ "addPrefix":"str", - "addSuffix":"str" + "addSuffix":"str", + "redactAs":"str" }, "location":{ "addPrefix":"str", - "addSuffix":"str" + "addSuffix":"str", + "redactAs":"str" } } } diff --git a/app/tools/tools.py b/app/tools/tools.py index 262666b..91f253d 100644 --- a/app/tools/tools.py +++ b/app/tools/tools.py @@ -422,7 +422,7 @@ def load_cal(entry: dict) -> Calendar: else: cal = Calendar(imports=r.content.decode()) - cal = horodate(cal, 'Downloaded at') + cal = horodate(cal, 'Event last fetched: ') return cal -- 2.45.3 From 27064a9c61637235f0fbfec00162842ec54eddf6 Mon Sep 17 00:00:00 2001 From: xelast418 Date: Wed, 20 Sep 2023 20:50:37 +1200 Subject: [PATCH 3/7] Adds support for extending a calendar config with another calendar config. --- README.md | 16 ++++++++++++++ app/tools/tools.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/README.md b/README.md index 4059092..03043f8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ The JSON configuration file should look like the following. ```json [ + { + "conf": true, + "extends": "str", + "extendFail": "str", + }, { "url":"str", "name":"str", @@ -104,9 +109,20 @@ Only the `url` and the `name` field are mandatory. - `redactAs`: Replaces the content of the field with the specified string If multiple calendars are specified in the configuration list, their events will be merged in the resulting ics feed. +The first dataset with {"conf": true,} specifies options that are globally applied to all calenders in the conf. Omit this set to disable. Options +- `extends`: string specifying the name (excluding .json) of another config file to extend. +- `extendFail`: string speciying the action to take if an extend fails, either "fail" or "ignore". Default is "fail". ## Usage Once the config file is created, the corresponding HTTP endpoint is accessible. For example, if the file `app/config/my-calendar.json` contains the configuration, the HTTP endpoint will be `http://localhost:8088/my-calendar`. +A config can extend another config file, to do this the extended config should contain begin with`{ + "conf": true, + "extends": , + "extendFail": "fail", + },` +For an extend to work calendars MUST share the same name between the configs +An extending config cannot remove data from a base calendar but can modify fields + ## Limitations Currently, the application only merges events of the ics feeds, the alarms and todos are not supported. diff --git a/app/tools/tools.py b/app/tools/tools.py index 91f253d..ca5ec91 100644 --- a/app/tools/tools.py +++ b/app/tools/tools.py @@ -348,6 +348,26 @@ def process(path: str, from_cache: bool = True) -> Calendar: data = [] for entry in config: + if entry.get("conf", False) == True: + if entry.get("extends", None) is not None: + try: + o = "app/config/" + sanitize_filename(entry["extends"]) + ".json" + print("Try to open " + o) + file = open(o, "r") + baseConfig = json.loads(file.read()) + file.close() + extendingConfig = config + try: + config = merge_json(baseConfig, extendingConfig) + except: + config = extendingConfig + + except: + if entry.get("extendFail", "fail") == "fail": + raise FileNotFoundError("The calendar is not cached") + else: + pass + continue cal = load_cal(entry) @@ -448,3 +468,35 @@ def horodate(cal: Calendar, prefix='') -> Calendar: if event.description is not None else prefix + ' ' + now return cal + +def merge_json(base, extention): + """Merges two config files by updating the value of base with the values in extention. + + + :param base: the base config file + :type base: dict + + :param extention: the config file to merge with the base + :type extention: dict + + :return: the merged config file + :rtype: dict + + """ + + newJson = base.copy() + + def update_json(target, source): + + for key, value in source.items(): + if isinstance(value, dict) and key in target and isinstance(target[key], dict): + update_json(target[key], value) + else: + target[key] = value + + for dataset in newJson: + for dset in extention: + if newJson["name"] == dset["name"]: + update_json(newJson, dset) + + return newJson \ No newline at end of file -- 2.45.3 From 3975e37fa70e32d09d962e9535209714e0d6ea2e Mon Sep 17 00:00:00 2001 From: xelast418 Date: Wed, 20 Sep 2023 20:58:25 +1200 Subject: [PATCH 4/7] Updated README with fork credit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4059092..419cc52 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # ICS Fusion +Forked from [https://github.com/jdejaegh/ics-fusion](https://github.com/jdejaegh/ics-fusion) ## Introduction ICS Fusion is a tool to merge multiple ics feed into a single ics calendar. Filters and modifications may be applied on the incoming feeds. The resulting ics can be accessed via an HTTP endpoint. -- 2.45.3 From c4623f8808f63d72e6ae8ab2063af3c9ace10e9b Mon Sep 17 00:00:00 2001 From: xelast418 Date: Thu, 21 Sep 2023 09:49:07 +1200 Subject: [PATCH 5/7] Replaces True with yes for specifying additional conf to avoid JSON/Python conflict ``` --- README.md | 6 +++--- app/tools/tools.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f681268..609c0cb 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The JSON configuration file should look like the following. ```json [ { - "conf": true, + "conf": "str", "extends": "str", "extendFail": "str", }, @@ -110,7 +110,7 @@ Only the `url` and the `name` field are mandatory. - `redactAs`: Replaces the content of the field with the specified string If multiple calendars are specified in the configuration list, their events will be merged in the resulting ics feed. -The first dataset with {"conf": true,} specifies options that are globally applied to all calenders in the conf. Omit this set to disable. Options +The first dataset with {"conf": "yes",} specifies options that are globally applied to all calenders in the conf. Omit this set to disable. Options - `extends`: string specifying the name (excluding .json) of another config file to extend. - `extendFail`: string speciying the action to take if an extend fails, either "fail" or "ignore". Default is "fail". @@ -118,7 +118,7 @@ The first dataset with {"conf": true,} specifies options that are globally appli Once the config file is created, the corresponding HTTP endpoint is accessible. For example, if the file `app/config/my-calendar.json` contains the configuration, the HTTP endpoint will be `http://localhost:8088/my-calendar`. A config can extend another config file, to do this the extended config should contain begin with`{ - "conf": true, + "conf": "yes", "extends": , "extendFail": "fail", },` diff --git a/app/tools/tools.py b/app/tools/tools.py index ca5ec91..06aadac 100644 --- a/app/tools/tools.py +++ b/app/tools/tools.py @@ -348,7 +348,7 @@ def process(path: str, from_cache: bool = True) -> Calendar: data = [] for entry in config: - if entry.get("conf", False) == True: + if entry.get("conf", "no") == "yes": if entry.get("extends", None) is not None: try: o = "app/config/" + sanitize_filename(entry["extends"]) + ".json" -- 2.45.3 From 811382fbb724afd5819a6e198f6aa912b23ad749 Mon Sep 17 00:00:00 2001 From: xelast418 Date: Sat, 23 Sep 2023 17:50:23 +1200 Subject: [PATCH 6/7] Bug fix: Fixes extends throwing errors with unexpected values --- README.md | 6 +-- app/tools/tools.py | 93 +++++++++++++++++++++++----------------------- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 609c0cb..3570bcd 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The JSON configuration file should look like the following. ```json [ { - "conf": "str", + "conf": true, "extends": "str", "extendFail": "str", }, @@ -110,7 +110,7 @@ Only the `url` and the `name` field are mandatory. - `redactAs`: Replaces the content of the field with the specified string If multiple calendars are specified in the configuration list, their events will be merged in the resulting ics feed. -The first dataset with {"conf": "yes",} specifies options that are globally applied to all calenders in the conf. Omit this set to disable. Options +The first dataset with {"conf": "true",} specifies options that are globally applied to all calenders in the conf. Omit this set to disable. Options - `extends`: string specifying the name (excluding .json) of another config file to extend. - `extendFail`: string speciying the action to take if an extend fails, either "fail" or "ignore". Default is "fail". @@ -118,7 +118,7 @@ The first dataset with {"conf": "yes",} specifies options that are globally appl Once the config file is created, the corresponding HTTP endpoint is accessible. For example, if the file `app/config/my-calendar.json` contains the configuration, the HTTP endpoint will be `http://localhost:8088/my-calendar`. A config can extend another config file, to do this the extended config should contain begin with`{ - "conf": "yes", + "conf": true, "extends": , "extendFail": "fail", },` diff --git a/app/tools/tools.py b/app/tools/tools.py index 06aadac..5bc7348 100644 --- a/app/tools/tools.py +++ b/app/tools/tools.py @@ -346,29 +346,31 @@ def process(path: str, from_cache: bool = True) -> Calendar: file.close() data = [] - - for entry in config: - if entry.get("conf", "no") == "yes": - if entry.get("extends", None) is not None: + + for entry in config: + if "conf" in entry: + if entry.get("extends", None) is not None: + try: + o = "app/config/" + sanitize_filename(entry["extends"]) + ".json" + print("Try to open " + o) + file = open(o, "r") + baseConfig = json.loads(file.read()) + file.close() + extendingConfig = config try: - o = "app/config/" + sanitize_filename(entry["extends"]) + ".json" - print("Try to open " + o) - file = open(o, "r") - baseConfig = json.loads(file.read()) - file.close() - extendingConfig = config - try: - config = merge_json(baseConfig, extendingConfig) - except: - config = extendingConfig - + config = merge_json(baseConfig, extendingConfig) except: - if entry.get("extendFail", "fail") == "fail": - raise FileNotFoundError("The calendar is not cached") - else: - pass - continue + config = extendingConfig + + except: + if entry.get("extendFail", "fail") == "fail": + raise FileNotFoundError("The calendar is not cached") + else: + pass + + for entry in config: + if "conf" not in entry: cal = load_cal(entry) if "filters" in entry: @@ -379,7 +381,7 @@ def process(path: str, from_cache: bool = True) -> Calendar: data.append(cal) - return merge(data) + return merge(data) def get_from_cache(entry: dict) -> Calendar: @@ -469,34 +471,33 @@ def horodate(cal: Calendar, prefix='') -> Calendar: return cal -def merge_json(base, extention): - """Merges two config files by updating the value of base with the values in extention. - - +def merge_json(base, extension): + """Merges two config files by updating the value of base with the values in extension. + :param base: the base config file :type base: dict - - :param extention: the config file to merge with the base - :type extention: dict - + + :param extension: the config file to merge with the base + :type extension: dict + :return: the merged config file :rtype: dict - """ - - newJson = base.copy() - - def update_json(target, source): - - for key, value in source.items(): - if isinstance(value, dict) and key in target and isinstance(target[key], dict): - update_json(target[key], value) - else: - target[key] = value - for dataset in newJson: - for dset in extention: - if newJson["name"] == dset["name"]: - update_json(newJson, dset) - - return newJson \ No newline at end of file + new_json = base.copy() + + def update_json(base_set, updates): + for key, value in updates.items(): + if not key == "conf": + if isinstance(value, dict) and key in base_set and isinstance(base_set[key], dict): + update_json(base_set[key], value) + else: + base_set[key] = value + + for base_dataset in new_json: + if "conf" not in base_dataset: + for ext_dataset in extension: + if base_dataset.get("name") == ext_dataset.get("name"): + update_json(base_dataset, ext_dataset) + + return new_json -- 2.45.3 From 72adb809a4d4375334f9530b0f890c480f64ea11 Mon Sep 17 00:00:00 2001 From: xelast418 Date: Sat, 23 Sep 2023 20:11:10 +1200 Subject: [PATCH 7/7] Fixes modify/shift to allow a negative time shift --- app/tools/tools.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/tools/tools.py b/app/tools/tools.py index 5bc7348..c9f6154 100644 --- a/app/tools/tools.py +++ b/app/tools/tools.py @@ -196,9 +196,16 @@ def modify_time(cal: Calendar, modify: dict) -> Calendar: hour = 0 if not ("hour" in shift) else shift["hour"] minute = 0 if not ("minute" in shift) else shift["minute"] - for event in cal.events: - event.end = event.end.shift(years=year, months=month, days=day, hours=hour, minutes=minute) - event.begin = event.begin.shift(years=year, months=month, days=day, hours=hour, minutes=minute) + shift_minutes = (year * 365 * 24 * 60) + (month * 30 * 24 * 60) + (day * 24 * 60) + (hour * 60) + minute + + if shift_minutes > 0: + for event in cal.events: + event.end = event.end.shift(minutes=shift_minutes) + event.begin = event.begin.shift(minutes=shift_minutes) + elif shift_minutes < 0: + for event in cal.events: + event.begin = event.begin.shift(minutes=shift_minutes) + event.end = event.end.shift(minutes=shift_minutes) return cal -- 2.45.3