diff --git a/.gitignore b/.gitignore index a054264..d901686 100644 --- a/.gitignore +++ b/.gitignore @@ -1,134 +1,88 @@ -.idea/ -__pycache__/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder +# Generated by Cargo +# will have compiled files and executables +debug/ target/ -# Jupyter Notebook -.ipynb_checkpoints +# These are backup files generated by rustfmt +**/*.rs.bk -# IPython -profile_default/ -ipython_config.py +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb -# pyenv -.python-version +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ +# AWS User-specific +.idea/**/aws.xml -# Celery stuff -celerybeat-schedule -celerybeat.pid +# Generated files +.idea/**/contentModel.xml -# SageMath parsed files -*.sage.py +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ +# Gradle +.idea/**/gradle.xml +.idea/**/libraries -# Spyder project settings -.spyderproject -.spyproject +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr -# Rope project settings -.ropeproject +# CMake +cmake-build-*/ -# mkdocs documentation -/site +# Mongo Explorer plugin +.idea/**/mongoSettings.xml -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json +# File-based project format +*.iws -# Pyre type checker -.pyre/ -app/config/calendar.json -/app/cache/ +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/ics-fusion.iml b/.idea/ics-fusion.iml new file mode 100644 index 0000000..d6ebd48 --- /dev/null +++ b/.idea/ics-fusion.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..1c03483 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/server.py b/app/server.py deleted file mode 100644 index ea2665c..0000000 --- a/app/server.py +++ /dev/null @@ -1,30 +0,0 @@ -from flask import Flask, make_response - -from tools.caching import CacheThread -from tools.tools import * - -app = Flask(__name__) - - -@app.route('/') -def main(calendar): - conf = calendar + ".json" - - print("Opening " + conf) - - try: - result = str(process(conf)) - response = make_response(result, 200) - response.headers["Content-Disposition"] = "attachment; filename=calendar.ics" - response.headers["Content-Type"] = "text/calendar; charset=utf-8" - - except FileNotFoundError: - response = make_response("Calendar not cached", 425) - - return response - - -thread = CacheThread() -thread.start() - -app.run(host='0.0.0.0', port=8088) diff --git a/app/tools/__init__.py b/app/tools/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/tools/caching.py b/app/tools/caching.py deleted file mode 100644 index f88463a..0000000 --- a/app/tools/caching.py +++ /dev/null @@ -1,151 +0,0 @@ -import json -import os -import sched -import threading -import time -from hashlib import sha256 - -import traceback -import arrow -import requests -from ics import Calendar -from tatsu.exceptions import FailedParse -from tools.tools import horodate, process - - -def cache(entry: dict, scheduler: sched.scheduler = None) -> None: - """Cache an .ics feed in the app/cache directory. - Different entries with the same URL will be cached in the same file. - The cached calendar contains a new line in the description with the current time when cached prefixed by the - 'Cached at' mention - - - - :param entry: representation of the entry to cache. This is the Python representation of the corresponding entry - in the config file - :type entry: dict - - :param scheduler: scheduler used to relaunch the caching task in the future. If not scheduler is specified, - the task will not be relaunched - :type scheduler: sched.scheduler - """ - - try: - if not os.path.isdir('app/cache'): - os.mkdir('app/cache') - - url = entry['url'] - path = "app/cache/" + sha256(url.encode()).hexdigest() + ".ics" - - r = requests.get(entry["url"], allow_redirects=True) - - if "encoding" in entry: - cal = Calendar(imports=r.content.decode(encoding=entry["encoding"])) - else: - cal = Calendar(imports=r.content.decode()) - - cal = horodate(cal, 'Cached at') - open(path, 'w').writelines(cal) - print(arrow.now().format("YYYY-MM-DD HH:mm:ss"), "Cached", entry['name']) - - except FailedParse: - print("Could not parse", entry['name']) - - # Save stack trace when an unknown error occurs - except Exception as e: - with open("error " + arrow.now().format("YYYY-MM-DD HH:mm:ss")+".txt", 'w') as file: - file.write(arrow.now().format("YYYY-MM-DD HH:mm:ss") + "\nCould not cache : " + str(entry)) - file.write(str(e)) - file.write(str(traceback.format_exc())) - finally: - if scheduler is not None: - delay = entry['cache'] if entry['cache'] > 0 else 10 - delay *= 60 - scheduler.enter(delay=delay, priority=1, action=cache, argument=(entry, scheduler)) - - -def precompute(config: str, scheduler: sched.scheduler = None) -> None: - """Precompute a configuration file result to serve it faster when it is requested. This function - should be used with a scheduler to be repeated over time. - - :param config: name of the configuration file to precompute the result for - :type config: str - - scheduler used to relaunch the precomputing task in the future. If not scheduler is specified, - the task will not be relaunched - :type scheduler: sched.scheduler - """ - try: - cal = process(os.path.basename(config), False) - path = "app/cache/" + os.path.basename(config).rstrip('.json') + ".ics" - open(path, 'w').writelines(cal) - print(arrow.now().format("YYYY-MM-DD HH:mm:ss"), "Precomputed", os.path.basename(config).rstrip('.json')) - - except Exception as e: - with open("error " + arrow.now().format("YYYY-MM-DD HH:mm:ss")+".txt", 'w') as file: - file.write(arrow.now().format("YYYY-MM-DD HH:mm:ss") + "\nCould not precompute : " + str(config)) - file.write(str(e)) - file.write(str(traceback.format_exc())) - finally: - if scheduler is not None: - delay = get_min_cache(config) - delay *= 60 - scheduler.enter(delay=delay, priority=1, action=precompute, argument=(config, scheduler)) - - -def get_min_cache(path: str) -> float: - """Get the minimum caching time of all the entries in a config file. - - :param path: path of the config file to use - :type path: str - - :return: float number representing the smallest caching time. - """ - result = float('inf') - - with open(path, 'r') as config_file: - file = json.loads(config_file.read()) - - for entry in file: - if 'cache' in entry and entry['cache'] < result: - result = entry['cache'] - - return result - - -def start_scheduler(scheduler: sched.scheduler) -> None: - """Start the caching of every config file found in the app/config directory - - - :param scheduler: scheduler object to use to schedule the caching - :type scheduler: sched.scheduler - """ - - path = "app/config" - files = [os.path.join(path, f) for f in os.listdir(path) - if os.path.isfile(os.path.join(path, f)) and f.endswith('.json')] - - for file in files: - with open(file, 'r') as config_file: - config = json.loads(config_file.read()) - - for entry in config: - if 'cache' in entry: - scheduler.enter(delay=0, priority=1, action=cache, argument=(entry, scheduler)) - - if get_min_cache(file) < float('inf'): - scheduler.enter(delay=get_min_cache(file)*60, priority=1, action=precompute, argument=(file, scheduler)) - - scheduler.run() - - -class CacheThread(threading.Thread): - """Child class of the threading.Thread class to run the caching process every 10 minutes - """ - - def __init__(self): - threading.Thread.__init__(self) - - def run(self): - print("Starting cache process") - start_scheduler(sched.scheduler(time.time, time.sleep)) diff --git a/app/tools/tools.py b/app/tools/tools.py deleted file mode 100644 index e0aa8e2..0000000 --- a/app/tools/tools.py +++ /dev/null @@ -1,434 +0,0 @@ -"""This module provides methods to combines multiples .ics feeds into a single object. - -The methods allow to filter the events to keep in the final object and to modify the remaining event -according to a configuration file. - -The JSON configuration file used in this module has the following structure - -[ - { - "url":"str", - "name":"str", - "encoding":"str", - "filters":{ - "name":{ - "exclude":"RegEx", - "includeOnly":"RegEx", - "ignoreCase":true - }, - "description":{ - "exclude":"RegEx", - "includeOnly":"RegEx", - "ignoreCase":true - } - }, - "modify":{ - "time":{ - "shift":{ - "year":0, - "month":0, - "day":0, - "hour":0, - "minute":0 - } - }, - "name":{ - "addPrefix":"str", - "addSuffix":"str" - }, - "description":{ - "addPrefix":"str", - "addSuffix":"str" - }, - "location":{ - "addPrefix":"str", - "addSuffix":"str" - } - } - } -] - -Only the url and the name field are mandatory. -- url: specify the url to find the calendar -- name: name to identify the calendar -- encoding: specify the encoding to use - -- filters: structure defining the filters to apply to the calendar -- name: filters to apply to the name field of the events -- description: filters to apply to the name field of the events -- exclude: RegEx to describe the events to exclude - cannot be specified with includeOnly -- includeOnly: RegEx to describe the events to include - cannot be specified with exclude -- ignoreCase: if true the RegEx will ignore the case of the field - -- modify: structure defining the modifications to the events of the calendar -- time: describe the modifications to apply to the timing of the event -- shift: shift the event of a certain amount of time -- year, month, day, hour, minute: amount of time to add to the events -- name: modifications to apply to the name of the events -- description: modifications to apply to the description of the events -- location: modification to apply to the location of the events -- addPrefix: string to add at the beginning of the field -- addSuffix: string to add at the end of the field -""" - -import json -import re -import arrow -import os -from hashlib import sha256 -from typing import List - -import requests -from ics import Calendar -from pathvalidate import sanitize_filename - - -def filtering(cal: Calendar, filters: dict, field_name: str) -> Calendar: - """Filter the event of a calendar according to the filters and the field_name - - - :param cal: the calendar to apply filters to - :type cal: Calendar - - :param filters: the filters to apply to the calendar - :type filters: dict - - :param field_name: the of the field in the filters to consider - :type field_name: str - - - :return: the modified cal argument after filtering out the events - :rtype: Calendar - - - :raises SyntaxError: if both exclude and includeOnly are specified in the filters - """ - - if field_name in filters: - field = filters[field_name] - - if ("exclude" in field) and ("includeOnly" in field): - raise SyntaxError("Cannot specify both exclude and includeOnly") - - if ("exclude" not in field) and ("includeOnly" not in field): - return cal - - new = Calendar() - - ignore_case = True if ("ignoreCase" in field and field["ignoreCase"]) else False - - if "exclude" in field: - p = re.compile(field["exclude"], re.IGNORECASE | re.DOTALL) \ - if ignore_case else re.compile(field["exclude"], re.DOTALL) - - for event in cal.events: - if event.name is None or (field_name == "name" and p.match(event.name) is None): - new.events.add(event) - elif event.description is None or (field_name == "description" and p.match(event.description) is None): - new.events.add(event) - - if "includeOnly" in field: - p = re.compile(field["includeOnly"], re.IGNORECASE | re.DOTALL) \ - if ignore_case else re.compile(field["includeOnly"], re.DOTALL) - - for event in cal.events: - if field_name == "name" and event.name is not None and p.match(event.name) is not None: - new.events.add(event) - elif field_name == "description" and event.description is not None \ - and p.match(event.description) is not None: - new.events.add(event) - - cal = new - return cal - - else: - return cal - - -def apply_filters(cal: Calendar, filters: dict) -> Calendar: - """Apply all the filters to a calendar and returns the resulting calendar - - - :param cal: the calendar to apply filters to - :type cal: Calendar - - :param filters: the filters to apply - :type filters: dict - - - :return: the modified cal parameter to satisfy the filters - :rtype: Calendar - - :raises SyntaxError: if both exclude and includeOnly are specified for the same field in the filters - """ - - cal = filtering(cal, filters, "name") - cal = filtering(cal, filters, "description") - - return cal - - -def modify_time(cal: Calendar, modify: dict) -> Calendar: - """Modify the time of all the events in a calendar as specified in the modify structure - - - :param cal: the calendar where it is needed to modify the time of the events - :type cal: Calendar - - :param modify: the structure defining how to modify the time - :type modify: dict - - - :return: the modified cal parameter - :rtype: Calendar - """ - - if ("time" in modify) and ("shift" in modify["time"]): - shift = modify["time"]["shift"] - - year = 0 if not ("year" in shift) else shift["year"] - month = 0 if not ("month" in shift) else shift["month"] - day = 0 if not ("day" in shift) else shift["day"] - hour = 0 if not ("hour" in shift) else shift["hour"] - minute = 0 if not ("minute" in shift) else shift["minute"] - - for event in cal.events: - event.end = event.end.shift(years=year, months=month, days=day, hours=hour, minutes=minute) - event.begin = event.begin.shift(years=year, months=month, days=day, hours=hour, minutes=minute) - - return cal - - -def modify_text(cal: Calendar, modify: dict, field_name: str) -> Calendar: - """Modify one text field (name, location, description) of all the events in the cal parameter - according to the modify structure and the field_name - - - :param cal: the calendar where it is needed to modify the text field - :type cal: Calendar - - :param modify: the structure defining how to modify the time - :type modify: dict - - :param field_name: the name of the field to modify - :type field_name: str - - - :return: the modified cal parameter - :rtype: Calendar - """ - - if field_name in modify: - change = modify[field_name] - - if "addPrefix" in change: - for event in cal.events: - - if field_name == "name": - event.name = change["addPrefix"] + event.name \ - if event.name is not None else change["addPrefix"] - - elif field_name == "description": - event.description = change["addPrefix"] + event.description \ - if event.description is not None else change["addPrefix"] - - elif field_name == "location": - event.location = change["addPrefix"] + event.location \ - if event.location is not None else change["addPrefix"] - - if "addSuffix" in change: - for event in cal.events: - - if field_name == "name": - event.name = event.name + change["addSuffix"] \ - if event.name is not None else change["addSuffix"] - - elif field_name == "description": - event.description = event.description + change["addSuffix"] \ - if event.description is not None else change["addSuffix"] - - elif field_name == "location": - event.location = event.location + change["addSuffix"] \ - if event.location is not None else change["addSuffix"] - - return cal - - -def apply_modify(cal: Calendar, modify: dict) -> Calendar: - """Apply all the needed modifications to a calendar and returns the resulting calendar - - - :param cal: the calendar to apply modifications to - :type cal: Calendar - - :param modify: the structure containing the modifications to apply - :type modify: dict - - - :return: the modified cal parameter - :rtype: Calendar - """ - - cal = modify_time(cal, modify) - cal = modify_text(cal, modify, "name") - cal = modify_text(cal, modify, "description") - cal = modify_text(cal, modify, "location") - return cal - - -def merge(cals: List[Calendar]) -> Calendar: - """Merge a list of calendars into a single calendar - Only takes the event into account, not the tasks or the alarms - - - :param cals: the list of calendars to merge - :type cals: List[Calendar] - - - :return: the calendar containing the union of the events contained in the cals list - :rtype: Calendar - - - :raises ValueError: if an element of the list is not a Calendar - """ - - result = Calendar() - - for cal in cals: - if not isinstance(cal, Calendar): - raise ValueError("All elements should be Calendar") - result.events = result.events.union(cal.events) - - return result - - -def process(path: str, from_cache: bool = True) -> Calendar: - """Open a config file from the specified path, download the calendars, - apply the filters, modify and merge the calendars as specified in the config file - - - :param from_cache: - :param path: name of the file to open. The file should be in the config/ folder - :type path: str - - - :return: the resulting calendar - :rtype: Calendar - """ - print("app/cache/" + sanitize_filename(path).rstrip(".json") + ".ics") - if from_cache and os.path.isfile("app/cache/" + sanitize_filename(path).rstrip(".json") + ".ics"): - with open("app/cache/" + sanitize_filename(path).rstrip(".json") + ".ics") as file: - data = file.read() - print("Serving precomputed file") - return Calendar(imports=data) - - else: - o = "app/config/" + sanitize_filename(path) - print("Try to open " + o) - file = open(o, "r") - config = json.loads(file.read()) - file.close() - - data = [] - - for entry in config: - - cal = load_cal(entry) - - if "filters" in entry: - cal = apply_filters(cal, entry["filters"]) - - if "modify" in entry: - cal = apply_modify(cal, entry["modify"]) - - data.append(cal) - - return merge(data) - - -def get_from_cache(entry: dict) -> Calendar: - """Retrieve the entry from cache. If the entry is not found, an exception is raised - - - :param entry: representation of the entry to cache. This is the Python representation of the corresponding entry - in the config file - :type entry: dict - - - :return: the corresponding calendar in cache - :rtype: Calendar - - - :raises FileNotfoundError: if the entry has not been cached before - """ - - url = entry['url'] - path = "app/cache/" + sha256(url.encode()).hexdigest() + ".ics" - if not os.path.isfile(path): - print("Not cached") - raise FileNotFoundError("The calendar is not cached") - - with open(path, 'r') as file: - data = file.read() - - return Calendar(imports=data) - - -def load_cal(entry: dict) -> Calendar: - """Load the calendar from the cache or from remote according to the entry. If the calendar is supposed to be in - cached but could not be found in cache, an error is thrown - - - :param entry: representation of the entry to cache. This is the Python representation of the corresponding entry - in the config file - :type entry: dict - - - :return: the calendar corresponding to the entry - :rtype: Calendar - - - :raises FileNotfoundError: if the entry was supposed to be cached but has not been cached before - """ - - if "cache" in entry and entry["cache"]: - print("Getting", entry["name"], "from cache") - try: - return get_from_cache(entry) - except FileNotFoundError: - return Calendar() - - else: - print("Getting", entry["name"], "from remote") - r = requests.get(entry["url"], allow_redirects=True) - if "encoding" in entry: - cal = Calendar(imports=r.content.decode(encoding=entry["encoding"])) - else: - cal = Calendar(imports=r.content.decode()) - - cal = horodate(cal, 'Downloaded at') - return cal - - -def horodate(cal: Calendar, prefix='') -> Calendar: - """Add a new line at the end of the description of every event in the calendar with the current time prefixed by - the prefix parameter and a space - The date is added with the following format: YYYY-MM-DD HH:mm:ss - - - :param cal: calendar to process - :type cal: Calendar - - :param prefix: the prefix to add in front of the date - :type prefix: str - - - :return: the modified calendar - :rtype: Calendar - """ - now = arrow.now().format("YYYY-MM-DD HH:mm:ss") - for event in cal.events: - event.description = event.description + '\n' + prefix + ' ' + now \ - if event.description is not None else prefix + ' ' + now - - return cal diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c3b9d5a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -requests~=2.22.0 -ics~=0.7 -pathvalidate~=2.3.0 -flask~=1.1.1 -arrow~=0.14.7 -tatsu~=4.4.0 \ No newline at end of file