jdejaegh/ics-fusion
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Clean for rust project

Browse source
This commit is contained in:
Jules 2023-01-14 14:11:38 +01:00
parent 7613441836
commit 317f32565b
11 changed files with 105 additions and 741 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

194
.gitignore vendored
View file

@ -1,134 +1,88 @@
.idea/
__pycache__/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# Jupyter Notebook
.ipynb_checkpoints
# These are backup files generated by rustfmt
**/*.rs.bk
# IPython
profile_default/
ipython_config.py
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# pyenv
.python-version
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# AWS User-specific
.idea/**/aws.xml
# Celery stuff
celerybeat-schedule
celerybeat.pid
# Generated files
.idea/**/contentModel.xml
# SageMath parsed files
*.sage.py
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Spyder project settings
.spyderproject
.spyproject
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# Rope project settings
.ropeproject
# CMake
cmake-build-*/
# mkdocs documentation
/site
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# File-based project format
*.iws
# Pyre type checker
.pyre/
app/config/calendar.json
/app/cache/
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# SonarLint plugin
.idea/sonarlint/
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser

8
.idea/.gitignore generated vendored Normal file
View file

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

9
.idea/ics-fusion.iml generated Normal file
View file

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

8
.idea/modules.xml generated Normal file
View file

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/ics-fusion.iml" filepath="$PROJECT_DIR$/.idea/ics-fusion.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml generated Normal file
View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

0
app/__init__.py
View file

30
app/server.py
View file

@ -1,30 +0,0 @@
from flask import Flask, make_response
from tools.caching import CacheThread
from tools.tools import *
app = Flask(__name__)
@app.route('/<calendar>')
def main(calendar):
conf = calendar + ".json"
print("Opening " + conf)
try:
result = str(process(conf))
response = make_response(result, 200)
response.headers["Content-Disposition"] = "attachment; filename=calendar.ics"
response.headers["Content-Type"] = "text/calendar; charset=utf-8"
except FileNotFoundError:
response = make_response("Calendar not cached", 425)
return response
thread = CacheThread()
thread.start()
app.run(host='0.0.0.0', port=8088)

0
app/tools/__init__.py
View file

151
app/tools/caching.py
View file

@ -1,151 +0,0 @@
import json
import os
import sched
import threading
import time
from hashlib import sha256
import traceback
import arrow
import requests
from ics import Calendar
from tatsu.exceptions import FailedParse
from tools.tools import horodate, process
def cache(entry: dict, scheduler: sched.scheduler = None) -> None:
"""Cache an .ics feed in the app/cache directory.
Different entries with the same URL will be cached in the same file.
The cached calendar contains a new line in the description with the current time when cached prefixed by the
'Cached at' mention
:param entry: representation of the entry to cache. This is the Python representation of the corresponding entry
in the config file
:type entry: dict
:param scheduler: scheduler used to relaunch the caching task in the future. If not scheduler is specified,
the task will not be relaunched
:type scheduler: sched.scheduler
"""
try:
if not os.path.isdir('app/cache'):
os.mkdir('app/cache')
url = entry['url']
path = "app/cache/" + sha256(url.encode()).hexdigest() + ".ics"
r = requests.get(entry["url"], allow_redirects=True)
if "encoding" in entry:
cal = Calendar(imports=r.content.decode(encoding=entry["encoding"]))
else:
cal = Calendar(imports=r.content.decode())
cal = horodate(cal, 'Cached at')
open(path, 'w').writelines(cal)
print(arrow.now().format("YYYY-MM-DD HH:mm:ss"), "Cached", entry['name'])
except FailedParse:
print("Could not parse", entry['name'])
# Save stack trace when an unknown error occurs
except Exception as e:
with open("error " + arrow.now().format("YYYY-MM-DD HH:mm:ss")+".txt", 'w') as file:
file.write(arrow.now().format("YYYY-MM-DD HH:mm:ss") + "\nCould not cache : " + str(entry))
file.write(str(e))
file.write(str(traceback.format_exc()))
finally:
if scheduler is not None:
delay = entry['cache'] if entry['cache'] > 0 else 10
delay *= 60
scheduler.enter(delay=delay, priority=1, action=cache, argument=(entry, scheduler))
def precompute(config: str, scheduler: sched.scheduler = None) -> None:
"""Precompute a configuration file result to serve it faster when it is requested. This function
should be used with a scheduler to be repeated over time.
:param config: name of the configuration file to precompute the result for
:type config: str
scheduler used to relaunch the precomputing task in the future. If not scheduler is specified,
the task will not be relaunched
:type scheduler: sched.scheduler
"""
try:
cal = process(os.path.basename(config), False)
path = "app/cache/" + os.path.basename(config).rstrip('.json') + ".ics"
open(path, 'w').writelines(cal)
print(arrow.now().format("YYYY-MM-DD HH:mm:ss"), "Precomputed", os.path.basename(config).rstrip('.json'))
except Exception as e:
with open("error " + arrow.now().format("YYYY-MM-DD HH:mm:ss")+".txt", 'w') as file:
file.write(arrow.now().format("YYYY-MM-DD HH:mm:ss") + "\nCould not precompute : " + str(config))
file.write(str(e))
file.write(str(traceback.format_exc()))
finally:
if scheduler is not None:
delay = get_min_cache(config)
delay *= 60
scheduler.enter(delay=delay, priority=1, action=precompute, argument=(config, scheduler))
def get_min_cache(path: str) -> float:
"""Get the minimum caching time of all the entries in a config file.
:param path: path of the config file to use
:type path: str
:return: float number representing the smallest caching time.
"""
result = float('inf')
with open(path, 'r') as config_file:
file = json.loads(config_file.read())
for entry in file:
if 'cache' in entry and entry['cache'] < result:
result = entry['cache']
return result
def start_scheduler(scheduler: sched.scheduler) -> None:
"""Start the caching of every config file found in the app/config directory
:param scheduler: scheduler object to use to schedule the caching
:type scheduler: sched.scheduler
"""
path = "app/config"
files = [os.path.join(path, f) for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f)) and f.endswith('.json')]
for file in files:
with open(file, 'r') as config_file:
config = json.loads(config_file.read())
for entry in config:
if 'cache' in entry:
scheduler.enter(delay=0, priority=1, action=cache, argument=(entry, scheduler))
if get_min_cache(file) < float('inf'):
scheduler.enter(delay=get_min_cache(file)*60, priority=1, action=precompute, argument=(file, scheduler))
scheduler.run()
class CacheThread(threading.Thread):
"""Child class of the threading.Thread class to run the caching process every 10 minutes
"""
def __init__(self):
threading.Thread.__init__(self)
def run(self):
print("Starting cache process")
start_scheduler(sched.scheduler(time.time, time.sleep))

434
app/tools/tools.py
View file

@ -1,434 +0,0 @@
"""This module provides methods to combines multiples .ics feeds into a single object.
The methods allow to filter the events to keep in the final object and to modify the remaining event
according to a configuration file.
The JSON configuration file used in this module has the following structure
[
{
"url":"str",
"name":"str",
"encoding":"str",
"filters":{
"name":{
"exclude":"RegEx",
"includeOnly":"RegEx",
"ignoreCase":true
},
"description":{
"exclude":"RegEx",
"includeOnly":"RegEx",
"ignoreCase":true
}
},
"modify":{
"time":{
"shift":{
"year":0,
"month":0,
"day":0,
"hour":0,
"minute":0
}
},
"name":{
"addPrefix":"str",
"addSuffix":"str"
},
"description":{
"addPrefix":"str",
"addSuffix":"str"
},
"location":{
"addPrefix":"str",
"addSuffix":"str"
}
}
}
]
Only the url and the name field are mandatory.
- url: specify the url to find the calendar
- name: name to identify the calendar
- encoding: specify the encoding to use
- filters: structure defining the filters to apply to the calendar
- name: filters to apply to the name field of the events
- description: filters to apply to the name field of the events
- exclude: RegEx to describe the events to exclude - cannot be specified with includeOnly
- includeOnly: RegEx to describe the events to include - cannot be specified with exclude
- ignoreCase: if true the RegEx will ignore the case of the field
- modify: structure defining the modifications to the events of the calendar
- time: describe the modifications to apply to the timing of the event
- shift: shift the event of a certain amount of time
- year, month, day, hour, minute: amount of time to add to the events
- name: modifications to apply to the name of the events
- description: modifications to apply to the description of the events
- location: modification to apply to the location of the events
- addPrefix: string to add at the beginning of the field
- addSuffix: string to add at the end of the field
"""
import json
import re
import arrow
import os
from hashlib import sha256
from typing import List
import requests
from ics import Calendar
from pathvalidate import sanitize_filename
def filtering(cal: Calendar, filters: dict, field_name: str) -> Calendar:
"""Filter the event of a calendar according to the filters and the field_name
:param cal: the calendar to apply filters to
:type cal: Calendar
:param filters: the filters to apply to the calendar
:type filters: dict
:param field_name: the of the field in the filters to consider
:type field_name: str
:return: the modified cal argument after filtering out the events
:rtype: Calendar
:raises SyntaxError: if both exclude and includeOnly are specified in the filters
"""
if field_name in filters:
field = filters[field_name]
if ("exclude" in field) and ("includeOnly" in field):
raise SyntaxError("Cannot specify both exclude and includeOnly")
if ("exclude" not in field) and ("includeOnly" not in field):
return cal
new = Calendar()
ignore_case = True if ("ignoreCase" in field and field["ignoreCase"]) else False
if "exclude" in field:
p = re.compile(field["exclude"], re.IGNORECASE | re.DOTALL) \
if ignore_case else re.compile(field["exclude"], re.DOTALL)
for event in cal.events:
if event.name is None or (field_name == "name" and p.match(event.name) is None):
new.events.add(event)
elif event.description is None or (field_name == "description" and p.match(event.description) is None):
new.events.add(event)
if "includeOnly" in field:
p = re.compile(field["includeOnly"], re.IGNORECASE | re.DOTALL) \
if ignore_case else re.compile(field["includeOnly"], re.DOTALL)
for event in cal.events:
if field_name == "name" and event.name is not None and p.match(event.name) is not None:
new.events.add(event)
elif field_name == "description" and event.description is not None \
and p.match(event.description) is not None:
new.events.add(event)
cal = new
return cal
else:
return cal
def apply_filters(cal: Calendar, filters: dict) -> Calendar:
"""Apply all the filters to a calendar and returns the resulting calendar
:param cal: the calendar to apply filters to
:type cal: Calendar
:param filters: the filters to apply
:type filters: dict
:return: the modified cal parameter to satisfy the filters
:rtype: Calendar
:raises SyntaxError: if both exclude and includeOnly are specified for the same field in the filters
"""
cal = filtering(cal, filters, "name")
cal = filtering(cal, filters, "description")
return cal
def modify_time(cal: Calendar, modify: dict) -> Calendar:
"""Modify the time of all the events in a calendar as specified in the modify structure
:param cal: the calendar where it is needed to modify the time of the events
:type cal: Calendar
:param modify: the structure defining how to modify the time
:type modify: dict
:return: the modified cal parameter
:rtype: Calendar
"""
if ("time" in modify) and ("shift" in modify["time"]):
shift = modify["time"]["shift"]
year = 0 if not ("year" in shift) else shift["year"]
month = 0 if not ("month" in shift) else shift["month"]
day = 0 if not ("day" in shift) else shift["day"]
hour = 0 if not ("hour" in shift) else shift["hour"]
minute = 0 if not ("minute" in shift) else shift["minute"]
for event in cal.events:
event.end = event.end.shift(years=year, months=month, days=day, hours=hour, minutes=minute)
event.begin = event.begin.shift(years=year, months=month, days=day, hours=hour, minutes=minute)
return cal
def modify_text(cal: Calendar, modify: dict, field_name: str) -> Calendar:
"""Modify one text field (name, location, description) of all the events in the cal parameter
according to the modify structure and the field_name
:param cal: the calendar where it is needed to modify the text field
:type cal: Calendar
:param modify: the structure defining how to modify the time
:type modify: dict
:param field_name: the name of the field to modify
:type field_name: str
:return: the modified cal parameter
:rtype: Calendar
"""
if field_name in modify:
change = modify[field_name]
if "addPrefix" in change:
for event in cal.events:
if field_name == "name":
event.name = change["addPrefix"] + event.name \
if event.name is not None else change["addPrefix"]
elif field_name == "description":
event.description = change["addPrefix"] + event.description \
if event.description is not None else change["addPrefix"]
elif field_name == "location":
event.location = change["addPrefix"] + event.location \
if event.location is not None else change["addPrefix"]
if "addSuffix" in change:
for event in cal.events:
if field_name == "name":
event.name = event.name + change["addSuffix"] \
if event.name is not None else change["addSuffix"]
elif field_name == "description":
event.description = event.description + change["addSuffix"] \
if event.description is not None else change["addSuffix"]
elif field_name == "location":
event.location = event.location + change["addSuffix"] \
if event.location is not None else change["addSuffix"]
return cal
def apply_modify(cal: Calendar, modify: dict) -> Calendar:
"""Apply all the needed modifications to a calendar and returns the resulting calendar
:param cal: the calendar to apply modifications to
:type cal: Calendar
:param modify: the structure containing the modifications to apply
:type modify: dict
:return: the modified cal parameter
:rtype: Calendar
"""
cal = modify_time(cal, modify)
cal = modify_text(cal, modify, "name")
cal = modify_text(cal, modify, "description")
cal = modify_text(cal, modify, "location")
return cal
def merge(cals: List[Calendar]) -> Calendar:
"""Merge a list of calendars into a single calendar
Only takes the event into account, not the tasks or the alarms
:param cals: the list of calendars to merge
:type cals: List[Calendar]
:return: the calendar containing the union of the events contained in the cals list
:rtype: Calendar
:raises ValueError: if an element of the list is not a Calendar
"""
result = Calendar()
for cal in cals:
if not isinstance(cal, Calendar):
raise ValueError("All elements should be Calendar")
result.events = result.events.union(cal.events)
return result
def process(path: str, from_cache: bool = True) -> Calendar:
"""Open a config file from the specified path, download the calendars,
apply the filters, modify and merge the calendars as specified in the config file
:param from_cache:
:param path: name of the file to open. The file should be in the config/ folder
:type path: str
:return: the resulting calendar
:rtype: Calendar
"""
print("app/cache/" + sanitize_filename(path).rstrip(".json") + ".ics")
if from_cache and os.path.isfile("app/cache/" + sanitize_filename(path).rstrip(".json") + ".ics"):
with open("app/cache/" + sanitize_filename(path).rstrip(".json") + ".ics") as file:
data = file.read()
print("Serving precomputed file")
return Calendar(imports=data)
else:
o = "app/config/" + sanitize_filename(path)
print("Try to open " + o)
file = open(o, "r")
config = json.loads(file.read())
file.close()
data = []
for entry in config:
cal = load_cal(entry)
if "filters" in entry:
cal = apply_filters(cal, entry["filters"])
if "modify" in entry:
cal = apply_modify(cal, entry["modify"])
data.append(cal)
return merge(data)
def get_from_cache(entry: dict) -> Calendar:
"""Retrieve the entry from cache. If the entry is not found, an exception is raised
:param entry: representation of the entry to cache. This is the Python representation of the corresponding entry
in the config file
:type entry: dict
:return: the corresponding calendar in cache
:rtype: Calendar
:raises FileNotfoundError: if the entry has not been cached before
"""
url = entry['url']
path = "app/cache/" + sha256(url.encode()).hexdigest() + ".ics"
if not os.path.isfile(path):
print("Not cached")
raise FileNotFoundError("The calendar is not cached")
with open(path, 'r') as file:
data = file.read()
return Calendar(imports=data)
def load_cal(entry: dict) -> Calendar:
"""Load the calendar from the cache or from remote according to the entry. If the calendar is supposed to be in
cached but could not be found in cache, an error is thrown
:param entry: representation of the entry to cache. This is the Python representation of the corresponding entry
in the config file
:type entry: dict
:return: the calendar corresponding to the entry
:rtype: Calendar
:raises FileNotfoundError: if the entry was supposed to be cached but has not been cached before
"""
if "cache" in entry and entry["cache"]:
print("Getting", entry["name"], "from cache")
try:
return get_from_cache(entry)
except FileNotFoundError:
return Calendar()
else:
print("Getting", entry["name"], "from remote")
r = requests.get(entry["url"], allow_redirects=True)
if "encoding" in entry:
cal = Calendar(imports=r.content.decode(encoding=entry["encoding"]))
else:
cal = Calendar(imports=r.content.decode())
cal = horodate(cal, 'Downloaded at')
return cal
def horodate(cal: Calendar, prefix='') -> Calendar:
"""Add a new line at the end of the description of every event in the calendar with the current time prefixed by
the prefix parameter and a space
The date is added with the following format: YYYY-MM-DD HH:mm:ss
:param cal: calendar to process
:type cal: Calendar
:param prefix: the prefix to add in front of the date
:type prefix: str
:return: the modified calendar
:rtype: Calendar
"""
now = arrow.now().format("YYYY-MM-DD HH:mm:ss")
for event in cal.events:
event.description = event.description + '\n' + prefix + ' ' + now \
if event.description is not None else prefix + ' ' + now
return cal

6
requirements.txt
View file

@ -1,6 +0,0 @@
requests~=2.22.0
ics~=0.7
pathvalidate~=2.3.0
flask~=1.1.1
arrow~=0.14.7
tatsu~=4.4.0