diff --git a/.gitignore b/.gitignore index 8b94239..ed1cd45 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,6 @@ dmypy.json cython_debug/ .devcontainer/devcontainer.json +.vscode/launch.json +dev.env +.vscode/launch.json diff --git a/Dockerfile b/Dockerfile index 720e5f0..9cb2276 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,17 @@ FROM python:3.8 RUN pip install ofxparse RUN pip install watchdog +RUN pip install pymongo -RUN useradd --uid 99 --gid 100 hsbc + +RUN useradd --uid 99 --gid 100 -m hsbc USER hsbc +# Creates a non-root user and adds permission to access the /app folder +# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers +#RUN useradd -u 1000 -m ubuntu && chown -R ubuntu /home/ubuntu +#USER ubuntu + ADD converter.py /app/ CMD [ "python", "./app/converter.py" ] \ No newline at end of file diff --git a/converter.py b/converter.py index fd0571d..f74536c 100644 --- a/converter.py +++ b/converter.py @@ -6,17 +6,36 @@ from glob import glob from ofxparse import OfxParser from pathlib import Path +from decimal import Decimal +from bson.decimal128 import Decimal128 + + import os +import pymongo import time import watchdog.events import watchdog.observers DATE_FORMAT = "%d/%m/%Y" -WATCH_DIR = '/mnt/data/' + +if 'WATCH_DIR' in os.environ: + WATCH_DIR = os.environ['WATCH_DIR'] +else: + WATCH_DIR = '/mnt/data/' + PATTERN = '*.qfx' BACKUP_DIR = 'Imported' CONVERTED_DIR = 'Converted' +MONGO_URL = os.environ['DB_HOST'] +MONGO_PORT = os.environ['DB_PORT'] +MONGO_DB = os.environ['DB_NAME'] +MONGO_COL = os.environ['DB_COL'] + +MONGO_URL = "mongodb://{}:{}".format(MONGO_URL, MONGO_PORT) +myclient = pymongo.MongoClient(MONGO_URL) +mydb = myclient[MONGO_DB] +mongo_col = mydb[MONGO_COL] class Handler(watchdog.events.PatternMatchingEventHandler): def __init__(self): @@ -27,6 +46,11 @@ class Handler(watchdog.events.PatternMatchingEventHandler): @staticmethod def write_csv(statement, out_file): print("Writing: " + out_file) + + if len(statement) == 0: + print("No transactions to write.") + return + fields = ['date', 'memo', 'category', 'amount', 'name'] with open(out_file, 'w') as f: f.write("Date,Original Description,Category,Amount,Account Name") @@ -35,17 +59,35 @@ class Handler(watchdog.events.PatternMatchingEventHandler): for line in statement: writer.writerow(line) + @staticmethod + def transaction_exists(line): + existing_trans = mongo_col.find_one(line) + + return existing_trans is not None + + @staticmethod + def convert_decimal(dict_item): + # This function iterates a dictionary looking for types of Decimal and converts them to Decimal128 + # Embedded dictionaries and lists are called recursively. + if dict_item is None: return None + + for k, v in list(dict_item.items()): + if isinstance(v, dict): + Handler.convert_decimal(v) + elif isinstance(v, list): + for l in v: + Handler.convert_decimal(l) + elif isinstance(v, Decimal): + dict_item[k] = Decimal128(str(v)) + + return dict_item + @staticmethod def get_statement_from_qfx(qfx): - balance = qfx.account.statement.balance statement = [] - credit_transactions = ['credit', 'dep', 'int'] - debit_transactions = ['debit', 'atm', 'pos', 'xfer', 'check'] for transaction in qfx.account.statement.transactions: - amount = "" - balance = balance + transaction.amount - + if transaction.payee.startswith("PENDING:"): continue @@ -56,7 +98,17 @@ class Handler(watchdog.events.PatternMatchingEventHandler): 'amount': transaction.amount, 'name': 'HSBC Everyday Global' } + + #mongo needs the decimal values in Decimal128, so create a version for it + line_d128 = Handler.convert_decimal(line.copy()) + + if Handler.transaction_exists(line_d128): + continue + statement.append(line) + result = mongo_col.insert_one(line_d128) + print("New db entry stored: {}".format(result.inserted_id)) + return statement @staticmethod @@ -84,10 +136,10 @@ class Handler(watchdog.events.PatternMatchingEventHandler): historicalSize = -1 while (historicalSize != os.path.getsize(event.src_path)): - historicalSize = os.path.getsize(event.src_path) - print('waiting....') - time.sleep(1) - print("file copy has now finished") + historicalSize = os.path.getsize(event.src_path) + print('waiting for copy to finish....') + time.sleep(1) + print("file copy has now finished") with open(event.src_path, 'r') as file: @@ -115,6 +167,7 @@ class Handler(watchdog.events.PatternMatchingEventHandler): if not destination.exists(): path.replace(destination) + print("Processing successfully finished for {}".format(event.src_path)) if __name__ == "__main__": event_handler = Handler() @@ -122,6 +175,7 @@ if __name__ == "__main__": observer.schedule(event_handler, path=WATCH_DIR, recursive=False) observer.start() + print('Converter running, waiting for files in {}'.format(WATCH_DIR)) try: while True: time.sleep(1)