Save the processed transactions so they don't get processed a second time

.gitignore

@@ -139,3 +139,6 @@ dmypy.json
 cython_debug/
 
 .devcontainer/devcontainer.json
+.vscode/launch.json
+dev.env
+.vscode/launch.json

Dockerfile

@@ -2,10 +2,17 @@ FROM python:3.8
 
 RUN pip install ofxparse
 RUN pip install watchdog
+RUN pip install pymongo
 
-RUN useradd --uid 99 --gid 100 hsbc
+
+RUN useradd --uid 99 --gid 100 -m hsbc
 USER hsbc
 
+# Creates a non-root user and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+#RUN useradd -u 1000 -m ubuntu && chown -R ubuntu /home/ubuntu
+#USER ubuntu
+
 ADD converter.py /app/
 
 CMD [ "python", "./app/converter.py" ]

converter.py

@@ -6,17 +6,36 @@ from glob import glob
 from ofxparse import OfxParser
 from pathlib import Path
 
+from decimal import Decimal
+from bson.decimal128 import Decimal128
+
+
 import os 
+import pymongo
 import time
 import watchdog.events
 import watchdog.observers
 
 DATE_FORMAT = "%d/%m/%Y"
-WATCH_DIR = '/mnt/data/'
+
+if 'WATCH_DIR' in os.environ:
+    WATCH_DIR = os.environ['WATCH_DIR']
+else:
+    WATCH_DIR = '/mnt/data/'
+
 PATTERN = '*.qfx'
 BACKUP_DIR = 'Imported'
 CONVERTED_DIR = 'Converted'
 
+MONGO_URL = os.environ['DB_HOST']
+MONGO_PORT = os.environ['DB_PORT']
+MONGO_DB = os.environ['DB_NAME']
+MONGO_COL = os.environ['DB_COL']
+
+MONGO_URL = "mongodb://{}:{}".format(MONGO_URL, MONGO_PORT)
+myclient = pymongo.MongoClient(MONGO_URL)
+mydb = myclient[MONGO_DB]
+mongo_col = mydb[MONGO_COL]
 
 class Handler(watchdog.events.PatternMatchingEventHandler):
     def __init__(self):
@@ -27,6 +46,11 @@ class Handler(watchdog.events.PatternMatchingEventHandler):
     @staticmethod
     def write_csv(statement, out_file):
         print("Writing: " + out_file)
+
+        if len(statement) == 0:
+            print("No transactions to write.")
+            return
+
         fields = ['date', 'memo', 'category', 'amount', 'name']
         with open(out_file, 'w') as f:
             f.write("Date,Original Description,Category,Amount,Account Name")
@@ -35,16 +59,34 @@ class Handler(watchdog.events.PatternMatchingEventHandler):
             for line in statement:
                 writer.writerow(line)
 
+    @staticmethod
+    def transaction_exists(line):
+        existing_trans = mongo_col.find_one(line)
+
+        return existing_trans is not None
+
+    @staticmethod
+    def convert_decimal(dict_item):
+    # This function iterates a dictionary looking for types of Decimal and converts them to Decimal128
+    # Embedded dictionaries and lists are called recursively.
+        if dict_item is None: return None
+
+        for k, v in list(dict_item.items()):
+            if isinstance(v, dict):
+                Handler.convert_decimal(v)
+            elif isinstance(v, list):
+                for l in v:
+                    Handler.convert_decimal(l)
+            elif isinstance(v, Decimal):
+                dict_item[k] = Decimal128(str(v))
+
+        return dict_item
+
     @staticmethod
     def get_statement_from_qfx(qfx):
-        balance = qfx.account.statement.balance
 
         statement = []
-        credit_transactions = ['credit', 'dep', 'int']
-        debit_transactions  = ['debit', 'atm', 'pos', 'xfer', 'check']
         for transaction in qfx.account.statement.transactions:
-            amount = ""
-            balance = balance + transaction.amount
            
             if transaction.payee.startswith("PENDING:"):
                 continue
@@ -56,7 +98,17 @@ class Handler(watchdog.events.PatternMatchingEventHandler):
                 'amount': transaction.amount,
                 'name': 'HSBC Everyday Global'
             }
+
+            #mongo needs the decimal values in Decimal128, so create a version for it
+            line_d128 = Handler.convert_decimal(line.copy())
+
+            if Handler.transaction_exists(line_d128):
+                continue
+
             statement.append(line)
+            result = mongo_col.insert_one(line_d128)
+            print("New db entry stored: {}".format(result.inserted_id))
+
         return statement        
 
     @staticmethod
@@ -84,10 +136,10 @@ class Handler(watchdog.events.PatternMatchingEventHandler):
         
         historicalSize = -1
         while (historicalSize != os.path.getsize(event.src_path)):
-          historicalSize = os.path.getsize(event.src_path)
+            historicalSize = os.path.getsize(event.src_path)
-          print('waiting....')
+            print('waiting for copy to finish....')
-          time.sleep(1)
+            time.sleep(1)
-        print("file copy has now finished")
+            print("file copy has now finished")
         
         with open(event.src_path, 'r') as file:
             
@@ -115,6 +167,7 @@ class Handler(watchdog.events.PatternMatchingEventHandler):
             if not destination.exists():
                 path.replace(destination)
 
+        print("Processing successfully finished for {}".format(event.src_path))
   
 if __name__ == "__main__":
     event_handler = Handler()
@@ -122,6 +175,7 @@ if __name__ == "__main__":
     observer.schedule(event_handler, path=WATCH_DIR, recursive=False)
 
     observer.start()
+    print('Converter running, waiting for files in {}'.format(WATCH_DIR))
     try:
         while True:
             time.sleep(1)