FHEM-Log-Parser/logParser.py

from sqlalchemy import Column, Integer, String, REAL
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import create_engine
from datetime import datetime
from math import inf

engine = create_engine('sqlite:///heaters.db')
session_factory = sessionmaker(bind=engine)
Session = lambda : scoped_session(session_factory)

Base = declarative_base()
heaters = {}

class HeaterStatProto():
    time = Column(Integer, primary_key=True)
    actuator = Column(Integer)
    desired_temp = Column(REAL)
    measured_temp = Column(REAL)
    dewpoint = Column(REAL)
    humidity = Column(REAL)

class Heater(Base):
    __tablename__ = 'heater_list'
    name = Column(String, primary_key=True)

Base.metadata.create_all(engine)

def getHeater(name):
    if name in heaters:
        return heaters[name]

    session = Session()
    heater = Heater(name=name)
    session.merge(heater)
    session.commit()
    session.close()

    heater = type(name, (Base, HeaterStatProto), {'__tablename__': 'heater_' + name})

    Base.metadata.create_all(engine)
    heaters[name] = heater
    return heater

def getHeaterList():
    session = Session()
    heaters = session.query(Heater).all()
    session.close()
    return [heater.name for heater in heaters]

def getHeaterSum(heater, mintime = 0, maxtime = inf):
    if not heater in getHeaterList():
        return 'Heater not found!'

    heater = getHeater(heater)
    session = Session()
    heaterData = session.query(heater).filter(heater.time > mintime, heater.time < maxtime).order_by(heater.time.asc()).all()
    session.close()

    totalHeat = 0
    
    # Filter out invalid
    heaterData = [heater for heater in heaterData if not heater.actuator is None]
    for h in range(0, len(heaterData) - 1):
        try:
            totalHeat += heaterData[h].actuator * (heaterData[h+1].time - heaterData[h].time)
        except:
            continue

    return totalHeat

def getAllHeaterSums(*args, **kwargs):
    sums = {}
    for heater in getHeaterList():
        sums[heater] = getHeaterSum(heater, *args, **kwargs)

    return sums

class LogParser:
    # Some constants...
    def parseLog(self, data):
        totalLines = len(data)
        print('Parsing', totalLines, 'lines.')

        heaterData = {}

        lineNum = 0
        for line in data:
            lineNum += 1
            print('\r  Progress:', int((lineNum/totalLines)*100), end='')
            line = line.strip().split(' ')

            if not (len(line) == 4):
                continue

            timeStamp, name, prop, value = line
            prop = prop[:-1].replace('-', '_')

            if hasattr(HeaterStatProto, prop):
                timeStamp = datetime.strptime(timeStamp, '%Y-%m-%d_%H:%M:%S').timestamp()

                if not name in heaterData:
                    heaterData[name] = {}

                if not timeStamp in heaterData[name]:
                    heaterData[name][timeStamp] = {}

                heaterData[name][timeStamp][prop] = value

        print('\n')
        self.simplify(heaterData)
        return heaterData

    def simplify(self, data):
        print('Removing redundant data...')
        for heater in data.values():
            lastValue = None
            for key, value in list(heater.items()):
                if value == lastValue:
                    del heater[key]
                    continue

                lastValue = value

    def writeHeaters(self, data):
        print('Writing the parsed data to the Database...')
        heaters = []

        for heater, heaterHistory in data.items():
            dbHeater = getHeater(heater)
            for time, heaterData in heaterHistory.items():
                heaters.append(dbHeater(time=time, **heaterData))

        session = Session()
        #session.add_all(heaters)
        for heater in heaters:
            try:
                session.merge(heater)
            except:
                print('Could not write: ', heaterx)

        session.commit()
        session.close()

    def parse(self):
        try:
            for logfile in self._logfiles:
                with open(logfile) as lf:
                    try:
                        print('Reading:', logfile)
                        logdata = lf.readlines()
                        self.writeHeaters(self.parseLog(logdata))

                    except (IOError, UnicodeDecodeError) as e:
                        print('Could not read: ', logfile)

                    print()

        except EnvironmentError:
            raise ValueError('Invalid log file!')

        print('Done')

    def __init__(self, logfiles):
        # Try to read the heater log.
        self._logData = []
        self._logfiles = logfiles