From 10860c7dc8c6f8a858c50e9611e84c5021412575 Mon Sep 17 00:00:00 2001 From: Valentin Boettcher Date: Sun, 13 Aug 2017 18:52:10 +0200 Subject: [PATCH] Clean --- .gitignore | 2 + logParser.py | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++ parseLogs.py | 39 ++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 .gitignore create mode 100644 logParser.py create mode 100644 parseLogs.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aa28b5b --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.db +__pycache__ diff --git a/logParser.py b/logParser.py new file mode 100644 index 0000000..9e9b29b --- /dev/null +++ b/logParser.py @@ -0,0 +1,165 @@ +from sqlalchemy import Column, Integer, String, REAL +from sqlalchemy.orm import sessionmaker, scoped_session +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship +from sqlalchemy import create_engine +from datetime import datetime +from math import inf + +engine = create_engine('sqlite:///heaters.db') +session_factory = sessionmaker(bind=engine) +Session = lambda : scoped_session(session_factory) + +Base = declarative_base() +heaters = {} + +class HeaterStatProto(): + time = Column(Integer, primary_key=True) + actuator = Column(Integer) + desired_temp = Column(REAL) + measured_temp = Column(REAL) + +class Heater(Base): + __tablename__ = 'heater_list' + name = Column(String, primary_key=True) + +Base.metadata.create_all(engine) + +def getHeater(name): + if name in heaters: + return heaters[name] + + session = Session() + heater = Heater(name=name) + session.merge(heater) + session.commit() + session.close() + + heater = type(name, (Base, HeaterStatProto), {'__tablename__': 'heater_' + name}) + + Base.metadata.create_all(engine) + heaters[name] = heater + return heater + +def getHeaterList(): + session = Session() + heaters = session.query(Heater).all() + session.close() + return [heater.name for heater in heaters] + +def getHeaterSum(heater, mintime = 0, maxtime = inf): + if not heater in getHeaterList(): + return 'Heater not found!' + + heater = getHeater(heater) + session = Session() + heaterData = session.query(heater).filter(heater.time > mintime, heater.time < maxtime).order_by(heater.time.asc()).all() + session.close() + + totalHeat = 0 + for h in range(0, len(heaterData) - 1): + try: + totalHeat += heaterData[h].actuator * (heaterData[h+1].time - heaterData[h].time) + except: + continue + + return totalHeat + +def getAllHeaterSums(*args, **kwargs): + sums = {} + for heater in getHeaterList(): + sums[heater] = getHeaterSum(heater, *args, **kwargs) + + return sums + +class LogParser: + # Some constants... + def parseLog(self, data): + totalLines = len(data) + print('Parsing', totalLines, 'lines.') + + heaterData = {} + + lineNum = 0 + for line in data: + lineNum += 1 + print('\r Progress:', int((lineNum/totalLines)*100), end='') + line = line.strip().split(' ') + + if not (len(line) == 4): + continue + + timeStamp, name, prop, value = line + prop = prop[:-1].replace('-', '_') + + if hasattr(HeaterStatProto, prop): + timeStamp = datetime.strptime(timeStamp, '%Y-%m-%d_%H:%M:%S').timestamp() + + if not name in heaterData: + heaterData[name] = {} + + if not timeStamp in heaterData[name]: + heaterData[name][timeStamp] = {} + + heaterData[name][timeStamp][prop] = value + + print('\n') + self.simplify(heaterData) + return heaterData + + def simplify(self, data): + print('Removing redundant data...') + for heater in data.values(): + lastValue = None + for key, value in list(heater.items()): + if value == lastValue: + del heater[key] + continue + + lastValue = value + + def writeHeaters(self, data): + print('Writing the parsed data to the Database...') + heaters = [] + + for heater, heaterHistory in data.items(): + dbHeater = getHeater(heater) + for time, heaterData in heaterHistory.items(): + heaters.append(dbHeater(time=time, **heaterData)) + + session = Session() + #session.add_all(heaters) + for heater in heaters: + try: + session.merge(heater) + except: + print('Could not write: ', heaterx) + + session.commit() + session.close() + + def parse(self): + try: + for logfile in self._logfiles: + with open(logfile) as lf: + try: + print('Reading:', logfile) + logdata = lf.readlines() + self.writeHeaters(self.parseLog(logdata)) + + except (IOError, UnicodeDecodeError) as e: + print('Could not read: ', logfile) + + print() + + except EnvironmentError: + raise ValueError('Invalid log file!') + + print('Done') + + def __init__(self, logfiles): + # Try to read the heater log. + self._logData = [] + self._logfiles = logfiles + + diff --git a/parseLogs.py b/parseLogs.py new file mode 100644 index 0000000..b2ae797 --- /dev/null +++ b/parseLogs.py @@ -0,0 +1,39 @@ +import sys +import datetime +import argparse +import logParser +from math import inf + +argparser = argparse.ArgumentParser(description='Parse the FHEM logfiles into a Database.') +argparser.add_argument('--parse', '-p', nargs='+', help='List of Logfiles to Parse.', required=False, metavar='FILE') +argparser.add_argument('--list', '-l', help='List all Heaters in the Database.', action='store_true') +argparser.add_argument('--heatAmmount', help='Sum of the actuator value times a time slice.', nargs=1, metavar='HEATER') +argparser.add_argument('--allHeatAmmount', help='Get the heat ammount from all known heaters.', action='store_true') +argparser.add_argument('--timeSpan', help='The time span for the heatAmmount calculation. Either t1/t2 or both may be supplied.', nargs=1, metavar='t1..t2 | t1 | ..t2') + +# Parse args. Set name to default. +args = argparser.parse_args() +mintime, maxtime = 0, inf +if args.timeSpan: + times = args.timeSpan[0].split('..') + + if len(times) < 1 or len(times) > 2: + print('Invalid Timespan!') + exit(1) + elif len(times) == 2: + if times[0] == '': + maxtime = int(times[1]) + else: + mintime, maxtime = times + else: + mintime = int(times[0]) + +if not args.parse is None: + parser = logParser.LogParser(args.parse) + parser.parse() +elif not args.list is False: + print('\n'.join(logParser.getHeaterList())) +elif not args.heatAmmount is None: + print(logParser.getHeaterSum(args.heatAmmount[0], mintime, maxtime)) +elif not args.allHeatAmmount is None: + print(logParser.getAllHeaterSums(mintime, maxtime))