From 8b1840742788f110de0a471d262d2e2a9d157dbb Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 09:38:50 +0200 Subject: [PATCH 01/95] added a generic file handler --- can/io/generic.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 can/io/generic.py diff --git a/can/io/generic.py b/can/io/generic.py new file mode 100644 index 000000000..97bc76555 --- /dev/null +++ b/can/io/generic.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# coding: utf-8 + +""" +Contains generic classes for file IO. +""" + +from abc import ABCMeta, abstractmethod + +from can import Listener + + +class BaseIOHandler(object): + """A generic file handler that can be used for reading and writing. + """ + + __metaclass__ = ABCMeta + + def __init__(self, open_file, filename='can_data', mode='Urt'): + """ + TODO docs + """ + if open_file: + self.file = open(filename, mode) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.stop() + + def stop(self): + if hasattr(self, 'file') and self.file: + # this also implies a flush() + self.file.close() From deee1b62bb14d09669017ba6633d47cd1fd9c8fd Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 09:39:18 +0200 Subject: [PATCH 02/95] better docs in listener.py and made the Listener class abstract --- can/listener.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/can/listener.py b/can/listener.py index d7e8e7ced..279dcfd8e 100644 --- a/can/listener.py +++ b/can/listener.py @@ -5,6 +5,8 @@ This module contains the implementation of `can.Listener` and some readers. """ +from abc import ABCMeta, abstractmethod + try: # Python 3 import queue @@ -14,12 +16,28 @@ class Listener(object): + """The basic listener that can be called directly to deliver a message:: + + listener = SomeListener() + msg = my_bus.recv() + + # now either call + listener(msg) + # or + listener.on_message_received(msg) + + """ + __metaclass__ = ABCMeta + + @abstractmethod def on_message_received(self, msg): - raise NotImplementedError( - "{} has not implemented on_message_received".format( - self.__class__.__name__) - ) + """This method is called to handle the given message. + + :param can.Message msg: the delivered message + + """ + pass def __call__(self, msg): return self.on_message_received(msg) @@ -32,8 +50,7 @@ def stop(self): class RedirectReader(Listener): """ - A RedirectReader sends all received messages - to another Bus. + A RedirectReader sends all received messages to another Bus. """ @@ -49,10 +66,12 @@ class BufferedReader(Listener): A BufferedReader is a subclass of :class:`~can.Listener` which implements a **message buffer**: that is, when the :class:`can.BufferedReader` instance is notified of a new message it pushes it into a queue of messages waiting to - be serviced. + be serviced. The messages can then be fetched with + :meth:`~can.BufferedReader.get_message` """ def __init__(self): + # 0 is "infinite" size self.buffer = queue.Queue(0) def on_message_received(self, msg): @@ -65,7 +84,8 @@ def get_message(self, timeout=0.5): is shorter), :param float timeout: The number of seconds to wait for a new message. - :return: the :class:`~can.Message` if there is one, or None if there is not. + :rytpe: can.Message + :return: the message if there is one, or None if there is not. """ try: return self.buffer.get(block=True, timeout=timeout) From 3cf6368278eb575599d2952557b81af7c454745e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 09:40:37 +0200 Subject: [PATCH 03/95] make CSV reader & writer is the new BaseIOHandler --- can/io/csv.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/can/io/csv.py b/can/io/csv.py index 1933648ac..b7b4c4d7c 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -5,19 +5,22 @@ This module contains handling for CSV (comma seperated values) files. TODO: CAN FD messages are not yet supported. + TODO: This module could use https://docs.python.org/2/library/csv.html#module-csv to allow different delimiters for writing, special escape chars to circumvent the base64 encoding and use csv.Sniffer to automatically deduce the delimiters of a CSV file. """ +from __future__ import absolute_import + from base64 import b64encode, b64decode from can.message import Message from can.listener import Listener +from .generic import BaseIOHandler - -class CSVWriter(Listener): +class CSVWriter(BaseIOHandler, Listener): """Writes a comma separated text file with a line for each message. @@ -38,11 +41,12 @@ class CSVWriter(Listener): Each line is terminated with a platform specific line seperator. """ - def __init__(self, filename): - self.csv_file = open(filename, 'wt') + def __init__(self, filename, append=False): + mode = 'Uat' if append else 'Uwt' + super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode=mode) # Write a header row - self.csv_file.write("timestamp,arbitration_id,extended,remote,error,dlc,data\n") + self.file.write("timestamp,arbitration_id,extended,remote,error,dlc,data\n") def on_message_received(self, msg): row = ','.join([ @@ -54,37 +58,37 @@ def on_message_received(self, msg): str(msg.dlc), b64encode(msg.data).decode('utf8') ]) - self.csv_file.write(row + '\n') + self.file.write(row) + self.file.write('\n') - def stop(self): - self.csv_file.flush() - self.csv_file.close() -class CSVReader(): +class CSVReader(BaseIOHandler): """Iterator over CAN messages from a .csv file that was generated by :class:`~can.CSVWriter` or that uses the same - format that is described there. + format as described there. + + Any line seperator is accepted. """ def __init__(self, filename): - self.csv_file = open(filename, 'rt') + super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode='Urt') + def __iter__(self): # skip the header line - self.header_line = next(self.csv_file).split(',') + next(self.file) - def __iter__(self): - for line in self.csv_file: + for line in self.file: timestamp, arbitration_id, extended, remote, error, dlc, data = line.split(',') yield Message( timestamp=float(timestamp), - is_remote_frame=(remote == '1'), - extended_id=(extended == '1'), - is_error_frame=(error == '1'), + is_remote_frame=bool(remote), + extended_id=bool(extended), + is_error_frame=bool(error), arbitration_id=int(arbitration_id, base=16), dlc=int(dlc), data=b64decode(data), ) - self.csv_file.close() + self.stop() From ebb36fbbdaba52da9d6d0e2694eacd594c09fcdc Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 09:40:44 +0200 Subject: [PATCH 04/95] make Canutils reader & writer is the new BaseIOHandler --- can/io/canutils.py | 103 +++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/can/io/canutils.py b/can/io/canutils.py index 564b386e1..0553e8a49 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -7,12 +7,15 @@ (https://github.com/linux-can/can-utils). """ +from __future__ import absolute_import, division + import time import datetime import logging from can.message import Message from can.listener import Listener +from .generic import BaseIOHandler log = logging.getLogger('can.io.canutils') @@ -23,7 +26,7 @@ CAN_ERR_DLC = 8 -class CanutilsLogReader(object): +class CanutilsLogReader(BaseIOHandler): """ Iterator over CAN messages from a .log Logging File (candump -L). @@ -34,50 +37,54 @@ class CanutilsLogReader(object): """ def __init__(self, filename): - self.fp = open(filename, 'r') + super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode='Urt') def __iter__(self): - for line in self.fp: + for line in self.file: + + # skip empty lines temp = line.strip() + if not temp: + continue - if temp: + timestamp, channel, frame = temp.split() + timestamp = float(timestamp[1:-1]) + canId, data = frame.split('#') + if channel.isdigit(): + channel = int(channel) - (timestamp, channel, frame) = temp.split() - timestamp = float(timestamp[1:-1]) - (canId, data) = frame.split('#') - if channel.isdigit(): - channel = int(channel) + if len(canId) > 3: + isExtended = True + else: + isExtended = False + canId = int(canId, 16) - if len(canId) > 3: - isExtended = True + if data and data[0].lower() == 'r': + isRemoteFrame = True + if len(data) > 1: + dlc = int(data[1:]) else: - isExtended = False - canId = int(canId, 16) - - if data and data[0].lower() == 'r': - isRemoteFrame = True - if len(data) > 1: - dlc = int(data[1:]) - else: - dlc = 0 - else: - isRemoteFrame = False + dlc = 0 + else: + isRemoteFrame = False - dlc = int(len(data) / 2) - dataBin = bytearray() - for i in range(0, 2 * dlc, 2): - dataBin.append(int(data[i:(i + 2)], 16)) + dlc = len(data) // 2 + dataBin = bytearray() + for i in range(0, len(data), 2): + dataBin.append(int(data[i:(i + 2)], 16)) - if canId & CAN_ERR_FLAG and canId & CAN_ERR_BUSERROR: - msg = Message(timestamp=timestamp, is_error_frame=True) - else: - msg = Message(timestamp=timestamp, arbitration_id=canId & 0x1FFFFFFF, - extended_id=isExtended, is_remote_frame=isRemoteFrame, - dlc=dlc, data=dataBin, channel=channel) - yield msg + if canId & CAN_ERR_FLAG and canId & CAN_ERR_BUSERROR: + msg = Message(timestamp=timestamp, is_error_frame=True) + else: + msg = Message(timestamp=timestamp, arbitration_id=canId & 0x1FFFFFFF, + extended_id=isExtended, is_remote_frame=isRemoteFrame, + dlc=dlc, data=dataBin, channel=channel) + yield msg + + self.stop() -class CanutilsLogWriter(Listener): +class CanutilsLogWriter(BaseIOHandler, Listener): """Logs CAN data to an ASCII log file (.log). This class is is compatible with "candump -L". @@ -86,21 +93,16 @@ class CanutilsLogWriter(Listener): It the first message does not have a timestamp, it is set to zero. """ - def __init__(self, filename, channel="vcan0"): + def __init__(self, filename, channel="vcan0", append=False): + mode = 'Uat' if append else 'Uwt' + super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode=mode) + self.channel = channel - self.log_file = open(filename, 'w') self.last_timestamp = None - def stop(self): - """Stops logging and closes the file.""" - if self.log_file is not None: - self.log_file.close() - self.log_file = None - else: - log.warn("ignoring attempt to colse a already closed file") - def on_message_received(self, msg): - if self.log_file is None: + # TODO handle uniform + if self.file is None: log.warn("ignoring write attempt to closed file") return @@ -117,18 +119,17 @@ def on_message_received(self, msg): channel = msg.channel if msg.channel is not None else self.channel if msg.is_error_frame: - self.log_file.write("(%f) %s %08X#0000000000000000\n" % (timestamp, channel, CAN_ERR_FLAG | CAN_ERR_BUSERROR)) + self.file.write("(%f) %s %08X#0000000000000000\n" % (timestamp, channel, CAN_ERR_FLAG | CAN_ERR_BUSERROR)) elif msg.is_remote_frame: - data = [] if msg.is_extended_id: - self.log_file.write("(%f) %s %08X#R\n" % (timestamp, channel, msg.arbitration_id)) + self.file.write("(%f) %s %08X#R\n" % (timestamp, channel, msg.arbitration_id)) else: - self.log_file.write("(%f) %s %03X#R\n" % (timestamp, channel, msg.arbitration_id)) + self.file.write("(%f) %s %03X#R\n" % (timestamp, channel, msg.arbitration_id)) else: data = ["{:02X}".format(byte) for byte in msg.data] if msg.is_extended_id: - self.log_file.write("(%f) %s %08X#%s\n" % (timestamp, channel, msg.arbitration_id, ''.join(data))) + self.file.write("(%f) %s %08X#%s\n" % (timestamp, channel, msg.arbitration_id, ''.join(data))) else: - self.log_file.write("(%f) %s %03X#%s\n" % (timestamp, channel, msg.arbitration_id, ''.join(data))) + self.file.write("(%f) %s %03X#%s\n" % (timestamp, channel, msg.arbitration_id, ''.join(data))) From 87984605e4bf90e848827a42ee0e84a2d814047b Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 09:48:54 +0200 Subject: [PATCH 05/95] docs & fixes --- can/io/canutils.py | 9 ++------- can/io/csv.py | 4 ++-- can/io/player.py | 10 +++++----- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/can/io/canutils.py b/can/io/canutils.py index 0553e8a49..0bf8dd519 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -37,7 +37,7 @@ class CanutilsLogReader(BaseIOHandler): """ def __init__(self, filename): - super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode='Urt') + super(CanutilsLogReader, self).__init__(open_file=True, filename=filename, mode='Urt') def __iter__(self): for line in self.file: @@ -95,17 +95,12 @@ class CanutilsLogWriter(BaseIOHandler, Listener): def __init__(self, filename, channel="vcan0", append=False): mode = 'Uat' if append else 'Uwt' - super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode=mode) + super(CanutilsLogWriter, self).__init__(open_file=True, filename=filename, mode=mode) self.channel = channel self.last_timestamp = None def on_message_received(self, msg): - # TODO handle uniform - if self.file is None: - log.warn("ignoring write attempt to closed file") - return - # this is the case for the very first message: if self.last_timestamp is None: self.last_timestamp = (msg.timestamp or 0.0) diff --git a/can/io/csv.py b/can/io/csv.py index b7b4c4d7c..4b1b8bd8b 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -43,7 +43,7 @@ class CSVWriter(BaseIOHandler, Listener): def __init__(self, filename, append=False): mode = 'Uat' if append else 'Uwt' - super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode=mode) + super(CSVWriter, self).__init__(open_file=True, filename=filename, mode=mode) # Write a header row self.file.write("timestamp,arbitration_id,extended,remote,error,dlc,data\n") @@ -71,7 +71,7 @@ class CSVReader(BaseIOHandler): """ def __init__(self, filename): - super(BaseIOHandler, self).__init__(open_file=True, filename=filename, mode='Urt') + super(CSVReader, self).__init__(open_file=True, filename=filename, mode='Urt') def __iter__(self): # skip the header line diff --git a/can/io/player.py b/can/io/player.py index 958f6a8dd..2a3b10fed 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -40,7 +40,7 @@ class LogReader(object): .. note:: There are no time delays, if you want to reproduce the measured delays between messages look at the - :class:`can.util.MessageSync` class. + :class:`can.MessageSync` class. """ @staticmethod @@ -67,12 +67,12 @@ class MessageSync(object): """ def __init__(self, messages, timestamps=True, gap=0.0001, skip=60): - """Creates an new `MessageSync` instance. + """Creates an new **MessageSync** instance. :param messages: An iterable of :class:`can.Message` instances. - :param timestamps: Use the messages' timestamps. - :param gap: Minimum time between sent messages - :param skip: Skip periods of inactivity greater than this. + :param bool timestamps: Use the messages' timestamps. + :param float gap: Minimum time between sent messages in seconds + :param float skip: Skip periods of inactivity greater than this (in seconds). """ self.raw_messages = messages self.timestamps = timestamps From 10ec6bf3950d4ce53f7e7a5d691938d74ec2e590 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 10:00:18 +0200 Subject: [PATCH 06/95] make ASC ahnders use BaseIOHandler --- can/io/asc.py | 69 +++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index 716f21472..f113cfdc8 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -7,6 +7,8 @@ Example .asc file: https://bitbucket.org/tobylorenz/vector_asc/src/47556e1a6d32c859224ca62d075e1efcc67fa690/src/Vector/ASC/tests/unittests/data/CAN_Log_Trigger_3_2.asc?at=master&fileviewer=file-view-default """ +from __future__ import absolute_import + from datetime import datetime import time import logging @@ -14,6 +16,7 @@ from can.listener import Listener from can.message import Message from can.util import channel2int +from .generic import BaseIOHandler CAN_MSG_EXT = 0x80000000 CAN_ID_MASK = 0x1FFFFFFF @@ -21,7 +24,7 @@ logger = logging.getLogger('can.io.asc') -class ASCReader(object): +class ASCReader(BaseIOHandler): """ Iterator of CAN messages from a ASC logging file. @@ -29,7 +32,7 @@ class ASCReader(object): """ def __init__(self, filename): - self.file = open(filename, 'r') + super(ASCReader, self).__init__(open_file=True, filename=filename, mode='Urt') @staticmethod def _extract_can_id(str_can_id): @@ -39,19 +42,19 @@ def _extract_can_id(str_can_id): else: is_extended = False can_id = int(str_can_id, 16) - logging.debug('ASCReader: _extract_can_id("%s") -> %x, %r', str_can_id, can_id, is_extended) - return (can_id, is_extended) + #logging.debug('ASCReader: _extract_can_id("%s") -> %x, %r', str_can_id, can_id, is_extended) + return can_id, is_extended def __iter__(self): for line in self.file: - logger.debug("ASCReader: parsing line: '%s'", line.splitlines()[0]) + #logger.debug("ASCReader: parsing line: '%s'", line.splitlines()[0]) temp = line.strip() if not temp or not temp[0].isdigit(): continue try: - (timestamp, channel, dummy) = temp.split(None, 2) # , frameType, dlc, frameData + timestamp, channel, dummy = temp.split(None, 2) # , frameType, dlc, frameData except ValueError: # we parsed an empty comment continue @@ -84,10 +87,10 @@ def __iter__(self): else: try: # this only works if dlc > 0 and thus data is availabe - (can_id_str, _, _, dlc, data) = dummy.split(None, 4) + can_id_str, _, _, dlc, data = dummy.split(None, 4) except ValueError: # but if not, we only want to get the stuff up to the dlc - (can_id_str, _, _, dlc ) = dummy.split(None, 3) + can_id_str, _, _, dlc = dummy.split(None, 3) # and we set data to an empty sequence manually data = '' @@ -97,20 +100,22 @@ def __iter__(self): for byte in data[0:dlc]: frame.append(int(byte, 16)) - (can_id_num, is_extended_id) = self._extract_can_id(can_id_str) + can_id_num, is_extended_id = self._extract_can_id(can_id_str) - msg = Message( - timestamp=timestamp, - arbitration_id=can_id_num & CAN_ID_MASK, - extended_id=is_extended_id, - is_remote_frame=False, - dlc=dlc, - data=frame, - channel=channel) - yield msg + yield Message( + timestamp=timestamp, + arbitration_id=can_id_num & CAN_ID_MASK, + extended_id=is_extended_id, + is_remote_frame=False, + dlc=dlc, + data=frame, + channel=channel + ) + self.stop() -class ASCWriter(Listener): + +class ASCWriter(BaseIOHandler, Listener): """Logs CAN data to an ASCII log file (.asc). The measurement starts with the timestamp of the first registered message. @@ -124,15 +129,14 @@ class ASCWriter(Listener): FORMAT_EVENT = "{timestamp: 9.4f} {message}\n" def __init__(self, filename, channel=1): - # setup + super(ASCWriter, self).__init__(open_file=True, filename=filename, mode='Uwt') self.channel = channel - self.log_file = open(filename, 'w') # write start of file header now = datetime.now().strftime("%a %b %m %I:%M:%S %p %Y") - self.log_file.write("date %s\n" % now) - self.log_file.write("base hex timestamps absolute\n") - self.log_file.write("internal events logged\n") + self.file.write("date %s\n" % now) + self.file.write("base hex timestamps absolute\n") + self.file.write("internal events logged\n") # the last part is written with the timestamp of the first message self.header_written = False @@ -140,10 +144,9 @@ def __init__(self, filename, channel=1): self.started = None def stop(self): - """Stops logging and closes the file.""" - if not self.log_file.closed: - self.log_file.write("End TriggerBlock\n") - self.log_file.close() + if not self.file.closed: + self.file.write("End TriggerBlock\n") + super(ASCWriter, self).stop() def log_event(self, message, timestamp=None): """Add a message to the log file. @@ -161,8 +164,8 @@ def log_event(self, message, timestamp=None): self.last_timestamp = (timestamp or 0.0) self.started = self.last_timestamp formatted_date = time.strftime(self.FORMAT_DATE, time.localtime(self.last_timestamp)) - self.log_file.write("base hex timestamps absolute\n") - self.log_file.write("Begin Triggerblock %s\n" % formatted_date) + self.file.write("base hex timestamps absolute\n") + self.file.write("Begin Triggerblock %s\n" % formatted_date) self.header_written = True self.log_event("Start of measurement") # recursive call @@ -175,11 +178,7 @@ def log_event(self, message, timestamp=None): timestamp -= self.started line = self.FORMAT_EVENT.format(timestamp=timestamp, message=message) - - if self.log_file.closed: - logger.warn("ASCWriter: ignoring write call to closed file") - else: - self.log_file.write(line) + self.file.write(line) def on_message_received(self, msg): From a6013911f1023472798ca8ae8aacf380de23a416 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 10:08:09 +0200 Subject: [PATCH 07/95] fix universal nelines support --- can/io/asc.py | 2 +- can/io/canutils.py | 2 +- can/io/csv.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index f113cfdc8..56d72954a 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -129,7 +129,7 @@ class ASCWriter(BaseIOHandler, Listener): FORMAT_EVENT = "{timestamp: 9.4f} {message}\n" def __init__(self, filename, channel=1): - super(ASCWriter, self).__init__(open_file=True, filename=filename, mode='Uwt') + super(ASCWriter, self).__init__(open_file=True, filename=filename, mode='wt') self.channel = channel # write start of file header diff --git a/can/io/canutils.py b/can/io/canutils.py index 0bf8dd519..0ead90243 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -94,7 +94,7 @@ class CanutilsLogWriter(BaseIOHandler, Listener): """ def __init__(self, filename, channel="vcan0", append=False): - mode = 'Uat' if append else 'Uwt' + mode = 'at' if append else 'wt' super(CanutilsLogWriter, self).__init__(open_file=True, filename=filename, mode=mode) self.channel = channel diff --git a/can/io/csv.py b/can/io/csv.py index 4b1b8bd8b..b7325f3c2 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -42,7 +42,7 @@ class CSVWriter(BaseIOHandler, Listener): """ def __init__(self, filename, append=False): - mode = 'Uat' if append else 'Uwt' + mode = 'at' if append else 'wt' super(CSVWriter, self).__init__(open_file=True, filename=filename, mode=mode) # Write a header row From c40ab53f692050c9325771f3c50b02f4f4190562 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 10:12:02 +0200 Subject: [PATCH 08/95] fix sphinx error --- can/listener.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/can/listener.py b/can/listener.py index 279dcfd8e..9a9f56ed2 100644 --- a/can/listener.py +++ b/can/listener.py @@ -16,7 +16,8 @@ class Listener(object): - """The basic listener that can be called directly to deliver a message:: + """The basic listener that can be called directly to handle some + CAN message:: listener = SomeListener() msg = my_bus.recv() From 190653bfd6c82595ecdf776141b95ca5fb386ced Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 16:23:54 +0200 Subject: [PATCH 09/95] make stdout use BaseIOHandler --- can/io/stdout.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/can/io/stdout.py b/can/io/stdout.py index c0a82ab5a..5d20a6c2e 100644 --- a/can/io/stdout.py +++ b/can/io/stdout.py @@ -5,36 +5,31 @@ This Listener simply prints to stdout / the terminal or a file. """ -from __future__ import print_function +from __future__ import print_function, absolute_import import logging from can.listener import Listener +from .generic import BaseIOHandler -log = logging.getLogger('can.io.stdout') +log = logging.getLogger('can.io.printer') -class Printer(Listener): +class Printer(BaseIOHandler, Listener): """ The Printer class is a subclass of :class:`~can.Listener` which simply prints - any messages it receives to the terminal (stdout). + any messages it receives to the terminal (stdout). A message is tunred into a + string using :meth:`~can.Message.__str__`. - :param output_file: An optional file to "print" to. + :param str output_file: An optional file to "print" to. """ - def __init__(self, output_file=None): - if output_file is not None: - log.info('Creating log file "{}"'.format(output_file)) - output_file = open(output_file, 'wt') - self.output_file = output_file + def __init__(self, filename=None): + self.write_to_file = filename is not None + super(Printer, self).__init__(open_file=self.write_to_file, filename=filename, mode='wt') def on_message_received(self, msg): - if self.output_file is not None: - self.output_file.write(str(msg) + '\n') + if self.write_to_file: + self.file.write(str(msg) + '\n') else: print(msg) - - def stop(self): - if self.output_file: - self.output_file.write('\n') - self.output_file.close() From 3368c24e702017f68603da2cec1897681171a0dc Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 16:24:06 +0200 Subject: [PATCH 10/95] docs --- can/io/generic.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/can/io/generic.py b/can/io/generic.py index 97bc76555..c50fb32f4 100644 --- a/can/io/generic.py +++ b/can/io/generic.py @@ -2,7 +2,7 @@ # coding: utf-8 """ -Contains generic classes for file IO. +Contains a generic class for file IO. """ from abc import ABCMeta, abstractmethod @@ -12,14 +12,20 @@ class BaseIOHandler(object): """A generic file handler that can be used for reading and writing. + + Can be used as a context manager. """ __metaclass__ = ABCMeta def __init__(self, open_file, filename='can_data', mode='Urt'): """ - TODO docs + :param bool open_file: opens a file if set to True + :param str filename: the path/filename of the file to open + :param str mode: the mode that should be used to open the file, see + :func:`builtin.open` """ + super(BaseIOHandler, self).__init__() # for multiple inheritance if open_file: self.file = open(filename, mode) From 4d9719491de1edd8584b7e96dbe9199765ad97ac Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 16:24:24 +0200 Subject: [PATCH 11/95] make SQlite reader/writer use the new BaseIOHandler --- can/io/sqlite.py | 156 ++++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 75 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 5f3255729..e4e8cac71 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -4,9 +4,11 @@ """ Implements an SQL database writer and reader for storing CAN messages. -The database schema is given in the documentation of the loggers. +.. note:: The database schema is given in the documentation of the loggers. """ +from __future__ import absolute_import + import sys import time import threading @@ -15,59 +17,57 @@ from can.listener import BufferedReader from can.message import Message +from .generic import BaseIOHandler -log = logging.getLogger('can.io.sql') +log = logging.getLogger('can.io.sqlite') # TODO comment on this if sys.version_info > (3,): buffer = memoryview -class SqliteReader: +class SqliteReader(BaseIOHandler): """ Reads recorded CAN messages from a simple SQL database. This class can be iterated over or used to fetch all messages in the database with :meth:`~SqliteReader.read_all`. - Calling len() on this object might not run in constant time. - """ + Calling :func:`~builtin.len` on this object might not run in constant time. - _SELECT_ALL_COMMAND = "SELECT * FROM messages" + .. note:: The database schema is given in the documentation of the loggers. + """ - def __init__(self, filename): - log.debug("Starting SqliteReader with %s", filename) + def __init__(self, filename, table_name="messages"): + super(SqliteReader, self).__init__(open_file=False) self.conn = sqlite3.connect(filename) self.cursor = self.conn.cursor() - - @staticmethod - def _create_frame_from_db_tuple(frame_data): - timestamp, can_id, is_extended, is_remote, is_error, dlc, data = frame_data - return Message( - timestamp, is_remote, is_extended, is_error, can_id, dlc, data - ) + self.table_name = table_name def __iter__(self): - log.debug("Iterating through messages from sql db") - for frame_data in self.cursor.execute(self._SELECT_ALL_COMMAND): - yield self._create_frame_from_db_tuple(frame_data) + for frame_data in self.cursor.execute("SELECT * FROM {}".format(self.table_name)): + timestamp, can_id, is_extended, is_remote, is_error, dlc, data = frame_data + yield Message(timestamp, is_remote, is_extended, is_error, can_id, dlc, data) def __len__(self): # this might not run in constant time - result = self.cursor.execute("SELECT COUNT(*) FROM messages") + result = self.cursor.execute("SELECT COUNT(*) FROM {}".format(self.table_name)) return int(result.fetchone()[0]) def read_all(self): - """Fetches all messages in the database.""" - result = self.cursor.execute(self._SELECT_ALL_COMMAND) + """Fetches all messages in the database. + """ + result = self.cursor.execute("SELECT * FROM {}".format(self.table_name)) return result.fetchall() - def close(self): - """Closes the connection to the database.""" + def stop(self): + """Closes the connection to the database. + """ + super(SqliteReader, self).stop() self.conn.close() -class SqliteWriter(BufferedReader): +class SqliteWriter(BaseIOHandler, BufferedReader): """Logs received CAN data to a simple SQL database. The sqlite database may already exist, otherwise it will @@ -84,42 +84,44 @@ class SqliteWriter(BufferedReader): :attr:`~SqliteWriter.GET_MESSAGE_TIMEOUT`. If the :attr:`~SqliteWriter.GET_MESSAGE_TIMEOUT` expires before a message - is received, the internal buffer is written out to the sql file. + is received, the internal buffer is written out to the databases file. However if the bus is still saturated with messages, the Listener will continue receiving until the :attr:`~SqliteWriter.MAX_TIME_BETWEEN_WRITES` timeout is reached. - """ + .. note:: The database schema is given in the documentation of the loggers. - _INSERT_MSG_TEMPLATE = ''' - INSERT INTO messages VALUES - (?, ?, ?, ?, ?, ?, ?) - ''' + """ GET_MESSAGE_TIMEOUT = 0.25 """Number of seconds to wait for messages from internal queue""" - MAX_TIME_BETWEEN_WRITES = 5 + MAX_TIME_BETWEEN_WRITES = 5.0 """Maximum number of seconds to wait between writes to the database""" - def __init__(self, filename): + def __init__(self, filename, table_name="messages"): super(SqliteWriter, self).__init__() - self.db_fn = filename + self.table_name = table_name + self.filename = filename self.stop_running_event = threading.Event() self.writer_thread = threading.Thread(target=self._db_writer_thread) self.writer_thread.start() def _create_db(self): - # Note: you can't share sqlite3 connections between threads - # hence we setup the db here. - log.info("Creating sqlite database") - self.conn = sqlite3.connect(self.db_fn) + """Creates a new databae or opens a connection to an existing one. + + .. note:: + You can't share sqlite3 connections between threads hence we + setup the db here. + """ + log.debug("Creating sqlite database") + self.conn = sqlite3.connect(self.filename) cursor = self.conn.cursor() # create table structure - cursor.execute(''' - CREATE TABLE IF NOT EXISTS messages + cursor.execute(""" + CREATE TABLE IF NOT EXISTS {} ( ts REAL, arbitration_id INTEGER, @@ -129,52 +131,56 @@ def _create_db(self): dlc INTEGER, data BLOB ) - ''') + """.format(self.table_name)) self.conn.commit() + self.insert_template = "INSERT INTO {} VALUES (?, ?, ?, ?, ?, ?, ?)".format(self.table_name) + def _db_writer_thread(self): num_frames = 0 last_write = time.time() self._create_db() - while not self.stop_running_event.is_set(): - messages = [] - - msg = self.get_message(self.GET_MESSAGE_TIMEOUT) - while msg is not None: - log.debug("SqliteWriter: buffering message") - - messages.append(( - msg.timestamp, - msg.arbitration_id, - msg.id_type, - msg.is_remote_frame, - msg.is_error_frame, - msg.dlc, - buffer(msg.data) - )) - - if time.time() - last_write > self.MAX_TIME_BETWEEN_WRITES: - log.debug("Max timeout between writes reached") - break + try: + while not self.stop_running_event.is_set(): + messages = [] msg = self.get_message(self.GET_MESSAGE_TIMEOUT) - - count = len(messages) - if count > 0: - with self.conn: - log.debug("Writing %s frames to db", count) - self.conn.executemany(SqliteWriter._INSERT_MSG_TEMPLATE, messages) - self.conn.commit() # make the changes visible to the entire database - num_frames += count - last_write = time.time() - - # go back up and check if we are still supposed to run - - self.conn.close() - log.info("Stopped sqlite writer after writing %s messages", num_frames) + while msg is not None: + #log.debug("SqliteWriter: buffering message") + + messages.append(( + msg.timestamp, + msg.arbitration_id, + msg.id_type, + msg.is_remote_frame, + msg.is_error_frame, + msg.dlc, + buffer(msg.data) + )) + + if time.time() - last_write > self.MAX_TIME_BETWEEN_WRITES: + #log.debug("Max timeout between writes reached") + break + + msg = self.get_message(self.GET_MESSAGE_TIMEOUT) + + count = len(messages) + if count > 0: + with self.conn: + #log.debug("Writing %s frames to db", count) + self.conn.executemany(self.insert_template, messages) + self.conn.commit() # make the changes visible to the entire database + num_frames += count + last_write = time.time() + + # go back up and check if we are still supposed to run + + finally: + self.conn.close() + log.info("Stopped sqlite writer after writing %s messages", num_frames) def stop(self): + super(SqliteWriter, self).stop() self.stop_running_event.set() - log.debug("Stopping sqlite writer") self.writer_thread.join() From 906cfb1716e47773d15de37b22d3b6610db4c89c Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 16:33:49 +0200 Subject: [PATCH 12/95] added BaseIOHandler in BLF reder/Writer --- can/io/blf.py | 47 +++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index 8115e04e2..b202eb99c 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -15,6 +15,8 @@ objects types. """ +from __future__ import absolute_import + import struct import zlib import datetime @@ -24,6 +26,7 @@ from can.message import Message from can.listener import Listener from can.util import len2dlc, dlc2len, channel2int +from .generic import BaseIOHandler class BLFParseError(Exception): @@ -112,7 +115,7 @@ def systemtime_to_timestamp(systemtime): return 0 -class BLFReader(object): +class BLFReader(BaseIOHandler): """ Iterator of CAN messages from a Binary Logging File. @@ -121,10 +124,9 @@ class BLFReader(object): """ def __init__(self, filename): - self.fp = open(filename, "rb") - data = self.fp.read(FILE_HEADER_STRUCT.size) + super(BLFReader, self).__init__(open_file=True, filename=filename, mode='Urt') + data = self.file.read(FILE_HEADER_STRUCT.size) header = FILE_HEADER_STRUCT.unpack(data) - #print(header) if header[0] != b"LOGG": raise BLFParseError("Unexpected file format") self.file_size = header[10] @@ -133,25 +135,24 @@ def __init__(self, filename): self.start_timestamp = systemtime_to_timestamp(header[14:22]) self.stop_timestamp = systemtime_to_timestamp(header[22:30]) # Read rest of header - self.fp.read(header[1] - FILE_HEADER_STRUCT.size) + self.file.read(header[1] - FILE_HEADER_STRUCT.size) def __iter__(self): tail = b"" while True: - data = self.fp.read(OBJ_HEADER_BASE_STRUCT.size) + data = self.file.read(OBJ_HEADER_BASE_STRUCT.size) if not data: # EOF break header = OBJ_HEADER_BASE_STRUCT.unpack(data) - #print(header) if header[0] != b"LOBJ": raise BLFParseError() obj_type = header[4] obj_data_size = header[3] - OBJ_HEADER_BASE_STRUCT.size - obj_data = self.fp.read(obj_data_size) + obj_data = self.file.read(obj_data_size) # Read padding bytes - self.fp.read(obj_data_size % 4) + self.file.read(obj_data_size % 4) if obj_type == LOG_CONTAINER: method, uncompressed_size = LOG_CONTAINER_STRUCT.unpack_from( @@ -245,13 +246,13 @@ def __iter__(self): pos = next_pos - # Save remaing data that could not be processed + # save remainig data that could not be processed tail = data[pos:] - self.fp.close() + self.stop() -class BLFWriter(Listener): +class BLFWriter(BaseIOHandler, Listener): """ Logs CAN data to a Binary Logging File compatible with Vector's tools. """ @@ -263,10 +264,10 @@ class BLFWriter(Listener): COMPRESSION_LEVEL = 9 def __init__(self, filename, channel=1): - self.fp = open(filename, "wb") + super(BLFWriter, self).__init__(open_file=True, filename=filename, mode='wt') self.channel = channel # Header will be written after log is done - self.fp.write(b"\x00" * FILE_HEADER_SIZE) + self.file.write(b"\x00" * FILE_HEADER_SIZE) self.cache = [] self.cache_size = 0 self.count_of_objects = 0 @@ -360,7 +361,7 @@ def _add_object(self, obj_type, data, timestamp=None): def _flush(self): """Compresses and writes data in the cache to file.""" - if self.fp.closed: + if self.file.closed: return cache = b"".join(self.cache) if not cache: @@ -379,21 +380,19 @@ def _flush(self): b"LOBJ", OBJ_HEADER_BASE_STRUCT.size, 1, obj_size, LOG_CONTAINER) container_header = LOG_CONTAINER_STRUCT.pack( ZLIB_DEFLATE, len(uncompressed_data)) - self.fp.write(base_header) - self.fp.write(container_header) - self.fp.write(compressed_data) + self.file.write(base_header) + self.file.write(container_header) + self.file.write(compressed_data) # Write padding bytes - self.fp.write(b"\x00" * (obj_size % 4)) + self.file.write(b"\x00" * (obj_size % 4)) self.uncompressed_size += OBJ_HEADER_V1_STRUCT.size + LOG_CONTAINER_STRUCT.size self.uncompressed_size += len(uncompressed_data) def stop(self): """Stops logging and closes the file.""" - if self.fp.closed: - return self._flush() - filesize = self.fp.tell() - self.fp.close() + filesize = self.file.tell() + super(BLFWriter, self).stop() # Write header in the beginning of the file header = [b"LOGG", FILE_HEADER_SIZE, @@ -403,5 +402,5 @@ def stop(self): self.count_of_objects, 0]) header.extend(timestamp_to_systemtime(self.start_timestamp)) header.extend(timestamp_to_systemtime(self.stop_timestamp)) - with open(self.fp.name, "r+b") as f: + with open(self.file.name, "r+b") as f: f.write(FILE_HEADER_STRUCT.pack(*header)) From d68c430bca696f045799143f1ed704506e83b719 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 16:36:25 +0200 Subject: [PATCH 13/95] sqlite fix --- can/io/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index e4e8cac71..1b1fcc7f5 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -101,7 +101,7 @@ class SqliteWriter(BaseIOHandler, BufferedReader): """Maximum number of seconds to wait between writes to the database""" def __init__(self, filename, table_name="messages"): - super(SqliteWriter, self).__init__() + super(SqliteWriter, self).__init__(open_file=False) self.table_name = table_name self.filename = filename self.stop_running_event = threading.Event() From b10080ed1f4559f7260d0c16162f6513bd849a60 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 16:48:17 +0200 Subject: [PATCH 14/95] fix file opening in BLF handlers --- can/io/blf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index b202eb99c..7796c279c 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -124,7 +124,7 @@ class BLFReader(BaseIOHandler): """ def __init__(self, filename): - super(BLFReader, self).__init__(open_file=True, filename=filename, mode='Urt') + super(BLFReader, self).__init__(open_file=True, filename=filename, mode='Urb') data = self.file.read(FILE_HEADER_STRUCT.size) header = FILE_HEADER_STRUCT.unpack(data) if header[0] != b"LOGG": @@ -264,7 +264,7 @@ class BLFWriter(BaseIOHandler, Listener): COMPRESSION_LEVEL = 9 def __init__(self, filename, channel=1): - super(BLFWriter, self).__init__(open_file=True, filename=filename, mode='wt') + super(BLFWriter, self).__init__(open_file=True, filename=filename, mode='wb') self.channel = channel # Header will be written after log is done self.file.write(b"\x00" * FILE_HEADER_SIZE) From 0fcbf072e0999acde48a4eb2063a47741a24d13c Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 17:00:11 +0200 Subject: [PATCH 15/95] rome "text mode" parameter form file open calls --- can/io/asc.py | 4 ++-- can/io/canutils.py | 4 ++-- can/io/csv.py | 4 ++-- can/io/stdout.py | 2 +- test/logformats_test.py | 5 ++--- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index 56d72954a..dc1a2daac 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -32,7 +32,7 @@ class ASCReader(BaseIOHandler): """ def __init__(self, filename): - super(ASCReader, self).__init__(open_file=True, filename=filename, mode='Urt') + super(ASCReader, self).__init__(open_file=True, filename=filename, mode='Ur') @staticmethod def _extract_can_id(str_can_id): @@ -129,7 +129,7 @@ class ASCWriter(BaseIOHandler, Listener): FORMAT_EVENT = "{timestamp: 9.4f} {message}\n" def __init__(self, filename, channel=1): - super(ASCWriter, self).__init__(open_file=True, filename=filename, mode='wt') + super(ASCWriter, self).__init__(open_file=True, filename=filename, mode='w') self.channel = channel # write start of file header diff --git a/can/io/canutils.py b/can/io/canutils.py index 0ead90243..d1d69486f 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -37,7 +37,7 @@ class CanutilsLogReader(BaseIOHandler): """ def __init__(self, filename): - super(CanutilsLogReader, self).__init__(open_file=True, filename=filename, mode='Urt') + super(CanutilsLogReader, self).__init__(open_file=True, filename=filename, mode='Ur') def __iter__(self): for line in self.file: @@ -94,7 +94,7 @@ class CanutilsLogWriter(BaseIOHandler, Listener): """ def __init__(self, filename, channel="vcan0", append=False): - mode = 'at' if append else 'wt' + mode = 'a' if append else 'w' super(CanutilsLogWriter, self).__init__(open_file=True, filename=filename, mode=mode) self.channel = channel diff --git a/can/io/csv.py b/can/io/csv.py index b7325f3c2..e7e81ee81 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -42,7 +42,7 @@ class CSVWriter(BaseIOHandler, Listener): """ def __init__(self, filename, append=False): - mode = 'at' if append else 'wt' + mode = 'a' if append else 'w' super(CSVWriter, self).__init__(open_file=True, filename=filename, mode=mode) # Write a header row @@ -71,7 +71,7 @@ class CSVReader(BaseIOHandler): """ def __init__(self, filename): - super(CSVReader, self).__init__(open_file=True, filename=filename, mode='Urt') + super(CSVReader, self).__init__(open_file=True, filename=filename, mode='Ur') def __iter__(self): # skip the header line diff --git a/can/io/stdout.py b/can/io/stdout.py index 5d20a6c2e..95c7466a0 100644 --- a/can/io/stdout.py +++ b/can/io/stdout.py @@ -26,7 +26,7 @@ class Printer(BaseIOHandler, Listener): def __init__(self, filename=None): self.write_to_file = filename is not None - super(Printer, self).__init__(open_file=self.write_to_file, filename=filename, mode='wt') + super(Printer, self).__init__(open_file=self.write_to_file, filename=filename, mode='w') def on_message_received(self, msg): if self.write_to_file: diff --git a/test/logformats_test.py b/test/logformats_test.py index e7d8b4bf7..67cb84b5f 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -13,8 +13,7 @@ TODO: implement CAN FD support testing """ -from __future__ import print_function -from __future__ import absolute_import +from __future__ import print_function, absolute_import import unittest import tempfile @@ -41,7 +40,7 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s check_comments=False, round_timestamps=False): """Tests a pair of writer and reader by writing all data first and then reading all data and checking if they could be reconstructed - correctly. + correctly. Optionally writes some comments as well. :param unittest.TestCase test_case: the test case the use the assert methods on :param Callable writer_constructor: the constructor of the writer class From 6d97d20afe1a33f709973a80735c86704ad1ad6a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 17:23:18 +0200 Subject: [PATCH 16/95] fix bad test cases --- test/listener_test.py | 13 +++++++------ test/logformats_test.py | 7 ++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/test/listener_test.py b/test/listener_test.py index b2a80382c..446fb4a37 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -69,13 +69,13 @@ def tearDown(self): class ListenerTest(BusTest): def testBasicListenerCanBeAddedToNotifier(self): - a_listener = can.Listener() + a_listener = can.Printer() notifier = can.Notifier(self.bus, [a_listener], 0.1) notifier.stop() self.assertIn(a_listener, notifier.listeners) - + def testAddListenerToNotifier(self): - a_listener = can.Listener() + a_listener = can.Printer() notifier = can.Notifier(self.bus, [], 0.1) notifier.stop() self.assertNotIn(a_listener, notifier.listeners) @@ -83,7 +83,7 @@ def testAddListenerToNotifier(self): self.assertIn(a_listener, notifier.listeners) def testRemoveListenerFromNotifier(self): - a_listener = can.Listener() + a_listener = can.Printer() notifier = can.Notifier(self.bus, [a_listener], 0.1) notifier.stop() self.assertIn(a_listener, notifier.listeners) @@ -113,7 +113,8 @@ def testLoggerTypeResolution(self): def test_filetype_to_instance(extension, klass): can_logger = can.Logger("test.{}".format(extension)) self.assertIsInstance(can_logger, klass) - can_logger.stop() + if hasattr(can_logger, "stop"): + can_logger.stop() test_filetype_to_instance("asc", can.ASCWriter) test_filetype_to_instance("blf", can.BLFWriter) @@ -122,7 +123,7 @@ def test_filetype_to_instance(extension, klass): test_filetype_to_instance("log", can.CanutilsLogWriter) test_filetype_to_instance("txt", can.Printer) - # test file extensions that should usa a fallback + # test file extensions that should use a fallback test_filetype_to_instance(None, can.Printer) test_filetype_to_instance("some_unknown_extention_42", can.Printer) diff --git a/test/logformats_test.py b/test/logformats_test.py index 67cb84b5f..d025d025a 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -95,17 +95,14 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s if comment is not None: print("writing comment: ", comment) writer.log_event(comment) # we already know that this method exists - print("writing comment: ", comment) if msg is not None: print("writing message: ", msg) writer(msg) - print("writing message: ", msg) else: # ony write messages for msg in original_messages: print("writing message: ", msg) writer(msg) - print("writing message: ", msg) # sleep and close the writer if sleep_time is not None: @@ -114,6 +111,7 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s writer.stop() # read all written messages + print("reading all messages ...") read_messages = list(reader_constructor(filename)) # check if at least the number of messages matches @@ -123,6 +121,9 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s # check the order and content of the individual messages for i, (read, original) in enumerate(zip(read_messages, original_messages)): try: + if read != original: + print("original message: {}".format(original)) + print("read message: {}".format(read)) # check everything except the timestamp test_case.assertEqual(read, original) # check the timestamp From 9e88d106371062cd87d64d31b6818ff3aabac456 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 17:37:44 +0200 Subject: [PATCH 17/95] only writer header if not appending --- can/io/csv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/can/io/csv.py b/can/io/csv.py index e7e81ee81..6a3ab34ac 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -46,7 +46,8 @@ def __init__(self, filename, append=False): super(CSVWriter, self).__init__(open_file=True, filename=filename, mode=mode) # Write a header row - self.file.write("timestamp,arbitration_id,extended,remote,error,dlc,data\n") + if not append: + self.file.write("timestamp,arbitration_id,extended,remote,error,dlc,data\n") def on_message_received(self, msg): row = ','.join([ From a9a961cd8b993f91f647991a617a664d8de1ed3b Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 22:32:51 +0200 Subject: [PATCH 18/95] address code review by @christiansandberg --- can/io/asc.py | 2 +- can/io/blf.py | 2 +- can/io/canutils.py | 2 +- can/io/csv.py | 8 ++++---- can/io/generic.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index dc1a2daac..30fffb996 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -32,7 +32,7 @@ class ASCReader(BaseIOHandler): """ def __init__(self, filename): - super(ASCReader, self).__init__(open_file=True, filename=filename, mode='Ur') + super(ASCReader, self).__init__(open_file=True, filename=filename, mode='r') @staticmethod def _extract_can_id(str_can_id): diff --git a/can/io/blf.py b/can/io/blf.py index 7796c279c..035465b44 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -124,7 +124,7 @@ class BLFReader(BaseIOHandler): """ def __init__(self, filename): - super(BLFReader, self).__init__(open_file=True, filename=filename, mode='Urb') + super(BLFReader, self).__init__(open_file=True, filename=filename, mode='rb') data = self.file.read(FILE_HEADER_STRUCT.size) header = FILE_HEADER_STRUCT.unpack(data) if header[0] != b"LOGG": diff --git a/can/io/canutils.py b/can/io/canutils.py index d1d69486f..26d19894c 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -37,7 +37,7 @@ class CanutilsLogReader(BaseIOHandler): """ def __init__(self, filename): - super(CanutilsLogReader, self).__init__(open_file=True, filename=filename, mode='Ur') + super(CanutilsLogReader, self).__init__(open_file=True, filename=filename, mode='r') def __iter__(self): for line in self.file: diff --git a/can/io/csv.py b/can/io/csv.py index 6a3ab34ac..ea2100336 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -72,7 +72,7 @@ class CSVReader(BaseIOHandler): """ def __init__(self, filename): - super(CSVReader, self).__init__(open_file=True, filename=filename, mode='Ur') + super(CSVReader, self).__init__(open_file=True, filename=filename, mode='r') def __iter__(self): # skip the header line @@ -84,9 +84,9 @@ def __iter__(self): yield Message( timestamp=float(timestamp), - is_remote_frame=bool(remote), - extended_id=bool(extended), - is_error_frame=bool(error), + is_remote_frame=(remote == '1'), + extended_id=(extended == '1'), + is_error_frame=(error == '1'), arbitration_id=int(arbitration_id, base=16), dlc=int(dlc), data=b64decode(data), diff --git a/can/io/generic.py b/can/io/generic.py index c50fb32f4..51518c060 100644 --- a/can/io/generic.py +++ b/can/io/generic.py @@ -18,7 +18,7 @@ class BaseIOHandler(object): __metaclass__ = ABCMeta - def __init__(self, open_file, filename='can_data', mode='Urt'): + def __init__(self, open_file, filename='can_data', mode='rt'): """ :param bool open_file: opens a file if set to True :param str filename: the path/filename of the file to open From def5a0400cb10e0ec8510434ad8877b6462f7e0c Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 3 Jul 2018 22:39:29 +0200 Subject: [PATCH 19/95] add docs --- can/io/canutils.py | 6 ++++++ can/io/csv.py | 11 +++++++++-- can/io/sqlite.py | 6 ++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/can/io/canutils.py b/can/io/canutils.py index 26d19894c..623d12538 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -94,6 +94,12 @@ class CanutilsLogWriter(BaseIOHandler, Listener): """ def __init__(self, filename, channel="vcan0", append=False): + """ + :param channel: a default channel to use when the message does not + have a channel set + :param bool append: if set to `True` messages are appended to + the file, else the file is truncated + """ mode = 'a' if append else 'w' super(CanutilsLogWriter, self).__init__(open_file=True, filename=filename, mode=mode) diff --git a/can/io/csv.py b/can/io/csv.py index ea2100336..50cd1c75f 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -22,7 +22,7 @@ class CSVWriter(BaseIOHandler, Listener): """Writes a comma separated text file with a line for - each message. + each message. Includes a header line. The columns are as follows: @@ -42,6 +42,12 @@ class CSVWriter(BaseIOHandler, Listener): """ def __init__(self, filename, append=False): + """ + :param bool append: if set to `True` messages are appended to + the file and no header line is written, else + the file is truncated and starts with a newly + written header line + """ mode = 'a' if append else 'w' super(CSVWriter, self).__init__(open_file=True, filename=filename, mode=mode) @@ -66,7 +72,8 @@ def on_message_received(self, msg): class CSVReader(BaseIOHandler): """Iterator over CAN messages from a .csv file that was generated by :class:`~can.CSVWriter` or that uses the same - format as described there. + format as described there. Assumes that there is a header + and thus skips the first line. Any line seperator is accepted. """ diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 1b1fcc7f5..5c00de918 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -39,6 +39,9 @@ class SqliteReader(BaseIOHandler): """ def __init__(self, filename, table_name="messages"): + """ + :param str table_name: the name of the table to look for the messages + """ super(SqliteReader, self).__init__(open_file=False) self.conn = sqlite3.connect(filename) self.cursor = self.conn.cursor() @@ -101,6 +104,9 @@ class SqliteWriter(BaseIOHandler, BufferedReader): """Maximum number of seconds to wait between writes to the database""" def __init__(self, filename, table_name="messages"): + """ + :param str table_name: the name of the table to store messages in + """ super(SqliteWriter, self).__init__(open_file=False) self.table_name = table_name self.filename = filename From 0490712fac3f5c0cb1a65d5e5487880425847acc Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Jul 2018 16:01:11 +0200 Subject: [PATCH 20/95] allow BaseIOHandler to use path-like objectas and file-like objects --- can/io/asc.py | 8 ++++++-- can/io/blf.py | 6 +++--- can/io/canutils.py | 4 ++-- can/io/csv.py | 4 ++-- can/io/generic.py | 26 ++++++++++++++++++-------- can/io/logger.py | 4 +--- can/io/player.py | 2 +- can/io/sqlite.py | 6 +++--- can/io/stdout.py | 7 ++++--- 9 files changed, 40 insertions(+), 27 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index 30fffb996..212a2e21e 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -32,7 +32,7 @@ class ASCReader(BaseIOHandler): """ def __init__(self, filename): - super(ASCReader, self).__init__(open_file=True, filename=filename, mode='r') + super(ASCReader, self).__init__(file=filename, mode='r') @staticmethod def _extract_can_id(str_can_id): @@ -129,7 +129,11 @@ class ASCWriter(BaseIOHandler, Listener): FORMAT_EVENT = "{timestamp: 9.4f} {message}\n" def __init__(self, filename, channel=1): - super(ASCWriter, self).__init__(open_file=True, filename=filename, mode='w') + """ + :param channel: a default channel to use when the message does not + have a channel set + """ + super(ASCWriter, self).__init__(file=filename, mode='w') self.channel = channel # write start of file header diff --git a/can/io/blf.py b/can/io/blf.py index 035465b44..aee87622a 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -124,7 +124,7 @@ class BLFReader(BaseIOHandler): """ def __init__(self, filename): - super(BLFReader, self).__init__(open_file=True, filename=filename, mode='rb') + super(BLFReader, self).__init__(file=filename, mode='rb') data = self.file.read(FILE_HEADER_STRUCT.size) header = FILE_HEADER_STRUCT.unpack(data) if header[0] != b"LOGG": @@ -246,7 +246,7 @@ def __iter__(self): pos = next_pos - # save remainig data that could not be processed + # save the remaining data that could not be processed tail = data[pos:] self.stop() @@ -264,7 +264,7 @@ class BLFWriter(BaseIOHandler, Listener): COMPRESSION_LEVEL = 9 def __init__(self, filename, channel=1): - super(BLFWriter, self).__init__(open_file=True, filename=filename, mode='wb') + super(BLFWriter, self).__init__(file=filename, mode='wb') self.channel = channel # Header will be written after log is done self.file.write(b"\x00" * FILE_HEADER_SIZE) diff --git a/can/io/canutils.py b/can/io/canutils.py index 623d12538..f150f8fc8 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -37,7 +37,7 @@ class CanutilsLogReader(BaseIOHandler): """ def __init__(self, filename): - super(CanutilsLogReader, self).__init__(open_file=True, filename=filename, mode='r') + super(CanutilsLogReader, self).__init__(file=filename, mode='r') def __iter__(self): for line in self.file: @@ -101,7 +101,7 @@ def __init__(self, filename, channel="vcan0", append=False): the file, else the file is truncated """ mode = 'a' if append else 'w' - super(CanutilsLogWriter, self).__init__(open_file=True, filename=filename, mode=mode) + super(CanutilsLogWriter, self).__init__(file=filename, mode=mode) self.channel = channel self.last_timestamp = None diff --git a/can/io/csv.py b/can/io/csv.py index 50cd1c75f..7d7f89a31 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -49,7 +49,7 @@ def __init__(self, filename, append=False): written header line """ mode = 'a' if append else 'w' - super(CSVWriter, self).__init__(open_file=True, filename=filename, mode=mode) + super(CSVWriter, self).__init__(file=filename, mode=mode) # Write a header row if not append: @@ -79,7 +79,7 @@ class CSVReader(BaseIOHandler): """ def __init__(self, filename): - super(CSVReader, self).__init__(open_file=True, filename=filename, mode='r') + super(CSVReader, self).__init__(file=filename, mode='r') def __iter__(self): # skip the header line diff --git a/can/io/generic.py b/can/io/generic.py index 51518c060..9f3df843f 100644 --- a/can/io/generic.py +++ b/can/io/generic.py @@ -14,20 +14,30 @@ class BaseIOHandler(object): """A generic file handler that can be used for reading and writing. Can be used as a context manager. + + :attr file-like file: + the file-like object that is kept internally, or None if none + was opened """ __metaclass__ = ABCMeta - def __init__(self, open_file, filename='can_data', mode='rt'): + def __init__(self, file='can.data', mode='rt'): """ - :param bool open_file: opens a file if set to True - :param str filename: the path/filename of the file to open + :param str file: a path-like object to open a file, a file-like object + to be used as a file or `None` to not use a file at all :param str mode: the mode that should be used to open the file, see - :func:`builtin.open` + :func:`builtin.open`, ignored if *file* is `None` """ - super(BaseIOHandler, self).__init__() # for multiple inheritance - if open_file: - self.file = open(filename, mode) + if file is None or (hasattr(file, 'read') and hasattr(file, 'write')): + # file is None or some file-like object + self.file = file + else: + # file is some path-like object + self.file = open(file, mode) + + # for multiple inheritance + super(BaseIOHandler, self).__init__() def __enter__(self): return self @@ -36,6 +46,6 @@ def __exit__(self, *args): self.stop() def stop(self): - if hasattr(self, 'file') and self.file: + if self.file is not None: # this also implies a flush() self.file.close() diff --git a/can/io/logger.py b/can/io/logger.py index c4b27815e..26b2abd06 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -36,9 +36,7 @@ class Logger(object): @staticmethod def __new__(cls, filename): - if not filename: - return Printer() - elif filename.endswith(".asc"): + if filename.endswith(".asc"): return ASCWriter(filename) elif filename.endswith(".blf"): return BLFWriter(filename) diff --git a/can/io/player.py b/can/io/player.py index 2a3b10fed..2a64cff18 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -45,7 +45,7 @@ class LogReader(object): @staticmethod def __new__(cls, filename): - if not filename: + if filename is None: raise TypeError("a filename must be given") elif filename.endswith(".asc"): return ASCReader(filename) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 5c00de918..ee11a8c97 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -42,7 +42,7 @@ def __init__(self, filename, table_name="messages"): """ :param str table_name: the name of the table to look for the messages """ - super(SqliteReader, self).__init__(open_file=False) + super(SqliteReader, self).__init__(file=None) self.conn = sqlite3.connect(filename) self.cursor = self.conn.cursor() self.table_name = table_name @@ -87,7 +87,7 @@ class SqliteWriter(BaseIOHandler, BufferedReader): :attr:`~SqliteWriter.GET_MESSAGE_TIMEOUT`. If the :attr:`~SqliteWriter.GET_MESSAGE_TIMEOUT` expires before a message - is received, the internal buffer is written out to the databases file. + is received, the internal buffer is written out to the database file. However if the bus is still saturated with messages, the Listener will continue receiving until the :attr:`~SqliteWriter.MAX_TIME_BETWEEN_WRITES` @@ -107,7 +107,7 @@ def __init__(self, filename, table_name="messages"): """ :param str table_name: the name of the table to store messages in """ - super(SqliteWriter, self).__init__(open_file=False) + super(SqliteWriter, self).__init__(file=None) self.table_name = table_name self.filename = filename self.stop_running_event = threading.Event() diff --git a/can/io/stdout.py b/can/io/stdout.py index 95c7466a0..e4adf2378 100644 --- a/can/io/stdout.py +++ b/can/io/stdout.py @@ -20,13 +20,14 @@ class Printer(BaseIOHandler, Listener): The Printer class is a subclass of :class:`~can.Listener` which simply prints any messages it receives to the terminal (stdout). A message is tunred into a string using :meth:`~can.Message.__str__`. - - :param str output_file: An optional file to "print" to. """ def __init__(self, filename=None): + """ + :param str output_file: An optional file to "print" to + """ self.write_to_file = filename is not None - super(Printer, self).__init__(open_file=self.write_to_file, filename=filename, mode='w') + super(Printer, self).__init__(file=filename, mode='w') def on_message_received(self, msg): if self.write_to_file: From e2cf45a099b1ed3e622e0498941b625babfaeddd Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Jul 2018 16:50:13 +0200 Subject: [PATCH 21/95] add tests for context manger in readers & writers --- test/back2back_test.py | 1 - test/contextmanager_test.py | 4 +- test/logformats_test.py | 135 +++++++++++++++++++++++------------- 3 files changed, 89 insertions(+), 51 deletions(-) diff --git a/test/back2back_test.py b/test/back2back_test.py index a93855dd2..5d0034330 100644 --- a/test/back2back_test.py +++ b/test/back2back_test.py @@ -17,7 +17,6 @@ import can from .config import * -from .data.example_data import generate_message class Back2BackTestCase(unittest.TestCase): diff --git a/test/contextmanager_test.py b/test/contextmanager_test.py index a69dfd5e4..ea9321502 100644 --- a/test/contextmanager_test.py +++ b/test/contextmanager_test.py @@ -16,7 +16,7 @@ def setUp(self): self.msg_send = can.Message(extended_id=False, arbitration_id=0x100, data=data) def test_open_buses(self): - with can.interface.Bus(bustype='virtual') as bus_send, can.interface.Bus(bustype='virtual') as bus_recv: + with can.Bus(interface='virtual') as bus_send, can.Bus(interface='virtual') as bus_recv: bus_send.send(self.msg_send) msg_recv = bus_recv.recv() @@ -24,7 +24,7 @@ def test_open_buses(self): self.assertTrue(msg_recv) def test_use_closed_bus(self): - with can.interface.Bus(bustype='virtual') as bus_send, can.interface.Bus(bustype='virtual') as bus_recv: + with can.Bus(interface='virtual') as bus_send, can.Bus(interface='virtual') as bus_recv: bus_send.send(self.msg_send) # Receiving a frame after bus has been closed should raise a CanException diff --git a/test/logformats_test.py b/test/logformats_test.py index d025d025a..77d9d745e 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -35,59 +35,88 @@ generate_message -def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=None, - check_remote_frames=True, check_error_frames=True, - check_comments=False, round_timestamps=False): - """Tests a pair of writer and reader by writing all data first and - then reading all data and checking if they could be reconstructed - correctly. Optionally writes some comments as well. - - :param unittest.TestCase test_case: the test case the use the assert methods on - :param Callable writer_constructor: the constructor of the writer class - :param Callable reader_constructor: the constructor of the reader class - - :param float sleep_time: specifies the time to sleep after writing all messages. - gets ignored when set to None +def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, + check_remote_frames=True, check_error_frames=True, check_comments=False, + **kwargs): + """ :param bool check_remote_frames: if True, also tests remote frames :param bool check_error_frames: if True, also tests error frames :param bool check_comments: if True, also inserts comments at some locations and checks if they are contained anywhere literally in the resulting file. The locations as selected randomly but deterministically, which makes the test reproducible. - :param bool round_timestamps: if True, rounds timestamps using :meth:`~builtin.round` - before comparing the read messages/events """ - assert isinstance(test_case, unittest.TestCase), \ - "test_case has to be a subclass of unittest.TestCase" + # get all test messages + original_messages = TEST_MESSAGES_BASE + if check_remote_frames: + original_messages += TEST_MESSAGES_REMOTE_FRAMES + if check_error_frames: + original_messages += TEST_MESSAGES_ERROR_FRAMES if check_comments: # we check this because of the lack of a common base class # we filter for not starts with '__' so we do not get all the builtin # methods when logging to the console - test_case.assertIn('log_event', [d for d in dir(writer_constructor) if not d.startswith('__')], + attrs = [attr for attr in dir(writer_constructor) if not attr.startswith('__')] + test_case.assertIn('log_event', attrs, "cannot check comments with this writer: {}".format(writer_constructor)) - # create a temporary file + # get all test comments + original_comments = TEST_COMMENTS if check_comments else () + + # TODO: use https://docs.python.org/3/library/unittest.html#unittest.TestCase.subTest + # once Python 2.7 gets dropped + + print("testing with path-like object and explicit stop() call") temp = tempfile.NamedTemporaryFile('w', delete=False) + filename = temp.name temp.close() + _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, + filename, original_messages, original_comments, + use_context_manager=False, **kwargs) + + print("testing with path-like object and context manager") + temp = tempfile.NamedTemporaryFile('w', delete=False) filename = temp.name + temp.close() + _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, + filename, original_messages, original_comments, + use_context_manager=True, **kwargs) - # get all test messages - original_messages = TEST_MESSAGES_BASE - if check_remote_frames: - original_messages += TEST_MESSAGES_REMOTE_FRAMES - if check_error_frames: - original_messages += TEST_MESSAGES_ERROR_FRAMES + print("testing with file-like object and explicit stop() call") - # get all test comments - original_comments = TEST_COMMENTS + print("testing with file-like object and context manager") - # create writer - writer = writer_constructor(filename) - # write - if check_comments: +def _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, + file, original_messages, original_comments, + use_context_manager=False, + sleep_time=None, round_timestamps=False): + """Tests a pair of writer and reader by writing all data first and + then reading all data and checking if they could be reconstructed + correctly. Optionally writes some comments as well. + + :param unittest.TestCase test_case: the test case the use the assert methods on + :param Callable writer_constructor: the constructor of the writer class + :param Callable reader_constructor: the constructor of the reader class + + :param bool use_context_manager: + if False, uses a explicit :meth:`~can.io.generic.BaseIOHandler.stop()` + call on the reader and writer when finished, and else used the reader + and writer as context managers + + :param float sleep_time: specifies the time to sleep after writing all messages. + gets ignored when set to None + :param bool round_timestamps: if True, rounds timestamps using :meth:`~builtin.round` + before comparing the read messages/events + + """ + + assert isinstance(test_case, unittest.TestCase), \ + "test_case has to be a subclass of unittest.TestCase" + + def _write_all(): # write messages and insert comments here and there # Note: we make no assumptions about the length of original_messages and original_comments for msg, comment in zip_longest(original_messages, original_comments, fillvalue=None): @@ -98,21 +127,30 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s if msg is not None: print("writing message: ", msg) writer(msg) - else: - # ony write messages - for msg in original_messages: - print("writing message: ", msg) - writer(msg) - # sleep and close the writer - if sleep_time is not None: - sleep(sleep_time) + # sleep and close the writer + if sleep_time is not None: + sleep(sleep_time) - writer.stop() + # create writer + print("writing all messages/comments") + if use_context_manager: + with writer_constructor(file) as writer: + _write_all() + else: + _write_all() + writer.stop() # read all written messages - print("reading all messages ...") - read_messages = list(reader_constructor(filename)) + print("reading all messages") + if use_context_manager: + with reader_constructor(file) as reader: + read_messages = list(reader) + else: + reader = reader_constructor(file) + read_messages = list(reader) + # redundant, but this checks if stop() can be called multiple times + reader.stop() # check if at least the number of messages matches test_case.assertEqual(len(read_messages), len(original_messages), @@ -121,11 +159,12 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s # check the order and content of the individual messages for i, (read, original) in enumerate(zip(read_messages, original_messages)): try: + # check everything except the timestamp if read != original: + # check like this to print the whole message print("original message: {}".format(original)) print("read message: {}".format(read)) - # check everything except the timestamp - test_case.assertEqual(read, original) + test_case.fail() # check the timestamp if round_timestamps: original.timestamp = round(original.timestamp) @@ -133,17 +172,17 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s test_case.assertAlmostEqual(read.timestamp, original.timestamp, places=6) except Exception as exception: # attach the index - exception.args += ("test failed at index #{}".format(i), ) + exception.args += ("messages are not equal at index #{}".format(i), ) raise exception # check if the comments are contained in the file - if check_comments: + if original_comments: # read the entire outout file - with open(filename, 'r') as file: + with open(file, 'r') as file: output_contents = file.read() # check each, if they can be found in there literally for comment in original_comments: - test_case.assertTrue(comment in output_contents) + test_case.assertIn(comment, output_contents) class TestCanutilsLog(unittest.TestCase): From fd624bfbe47e8528c5dcf43aed492440838ce27a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Jul 2018 17:00:59 +0200 Subject: [PATCH 22/95] fix error in test case --- test/logformats_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 77d9d745e..329db76fa 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -116,7 +116,7 @@ def _test_writer_and_reader_execute(test_case, writer_constructor, reader_constr assert isinstance(test_case, unittest.TestCase), \ "test_case has to be a subclass of unittest.TestCase" - def _write_all(): + def _write_all(writer): # write messages and insert comments here and there # Note: we make no assumptions about the length of original_messages and original_comments for msg, comment in zip_longest(original_messages, original_comments, fillvalue=None): @@ -136,9 +136,10 @@ def _write_all(): print("writing all messages/comments") if use_context_manager: with writer_constructor(file) as writer: - _write_all() + _write_all(writer) else: - _write_all() + writer = writer_constructor(file) + _write_all(writer) writer.stop() # read all written messages From aefefcd11e0a7e9892dc1ce1af7c1fa029b35b02 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Jul 2018 18:01:19 +0200 Subject: [PATCH 23/95] fix alignment of message timestamp printing --- can/message.py | 2 +- test/logformats_test.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/can/message.py b/can/message.py index 9154bc0b5..4453b0359 100644 --- a/can/message.py +++ b/can/message.py @@ -69,7 +69,7 @@ def __init__(self, timestamp=0.0, is_remote_frame=False, extended_id=True, logger.warning("data link count was %d but it should be less than or equal to 8", self.dlc) def __str__(self): - field_strings = ["Timestamp: {0:15.6f}".format(self.timestamp)] + field_strings = ["Timestamp: {0:>15.6f}".format(self.timestamp)] if self.id_type: # Extended arbitrationID arbitration_id_string = "ID: {0:08x}".format(self.arbitration_id) diff --git a/test/logformats_test.py b/test/logformats_test.py index 329db76fa..4bc3e1e07 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -66,7 +66,7 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, original_comments = TEST_COMMENTS if check_comments else () # TODO: use https://docs.python.org/3/library/unittest.html#unittest.TestCase.subTest - # once Python 2.7 gets dropped + # once Python 2.7 support gets dropped print("testing with path-like object and explicit stop() call") temp = tempfile.NamedTemporaryFile('w', delete=False) @@ -85,8 +85,10 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, use_context_manager=True, **kwargs) print("testing with file-like object and explicit stop() call") + # TODO print("testing with file-like object and context manager") + # TODO def _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, @@ -163,8 +165,8 @@ def _write_all(writer): # check everything except the timestamp if read != original: # check like this to print the whole message - print("original message: {}".format(original)) - print("read message: {}".format(read)) + print("original message: {!r}".format(original)) + print("read message: {!r}".format(read)) test_case.fail() # check the timestamp if round_timestamps: From 627424530b071ebfbe965a7ab377b03e81f2e245 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Jul 2018 19:21:23 +0200 Subject: [PATCH 24/95] add __ne__ method for message object --- can/io/canutils.py | 2 +- can/message.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/can/io/canutils.py b/can/io/canutils.py index f150f8fc8..490c9dcfe 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -116,7 +116,7 @@ def on_message_received(self, msg): timestamp = self.last_timestamp else: timestamp = msg.timestamp - + channel = msg.channel if msg.channel is not None else self.channel if msg.is_error_frame: diff --git a/can/message.py b/can/message.py index 4453b0359..6cc31b1de 100644 --- a/can/message.py +++ b/can/message.py @@ -131,7 +131,8 @@ def __repr__(self): return "can.Message({})".format(", ".join(args)) def __eq__(self, other): - return (isinstance(other, self.__class__) and + if isinstance(other, self.__class__): + return ( self.arbitration_id == other.arbitration_id and #self.timestamp == other.timestamp and # allow the timestamp to differ self.id_type == other.id_type and @@ -140,7 +141,13 @@ def __eq__(self, other): self.is_remote_frame == other.is_remote_frame and self.is_error_frame == other.is_error_frame and self.is_fd == other.is_fd and - self.bitrate_switch == other.bitrate_switch) + self.bitrate_switch == other.bitrate_switch + ) + else: + raise NotImplementedError() + + def __ne__(self, other): + return not self.__eq__(other) def __hash__(self): return hash(( From 31aab59639368639f6c4a94bd455a664259aa505 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 11:52:47 +0200 Subject: [PATCH 25/95] add docs and privatize attrs in Sqlite handlers --- can/io/sqlite.py | 57 ++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index ee11a8c97..bd0a235dd 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -38,36 +38,41 @@ class SqliteReader(BaseIOHandler): .. note:: The database schema is given in the documentation of the loggers. """ - def __init__(self, filename, table_name="messages"): + def __init__(self, file, table_name="messages"): """ + :param file: a `str` or since Python 3.7 a path like object that points + to the database file to use :param str table_name: the name of the table to look for the messages + + .. warning:: In contrary to all other readers/writers the Sqlite handlers + do not accept file-like objects as the `file` parameter. """ super(SqliteReader, self).__init__(file=None) - self.conn = sqlite3.connect(filename) - self.cursor = self.conn.cursor() + self._conn = sqlite3.connect(file) + self._cursor = self._conn.cursor() self.table_name = table_name def __iter__(self): - for frame_data in self.cursor.execute("SELECT * FROM {}".format(self.table_name)): + for frame_data in self._cursor.execute("SELECT * FROM {}".format(self.table_name)): timestamp, can_id, is_extended, is_remote, is_error, dlc, data = frame_data yield Message(timestamp, is_remote, is_extended, is_error, can_id, dlc, data) def __len__(self): # this might not run in constant time - result = self.cursor.execute("SELECT COUNT(*) FROM {}".format(self.table_name)) + result = self._cursor.execute("SELECT COUNT(*) FROM {}".format(self.table_name)) return int(result.fetchone()[0]) def read_all(self): """Fetches all messages in the database. """ - result = self.cursor.execute("SELECT * FROM {}".format(self.table_name)) + result = self._cursor.execute("SELECT * FROM {}".format(self.table_name)) return result.fetchall() def stop(self): """Closes the connection to the database. """ super(SqliteReader, self).stop() - self.conn.close() + self._conn.close() class SqliteWriter(BaseIOHandler, BufferedReader): @@ -103,16 +108,21 @@ class SqliteWriter(BaseIOHandler, BufferedReader): MAX_TIME_BETWEEN_WRITES = 5.0 """Maximum number of seconds to wait between writes to the database""" - def __init__(self, filename, table_name="messages"): + def __init__(self, file, table_name="messages"): """ + :param file: a `str` or since Python 3.7 a path like object that points + to the database file to use :param str table_name: the name of the table to store messages in + + .. warning:: In contrary to all other readers/writers the Sqlite handlers + do not accept file-like objects as the `file` parameter. """ super(SqliteWriter, self).__init__(file=None) self.table_name = table_name - self.filename = filename - self.stop_running_event = threading.Event() - self.writer_thread = threading.Thread(target=self._db_writer_thread) - self.writer_thread.start() + self._db_filename = file + self._stop_running_event = threading.Event() + self._writer_thread = threading.Thread(target=self._db_writer_thread) + self._writer_thread.start() def _create_db(self): """Creates a new databae or opens a connection to an existing one. @@ -122,11 +132,10 @@ def _create_db(self): setup the db here. """ log.debug("Creating sqlite database") - self.conn = sqlite3.connect(self.filename) - cursor = self.conn.cursor() + self._conn = sqlite3.connect(self._db_filename) # create table structure - cursor.execute(""" + self._conn.cursor().execute(""" CREATE TABLE IF NOT EXISTS {} ( ts REAL, @@ -138,9 +147,9 @@ def _create_db(self): data BLOB ) """.format(self.table_name)) - self.conn.commit() + self._conn.commit() - self.insert_template = "INSERT INTO {} VALUES (?, ?, ?, ?, ?, ?, ?)".format(self.table_name) + self._insert_template = "INSERT INTO {} VALUES (?, ?, ?, ?, ?, ?, ?)".format(self.table_name) def _db_writer_thread(self): num_frames = 0 @@ -148,7 +157,7 @@ def _db_writer_thread(self): self._create_db() try: - while not self.stop_running_event.is_set(): + while not self._stop_running_event.is_set(): messages = [] msg = self.get_message(self.GET_MESSAGE_TIMEOUT) @@ -173,20 +182,20 @@ def _db_writer_thread(self): count = len(messages) if count > 0: - with self.conn: + with self._conn: #log.debug("Writing %s frames to db", count) - self.conn.executemany(self.insert_template, messages) - self.conn.commit() # make the changes visible to the entire database + self._conn.executemany(self._insert_template, messages) + self._conn.commit() # make the changes visible to the entire database num_frames += count last_write = time.time() # go back up and check if we are still supposed to run finally: - self.conn.close() + self._conn.close() log.info("Stopped sqlite writer after writing %s messages", num_frames) def stop(self): super(SqliteWriter, self).stop() - self.stop_running_event.set() - self.writer_thread.join() + self._stop_running_event.set() + self._writer_thread.join() From 7234a76ee8f5d64cacca8b46c4c2d15780f140f1 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 11:53:06 +0200 Subject: [PATCH 26/95] add missing handlers to docs --- doc/listeners.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/listeners.rst b/doc/listeners.rst index 3f2b57425..81d84f8df 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -50,6 +50,9 @@ CSVWriter .. autoclass:: can.CSVWriter :members: +.. autoclass:: can.CSVReader + :members: + SqliteWriter ------------ @@ -57,11 +60,15 @@ SqliteWriter .. autoclass:: can.SqliteWriter :members: +.. autoclass:: can.SqliteReader + :members: + + Database table format ~~~~~~~~~~~~~~~~~~~~~ -The messages are written to the table ``messages`` in the sqlite database. -The table is created if it does not already exist. +The messages are written to the table ``messages`` in the sqlite database +by default. The table is created if it does not already exist. The entries are as follows: @@ -95,7 +102,7 @@ engineered from existing log files. One description of the format can be found ` .. autoclass:: can.ASCWriter :members: -ASCReader reads CAN data from ASCII log files .asc +ASCReader reads CAN data from ASCII log files .asc, as further references can-utils can be used: `asc2log `_, `log2asc `_. From 9ac38fe279165b86070513e985c89c48763b06ca Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 11:55:20 +0200 Subject: [PATCH 27/95] change param from filename ot file and comments --- can/io/asc.py | 8 ++++---- can/io/blf.py | 8 ++++---- can/io/canutils.py | 8 ++++---- can/io/csv.py | 8 ++++---- can/io/generic.py | 2 +- can/io/stdout.py | 8 ++++---- can/message.py | 2 +- can/thread_safe_bus.py | 2 ++ 8 files changed, 24 insertions(+), 22 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index 212a2e21e..f820c40df 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -31,8 +31,8 @@ class ASCReader(BaseIOHandler): TODO: turn relative timestamps back to absolute form """ - def __init__(self, filename): - super(ASCReader, self).__init__(file=filename, mode='r') + def __init__(self, file): + super(ASCReader, self).__init__(file, mode='r') @staticmethod def _extract_can_id(str_can_id): @@ -128,12 +128,12 @@ class ASCWriter(BaseIOHandler, Listener): FORMAT_DATE = "%a %b %m %I:%M:%S %p %Y" FORMAT_EVENT = "{timestamp: 9.4f} {message}\n" - def __init__(self, filename, channel=1): + def __init__(self, file, channel=1): """ :param channel: a default channel to use when the message does not have a channel set """ - super(ASCWriter, self).__init__(file=filename, mode='w') + super(ASCWriter, self).__init__(file, mode='w') self.channel = channel # write start of file header diff --git a/can/io/blf.py b/can/io/blf.py index aee87622a..678cfc161 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -123,8 +123,8 @@ class BLFReader(BaseIOHandler): silently ignored. """ - def __init__(self, filename): - super(BLFReader, self).__init__(file=filename, mode='rb') + def __init__(self, file): + super(BLFReader, self).__init__(file, mode='rb') data = self.file.read(FILE_HEADER_STRUCT.size) header = FILE_HEADER_STRUCT.unpack(data) if header[0] != b"LOGG": @@ -263,8 +263,8 @@ class BLFWriter(BaseIOHandler, Listener): #: ZLIB compression level COMPRESSION_LEVEL = 9 - def __init__(self, filename, channel=1): - super(BLFWriter, self).__init__(file=filename, mode='wb') + def __init__(self, file, channel=1): + super(BLFWriter, self).__init__(file, mode='wb') self.channel = channel # Header will be written after log is done self.file.write(b"\x00" * FILE_HEADER_SIZE) diff --git a/can/io/canutils.py b/can/io/canutils.py index 490c9dcfe..55ac50125 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -36,8 +36,8 @@ class CanutilsLogReader(BaseIOHandler): ``(0.0) vcan0 001#8d00100100820100`` """ - def __init__(self, filename): - super(CanutilsLogReader, self).__init__(file=filename, mode='r') + def __init__(self, file): + super(CanutilsLogReader, self).__init__(file, mode='r') def __iter__(self): for line in self.file: @@ -93,7 +93,7 @@ class CanutilsLogWriter(BaseIOHandler, Listener): It the first message does not have a timestamp, it is set to zero. """ - def __init__(self, filename, channel="vcan0", append=False): + def __init__(self, file, channel="vcan0", append=False): """ :param channel: a default channel to use when the message does not have a channel set @@ -101,7 +101,7 @@ def __init__(self, filename, channel="vcan0", append=False): the file, else the file is truncated """ mode = 'a' if append else 'w' - super(CanutilsLogWriter, self).__init__(file=filename, mode=mode) + super(CanutilsLogWriter, self).__init__(file, mode=mode) self.channel = channel self.last_timestamp = None diff --git a/can/io/csv.py b/can/io/csv.py index 7d7f89a31..ce3fff9a1 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -41,7 +41,7 @@ class CSVWriter(BaseIOHandler, Listener): Each line is terminated with a platform specific line seperator. """ - def __init__(self, filename, append=False): + def __init__(self, file, append=False): """ :param bool append: if set to `True` messages are appended to the file and no header line is written, else @@ -49,7 +49,7 @@ def __init__(self, filename, append=False): written header line """ mode = 'a' if append else 'w' - super(CSVWriter, self).__init__(file=filename, mode=mode) + super(CSVWriter, self).__init__(file, mode=mode) # Write a header row if not append: @@ -78,8 +78,8 @@ class CSVReader(BaseIOHandler): Any line seperator is accepted. """ - def __init__(self, filename): - super(CSVReader, self).__init__(file=filename, mode='r') + def __init__(self, file): + super(CSVReader, self).__init__(file, mode='r') def __iter__(self): # skip the header line diff --git a/can/io/generic.py b/can/io/generic.py index 9f3df843f..6d889437f 100644 --- a/can/io/generic.py +++ b/can/io/generic.py @@ -22,7 +22,7 @@ class BaseIOHandler(object): __metaclass__ = ABCMeta - def __init__(self, file='can.data', mode='rt'): + def __init__(self, file, mode='rt'): """ :param str file: a path-like object to open a file, a file-like object to be used as a file or `None` to not use a file at all diff --git a/can/io/stdout.py b/can/io/stdout.py index e4adf2378..f5680d7e2 100644 --- a/can/io/stdout.py +++ b/can/io/stdout.py @@ -22,12 +22,12 @@ class Printer(BaseIOHandler, Listener): string using :meth:`~can.Message.__str__`. """ - def __init__(self, filename=None): + def __init__(self, file=None): """ - :param str output_file: An optional file to "print" to + :param str file: An optional file to "print" to """ - self.write_to_file = filename is not None - super(Printer, self).__init__(file=filename, mode='w') + self.write_to_file = file is not None + super(Printer, self).__init__(file, mode='w') def on_message_received(self, msg): if self.write_to_file: diff --git a/can/message.py b/can/message.py index 6cc31b1de..dc9c80695 100644 --- a/can/message.py +++ b/can/message.py @@ -2,7 +2,7 @@ # coding: utf-8 """ -This module contains the implementation of `can.Message`. +This module contains the implementation of :class:`can.Message`. """ import logging diff --git a/can/thread_safe_bus.py b/can/thread_safe_bus.py index 3a126d90e..bec3a2284 100644 --- a/can/thread_safe_bus.py +++ b/can/thread_safe_bus.py @@ -20,6 +20,8 @@ class NullContextManager(object): """ A context manager that does nothing at all. """ + # could use https://docs.python.org/3/library/contextlib.html#contextlib.nullcontext + # beginning with Python 3.7 def __init__(self, resource=None): self.resource = resource From 86244c4722ce9b0c413592702dd8c040d9fb2407 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 11:57:18 +0200 Subject: [PATCH 28/95] rename loger.py to printer.py --- can/io/printer.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 can/io/printer.py diff --git a/can/io/printer.py b/can/io/printer.py new file mode 100644 index 000000000..f5680d7e2 --- /dev/null +++ b/can/io/printer.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding: utf-8 + +""" +This Listener simply prints to stdout / the terminal or a file. +""" + +from __future__ import print_function, absolute_import + +import logging + +from can.listener import Listener +from .generic import BaseIOHandler + +log = logging.getLogger('can.io.printer') + + +class Printer(BaseIOHandler, Listener): + """ + The Printer class is a subclass of :class:`~can.Listener` which simply prints + any messages it receives to the terminal (stdout). A message is tunred into a + string using :meth:`~can.Message.__str__`. + """ + + def __init__(self, file=None): + """ + :param str file: An optional file to "print" to + """ + self.write_to_file = file is not None + super(Printer, self).__init__(file, mode='w') + + def on_message_received(self, msg): + if self.write_to_file: + self.file.write(str(msg) + '\n') + else: + print(msg) From d857366ce24e9bea31a61d8fcd57995a609a12ab Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 11:58:18 +0200 Subject: [PATCH 29/95] fix import --- can/io/__init__.py | 2 +- can/io/stdout.py | 36 ------------------------------------ 2 files changed, 1 insertion(+), 37 deletions(-) delete mode 100644 can/io/stdout.py diff --git a/can/io/__init__.py b/can/io/__init__.py index 1dc412d52..1d5269912 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -18,4 +18,4 @@ from .canutils import CanutilsLogReader, CanutilsLogWriter from .csv import CSVWriter, CSVReader from .sqlite import SqliteReader, SqliteWriter -from .stdout import Printer +from .printer import Printer diff --git a/can/io/stdout.py b/can/io/stdout.py deleted file mode 100644 index f5680d7e2..000000000 --- a/can/io/stdout.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -""" -This Listener simply prints to stdout / the terminal or a file. -""" - -from __future__ import print_function, absolute_import - -import logging - -from can.listener import Listener -from .generic import BaseIOHandler - -log = logging.getLogger('can.io.printer') - - -class Printer(BaseIOHandler, Listener): - """ - The Printer class is a subclass of :class:`~can.Listener` which simply prints - any messages it receives to the terminal (stdout). A message is tunred into a - string using :meth:`~can.Message.__str__`. - """ - - def __init__(self, file=None): - """ - :param str file: An optional file to "print" to - """ - self.write_to_file = file is not None - super(Printer, self).__init__(file, mode='w') - - def on_message_received(self, msg): - if self.write_to_file: - self.file.write(str(msg) + '\n') - else: - print(msg) From 36b6edaad789629a7b9e6db77f1ae115d53c55a1 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 12:07:29 +0200 Subject: [PATCH 30/95] update docstrings for file parameter --- can/io/asc.py | 4 ++++ can/io/blf.py | 6 ++++++ can/io/canutils.py | 4 ++++ can/io/csv.py | 4 ++++ can/io/generic.py | 4 ++-- can/io/printer.py | 6 +++++- can/io/sqlite.py | 4 ++-- 7 files changed, 27 insertions(+), 5 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index f820c40df..6601e7c66 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -32,6 +32,9 @@ class ASCReader(BaseIOHandler): """ def __init__(self, file): + """ + :param file: a path-like object or as file-like object to read from + """ super(ASCReader, self).__init__(file, mode='r') @staticmethod @@ -130,6 +133,7 @@ class ASCWriter(BaseIOHandler, Listener): def __init__(self, file, channel=1): """ + :param file: a path-like object or as file-like object to write to :param channel: a default channel to use when the message does not have a channel set """ diff --git a/can/io/blf.py b/can/io/blf.py index 678cfc161..e2917e9ed 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -124,6 +124,9 @@ class BLFReader(BaseIOHandler): """ def __init__(self, file): + """ + :param file: a path-like object or as file-like object to read from + """ super(BLFReader, self).__init__(file, mode='rb') data = self.file.read(FILE_HEADER_STRUCT.size) header = FILE_HEADER_STRUCT.unpack(data) @@ -264,6 +267,9 @@ class BLFWriter(BaseIOHandler, Listener): COMPRESSION_LEVEL = 9 def __init__(self, file, channel=1): + """ + :param file: a path-like object or as file-like object to write to + """ super(BLFWriter, self).__init__(file, mode='wb') self.channel = channel # Header will be written after log is done diff --git a/can/io/canutils.py b/can/io/canutils.py index 55ac50125..846bb1f56 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -37,6 +37,9 @@ class CanutilsLogReader(BaseIOHandler): """ def __init__(self, file): + """ + :param file: a path-like object or as file-like object to read from + """ super(CanutilsLogReader, self).__init__(file, mode='r') def __iter__(self): @@ -95,6 +98,7 @@ class CanutilsLogWriter(BaseIOHandler, Listener): def __init__(self, file, channel="vcan0", append=False): """ + :param file: a path-like object or as file-like object to write to :param channel: a default channel to use when the message does not have a channel set :param bool append: if set to `True` messages are appended to diff --git a/can/io/csv.py b/can/io/csv.py index ce3fff9a1..64d7cb7e9 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -43,6 +43,7 @@ class CSVWriter(BaseIOHandler, Listener): def __init__(self, file, append=False): """ + :param file: a path-like object or as file-like object to write to :param bool append: if set to `True` messages are appended to the file and no header line is written, else the file is truncated and starts with a newly @@ -79,6 +80,9 @@ class CSVReader(BaseIOHandler): """ def __init__(self, file): + """ + :param file: a path-like object or as file-like object to read from + """ super(CSVReader, self).__init__(file, mode='r') def __iter__(self): diff --git a/can/io/generic.py b/can/io/generic.py index 6d889437f..4f278d223 100644 --- a/can/io/generic.py +++ b/can/io/generic.py @@ -24,8 +24,8 @@ class BaseIOHandler(object): def __init__(self, file, mode='rt'): """ - :param str file: a path-like object to open a file, a file-like object - to be used as a file or `None` to not use a file at all + :param file: a path-like object to open a file, a file-like object + to be used as a file or `None` to not use a file at all :param str mode: the mode that should be used to open the file, see :func:`builtin.open`, ignored if *file* is `None` """ diff --git a/can/io/printer.py b/can/io/printer.py index f5680d7e2..8662d0f95 100644 --- a/can/io/printer.py +++ b/can/io/printer.py @@ -20,11 +20,15 @@ class Printer(BaseIOHandler, Listener): The Printer class is a subclass of :class:`~can.Listener` which simply prints any messages it receives to the terminal (stdout). A message is tunred into a string using :meth:`~can.Message.__str__`. + + :attr bool write_to_file: `True` iff this instance prints to a file instead of + standard out """ def __init__(self, file=None): """ - :param str file: An optional file to "print" to + :param file: an optional path-like object or as file-like object to "print" + to instead of writing to standard out (stdout) """ self.write_to_file = file is not None super(Printer, self).__init__(file, mode='w') diff --git a/can/io/sqlite.py b/can/io/sqlite.py index bd0a235dd..7cfae3f64 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -128,8 +128,8 @@ def _create_db(self): """Creates a new databae or opens a connection to an existing one. .. note:: - You can't share sqlite3 connections between threads hence we - setup the db here. + You can't share sqlite3 connections between threads (by default) + hence we setup the db here. It has the upside of running async. """ log.debug("Creating sqlite database") self._conn = sqlite3.connect(self._db_filename) From d1ffbaf4d2c8901d9c80e8589a01f47d5fa66dc9 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 12:30:40 +0200 Subject: [PATCH 31/95] better docs --- can/io/logger.py | 8 +++++--- can/io/player.py | 7 ++++--- can/thread_safe_bus.py | 1 + 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 26b2abd06..ef6486b93 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -29,13 +29,15 @@ class Logger(object): * .log :class:`can.CanutilsLogWriter` * other: :class:`can.Printer` - Note this class itself is just a dispatcher, - an object that inherits from Listener will - be created when instantiating this class. + Note this class itself is just a dispatcher, an object that inherits + from Listener will be created when instantiating this class. """ @staticmethod def __new__(cls, filename): + """ + :param str filename: the filename/path the file to write to + """ if filename.endswith(".asc"): return ASCWriter(filename) elif filename.endswith(".blf"): diff --git a/can/io/player.py b/can/io/player.py index 2a64cff18..d03b0ccf4 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -45,9 +45,10 @@ class LogReader(object): @staticmethod def __new__(cls, filename): - if filename is None: - raise TypeError("a filename must be given") - elif filename.endswith(".asc"): + """ + :param str filename: the filename/path the file to read from + """ + if filename.endswith(".asc"): return ASCReader(filename) elif filename.endswith(".blf"): return BLFReader(filename) diff --git a/can/thread_safe_bus.py b/can/thread_safe_bus.py index bec3a2284..44ec10dee 100644 --- a/can/thread_safe_bus.py +++ b/can/thread_safe_bus.py @@ -2,6 +2,7 @@ # coding: utf-8 from __future__ import print_function, absolute_import + from threading import RLock try: From 86c1070c81895d73fd679fb2e2283c0541d2b78d Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 12:34:55 +0200 Subject: [PATCH 32/95] fix imports --- can/io/logger.py | 14 ++++++++------ can/io/player.py | 12 ++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index ef6486b93..e7cd6c1b2 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -5,14 +5,16 @@ See the :class:`Logger` class. """ +from __future__ import absolute_import + import logging -from .asc import ASCWriter -from .blf import BLFWriter -from .canutils import CanutilsLogWriter -from .csv import CSVWriter -from .sqlite import SqliteWriter -from .stdout import Printer +from . import ASCWriter +from . import BLFWriter +from . import CanutilsLogWriter +from . import CSVWriter +from . import SqliteWriter +from . import Printer log = logging.getLogger("can.io.logger") diff --git a/can/io/player.py b/can/io/player.py index d03b0ccf4..138f95656 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -7,16 +7,16 @@ in the recorded order an time intervals. """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import import time import logging -from .asc import ASCReader -from .blf import BLFReader -from .canutils import CanutilsLogReader -from .csv import CSVReader -from .sqlite import SqliteReader +from . import ASCReader +from . import BLFReader +from . import CanutilsLogReader +from . import CSVReader +from . import SqliteReader log = logging.getLogger('can.io.player') From 65ffce94cf02e6382b3164a7e76ba9c52d8ec430 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 5 Jul 2018 13:24:23 +0200 Subject: [PATCH 33/95] fix imports --- can/io/logger.py | 12 ++++++------ can/io/player.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index e7cd6c1b2..4f002060f 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -9,12 +9,12 @@ import logging -from . import ASCWriter -from . import BLFWriter -from . import CanutilsLogWriter -from . import CSVWriter -from . import SqliteWriter -from . import Printer +from .asc import ASCWriter +from .blf import BLFWriter +from .canutils import CanutilsLogWriter +from .csv import CSVWriter +from .sqlite import SqliteWriter +from .printer import Printer log = logging.getLogger("can.io.logger") diff --git a/can/io/player.py b/can/io/player.py index 138f95656..5089a6c91 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -12,11 +12,11 @@ import time import logging -from . import ASCReader -from . import BLFReader -from . import CanutilsLogReader -from . import CSVReader -from . import SqliteReader +from .asc import ASCReader +from .blf import BLFReader +from .canutils import CanutilsLogReader +from .csv import CSVReader +from .sqlite import SqliteReader log = logging.getLogger('can.io.player') From 5abea2627682d5e8d4745261f11f12f7dc5a5863 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 08:58:11 +0200 Subject: [PATCH 34/95] add possibility for testing appending of writers --- test/logformats_test.py | 55 +++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 4bc3e1e07..a9861dffd 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -13,7 +13,7 @@ TODO: implement CAN FD support testing """ -from __future__ import print_function, absolute_import +from __future__ import print_function, absolute_import, division import unittest import tempfile @@ -37,6 +37,7 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, check_remote_frames=True, check_error_frames=True, check_comments=False, + test_append=False, round_timestamps=False, **kwargs): """ :param bool check_remote_frames: if True, also tests remote frames @@ -45,6 +46,10 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, locations and checks if they are contained anywhere literally in the resulting file. The locations as selected randomly but deterministically, which makes the test reproducible. + :param bool test_append: tests the writer in append mode as well + :param bool round_timestamps: if True, rounds timestamps using :meth:`~builtin.round` + before comparing the read messages/events + """ # get all test messages @@ -90,6 +95,24 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, print("testing with file-like object and context manager") # TODO + if test_append: + print("testing append mode with context manager and path-like object") + count = len(original_messages) + first_part = original_messages[:count // 2] + second_part = original_messages[count // 2:] + temp = tempfile.NamedTemporaryFile('w', delete=False) + filename = temp.name + temp.close() + with writer_constructor(filename) as writer: + for message in first_part: + writer(message) + with writer_constructor(filename, append=True) as writer: + for message in second_part: + writer(message) + with reader_constructor(filename) as reader: + read_messages = list(reader) + _check_messages(test_case, original_messages, read_messages, round_timestamps) + def _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, file, original_messages, original_comments, @@ -159,8 +182,23 @@ def _write_all(writer): test_case.assertEqual(len(read_messages), len(original_messages), "the number of written messages does not match the number of read messages") - # check the order and content of the individual messages - for i, (read, original) in enumerate(zip(read_messages, original_messages)): + _check_messages(test_case, original_messages, read_messages, round_timestamps) + + # check if the comments are contained in the file + if original_comments: + # read the entire outout file + with open(file, 'r') as file: + output_contents = file.read() + # check each, if they can be found in there literally + for comment in original_comments: + test_case.assertIn(comment, output_contents) + + +def _check_messages(test_case, original_messages, read_messages, round_timestamps): + """ + Checks the order and content of the individual messages. + """ + for index, (original, read) in enumerate(zip(original_messages, read_messages)): try: # check everything except the timestamp if read != original: @@ -175,18 +213,9 @@ def _write_all(writer): test_case.assertAlmostEqual(read.timestamp, original.timestamp, places=6) except Exception as exception: # attach the index - exception.args += ("messages are not equal at index #{}".format(i), ) + exception.args += ("messages are not equal at index #{}".format(index), ) raise exception - # check if the comments are contained in the file - if original_comments: - # read the entire outout file - with open(file, 'r') as file: - output_contents = file.read() - # check each, if they can be found in there literally - for comment in original_comments: - test_case.assertIn(comment, output_contents) - class TestCanutilsLog(unittest.TestCase): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" From 32f01c834f376020fa0e60676e36d32821e35ba7 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 10:02:31 +0200 Subject: [PATCH 35/95] enable append testing --- test/logformats_test.py | 68 ++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index a9861dffd..e4229ef74 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -217,20 +217,45 @@ def _check_messages(test_case, original_messages, read_messages, round_timestamp raise exception -class TestCanutilsLog(unittest.TestCase): - """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" +class TestAscFileFormat(unittest.TestCase): + """Tests can.ASCWriter and can.ASCReader""" def test_writer_and_reader(self): - _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader, + _test_writer_and_reader(self, can.ASCWriter, can.ASCReader, + check_comments=True, round_timestamps=True) + + +class TestBlfFileFormat(unittest.TestCase): + """Tests can.BLFWriter and can.BLFReader""" + + def test_writer_and_reader(self): + _test_writer_and_reader(self, can.BLFWriter, can.BLFReader, check_comments=False) + def test_reader(self): + logfile = os.path.join(os.path.dirname(__file__), "data", "logfile.blf") + messages = list(can.BLFReader(logfile)) + self.assertEqual(len(messages), 2) + self.assertEqual(messages[0], + can.Message( + extended_id=False, + arbitration_id=0x64, + data=[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8])) + self.assertEqual(messages[0].channel, 0) + self.assertEqual(messages[1], + can.Message( + is_error_frame=True, + extended_id=True, + arbitration_id=0x1FFFFFFF)) + self.assertEqual(messages[1].channel, 0) -class TestAscFileFormat(unittest.TestCase): - """Tests can.ASCWriter and can.ASCReader""" + +class TestCanutilsLog(unittest.TestCase): + """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" def test_writer_and_reader(self): - _test_writer_and_reader(self, can.ASCWriter, can.ASCReader, - check_comments=True, round_timestamps=True) + _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader, + test_append=True, check_comments=False) class TestCsvFileFormat(unittest.TestCase): @@ -238,7 +263,7 @@ class TestCsvFileFormat(unittest.TestCase): def test_writer_and_reader(self): _test_writer_and_reader(self, can.CSVWriter, can.CSVReader, - check_comments=False) + test_append=True, check_comments=False) class TestSqliteDatabaseFormat(unittest.TestCase): @@ -247,7 +272,7 @@ class TestSqliteDatabaseFormat(unittest.TestCase): def test_writer_and_reader(self): _test_writer_and_reader(self, can.SqliteWriter, can.SqliteReader, sleep_time=can.SqliteWriter.MAX_TIME_BETWEEN_WRITES + 0.5, - check_comments=False) + test_append=True, check_comments=False) def testSQLWriterWritesToSameFile(self): f = tempfile.NamedTemporaryFile('w', delete=False) @@ -282,30 +307,5 @@ def testSQLWriterWritesToSameFile(self): self.assertEqual(msg2[1], 0x02) -class TestBlfFileFormat(unittest.TestCase): - """Tests can.BLFWriter and can.BLFReader""" - - def test_writer_and_reader(self): - _test_writer_and_reader(self, can.BLFWriter, can.BLFReader, - check_comments=False) - - def test_reader(self): - logfile = os.path.join(os.path.dirname(__file__), "data", "logfile.blf") - messages = list(can.BLFReader(logfile)) - self.assertEqual(len(messages), 2) - self.assertEqual(messages[0], - can.Message( - extended_id=False, - arbitration_id=0x64, - data=[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8])) - self.assertEqual(messages[0].channel, 0) - self.assertEqual(messages[1], - can.Message( - is_error_frame=True, - extended_id=True, - arbitration_id=0x1FFFFFFF)) - self.assertEqual(messages[1].channel, 0) - - if __name__ == '__main__': unittest.main() From c9ea0b3ed5e8b23799f058960da415790b0bfc1a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 10:15:31 +0200 Subject: [PATCH 36/95] fix append mode test for Sqlite --- test/logformats_test.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index e4229ef74..62b951d20 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -106,12 +106,24 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, with writer_constructor(filename) as writer: for message in first_part: writer(message) - with writer_constructor(filename, append=True) as writer: + # use append mode + try: + writer = writer_constructor(filename, append=True) + except TypeError as e: + # maybe "append" is not a formal parameter + try: + writer = writer_constructor(filename) + except TypeError: + # is the is still a problem, raise the initial error + raise e + with writer: for message in second_part: writer(message) with reader_constructor(filename) as reader: read_messages = list(reader) _check_messages(test_case, original_messages, read_messages, round_timestamps) + else: + print("do not test append mode") def _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, From b3ff334adddec0e937ec0f92aa362b810ec77be8 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 10:24:25 +0200 Subject: [PATCH 37/95] add test for can.Printer --- test/logformats_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/logformats_test.py b/test/logformats_test.py index 62b951d20..8bed06527 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -319,5 +319,22 @@ def testSQLWriterWritesToSameFile(self): self.assertEqual(msg2[1], 0x02) +class TestPrinter(unittest.TestCase): + """Tests that can.Printer does not crash""" + + messages = TEST_MESSAGES_BASE + TEST_MESSAGES_REMOTE_FRAMES + TEST_MESSAGES_ERROR_FRAMES + + def test_not_crashes_stdout(self): + with can.Printer() as printer: + for message in self.messages: + printer(message) + + def test_not_crashed_file(self): + with tempfile.NamedTemporaryFile('w', delete=False) as temp_file: + with can.Printer(temp_file) as printer: + for message in self.messages: + printer(message) + + if __name__ == '__main__': unittest.main() From eb501be496395ddf1dafabf47ad28cb8fed2f6e4 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 10:35:54 +0200 Subject: [PATCH 38/95] slightly simpler test case --- test/logformats_test.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 8bed06527..f2533b383 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -211,22 +211,19 @@ def _check_messages(test_case, original_messages, read_messages, round_timestamp Checks the order and content of the individual messages. """ for index, (original, read) in enumerate(zip(original_messages, read_messages)): - try: - # check everything except the timestamp - if read != original: - # check like this to print the whole message - print("original message: {!r}".format(original)) - print("read message: {!r}".format(read)) - test_case.fail() - # check the timestamp - if round_timestamps: - original.timestamp = round(original.timestamp) - read.timestamp = round(read.timestamp) - test_case.assertAlmostEqual(read.timestamp, original.timestamp, places=6) - except Exception as exception: - # attach the index - exception.args += ("messages are not equal at index #{}".format(index), ) - raise exception + # check everything except the timestamp + if read != original: + # check like this to print the whole message + print("original message: {!r}".format(original)) + print("read message: {!r}".format(read)) + test_case.fail("messages are not equal at index #{}".format(index)) + # check the timestamp + if round_timestamps: + original.timestamp = round(original.timestamp) + read.timestamp = round(read.timestamp) + test_case.assertAlmostEqual(read.timestamp, original.timestamp, places=6, + msg="message timestamps are not almost_equal at index #{} ({!r} !~= {!r})" + .format(index, original.timestamp, read.timestamp)) class TestAscFileFormat(unittest.TestCase): From e5df130a68214edf9f3b6cc0fd611de5c9d97645 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 10:41:02 +0200 Subject: [PATCH 39/95] fix sqlite parser --- can/io/sqlite.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 7cfae3f64..915f54472 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -55,7 +55,15 @@ def __init__(self, file, table_name="messages"): def __iter__(self): for frame_data in self._cursor.execute("SELECT * FROM {}".format(self.table_name)): timestamp, can_id, is_extended, is_remote, is_error, dlc, data = frame_data - yield Message(timestamp, is_remote, is_extended, is_error, can_id, dlc, data) + yield Message( + timestamp=timestamp, + is_remote_frame=bool(is_remote), + extended_id=bool(is_extended), + is_error_frame=bool(is_error), + arbitration_id=can_id, + dlc=dlc, + data=data + ) def __len__(self): # this might not run in constant time From b769137a415b9f0eab2da12cb4fe922273a4bd3f Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 11:00:40 +0200 Subject: [PATCH 40/95] add sleepingto append mode testing --- test/logformats_test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index f2533b383..4cd37924f 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -37,8 +37,8 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, check_remote_frames=True, check_error_frames=True, check_comments=False, - test_append=False, round_timestamps=False, - **kwargs): + test_append=False, + sleep_time=None, round_timestamps=False): """ :param bool check_remote_frames: if True, also tests remote frames :param bool check_error_frames: if True, also tests error frames @@ -79,7 +79,7 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, temp.close() _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, filename, original_messages, original_comments, - use_context_manager=False, **kwargs) + use_context_manager=False, sleep_time=sleep_time) print("testing with path-like object and context manager") temp = tempfile.NamedTemporaryFile('w', delete=False) @@ -87,7 +87,7 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, temp.close() _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, filename, original_messages, original_comments, - use_context_manager=True, **kwargs) + use_context_manager=True, sleep_time=sleep_time) print("testing with file-like object and explicit stop() call") # TODO @@ -106,6 +106,8 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, with writer_constructor(filename) as writer: for message in first_part: writer(message) + if sleep_time is not None: + sleep(sleep_time) # use append mode try: writer = writer_constructor(filename, append=True) @@ -119,6 +121,8 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, with writer: for message in second_part: writer(message) + if sleep_time is not None: + sleep(sleep_time) with reader_constructor(filename) as reader: read_messages = list(reader) _check_messages(test_case, original_messages, read_messages, round_timestamps) @@ -128,8 +132,8 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, def _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, file, original_messages, original_comments, - use_context_manager=False, - sleep_time=None, round_timestamps=False): + use_context_manager, + sleep_time, round_timestamps): """Tests a pair of writer and reader by writing all data first and then reading all data and checking if they could be reconstructed correctly. Optionally writes some comments as well. @@ -165,7 +169,6 @@ def _write_all(writer): print("writing message: ", msg) writer(msg) - # sleep and close the writer if sleep_time is not None: sleep(sleep_time) From b51b71984b6b74658fadbe7a7d667263e81e2518 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 11:08:20 +0200 Subject: [PATCH 41/95] fix test method call --- test/logformats_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 4cd37924f..ccbf9c469 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -79,7 +79,8 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, temp.close() _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, filename, original_messages, original_comments, - use_context_manager=False, sleep_time=sleep_time) + use_context_manager=False, sleep_time=sleep_time, + round_timestamps=round_timestamps) print("testing with path-like object and context manager") temp = tempfile.NamedTemporaryFile('w', delete=False) @@ -87,7 +88,8 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, temp.close() _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, filename, original_messages, original_comments, - use_context_manager=True, sleep_time=sleep_time) + use_context_manager=True, sleep_time=sleep_time, + round_timestamps=round_timestamps) print("testing with file-like object and explicit stop() call") # TODO From 54aa145a6d515f2ed40ab7bfe7ff9b12695bfb58 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 11:31:22 +0200 Subject: [PATCH 42/95] enuse data is written to disk --- test/logformats_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/logformats_test.py b/test/logformats_test.py index ccbf9c469..568eadbb4 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -104,12 +104,14 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, second_part = original_messages[count // 2:] temp = tempfile.NamedTemporaryFile('w', delete=False) filename = temp.name + fileno = temp.fileno() temp.close() with writer_constructor(filename) as writer: for message in first_part: writer(message) if sleep_time is not None: sleep(sleep_time) + os.fsync(fileno) # use append mode try: writer = writer_constructor(filename, append=True) From 732fbb68681afdbd2627a9ef24e1535ccbf7dcb8 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 12:39:08 +0200 Subject: [PATCH 43/95] restructured logformats_test --- test/logformats_test.py | 417 ++++++++++++++++++++-------------------- 1 file changed, 207 insertions(+), 210 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 568eadbb4..c58d03777 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -35,136 +35,156 @@ generate_message -def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, - check_remote_frames=True, check_error_frames=True, check_comments=False, - test_append=False, - sleep_time=None, round_timestamps=False): - """ - :param bool check_remote_frames: if True, also tests remote frames - :param bool check_error_frames: if True, also tests error frames - :param bool check_comments: if True, also inserts comments at some - locations and checks if they are contained anywhere literally - in the resulting file. The locations as selected randomly - but deterministically, which makes the test reproducible. - :param bool test_append: tests the writer in append mode as well - :param bool round_timestamps: if True, rounds timestamps using :meth:`~builtin.round` - before comparing the read messages/events +class ReaderWriterTest(unittest.TestCase): + """Tests a pair of writer and reader by writing all data first and + then reading all data and checking if they could be reconstructed + correctly. Optionally writes some comments as well. """ - # get all test messages - original_messages = TEST_MESSAGES_BASE - if check_remote_frames: - original_messages += TEST_MESSAGES_REMOTE_FRAMES - if check_error_frames: - original_messages += TEST_MESSAGES_ERROR_FRAMES - - if check_comments: - # we check this because of the lack of a common base class - # we filter for not starts with '__' so we do not get all the builtin - # methods when logging to the console - attrs = [attr for attr in dir(writer_constructor) if not attr.startswith('__')] - test_case.assertIn('log_event', attrs, - "cannot check comments with this writer: {}".format(writer_constructor)) - - # get all test comments - original_comments = TEST_COMMENTS if check_comments else () - - # TODO: use https://docs.python.org/3/library/unittest.html#unittest.TestCase.subTest - # once Python 2.7 support gets dropped - - print("testing with path-like object and explicit stop() call") - temp = tempfile.NamedTemporaryFile('w', delete=False) - filename = temp.name - temp.close() - _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, - filename, original_messages, original_comments, - use_context_manager=False, sleep_time=sleep_time, - round_timestamps=round_timestamps) - - print("testing with path-like object and context manager") - temp = tempfile.NamedTemporaryFile('w', delete=False) - filename = temp.name - temp.close() - _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, - filename, original_messages, original_comments, - use_context_manager=True, sleep_time=sleep_time, - round_timestamps=round_timestamps) - - print("testing with file-like object and explicit stop() call") - # TODO - - print("testing with file-like object and context manager") - # TODO - - if test_append: - print("testing append mode with context manager and path-like object") - count = len(original_messages) - first_part = original_messages[:count // 2] - second_part = original_messages[count // 2:] - temp = tempfile.NamedTemporaryFile('w', delete=False) - filename = temp.name - fileno = temp.fileno() - temp.close() - with writer_constructor(filename) as writer: + def __init__(self, writer_constructor, reader_constructor, + check_remote_frames=True, check_error_frames=True, check_comments=False, + test_append=False, round_timestamps=False): + """ + :param Callable writer_constructor: the constructor of the writer class + :param Callable reader_constructor: the constructor of the reader class + + :param bool check_remote_frames: if True, also tests remote frames + :param bool check_error_frames: if True, also tests error frames + :param bool check_comments: if True, also inserts comments at some + locations and checks if they are contained anywhere literally + in the resulting file. The locations as selected randomly + but deterministically, which makes the test reproducible. + :param bool test_append: tests the writer in append mode as well + :param bool round_timestamps: if True, rounds timestamps using :meth:`~builtin.round` + before comparing the read messages/events + + """ + + # get all test messages + self.original_messages = TEST_MESSAGES_BASE + if check_remote_frames: + self.original_messages += TEST_MESSAGES_REMOTE_FRAMES + if check_error_frames: + self.original_messages += TEST_MESSAGES_ERROR_FRAMES + + if check_comments: + # we check this because of the lack of a common base class + # we filter for not starts with '__' so we do not get all the builtin + # methods when logging to the console + attrs = [attr for attr in dir(writer_constructor) if not attr.startswith('__')] + self.assertIn('log_event', attrs, + "cannot check comments with this writer: {}".format(writer_constructor)) + + # get all test comments + self.original_comments = TEST_COMMENTS if check_comments else () + + self.writer_constructor = writer_constructor + self.reader_constructor = reader_constructor + self.test_append_enabled = test_append + self.round_timestamps = round_timestamps + + def test_path_like_explicit_stop(self): + """testing with path-like and explicit stop() call""" + filename = self._get_temp_filename() + + # create writer + print("writing all messages/comments") + writer = self.writer_constructor(filename) + self._write_all(writer) + os.fsync(writer.file.fileno()) + writer.stop() + + print("reading all messages") + reader = self.reader_constructor(filename) + read_messages = list(reader) + # redundant, but this checks if stop() can be called multiple times + reader.stop() + + # check if at least the number of messages matches + # could use assertCountEqual in later versions of Python and in the other methods + self.assertEqual(len(read_messages), len(self.original_messages), + "the number of written messages does not match the number of read messages") + + self.assertMessagesEqual(read_messages) + self.assertIncludesComments(filename) + + def test_path_like_context_manager(self): + """testing with path-like object and context manager""" + filename = self._get_temp_filename() + + # create writer + print("writing all messages/comments") + with self.writer_constructor(filename) as writer: + self._write_all(writer) + os.fsync(writer.file.fileno()) + + # read all written messages + print("reading all messages") + with self.reader_constructor(filename) as reader: + read_messages = list(reader) + + # check if at least the number of messages matches; + self.assertEqual(len(read_messages), len(self.original_messages), + "the number of written messages does not match the number of read messages") + + self.assertMessagesEqual(read_messages) + self.assertIncludesComments(filename) + + def test_file_like_explicit_stop(self): + """testing with file-like object and explicit stop() call""" + raise unittest.SkipTest("not yet implemented") + + def test_file_like_context_manager(self): + """testing with file-like object and context manager""" + raise unittest.SkipTest("not yet implemented") + + def test_append_mode(self): + """ + testing append mode with context manager and path-like object + """ + if not self.test_append_enabled: + raise unittest.SkipTest("do not test append mode") + + filename = self._get_temp_filename() + count = len(self.original_messages) + first_part = self.original_messages[:count // 2] + second_part = self.original_messages[count // 2:] + + # write first half + with self.writer_constructor(filename) as writer: for message in first_part: writer(message) - if sleep_time is not None: - sleep(sleep_time) - os.fsync(fileno) - # use append mode + os.fsync(writer.file.fileno()) + + # use append mode for second half try: - writer = writer_constructor(filename, append=True) + writer = self.writer_constructor(filename, append=True) except TypeError as e: # maybe "append" is not a formal parameter try: - writer = writer_constructor(filename) + writer = self.writer_constructor(filename) except TypeError: # is the is still a problem, raise the initial error raise e with writer: for message in second_part: writer(message) - if sleep_time is not None: - sleep(sleep_time) - with reader_constructor(filename) as reader: + os.fsync(writer.file.fileno()) + with self.reader_constructor(filename) as reader: read_messages = list(reader) - _check_messages(test_case, original_messages, read_messages, round_timestamps) - else: - print("do not test append mode") + self.assertMessagesEqual(read_messages) -def _test_writer_and_reader_execute(test_case, writer_constructor, reader_constructor, - file, original_messages, original_comments, - use_context_manager, - sleep_time, round_timestamps): - """Tests a pair of writer and reader by writing all data first and - then reading all data and checking if they could be reconstructed - correctly. Optionally writes some comments as well. - - :param unittest.TestCase test_case: the test case the use the assert methods on - :param Callable writer_constructor: the constructor of the writer class - :param Callable reader_constructor: the constructor of the reader class - - :param bool use_context_manager: - if False, uses a explicit :meth:`~can.io.generic.BaseIOHandler.stop()` - call on the reader and writer when finished, and else used the reader - and writer as context managers - - :param float sleep_time: specifies the time to sleep after writing all messages. - gets ignored when set to None - :param bool round_timestamps: if True, rounds timestamps using :meth:`~builtin.round` - before comparing the read messages/events - - """ + @staticmethod + def _get_temp_filename(): + with tempfile.NamedTemporaryFile('w+', delete=False) as temp: + return temp.name - assert isinstance(test_case, unittest.TestCase), \ - "test_case has to be a subclass of unittest.TestCase" - - def _write_all(writer): - # write messages and insert comments here and there + def _write_all(self, writer): + """Writes messages and insert comments here and there.""" # Note: we make no assumptions about the length of original_messages and original_comments - for msg, comment in zip_longest(original_messages, original_comments, fillvalue=None): + for msg, comment in zip_longest(self.original_messages, self.original_comments, fillvalue=None): # msg and comment might be None if comment is not None: print("writing comment: ", comment) @@ -173,84 +193,63 @@ def _write_all(writer): print("writing message: ", msg) writer(msg) - if sleep_time is not None: - sleep(sleep_time) - - # create writer - print("writing all messages/comments") - if use_context_manager: - with writer_constructor(file) as writer: - _write_all(writer) - else: - writer = writer_constructor(file) - _write_all(writer) - writer.stop() - - # read all written messages - print("reading all messages") - if use_context_manager: - with reader_constructor(file) as reader: - read_messages = list(reader) - else: - reader = reader_constructor(file) - read_messages = list(reader) - # redundant, but this checks if stop() can be called multiple times - reader.stop() - - # check if at least the number of messages matches - test_case.assertEqual(len(read_messages), len(original_messages), - "the number of written messages does not match the number of read messages") - - _check_messages(test_case, original_messages, read_messages, round_timestamps) - - # check if the comments are contained in the file - if original_comments: - # read the entire outout file - with open(file, 'r') as file: - output_contents = file.read() - # check each, if they can be found in there literally - for comment in original_comments: - test_case.assertIn(comment, output_contents) - - -def _check_messages(test_case, original_messages, read_messages, round_timestamps): - """ - Checks the order and content of the individual messages. - """ - for index, (original, read) in enumerate(zip(original_messages, read_messages)): - # check everything except the timestamp - if read != original: - # check like this to print the whole message - print("original message: {!r}".format(original)) - print("read message: {!r}".format(read)) - test_case.fail("messages are not equal at index #{}".format(index)) - # check the timestamp - if round_timestamps: - original.timestamp = round(original.timestamp) - read.timestamp = round(read.timestamp) - test_case.assertAlmostEqual(read.timestamp, original.timestamp, places=6, - msg="message timestamps are not almost_equal at index #{} ({!r} !~= {!r})" - .format(index, original.timestamp, read.timestamp)) - - -class TestAscFileFormat(unittest.TestCase): + def assertMessagesEqual(self, read_messages): + """ + Checks the order and content of the individual messages. + """ + for index, (original, read) in enumerate(zip(self.original_messages, read_messages)): + # check everything except the timestamp + if read != original: + # check like this to print the whole message + print("original message: {!r}".format(original)) + print("read message: {!r}".format(read)) + self.fail("messages are not equal at index #{}".format(index)) + # check the timestamp + if self.round_timestamps: + original.timestamp = round(original.timestamp) + read.timestamp = round(read.timestamp) + self.assertAlmostEqual(read.timestamp, original.timestamp, places=6, + msg="message timestamps are not almost_equal at index #{} ({!r} !~= {!r})" + .format(index, original.timestamp, read.timestamp)) + + def assertIncludesComments(self, filename): + """ + Ensures that all comments are literally contained in the given file. + + :param filename: the path-like object to use + """ + if self.original_comments: + # read the entire outout file + with open(filename, 'rt') as file: + output_contents = file.read() + # check each, if they can be found in there literally + for comment in self.original_comments: + self.assertIn(comment, output_contents) + + +class TestAscFileFormat(ReaderWriterTest): """Tests can.ASCWriter and can.ASCReader""" - def test_writer_and_reader(self): - _test_writer_and_reader(self, can.ASCWriter, can.ASCReader, - check_comments=True, round_timestamps=True) + def __init__(self): + super(TestAscFileFormat, self).__init__( + can.ASCWriter, can.ASCReader, + check_comments=True, round_timestamps=True + ) -class TestBlfFileFormat(unittest.TestCase): +class TestBlfFileFormat(ReaderWriterTest): """Tests can.BLFWriter and can.BLFReader""" - def test_writer_and_reader(self): - _test_writer_and_reader(self, can.BLFWriter, can.BLFReader, - check_comments=False) + def __init__(self): + super(TestBlfFileFormat, self).__init__( + can.BLFWriter, can.BLFReader, + check_comments=False + ) - def test_reader(self): + def test_read_known_file(self): logfile = os.path.join(os.path.dirname(__file__), "data", "logfile.blf") - messages = list(can.BLFReader(logfile)) + with can.BLFReader(logfile) as reader: + messages = list(reader) self.assertEqual(len(messages), 2) self.assertEqual(messages[0], can.Message( @@ -266,50 +265,48 @@ def test_reader(self): self.assertEqual(messages[1].channel, 0) -class TestCanutilsLog(unittest.TestCase): +class TestCanutilsFileFormat(ReaderWriterTest): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" - def test_writer_and_reader(self): - _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader, - test_append=True, check_comments=False) + def __init__(self): + super(TestCanutilsFileFormat, self).__init__( + can.CanutilsLogWriter, can.CanutilsLogReader, + test_append=True, check_comments=False + ) -class TestCsvFileFormat(unittest.TestCase): +class TestCsvFileFormat(ReaderWriterTest): """Tests can.ASCWriter and can.ASCReader""" - def test_writer_and_reader(self): - _test_writer_and_reader(self, can.CSVWriter, can.CSVReader, - test_append=True, check_comments=False) + def __init__(self): + super(TestCsvFileFormat, self).__init__( + can.CSVWriter, can.CSVReader, + test_append=True, check_comments=False + ) -class TestSqliteDatabaseFormat(unittest.TestCase): +class TestSqliteDatabaseFormat(ReaderWriterTest): """Tests can.SqliteWriter and can.SqliteReader""" - def test_writer_and_reader(self): - _test_writer_and_reader(self, can.SqliteWriter, can.SqliteReader, - sleep_time=can.SqliteWriter.MAX_TIME_BETWEEN_WRITES + 0.5, - test_append=True, check_comments=False) - - def testSQLWriterWritesToSameFile(self): - f = tempfile.NamedTemporaryFile('w', delete=False) - f.close() - - first_listener = can.SqliteWriter(f.name) - first_listener(generate_message(0x01)) - - sleep(first_listener.MAX_TIME_BETWEEN_WRITES) - first_listener.stop() - - second_listener = can.SqliteWriter(f.name) - second_listener(generate_message(0x02)) + def __init__(self): + super(TestSqliteDatabaseFormat, self).__init__( + can.SqliteWriter, can.SqliteReader, + sleep_time=can.SqliteWriter.MAX_TIME_BETWEEN_WRITES + 0.5, + test_append=True, check_comments=False + ) - sleep(second_listener.MAX_TIME_BETWEEN_WRITES) + def test_writes_to_same_file(self): + filename = self._get_temp_filename() - second_listener.stop() + with can.SqliteWriter(filename) as first_listener: + first_listener(generate_message(0x01)) + first_listener.stop() - con = sqlite3.connect(f.name) + with can.SqliteWriter(filename) as second_listener: + second_listener(generate_message(0x02)) + second_listener.stop() - with con: + with sqlite3.connect(filename) as con: c = con.cursor() c.execute("select COUNT() from messages") From 3b34dcde64fd93f57f3f139ba8a67db13a2fef1a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 13:00:41 +0200 Subject: [PATCH 44/95] exclude base class from tests --- test/logformats_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index c58d03777..414941a55 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -35,7 +35,7 @@ generate_message -class ReaderWriterTest(unittest.TestCase): +class ReaderWriterTest(object): """Tests a pair of writer and reader by writing all data first and then reading all data and checking if they could be reconstructed correctly. Optionally writes some comments as well. @@ -227,7 +227,7 @@ def assertIncludesComments(self, filename): self.assertIn(comment, output_contents) -class TestAscFileFormat(ReaderWriterTest): +class TestAscFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" def __init__(self): @@ -237,7 +237,7 @@ def __init__(self): ) -class TestBlfFileFormat(ReaderWriterTest): +class TestBlfFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.BLFWriter and can.BLFReader""" def __init__(self): @@ -265,7 +265,7 @@ def test_read_known_file(self): self.assertEqual(messages[1].channel, 0) -class TestCanutilsFileFormat(ReaderWriterTest): +class TestCanutilsFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" def __init__(self): @@ -275,7 +275,7 @@ def __init__(self): ) -class TestCsvFileFormat(ReaderWriterTest): +class TestCsvFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" def __init__(self): @@ -285,7 +285,7 @@ def __init__(self): ) -class TestSqliteDatabaseFormat(ReaderWriterTest): +class TestSqliteDatabaseFormat(ReaderWriterTest, unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" def __init__(self): From aa59dc06684a3420f12a0c5acdbf949677550dea Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 13:18:16 +0200 Subject: [PATCH 45/95] try to fix ReaderWriterTest --- test/logformats_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 414941a55..abc57ec1d 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -60,6 +60,7 @@ def __init__(self, writer_constructor, reader_constructor, before comparing the read messages/events """ + super(ReaderWriterTest, self).__init__() # get all test messages self.original_messages = TEST_MESSAGES_BASE @@ -73,8 +74,8 @@ def __init__(self, writer_constructor, reader_constructor, # we filter for not starts with '__' so we do not get all the builtin # methods when logging to the console attrs = [attr for attr in dir(writer_constructor) if not attr.startswith('__')] - self.assertIn('log_event', attrs, - "cannot check comments with this writer: {}".format(writer_constructor)) + assert 'log_event' in attrs, \ + "cannot check comments with this writer: {}".format(writer_constructor) # get all test comments self.original_comments = TEST_COMMENTS if check_comments else () From 3096feb74450ca3673d2724f0da99cb9e57362ed Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 13:44:04 +0200 Subject: [PATCH 46/95] pass along params to superclass in test case --- test/logformats_test.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index abc57ec1d..de99f8043 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -44,7 +44,8 @@ class ReaderWriterTest(object): def __init__(self, writer_constructor, reader_constructor, check_remote_frames=True, check_error_frames=True, check_comments=False, - test_append=False, round_timestamps=False): + test_append=False, round_timestamps=False, + *args, **kwargs): """ :param Callable writer_constructor: the constructor of the writer class :param Callable reader_constructor: the constructor of the reader class @@ -60,7 +61,7 @@ def __init__(self, writer_constructor, reader_constructor, before comparing the read messages/events """ - super(ReaderWriterTest, self).__init__() + super(ReaderWriterTest, self).__init__(*args, **kwargs) # get all test messages self.original_messages = TEST_MESSAGES_BASE @@ -231,20 +232,22 @@ def assertIncludesComments(self, filename): class TestAscFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" - def __init__(self): + def __init__(self, *args, **kwargs): super(TestAscFileFormat, self).__init__( can.ASCWriter, can.ASCReader, - check_comments=True, round_timestamps=True + check_comments=True, round_timestamps=True, + *args, **kwargs ) class TestBlfFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.BLFWriter and can.BLFReader""" - def __init__(self): + def __init__(self, *args, **kwargs): super(TestBlfFileFormat, self).__init__( can.BLFWriter, can.BLFReader, - check_comments=False + check_comments=False, + *args, **kwargs ) def test_read_known_file(self): @@ -269,31 +272,34 @@ def test_read_known_file(self): class TestCanutilsFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" - def __init__(self): + def __init__(self, *args, **kwargs): super(TestCanutilsFileFormat, self).__init__( can.CanutilsLogWriter, can.CanutilsLogReader, - test_append=True, check_comments=False + test_append=True, check_comments=False, + *args, **kwargs ) class TestCsvFileFormat(ReaderWriterTest, unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" - def __init__(self): + def __init__(self, *args, **kwargs): super(TestCsvFileFormat, self).__init__( can.CSVWriter, can.CSVReader, - test_append=True, check_comments=False + test_append=True, check_comments=False, + *args, **kwargs ) class TestSqliteDatabaseFormat(ReaderWriterTest, unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" - def __init__(self): + def __init__(self, *args, **kwargs): super(TestSqliteDatabaseFormat, self).__init__( can.SqliteWriter, can.SqliteReader, sleep_time=can.SqliteWriter.MAX_TIME_BETWEEN_WRITES + 0.5, - test_append=True, check_comments=False + test_append=True, check_comments=False, + *args, **kwargs ) def test_writes_to_same_file(self): From 65b2725a8b86dc22658d3b0b96b1e52f81806fb2 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 13:49:46 +0200 Subject: [PATCH 47/95] fix wrong param in init call --- test/logformats_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index de99f8043..22cd0243e 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -17,7 +17,6 @@ import unittest import tempfile -from time import sleep import sqlite3 import os @@ -297,7 +296,6 @@ class TestSqliteDatabaseFormat(ReaderWriterTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(TestSqliteDatabaseFormat, self).__init__( can.SqliteWriter, can.SqliteReader, - sleep_time=can.SqliteWriter.MAX_TIME_BETWEEN_WRITES + 0.5, test_append=True, check_comments=False, *args, **kwargs ) From 994497bc75dff83e50901d74d7bff969cd0885d1 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 14:26:57 +0200 Subject: [PATCH 48/95] attempt to fix test case invocation --- test/logformats_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 22cd0243e..84fb76008 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -228,7 +228,7 @@ def assertIncludesComments(self, filename): self.assertIn(comment, output_contents) -class TestAscFileFormat(ReaderWriterTest, unittest.TestCase): +class TestAscFileFormat(unittest.TestCase, ReaderWriterTest): """Tests can.ASCWriter and can.ASCReader""" def __init__(self, *args, **kwargs): From 7337055caaf84833a4987648eb6e1ff08f77ca9b Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 14:58:07 +0200 Subject: [PATCH 49/95] attempt different approach at ReaderWriterTest --- test/logformats_test.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 84fb76008..e1a438711 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -34,13 +34,15 @@ generate_message -class ReaderWriterTest(object): +class ReaderWriterTest(unittest.TestCase): """Tests a pair of writer and reader by writing all data first and then reading all data and checking if they could be reconstructed correctly. Optionally writes some comments as well. """ + __test__ = False + def __init__(self, writer_constructor, reader_constructor, check_remote_frames=True, check_error_frames=True, check_comments=False, test_append=False, round_timestamps=False, @@ -228,7 +230,7 @@ def assertIncludesComments(self, filename): self.assertIn(comment, output_contents) -class TestAscFileFormat(unittest.TestCase, ReaderWriterTest): +class TestAscFileFormat(ReaderWriterTest): """Tests can.ASCWriter and can.ASCReader""" def __init__(self, *args, **kwargs): @@ -239,7 +241,7 @@ def __init__(self, *args, **kwargs): ) -class TestBlfFileFormat(ReaderWriterTest, unittest.TestCase): +class TestBlfFileFormat(ReaderWriterTest): """Tests can.BLFWriter and can.BLFReader""" def __init__(self, *args, **kwargs): @@ -268,7 +270,7 @@ def test_read_known_file(self): self.assertEqual(messages[1].channel, 0) -class TestCanutilsFileFormat(ReaderWriterTest, unittest.TestCase): +class TestCanutilsFileFormat(ReaderWriterTest): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" def __init__(self, *args, **kwargs): @@ -279,7 +281,7 @@ def __init__(self, *args, **kwargs): ) -class TestCsvFileFormat(ReaderWriterTest, unittest.TestCase): +class TestCsvFileFormat(ReaderWriterTest): """Tests can.ASCWriter and can.ASCReader""" def __init__(self, *args, **kwargs): @@ -290,7 +292,7 @@ def __init__(self, *args, **kwargs): ) -class TestSqliteDatabaseFormat(ReaderWriterTest, unittest.TestCase): +class TestSqliteDatabaseFormat(ReaderWriterTest): """Tests can.SqliteWriter and can.SqliteReader""" def __init__(self, *args, **kwargs): From b937ab473f303d88708ef5acc330718371844591 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 6 Jul 2018 15:02:27 +0200 Subject: [PATCH 50/95] add magic "__test__ " attribute --- test/logformats_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/logformats_test.py b/test/logformats_test.py index e1a438711..5ec20d61a 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -233,6 +233,8 @@ def assertIncludesComments(self, filename): class TestAscFileFormat(ReaderWriterTest): """Tests can.ASCWriter and can.ASCReader""" + __test__ = True + def __init__(self, *args, **kwargs): super(TestAscFileFormat, self).__init__( can.ASCWriter, can.ASCReader, @@ -244,6 +246,8 @@ def __init__(self, *args, **kwargs): class TestBlfFileFormat(ReaderWriterTest): """Tests can.BLFWriter and can.BLFReader""" + __test__ = True + def __init__(self, *args, **kwargs): super(TestBlfFileFormat, self).__init__( can.BLFWriter, can.BLFReader, @@ -273,6 +277,8 @@ def test_read_known_file(self): class TestCanutilsFileFormat(ReaderWriterTest): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" + __test__ = True + def __init__(self, *args, **kwargs): super(TestCanutilsFileFormat, self).__init__( can.CanutilsLogWriter, can.CanutilsLogReader, @@ -284,6 +290,8 @@ def __init__(self, *args, **kwargs): class TestCsvFileFormat(ReaderWriterTest): """Tests can.ASCWriter and can.ASCReader""" + __test__ = True + def __init__(self, *args, **kwargs): super(TestCsvFileFormat, self).__init__( can.CSVWriter, can.CSVReader, @@ -295,6 +303,8 @@ def __init__(self, *args, **kwargs): class TestSqliteDatabaseFormat(ReaderWriterTest): """Tests can.SqliteWriter and can.SqliteReader""" + __test__ = True + def __init__(self, *args, **kwargs): super(TestSqliteDatabaseFormat, self).__init__( can.SqliteWriter, can.SqliteReader, From e91b0b31cb9a074b9929e09ab45e96a7476bd4fc Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 18 Jul 2018 15:04:50 +0200 Subject: [PATCH 51/95] change constructor for ReaderWriterTest --- test/logformats_test.py | 61 ++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 5ec20d61a..be964425d 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -19,6 +19,7 @@ import tempfile import sqlite3 import os +from abc import abstractmethod, ABCMeta try: # Python 3 @@ -43,10 +44,21 @@ class ReaderWriterTest(unittest.TestCase): __test__ = False - def __init__(self, writer_constructor, reader_constructor, - check_remote_frames=True, check_error_frames=True, check_comments=False, - test_append=False, round_timestamps=False, - *args, **kwargs): + __metaclass__ = ABCMeta + + def __init__(self, *args, **kwargs): + super(ReaderWriterTest, self).__init__(*args, **kwargs) + self._setup_instance() + + @abstractmethod + def _setup_instance(self): + """Hook for subclasses.""" + raise NotImplementedError() + + def _setup_instance_helper(self, + writer_constructor, reader_constructor, + check_remote_frames=True, check_error_frames=True, check_comments=False, + test_append=False, round_timestamps=False): """ :param Callable writer_constructor: the constructor of the writer class :param Callable reader_constructor: the constructor of the reader class @@ -62,8 +74,6 @@ def __init__(self, writer_constructor, reader_constructor, before comparing the read messages/events """ - super(ReaderWriterTest, self).__init__(*args, **kwargs) - # get all test messages self.original_messages = TEST_MESSAGES_BASE if check_remote_frames: @@ -235,11 +245,10 @@ class TestAscFileFormat(ReaderWriterTest): __test__ = True - def __init__(self, *args, **kwargs): - super(TestAscFileFormat, self).__init__( + def _setup_instance(self): + super(TestAscFileFormat, self)._setup_instance_helper( can.ASCWriter, can.ASCReader, - check_comments=True, round_timestamps=True, - *args, **kwargs + check_comments=True, round_timestamps=True ) @@ -248,11 +257,10 @@ class TestBlfFileFormat(ReaderWriterTest): __test__ = True - def __init__(self, *args, **kwargs): - super(TestBlfFileFormat, self).__init__( + def _setup_instance(self): + super(TestBlfFileFormat, self)._setup_instance_helper( can.BLFWriter, can.BLFReader, - check_comments=False, - *args, **kwargs + check_comments=False ) def test_read_known_file(self): @@ -279,11 +287,10 @@ class TestCanutilsFileFormat(ReaderWriterTest): __test__ = True - def __init__(self, *args, **kwargs): - super(TestCanutilsFileFormat, self).__init__( + def _setup_instance(self): + super(TestCanutilsFileFormat, self)._setup_instance_helper( can.CanutilsLogWriter, can.CanutilsLogReader, - test_append=True, check_comments=False, - *args, **kwargs + test_append=True, check_comments=False ) @@ -292,11 +299,10 @@ class TestCsvFileFormat(ReaderWriterTest): __test__ = True - def __init__(self, *args, **kwargs): - super(TestCsvFileFormat, self).__init__( + def _setup_instance(self): + super(TestCsvFileFormat, self)._setup_instance_helper( can.CSVWriter, can.CSVReader, - test_append=True, check_comments=False, - *args, **kwargs + test_append=True, check_comments=False ) @@ -305,11 +311,10 @@ class TestSqliteDatabaseFormat(ReaderWriterTest): __test__ = True - def __init__(self, *args, **kwargs): - super(TestSqliteDatabaseFormat, self).__init__( + def _setup_instance(self): + super(TestSqliteDatabaseFormat, self)._setup_instance_helper( can.SqliteWriter, can.SqliteReader, - test_append=True, check_comments=False, - *args, **kwargs + test_append=True, check_comments=False ) def test_writes_to_same_file(self): @@ -342,12 +347,12 @@ class TestPrinter(unittest.TestCase): messages = TEST_MESSAGES_BASE + TEST_MESSAGES_REMOTE_FRAMES + TEST_MESSAGES_ERROR_FRAMES - def test_not_crashes_stdout(self): + def test_not_crashes_with_stdout(self): with can.Printer() as printer: for message in self.messages: printer(message) - def test_not_crashed_file(self): + def test_not_crashes_with_file(self): with tempfile.NamedTemporaryFile('w', delete=False) as temp_file: with can.Printer(temp_file) as printer: for message in self.messages: From 35c4676aad75fe3f34e754122c496969cee0862b Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 18 Jul 2018 15:13:30 +0200 Subject: [PATCH 52/95] fix Sqlite test --- test/logformats_test.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index be964425d..70508026e 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -105,7 +105,8 @@ def test_path_like_explicit_stop(self): print("writing all messages/comments") writer = self.writer_constructor(filename) self._write_all(writer) - os.fsync(writer.file.fileno()) + if hasattr(writer.file, 'fileno'): + os.fsync(writer.file.fileno()) writer.stop() print("reading all messages") @@ -130,7 +131,8 @@ def test_path_like_context_manager(self): print("writing all messages/comments") with self.writer_constructor(filename) as writer: self._write_all(writer) - os.fsync(writer.file.fileno()) + if hasattr(writer.file, 'fileno'): + os.fsync(writer.file.fileno()) # read all written messages print("reading all messages") @@ -168,7 +170,8 @@ def test_append_mode(self): with self.writer_constructor(filename) as writer: for message in first_part: writer(message) - os.fsync(writer.file.fileno()) + if hasattr(writer.file, 'fileno'): + os.fsync(writer.file.fileno()) # use append mode for second half try: @@ -183,7 +186,8 @@ def test_append_mode(self): with writer: for message in second_part: writer(message) - os.fsync(writer.file.fileno()) + if hasattr(writer.file, 'fileno'): + os.fsync(writer.file.fileno()) with self.reader_constructor(filename) as reader: read_messages = list(reader) @@ -317,6 +321,14 @@ def _setup_instance(self): test_append=True, check_comments=False ) + @unittest.SkipTest("not implemented") + def test_file_like_explicit_stop(self): + pass + + @unittest.SkipTest("not implemented") + def test_file_like_context_manager(self): + pass + def test_writes_to_same_file(self): filename = self._get_temp_filename() From f2c3ee250f6247a2ebadb8dc115672ad1f539a48 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 18 Jul 2018 15:18:59 +0200 Subject: [PATCH 53/95] Sqlite test fixes --- test/logformats_test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 70508026e..c279ae549 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -334,19 +334,17 @@ def test_writes_to_same_file(self): with can.SqliteWriter(filename) as first_listener: first_listener(generate_message(0x01)) - first_listener.stop() with can.SqliteWriter(filename) as second_listener: second_listener(generate_message(0x02)) - second_listener.stop() with sqlite3.connect(filename) as con: c = con.cursor() - c.execute("select COUNT() from messages") + c.execute("SELECT COUNT(*) FROM messages") self.assertEqual(2, c.fetchone()[0]) - c.execute("select * from messages") + c.execute("SELECT * FROM messages") msg1 = c.fetchone() msg2 = c.fetchone() From 02f86e42be9cd26de2e90a32bca6e2725aa5a2b8 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 18 Jul 2018 21:39:19 +0200 Subject: [PATCH 54/95] fix skipping of tests --- test/logformats_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index c279ae549..0cf896e95 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -321,11 +321,11 @@ def _setup_instance(self): test_append=True, check_comments=False ) - @unittest.SkipTest("not implemented") + @unittest.skip("not implemented") def test_file_like_explicit_stop(self): pass - @unittest.SkipTest("not implemented") + @unittest.skip("not implemented") def test_file_like_context_manager(self): pass From baf2416400a1eac5bb3c58fa1b99805d5a27f82e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 19 Jul 2018 10:46:28 +0200 Subject: [PATCH 55/95] remove redundant Sqlite test --- test/logformats_test.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 0cf896e95..55437bbb4 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -329,28 +329,6 @@ def test_file_like_explicit_stop(self): def test_file_like_context_manager(self): pass - def test_writes_to_same_file(self): - filename = self._get_temp_filename() - - with can.SqliteWriter(filename) as first_listener: - first_listener(generate_message(0x01)) - - with can.SqliteWriter(filename) as second_listener: - second_listener(generate_message(0x02)) - - with sqlite3.connect(filename) as con: - c = con.cursor() - - c.execute("SELECT COUNT(*) FROM messages") - self.assertEqual(2, c.fetchone()[0]) - - c.execute("SELECT * FROM messages") - msg1 = c.fetchone() - msg2 = c.fetchone() - - self.assertEqual(msg1[1], 0x01) - self.assertEqual(msg2[1], 0x02) - class TestPrinter(unittest.TestCase): """Tests that can.Printer does not crash""" From a1ed6bb77196a2f0d6da9f1b4e07f030fe27bd34 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:02:08 +0200 Subject: [PATCH 56/95] improved logformats_test and added test_file_like_explicit_stop --- test/logformats_test.py | 78 ++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 20 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 55437bbb4..2c90594e9 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -17,6 +17,7 @@ import unittest import tempfile +import os import sqlite3 import os from abc import abstractmethod, ABCMeta @@ -97,23 +98,34 @@ def _setup_instance_helper(self, self.test_append_enabled = test_append self.round_timestamps = round_timestamps + def setUp(self): + with tempfile.NamedTemporaryFile('w+', delete=False) as test_file: + self.test_file_name = test_file.name + + def tearDown(self): + os.remove(self.test_file_name) + del self.test_file_name + def test_path_like_explicit_stop(self): """testing with path-like and explicit stop() call""" - filename = self._get_temp_filename() # create writer print("writing all messages/comments") - writer = self.writer_constructor(filename) + writer = self.writer_constructor(self.test_file_name) self._write_all(writer) if hasattr(writer.file, 'fileno'): os.fsync(writer.file.fileno()) writer.stop() + if hasattr(writer.file, 'closed'): + self.assertTrue(writer.file.closed) print("reading all messages") - reader = self.reader_constructor(filename) + reader = self.reader_constructor(self.test_file_name) read_messages = list(reader) # redundant, but this checks if stop() can be called multiple times reader.stop() + if hasattr(writer.file, 'closed'): + self.assertTrue(writer.file.closed) # check if at least the number of messages matches # could use assertCountEqual in later versions of Python and in the other methods @@ -121,34 +133,66 @@ def test_path_like_explicit_stop(self): "the number of written messages does not match the number of read messages") self.assertMessagesEqual(read_messages) - self.assertIncludesComments(filename) + self.assertIncludesComments(self.test_file_name) def test_path_like_context_manager(self): """testing with path-like object and context manager""" - filename = self._get_temp_filename() # create writer print("writing all messages/comments") - with self.writer_constructor(filename) as writer: + with self.writer_constructor(self.test_file_name) as writer: self._write_all(writer) if hasattr(writer.file, 'fileno'): os.fsync(writer.file.fileno()) + w = writer + if hasattr(w.file, 'closed'): + self.assertTrue(w.file.closed) # read all written messages print("reading all messages") - with self.reader_constructor(filename) as reader: + with self.reader_constructor(self.test_file_name) as reader: read_messages = list(reader) + r = reader + if hasattr(r.file, 'closed'): + self.assertTrue(r.file.closed) # check if at least the number of messages matches; self.assertEqual(len(read_messages), len(self.original_messages), "the number of written messages does not match the number of read messages") self.assertMessagesEqual(read_messages) - self.assertIncludesComments(filename) + self.assertIncludesComments(self.test_file_name) def test_file_like_explicit_stop(self): """testing with file-like object and explicit stop() call""" - raise unittest.SkipTest("not yet implemented") + + # create writer + print("writing all messages/comments") + my_file = open(self.test_file_name, 'w') + writer = self.writer_constructor(my_file) + self._write_all(writer) + if hasattr(writer.file, 'fileno'): + os.fsync(writer.file.fileno()) + writer.stop() + if hasattr(my_file, 'closed'): + self.assertTrue(my_file.closed) + + print("reading all messages") + my_file = open(self.test_file_name, 'r') + reader = self.reader_constructor(my_file) + read_messages = list(reader) + # redundant, but this checks if stop() can be called multiple times + reader.stop() + if hasattr(my_file, 'closed'): + self.assertTrue(my_file.closed) + + # check if at least the number of messages matches + # could use assertCountEqual in later versions of Python and in the other methods + self.assertEqual(len(read_messages), len(self.original_messages), + "the number of written messages does not match the number of read messages") + + self.assertMessagesEqual(read_messages) + self.assertIncludesComments(self.test_file_name) def test_file_like_context_manager(self): """testing with file-like object and context manager""" @@ -161,13 +205,12 @@ def test_append_mode(self): if not self.test_append_enabled: raise unittest.SkipTest("do not test append mode") - filename = self._get_temp_filename() count = len(self.original_messages) first_part = self.original_messages[:count // 2] second_part = self.original_messages[count // 2:] # write first half - with self.writer_constructor(filename) as writer: + with self.writer_constructor(self.test_file_name) as writer: for message in first_part: writer(message) if hasattr(writer.file, 'fileno'): @@ -175,11 +218,11 @@ def test_append_mode(self): # use append mode for second half try: - writer = self.writer_constructor(filename, append=True) + writer = self.writer_constructor(self.test_file_name, append=True) except TypeError as e: # maybe "append" is not a formal parameter try: - writer = self.writer_constructor(filename) + writer = self.writer_constructor(self.test_file_name) except TypeError: # is the is still a problem, raise the initial error raise e @@ -188,16 +231,11 @@ def test_append_mode(self): writer(message) if hasattr(writer.file, 'fileno'): os.fsync(writer.file.fileno()) - with self.reader_constructor(filename) as reader: + with self.reader_constructor(self.test_file_name) as reader: read_messages = list(reader) self.assertMessagesEqual(read_messages) - @staticmethod - def _get_temp_filename(): - with tempfile.NamedTemporaryFile('w+', delete=False) as temp: - return temp.name - def _write_all(self, writer): """Writes messages and insert comments here and there.""" # Note: we make no assumptions about the length of original_messages and original_comments @@ -237,7 +275,7 @@ def assertIncludesComments(self, filename): """ if self.original_comments: # read the entire outout file - with open(filename, 'rt') as file: + with open(filename, 'r') as file: output_contents = file.read() # check each, if they can be found in there literally for comment in self.original_comments: From 4afbd0ce3d59ba77adfcab4f23e2ed667a869792 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:07:10 +0200 Subject: [PATCH 57/95] add test_file_like_context_manager --- test/logformats_test.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 2c90594e9..e4e6c0f7f 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -196,7 +196,33 @@ def test_file_like_explicit_stop(self): def test_file_like_context_manager(self): """testing with file-like object and context manager""" - raise unittest.SkipTest("not yet implemented") + + # create writer + print("writing all messages/comments") + my_file = open(self.test_file_name, 'w') + with self.writer_constructor(my_file) as writer: + self._write_all(writer) + if hasattr(writer.file, 'fileno'): + os.fsync(writer.file.fileno()) + w = writer + if hasattr(my_file, 'closed'): + self.assertTrue(my_file.closed) + + # read all written messages + print("reading all messages") + my_file = open(self.test_file_name, 'w') + with self.reader_constructor(self.test_file_name) as reader: + read_messages = list(reader) + r = reader + if hasattr(my_file, 'closed'): + self.assertTrue(my_file.closed) + + # check if at least the number of messages matches; + self.assertEqual(len(read_messages), len(self.original_messages), + "the number of written messages does not match the number of read messages") + + self.assertMessagesEqual(read_messages) + self.assertIncludesComments(self.test_file_name) def test_append_mode(self): """ From 9c19bcd1c3677348f5afabcaa17143b8004c2f83 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:18:59 +0200 Subject: [PATCH 58/95] added support for testing binary readers/writers + added docs on file modes --- can/io/asc.py | 4 ++++ can/io/blf.py | 4 ++++ can/io/canutils.py | 4 ++++ can/io/csv.py | 4 ++++ can/io/printer.py | 2 ++ test/logformats_test.py | 14 ++++++++------ 6 files changed, 26 insertions(+), 6 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index 7aec40763..b9409ad39 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -34,6 +34,8 @@ class ASCReader(BaseIOHandler): def __init__(self, file): """ :param file: a path-like object or as file-like object to read from + If this is a file-like object, is has to opened in text + read mode, not binary read mode. """ super(ASCReader, self).__init__(file, mode='r') @@ -134,6 +136,8 @@ class ASCWriter(BaseIOHandler, Listener): def __init__(self, file, channel=1): """ :param file: a path-like object or as file-like object to write to + If this is a file-like object, is has to opened in text + write mode, not binary write mode. :param channel: a default channel to use when the message does not have a channel set """ diff --git a/can/io/blf.py b/can/io/blf.py index e2917e9ed..9de76aec4 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -126,6 +126,8 @@ class BLFReader(BaseIOHandler): def __init__(self, file): """ :param file: a path-like object or as file-like object to read from + If this is a file-like object, is has to opened in binary + read mode, not text read mode. """ super(BLFReader, self).__init__(file, mode='rb') data = self.file.read(FILE_HEADER_STRUCT.size) @@ -269,6 +271,8 @@ class BLFWriter(BaseIOHandler, Listener): def __init__(self, file, channel=1): """ :param file: a path-like object or as file-like object to write to + If this is a file-like object, is has to opened in binary + write mode, not text write mode. """ super(BLFWriter, self).__init__(file, mode='wb') self.channel = channel diff --git a/can/io/canutils.py b/can/io/canutils.py index 846bb1f56..7a4f4fa4f 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -39,6 +39,8 @@ class CanutilsLogReader(BaseIOHandler): def __init__(self, file): """ :param file: a path-like object or as file-like object to read from + If this is a file-like object, is has to opened in text + read mode, not binary read mode. """ super(CanutilsLogReader, self).__init__(file, mode='r') @@ -99,6 +101,8 @@ class CanutilsLogWriter(BaseIOHandler, Listener): def __init__(self, file, channel="vcan0", append=False): """ :param file: a path-like object or as file-like object to write to + If this is a file-like object, is has to opened in text + write mode, not binary write mode. :param channel: a default channel to use when the message does not have a channel set :param bool append: if set to `True` messages are appended to diff --git a/can/io/csv.py b/can/io/csv.py index 64d7cb7e9..e108679b8 100644 --- a/can/io/csv.py +++ b/can/io/csv.py @@ -44,6 +44,8 @@ class CSVWriter(BaseIOHandler, Listener): def __init__(self, file, append=False): """ :param file: a path-like object or as file-like object to write to + If this is a file-like object, is has to opened in text + write mode, not binary write mode. :param bool append: if set to `True` messages are appended to the file and no header line is written, else the file is truncated and starts with a newly @@ -82,6 +84,8 @@ class CSVReader(BaseIOHandler): def __init__(self, file): """ :param file: a path-like object or as file-like object to read from + If this is a file-like object, is has to opened in text + read mode, not binary read mode. """ super(CSVReader, self).__init__(file, mode='r') diff --git a/can/io/printer.py b/can/io/printer.py index 8662d0f95..4e9333fa2 100644 --- a/can/io/printer.py +++ b/can/io/printer.py @@ -29,6 +29,8 @@ def __init__(self, file=None): """ :param file: an optional path-like object or as file-like object to "print" to instead of writing to standard out (stdout) + If this is a file-like object, is has to opened in text + write mode, not binary write mode. """ self.write_to_file = file is not None super(Printer, self).__init__(file, mode='w') diff --git a/test/logformats_test.py b/test/logformats_test.py index e4e6c0f7f..1cdee12ec 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -57,7 +57,7 @@ def _setup_instance(self): raise NotImplementedError() def _setup_instance_helper(self, - writer_constructor, reader_constructor, + writer_constructor, reader_constructor, binary_file=False, check_remote_frames=True, check_error_frames=True, check_comments=False, test_append=False, round_timestamps=False): """ @@ -95,6 +95,7 @@ def _setup_instance_helper(self, self.writer_constructor = writer_constructor self.reader_constructor = reader_constructor + self.binary_file = binary_file self.test_append_enabled = test_append self.round_timestamps = round_timestamps @@ -168,7 +169,7 @@ def test_file_like_explicit_stop(self): # create writer print("writing all messages/comments") - my_file = open(self.test_file_name, 'w') + my_file = open(self.test_file_name, 'wb' if self.binary_file else 'w') writer = self.writer_constructor(my_file) self._write_all(writer) if hasattr(writer.file, 'fileno'): @@ -178,7 +179,7 @@ def test_file_like_explicit_stop(self): self.assertTrue(my_file.closed) print("reading all messages") - my_file = open(self.test_file_name, 'r') + my_file = open(self.test_file_name, 'rb' if self.binary_file else 'r') reader = self.reader_constructor(my_file) read_messages = list(reader) # redundant, but this checks if stop() can be called multiple times @@ -199,7 +200,7 @@ def test_file_like_context_manager(self): # create writer print("writing all messages/comments") - my_file = open(self.test_file_name, 'w') + my_file = open(self.test_file_name, 'wb' if self.binary_file else 'w') with self.writer_constructor(my_file) as writer: self._write_all(writer) if hasattr(writer.file, 'fileno'): @@ -210,7 +211,7 @@ def test_file_like_context_manager(self): # read all written messages print("reading all messages") - my_file = open(self.test_file_name, 'w') + my_file = open(self.test_file_name, 'rb' if self.binary_file else 'r') with self.reader_constructor(self.test_file_name) as reader: read_messages = list(reader) r = reader @@ -301,7 +302,7 @@ def assertIncludesComments(self, filename): """ if self.original_comments: # read the entire outout file - with open(filename, 'r') as file: + with open(filename, 'rb' if self.binary_file else 'r') as file: output_contents = file.read() # check each, if they can be found in there literally for comment in self.original_comments: @@ -328,6 +329,7 @@ class TestBlfFileFormat(ReaderWriterTest): def _setup_instance(self): super(TestBlfFileFormat, self)._setup_instance_helper( can.BLFWriter, can.BLFReader, + binary_file=True, check_comments=False ) From 4c835cd63f700e20a081d2d07cffecc5690f2afe Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:20:37 +0200 Subject: [PATCH 59/95] added *args and **kwargs forwaring to LogReader and Logger --- can/io/logger.py | 14 +++++++------- can/io/player.py | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 4f002060f..bb4a966ed 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -36,20 +36,20 @@ class Logger(object): """ @staticmethod - def __new__(cls, filename): + def __new__(cls, filename, *args, **kwargs): """ :param str filename: the filename/path the file to write to """ if filename.endswith(".asc"): - return ASCWriter(filename) + return ASCWriter(filename, *args, **kwargs) elif filename.endswith(".blf"): - return BLFWriter(filename) + return BLFWriter(filename, *args, **kwargs) elif filename.endswith(".csv"): - return CSVWriter(filename) + return CSVWriter(filename, *args, **kwargs) elif filename.endswith(".db"): - return SqliteWriter(filename) + return SqliteWriter(filename, *args, **kwargs) elif filename.endswith(".log"): - return CanutilsLogWriter(filename) + return CanutilsLogWriter(filename, *args, **kwargs) else: log.info('unknown file type "%s", falling pack to can.Printer', filename) - return Printer(filename) + return Printer(filename, *args, **kwargs) diff --git a/can/io/player.py b/can/io/player.py index 5089a6c91..186442bd2 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -44,20 +44,20 @@ class LogReader(object): """ @staticmethod - def __new__(cls, filename): + def __new__(cls, filename, *args, **kwargs): """ :param str filename: the filename/path the file to read from """ if filename.endswith(".asc"): - return ASCReader(filename) + return ASCReader(filename, *args, **kwargs) elif filename.endswith(".blf"): - return BLFReader(filename) + return BLFReader(filename, *args, **kwargs) elif filename.endswith(".csv"): - return CSVReader(filename) + return CSVReader(filename, *args, **kwargs) elif filename.endswith(".db"): - return SqliteReader(filename) + return SqliteReader(filename, *args, **kwargs) elif filename.endswith(".log"): - return CanutilsLogReader(filename) + return CanutilsLogReader(filename, *args, **kwargs) else: raise NotImplementedError("No read support for this log format: {}".format(filename)) From c4fde3bce7274b1bd73ecaff0f63f2446b93622d Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:23:40 +0200 Subject: [PATCH 60/95] add some documentation to Sqlite --- can/io/sqlite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 915f54472..0fe2d8d68 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -46,6 +46,7 @@ def __init__(self, file, table_name="messages"): .. warning:: In contrary to all other readers/writers the Sqlite handlers do not accept file-like objects as the `file` parameter. + It also runs in ``append=True`` mode all the time. """ super(SqliteReader, self).__init__(file=None) self._conn = sqlite3.connect(file) From ff9d8213c1d58471681748fb823266da2267c3a1 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:29:21 +0200 Subject: [PATCH 61/95] fix bug in test call --- test/logformats_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 1cdee12ec..131d6b7dd 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -212,7 +212,7 @@ def test_file_like_context_manager(self): # read all written messages print("reading all messages") my_file = open(self.test_file_name, 'rb' if self.binary_file else 'r') - with self.reader_constructor(self.test_file_name) as reader: + with self.reader_constructor(my_file) as reader: read_messages = list(reader) r = reader if hasattr(my_file, 'closed'): From 75ff215324a6279a6ec1af56d1950f553df6b4e0 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:32:31 +0200 Subject: [PATCH 62/95] add more xtensive logging in test (in assertMessagesEqual) --- test/logformats_test.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 131d6b7dd..3d37acb31 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -280,12 +280,10 @@ def assertMessagesEqual(self, read_messages): Checks the order and content of the individual messages. """ for index, (original, read) in enumerate(zip(self.original_messages, read_messages)): + print("Comapring: original message: {!r}".format(original)) + print(" read message: {!r}".format(read)) # check everything except the timestamp - if read != original: - # check like this to print the whole message - print("original message: {!r}".format(original)) - print("read message: {!r}".format(read)) - self.fail("messages are not equal at index #{}".format(index)) + self.assertEqual(original, read, "messages are not equal at index #{}".format(index)) # check the timestamp if self.round_timestamps: original.timestamp = round(original.timestamp) From 0a7ab37176b760d85cb9be28152a328fb3e5bfcc Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 12:38:41 +0200 Subject: [PATCH 63/95] less verbose logging --- test/logformats_test.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 3d37acb31..8d562fcfa 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -280,17 +280,20 @@ def assertMessagesEqual(self, read_messages): Checks the order and content of the individual messages. """ for index, (original, read) in enumerate(zip(self.original_messages, read_messages)): - print("Comapring: original message: {!r}".format(original)) - print(" read message: {!r}".format(read)) - # check everything except the timestamp - self.assertEqual(original, read, "messages are not equal at index #{}".format(index)) - # check the timestamp - if self.round_timestamps: - original.timestamp = round(original.timestamp) - read.timestamp = round(read.timestamp) - self.assertAlmostEqual(read.timestamp, original.timestamp, places=6, - msg="message timestamps are not almost_equal at index #{} ({!r} !~= {!r})" - .format(index, original.timestamp, read.timestamp)) + try: + # check everything except the timestamp + self.assertEqual(original, read, "messages are not equal at index #{}".format(index)) + # check the timestamp + if self.round_timestamps: + original.timestamp = round(original.timestamp) + read.timestamp = round(read.timestamp) + self.assertAlmostEqual(read.timestamp, original.timestamp, places=6, + msg="message timestamps are not almost_equal at index #{} ({!r} !~= {!r})" + .format(index, original.timestamp, read.timestamp)) + except: + print("Comparing: original message: {!r}".format(original)) + print(" read message: {!r}".format(read)) + raise def assertIncludesComments(self, filename): """ From f2fff0073fb421ebe04a88eb802f9c1f442aeb80 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 13:11:24 +0200 Subject: [PATCH 64/95] added simple _ensure_fsync method to logformats_test --- can/io/canutils.py | 4 ++-- test/logformats_test.py | 23 +++++++++++------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/can/io/canutils.py b/can/io/canutils.py index 7a4f4fa4f..f3b436b13 100644 --- a/can/io/canutils.py +++ b/can/io/canutils.py @@ -82,8 +82,8 @@ def __iter__(self): msg = Message(timestamp=timestamp, is_error_frame=True) else: msg = Message(timestamp=timestamp, arbitration_id=canId & 0x1FFFFFFF, - extended_id=isExtended, is_remote_frame=isRemoteFrame, - dlc=dlc, data=dataBin, channel=channel) + extended_id=isExtended, is_remote_frame=isRemoteFrame, + dlc=dlc, data=dataBin, channel=channel) yield msg self.stop() diff --git a/test/logformats_test.py b/test/logformats_test.py index 8d562fcfa..caa786430 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -114,8 +114,7 @@ def test_path_like_explicit_stop(self): print("writing all messages/comments") writer = self.writer_constructor(self.test_file_name) self._write_all(writer) - if hasattr(writer.file, 'fileno'): - os.fsync(writer.file.fileno()) + self._ensure_fsync(writer) writer.stop() if hasattr(writer.file, 'closed'): self.assertTrue(writer.file.closed) @@ -143,8 +142,7 @@ def test_path_like_context_manager(self): print("writing all messages/comments") with self.writer_constructor(self.test_file_name) as writer: self._write_all(writer) - if hasattr(writer.file, 'fileno'): - os.fsync(writer.file.fileno()) + self._ensure_fsync(writer) w = writer if hasattr(w.file, 'closed'): self.assertTrue(w.file.closed) @@ -172,8 +170,7 @@ def test_file_like_explicit_stop(self): my_file = open(self.test_file_name, 'wb' if self.binary_file else 'w') writer = self.writer_constructor(my_file) self._write_all(writer) - if hasattr(writer.file, 'fileno'): - os.fsync(writer.file.fileno()) + self._ensure_fsync(writer) writer.stop() if hasattr(my_file, 'closed'): self.assertTrue(my_file.closed) @@ -203,8 +200,7 @@ def test_file_like_context_manager(self): my_file = open(self.test_file_name, 'wb' if self.binary_file else 'w') with self.writer_constructor(my_file) as writer: self._write_all(writer) - if hasattr(writer.file, 'fileno'): - os.fsync(writer.file.fileno()) + self._ensure_fsync(writer) w = writer if hasattr(my_file, 'closed'): self.assertTrue(my_file.closed) @@ -240,8 +236,7 @@ def test_append_mode(self): with self.writer_constructor(self.test_file_name) as writer: for message in first_part: writer(message) - if hasattr(writer.file, 'fileno'): - os.fsync(writer.file.fileno()) + self._ensure_fsync(writer) # use append mode for second half try: @@ -256,8 +251,7 @@ def test_append_mode(self): with writer: for message in second_part: writer(message) - if hasattr(writer.file, 'fileno'): - os.fsync(writer.file.fileno()) + self._ensure_fsync(writer) with self.reader_constructor(self.test_file_name) as reader: read_messages = list(reader) @@ -275,6 +269,11 @@ def _write_all(self, writer): print("writing message: ", msg) writer(msg) + def _ensure_fsync(self, io_handler): + if hasattr(io_handler.file, 'fileno'): + io_handler.file.flush() + os.fsync(io_handler.file.fileno()) + def assertMessagesEqual(self, read_messages): """ Checks the order and content of the individual messages. From c86fed3ca8e9c558478fb2f0d20a45d5b6842f19 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 13:13:28 +0200 Subject: [PATCH 65/95] removed unused imports --- test/logformats_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index caa786430..0fa989d4b 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -18,8 +18,6 @@ import unittest import tempfile import os -import sqlite3 -import os from abc import abstractmethod, ABCMeta try: @@ -32,8 +30,7 @@ import can from .data.example_data import TEST_MESSAGES_BASE, TEST_MESSAGES_REMOTE_FRAMES, \ - TEST_MESSAGES_ERROR_FRAMES, TEST_COMMENTS, \ - generate_message + TEST_MESSAGES_ERROR_FRAMES, TEST_COMMENTS class ReaderWriterTest(unittest.TestCase): From 796fb8d8f455fc91a36e4b1f2b20a60176d93309 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 16:49:35 +0200 Subject: [PATCH 66/95] use more lightwight SimpleQueue in BufferedReader --- can/listener.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/can/listener.py b/can/listener.py index 9a9f56ed2..e1940a05e 100644 --- a/can/listener.py +++ b/can/listener.py @@ -9,11 +9,10 @@ try: # Python 3 - import queue + from queue import SimpleQueue, Empty except ImportError: # Python 2 - import Queue as queue - + from Queue import Queue as SimpleQueue, Empty class Listener(object): """The basic listener that can be called directly to handle some @@ -73,7 +72,7 @@ class BufferedReader(Listener): def __init__(self): # 0 is "infinite" size - self.buffer = queue.Queue(0) + self.buffer = SimpleQueue(0) def on_message_received(self, msg): self.buffer.put(msg) @@ -90,5 +89,5 @@ def get_message(self, timeout=0.5): """ try: return self.buffer.get(block=True, timeout=timeout) - except queue.Empty: + except Empty: return None From 1975d5205a7ec87f2e6dde9da5a1860ee67a359d Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 17:27:40 +0200 Subject: [PATCH 67/95] add a stop() method to BufferedReader --- can/listener.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/can/listener.py b/can/listener.py index e1940a05e..1033069a7 100644 --- a/can/listener.py +++ b/can/listener.py @@ -67,7 +67,12 @@ class BufferedReader(Listener): **message buffer**: that is, when the :class:`can.BufferedReader` instance is notified of a new message it pushes it into a queue of messages waiting to be serviced. The messages can then be fetched with - :meth:`~can.BufferedReader.get_message` + :meth:`~can.BufferedReader.get_message`. + + Putting in messages after :meth:`~can.BufferedReader.stop` has be called will raise + an exception, see :meth:`~can.BufferedReader.on_message_received`. + + :attr bool is_stopped: ``True`` iff the reader has been stopped """ def __init__(self): @@ -75,19 +80,33 @@ def __init__(self): self.buffer = SimpleQueue(0) def on_message_received(self, msg): - self.buffer.put(msg) + """Append a message to the buffer. + + :raises: BufferError + if the reader has already been stopped + """ + if self.is_stopped: + raise BufferError("reader has already been stopped") + else: + self.buffer.put(msg) def get_message(self, timeout=0.5): """ Attempts to retrieve the latest message received by the instance. If no message is - available it blocks for given timeout or until a message is received (whichever - is shorter), + available it blocks for given timeout or until a message is received, or else + returns None (whichever is shorter). This method does not block after + :meth:`can.BufferedReader.stop` has been called. :param float timeout: The number of seconds to wait for a new message. - :rytpe: can.Message + :rytpe: can.Message or None :return: the message if there is one, or None if there is not. """ try: - return self.buffer.get(block=True, timeout=timeout) + return self.buffer.get(block=not self.is_stopped, timeout=timeout) except Empty: return None + + def stop(self): + """Prohibits any more additions to this reader. + """ + self.is_stopped = True From bd060892d50a354eba8da955c62a8960d147f1ec Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 17:28:01 +0200 Subject: [PATCH 68/95] various SqliteWriter improvements --- can/io/sqlite.py | 59 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 0fe2d8d68..e071de7d1 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -35,6 +35,8 @@ class SqliteReader(BaseIOHandler): Calling :func:`~builtin.len` on this object might not run in constant time. + :attr str table_name: the name of the database table used for storing the messages + .. note:: The database schema is given in the documentation of the loggers. """ @@ -91,12 +93,20 @@ class SqliteWriter(BaseIOHandler, BufferedReader): be created when the first message arrives. Messages are internally buffered and written to the SQL file in a background - thread. + thread. Ensures that all messages that are added before calling :meth:`~can.SqliteWriter.stop()` + are actually written to the database after that call returns. Thus, calling + :meth:`~can.SqliteWriter.stop()` may take a while. + + :attr str table_name: the name of the database table used for storing the messages + :attr int num_frames: the number of frames actally writtem to the database, this + excludes messages that are still buffered + :attr float last_write: the last time a message war actually written to the database, + as given by ``time.time()`` .. note:: When the listener's :meth:`~SqliteWriter.stop` method is called the - thread writing to the sql file will continue to receive and internally + thread writing to the database will continue to receive and internally buffer messages if they continue to arrive before the :attr:`~SqliteWriter.GET_MESSAGE_TIMEOUT`. @@ -104,8 +114,9 @@ class SqliteWriter(BaseIOHandler, BufferedReader): is received, the internal buffer is written out to the database file. However if the bus is still saturated with messages, the Listener - will continue receiving until the :attr:`~SqliteWriter.MAX_TIME_BETWEEN_WRITES` - timeout is reached. + will continue receiving until the :attr:`~can.SqliteWriter.MAX_TIME_BETWEEN_WRITES` + timeout is reached or more than + :attr:`~can.SqliteWriter.MAX_BUFFER_SIZE_BEFORE_WRITES` messages are buffered. .. note:: The database schema is given in the documentation of the loggers. @@ -117,6 +128,9 @@ class SqliteWriter(BaseIOHandler, BufferedReader): MAX_TIME_BETWEEN_WRITES = 5.0 """Maximum number of seconds to wait between writes to the database""" + MAX_BUFFER_SIZE_BEFORE_WRITES = 500 + """Maximum number of messages to buffer before writing to the database""" + def __init__(self, file, table_name="messages"): """ :param file: a `str` or since Python 3.7 a path like object that points @@ -132,6 +146,8 @@ def __init__(self, file, table_name="messages"): self._stop_running_event = threading.Event() self._writer_thread = threading.Thread(target=self._db_writer_thread) self._writer_thread.start() + self.num_frames = 0 + self.last_write = time.time() def _create_db(self): """Creates a new databae or opens a connection to an existing one. @@ -161,17 +177,15 @@ def _create_db(self): self._insert_template = "INSERT INTO {} VALUES (?, ?, ?, ?, ?, ?, ?)".format(self.table_name) def _db_writer_thread(self): - num_frames = 0 - last_write = time.time() self._create_db() try: - while not self._stop_running_event.is_set(): - messages = [] + while True: + messages = [] # reset buffer msg = self.get_message(self.GET_MESSAGE_TIMEOUT) while msg is not None: - #log.debug("SqliteWriter: buffering message") + log.debug("SqliteWriter: buffering message") messages.append(( msg.timestamp, @@ -183,11 +197,12 @@ def _db_writer_thread(self): buffer(msg.data) )) - if time.time() - last_write > self.MAX_TIME_BETWEEN_WRITES: - #log.debug("Max timeout between writes reached") - break - - msg = self.get_message(self.GET_MESSAGE_TIMEOUT) + if time.time() - self.last_write > self.MAX_TIME_BETWEEN_WRITES or \ + len(messages) > self.MAX_BUFFER_SIZE_BEFORE_WRITES: + break + else: + # just go on + msg = self.get_message(self.GET_MESSAGE_TIMEOUT) count = len(messages) if count > 0: @@ -195,16 +210,22 @@ def _db_writer_thread(self): #log.debug("Writing %s frames to db", count) self._conn.executemany(self._insert_template, messages) self._conn.commit() # make the changes visible to the entire database - num_frames += count - last_write = time.time() + self.num_frames += count + self.last_write = time.time() - # go back up and check if we are still supposed to run + # check if we are still supposed to run and go back up if yes + if self._stop_running_event.is_set(): + break finally: self._conn.close() - log.info("Stopped sqlite writer after writing %s messages", num_frames) + log.info("Stopped sqlite writer after writing %d messages", self.num_frames) def stop(self): - super(SqliteWriter, self).stop() + """Stops the reader an writes all remaining messages to the database. Thus, this + might take a while an block. + """ + BufferedReader.stop(self) self._stop_running_event.set() self._writer_thread.join() + BaseIOHandler.stop(self) From ea7344590819386c7774e16020b761400848a0c3 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 17:35:19 +0200 Subject: [PATCH 69/95] corrected import for queues in listener.py --- can/listener.py | 11 ++++++++--- test/logformats_test.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/can/listener.py b/can/listener.py index 1033069a7..9f46f6e8d 100644 --- a/can/listener.py +++ b/can/listener.py @@ -8,11 +8,16 @@ from abc import ABCMeta, abstractmethod try: - # Python 3 + # Python 3.7 from queue import SimpleQueue, Empty except ImportError: - # Python 2 - from Queue import Queue as SimpleQueue, Empty + try: + # Python 3.0 - 3.6 + from queue import Queue as SimpleQueue, Empty + except ImportError: + # Python 2 + from Queue import Queue as SimpleQueue, Empty + class Listener(object): """The basic listener that can be called directly to handle some diff --git a/test/logformats_test.py b/test/logformats_test.py index 0fa989d4b..9ec9264de 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -239,7 +239,7 @@ def test_append_mode(self): try: writer = self.writer_constructor(self.test_file_name, append=True) except TypeError as e: - # maybe "append" is not a formal parameter + # maybe "append" is not a formal parameter (this is the case for SqliteWriter) try: writer = self.writer_constructor(self.test_file_name) except TypeError: From 9e38a95d8d2dff1404b1d870ad6c9aa829ac7517 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 17:38:53 +0200 Subject: [PATCH 70/95] bugfix in BufferedReader --- can/listener.py | 1 + 1 file changed, 1 insertion(+) diff --git a/can/listener.py b/can/listener.py index 9f46f6e8d..1642eaca0 100644 --- a/can/listener.py +++ b/can/listener.py @@ -83,6 +83,7 @@ class BufferedReader(Listener): def __init__(self): # 0 is "infinite" size self.buffer = SimpleQueue(0) + self.is_stopped = False def on_message_received(self, msg): """Append a message to the buffer. From a3001dc5a53bad99d4cbdc612799de09a5564d35 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 18:11:25 +0200 Subject: [PATCH 71/95] added cleanups for listener tests --- test/listener_test.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/listener_test.py b/test/listener_test.py index 446fb4a37..096815545 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -92,10 +92,9 @@ def testRemoveListenerFromNotifier(self): def testPlayerTypeResolution(self): def test_filetype_to_instance(extension, klass): - can_player = can.LogReader("test.{}".format(extension)) - self.assertIsInstance(can_player, klass) - if hasattr(can_player, "stop"): - can_player.stop() + with tempfile.NamedTemporaryFile(suffix=extension) as my_file: + with can.LogReader(my_file.name) as reader: + self.assertIsInstance(reader, klass) test_filetype_to_instance("asc", can.ASCReader) test_filetype_to_instance("blf", can.BLFReader) @@ -111,10 +110,9 @@ def test_filetype_to_instance(extension, klass): def testLoggerTypeResolution(self): def test_filetype_to_instance(extension, klass): - can_logger = can.Logger("test.{}".format(extension)) - self.assertIsInstance(can_logger, klass) - if hasattr(can_logger, "stop"): - can_logger.stop() + with tempfile.NamedTemporaryFile(suffix=extension) as my_file: + with can.Logger(my_file.name) as writer: + self.assertIsInstance(writer, klass) test_filetype_to_instance("asc", can.ASCWriter) test_filetype_to_instance("blf", can.BLFWriter) @@ -130,8 +128,10 @@ def test_filetype_to_instance(extension, klass): def testBufferedListenerReceives(self): a_listener = can.BufferedReader() a_listener(generate_message(0xDADADA)) - m = a_listener.get_message(0.1) - self.assertIsNotNone(m) + a_listener(generate_message(0xDADADA)) + self.assertIsNotNone(a_listener.get_message(0.1)) + a_listener.stop() + self.assertIsNotNone(a_listener.get_message(0.1)) if __name__ == '__main__': From f11e165830701bfd9d7e3e818dc681a0fcddc3ab Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 18:14:35 +0200 Subject: [PATCH 72/95] added superclasses for Logger and LogReader --- can/io/logger.py | 4 +++- can/io/player.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index bb4a966ed..74ab93420 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -9,6 +9,8 @@ import logging +from ..listener import Listener +from .generic import BaseIOHandler from .asc import ASCWriter from .blf import BLFWriter from .canutils import CanutilsLogWriter @@ -19,7 +21,7 @@ log = logging.getLogger("can.io.logger") -class Logger(object): +class Logger(BaseIOHandler, Listener): """ Logs CAN messages to a file. diff --git a/can/io/player.py b/can/io/player.py index 186442bd2..c6444dde2 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -12,6 +12,7 @@ import time import logging +from .generic import BaseIOHandler from .asc import ASCReader from .blf import BLFReader from .canutils import CanutilsLogReader @@ -21,7 +22,7 @@ log = logging.getLogger('can.io.player') -class LogReader(object): +class LogReader(BaseIOHandler): """ Replay logged CAN messages from a file. From 3c87b3e2c1f9e19c57f72c9e04cd8fb995bb0a65 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 18:20:56 +0200 Subject: [PATCH 73/95] doc improvements --- can/io/logger.py | 5 +++-- can/io/player.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 74ab93420..72487cc9e 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -33,8 +33,9 @@ class Logger(BaseIOHandler, Listener): * .log :class:`can.CanutilsLogWriter` * other: :class:`can.Printer` - Note this class itself is just a dispatcher, an object that inherits - from Listener will be created when instantiating this class. + .. note:: + This class itself is just a dispatcher, and any positional an keyword + arguments are passed on to the returned instance. """ @staticmethod diff --git a/can/io/player.py b/can/io/player.py index c6444dde2..4af42c479 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -35,13 +35,16 @@ class LogReader(BaseIOHandler): Exposes a simple iterator interface, to use simply: - >>> for m in LogReader(my_file): - ... print(m) + >>> for msg in LogReader("some/path/to/my_file.log"): + ... print(msg) .. note:: - There are no time delays, if you want to reproduce - the measured delays between messages look at the - :class:`can.MessageSync` class. + There are no time delays, if you want to reproduce the measured + delays between messages look at the :class:`can.MessageSync` class. + + .. note:: + This class itself is just a dispatcher, and any positional an keyword + arguments are passed on to the returned instance. """ @staticmethod From f53aa0ad473aa6abd40cf2d9642999e60643259a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 18:45:18 +0200 Subject: [PATCH 74/95] updated history --- doc/history.rst | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/doc/history.rst b/doc/history.rst index dfc7ad532..ee8667920 100644 --- a/doc/history.rst +++ b/doc/history.rst @@ -22,24 +22,36 @@ who wrote a leaf-socketcan driver for Linux. The pcan interface was contributed by Albert Bloomfield in 2013. -The usb2can interface was contributed by Joshua Villyard in 2015 +The usb2can interface was contributed by Joshua Villyard in 2015. The IXXAT VCI interface was contributed by Giuseppe Corbelli and funded -by `Weightpack `__ in 2016 +by `Weightpack `__ in 2016. The NI-CAN and virtual interfaces plus the ASCII and BLF loggers were contributed by Christian Sandberg in 2016 and 2017. The BLF format is based on a C++ library by Toby Lorenz. -The slcan interface, ASCII listener and log logger and listener were contributed by Eduard Bröcker in 2017. +The slcan interface, ASCII listener and log logger and listener were contributed +by Eduard Bröcker in 2017. The NeoVi interface for ICS (Intrepid Control Systems) devices was contributed by Pierre-Luc Tessier Gagné in 2017. +Many improvements all over the library, cleanups, unifications as well as more +comprehensive documentation and CI testing was contributed by Felix Divo in 2017 +and 2018. + Support for CAN within Python ----------------------------- -The 'socket' module contains support for SocketCAN from Python 3.3. +Python natively supports the CAN protocol from version 3.3 on, if running on Lniux: -From Python 3.4 broadcast management commands are natively supported. +============== ============================================================== ==== +Python version Feature Link +============== ============================================================== ==== +3.3 Initial SocketCAN support `Docs `__ +3.4 Broadcast Banagement (BCM) commands are natively supported `Docs `__ +3.5 CAN FD support `Docs `__ +3.7 Support for CAN ISO-TP `Docs `__ +============== ============================================================== ==== From a62c9179b7bd77ecc2d5c5b21892d78ba393e8f2 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 18:48:32 +0200 Subject: [PATCH 75/95] fix for listener test --- test/listener_test.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/listener_test.py b/test/listener_test.py index 096815545..6cf69b109 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -96,15 +96,15 @@ def test_filetype_to_instance(extension, klass): with can.LogReader(my_file.name) as reader: self.assertIsInstance(reader, klass) - test_filetype_to_instance("asc", can.ASCReader) - test_filetype_to_instance("blf", can.BLFReader) - test_filetype_to_instance("csv", can.CSVReader) - test_filetype_to_instance("db" , can.SqliteReader) - test_filetype_to_instance("log", can.CanutilsLogReader) + test_filetype_to_instance(".asc", can.ASCReader) + test_filetype_to_instance(".blf", can.BLFReader) + test_filetype_to_instance(".csv", can.CSVReader) + test_filetype_to_instance(".db" , can.SqliteReader) + test_filetype_to_instance(".log", can.CanutilsLogReader) # test file extensions that are not supported - with self.assertRaisesRegexp(NotImplementedError, "xyz_42"): - test_filetype_to_instance("xyz_42", can.Printer) + with self.assertRaisesRegexp(NotImplementedError, ".xyz_42"): + test_filetype_to_instance(".xyz_42", can.Printer) with self.assertRaises(Exception): test_filetype_to_instance(None, can.Printer) @@ -114,16 +114,16 @@ def test_filetype_to_instance(extension, klass): with can.Logger(my_file.name) as writer: self.assertIsInstance(writer, klass) - test_filetype_to_instance("asc", can.ASCWriter) - test_filetype_to_instance("blf", can.BLFWriter) - test_filetype_to_instance("csv", can.CSVWriter) - test_filetype_to_instance("db" , can.SqliteWriter) - test_filetype_to_instance("log", can.CanutilsLogWriter) - test_filetype_to_instance("txt", can.Printer) + test_filetype_to_instance(".asc", can.ASCWriter) + test_filetype_to_instance(".blf", can.BLFWriter) + test_filetype_to_instance(".csv", can.CSVWriter) + test_filetype_to_instance(".db" , can.SqliteWriter) + test_filetype_to_instance(".log", can.CanutilsLogWriter) + test_filetype_to_instance(".txt", can.Printer) # test file extensions that should use a fallback + test_filetype_to_instance(".some_unknown_extention_42", can.Printer) test_filetype_to_instance(None, can.Printer) - test_filetype_to_instance("some_unknown_extention_42", can.Printer) def testBufferedListenerReceives(self): a_listener = can.BufferedReader() From 118a455194f09c2eb6bf2fbc84f0af8989e9fbb2 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 23:17:21 +0200 Subject: [PATCH 76/95] add tests for SqliteReader.read_all() --- test/logformats_test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/logformats_test.py b/test/logformats_test.py index 9ec9264de..e7a77f0ca 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -392,6 +392,26 @@ def test_file_like_explicit_stop(self): def test_file_like_context_manager(self): pass + def test_read_all(self): + """ + testing :meth:`can.SqliteReader.read_all` with context manager and path-like object + """ + # create writer + print("writing all messages/comments") + with self.writer_constructor(self.test_file_name) as writer: + self._write_all(writer) + + # read all written messages + print("reading all messages") + with self.reader_constructor(self.test_file_name) as reader: + read_messages = reader.read_all() + + # check if at least the number of messages matches; + self.assertEqual(len(read_messages), len(self.original_messages), + "the number of written messages does not match the number of read messages") + + self.assertMessagesEqual(read_messages) + class TestPrinter(unittest.TestCase): """Tests that can.Printer does not crash""" From fe638e5d64bee8c2f16e671825459ec00926af28 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 23:17:46 +0200 Subject: [PATCH 77/95] fix SqliteReader.read_all() and fix usage of memoryview --- can/io/sqlite.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index e071de7d1..0f2069aa6 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -21,10 +21,6 @@ log = logging.getLogger('can.io.sqlite') -# TODO comment on this -if sys.version_info > (3,): - buffer = memoryview - class SqliteReader(BaseIOHandler): """ @@ -57,16 +53,20 @@ def __init__(self, file, table_name="messages"): def __iter__(self): for frame_data in self._cursor.execute("SELECT * FROM {}".format(self.table_name)): - timestamp, can_id, is_extended, is_remote, is_error, dlc, data = frame_data - yield Message( - timestamp=timestamp, - is_remote_frame=bool(is_remote), - extended_id=bool(is_extended), - is_error_frame=bool(is_error), - arbitration_id=can_id, - dlc=dlc, - data=data - ) + yield SqliteReader._assemble_message(frame_data) + + @staticmethod + def _assemble_message(frame_data): + timestamp, can_id, is_extended, is_remote, is_error, dlc, data = frame_data + return Message( + timestamp=timestamp, + is_remote_frame=bool(is_remote), + extended_id=bool(is_extended), + is_error_frame=bool(is_error), + arbitration_id=can_id, + dlc=dlc, + data=data + ) def __len__(self): # this might not run in constant time @@ -75,9 +75,11 @@ def __len__(self): def read_all(self): """Fetches all messages in the database. + + :rtype: Generator[can.Message] """ - result = self._cursor.execute("SELECT * FROM {}".format(self.table_name)) - return result.fetchall() + result = self._cursor.execute("SELECT * FROM {}".format(self.table_name)).fetchall() + return (SqliteReader._assemble_message(frame) for frame in result) def stop(self): """Closes the connection to the database. @@ -194,7 +196,7 @@ def _db_writer_thread(self): msg.is_remote_frame, msg.is_error_frame, msg.dlc, - buffer(msg.data) + memoryview(msg.data) )) if time.time() - self.last_write > self.MAX_TIME_BETWEEN_WRITES or \ From 5a75ef7044318d056f0ad38e6589fb64b4714efa Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 23:24:27 +0200 Subject: [PATCH 78/95] fix test_read_all --- test/logformats_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index e7a77f0ca..7a88fe703 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -404,7 +404,7 @@ def test_read_all(self): # read all written messages print("reading all messages") with self.reader_constructor(self.test_file_name) as reader: - read_messages = reader.read_all() + read_messages = list(reader.read_all()) # check if at least the number of messages matches; self.assertEqual(len(read_messages), len(self.original_messages), From 87ddb616c3c1729fcf6c8479e6faf7e1c9cc7a2f Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 20 Jul 2018 23:38:17 +0200 Subject: [PATCH 79/95] logging adjustments --- can/io/sqlite.py | 4 ++-- test/listener_test.py | 3 +-- test/logformats_test.py | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 0f2069aa6..79189e7a1 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -187,7 +187,7 @@ def _db_writer_thread(self): msg = self.get_message(self.GET_MESSAGE_TIMEOUT) while msg is not None: - log.debug("SqliteWriter: buffering message") + #log.debug("SqliteWriter: buffering message") messages.append(( msg.timestamp, @@ -209,7 +209,7 @@ def _db_writer_thread(self): count = len(messages) if count > 0: with self._conn: - #log.debug("Writing %s frames to db", count) + #log.debug("Writing %d frames to db", count) self._conn.executemany(self._insert_template, messages) self._conn.commit() # make the changes visible to the entire database self.num_frames += count diff --git a/test/listener_test.py b/test/listener_test.py index 6cf69b109..137111e3e 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -11,7 +11,6 @@ import random import logging import tempfile -import os.path import sqlite3 import can @@ -21,7 +20,7 @@ channel = 'virtual_channel_0' can.rc['interface'] = 'virtual' -logging.getLogger('').setLevel(logging.DEBUG) +logging.basicConfig(level=logging.DEBUG) # makes the random number generator deterministic random.seed(13339115) diff --git a/test/logformats_test.py b/test/logformats_test.py index 7a88fe703..c9a1ff27d 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -15,6 +15,7 @@ from __future__ import print_function, absolute_import, division +import logging import unittest import tempfile import os @@ -32,6 +33,8 @@ from .data.example_data import TEST_MESSAGES_BASE, TEST_MESSAGES_REMOTE_FRAMES, \ TEST_MESSAGES_ERROR_FRAMES, TEST_COMMENTS +logging.basicConfig(level=logging.DEBUG) + class ReaderWriterTest(unittest.TestCase): """Tests a pair of writer and reader by writing all data first and From 08666575c5c3fa1dfd1aa1496a60fdcc187027d5 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sat, 28 Jul 2018 16:07:23 +0200 Subject: [PATCH 80/95] fix typo --- doc/history.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/history.rst b/doc/history.rst index ee8667920..70d0460c0 100644 --- a/doc/history.rst +++ b/doc/history.rst @@ -45,7 +45,7 @@ and 2018. Support for CAN within Python ----------------------------- -Python natively supports the CAN protocol from version 3.3 on, if running on Lniux: +Python natively supports the CAN protocol from version 3.3 on, if running on Linux: ============== ============================================================== ==== Python version Feature Link From 74b59cf1ba82c60e778180e273279828136bdd2e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sun, 29 Jul 2018 12:04:01 +0200 Subject: [PATCH 81/95] fix logger_test.py --- can/io/logger.py | 34 ++++++++++++++++++++-------------- test/listener_test.py | 18 ++++++++++++++---- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 72487cc9e..9095c5898 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -41,18 +41,24 @@ class Logger(BaseIOHandler, Listener): @staticmethod def __new__(cls, filename, *args, **kwargs): """ - :param str filename: the filename/path the file to write to + :type filename: str or None or path-like + :param filename: the filename/path the file to write to, + may be a path-like object if the target logger supports + it, and may be None to instantiate a :class:`~can.Printer` + """ - if filename.endswith(".asc"): - return ASCWriter(filename, *args, **kwargs) - elif filename.endswith(".blf"): - return BLFWriter(filename, *args, **kwargs) - elif filename.endswith(".csv"): - return CSVWriter(filename, *args, **kwargs) - elif filename.endswith(".db"): - return SqliteWriter(filename, *args, **kwargs) - elif filename.endswith(".log"): - return CanutilsLogWriter(filename, *args, **kwargs) - else: - log.info('unknown file type "%s", falling pack to can.Printer', filename) - return Printer(filename, *args, **kwargs) + if filename: + if filename.endswith(".asc"): + return ASCWriter(filename, *args, **kwargs) + elif filename.endswith(".blf"): + return BLFWriter(filename, *args, **kwargs) + elif filename.endswith(".csv"): + return CSVWriter(filename, *args, **kwargs) + elif filename.endswith(".db"): + return SqliteWriter(filename, *args, **kwargs) + elif filename.endswith(".log"): + return CanutilsLogWriter(filename, *args, **kwargs) + + # else: + log.info('unknown file type "%s", falling pack to can.Printer', filename) + return Printer(filename, *args, **kwargs) diff --git a/test/listener_test.py b/test/listener_test.py index 137111e3e..090cc62a1 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -12,6 +12,7 @@ import logging import tempfile import sqlite3 +from os.path import join, dirname import can @@ -91,7 +92,14 @@ def testRemoveListenerFromNotifier(self): def testPlayerTypeResolution(self): def test_filetype_to_instance(extension, klass): - with tempfile.NamedTemporaryFile(suffix=extension) as my_file: + print("testing: {}".format(extension)) + + if extension == ".blf": + file_handler = open(join(dirname(__file__), "data/logfile.blf")) + else: + file_handler = tempfile.NamedTemporaryFile(suffix=extension) + + with file_handler as my_file: with can.LogReader(my_file.name) as reader: self.assertIsInstance(reader, klass) @@ -104,11 +112,10 @@ def test_filetype_to_instance(extension, klass): # test file extensions that are not supported with self.assertRaisesRegexp(NotImplementedError, ".xyz_42"): test_filetype_to_instance(".xyz_42", can.Printer) - with self.assertRaises(Exception): - test_filetype_to_instance(None, can.Printer) def testLoggerTypeResolution(self): def test_filetype_to_instance(extension, klass): + print("testing: {}".format(extension)) with tempfile.NamedTemporaryFile(suffix=extension) as my_file: with can.Logger(my_file.name) as writer: self.assertIsInstance(writer, klass) @@ -121,8 +128,11 @@ def test_filetype_to_instance(extension, klass): test_filetype_to_instance(".txt", can.Printer) # test file extensions that should use a fallback + test_filetype_to_instance("", can.Printer) + test_filetype_to_instance(".", can.Printer) test_filetype_to_instance(".some_unknown_extention_42", can.Printer) - test_filetype_to_instance(None, can.Printer) + with can.Logger(None) as logger: + self.assertIsInstance(logger, can.Printer) def testBufferedListenerReceives(self): a_listener = can.BufferedReader() From 31734d05ec4e117f8ed1276a455bb094adf49a27 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 30 Jul 2018 11:53:30 +0200 Subject: [PATCH 82/95] use RuntimeError on read attempt on a stopped BufferedReader --- can/listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/can/listener.py b/can/listener.py index 1642eaca0..1c6983a0f 100644 --- a/can/listener.py +++ b/can/listener.py @@ -92,7 +92,7 @@ def on_message_received(self, msg): if the reader has already been stopped """ if self.is_stopped: - raise BufferError("reader has already been stopped") + raise RuntimeError("reader has already been stopped") else: self.buffer.put(msg) From bf913641f7e28d7b891506f108e74a4f0f18046a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 30 Jul 2018 12:43:31 +0200 Subject: [PATCH 83/95] add writable tempdir to appveyor tests --- .appveyor.yml | 1 + .gitignore | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 10d8a6ca5..b4b0af7f2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -28,5 +28,6 @@ install: build: off test_script: + - set TMPDIR=%cd%\test\__tempdir__ - "pytest" - "codecov" diff --git a/.gitignore b/.gitignore index 96acb31a4..10aacc415 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ -# https://github.com/github/gitignore/blob/da00310ccba9de9a988cc973ef5238ad2c1460e9/Python.gitignore +__tempdir__ + + +# ------------------------- +# below: https://github.com/github/gitignore/blob/da00310ccba9de9a988cc973ef5238ad2c1460e9/Python.gitignore # Byte-compiled / optimized / DLL files __pycache__/ From 5449ac4b8511af803eacd7421a4068cd9913a34f Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 30 Jul 2018 12:54:32 +0200 Subject: [PATCH 84/95] use memoryview = buffer in Sqlite in Python 2 --- can/io/sqlite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 79189e7a1..23be2b8f5 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -21,6 +21,10 @@ log = logging.getLogger('can.io.sqlite') +if sys.version_info.major < 3: + # legacy fallback for Python 2 + memoryview = buffer + class SqliteReader(BaseIOHandler): """ From 3beb5b5d281d1f4ba88ccad2e0de37629a33ca7d Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 31 Jul 2018 16:45:00 +0200 Subject: [PATCH 85/95] fix Message's __eq__() and __ne__() --- can/message.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/can/message.py b/can/message.py index dc9c80695..9bbf82a0e 100644 --- a/can/message.py +++ b/can/message.py @@ -144,10 +144,13 @@ def __eq__(self, other): self.bitrate_switch == other.bitrate_switch ) else: - raise NotImplementedError() + return NotImplemented def __ne__(self, other): - return not self.__eq__(other) + if isinstance(other, self.__class__): + return not self.__eq__(other) + else: + return NotImplemented def __hash__(self): return hash(( From 201bf229d4c89cc16df3bd4c07aae2524f569578 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 31 Jul 2018 16:49:56 +0200 Subject: [PATCH 86/95] fix a bug in Python 3.7 --- can/listener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/can/listener.py b/can/listener.py index 1c6983a0f..1388bfedc 100644 --- a/can/listener.py +++ b/can/listener.py @@ -81,8 +81,8 @@ class BufferedReader(Listener): """ def __init__(self): - # 0 is "infinite" size - self.buffer = SimpleQueue(0) + # set to "infinite" size + self.buffer = SimpleQueue() self.is_stopped = False def on_message_received(self, msg): From 39eaa66c50f1b2ddb0c46a4b750c72f6cd69d30a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 31 Jul 2018 17:07:14 +0200 Subject: [PATCH 87/95] new attempt at making the TEMPDIR writable on AppVeyor --- .appveyor.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index b4b0af7f2..cfd26c61d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -27,7 +27,13 @@ install: build: off +before_test: + # see: https://help.appveyor.com/discussions/problems/938-getting-permission-denied-errors-when-trying-to-make-a-temp-directory + - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tempdir__ + - icacls %TMPDIR% /inheritance:r /grant Everyone:F + test_script: - - set TMPDIR=%cd%\test\__tempdir__ + # TODO: is this required? + - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tempdir__ - "pytest" - "codecov" From 53c7ec96060bdd4402cffe967219073d8a8b1d97 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 31 Jul 2018 17:14:06 +0200 Subject: [PATCH 88/95] create test folder in AppVeyor --- .appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index cfd26c61d..e0120be4a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -30,6 +30,7 @@ build: off before_test: # see: https://help.appveyor.com/discussions/problems/938-getting-permission-denied-errors-when-trying-to-make-a-temp-directory - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tempdir__ + - mkdir %TMPDIR% - icacls %TMPDIR% /inheritance:r /grant Everyone:F test_script: @@ -37,3 +38,5 @@ test_script: - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tempdir__ - "pytest" - "codecov" + +# TODO delete at the end (?) From 3bf5a6a22fdb5c1275087352a3abe2e8c86a1526 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 31 Jul 2018 22:00:36 +0200 Subject: [PATCH 89/95] also give permissions to future temp files & clean up afterwards --- .appveyor.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index e0120be4a..03dc7c680 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -29,14 +29,14 @@ build: off before_test: # see: https://help.appveyor.com/discussions/problems/938-getting-permission-denied-errors-when-trying-to-make-a-temp-directory - - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tempdir__ + # TMPDIR is used by Python to detect the temporary directory: https://docs.python.org/3/library/tempfile.html#tempfile.tempdir + - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tmpdir__ - mkdir %TMPDIR% - - icacls %TMPDIR% /inheritance:r /grant Everyone:F + - icacls %TMPDIR% /inheritance:r /grant Everyone:(OI)(CI)F test_script: - # TODO: is this required? - - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tempdir__ - "pytest" - "codecov" -# TODO delete at the end (?) +after_test: + - rmdir /S /Q %TMPDIR% From 767838bab0659c28f2e26b1b682769e596c96a33 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 31 Jul 2018 23:39:04 +0200 Subject: [PATCH 90/95] fix #379 --- can/io/asc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index b9409ad39..58d77bea8 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -4,7 +4,9 @@ """ Contains handling of ASC logging files. -Example .asc file: https://bitbucket.org/tobylorenz/vector_asc/src/47556e1a6d32c859224ca62d075e1efcc67fa690/src/Vector/ASC/tests/unittests/data/CAN_Log_Trigger_3_2.asc?at=master&fileviewer=file-view-default +Example .asc files: + - https://bitbucket.org/tobylorenz/vector_asc/src/47556e1a6d32c859224ca62d075e1efcc67fa690/src/Vector/ASC/tests/unittests/data/CAN_Log_Trigger_3_2.asc?at=master&fileviewer=file-view-default + - under `test/data/logfile.asc` """ from __future__ import absolute_import @@ -176,10 +178,9 @@ def log_event(self, message, timestamp=None): self.last_timestamp = (timestamp or 0.0) self.started = self.last_timestamp formatted_date = time.strftime(self.FORMAT_DATE, time.localtime(self.last_timestamp)) - self.file.write("base hex timestamps absolute\n") self.file.write("Begin Triggerblock %s\n" % formatted_date) self.header_written = True - self.log_event("Start of measurement") # recursive call + self.log_event("Start of measurement") # caution: this is a recursive call! # figure out the correct timestamp if timestamp is None or timestamp < self.last_timestamp: From 34a640b328410ab9588492f00d2c2793ded3ae05 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 1 Aug 2018 09:55:17 +0200 Subject: [PATCH 91/95] fix wrong timestamps problem in tests --- can/io/asc.py | 6 +++--- test/data/example_data.py | 25 +++++++++++++++++++++++-- test/logformats_test.py | 6 +++++- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index 58d77bea8..bbc8807b7 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -82,8 +82,8 @@ def __iter__(self): pass elif dummy[-1:].lower() == 'r': - (can_id_str, _) = dummy.split(None, 1) - (can_id_num, is_extended_id) = self._extract_can_id(can_id_str) + can_id_str, _ = dummy.split(None, 1) + can_id_num, is_extended_id = self._extract_can_id(can_id_str) msg = Message(timestamp=timestamp, arbitration_id=can_id_num & CAN_ID_MASK, extended_id=is_extended_id, @@ -126,7 +126,7 @@ class ASCWriter(BaseIOHandler, Listener): """Logs CAN data to an ASCII log file (.asc). The measurement starts with the timestamp of the first registered message. - If a message has a timestamp smaller than the previous one (or 0 or None), + If a message has a timestamp smaller than the previous one or None, it gets assigned the timestamp that was written for the last message. It the first message does not have a timestamp, it is set to zero. """ diff --git a/test/data/example_data.py b/test/data/example_data.py index c3290683a..e1a446384 100644 --- a/test/data/example_data.py +++ b/test/data/example_data.py @@ -7,15 +7,28 @@ """ import random +from operator import attrgetter from can import Message # make tests more reproducible random.seed(13339115) + +def sort_messages(messages): + """ + Sorts the given messages by timestamps (ascending). + + :param Iterable[can.Message] messages: a sequence of messages to sort + :rtype: list + """ + return list(sorted(messages, key=attrgetter('timestamp'))) + + # some random number TEST_TIME = 1483389946.197 + # List of messages of different types that can be used in tests TEST_MESSAGES_BASE = [ Message( @@ -70,6 +83,8 @@ timestamp=TEST_TIME + 3.165 ), ] +TEST_MESSAGES_BASE = sort_messages(TEST_MESSAGES_BASE) + TEST_MESSAGES_REMOTE_FRAMES = [ Message( @@ -91,6 +106,8 @@ timestamp=TEST_TIME + 7858.67 ), ] +TEST_MESSAGES_REMOTE_FRAMES = sort_messages(TEST_MESSAGES_REMOTE_FRAMES) + TEST_MESSAGES_ERROR_FRAMES = [ Message( @@ -105,8 +122,12 @@ timestamp=TEST_TIME + 17.157 ) ] +TEST_MESSAGES_ERROR_FRAMES = sort_messages(TEST_MESSAGES_ERROR_FRAMES) + + +TEST_ALL_MESSAGES = sort_messages(TEST_MESSAGES_BASE + TEST_MESSAGES_REMOTE_FRAMES + \ + TEST_MESSAGES_ERROR_FRAMES) -TEST_ALL_MESSAGES = TEST_MESSAGES_BASE + TEST_MESSAGES_REMOTE_FRAMES + TEST_MESSAGES_ERROR_FRAMES TEST_COMMENTS = [ "This is the first comment", @@ -127,4 +148,4 @@ def generate_message(arbitration_id): and a non-extended ID. """ data = bytearray([random.randrange(0, 2 ** 8 - 1) for _ in range(8)]) - return Message(arbitration_id=arbitration_id, data=data, extended_id=False) + return Message(arbitration_id=arbitration_id, data=data, extended_id=False, timestamp=TEST_TIME) diff --git a/test/logformats_test.py b/test/logformats_test.py index c9a1ff27d..2a315352d 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -31,7 +31,8 @@ import can from .data.example_data import TEST_MESSAGES_BASE, TEST_MESSAGES_REMOTE_FRAMES, \ - TEST_MESSAGES_ERROR_FRAMES, TEST_COMMENTS + TEST_MESSAGES_ERROR_FRAMES, TEST_COMMENTS, \ + sort_messages logging.basicConfig(level=logging.DEBUG) @@ -82,6 +83,9 @@ def _setup_instance_helper(self, if check_error_frames: self.original_messages += TEST_MESSAGES_ERROR_FRAMES + # sort them so that for example ASCWriter does not "fix" any messages with timestamp 0.0 + self.original_messages = sort_messages(self.original_messages) + if check_comments: # we check this because of the lack of a common base class # we filter for not starts with '__' so we do not get all the builtin From d1b0503eb3310f2ac6c37e8abe9374902a370950 Mon Sep 17 00:00:00 2001 From: Felix D Date: Wed, 1 Aug 2018 19:15:42 +0200 Subject: [PATCH 92/95] fix apparent permissions problem on windows --- .gitignore | 4 ++-- test/listener_test.py | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 10aacc415..b0f71da4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -__tempdir__ - +test/__tempdir__/ +.pytest_cache/ # ------------------------- # below: https://github.com/github/gitignore/blob/da00310ccba9de9a988cc973ef5238ad2c1460e9/Python.gitignore diff --git a/test/listener_test.py b/test/listener_test.py index 090cc62a1..64371def2 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -95,13 +95,18 @@ def test_filetype_to_instance(extension, klass): print("testing: {}".format(extension)) if extension == ".blf": + delete = False file_handler = open(join(dirname(__file__), "data/logfile.blf")) else: - file_handler = tempfile.NamedTemporaryFile(suffix=extension) + delete = True + file_handler = tempfile.NamedTemporaryFile(suffix=extension, delete=False) with file_handler as my_file: - with can.LogReader(my_file.name) as reader: - self.assertIsInstance(reader, klass) + filename = my_file.name + with can.LogReader(filename) as reader: + self.assertIsInstance(reader, klass) + + # TODO: delete test_filetype_to_instance(".asc", can.ASCReader) test_filetype_to_instance(".blf", can.BLFReader) @@ -116,9 +121,12 @@ def test_filetype_to_instance(extension, klass): def testLoggerTypeResolution(self): def test_filetype_to_instance(extension, klass): print("testing: {}".format(extension)) - with tempfile.NamedTemporaryFile(suffix=extension) as my_file: - with can.Logger(my_file.name) as writer: - self.assertIsInstance(writer, klass) + with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as my_file: + filename = my_file.name + with can.Logger(filename) as writer: + self.assertIsInstance(writer, klass) + + # TODO: delete test_filetype_to_instance(".asc", can.ASCWriter) test_filetype_to_instance(".blf", can.BLFWriter) From 7be909dd951b4b011ab521de0959f52ab1b00d8e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 1 Aug 2018 19:22:30 +0200 Subject: [PATCH 93/95] add removal of temporyry file in listener_test.py --- test/listener_test.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/test/listener_test.py b/test/listener_test.py index 64371def2..a10ef611e 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -12,6 +12,7 @@ import logging import tempfile import sqlite3 +import os from os.path import join, dirname import can @@ -93,20 +94,20 @@ def testRemoveListenerFromNotifier(self): def testPlayerTypeResolution(self): def test_filetype_to_instance(extension, klass): print("testing: {}".format(extension)) - - if extension == ".blf": - delete = False - file_handler = open(join(dirname(__file__), "data/logfile.blf")) - else: - delete = True - file_handler = tempfile.NamedTemporaryFile(suffix=extension, delete=False) - - with file_handler as my_file: - filename = my_file.name - with can.LogReader(filename) as reader: - self.assertIsInstance(reader, klass) - - # TODO: delete + try: + if extension == ".blf": + delete = False + file_handler = open(join(dirname(__file__), "data/logfile.blf")) + else: + delete = True + file_handler = tempfile.NamedTemporaryFile(suffix=extension, delete=False) + + with file_handler as my_file: + filename = my_file.name + with can.LogReader(filename) as reader: + self.assertIsInstance(reader, klass) + finally: + os.remove(filename) test_filetype_to_instance(".asc", can.ASCReader) test_filetype_to_instance(".blf", can.BLFReader) @@ -121,12 +122,13 @@ def test_filetype_to_instance(extension, klass): def testLoggerTypeResolution(self): def test_filetype_to_instance(extension, klass): print("testing: {}".format(extension)) - with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as my_file: - filename = my_file.name - with can.Logger(filename) as writer: - self.assertIsInstance(writer, klass) - - # TODO: delete + try: + with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as my_file: + filename = my_file.name + with can.Logger(filename) as writer: + self.assertIsInstance(writer, klass) + finally: + os.remove(filename) test_filetype_to_instance(".asc", can.ASCWriter) test_filetype_to_instance(".blf", can.BLFWriter) From 23a9d9a72e234d240486e3b241e1336696dc2c6e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 1 Aug 2018 19:26:04 +0200 Subject: [PATCH 94/95] removed debugging code from .appveyor.yml --- .appveyor.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 03dc7c680..10d8a6ca5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -27,16 +27,6 @@ install: build: off -before_test: - # see: https://help.appveyor.com/discussions/problems/938-getting-permission-denied-errors-when-trying-to-make-a-temp-directory - # TMPDIR is used by Python to detect the temporary directory: https://docs.python.org/3/library/tempfile.html#tempfile.tempdir - - set TMPDIR=%APPVEYOR_BUILD_FOLDER%\test\__tmpdir__ - - mkdir %TMPDIR% - - icacls %TMPDIR% /inheritance:r /grant Everyone:(OI)(CI)F - test_script: - "pytest" - "codecov" - -after_test: - - rmdir /S /Q %TMPDIR% From a839a0ac960d4abece565e38c3bacc29e66f354e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 1 Aug 2018 19:39:01 +0200 Subject: [PATCH 95/95] only delete some temp files --- test/listener_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/listener_test.py b/test/listener_test.py index a10ef611e..c25a6fb56 100644 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -107,7 +107,8 @@ def test_filetype_to_instance(extension, klass): with can.LogReader(filename) as reader: self.assertIsInstance(reader, klass) finally: - os.remove(filename) + if delete: + os.remove(filename) test_filetype_to_instance(".asc", can.ASCReader) test_filetype_to_instance(".blf", can.BLFReader)