Commit b942e203 authored by Michael Büsch's avatar Michael Büsch

Identify sources by content hash

Signed-off-by: Michael Büsch's avatarMichael Buesch <m@bues.ch>
parent be2922f5
......@@ -151,8 +151,7 @@ def readInputFile(inputFile):
project = Project.fromFile(inputFile)
else:
# make a fake project
awlSrc = AwlSource.fromFile(identNr = AwlSource.newIdentNr(),
name = inputFile,
awlSrc = AwlSource.fromFile(name = inputFile,
filepath = inputFile)
project = Project(projectFile = None,
awlSources = [ awlSrc, ])
......
......@@ -198,8 +198,7 @@ def main():
mainwnd = MainWindow.start(initialAwlSource = awlSource)
s = mainwnd.getSim()
else:
source = AwlSource(identNr = AwlSource.newIdentNr(),
name = awlFile,
source = AwlSource(name = awlFile,
filepath = awlFile,
sourceBytes = awlFileRead(awlFile, encoding="binary"))
p = AwlParser()
......
......@@ -28,6 +28,7 @@ from awlsim.common.util import *
import base64, binascii
import datetime
import os
import hashlib
if isPy2Compat:
from ConfigParser import SafeConfigParser as _ConfigParser
......@@ -38,21 +39,25 @@ else:
class GenericSource(object):
SRCTYPE = "<generic>"
SRCTYPE = "<generic>"
IDENT_HASH = "sha256"
__nextIdentNr = 0
def __init__(self, identNr, name="", filepath="", sourceBytes=b""):
self.identNr = identNr
def __init__(self, name="", filepath="", sourceBytes=b""):
self.name = name
self.filepath = filepath
self.sourceBytes = sourceBytes
@staticmethod
def newIdentNr():
identNr = GenericSource.__nextIdentNr
GenericSource.__nextIdentNr = (GenericSource.__nextIdentNr + 1) & 0x7FFFFFFF
return identNr
def getIdentHash(self):
h = hashlib.new(self.IDENT_HASH, self.SRCTYPE.encode("utf-8"))
if self.name is not None:
h.update(self.name.encode("utf-8"))
if self.filepath is not None:
h.update(self.filepath.encode("utf-8"))
h.update(self.sourceBytes)
return h.digest()
def getIdentHashStr(self):
return binascii.b2a_hex(self.getIdentHash())
def dup(self):
raise NotImplementedError
......@@ -76,41 +81,41 @@ class GenericSource(object):
return base64.b64encode(self.sourceBytes).decode("ascii")
@classmethod
def fromFile(cls, identNr, name, filepath):
def fromFile(cls, name, filepath):
try:
data = awlFileRead(filepath, encoding="binary")
except AwlSimError as e:
raise AwlSimError("Project: Could not read %s "
"source file '%s':\n%s" %\
(cls.SRCTYPE, filepath, str(e)))
return cls(identNr, name, filepath, data)
return cls(name, filepath, data)
@classmethod
def fromBase64(cls, identNr, name, b64):
def fromBase64(cls, name, b64):
try:
data = base64.b64decode(b64.encode("ascii"))
except (TypeError, binascii.Error, UnicodeError) as e:
raise AwlSimError("Project: %s source '%s' "
"has invalid base64 encoding." %\
(cls.SRCTYPE, name))
return cls(identNr, name, None, data)
return cls(name, None, data)
def __repr__(self):
return "%s%s %d %s" % ("" if self.isFileBacked() else "project ",
self.SRCTYPE, self.identNr, self.name)
return "%s%s %s %s" % ("" if self.isFileBacked() else "project ",
self.SRCTYPE, self.name, self.getIdentHashStr())
class AwlSource(GenericSource):
SRCTYPE = "AWL/STL"
def dup(self):
return AwlSource(self.identNr, self.name, self.filepath,
return AwlSource(self.name, self.filepath,
self.sourceBytes[:])
class SymTabSource(GenericSource):
SRCTYPE = "symbol table"
def dup(self):
return SymTabSource(self.identNr, self.name, self.filepath,
return SymTabSource(self.name, self.filepath,
self.sourceBytes[:])
class Project(object):
......@@ -204,8 +209,7 @@ class Project(object):
if not p.has_option("CPU", option):
break
path = p.get("CPU", option)
sourceId = AwlSource.newIdentNr()
src = AwlSource.fromFile(sourceId, path, cls.__generic2path(path, projectDir))
src = AwlSource.fromFile(path, cls.__generic2path(path, projectDir))
awlSources.append(src)
for i in range(0xFFFF):
srcOption = "awl_%d" % i
......@@ -222,8 +226,7 @@ class Project(object):
pass
if name is None:
name = "AWL/STL #%d" % i
sourceId = AwlSource.newIdentNr()
src = AwlSource.fromBase64(sourceId, name, awlBase64)
src = AwlSource.fromBase64(name, awlBase64)
awlSources.append(src)
if p.has_option("CPU", "mnemonics"):
mnemonics = p.getint("CPU", "mnemonics")
......@@ -245,8 +248,7 @@ class Project(object):
if not p.has_option("SYMBOLS", option):
break
path = p.get("SYMBOLS", option)
sourceId = SymTabSource.newIdentNr()
src = SymTabSource.fromFile(sourceId, path, cls.__generic2path(path, projectDir))
src = SymTabSource.fromFile(path, cls.__generic2path(path, projectDir))
symTabSources.append(src)
for i in range(0xFFFF):
srcOption = "sym_tab_%d" % i
......@@ -263,8 +265,7 @@ class Project(object):
pass
if name is None:
name = "Symbol table #%d" % i
sourceId = SymTabSource.newIdentNr()
src = SymTabSource.fromBase64(sourceId, name, symTabBase64)
src = SymTabSource.fromBase64(name, symTabBase64)
symTabSources.append(src)
except _ConfigParserError as e:
......
# -*- coding: utf-8 -*-
#
# AWL simulator - Object identification
#
# Copyright 2014 Michael Buesch <m@bues.ch>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
from __future__ import division, absolute_import, print_function, unicode_literals
from awlsim.common.compat import *
import hashlib
import binascii
class ObjIdent(object):
IDENT_HASH = "sha256"
IDENT_HASH_INIT = b""
@classmethod
def __hashFunc(cls, initData=b''):
return hashlib.new(cls.IDENTHASH, initData)
# Get the identification hash for this object.
def getIdentHash(self):
digest = self.__hashFunc(self.IDENT_HASH_INIT)
for data in self.getIdentData():
h = self.__hashFunc(digest)
h.update(data)
digest = h.digest()
return binascii.b2a_hex(digest)
# Get the data blobs used to identify the object.
# Reimplement this in the subclass to provide the data.
def getIdentData(self):
# Return no data blobs by default.
return []
......@@ -29,13 +29,10 @@ from awlsim.common.project import *
from awlsim.core.util import *
from awlsim.core.datatypes import *
from awlsim.core.objectident import *
class RawAwlInsn(ObjIdent):
class RawAwlInsn(object):
def __init__(self, block):
ObjIdent.__init__(self)
self.block = block
self.sourceId = None
self.lineNr = 0
......@@ -95,16 +92,6 @@ class RawAwlInsn(ObjIdent):
def hasOperators(self):
return bool(self.getOperators())
def getIdentData(self):
#FIXME encodings
yield self.name
if self.hasLabel():
yield self.label
#TODO lineNr
for oper in self.ops:
yield oper
pass#TODO
class RawAwlBlock(object):
def __init__(self, tree, index):
self.tree = tree
......@@ -1014,7 +1001,7 @@ class AwlParser(object):
"""Parse an AWL source.
awlSource is an AwlSource instance."""
self.parseData(awlSource.sourceBytes,
sourceId = awlSource.identNr,
sourceId = awlSource.getIdentHash(),
sourceName = awlSource.name)
@classmethod
......
......@@ -46,7 +46,7 @@ class AwlSimMessage(object):
# Payload (optional)
hdrStruct = struct.Struct(str(">HHHHI"))
HDR_MAGIC = 0x5711
HDR_MAGIC = 0x5712
HDR_LENGTH = hdrStruct.size
EnumGen.start
......@@ -226,18 +226,13 @@ class AwlSimMessage_EXCEPTION(AwlSimMessage):
class _AwlSimMessage_source(AwlSimMessage):
sourceClass = None
# Payload header struct:
# Source identifier (32 bit)
plStruct = struct.Struct(str(">I"))
def __init__(self, msgId, source):
AwlSimMessage.__init__(self, msgId)
self.source = source
def toBytes(self):
try:
pl = self.plStruct.pack(self.source.identNr) +\
self.packString(self.source.name) +\
pl = self.packString(self.source.name) +\
self.packString(self.source.filepath) +\
self.packBytes(self.source.sourceBytes)
return AwlSimMessage.toBytes(self, len(pl)) + pl
......@@ -247,8 +242,7 @@ class _AwlSimMessage_source(AwlSimMessage):
@classmethod
def fromBytes(cls, payload):
try:
(identNr, ) = cls.plStruct.unpack_from(payload, 0)
count = cls.plStruct.size
count = 0
name, cnt = cls.unpackString(payload, count)
count += cnt
filepath, cnt = cls.unpackString(payload, count)
......@@ -256,7 +250,7 @@ class _AwlSimMessage_source(AwlSimMessage):
sourceBytes, cnt = cls.unpackBytes(payload, count)
except (ValueError, struct.error) as e:
raise TransferError("SOURCE: Data format error")
return cls(cls.sourceClass(identNr, name, filepath, sourceBytes))
return cls(cls.sourceClass(name, filepath, sourceBytes))
class AwlSimMessage_LOAD_SYMTAB(_AwlSimMessage_source):
sourceClass = SymTabSource
......@@ -551,7 +545,6 @@ class AwlSimMessage_MEMORY(AwlSimMessage):
class AwlSimMessage_INSNSTATE(AwlSimMessage):
# Payload data struct:
# AWL source ident number (32 bit)
# AWL line number (32 bit)
# Serial number. Reset to 0 on cycle exit. (32 bit)
# Flags (16 bit) (currently unused. Set to 0)
......@@ -564,7 +557,8 @@ class AwlSimMessage_INSNSTATE(AwlSimMessage):
# CPU AR 2 (32 bit)
# CPU DB register (16 bit)
# CPU DI register (16 bit)
plDataStruct = struct.Struct(str(">IIIHHIIIIIIHH"))
# AWL source ident hash bytes (variable length)
plDataStruct = struct.Struct(str(">IIHHIIIIIIHH"))
def __init__(self, sourceId, lineNr, serial, flags, stw, accu1, accu2, accu3, accu4, ar1, ar2, db, di):
AwlSimMessage.__init__(self, AwlSimMessage.MSG_ID_INSNSTATE)
......@@ -584,17 +578,19 @@ class AwlSimMessage_INSNSTATE(AwlSimMessage):
def toBytes(self):
pl = self.plDataStruct.pack(
self.sourceId, self.lineNr, self.serial,
self.lineNr, self.serial,
self.flags, self.stw, self.accu1, self.accu2,
self.accu3, self.accu4, self.ar1, self.ar2,
self.db, self.di)
pl += self.packBytes(self.sourceId)
return AwlSimMessage.toBytes(self, len(pl)) + pl
@classmethod
def fromBytes(cls, payload):
try:
sourceId, lineNr, serial, flags, stw, accu1, accu2, accu3, accu4, ar1, ar2, db, di =\
lineNr, serial, flags, stw, accu1, accu2, accu3, accu4, ar1, ar2, db, di =\
cls.plDataStruct.unpack_from(payload, 0)
sourceId, offset = cls.unpackBytes(payload, cls.plDataStruct.size)
except (struct.error, IndexError) as e:
raise TransferError("INSNSTATE: Invalid data format")
return cls(sourceId, lineNr, serial, flags, stw, accu1, accu2, accu3, accu4, ar1, ar2, db, di)
......@@ -602,10 +598,10 @@ class AwlSimMessage_INSNSTATE(AwlSimMessage):
class AwlSimMessage_INSNSTATE_CONFIG(AwlSimMessage):
# Payload data struct:
# Flags (32 bit)
# AWL source ident number (32 bit)
# From AWL line (32 bit)
# To AWL line (32 bit)
plDataStruct = struct.Struct(str(">IIII"))
# AWL source ident hash bytes (variable length)
plDataStruct = struct.Struct(str(">III"))
# Flags:
FLG_SYNC = 1 << 0 # Synchronous status reply.
......@@ -621,15 +617,16 @@ class AwlSimMessage_INSNSTATE_CONFIG(AwlSimMessage):
def toBytes(self):
pl = self.plDataStruct.pack(
self.flags, self.sourceId,
self.fromLine, self.toLine)
self.flags, self.fromLine, self.toLine)
pl += self.packBytes(self.sourceId)
return AwlSimMessage.toBytes(self, len(pl)) + pl
@classmethod
def fromBytes(cls, payload):
try:
flags, sourceId, fromLine, toLine =\
flags, fromLine, toLine =\
cls.plDataStruct.unpack_from(payload, 0)
sourceId, offset = cls.unpackBytes(payload, cls.plDataStruct.size)
except (struct.error, IndexError) as e:
raise TransferError("INSNSTATE_CONFIG: Invalid data format")
return cls(flags, sourceId, fromLine, toLine)
......
......@@ -267,7 +267,7 @@ class AwlSimServer(object):
continue
if not msg:
msg = AwlSimMessage_INSNSTATE(
sourceId & 0xFFFFFFFF,
sourceId,
lineNr & 0xFFFFFFFF,
self.__insnSerial,
0,
......
......@@ -372,7 +372,7 @@ class CpuWidget(QWidget):
client = self.mainWidget.getSimClient()
if onlineDiagEn and source:
client.setInsnStateDump(enable=True,
sourceId=source.identNr,
sourceId=source.getIdentHash(),
fromLine=fromLine, toLine=toLine,
sync=False)
else:
......
......@@ -124,8 +124,9 @@ class EditWidget(QPlainTextEdit):
self.lineNumWidget = LineNumSubWidget(self)
self.cpuStatsWidget = CpuStatsSubWidget(self)
self.__source = AwlSource(identNr = AwlSource.newIdentNr(),
name = "Unnamed source")
self.__source = AwlSource(name = "Unnamed source")
self.__needSourceUpdate = True
self.__runStateCopy = CpuWidget.STATE_STOP
self.__nextHdrUpdate = 0
self.__hdrAniStat = 0
......@@ -163,9 +164,9 @@ class EditWidget(QPlainTextEdit):
self.resetCpuStats()
finally:
self.__textChangeBlocked -= 1
self.__needSourceUpdate = True
def getFullSource(self):
source = self.__source.dup()
def __updateSource(self):
sourceText = self.toPlainText()
# Convert to DOS-style line endings
sourceText = "\r\n".join(sourceText.splitlines()) + "\r\n"
......@@ -173,18 +174,20 @@ class EditWidget(QPlainTextEdit):
try:
sourceBytes = sourceText.encode(AwlParser.TEXT_ENCODING,
errors="strict")
source.sourceBytes = sourceBytes
self.__source.sourceBytes = sourceBytes
except UnicodeError:
MessageBox.error(self, "The AWL/STL code contains "
"non-%s-characters. These were ignored and stripped "
"from the code." % AwlParser.TEXT_ENCODING)
sourceBytes = sourceText.encode(AwlParser.TEXT_ENCODING,
errors="ignore")
source.sourceBytes = sourceBytes
self.setSource(source)
return source
self.__source.sourceBytes = sourceBytes
self.setSource(self.__source)
self.__needSourceUpdate = False
def getSourceRef(self):
def getSource(self):
if self.__needSourceUpdate:
self.__updateSource()
return self.__source
def runStateChanged(self, newState):
......@@ -255,7 +258,7 @@ class EditWidget(QPlainTextEdit):
# insnDumpMsg => AwlSimMessage_INSNDUMP instance
if not self.__cpuStatsEnabled:
return
if insnDumpMsg.sourceId != self.__source.identNr:
if insnDumpMsg.sourceId != self.__source.getIdentHash():
# Discard old messages that were still in the queue.
return
# Save the instruction dump
......@@ -453,5 +456,6 @@ class EditWidget(QPlainTextEdit):
self.__updateFonts()
if self.__textChangeBlocked:
return
self.__needSourceUpdate = True
self.codeChanged.emit()
self.resetCpuStats()
......@@ -142,8 +142,7 @@ class ProjectWidget(QTabWidget):
def __loadPlainAwlSource(self, filename):
project = Project(None) # Create an ad-hoc project
srcs = [ AwlSource.fromFile(identNr = AwlSource.newIdentNr(),
name = filename,
srcs = [ AwlSource.fromFile(name = filename,
filepath = filename), ]
project.setAwlSources(srcs)
self.__loadProject(project)
......
......@@ -129,12 +129,12 @@ class SourceTabWidget(QTabWidget):
curWidget = self.currentWidget()
showIntegrate = False
if curWidget:
showIntegrate = curWidget.getSourceRef().isFileBacked()
showIntegrate = curWidget.getSource().isFileBacked()
self.contextMenu.showIntegrateButton(showIntegrate)
def updateTabTexts(self):
for i in range(self.count()):
self.setTabText(i, self.widget(i).getSourceRef().name)
self.setTabText(i, self.widget(i).getSource().name)
self.sourceChanged.emit()
def allTabWidgets(self):
......@@ -151,7 +151,7 @@ class SourceTabWidget(QTabWidget):
def getSources(self):
"Returns a list of sources"
return [ w.getFullSource() for w in self.allTabWidgets() ]
return [ w.getSource() for w in self.allTabWidgets() ]
def setSources(self, sources):
raise NotImplementedError
......@@ -159,7 +159,7 @@ class SourceTabWidget(QTabWidget):
def integrateSource(self):
curWidget = self.currentWidget()
if curWidget:
curWidget.getSourceRef().forceNonFileBacked(self.contextMenu.itemName)
curWidget.getSource().forceNonFileBacked(self.contextMenu.itemName)
self.updateActionMenu()
self.updateTabTexts()
......@@ -198,7 +198,7 @@ class AwlSourceTabWidget(SourceTabWidget):
editWidget = self.currentWidget()
if editWidget:
fromLine, toLine = editWidget.getVisibleLineRange()
source = editWidget.getSourceRef()
source = editWidget.getSource()
self.visibleLinesChanged.emit(source, fromLine, toLine)
else:
self.visibleLinesChanged.emit(None, -1, -1)
......@@ -245,7 +245,7 @@ class AwlSourceTabWidget(SourceTabWidget):
editWidget = EditWidget(self)
editWidget.codeChanged.connect(self.sourceChanged)
editWidget.visibleRangeChanged.connect(self.__emitVisibleLinesSignal)
index = self.addTab(editWidget, editWidget.getSourceRef().name)
index = self.addTab(editWidget, editWidget.getSource().name)
self.setCurrentIndex(index)
self.updateActionMenu()
self.sourceChanged.emit()
......@@ -274,7 +274,7 @@ class AwlSourceTabWidget(SourceTabWidget):
text)
if ok and newText != text:
editWidget = self.widget(index)
source = editWidget.getSourceRef()
source = editWidget.getSource()
source.name = newText
self.updateTabTexts()
......@@ -282,7 +282,7 @@ class AwlSourceTabWidget(SourceTabWidget):
editWidget = self.currentWidget()
if not editWidget:
return
source = editWidget.getFullSource()
source = editWidget.getSource()
if not source:
return
fn, fil = QFileDialog.getSaveFileName(self,
......@@ -306,8 +306,7 @@ class AwlSourceTabWidget(SourceTabWidget):
"All files (*)")
if not fn:
return
source = AwlSource.fromFile(AwlSource.newIdentNr(),
"Imported source",
source = AwlSource.fromFile("Imported source",
fn)
index, editWidget = self.addEditWidget()
editWidget.setSource(source)
......@@ -353,7 +352,7 @@ class SymSourceTabWidget(SourceTabWidget):
symTabView = SymTabView(self)
symTabView.setSymTab(SymbolTable())
symTabView.model().sourceChanged.connect(self.sourceChanged)
index = self.addTab(symTabView, symTabView.model().getSourceRef().name)
index = self.addTab(symTabView, symTabView.model().getSource().name)
self.setCurrentIndex(index)
self.updateActionMenu()
self.sourceChanged.emit()
......@@ -382,7 +381,7 @@ class SymSourceTabWidget(SourceTabWidget):
text)
if ok and newText != text:
symTabView = self.widget(index)
source = symTabView.getSourceRef()
source = symTabView.getSource()
source.name = newText
self.updateTabTexts()
......@@ -390,7 +389,7 @@ class SymSourceTabWidget(SourceTabWidget):
symTabView = self.currentWidget()
if not symTabView:
return
source = symTabView.getFullSource()
source = symTabView.getSource()
if not source:
return
fn, fil = QFileDialog.getSaveFileName(self,
......@@ -414,8 +413,7 @@ class SymSourceTabWidget(SourceTabWidget):
"All files (*)")
if not fn:
return
source = SymTabSource.fromFile(SymTabSource.newIdentNr(),
"Imported symbol table",
source = SymTabSource.fromFile("Imported symbol table",
fn)
index, symTabView = self.addSymTable()
symTabView.setSource(source)
......
......@@ -32,8 +32,12 @@ class SymTabModel(QAbstractTableModel):
def __init__(self, symTab):
QAbstractTableModel.__init__(self)
self.symTab = symTab
self.__source = SymTabSource(SymTabSource.newIdentNr(),
"Unnamed symbol table")
self.__source = SymTabSource("Unnamed symbol table")
self.__needSourceUpdate = True
def emitSourceChanged(self):
self.__needSourceUpdate = True
self.sourceChanged.emit()
def getSymTab(self):
return self.symTab
......@@ -43,14 +47,14 @@ class SymTabModel(QAbstractTableModel):
self.beginResetModel()
del self.symTab.symbols[row]
self.endResetModel()
self.sourceChanged.emit()
self.emitSourceChanged()
def moveSymbol(self, fromRow, toRow):
self.beginResetModel()
sym = self.symTab.symbols.pop(fromRow)
self.symTab.symbols.insert(toRow, sym)
self.endResetModel()
self.sourceChanged.emit()
self.emitSourceChanged()
def rowCount(self, parent=QModelIndex()):
return len(self.symTab.symbols) + 1
......@@ -124,7 +128,7 @@ class SymTabModel(QAbstractTableModel):
MessageBox.handleAwlSimError(None,
"Invalid symbol information", e)
return False
self.sourceChanged.emit()
self.emitSourceChanged()
return True
return False
......@@ -133,19 +137,19 @@ class SymTabModel(QAbstractTableModel):
return Qt.ItemIsEnabled
return Qt.ItemIsEnabled | Qt.ItemIsSelectable | Qt.ItemIsEditable
def getSourceRef(self):
def getSource(self):
if self.__needSourceUpdate:
self.__updateSource()
return self.__source
def getFullSource(self):
source = self.__source.dup()
def __updateSource(self):
try:
source.sourceBytes = self.symTab.toASC()
self.__source.sourceBytes = self.symTab.toASC()
except AwlSimError as e:
MessageBox.handleAwlSimError(None,
"Symbol table contains invalid characters", e)
return None
source.sourceBytes = self.symTab.toBytes("ignore")
return source
return
self.__needSourceUpdate = False
def setSource(self, newSource):
self.beginResetModel()
......@@ -159,6 +163,7 @@ class SymTabModel(QAbstractTableModel):
"Could not parse symbol table information", e)
finally:
self.endResetModel()
self.__needSourceUpdate = True
class SymTabView(QTableView):
def __init__(self, parent=None):
......@@ -208,11 +213,8 @@ class SymTabView(QTableView):
def setSymTab(self, symTab):
self.setModel(SymTabModel(symTab))
def getSourceRef(self):
return self.model().getSourceRef()
def getFullSource(self):
return self.model().getFullSource()
def getSource(self):
return self.model().getSource()
def setSource(self, newSource):
return self.model().setSource(newSource)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment